diff --git a/.gcloudignore b/.gcloudignore new file mode 100644 index 0000000000..f3cc65ba95 --- /dev/null +++ b/.gcloudignore @@ -0,0 +1,10 @@ +.gcloudignore +.git +.gitignore +#!include:.gitignore + +perfmetrics/ +.github/ +benchmarks/ +docs/ +samples/ diff --git a/.gemini/config.yaml b/.gemini/config.yaml new file mode 100644 index 0000000000..e7b78c2a22 --- /dev/null +++ b/.gemini/config.yaml @@ -0,0 +1,21 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Custom configuration guide: https://developers.google.com/gemini-code-assist/docs/customize-gemini-behavior-github?content_ref=the+gemini+folder+hosts+any+gemini+code+assist+related+configuration+files+like+config+yaml+and+styleguide+md#custom-configuration + +code_review: + pull_request_opened: + code_review: true + summary: true + include_drafts: false diff --git a/.gemini/skills/build/SKILL.md b/.gemini/skills/build/SKILL.md new file mode 100644 index 0000000000..d707f375ae --- /dev/null +++ b/.gemini/skills/build/SKILL.md @@ -0,0 +1,80 @@ +# GCSFuse Build and Style Verification Skill + +This skill provides a highly streamlined workflow for Antigravity agents to compile the GCSFuse codebase, verify layout formats, resolve imports, and perform static analysis checks. Executing this unified validation runbook ensures all changes are cleanly compiled and formatted exactly to the repository's strict standards before proposing pull requests. + +--- + +## 1. Context and Target Standards + +GCSFuse uses a strict presubmit verification pipeline to ensure code hygiene and layout correctness. The root-level [Makefile](../../../Makefile) is the single-source-of-truth configuration orchestrator. + +Running `make build` automatically handles the entire verification chain. Agents must not execute individual compilation or formatting steps manually unless explicitly instructed by this runbook. + +```mermaid +graph TD + A[make build] --> B[Run imports, formatting, and dependency sorting] + B --> C[Run code compiler verification checks] +``` + +Executing `make build` automatically executes and verifies the following operations: +* **Code Generation**: Triggers `go generate` to build auto-generated structures. +* **Auto-Imports & Formatting**: Automatically runs `goimports` and `go fmt` to align headers and style of all updated files. +* **Dependency Hygiene**: Runs `go mod tidy` to clean up and sync local package lists. +* **Static Analysis Checks**: Runs `go vet` to intercept logical and syntax issues early. +* **Style Compliance**: Runs `golangci-lint` to check code cleanliness against `master`. +* **Binary Compilation**: Builds all target packages under the root directory. + +--- + +## 2. Step-by-Step Verification Workflow + +### Step 1: Align Go Toolchain +1. Inspect the targeted Go compiler version specified in the root [.go-version](../../../.go-version) file (currently **`1.26.3`**). +2. Check the active compiler version: + ```bash + go version + ``` +3. If there is a major/minor version mismatch with the target baseline series, notify the user immediately before compiling, as version mismatches can introduce syntax regressions. + +### Step 2: Run Unified Verification Target +Run the comprehensive validation suite from the root of the workspace: +```bash +make build +``` +* **Verification Success**: The build target finishes with exit code `0`. +* **Verification Failure**: If the build target fails, analyze the output: + * **Compilation/Syntax Errors**: Correct the source files and re-run `make build`. + * **Lint Issues**: Resolve style warnings flagged by the linter. + +### Step 3: Stage & Commit Automatic Formatting +Because `make build` runs auto-formatters and import organizers in-place, the workspace might receive automatic layout corrections. + +1. Inspect the working tree for any modifications automatically made by the build: + ```bash + git status + ``` +2. Inspect formatting diffs: + ```bash + git diff --exit-code --name-only + ``` +3. **Commit Rule**: If formatting changes exist, **commit these automatic corrections** directly alongside your functional changes. Clean formatting is a mandatory PR check block. + +--- + +## 3. Graceful Warning Fallbacks + +### Style Linters Failure +`golangci-lint` is required under the `make build` pipeline. In sandboxed or offline local developer environments, this check can occasionally fail or time out: +* > [!WARNING] +* > Do **NOT** treat local linter warnings or execution failures as a blocking compilation failure. +* If standard compiling passes but the linter blocks execution, log a warning to the user noting that linting was bypassed, and proceed. + +--- + +## 4. Compilation Verification Checklist + +Before final project handoff, verify the following checks: + +- [ ] **Toolchain Sync**: My active `go version` aligns with the project’s [.go-version](../../../.go-version) file directive. +- [ ] **Verification Clean**: Successfully compiled using `make build` without errors. +- [ ] **Hygiene Staged**: Any auto-formatting, import sorting, or dependency adjustments made by the Makefile are committed to the branch tree. diff --git a/.gemini/skills/dep-upgrade/SKILL.md b/.gemini/skills/dep-upgrade/SKILL.md new file mode 100644 index 0000000000..8b69352c57 --- /dev/null +++ b/.gemini/skills/dep-upgrade/SKILL.md @@ -0,0 +1,195 @@ +# GCSFuse Dependency Upgrading Skill + +This skill provides a comprehensive, step-by-step workflow for Antigravity agents to identify, upgrade, and verify external dependencies and Go toolchain versions in the GCSFuse repository, specifically targeting the remediation of CVEs and security vulnerabilities. + +--- + +## 1. Parsing Input & Discovering Vulnerabilities + +The user initiates this skill by providing one of the following inputs: + +### Input Option A: Specific Dependencies to Upgrade +* The user explicitly specifies the module(s) and/or Go version to upgrade (e.g., "Upgrade `google.golang.org/protobuf` to latest", "Upgrade Go to 1.27.0"). +* Identify the exact module path(s) and target version(s). + +### Input Option B: Bug / Vulnerability Reference +* The user specifies a bug reference, issue ID, or CVE ID (e.g., "Fix security issues mentioned in b/123456" or "Fix CVE-2026-12345"). +* **Workflow for Option B**: + 1. Retrieve the full details of the issue, bug description, or external CVE details. + 2. Analyze the issue content or logs to find the vulnerable dependencies (e.g., `golang.org/x/net` before `v0.17.0`) or Go standard library vulnerabilities. + 3. If the bug mentions Go standard library vulnerabilities (e.g., `Found in: net/http@go1.26.2`, `Fixed in: net/http@go1.26.3`), identify the new target Go compiler version. + 4. Produce a list of target packages and their minimum safe versions. + +--- + +## 2. Querying Versions & Major Upgrade Safety Checks + +Before executing any dependency upgrades, identify the target versions and evaluate them for major breaking changes. + +### Core Directive +* **Default Target**: Always aim to upgrade dependencies to their **latest stable version**. +* **Upgrade Limit**: You may automatically upgrade dependencies to their latest minor or patch versions (e.g., `v1.2.0` -> `v1.5.3`). However, if upgrading to the latest stable version requires a **major version change**, you must halt and ask the user for permission (as detailed in Step 2). + +### Step 1: Query Available Versions +List all available stable versions of the target Go module to find the latest stable release: +```bash +go list -m -versions +``` +*Example:* +```bash +go list -m -versions go.opentelemetry.io/otel/sdk +``` + +### Step 2: Version Comparison & Safety Checks +Compare the current version of the dependency in `go.mod` against the proposed target/latest version. + +* **Major Version Check Rule**: + If the proposed upgrade involves a **major version change** (e.g., upgrading `github.com/pkg/errors` from `v0.9.1` to `v2.0.0`, or any package whose import path or major version number increments): + 1. **Stop immediately.** + 2. **Alert the user**: Notify them that a major version upgrade is breaking, will require changing module import paths in `.go` files, and could introduce API incompatibilities. + 3. **Ask for permission** before proceeding. + 4. Do NOT proceed with the upgrade unless the user explicitly approves. + +* **No Downgrades Rule**: + If the proposed target version is lower/older than the current version in `go.mod`: + 1. **Stop immediately.** + 2. **Alert the user**: Notify them that the target version represents a downgrade (e.g., going from `v1.5.0` to `v1.3.0`). + 3. **Ask for permission**: Do NOT proceed with a downgrade unless the user explicitly instructs or approves it. + +* If the upgrade to the latest version is a minor or patch version change (e.g., `v1.2.0` to `v1.3.5`) and is a strictly higher version, proceed automatically. + +--- + +## 3. Dependency Upgrade Workflow + +To upgrade standard Go modules (non-toolchain libraries): + +### Guidelines +1. **Focus on Direct Dependencies**: Only upgrade direct dependencies directly. Do not manually upgrade indirect dependencies unless the corresponding direct dependency is also being upgraded. Let `go mod tidy` resolve and update indirect dependencies automatically based on the upgraded direct dependencies. +2. **Lock Version Precisely**: Do not use wildcard upgrades if a specific fixed version is known. +3. **Upgrade Co-dependencies (Peer Packages)**: Upgrading a core library (like `go.opentelemetry.io/otel/sdk`) often requires upgrading peer packages (like `go.opentelemetry.io/otel` or contribution/instrumentation packages like `otelhttp`) to the same version to prevent Skew/API conflicts. + +### Commands + +#### Case A: Upgrading All/General Dependencies +If you are asked to upgrade all or general dependencies, run the following command to upgrade **only the direct dependencies** to their latest minor/patch versions: +```bash +go get $(go list -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' -m all) +``` +Then, run `go mod tidy` to let Go resolve and update the indirect dependencies and clean up `go.mod` and `go.sum`: +```bash +go mod tidy +``` + +#### Case B: Upgrading Specific Dependencies +If you are asked to upgrade specific dependencies: +1. Run the `go get` command for the target package(s) and its direct dependencies: + ```bash + go get @ [peer_package]@[target_version] ... + ``` +2. Tidy up the module definition files (`go.mod` and `go.sum`): + ```bash + go mod tidy + ``` + +--- + +## 4. Go Toolchain & Standard Library Upgrades + +Standard library vulnerabilities (e.g., in `net/http` or `html/template`) cannot be patched via `go get`. They require upgrading the Go toolchain/compiler version itself across the entire repository. + +### Core Directives for Go Version Upgrades + +1. **Target Go Version Rule**: + * **Default Target**: When upgrading the Go compiler/toolchain (whether for standard library CVE fixes, general feature upgrades, or dependency requirements), **always upgrade to the latest stable available version of Golang**, unless a specific target version is explicitly requested by the user or hardcoded in the bug description. + * > [!IMPORTANT] + * > If the upgrade involves a major/minor version change (e.g., moving the repository from Go `1.23.x` to a newer series like `1.26.x`), **upgrade to the latest available stable release of that new series** (e.g., the absolute latest stable release `1.26.3` or newer), instead of just targetting the baseline first release of the series (e.g., `1.26.0`), unless explicitly specified otherwise by the user or the bug. + * Verify the latest stable version of Go via the web search tool or Go's official download site before finalizing the target version. + +2. **Absolute Go Version Consistency**: + * **Strict Consistency Requirement**: The **exact same** three-part (patch-level) Go version string (e.g., `1.26.3`) must be applied globally across the entire repository. There must be absolutely no version mismatch or inconsistency. + * **Consistency in [go.mod](../../../go.mod)**: The `go` directive in [go.mod](../../../go.mod) **MUST be updated to match the exact patch-level version string** (e.g., `go 1.26.3`), aligning it completely with [.go-version](../../../.go-version). Do NOT use minor-only shorthand (e.g., `go 1.26`) in [go.mod](../../../go.mod) while using a three-part version string elsewhere. + * Every single reference to the Go compiler in main builder images, packaging Dockerfiles, presubmit checks, performance suites, and CD scripts must be kept strictly uniform. + +### Step 1: Retrieve Current Go Version +Look at the [.go-version](../../../.go-version) file in the root directory, or the `go` directive in [go.mod](../../../go.mod) (e.g., `go 1.26.3`) to identify the old Go version string. + +### Step 2: Determine and Query the Target Go Version +1. Identify the minimum safe version required (e.g., if remediating a CVE). +2. Query the web or official resources to find the **latest stable available version** of the Go compiler (within the target major/minor release, or a newer release series if upgrading Go major/minor versions). +3. Set this **latest available version** as the exact target Go version string (e.g., `1.26.3`). + +### Step 3: Search the Entire Repository +Search the codebase using `grep` to find all instances of the old Go version string (e.g., `1.26.3`): +```bash +# Example grep query using the search tool: +Query: "1.26.3" +SearchPath: "" +``` + +*Key Go Version Reference Locations that MUST match exactly:* +1. **[.go-version](../../../.go-version)**: + *Located at the root of the repository.* + * **Mandatory Action**: Ensure this file contains the exact target Go version (e.g., `1.26.3`). +2. **[go.mod](../../../go.mod)**: + * **Mandatory Action**: Ensure the `go` directive matches the exact patch-level version string (e.g., `go 1.26.3`). Do NOT use two-part versions here. + ```diff + -go 1.26.3 + +go 1.27.1 + ``` +3. **[Dockerfile](../../../Dockerfile)** (Main Builder Image at root) +4. **Packaging Dockerfiles**: + * **[Dockerfile](../../../tools/containerize_gcsfuse_docker/Dockerfile)** (Containerization base image) + * **[Dockerfile](../../../tools/package_gcsfuse_docker/Dockerfile)** (Packaging base image. Make sure to align *both* the builder baseline tag `FROM golang:X.Y.Z` and the download command `goX.Y.Z.linux` to match the exact same version string). +5. **E2E & Integration Test Scripts**: + * **[e2e_test.sh](../../../tools/cd_scripts/e2e_test.sh)** + * **[run_e2e_tests.sh](../../../tools/integration_tests/run_e2e_tests.sh)** + * **[improved_run_e2e_tests.sh](../../../tools/integration_tests/improved_run_e2e_tests.sh)** +6. **Performance & Presubmit Test Scripts**: + * **[build.sh](../../../perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh)** + * **[setup.sh](../../../perfmetrics/scripts/read_cache/setup.sh)** + * **[run_checkpoints.sh](../../../perfmetrics/scripts/ml_tests/checkpoint/Jax/run_checkpoints.sh)** + +### Step 4: Replace All References +Update every file identified in Step 3, substituting the old Go version with the new exact target Go version. Ensure that no minor-only (e.g. `1.26` instead of `1.26.3`) or mismatched version strings exist. + +### Step 5: Sync and Tidy +Run `go mod tidy` to ensure `go.mod` and `go.sum` are consistent and fully synchronized. + +--- + +## 5. Compilation & Verification + +After upgrading dependencies and/or the Go toolchain, verify the integrity and safety of the changes. + +### Compile Check +Verify that the entire codebase builds cleanly with no compile-time regressions: +```bash +make build +``` + +### Run Vulnerability Scan +Verify that all active CVEs are completely cleared. First, ensure `govulncheck` is installed on the system, and install it if it is missing: +```bash +# Check if govulncheck is installed, if not, attempt to install it and expose $(go env GOPATH)/bin in PATH +if ! command -v govulncheck &> /dev/null; then + go install golang.org/x/vuln/cmd/govulncheck@latest || true + export PATH=$PATH:$(go env GOPATH)/bin +fi +``` +Then attempt to run the vulnerability scan. +> [!IMPORTANT] +> If `govulncheck` fails to install or run (e.g., due to offline environments, missing access to external proxies, or toolchain configuration limits), **do NOT treat this failure as a critical blocker**. Log a warning to the user, note the issue in your summary, and proceed to running the test suite to complete the dependency upgrade. + +If the utility is available, run the scan: +```bash +govulncheck ./... +``` +*Result:* The scan should report `Your code is affected by 0 vulnerabilities`. + +### Run Tests +Verify that the test suite passes successfully with the upgraded dependencies: +```bash +go test ./... +``` + diff --git a/.gemini/skills/pull-request/SKILL.md b/.gemini/skills/pull-request/SKILL.md new file mode 100644 index 0000000000..0cd09d682a --- /dev/null +++ b/.gemini/skills/pull-request/SKILL.md @@ -0,0 +1,134 @@ +# GCSFuse Pull Request Creation Skill + +This skill provides a highly automated, step-by-step workflow for Antigravity agents to construct, format, push, and create pull requests for their changes in the GCSFuse repository. Executing this runbook ensures that commits and pull requests strictly follow the repository's standardized templates, author identity alignments, and style checking procedures before PR creation. + +--- + +## 1. Context and Formatting Standards + +GCSFuse enforces a structured PR style and title system. Agents must align their branch work with the repository's standard [PR Template](../../../.github/pull_request_template.md) to facilitate clean, review-ready pull requests. + +### Pull Request Title Standard +The pull request title and git commit message **MUST** strictly follow the Conventional Commits specification: +``` +type(scope): subject +``` +* **Types list**: + * `feat`: A new feature + * `fix`: A bug fix + * `docs`: Documentation only changes + * `style`: Structural style updates (white-space, formatting, etc.) + * `refactor`: A code modification that neither fixes a bug nor adds a feature + * `perf`: Performance improvements + * `test`: Correcting or creating missing tests + * `build`: Build systems or dependency updates (example scope: `deps`) + * `ci`: Verification runner scripts and presubmits (example scope: `workflows`) + * `chore`: Other maintenance shifts that don't modify src or test structures + * `revert`: Reverting a past merge or commit +* **Scope**: A bracketed descriptor highlighting the specific package or component modified (e.g., `skills`, `internal/cache`, `metrics`, `tools`). +* **Subject**: A clear, concise present-tense statement of the changes made (e.g., `add PR creation skill for GCSFuse`). + +--- + +## 2. Compiling the PR Fields + +Agents must synthesize a structured PR body based on the original task requirements and development results, mapping them onto the [pull_request_template.md](../../../.github/pull_request_template.md) sections: + +> [!IMPORTANT] +> **Instructional Block Omission Rule**: +> The [PR Template](../../../.github/pull_request_template.md) contains an initial introductory/instructional block (lines 1 to 20) detailing conventional commit rules and types. This block is strictly for reference. +> When compiling and generating the pull request body payload, the agent **MUST REMOVE / OMIT this instructional block** completely. The final PR description body must strictly start with the `### Description` header down to the end of the file. + +### Section A: Description +* **Guideline**: Summarize *what* was updated and *why*. +* Detail functional logic changes, additions, structural moves, and documentation additions. +* Clearly state architectural transitions or optimizations applied. + +### Section B: Link to Issue +* **Guideline**: Reference the bug tracking index, such as internal bug targets (`b/514754560`) or external GitHub Issue URLs. +* Example: `Fixes b/514754560` or `Closes #123`. + +### Section C: Testing Details +* **Guideline**: Report test validation outcomes to ensure functional coverage. +* Divide into: + 1. **Manual**: Detail manual tests ran. E.g., manual compilation verification or verification run commands. + 2. **Unit tests**: Report outcome of unit tests (e.g., package test runs or checkpoint suite coverage). + 3. **Integration tests**: Detail E2E or system suite validations run. + +### Section D: Backward Incompatibility +* **Guideline**: Inspect modifications to identify any backwards-incompatible changes. +* Analyze signature changes on exported APIs, removal of support parameters, or config skews. If none, state: `N/A`. + +--- + +## 3. Step-by-Step Execution Workflow + +### Step 1: Compilation and Layout Pre-Check +Before creating a PR, you **MUST** ensure the branch is fully compliant with repository verification targets. + +1. Invoke the **Build and Style Verification Skill** by executing the single native target: + ```bash + make build + ``` +2. Ensure the build finishes with exit code `0`. If not, resolve the compilation or lint warnings immediately before proceeding. + +### Step 2: Commit Auto-Format & Skill Changes +Because build verifiers run layout formatters in-place, stage all automatic layout alignment differences. + +1. Stash or add untracked/modified updates to the branch: + ```bash + git add . + ``` +2. Verify staged items match expected additions: + ```bash + git status + ``` +3. Commit local staged components using a commit message that maps exactly to the PR Title format compiled in Section 1: + ```bash + git commit -m "type(scope): subject" + ``` + +### Step 3: Establish Remote Upstream & Push +Push the local branch to your fork or branch tree upstream: +```bash +# Example syntax: git push -u origin +git push -u origin $(git branch --show-current) +``` + +### Step 4: Raise PR via GitHub CLI +Utilize the authenticated GitHub CLI (`gh`) to trigger unified PR creation: + +1. Draft the PR body by populating the template values compiled in Section 2 into a clean markdown structure. +2. Trigger the PR creation in **Draft** status: + ```bash + # Syntax: gh pr create --title "" --body "<body>" --draft + gh pr create --title "type(scope): subject" --body-file - --draft <<EOF + ### Description + [Synthesized PR Description] + + ### Link to the issue in case of a bug fix. + [Bug tracker reference] + + ### Testing details + 1. Manual - [Manual validation statement] + 2. Unit tests - [Unit validations statement] + 3. Integration tests - [E2E test suite validation statement] + + ### Any backward incompatible change? If so, please explain. + [Incompatibility statement] + EOF + ``` + * > [!IMPORTANT] + * > **Draft PR Mode Enforcement**: Always pass the `--draft` flag to the creation tool. Publishing in draft mode allows the developer to review diff lines, inspect layout annotations, and verify final pull details inside their browser before officially launching review gates. + +--- + +## 4. Fallback Execution + +If the GitHub CLI is unauthenticated or missing scopes: +1. Log a warning in the execution session describing the CLI authentication limit. +2. Construct and print a pre-filled direct web link to create a pull request via the browser: + ``` + https://github.com/GoogleCloudPlatform/gcsfuse/pull/new/<branch_name>?title=<urlencoded_title>&body=<urlencoded_body> + ``` +3. Output the generated PR body text (omitting the top instructions block as per the Omission Rule) directly in markdown so the user can easily copy and paste it into the browser window if requested. diff --git a/.github/.codecov.yml b/.github/.codecov.yml index cf6445d4fc..0fc03a30ef 100644 --- a/.github/.codecov.yml +++ b/.github/.codecov.yml @@ -1,6 +1,22 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + ignore: - "tools" - "benchmarks" + - "perfmetrics" + - "cfg/config.go" coverage: round: down precision: 2 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d254b57bfa..e57a00c6ac 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,7 +1,6 @@ # These owners will be the default owners for everything in # the repo. * @GoogleCloudPlatform/gcsfuse-codeowners - -perfmetrics/scripts/testing_on_gke/ @gargnitingoogle @kislaykishore @tulsishah @charith87 @GoogleCloudPlatform/gcsfuse-codeowners -perfmetrics/scripts/ @tulsishah @GoogleCloudPlatform/gcsfuse-codeowners -tools/ @tulsishah @GoogleCloudPlatform/gcsfuse-codeowners +.gemini @alleaditya @thrivikram-karur-g @PranjalC100 @meet2mky @anushka567 @vipnydav @GoogleCloudPlatform/gcsfuse-codeowners +perfmetrics/scripts/ @meet2mky @GoogleCloudPlatform/gcsfuse-codeowners +tools/ @meet2mky @GoogleCloudPlatform/gcsfuse-codeowners diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 7de4c21764..abd809dc72 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,9 +11,16 @@ updates: - dependency-type: "direct" schedule: interval: weekly - open-pull-requests-limit: 10 + groups: + # Group all Go updates together in a single PR. + go-dependencies: + patterns: + - "*" - package-ecosystem: pip directory: "/" schedule: interval: weekly - open-pull-requests-limit: 10 + groups: + python-dependencies: + patterns: + - "*" diff --git a/.github/header-checker-lint.yml b/.github/header-checker-lint.yml index 35f6e0e573..81625d9f9a 100644 --- a/.github/header-checker-lint.yml +++ b/.github/header-checker-lint.yml @@ -32,6 +32,7 @@ sourceFileExtensions: ignoreFiles: - 'cmd/testdata/**' - 'internal/config/testdata/*' + - 'internal/workloadinsight/testdata/**' - 'internal/storage/caching/mock_gcscaching/mock_stat_cache.go' - 'internal/storage/mock_bucket.go' - 'perfmetrics/scripts/load_tests/python/sample_tasks.yaml' diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 155a7c4c23..abd95e8498 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,9 +1,30 @@ +**Please ensure your PR title follows the format:** +``` +type(scope): subject +``` +Example: +`feat(api): add user login endpoint` + +Available types: +- `feat`: A new feature +- `fix`: A bug fix +- `docs`: Documentation only changes +- `style`: Changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc) +- `refactor`: A code change that neither fixes a bug nor adds a feature +- `perf`: A code change that improves performance +- `test`: Adding missing tests or correcting existing tests +- `build`: Changes that affect the build system or external dependencies (example scopes: gulp, broccoli, npm) +- `ci`: Changes to our CI configuration files and scripts (example scopes: Travis, Circle, BrowserStack, SauceLabs) +- `chore`: Other changes that don't modify src or test files +- `revert`: Reverts a previous commit + ### Description ### Link to the issue in case of a bug fix. -NA ### Testing details 1. Manual - NA 2. Unit tests - NA 3. Integration tests - NA + +### Any backward incompatible change? If so, please explain. diff --git a/.github/scripts/reminder.js b/.github/scripts/reminder.js new file mode 100644 index 0000000000..cc2725e511 --- /dev/null +++ b/.github/scripts/reminder.js @@ -0,0 +1,82 @@ +// This script uses the Octokit library to interact with the GitHub API. +const { getOctokit } = require('@actions/github'); +// The context object provides information about the workflow run. +const { context } = require('@actions/github'); + +// This is the main function that will be executed. +async function run() { + try { + // ------------------ CONFIGURATION ------------------ + // Label that triggers the reminder. + const REMINDER_LABEL = 'remind-reviewers'; + // Inactivity time in hours. + const INACTIVITY_HOURS = 24; + // The message to post on the pull request. + const REMINDER_MESSAGE = `Hi {reviewers}, your feedback is needed to move this pull request forward. This automated reminder was triggered because there has been no activity for over ${INACTIVITY_HOURS} hours. Please provide your input when you have a moment. Thank you!`; + // --------------------------------------------------- + + // Get the GitHub token from environment variables. It's passed in from the workflow file. + const token = process.env.GITHUB_TOKEN; + if (!token) { + throw new Error("GITHUB_TOKEN is not set. The workflow must pass it as an environment variable."); + } + + // Create an authenticated Octokit client. + const octokit = getOctokit(token); + // Remove 10 minutes from inactivity time to not account previous reminders as activity. + const INACTIVITY_MS = INACTIVITY_HOURS * 60 * 60 * 1000 - 10 * 60 * 1000; + const now = new Date().getTime(); + + // Get all open pull requests + const pullRequests = await octokit.paginate(octokit.rest.pulls.list, { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + }); + + console.log(`Found ${pullRequests.length} open pull requests.`); + + for (const pr of pullRequests) { + const hasReminderLabel = pr.labels.some(label => label.name === REMINDER_LABEL); + if (!hasReminderLabel) { + console.log(`PR #${pr.number} ('${pr.title}') does not have the '${REMINDER_LABEL}' label. Skipping.`); + continue; + } + + if (pr.draft) { + console.log(`PR #${pr.number} ('${pr.title}') is a draft. Skipping.`); + continue; + } + + const updatedAt = new Date(pr.updated_at).getTime(); + const isInactive = (now - updatedAt) > INACTIVITY_MS; + if (!isInactive) { + console.log(`PR #${pr.number} ('${pr.title}') is not inactive yet. Last updated at ${pr.updated_at}. Skipping.`); + continue; + } + + const requestedReviewers = pr.requested_reviewers.map(reviewer => `@${reviewer.login}`).join(', '); + if (requestedReviewers.length === 0) { + console.log(`PR #${pr.number} ('${pr.title}') is inactive but has no requested reviewers. Skipping.`); + continue; + } + + const finalMessage = REMINDER_MESSAGE.replace('{reviewers}', requestedReviewers); + console.log(`PR #${pr.number} ('${pr.title}') is inactive. Posting a reminder comment.`); + + await octokit.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + body: finalMessage + }); + } + } catch (error) { + // If the script fails, log the error message. + console.error(error.message); + process.exit(1); + } +} + +// Execute the main function. +run(); diff --git a/.github/workflows/auto-pr-reminder.yml b/.github/workflows/auto-pr-reminder.yml new file mode 100644 index 0000000000..74cecd59b1 --- /dev/null +++ b/.github/workflows/auto-pr-reminder.yml @@ -0,0 +1,45 @@ +name: Auto Add Label on Ready for Review or Reopen PR + +on: + pull_request: + types: + - opened + - reopened + - ready_for_review + branches: + - master + +jobs: + add-label: + # This condition ensures the label is only added if the PR is not a draft. + # When a PR is reopened, it is never in a draft state. + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + permissions: + pull-requests: write + issues: write + steps: + - name: Add 'remind-reviewers' label + uses: actions/github-script@v6 + with: + script: | + // List of authors to exclude from reminders. + const excludedAuthors = [ + 'dependabot[bot]', + ]; + + const author = context.payload.pull_request.user.login; + console.log(`Pull Request author is: ${author}`); + + // Check if the author is in the exclusion list. + if (excludedAuthors.includes(author)) { + console.log(`Author '${author}' is in the exclusion list. Skipping label addition.`); + return; + } + + github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: ['remind-reviewers'] + }) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eda21ef14a..ae5523d2a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,8 +7,36 @@ on: pull_request: branches: - '*' +permissions: + contents: read + +# Ensure only the latest run per branch is active (cancel older in-progress runs), +# but allow every master commit to run (don't cancel or group master runs together). +concurrency: + # For master we use a unique group per run (github.run_id) so runs don't cancel each other. + # For other refs we use the ref as the group so a new run cancels any previous in-progress run on that ref. + group: ${{ github.ref == 'refs/heads/master' && github.run_id || github.ref }} + # Cancel in-progress runs for non-master refs; do not cancel on master. + cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} jobs: + filter: + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + run_tests: ${{ steps.filter.outputs.run_tests }} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3.0.2 + id: filter + with: + predicate-quantifier: 'every' + filters: | + run_tests: + - '!**/*.md' + - '!tools/**' + - '!perfmetrics/**' + - '!samples/**' format-test: runs-on: ubuntu-latest timeout-minutes: 5 @@ -17,59 +45,64 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: "1.23" + go-version-file: '.go-version' + - name: Install goimports + run: go install golang.org/x/tools/cmd/goimports@latest - name: CodeGen run: go generate ./... - - name: Formatting diff - run: go fmt ./... && git diff --exit-code --name-only + - name: Formatting and Imports check + run: | + goimports -w . + go fmt ./... + go mod tidy + git diff --exit-code --name-only linux-tests: - strategy: - matrix: - go: [ 1.23.x ] - runs-on: ubuntu-latest - timeout-minutes: 15 + needs: filter + runs-on: ubuntu-22.04 + timeout-minutes: 25 steps: - - uses: actions/checkout@v2 - - name: Set up Go ${{ matrix.go }} - uses: actions/setup-go@v2.1.4 + - uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 with: - go-version: ${{ matrix.go }} + go-version-file: '.go-version' - name: Install fuse run: sudo apt-get update && sudo apt-get install -y fuse3 libfuse-dev - name: Build + if: ${{ needs.filter.outputs.run_tests == 'true' }} run: | CGO_ENABLED=0 go build ./... go install ./tools/build_gcsfuse build_gcsfuse . /tmp ${GITHUB_SHA} - - name: Test all except caching parallely - run: CGO_ENABLED=0 go test -count 1 -covermode=atomic -coverprofile=coverage.out -coverpkg=./... -v -skip `cat flaky_tests.lst | go run tools/scripts/skip_tests/main.go` `go list ./... | grep -v internal/cache/...` - - name: Test caching - run: CGO_ENABLED=0 go test -p 1 -count 1 -covermode=atomic -coverprofile=coverage_cache.out -coverpkg=./... -v -skip `cat flaky_tests.lst | go run tools/scripts/skip_tests/main.go` ./internal/cache/... - - name: Cache RaceDetector Test - run: go test -p 1 -count 1 -v -race -skip `cat flaky_tests.lst | go run tools/scripts/skip_tests/main.go` ./internal/cache/... + - name: Test all + if: ${{ needs.filter.outputs.run_tests == 'true' }} + run: CGO_ENABLED=0 go test -p 1 -count 1 -covermode=atomic -coverprofile=coverage.out -coverpkg=./... -v -skip `cat flaky_tests.lst | go run tools/scripts/skip_tests/main.go` `go list ./... | grep -v 'tools/integration_tests'` + - name: RaceDetector Test + if: ${{ needs.filter.outputs.run_tests == 'true' }} + run: go test -p 1 -count 1 -v -race -skip `cat flaky_tests.lst | go run tools/scripts/skip_tests/main.go` ./internal/cache/... ./internal/gcsx/... - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v4.3.1 timeout-minutes: 5 with: - fail_ci_if_error: true + fail_ci_if_error: false token: ${{ secrets.CODECOV_TOKEN }} flags: unittests lint: name: Lint + if: github.ref != 'refs/heads/master' runs-on: ubuntu-latest steps: - - name: Setup Go - uses: actions/setup-go@v4 - with: - go-version: "1.23" - name: checkout code uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: '.go-version' - name: golangci-lint - uses: golangci/golangci-lint-action@032fa5c5e48499f06cf9d32c02149bfac1284239 + uses: golangci/golangci-lint-action@v8 with: - args: -E=goimports --timeout 2m0s + args: -E=unused --timeout 3m0s only-new-issues: true - diff --git a/.github/workflows/flake-detector.yml b/.github/workflows/flake-detector.yml index 9df870fad4..b3bea0c304 100644 --- a/.github/workflows/flake-detector.yml +++ b/.github/workflows/flake-detector.yml @@ -3,12 +3,15 @@ on: # Enable triggering the workflow manually workflow_dispatch: push: - branches: [ "master" ] + branches: ["master"] + +permissions: + contents: read jobs: flake-detector: - runs-on: ubuntu-latest - timeout-minutes: 75 + runs-on: ubuntu-22.04 + timeout-minutes: 20 steps: - name: Checkout uses: actions/checkout@v4 @@ -18,7 +21,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: 1.23.x + go-version-file: '.go-version' cache: false - name: Install fuse run: sudo apt-get update && sudo apt-get install -y fuse3 libfuse-dev @@ -31,11 +34,8 @@ jobs: - name: Download dependencies run: go mod download - - name: Test all except caching parallely - run: CGO_ENABLED=0 go test -timeout 75m -count 20 -skip `cat flaky_tests.lst | go run tools/scripts/skip_tests/main.go` `go list ./... | grep -v internal/cache/...` - - - name: Test caching - run: CGO_ENABLED=0 go test -p 1 -timeout 75m -count 20 -v -skip `cat flaky_tests.lst | go run tools/scripts/skip_tests/main.go` ./internal/cache/... + - name: Test all + run: CGO_ENABLED=0 go test -p 1 -count 5 -skip `cat flaky_tests.lst | go run tools/scripts/skip_tests/main.go` `go list ./...` - - name: Cache RaceDetector Test - run: CGO_ENABLED=0 go test -p 1 -timeout 75m -count 20 ./internal/cache/... + - name: RaceDetector Test + run: CGO_ENABLED=0 go test -p 1 -count 5 ./internal/cache/... ./internal/gcsx/... diff --git a/.github/workflows/pr-reminder.yml b/.github/workflows/pr-reminder.yml new file mode 100644 index 0000000000..f74f3260ce --- /dev/null +++ b/.github/workflows/pr-reminder.yml @@ -0,0 +1,35 @@ +name: PR Review Reminder + +on: + # Allow manual runs. + workflow_dispatch: + schedule: + # Runs every hour from 9AM to 5PM IST weekdays. + - cron: '30 3-11 * * 1-5' + +jobs: + remind: + runs-on: ubuntu-latest + # These permissions are required for the action to post comments on pull requests. + permissions: + pull-requests: write + issues: write + + steps: + - name: Check out repository code + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install Dependencies + # Pin to latest version supporting commonJS syntax used in reminder.js + run: npm install @actions/github@5.1.1 @actions/core + + - name: Run Reminder Script + run: node .github/scripts/reminder.js + env: + # The GITHUB_TOKEN is automatically created by Actions. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/pr-title-check.yml b/.github/workflows/pr-title-check.yml new file mode 100644 index 0000000000..4511056475 --- /dev/null +++ b/.github/workflows/pr-title-check.yml @@ -0,0 +1,42 @@ +name: PR Title Check + +on: + pull_request_target: + types: + - opened + - edited + +permissions: + pull-requests: write + +jobs: + main: + name: Validate PR title + runs-on: ubuntu-latest + + steps: + - uses: amannn/action-semantic-pull-request@v5 + id: lint_pr_title + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: marocchino/sticky-pull-request-comment@v2 + if: always() && (steps.lint_pr_title.outputs.error_message != null) + with: + header: pr-title-lint-error + message: | + Hey there and thank you for opening this pull request! 👋🏼 + + We require pull request titles to follow the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) and it looks like your proposed title needs to be adjusted. + + Details: + + ``` + ${{ steps.lint_pr_title.outputs.error_message }} + ``` + + - if: ${{ steps.lint_pr_title.outputs.error_message == null }} + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: pr-title-lint-error + delete: true diff --git a/.github/workflows/shadow.yml b/.github/workflows/shadow.yml deleted file mode 100644 index f3a2a78cd8..0000000000 --- a/.github/workflows/shadow.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: "Shadow reviews" - -on: - pull_request: - types: - - review_requested - branches: - - master - -jobs: - shadow-reviewer: - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - name: Add shadow reviewer - run: gh pr edit --add-reviewer @GoogleCloudPlatform/gcsfuse-shadow-reviewers "$PR_URL" - env: - GH_TOKEN: ${{ secrets.SHADOW_REVIEWER_CLASSIC }} - PR_URL: ${{github.event.pull_request.html_url}} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index a9cdccc5a7..f639ee4495 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -14,9 +14,9 @@ jobs: with: only-labels: "pending customer action" days-before-issue-stale: -1 - days-before-issue-close: 30 + days-before-issue-close: 14 stale-issue-label: "pending customer action" - close-issue-message: "Closing this issue as we haven't received any response in 30 days. Please reopen if you are still experiencing this issue." + close-issue-message: "Closing this issue as we haven't received any response in 14 days. Please reopen if you are still experiencing this issue." days-before-pr-stale: -1 days-before-pr-close: -1 repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 8656d8cf65..42f2abb3fc 100644 --- a/.gitignore +++ b/.gitignore @@ -39,7 +39,7 @@ _testmain.go # Editors .idea/ -.vscode/ +.vscode # External folders vendor/ diff --git a/.go-version b/.go-version new file mode 100644 index 0000000000..f8f7381409 --- /dev/null +++ b/.go-version @@ -0,0 +1 @@ +1.26.3 diff --git a/Dockerfile b/Dockerfile index f915c96c1e..81b52b5bdf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,8 +16,9 @@ # > docker build . -t gcsfuse # Mount the gcsfuse to /mnt/gcs: # > docker run --privileged --device /fuse -v /mnt/gcs:/gcs:rw,rshared gcsfuse +ARG GO_VERSION -FROM golang:1.23.3-alpine AS builder +FROM golang:${GO_VERSION}-alpine AS builder RUN apk add git @@ -27,7 +28,7 @@ WORKDIR ${GCSFUSE_REPO} RUN go install ./tools/build_gcsfuse RUN build_gcsfuse . /tmp $(git log -1 --format=format:"%H") -FROM alpine:3.20 +FROM alpine:3.21 RUN apk add --update --no-cache bash ca-certificates fuse diff --git a/Makefile b/Makefile index 78d60b4978..a001cb5492 100644 --- a/Makefile +++ b/Makefile @@ -12,9 +12,33 @@ # See the License for the specific language governing permissions and # limitations under the License. +CSI_VERSION ?= main +GCSFUSE_VERSION ?= $(shell HASH=$$(git rev-parse --short=6 HEAD 2>/dev/null); if [ -z "$$HASH" ]; then echo "unknown"; else if [ -n "$$(git status --porcelain)" ]; then echo "$$HASH-dirty"; else echo "$$HASH"; fi; fi) +GOLANG_VERSION := $(shell cat .go-version) +BUILD_ARM ?= true + +# The following section is to set the value of STAGINGVERSION to be used in build-csi target. +# Define the mandatory prefix, needed to allow passing machine-type from gke csi driver to gcsfuse, +# bypassing the check at +# https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/blob/15afd00dcc2cfe0f9753ddc53c81631ff037c3f2/pkg/csi_driver/utils.go#L532. +STAGINGVERSIONPREFIX := prow-gob-internal-boskos- +# Define the fallback logic in case uuidgen is not available. +# 1. Try 'uuidgen'. +# 2. If 'uuidgen' fails or is missing, construct: [GitHash][Dirty?]-[Epoch] +# Note: We use '=' so this shell command only executes if STAGINGVERSION was not provided. +_STAGINGVERSION_FALLBACK = $(shell \ + uuidgen 2>/dev/null || \ + echo "$$(git rev-parse --short HEAD)$$(git diff --quiet HEAD || echo '+')-$$(date +%s)" \ +) +# Apply default if not provided by user +STAGINGVERSION ?= $(_STAGINGVERSION_FALLBACK) +# Enforce the prefix (Idempotent: removes prefix if present, then adds it) +override STAGINGVERSION := $(STAGINGVERSIONPREFIX)$(patsubst $(STAGINGVERSIONPREFIX)%,%,$(STAGINGVERSION)) + +PROJECT ?= $(shell gcloud config get-value project 2>/dev/null) .DEFAULT_GOAL := build -.PHONY: generate imports fmt vet build buildTest install test clean-gen clean clean-all +.PHONY: generate imports fmt vet lint build buildTest install test clean-gen clean clean-all build-csi generate: go generate ./... @@ -23,13 +47,16 @@ imports: generate goimports -w . fmt: imports - go fmt ./... + go mod tidy && go fmt ./... vet: fmt go vet ./... -build: vet - go build . +lint: vet + golangci-lint run -E=unused --timeout 3m0s --new-from-rev=master + +build: lint + go build ./... buildTest: vet go test -run=PATTERN_THAT_DOES_NOT_MATCH_ANYTHING ./... @@ -41,10 +68,24 @@ test: fmt CGO_ENABLED=0 go test -timeout 5m -count 1 `go list ./... | grep -v internal/cache/...` && CGO_ENABLED=0 go test -timeout 5m -p 1 -count 1 ./internal/cache/... clean-gen: - rm -rf cfg/config.go + rm -rf cfg/config.go cfg/config_test.go clean: clean-gen go clean clean-all: clean-gen go clean -i ./... + +build-csi: + @echo "--------------------------------------" + @echo "Starting build for version: $(STAGINGVERSION)" + @echo "--------------------------------------" + # Actual build commands would go here... + gcloud builds submit --config csi_driver_build.yml --project=$(PROJECT) --substitutions=_GOLANG_VERSION=$(GOLANG_VERSION),_CSI_VERSION=$(CSI_VERSION),_GCSFUSE_VERSION=$(GCSFUSE_VERSION),_BUILD_ARM=$(BUILD_ARM),_STAGINGVERSION=$(STAGINGVERSION) + +e2e-test: + ZONE=$$(curl -H "Metadata-Flavor: Google" metadata.google.internal/computeMetadata/v1/instance/zone | awk -F'/' '{print $$NF}'); \ + echo $$ZONE; \ + REGION=$$(echo $$ZONE | sed 's/-[a-z]$$//'); \ + echo $$REGION; \ + tools/integration_tests/improved_run_e2e_tests.sh --bucket-location $$REGION diff --git a/README.md b/README.md index 2b066a6156..5fdfcdd04b 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,47 @@ # Current status -Starting with V1.0, Cloud Storage FUSE is Generally Available and supported by Google, provided that it is used within its documented supported applications, platforms, and limits. Support requests, feature requests, and general questions should be submitted as a support request via Google Cloud support channels or via GitHub [here](https://github.com/GoogleCloudPlatform/gcsfuse/issues). +Cloud Storage FUSE continues to evolve with significant enhancements in v2 and v3, and is Generally Available and +supported by Google starting with v1.0, Cloud Storage FUSE is Generally Available and supported by Google, provided that +it is used within its documented supported applications, platforms, and limits. Support requests, feature requests, and +general questions should be submitted as a support request via Google Cloud support channels or via +GitHub[here](https://github.com/GoogleCloudPlatform/gcsfuse/issues). -Cloud Storage FUSE is open source software, released under the +Cloud Storage FUSE is open source software, released under the [Apache license](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/LICENSE). -## _New_ Cloud Storage FUSE V2 -Cloud Storage FUSE V2 provides important stability, functionality, and performance enhancements, including the introduction of a file cache that allows repeat file reads to be served from a local, faster cache storage of choice, such as a Local SSD, Persistent Disk, or even in-memory /tmpfs. The Cloud Storage FUSE file cache makes AI/ML training faster and more cost-effective by reducing the time spent waiting for data, with up to _**2.3x faster training time and 3.4x higher throughput**_ observed in training runs. This is especially valuable for multi epoch training and can serve small and random I/O operations significantly faster. The file cache feature is disabled by default and is enabled by passing a directory to 'cache-dir'. See [overview of caching](https://cloud.google.com/storage/docs/gcsfuse-cache) for more details. +## Cloud Storage Fuse v3 features + +### Streaming Writes + +Streaming writes is the new default write path that uploads data directly to Google Cloud Storage (GCS) as it is +written. +The previous write path temporarily staged the entire write in a local file, uploading to GCS on close or fsync. +This reduces both latency and disk space usage, making it particularly beneficial for large, sequential writes, such as +checkpoint writes, which can be up to _**40% faster**_, as observed in training runs. +See [streaming writes](https://github.com/googlecloudplatform/gcsfuse/blob/master/docs/semantics.md#with-streaming-writes) +for more details. + +### File Cache Parallel Downloads (Default) + +Parallel downloads uses multiple workers to download a file in parallel using the file cache directory as a prefetch +buffer. We recommend using parallel downloads for single-threaded read scenarios that load large files such as model +serving and checkpoint restores, with up to _**9x faster model load times**_. +See [Using Parallel Downloads](https://cloud.google.com/storage/docs/cloud-storage-fuse/file-caching#configure-parallel-downloads) +for more details. + +### Automatic Optimization for High-Performance Machine Types + +GCSFuse now automatically optimizes its configuration when running on specific high-performance Google Cloud machine +types to maximize performance for demanding workloads and effectively utilize the machine's capability. Manually set +values at the time of mount will override these defaults. + +## Cloud Storage FUSE v2 features + +Cloud Storage FUSE V2 provides important stability, functionality, and performance enhancements. + +### File Cache +The file cache allows repeat file reads to be served from a local, faster cache storage of choice, such as a Local SSD, Persistent Disk, or even in-memory /tmpfs. The Cloud Storage FUSE file cache makes AI/ML training faster and more cost-effective by reducing the time spent waiting for data, with up to _**2.3x faster training time and 3.4x higher throughput**_ observed in training runs. This is especially valuable for multi epoch training and can serve small and random I/O operations significantly faster. The file cache feature is disabled by default and is enabled by passing a directory to 'cache-dir'. See [overview of caching](https://cloud.google.com/storage/docs/gcsfuse-cache) for more details. # ABOUT ## What is Cloud Storage FUSE? diff --git a/benchmarks/internal/percentile/duration_test.go b/benchmarks/internal/percentile/duration_test.go index 2d699132f9..4c1efaa6cd 100644 --- a/benchmarks/internal/percentile/duration_test.go +++ b/benchmarks/internal/percentile/duration_test.go @@ -18,7 +18,7 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/benchmarks/internal/percentile" + "github.com/googlecloudplatform/gcsfuse/v3/benchmarks/internal/percentile" . "github.com/jacobsa/ogletest" ) diff --git a/benchmarks/read_full_file/main.go b/benchmarks/read_full_file/main.go index 5ad4cdc903..24542ea208 100644 --- a/benchmarks/read_full_file/main.go +++ b/benchmarks/read_full_file/main.go @@ -30,8 +30,8 @@ import ( "sort" "time" - "github.com/googlecloudplatform/gcsfuse/v2/benchmarks/internal/format" - "github.com/googlecloudplatform/gcsfuse/v2/benchmarks/internal/percentile" + "github.com/googlecloudplatform/gcsfuse/v3/benchmarks/internal/format" + "github.com/googlecloudplatform/gcsfuse/v3/benchmarks/internal/percentile" ) var fDir = flag.String("dir", "", "Directory within which to write the file.") diff --git a/benchmarks/read_within_file/main.go b/benchmarks/read_within_file/main.go index 2f480bc47b..68798d88db 100644 --- a/benchmarks/read_within_file/main.go +++ b/benchmarks/read_within_file/main.go @@ -24,7 +24,7 @@ import ( "os" "time" - "github.com/googlecloudplatform/gcsfuse/v2/benchmarks/internal/format" + "github.com/googlecloudplatform/gcsfuse/v3/benchmarks/internal/format" ) var fFile = flag.String("file", "", "Path to file to read.") diff --git a/benchmarks/stat_files/main.go b/benchmarks/stat_files/main.go index 1ab6f988b1..31e71468dc 100644 --- a/benchmarks/stat_files/main.go +++ b/benchmarks/stat_files/main.go @@ -29,7 +29,7 @@ import ( "golang.org/x/net/context" "golang.org/x/sync/errgroup" - "github.com/googlecloudplatform/gcsfuse/v2/benchmarks/internal/format" + "github.com/googlecloudplatform/gcsfuse/v3/benchmarks/internal/format" "github.com/jacobsa/fuse/fsutil" ) @@ -59,7 +59,7 @@ func createFiles( fileChan := make(chan *os.File) var wg sync.WaitGroup - for i := 0; i < parallelism; i++ { + for range parallelism { wg.Add(1) group.Go(func() (err error) { defer wg.Done() diff --git a/benchmarks/write_locally/main.go b/benchmarks/write_locally/main.go index b3f49d5707..eae51ef199 100644 --- a/benchmarks/write_locally/main.go +++ b/benchmarks/write_locally/main.go @@ -26,7 +26,7 @@ import ( "os" "time" - "github.com/googlecloudplatform/gcsfuse/v2/benchmarks/internal/format" + "github.com/googlecloudplatform/gcsfuse/v3/benchmarks/internal/format" ) var fDir = flag.String("dir", "", "Directory within which to write the file.") diff --git a/benchmarks/write_to_gcs/main.go b/benchmarks/write_to_gcs/main.go index 800daa1e49..270f480089 100644 --- a/benchmarks/write_to_gcs/main.go +++ b/benchmarks/write_to_gcs/main.go @@ -26,7 +26,7 @@ import ( "os" "time" - "github.com/googlecloudplatform/gcsfuse/v2/benchmarks/internal/format" + "github.com/googlecloudplatform/gcsfuse/v3/benchmarks/internal/format" ) var fDir = flag.String("dir", "", "Directory within which to write the file.") @@ -71,10 +71,7 @@ func run() (err error) { for bytesWritten < *fFileSize { // Decide how many bytes to write. - toWrite := *fFileSize - bytesWritten - if toWrite > *fWriteSize { - toWrite = *fWriteSize - } + toWrite := min(*fFileSize-bytesWritten, *fWriteSize) // Write them. _, err = f.Write(buf) diff --git a/cfg/config.go b/cfg/config.go index eb2af698b2..a0860631a9 100644 --- a/cfg/config.go +++ b/cfg/config.go @@ -19,19 +19,445 @@ package cfg import ( "time" + "github.com/googlecloudplatform/gcsfuse/v3/cfg/shared" "github.com/spf13/pflag" "github.com/spf13/viper" ) +// AllFlagOptimizationRules is the generated map from a flag's config-path to its specific rules. +var AllFlagOptimizationRules = map[string]shared.OptimizationRules{"file-system.congestion-threshold": { + BucketTypeOptimization: []shared.BucketTypeOptimization{ + { + BucketType: "zonal", + Value: int64(DefaultCongestionThreshold()), + }, + { + BucketType: "pirlo", + Value: int64(DefaultCongestionThreshold()), + }, + }, +}, "file-system.enable-kernel-reader": { + BucketTypeOptimization: []shared.BucketTypeOptimization{ + { + BucketType: "zonal", + Value: bool(true), + }, + { + BucketType: "pirlo", + Value: bool(true), + }, + }, +}, "file-cache.cache-file-for-range-read": { + Profiles: []shared.ProfileOptimization{ + { + Name: "aiml-serving", + Value: bool(true), + }, + { + Name: "aiml-checkpointing", + Value: bool(true), + }, + }, +}, "write.finalize-file-on-close": { + BucketTypeOptimization: []shared.BucketTypeOptimization{ + { + BucketType: "zonal", + Value: bool(false), + }, + { + BucketType: "pirlo", + Value: bool(true), + }, + }, +}, "implicit-dirs": { + MachineBasedOptimization: []shared.MachineBasedOptimization{ + { + Group: "high-performance", + Value: bool(true), + }, + }, + Profiles: []shared.ProfileOptimization{ + { + Name: "aiml-training", + Value: bool(true), + }, + { + Name: "aiml-serving", + Value: bool(true), + }, + { + Name: "aiml-checkpointing", + Value: bool(true), + }, + }, +}, "file-system.kernel-list-cache-ttl-secs": { + Profiles: []shared.ProfileOptimization{ + { + Name: "aiml-serving", + Value: int64(-1), + }, + }, +}, "file-system.max-background": { + BucketTypeOptimization: []shared.BucketTypeOptimization{ + { + BucketType: "zonal", + Value: int64(DefaultMaxBackground()), + }, + { + BucketType: "pirlo", + Value: int64(DefaultMaxBackground()), + }, + }, +}, "file-system.max-read-ahead-kb": { + BucketTypeOptimization: []shared.BucketTypeOptimization{ + { + BucketType: "zonal", + Value: int64(16384), + }, + { + BucketType: "pirlo", + Value: int64(16384), + }, + }, +}, "metadata-cache.negative-ttl-secs": { + MachineBasedOptimization: []shared.MachineBasedOptimization{ + { + Group: "high-performance", + Value: int64(0), + }, + }, + Profiles: []shared.ProfileOptimization{ + { + Name: "aiml-training", + Value: int64(0), + }, + { + Name: "aiml-serving", + Value: int64(0), + }, + { + Name: "aiml-checkpointing", + Value: int64(0), + }, + }, +}, "metadata-cache.ttl-secs": { + MachineBasedOptimization: []shared.MachineBasedOptimization{ + { + Group: "high-performance", + Value: int64(-1), + }, + }, + Profiles: []shared.ProfileOptimization{ + { + Name: "aiml-training", + Value: int64(-1), + }, + { + Name: "aiml-serving", + Value: int64(-1), + }, + { + Name: "aiml-checkpointing", + Value: int64(-1), + }, + }, +}, "file-system.rename-dir-limit": { + MachineBasedOptimization: []shared.MachineBasedOptimization{ + { + Group: "high-performance", + Value: int64(200000), + }, + }, + Profiles: []shared.ProfileOptimization{ + { + Name: "aiml-checkpointing", + Value: int64(200000), + }, + }, +}, "metadata-cache.stat-cache-max-size-mb": { + MachineBasedOptimization: []shared.MachineBasedOptimization{ + { + Group: "high-performance", + Value: int64(1024), + }, + }, + Profiles: []shared.ProfileOptimization{ + { + Name: "aiml-training", + Value: int64(-1), + }, + { + Name: "aiml-serving", + Value: int64(-1), + }, + { + Name: "aiml-checkpointing", + Value: int64(-1), + }, + }, +}, "write.global-max-blocks": { + MachineBasedOptimization: []shared.MachineBasedOptimization{ + { + Group: "high-performance", + Value: int64(1600), + }, + }, +}, +} + +// machineTypeToGroupMap is the generated map from machine type to the group it belongs to. +var machineTypeToGroupMap = map[string]string{ + "a2-megagpu-16g": "high-performance", + "a2-ultragpu-8g": "high-performance", + "a3-edgegpu-8g": "high-performance", + "a3-highgpu-8g": "high-performance", + "a3-megagpu-8g": "high-performance", + "a3-ultragpu-8g": "high-performance", + "a4-highgpu-8g": "high-performance", + "a4-highgpu-8g-lowmem": "high-performance", + "a4-highgpu-8g-nolssd": "high-performance", + "a4x-highgpu-4g": "high-performance", + "a4x-highgpu-4g-nolssd": "high-performance", + "a4x-maxgpu-4g-metal": "high-performance", + "ct5l-hightpu-8t": "high-performance", + "ct5lp-hightpu-8t": "high-performance", + "ct5p-hightpu-4t": "high-performance", + "ct5p-hightpu-4t-tpu": "high-performance", + "ct6e-standard-4t": "high-performance", + "ct6e-standard-4t-tpu": "high-performance", + "ct6e-standard-8t": "high-performance", + "ct6e-standard-8t-tpu": "high-performance", + "tpu7x-standard-4t": "high-performance", + "tpu7x-standard-4t-tpu": "high-performance", + "tpu7x-ultranet-4t": "high-performance", + "tpu7x-ultranet-4t-tpu": "high-performance", +} + +// ApplyOptimizations modifies the config in-place with optimized values. +// input parameter is optional and provides runtime context for optimizations +// such as bucket type. Pass nil if not available. +func (c *Config) ApplyOptimizations(v *viper.Viper, input *OptimizationInput) map[string]OptimizationResult { + var optimizedFlags = make(map[string]OptimizationResult) + // Skip all optimizations if autoconfig is disabled. + if c.DisableAutoconfig { + return nil + } + + profileName := c.Profile + machineType, err := getMachineType(v) + if err != nil { + // Non-fatal, just means machine-based optimizations won't apply. + machineType = "" + } + c.MachineType = machineType + + // Apply optimizations for each flag that has rules defined. + if !v.IsSet("file-system.congestion-threshold") { + rules := AllFlagOptimizationRules["file-system.congestion-threshold"] + result := getOptimizedValue(&rules, c.FileSystem.CongestionThreshold, profileName, machineType, input, machineTypeToGroupMap) + if result.Optimized { + if val, ok := result.FinalValue.(int64); ok { + if c.FileSystem.CongestionThreshold != val { + c.FileSystem.CongestionThreshold = val + optimizedFlags["file-system.congestion-threshold"] = result + } + } + } + } + if !v.IsSet("file-system.enable-kernel-reader") { + rules := AllFlagOptimizationRules["file-system.enable-kernel-reader"] + result := getOptimizedValue(&rules, c.FileSystem.EnableKernelReader, profileName, machineType, input, machineTypeToGroupMap) + if result.Optimized { + if val, ok := result.FinalValue.(bool); ok { + if c.FileSystem.EnableKernelReader != val { + c.FileSystem.EnableKernelReader = val + optimizedFlags["file-system.enable-kernel-reader"] = result + } + } + } + } + if !v.IsSet("file-cache.cache-file-for-range-read") { + rules := AllFlagOptimizationRules["file-cache.cache-file-for-range-read"] + result := getOptimizedValue(&rules, c.FileCache.CacheFileForRangeRead, profileName, machineType, input, machineTypeToGroupMap) + if result.Optimized { + if val, ok := result.FinalValue.(bool); ok { + if c.FileCache.CacheFileForRangeRead != val { + c.FileCache.CacheFileForRangeRead = val + optimizedFlags["file-cache.cache-file-for-range-read"] = result + } + } + } + } + if !v.IsSet("write.finalize-file-on-close") { + rules := AllFlagOptimizationRules["write.finalize-file-on-close"] + result := getOptimizedValue(&rules, c.Write.FinalizeFileOnClose, profileName, machineType, input, machineTypeToGroupMap) + if result.Optimized { + if val, ok := result.FinalValue.(bool); ok { + if c.Write.FinalizeFileOnClose != val { + c.Write.FinalizeFileOnClose = val + optimizedFlags["write.finalize-file-on-close"] = result + } + } + } + } + if !v.IsSet("implicit-dirs") { + rules := AllFlagOptimizationRules["implicit-dirs"] + result := getOptimizedValue(&rules, c.ImplicitDirs, profileName, machineType, input, machineTypeToGroupMap) + if result.Optimized { + if val, ok := result.FinalValue.(bool); ok { + if c.ImplicitDirs != val { + c.ImplicitDirs = val + optimizedFlags["implicit-dirs"] = result + } + } + } + } + if !v.IsSet("file-system.kernel-list-cache-ttl-secs") { + rules := AllFlagOptimizationRules["file-system.kernel-list-cache-ttl-secs"] + result := getOptimizedValue(&rules, c.FileSystem.KernelListCacheTtlSecs, profileName, machineType, input, machineTypeToGroupMap) + if result.Optimized { + if val, ok := result.FinalValue.(int64); ok { + if c.FileSystem.KernelListCacheTtlSecs != val { + c.FileSystem.KernelListCacheTtlSecs = val + optimizedFlags["file-system.kernel-list-cache-ttl-secs"] = result + } + } + } + } + if !v.IsSet("file-system.max-background") { + rules := AllFlagOptimizationRules["file-system.max-background"] + result := getOptimizedValue(&rules, c.FileSystem.MaxBackground, profileName, machineType, input, machineTypeToGroupMap) + if result.Optimized { + if val, ok := result.FinalValue.(int64); ok { + if c.FileSystem.MaxBackground != val { + c.FileSystem.MaxBackground = val + optimizedFlags["file-system.max-background"] = result + } + } + } + } + if !v.IsSet("file-system.max-read-ahead-kb") { + rules := AllFlagOptimizationRules["file-system.max-read-ahead-kb"] + result := getOptimizedValue(&rules, c.FileSystem.MaxReadAheadKb, profileName, machineType, input, machineTypeToGroupMap) + if result.Optimized { + if val, ok := result.FinalValue.(int64); ok { + if c.FileSystem.MaxReadAheadKb != val { + c.FileSystem.MaxReadAheadKb = val + optimizedFlags["file-system.max-read-ahead-kb"] = result + } + } + } + } + if !v.IsSet("metadata-cache.negative-ttl-secs") { + rules := AllFlagOptimizationRules["metadata-cache.negative-ttl-secs"] + result := getOptimizedValue(&rules, c.MetadataCache.NegativeTtlSecs, profileName, machineType, input, machineTypeToGroupMap) + if result.Optimized { + if val, ok := result.FinalValue.(int64); ok { + if c.MetadataCache.NegativeTtlSecs != val { + c.MetadataCache.NegativeTtlSecs = val + optimizedFlags["metadata-cache.negative-ttl-secs"] = result + } + } + } + } + if !v.IsSet("metadata-cache.ttl-secs") { + rules := AllFlagOptimizationRules["metadata-cache.ttl-secs"] + result := getOptimizedValue(&rules, c.MetadataCache.TtlSecs, profileName, machineType, input, machineTypeToGroupMap) + if result.Optimized { + if val, ok := result.FinalValue.(int64); ok { + if c.MetadataCache.TtlSecs != val { + c.MetadataCache.TtlSecs = val + optimizedFlags["metadata-cache.ttl-secs"] = result + } + } + } + } + if !v.IsSet("file-system.rename-dir-limit") { + rules := AllFlagOptimizationRules["file-system.rename-dir-limit"] + result := getOptimizedValue(&rules, c.FileSystem.RenameDirLimit, profileName, machineType, input, machineTypeToGroupMap) + if result.Optimized { + if val, ok := result.FinalValue.(int64); ok { + if c.FileSystem.RenameDirLimit != val { + c.FileSystem.RenameDirLimit = val + optimizedFlags["file-system.rename-dir-limit"] = result + } + } + } + } + if !v.IsSet("metadata-cache.stat-cache-max-size-mb") { + rules := AllFlagOptimizationRules["metadata-cache.stat-cache-max-size-mb"] + result := getOptimizedValue(&rules, c.MetadataCache.StatCacheMaxSizeMb, profileName, machineType, input, machineTypeToGroupMap) + if result.Optimized { + if val, ok := result.FinalValue.(int64); ok { + if c.MetadataCache.StatCacheMaxSizeMb != val { + c.MetadataCache.StatCacheMaxSizeMb = val + optimizedFlags["metadata-cache.stat-cache-max-size-mb"] = result + } + } + } + } + if !v.IsSet("write.global-max-blocks") { + rules := AllFlagOptimizationRules["write.global-max-blocks"] + result := getOptimizedValue(&rules, c.Write.GlobalMaxBlocks, profileName, machineType, input, machineTypeToGroupMap) + if result.Optimized { + if val, ok := result.FinalValue.(int64); ok { + if c.Write.GlobalMaxBlocks != val { + c.Write.GlobalMaxBlocks = val + optimizedFlags["write.global-max-blocks"] = result + } + } + } + } + return optimizedFlags +} + +type CloudProfilerConfig struct { + AllocatedHeap bool `yaml:"allocated-heap"` + + Cpu bool `yaml:"cpu"` + + Enabled bool `yaml:"enabled"` + + Goroutines bool `yaml:"goroutines"` + + Heap bool `yaml:"heap"` + + Label string `yaml:"label"` + + Mutex bool `yaml:"mutex"` + + ServiceName string `yaml:"service-name"` +} + type Config struct { AppName string `yaml:"app-name"` CacheDir ResolvedPath `yaml:"cache-dir"` + CloudProfiler CloudProfilerConfig `yaml:"cloud-profiler"` + Debug DebugConfig `yaml:"debug"` + DisableAutoconfig bool `yaml:"disable-autoconfig"` + + DisableListAccessCheck bool `yaml:"disable-list-access-check"` + + DummyIo DummyIoConfig `yaml:"dummy-io"` + + EnableAtomicRenameObject bool `yaml:"enable-atomic-rename-object"` + + EnableGoogleLibAuth bool `yaml:"enable-google-lib-auth"` + EnableHns bool `yaml:"enable-hns"` + EnableNewReader bool `yaml:"enable-new-reader"` + + EnableStandardSymlinks bool `yaml:"enable-standard-symlinks"` + + EnableTypeCacheDeprecation bool `yaml:"enable-type-cache-deprecation"` + + EnableUnsupportedPathSupport bool `yaml:"enable-unsupported-path-support"` + FileCache FileCacheConfig `yaml:"file-cache"` FileSystem FileSystemConfig `yaml:"file-system"` @@ -50,14 +476,24 @@ type Config struct { Logging LoggingConfig `yaml:"logging"` + MachineType string `yaml:"machine-type"` + MetadataCache MetadataCacheConfig `yaml:"metadata-cache"` Metrics MetricsConfig `yaml:"metrics"` - Monitoring MonitoringConfig `yaml:"monitoring"` + Mrd MrdConfig `yaml:"mrd"` OnlyDir string `yaml:"only-dir"` + Profile string `yaml:"profile"` + + Read ReadConfig `yaml:"read"` + + Trace TraceConfig `yaml:"trace"` + + WorkloadInsight WorkloadInsightConfig `yaml:"workload-insight"` + Write WriteConfig `yaml:"write"` } @@ -71,6 +507,14 @@ type DebugConfig struct { LogMutex bool `yaml:"log-mutex"` } +type DummyIoConfig struct { + Enable bool `yaml:"enable"` + + PerMbLatency time.Duration `yaml:"per-mb-latency"` + + ReaderLatency time.Duration `yaml:"reader-latency"` +} + type FileCacheConfig struct { CacheFileForRangeRead bool `yaml:"cache-file-for-range-read"` @@ -78,37 +522,67 @@ type FileCacheConfig struct { EnableCrc bool `yaml:"enable-crc"` + EnableExperimentalSharedChunkCache bool `yaml:"enable-experimental-shared-chunk-cache"` + EnableODirect bool `yaml:"enable-o-direct"` EnableParallelDownloads bool `yaml:"enable-parallel-downloads"` + ExcludeRegex string `yaml:"exclude-regex"` + + ExperimentalDisableSizeCalculationFix bool `yaml:"experimental-disable-size-calculation-fix"` + + ExperimentalEnableChunkCache bool `yaml:"experimental-enable-chunk-cache"` + + ExperimentalParallelDownloadsDefaultOn bool `yaml:"experimental-parallel-downloads-default-on"` + + IncludeRegex string `yaml:"include-regex"` + MaxParallelDownloads int64 `yaml:"max-parallel-downloads"` MaxSizeMb int64 `yaml:"max-size-mb"` ParallelDownloadsPerFile int64 `yaml:"parallel-downloads-per-file"` + SharedCacheChunkSizeMb int64 `yaml:"shared-cache-chunk-size-mb"` + WriteBufferSize int64 `yaml:"write-buffer-size"` } type FileSystemConfig struct { + CongestionThreshold int64 `yaml:"congestion-threshold"` + DirMode Octal `yaml:"dir-mode"` DisableParallelDirops bool `yaml:"disable-parallel-dirops"` + EnableKernelReader bool `yaml:"enable-kernel-reader"` + + ExperimentalEnableDentryCache bool `yaml:"experimental-enable-dentry-cache"` + + ExperimentalEnablePirlo bool `yaml:"experimental-enable-pirlo"` + + ExperimentalEnableReaddirplus bool `yaml:"experimental-enable-readdirplus"` + + ExperimentalODirect bool `yaml:"experimental-o-direct"` + FileMode Octal `yaml:"file-mode"` FuseOptions []string `yaml:"fuse-options"` Gid int64 `yaml:"gid"` - HandleSigterm bool `yaml:"handle-sigterm"` - IgnoreInterrupts bool `yaml:"ignore-interrupts"` + InactiveMrdCacheSize int64 `yaml:"inactive-mrd-cache-size"` + KernelListCacheTtlSecs int64 `yaml:"kernel-list-cache-ttl-secs"` - PreconditionErrors bool `yaml:"precondition-errors"` + KernelParamsFile ResolvedPath `yaml:"kernel-params-file"` + + MaxBackground int64 `yaml:"max-background"` + + MaxReadAheadKb int64 `yaml:"max-read-ahead-kb"` RenameDirLimit int64 `yaml:"rename-dir-limit"` @@ -134,10 +608,16 @@ type GcsConnectionConfig struct { CustomEndpoint string `yaml:"custom-endpoint"` + EnableHttpDnsCache bool `yaml:"enable-http-dns-cache"` + ExperimentalEnableJsonRead bool `yaml:"experimental-enable-json-read"` + ExperimentalLocalSocketAddress string `yaml:"experimental-local-socket-address"` + GrpcConnPoolSize int64 `yaml:"grpc-conn-pool-size"` + GrpcPathStrategy DirectPathStrategy `yaml:"grpc-path-strategy"` + HttpClientTimeout time.Duration `yaml:"http-client-timeout"` LimitBytesPerSec float64 `yaml:"limit-bytes-per-sec"` @@ -152,6 +632,14 @@ type GcsConnectionConfig struct { } type GcsRetriesConfig struct { + ChunkRetryDeadlineSecs int64 `yaml:"chunk-retry-deadline-secs"` + + ChunkTransferTimeoutSecs int64 `yaml:"chunk-transfer-timeout-secs"` + + EnableMountRetries bool `yaml:"enable-mount-retries"` + + ExperimentalNonrapidFolderApiStallRetry bool `yaml:"experimental-nonrapid-folder-api-stall-retry"` + MaxRetryAttempts int64 `yaml:"max-retry-attempts"` MaxRetrySleep time.Duration `yaml:"max-retry-sleep"` @@ -181,6 +669,8 @@ type LoggingConfig struct { LogRotate LogRotateLoggingConfig `yaml:"log-rotate"` Severity LogSeverity `yaml:"severity"` + + WireLog ResolvedPath `yaml:"wire-log"` } type MetadataCacheConfig struct { @@ -190,10 +680,18 @@ type MetadataCacheConfig struct { DeprecatedTypeCacheTtl time.Duration `yaml:"deprecated-type-cache-ttl"` + EnableMetadataPrefetch bool `yaml:"enable-metadata-prefetch"` + EnableNonexistentTypeCache bool `yaml:"enable-nonexistent-type-cache"` ExperimentalMetadataPrefetchOnMount string `yaml:"experimental-metadata-prefetch-on-mount"` + MetadataPrefetchEntriesLimit int64 `yaml:"metadata-prefetch-entries-limit"` + + MetadataPrefetchMaxWorkers int64 `yaml:"metadata-prefetch-max-workers"` + + NegativeTtlSecs int64 `yaml:"negative-ttl-secs"` + StatCacheMaxSizeMb int64 `yaml:"stat-cache-max-size-mb"` TtlSecs int64 `yaml:"ttl-secs"` @@ -202,21 +700,41 @@ type MetadataCacheConfig struct { } type MetricsConfig struct { + BufferSize int64 `yaml:"buffer-size"` + CloudMetricsExportIntervalSecs int64 `yaml:"cloud-metrics-export-interval-secs"` - EnableOtel bool `yaml:"enable-otel"` + ExperimentalEnableGrpcMetrics bool `yaml:"experimental-enable-grpc-metrics"` PrometheusPort int64 `yaml:"prometheus-port"` StackdriverExportInterval time.Duration `yaml:"stackdriver-export-interval"` + + UseNewNames bool `yaml:"use-new-names"` + + Workers int64 `yaml:"workers"` +} + +type MrdConfig struct { + PoolSize int64 `yaml:"pool-size"` } -type MonitoringConfig struct { - ExperimentalOpentelemetryCollectorAddress string `yaml:"experimental-opentelemetry-collector-address"` +type ReadConfig struct { + BlockSizeMb int64 `yaml:"block-size-mb"` + + EnableBufferedRead bool `yaml:"enable-buffered-read"` + + GlobalMaxBlocks int64 `yaml:"global-max-blocks"` + + InactiveStreamTimeout time.Duration `yaml:"inactive-stream-timeout"` + + MaxBlocksPerHandle int64 `yaml:"max-blocks-per-handle"` - ExperimentalTracingMode string `yaml:"experimental-tracing-mode"` + MinBlocksPerHandle int64 `yaml:"min-blocks-per-handle"` - ExperimentalTracingSamplingRatio float64 `yaml:"experimental-tracing-sampling-ratio"` + RandomSeekThreshold int64 `yaml:"random-seek-threshold"` + + StartBlocksPerHandle int64 `yaml:"start-blocks-per-handle"` } type ReadStallGcsRetriesConfig struct { @@ -233,12 +751,34 @@ type ReadStallGcsRetriesConfig struct { ReqTargetPercentile float64 `yaml:"req-target-percentile"` } +type TraceConfig struct { + Exporters []string `yaml:"exporters"` + + ProjectId string `yaml:"project-id"` + + SamplingRatio float64 `yaml:"sampling-ratio"` +} + +type WorkloadInsightConfig struct { + ForwardMergeThresholdMb int64 `yaml:"forward-merge-threshold-mb"` + + OutputFile string `yaml:"output-file"` + + Visualize bool `yaml:"visualize"` +} + type WriteConfig struct { BlockSizeMb int64 `yaml:"block-size-mb"` CreateEmptyFile bool `yaml:"create-empty-file"` - ExperimentalEnableStreamingWrites bool `yaml:"experimental-enable-streaming-writes"` + EnableRapidAppends bool `yaml:"enable-rapid-appends"` + + EnableRapidWrites bool `yaml:"enable-rapid-writes"` + + EnableStreamingWrites bool `yaml:"enable-streaming-writes"` + + FinalizeFileOnClose bool `yaml:"finalize-file-on-close"` GlobalMaxBlocks int64 `yaml:"global-max-blocks"` @@ -247,21 +787,85 @@ type WriteConfig struct { func BuildFlagSet(flagSet *pflag.FlagSet) error { - flagSet.BoolP("anonymous-access", "", false, "Authentication is enabled by default. This flag disables authentication") + flagSet.BoolP("anonymous-access", "", false, "This flag disables authentication.") flagSet.StringP("app-name", "", "", "The application name of this mount.") - flagSet.StringP("billing-project", "", "", "Project to use for billing when accessing a bucket enabled with \"Requester Pays\". (The default is none)") + flagSet.StringP("billing-project", "", "", "Project to use for billing when accessing a bucket enabled with \"Requester Pays\".") flagSet.StringP("cache-dir", "", "", "Enables file-caching. Specifies the directory to use for file-cache.") + flagSet.IntP("chunk-retry-deadline-secs", "", 120, "We send larger file uploads in 16 MiB (Legacy Writes) or 32MiB (Streaming Writes) chunks. This flag controls the overall duration that GCSFuse would keep retrying for a single chunk upload completion. 0 means infinity duration for chunk retries.") + + if err := flagSet.MarkHidden("chunk-retry-deadline-secs"); err != nil { + return err + } + + flagSet.IntP("chunk-transfer-timeout-secs", "", 10, "We send larger file uploads in 16 MiB (Legacy Writes) or 32MiB (Streaming Writes) chunks. This flag controls the duration that the HTTP client will wait for a response after making a request to upload a chunk. As an example, a value of 10 indicates that the client will wait 10 seconds for upload completion; otherwise, it cancels the request and retries for that chunk till chunk retry deadline duration. 0 means no timeout.") + + if err := flagSet.MarkHidden("chunk-transfer-timeout-secs"); err != nil { + return err + } + flagSet.StringP("client-protocol", "", "http1", "The protocol used for communicating with the GCS backend. Value can be 'http1' (HTTP/1.1), 'http2' (HTTP/2) or 'grpc'.") flagSet.IntP("cloud-metrics-export-interval-secs", "", 0, "Specifies the interval at which the metrics are uploaded to cloud monitoring") + flagSet.BoolP("cloud-profiler-allocated-heap", "", true, "Enables allocated heap (HeapProfileAllocs) profiling. This only works when --enable-cloud-profiler is set to true.") + + if err := flagSet.MarkHidden("cloud-profiler-allocated-heap"); err != nil { + return err + } + + flagSet.BoolP("cloud-profiler-cpu", "", true, "Enables cpu profiling. This only works when --enable-cloud-profiler is set to true.") + + if err := flagSet.MarkHidden("cloud-profiler-cpu"); err != nil { + return err + } + + flagSet.BoolP("cloud-profiler-goroutines", "", false, "Enables goroutines cloud-profiler. This only works when --enable-cloud-profiler is set to true.") + + if err := flagSet.MarkHidden("cloud-profiler-goroutines"); err != nil { + return err + } + + flagSet.BoolP("cloud-profiler-heap", "", true, "Enables heap cloud-profiler. This only works when --enable-cloud-profiler is set to true.") + + if err := flagSet.MarkHidden("cloud-profiler-heap"); err != nil { + return err + } + + flagSet.StringP("cloud-profiler-label", "", "gcsfuse-0.0.0", "Allow setting a profile label to uniquely identify and compare cloud-profiler data with other profiles. This only works when --enable-cloud-profiler is set to true.") + + if err := flagSet.MarkHidden("cloud-profiler-label"); err != nil { + return err + } + + flagSet.BoolP("cloud-profiler-mutex", "", false, "Enables mutex cloud-profiler. This only works when --enable-cloud-profiler is set to true.") + + if err := flagSet.MarkHidden("cloud-profiler-mutex"); err != nil { + return err + } + + flagSet.StringP("cloud-profiler-service-name", "", "gcsfuse", "The service name for cloud-profiler. This only works when --enable-cloud-profiler is set to true.") + + if err := flagSet.MarkHidden("cloud-profiler-service-name"); err != nil { + return err + } + + flagSet.IntP("congestion-threshold", "", 0, "Sets the congestion threshold for background requests. When the number of outstanding requests exceeds this threshold, the kernel may start blocking new requests. 0 means system default (typically 75% of max-background; 9).") + + if err := flagSet.MarkHidden("congestion-threshold"); err != nil { + return err + } + flagSet.BoolP("create-empty-file", "", false, "For a new file, it creates an empty file in Cloud Storage bucket as a hold.") - flagSet.StringP("custom-endpoint", "", "", "Specifies an alternative custom endpoint for fetching data. Should only be used for testing. The custom endpoint must support the equivalent resources and operations as the GCS JSON endpoint, https://storage.googleapis.com/storage/v1. If a custom endpoint is not specified, GCSFuse uses the global GCS JSON API endpoint, https://storage.googleapis.com/storage/v1.") + if err := flagSet.MarkHidden("create-empty-file"); err != nil { + return err + } + + flagSet.StringP("custom-endpoint", "", "", "To specify a custom storage endpoint, ensure it supports the same resources as the default storage.googleapis.com:443 and includes the port number.") flagSet.BoolP("debug_fs", "", false, "This flag is unused.") @@ -299,47 +903,161 @@ func BuildFlagSet(flagSet *pflag.FlagSet) error { flagSet.StringP("dir-mode", "", "0755", "Permissions bits for directories, in octal.") + flagSet.BoolP("disable-autoconfig", "", false, "Disable optimizing configuration automatically for a machine") + + if err := flagSet.MarkHidden("disable-autoconfig"); err != nil { + return err + } + + flagSet.BoolP("disable-list-access-check", "", true, "Disables the list object based access check during mount operation") + + if err := flagSet.MarkHidden("disable-list-access-check"); err != nil { + return err + } + flagSet.BoolP("disable-parallel-dirops", "", false, "Specifies whether to allow parallel dir operations (lookups and readers)") if err := flagSet.MarkHidden("disable-parallel-dirops"); err != nil { return err } + flagSet.DurationP("dummy-io-per-mb-latency", "", 0*time.Nanosecond, "Simulates reading from the reader latency in dummy I/O mode. This value is only used when dummy I/O mode is enabled.") + + if err := flagSet.MarkHidden("dummy-io-per-mb-latency"); err != nil { + return err + } + + flagSet.DurationP("dummy-io-reader-latency", "", 0*time.Nanosecond, "Simulates reader creation latency in dummy I/O mode. This value is only used when dummy I/O mode is enabled.") + + if err := flagSet.MarkHidden("dummy-io-reader-latency"); err != nil { + return err + } + + flagSet.BoolP("enable-atomic-rename-object", "", true, "Enables support for atomic rename object operation on HNS bucket.") + + if err := flagSet.MarkHidden("enable-atomic-rename-object"); err != nil { + return err + } + + flagSet.BoolP("enable-buffered-read", "", false, "When enabled, read starts using buffer to prefetch (asynchronous and in parallel) data from GCS. This improves performance for large file sequential reads. Note: Enabling this flag can increase the memory usage significantly.") + + flagSet.BoolP("enable-cloud-profiler", "", false, "Enables cloud-profiler, by default disabled.") + + if err := flagSet.MarkHidden("enable-cloud-profiler"); err != nil { + return err + } + + flagSet.BoolP("enable-dummy-io", "", false, "Enable dummy I/O mode for testing purposes. In this mode all reads and writes are simulated and no actual data is transferred to or from Cloud Storage. All the metadata operations like object listing and stats are real.") + + if err := flagSet.MarkHidden("enable-dummy-io"); err != nil { + return err + } + flagSet.BoolP("enable-empty-managed-folders", "", false, "This handles the corner case in listing managed folders. There are two corner cases (a) empty managed folder (b) nested managed folder which doesn't contain any descendent as object. This flag always works in conjunction with --implicit-dirs flag. (a) If only ImplicitDirectories is true, all managed folders are listed other than above two mentioned cases. (b) If both ImplicitDirectories and EnableEmptyManagedFolders are true, then all the managed folders are listed including the above-mentioned corner case. (c) If ImplicitDirectories is false then no managed folders are listed irrespective of enable-empty-managed-folders flag.") if err := flagSet.MarkHidden("enable-empty-managed-folders"); err != nil { return err } + flagSet.BoolP("enable-experimental-shared-chunk-cache", "", false, "[EXPERIMENTAL] Enable chunk-based shared cache that allows multiple gcsfuse mount instances to safely share the same cache directory (e.g., on NFS). Uses fixed size chunks with atomic operations (write + rename) to download chunk file without locks. Ideal for distributed environments where multiple nodes need to share cached GCS data.") + + flagSet.BoolP("enable-google-lib-auth", "", true, "Enable google library authentication method to fetch the credentials") + + if err := flagSet.MarkHidden("enable-google-lib-auth"); err != nil { + return err + } + flagSet.BoolP("enable-hns", "", true, "Enables support for HNS buckets") if err := flagSet.MarkHidden("enable-hns"); err != nil { return err } - flagSet.BoolP("enable-nonexistent-type-cache", "", false, "Once set, if an inode is not found in GCS, a type cache entry with type NonexistentType will be created. This also means new file/dir created might not be seen. For example, if this flag is set, and metadata-cache-ttl-secs is set, then if we create the same file/node in the meantime using the same mount, since we are not refreshing the cache, it will still return nil.") + flagSet.BoolP("enable-http-dns-cache", "", true, "Enables DNS cache for HTTP/1 connections") + + if err := flagSet.MarkHidden("enable-http-dns-cache"); err != nil { + return err + } + + flagSet.BoolP("enable-kernel-reader", "", false, "Enables kernel reader, disables prefetching gcsfuse side and relies on kernel read-ahead and page-cache.") + + if err := flagSet.MarkHidden("enable-kernel-reader"); err != nil { + return err + } + + flagSet.BoolP("enable-metadata-prefetch", "", false, "Enables background prefetching of object metadata when a directory is first opened. This reduces latency for subsequent file lookups by pre-filling the metadata cache.") + + flagSet.BoolP("enable-mount-retries", "", false, "If true, enables retry logic in GCSFuse during the mount sequence for additional errors (such as metadata server readiness delays, IAM propagation delays, and temporary bucket non-existence). Intended specifically for the GKE GCSFuse CSI Driver.") + + if err := flagSet.MarkHidden("enable-mount-retries"); err != nil { + return err + } - flagSet.BoolP("enable-otel", "", false, "Specifies whether to use OpenTelemetry for capturing and exporting metrics. If false, use OpenCensus.") + flagSet.BoolP("enable-new-reader", "", true, "Enables support for new reader implementation.") - if err := flagSet.MarkHidden("enable-otel"); err != nil { + if err := flagSet.MarkHidden("enable-new-reader"); err != nil { return err } - flagSet.BoolP("enable-read-stall-retry", "", false, "To turn on/off retries for stalled read requests. This is based on a timeout that changes depending on how long similar requests took in the past.") + flagSet.BoolP("enable-nonexistent-type-cache", "", false, "Once set, if an inode is not found in GCS, a type cache entry with type NonexistentType will be created. This also means new file/dir created might not be seen. For example, if this flag is set, and metadata-cache-ttl-secs is set, then if we create the same file/node in the meantime using the same mount, since we are not refreshing the cache, it will still return nil. This flag has been deprecated in favour of a single unified flag metadata-cache-negative-ttl-secs.") + + flagSet.BoolP("enable-rapid-appends", "", true, "Enables support for appends to unfinalized object using streaming writes") + + flagSet.BoolP("enable-rapid-writes", "", false, "For pirlo, toggles between using STANDARD class and RAPID class for writes.") + + flagSet.BoolP("enable-read-stall-retry", "", true, "To turn on/off retries for stalled read requests. This is based on a timeout that changes depending on how long similar requests took in the past.") if err := flagSet.MarkHidden("enable-read-stall-retry"); err != nil { return err } + flagSet.BoolP("enable-standard-symlinks", "", true, "Enables the creation and reading of symbolic links using the standard GCS representation. When enabled, new symlinks created via GCSFuse mount ensure compatibility with other GCS clients like Storage Transfer Service (STS).") + + if err := flagSet.MarkHidden("enable-standard-symlinks"); err != nil { + return err + } + + flagSet.BoolP("enable-streaming-writes", "", true, "Enables streaming uploads during write file operation.") + + flagSet.BoolP("enable-type-cache-deprecation", "", true, "Enables support to deprecate type cache.") + + if err := flagSet.MarkHidden("enable-type-cache-deprecation"); err != nil { + return err + } + + flagSet.BoolP("enable-unsupported-path-support", "", true, "Enables support for file system paths with unsupported GCS names (e.g., names containing '//' or starting with /). When set, GCSFuse will ignore these objects during listing and copying operations. For rename and delete operations, the flag allows the action to proceed for all specified objects, including those with unsupported names.") + + if err := flagSet.MarkHidden("enable-unsupported-path-support"); err != nil { + return err + } + + flagSet.BoolP("experimental-enable-dentry-cache", "", false, "When enabled, it sets the Dentry cache entry timeout same as metadata-cache-ttl. This enables kernel to use cached entry to map the file paths to inodes, instead of making LookUpInode calls to GCSFuse.") + + if err := flagSet.MarkHidden("experimental-enable-dentry-cache"); err != nil { + return err + } + + flagSet.BoolP("experimental-enable-grpc-metrics", "", true, "Enables support for gRPC metrics") + + if err := flagSet.MarkHidden("experimental-enable-grpc-metrics"); err != nil { + return err + } + flagSet.BoolP("experimental-enable-json-read", "", false, "By default, GCSFuse uses the GCS XML API to get and read objects. When this flag is specified, GCSFuse uses the GCS JSON API instead.\"") if err := flagSet.MarkDeprecated("experimental-enable-json-read", "Experimental flag: could be dropped even in a minor release."); err != nil { return err } - flagSet.BoolP("experimental-enable-streaming-writes", "", false, "Enables streaming uploads during write file operation.") + flagSet.BoolP("experimental-enable-pirlo", "", false, "Enables support for pirlo.") - if err := flagSet.MarkHidden("experimental-enable-streaming-writes"); err != nil { + if err := flagSet.MarkHidden("experimental-enable-pirlo"); err != nil { + return err + } + + flagSet.BoolP("experimental-enable-readdirplus", "", false, "Enables ReadDirPlus capability") + + if err := flagSet.MarkHidden("experimental-enable-readdirplus"); err != nil { return err } @@ -349,35 +1067,35 @@ func BuildFlagSet(flagSet *pflag.FlagSet) error { return err } - flagSet.StringP("experimental-metadata-prefetch-on-mount", "", "disabled", "Experimental: This indicates whether or not to prefetch the metadata (prefilling of metadata caches and creation of inodes) of the mounted bucket at the time of mounting the bucket. Supported values: \"disabled\", \"sync\" and \"async\". Any other values will return error on mounting. This is applicable only to static mounting, and not to dynamic mounting.") + flagSet.StringP("experimental-local-socket-address", "", "", "The local socket address to bind to. This is useful in multi-NIC scenarios. This is an experimental flag.") - if err := flagSet.MarkDeprecated("experimental-metadata-prefetch-on-mount", "Experimental flag: could be removed even in a minor release."); err != nil { + if err := flagSet.MarkHidden("experimental-local-socket-address"); err != nil { return err } - flagSet.StringP("experimental-opentelemetry-collector-address", "", "", "Experimental: Export metrics to the OpenTelemetry collector at this address.") + flagSet.StringP("experimental-metadata-prefetch-on-mount", "", "disabled", "Experimental: This indicates whether or not to prefetch the metadata (prefilling of metadata caches and creation of inodes) of the mounted bucket at the time of mounting the bucket. Supported values: \"disabled\", \"sync\" and \"async\". Any other values will return error on mounting. This is applicable only to static mounting, and not to dynamic mounting.") - if err := flagSet.MarkDeprecated("experimental-opentelemetry-collector-address", "Experimental flag: could be dropped even in a minor release."); err != nil { + if err := flagSet.MarkDeprecated("experimental-metadata-prefetch-on-mount", "Experimental flag: could be removed even in a minor release."); err != nil { return err } - flagSet.StringP("experimental-tracing-mode", "", "", "Experimental: specify tracing mode") + flagSet.BoolP("experimental-nonrapid-folder-api-stall-retry", "", false, "Enables stall-retry-fix for folder APIs for non-rapid buckets.") - if err := flagSet.MarkHidden("experimental-tracing-mode"); err != nil { + if err := flagSet.MarkHidden("experimental-nonrapid-folder-api-stall-retry"); err != nil { return err } - flagSet.Float64P("experimental-tracing-sampling-ratio", "", 0, "Experimental: Trace sampling ratio") + flagSet.BoolP("experimental-o-direct", "", false, "Experimental: Bypasses the kernel's page cache for file reads and writes. When enabled, all I/O operations are sent directly to the GCSFuse process.") - if err := flagSet.MarkHidden("experimental-tracing-sampling-ratio"); err != nil { + if err := flagSet.MarkHidden("experimental-o-direct"); err != nil { return err } flagSet.BoolP("file-cache-cache-file-for-range-read", "", false, "Whether to cache file for range reads.") - flagSet.IntP("file-cache-download-chunk-size-mb", "", 50, "Size of chunks in MiB that each concurrent request downloads.") + flagSet.IntP("file-cache-download-chunk-size-mb", "", 200, "Size of chunks in MiB that each concurrent request downloads.") - flagSet.BoolP("file-cache-enable-crc", "", false, "Performs CRC to ensure that file is correctly downloaded into cache.") + flagSet.BoolP("file-cache-enable-crc", "", false, "Performs CRC to ensure that file is correctly downloaded into cache. No op for rapid storage.") if err := flagSet.MarkHidden("file-cache-enable-crc"); err != nil { return err @@ -391,12 +1109,40 @@ func BuildFlagSet(flagSet *pflag.FlagSet) error { flagSet.BoolP("file-cache-enable-parallel-downloads", "", false, "Enable parallel downloads.") + flagSet.StringP("file-cache-exclude-regex", "", "", "Exclude file paths (in the format bucket_name/object_key) specified by this regex from file caching.") + + flagSet.BoolP("file-cache-experimental-disable-size-calculation-fix", "", false, "Disable the fix in calculation of disk-utilization of file-cache.") + + if err := flagSet.MarkHidden("file-cache-experimental-disable-size-calculation-fix"); err != nil { + return err + } + + flagSet.BoolP("file-cache-experimental-enable-chunk-cache", "", false, "Enable chunk cache mode for random I/O optimization that downloads only requested blocks.") + + if err := flagSet.MarkHidden("file-cache-experimental-enable-chunk-cache"); err != nil { + return err + } + + flagSet.BoolP("file-cache-experimental-parallel-downloads-default-on", "", true, "Enable parallel downloads by default on experimental basis.") + + if err := flagSet.MarkHidden("file-cache-experimental-parallel-downloads-default-on"); err != nil { + return err + } + + flagSet.StringP("file-cache-include-regex", "", "", "Include file paths (in the format bucket_name/object_key) specified by this regex for file caching.") + flagSet.IntP("file-cache-max-parallel-downloads", "", DefaultMaxParallelDownloads(), "Sets an uber limit of number of concurrent file download requests that are made across all files.") flagSet.IntP("file-cache-max-size-mb", "", -1, "Maximum size of the file-cache in MiBs") flagSet.IntP("file-cache-parallel-downloads-per-file", "", 16, "Number of concurrent download requests per file.") + flagSet.IntP("file-cache-shared-cache-chunk-size-mb", "", 8, "Chunk size in MiBs for shared chunk cache. Each chunk is downloaded on-demand.") + + if err := flagSet.MarkHidden("file-cache-shared-cache-chunk-size-mb"); err != nil { + return err + } + flagSet.IntP("file-cache-write-buffer-size", "", 4194304, "Size of in-memory buffer that is used per goroutine in parallel downloads while writing to file-cache.") if err := flagSet.MarkHidden("file-cache-write-buffer-size"); err != nil { @@ -405,71 +1151,161 @@ func BuildFlagSet(flagSet *pflag.FlagSet) error { flagSet.StringP("file-mode", "", "0644", "Permissions bits for files, in octal.") + flagSet.BoolP("finalize-file-on-close", "", false, "Finalizes the files on close for Rapid storage. Appends will be slower on finalized files.") + + if err := flagSet.MarkHidden("finalize-file-on-close"); err != nil { + return err + } + flagSet.BoolP("foreground", "", false, "Stay in the foreground after mounting.") flagSet.IntP("gid", "", -1, "GID owner of all inodes.") - flagSet.BoolP("handle-sigterm", "", true, "Instructs gcsfuse to handle SIGTERM to gracefully shutdown") + flagSet.StringP("grpc-path-strategy", "", "direct-path-with-fallback", "Strategy for DirectPath connectivity when client-protocol=grpc. Options: 'direct-path-only' (fail if unavailable), 'direct-path-with-fallback' (always fallback to HTTP/1 when direct path is not available).") - if err := flagSet.MarkHidden("handle-sigterm"); err != nil { + if err := flagSet.MarkHidden("grpc-path-strategy"); err != nil { return err } - flagSet.DurationP("http-client-timeout", "", 0*time.Nanosecond, "The time duration that http client will wait to get response from the server. The default value 0 indicates no timeout.") + flagSet.DurationP("http-client-timeout", "", 0*time.Nanosecond, "The time duration that http client will wait to get response from the server. A value of 0 indicates no timeout.") - flagSet.BoolP("ignore-interrupts", "", true, "Instructs gcsfuse to ignore system interrupt signals (like SIGINT, triggered by Ctrl+C). This prevents those signals from immediately terminating gcsfuse inflight operations. (default: true)") + flagSet.BoolP("ignore-interrupts", "", true, "Instructs gcsfuse to ignore system interrupt signals (like SIGINT, triggered by Ctrl+C). This prevents those signals from immediately terminating gcsfuse inflight operations.") flagSet.BoolP("implicit-dirs", "", false, "Implicitly define directories based on content. See files and directories in docs/semantics for more information") + flagSet.IntP("inactive-mrd-cache-size", "", 1000, "Sets the cache-size of inactive (no open file) MRD instances. When this limit is exceeded, the least recently inactive MRD instances will be closed. Set to 0 to disable the cache, which will keep all the inactive MRD instances open forever.") + + if err := flagSet.MarkHidden("inactive-mrd-cache-size"); err != nil { + return err + } + flagSet.IntP("kernel-list-cache-ttl-secs", "", 0, "How long the directory listing (output of ls <dir>) should be cached in the kernel page cache. If a particular directory cache entry is kept by kernel for longer than TTL, then it will be sent for invalidation by gcsfuse on next opendir (comes in the start, as part of next listing) call. 0 means no caching. Use -1 to cache for lifetime (no ttl). Negative value other than -1 will throw error.") - flagSet.StringP("key-file", "", "", "Absolute path to JSON key file for use with GCS. (The default is none, Google application default credentials used)") + flagSet.StringP("kernel-params-file", "", "", "File path used to communicate various kernel parameters to CSI Driver in GKE environment.") + + if err := flagSet.MarkHidden("kernel-params-file"); err != nil { + return err + } + + flagSet.StringP("key-file", "", "", "Absolute path to JSON key file for use with GCS. If this flag is left unset, Google application default credentials are used.") flagSet.Float64P("limit-bytes-per-sec", "", -1, "Bandwidth limit for reading data, measured over a 30-second window. (use -1 for no limit)") - flagSet.Float64P("limit-ops-per-sec", "", -1, "Operations per second limit, measured over a 30-second window (use -1 for no limit)") + flagSet.Float64P("limit-ops-per-sec", "", -1, "Operations per second limit, measured over a 30-second window (use -1 for no limit)") + + flagSet.StringP("log-file", "", "", "The file for storing logs that can be parsed by fluentd. When not provided, plain text logs are printed to stdout when Cloud Storage FUSE is run in the foreground, or to syslog when Cloud Storage FUSE is run in the background.") + + flagSet.StringP("log-format", "", "json", "The format of the log file: 'text' or 'json'.") + + flagSet.IntP("log-rotate-backup-file-count", "", 10, "The maximum number of backup log files to retain after they have been rotated. A value of 0 indicates all backup files are retained.") + + flagSet.BoolP("log-rotate-compress", "", true, "Controls whether the rotated log files should be compressed using gzip.") + + flagSet.IntP("log-rotate-max-file-size-mb", "", 512, "The maximum size in megabytes that a log file can reach before it is rotated.") + + flagSet.StringP("log-severity", "", "info", "Specifies the logging severity expressed as one of [trace, debug, info, warning, error, off]") + + flagSet.StringP("machine-type", "", "", "Type of the machine on which gcsfuse is being run e.g. a3-highgpu-4g") + + if err := flagSet.MarkHidden("machine-type"); err != nil { + return err + } + + flagSet.IntP("max-background", "", 0, "Sets the maximum number of outstanding background requests (e.g., request corresponding to kernel readahead, writeback cache etc.) that the kernel will send to the FUSE daemon. 0 means system default (typically 12).") + + if err := flagSet.MarkHidden("max-background"); err != nil { + return err + } + + flagSet.IntP("max-conns-per-host", "", 0, "The max number of TCP connections allowed per server. This is effective when client-protocol is set to 'http1'. A value of 0 indicates no limit on TCP connections (limited by the machine specifications).") + + flagSet.IntP("max-idle-conns-per-host", "", 100, "The number of maximum idle connections allowed per server.") + + flagSet.IntP("max-read-ahead-kb", "", 0, "Sets max kernel-read-ahead for the mount in KiB. 0 means system default. Requires sudo permission to set this value, otherwise the value will be ignored and system default will be used.") + + if err := flagSet.MarkHidden("max-read-ahead-kb"); err != nil { + return err + } + + flagSet.IntP("max-retry-attempts", "", 0, "It sets a limit on the total number of attempts (including the initial call) made for an operation if it fails, preventing endless retry loops. For example, a value of 5 means up to 5 total attempts (1 initial call plus 4 retries). A value of 0 indicates unlimited attempts.") + + flagSet.DurationP("max-retry-duration", "", 0*time.Nanosecond, "This is currently unused.") + + if err := flagSet.MarkDeprecated("max-retry-duration", "This is currently unused."); err != nil { + return err + } + + flagSet.DurationP("max-retry-sleep", "", 30000000000*time.Nanosecond, "The maximum backoff sleep duration allowed between retry attempts. Once the exponential backoff exceeds this limit, subsequent retries will use this constant sleep value.") + + flagSet.IntP("metadata-cache-negative-ttl-secs", "", 5, "The negative-ttl-secs value in seconds to be used for expiring negative entries in metadata-cache. It can be set to -1 for no-ttl, 0 for no cache and > 0 for ttl-controlled negative entries in metadata-cache. Any value set below -1 will throw an error.") + + flagSet.IntP("metadata-cache-ttl-secs", "", 60, "The ttl value in seconds to be used for expiring items in metadata-cache. It can be set to -1 for no-ttl, 0 for no cache and > 0 for ttl-controlled metadata-cache. Any value set below -1 will throw an error.") + + flagSet.IntP("metadata-prefetch-entries-limit", "", 5000, "The maximum number of metadata entries (files and directories) to prefetch into the cache upon a prefetch trigger. Since a single GCS List call is capped at 5000 results, values higher than 5000 will trigger multiple sequential GCS List calls per directory.\n") + + flagSet.IntP("metadata-prefetch-max-workers", "", 10, "The maximum number of concurrent goroutines (workers) allowed to perform metadata prefetching across all directories.\n") + + flagSet.IntP("metrics-buffer-size", "", 256, "The maximum number of histogram metric updates in the queue.") + + if err := flagSet.MarkHidden("metrics-buffer-size"); err != nil { + return err + } + + flagSet.BoolP("metrics-use-new-names", "", false, "Use the new metric names.") - flagSet.StringP("log-file", "", "", "The file for storing logs that can be parsed by fluentd. When not provided, plain text logs are printed to stdout when Cloud Storage FUSE is run in the foreground, or to syslog when Cloud Storage FUSE is run in the background.") + if err := flagSet.MarkHidden("metrics-use-new-names"); err != nil { + return err + } - flagSet.StringP("log-format", "", "json", "The format of the log file: 'text' or 'json'.") + flagSet.IntP("metrics-workers", "", 3, "The number of workers that update histogram metrics concurrently.") - flagSet.IntP("log-rotate-backup-file-count", "", 10, "The maximum number of backup log files to retain after they have been rotated. The default value is 10. When value is set to 0, all backup files are retained.") + if err := flagSet.MarkHidden("metrics-workers"); err != nil { + return err + } - flagSet.BoolP("log-rotate-compress", "", true, "Controls whether the rotated log files should be compressed using gzip.") + flagSet.IntP("mrd-pool-size", "", 4, "Specifies the MRD pool size to be used for zonal buckets. The value should be more than 0.") - flagSet.IntP("log-rotate-max-file-size-mb", "", 512, "The maximum size in megabytes that a log file can reach before it is rotated.") + if err := flagSet.MarkHidden("mrd-pool-size"); err != nil { + return err + } - flagSet.StringP("log-severity", "", "info", "Specifies the logging severity expressed as one of [trace, debug, info, warning, error, off]") + flagSet.StringSliceP("o", "", []string{}, "Additional system-specific mount options. Multiple options can be passed as comma separated. For readonly, use --o ro") - flagSet.IntP("max-conns-per-host", "", 0, "The max number of TCP connections allowed per server. This is effective when client-protocol is set to 'http1'. The default value 0 indicates no limit on TCP connections (limited by the machine specifications).") + flagSet.StringP("only-dir", "", "", "Mount only a specific directory within the bucket. See docs/mounting for more information") - flagSet.IntP("max-idle-conns-per-host", "", 100, "The number of maximum idle connections allowed per server.") + flagSet.StringP("profile", "", "", "The name of the profile to apply. e.g. aiml-training, aiml-serving, aiml-checkpointing") - flagSet.IntP("max-retry-attempts", "", 0, "It sets a limit on the number of times an operation will be retried if it fails, preventing endless retry loops. The default value 0 indicates no limit.") + flagSet.IntP("prometheus-port", "", 0, "Expose Prometheus metrics endpoint on this port and a path of /metrics.") - flagSet.DurationP("max-retry-duration", "", 0*time.Nanosecond, "This is currently unused.") + flagSet.IntP("read-block-size-mb", "", 16, "Specifies the block size for buffered reads. The value should be more than 0. This is used to read data in chunks from GCS.") - if err := flagSet.MarkDeprecated("max-retry-duration", "This is currently unused."); err != nil { + if err := flagSet.MarkHidden("read-block-size-mb"); err != nil { return err } - flagSet.DurationP("max-retry-sleep", "", 30000000000*time.Nanosecond, "The maximum duration allowed to sleep in a retry loop with exponential backoff for failed requests to GCS backend. Once the backoff duration exceeds this limit, the retry continues with this specified maximum value.") + flagSet.IntP("read-global-max-blocks", "", 40, "Specifies the maximum number of blocks available for buffered reads across all file-handles. The value should be >= 0 or -1 (for infinite blocks). A value of 0 disables buffered reads.") - flagSet.IntP("metadata-cache-ttl-secs", "", 60, "The ttl value in seconds to be used for expiring items in metadata-cache. It can be set to -1 for no-ttl, 0 for no cache and > 0 for ttl-controlled metadata-cache. Any value set below -1 will throw an error.") + flagSet.DurationP("read-inactive-stream-timeout", "", 10000000000*time.Nanosecond, "Duration of inactivity after which an open GCS read stream is automatically closed. This helps conserve resources when a file handle remains open without active Read calls. A value of '0s' disables this timeout.") - flagSet.StringSliceP("o", "", []string{}, "Additional system-specific mount options. Multiple options can be passed as comma separated. For readonly, use --o ro") + if err := flagSet.MarkHidden("read-inactive-stream-timeout"); err != nil { + return err + } - flagSet.StringP("only-dir", "", "", "Mount only a specific directory within the bucket. See docs/mounting for more information") + flagSet.IntP("read-max-blocks-per-handle", "", 20, "Specifies the maximum number of blocks to be used by a single file handle for buffered reads. The value should be >= 0 or -1 (for infinite blocks). A value of 0 disables buffered reads.") - flagSet.BoolP("precondition-errors", "", true, "Throw Stale NFS file handle error in case the object being synced or read from is modified by some other concurrent process. This helps prevent silent data loss or data corruption.") + if err := flagSet.MarkHidden("read-max-blocks-per-handle"); err != nil { + return err + } - if err := flagSet.MarkHidden("precondition-errors"); err != nil { + flagSet.IntP("read-min-blocks-per-handle", "", 4, "Specifies the minimum number of blocks required by a file-handle to start reading via buffered reads. The value should be >= 1 or \"read-max-blocks-per-handle\".") + + if err := flagSet.MarkHidden("read-min-blocks-per-handle"); err != nil { return err } - flagSet.IntP("prometheus-port", "", 0, "Expose Prometheus metrics endpoint on this port and a path of /metrics.") + flagSet.IntP("read-random-seek-threshold", "", 3, "Specifies the random seek threshold to switch to another reader when random reads are detected.") - if err := flagSet.MarkHidden("prometheus-port"); err != nil { + if err := flagSet.MarkHidden("read-random-seek-threshold"); err != nil { return err } @@ -503,27 +1339,33 @@ func BuildFlagSet(flagSet *pflag.FlagSet) error { return err } + flagSet.IntP("read-start-blocks-per-handle", "", 1, "Specifies the number of blocks to be prefetched on the first read.") + + if err := flagSet.MarkHidden("read-start-blocks-per-handle"); err != nil { + return err + } + flagSet.IntP("rename-dir-limit", "", 0, "Allow rename a directory containing fewer descendants than this limit.") - flagSet.Float64P("retry-multiplier", "", 2, "Param for exponential backoff algorithm, which is used to increase waiting time b/w two consecutive retries.") + flagSet.Float64P("retry-multiplier", "", 2, "The multiplier factor by which the retry backoff duration increases after each failed attempt. For example, a multiplier of 2.0 doubles the backoff sleep duration for each subsequent retry.") flagSet.BoolP("reuse-token-from-url", "", true, "If false, the token acquired from token-url is not reused.") flagSet.IntP("sequential-read-size-mb", "", 200, "File chunk size to read from GCS in one call. Need to specify the value in MB. ChunkSize less than 1MB is not supported") - flagSet.DurationP("stackdriver-export-interval", "", 0*time.Nanosecond, "Export metrics to stackdriver with this interval. The default value 0 indicates no exporting.") + flagSet.DurationP("stackdriver-export-interval", "", 0*time.Nanosecond, "Export metrics to stackdriver with this interval. A value of 0 indicates no exporting.") if err := flagSet.MarkDeprecated("stackdriver-export-interval", "Please use --cloud-metrics-export-interval-secs instead."); err != nil { return err } - flagSet.IntP("stat-cache-capacity", "", 20460, "How many entries can the stat-cache hold (impacts memory consumption). This flag has been deprecated (starting v2.0) and in favor of stat-cache-max-size-mb. For now, the value of stat-cache-capacity will be translated to the next higher corresponding value of stat-cache-max-size-mb (assuming stat-cache entry-size ~= 1640 bytes, including 1400 for positive entry and 240 for corresponding negative entry), if stat-cache-max-size-mb is not set.\"") + flagSet.IntP("stat-cache-capacity", "", 20460, "How many entries can the stat-cache hold (impacts memory consumption). This flag has been deprecated (starting v2.0) and in favor of stat-cache-max-size-mb. For now, the value of stat-cache-capacity will be translated to the next higher corresponding value of stat-cache-max-size-mb (assuming stat-cache entry-size ~= 1720 bytes, including 1464 for positive entry and 256 for corresponding negative entry), if stat-cache-max-size-mb is not set.\"") if err := flagSet.MarkDeprecated("stat-cache-capacity", "Please use --stat-cache-max-size-mb instead."); err != nil { return err } - flagSet.IntP("stat-cache-max-size-mb", "", 32, "The maximum size of stat-cache in MiBs. It can also be set to -1 for no-size-limit, 0 for no cache. Values below -1 are not supported.") + flagSet.IntP("stat-cache-max-size-mb", "", 34, "The maximum size of stat-cache in MiBs. It can also be set to -1 for no-size-limit, 0 for no cache. Values below -1 are not supported.") flagSet.DurationP("stat-cache-ttl", "", 60000000000*time.Nanosecond, "How long to cache StatObject results and inode attributes. This flag has been deprecated (starting v2.0) in favor of metadata-cache-ttl-secs. For now, the minimum of stat-cache-ttl and type-cache-ttl values, rounded up to the next higher multiple of a second is used as ttl for both stat-cache and type-cache, when metadata-cache-ttl-secs is not set.") @@ -535,7 +1377,21 @@ func BuildFlagSet(flagSet *pflag.FlagSet) error { flagSet.StringP("token-url", "", "", "A url for getting an access token when the key-file is absent.") - flagSet.IntP("type-cache-max-size-mb", "", 4, "Max size of type-cache maps which are maintained at a per-directory level.") + flagSet.StringSliceP("trace-exporters", "", []string{"gcpexporter"}, "Specify comma separated value of the exporters where traces are exported to. Supported values: stdout(writes traces to stdout), gcpexporter(exports traces to google cloud trace)") + + if err := flagSet.MarkHidden("trace-exporters"); err != nil { + return err + } + + flagSet.StringP("trace-project-id", "", "", "Specify the GCP project id to which traces will be exported. When unset, a project id will be inferred as per the default credential detection process") + + if err := flagSet.MarkHidden("trace-project-id"); err != nil { + return err + } + + flagSet.Float64P("trace-sampling-ratio", "", 0, "Specifies the fraction of traces to export, ranging from 0.0 to 1.0. Setting a value greater than 0 enables tracing; 1.0 exports all traces, while 0.0 (default) disables them. Use this to balance the number of traces exported with the tradeoff of higher perf and cost impact.") + + flagSet.IntP("type-cache-max-size-mb", "", 4, "Max size of type-cache maps which are maintained at a per-directory level. This flag has been deprecated in favour of a single unified flag stat-cache-max-size-mb.") flagSet.DurationP("type-cache-ttl", "", 60000000000*time.Nanosecond, "Usage: How long to cache StatObject results and inode attributes. This flag has been deprecated (starting v2.0) in favor of metadata-cache-ttl-secs. For now, the minimum of stat-cache-ttl and type-cache-ttl values, rounded up to the next higher multiple of a second is used as ttl for both stat-cache and type-cache, when metadata-cache-ttl-secs is not set.") @@ -545,19 +1401,39 @@ func BuildFlagSet(flagSet *pflag.FlagSet) error { flagSet.IntP("uid", "", -1, "UID owner of all inodes.") - flagSet.IntP("write-block-size-mb", "", 64, "Specifies the block size for streaming writes. The value should be more than 0.") + flagSet.BoolP("visualize-workload-insight", "", false, "A flag to enable workload visualization. When enabled, workload insights will include visualizations to help understand access patterns. Insights will be written to the file specified by --workload-insight-output-file.") - if err := flagSet.MarkHidden("write-block-size-mb"); err != nil { + if err := flagSet.MarkHidden("visualize-workload-insight"); err != nil { + return err + } + + flagSet.StringP("wire-log", "", "", "The file name of the wire log. When specified, GCSFuse will serialize each FUSE operation as a JSON object and append it to this file.") + + if err := flagSet.MarkHidden("wire-log"); err != nil { + return err + } + + flagSet.IntP("workload-insight-forward-merge-threshold-mb", "", 0, "The threshold in MB for merging forward sequential reads for workload insights visualization.Reads within this threshold will be merged into a single read operation. Applicable only when --visualize-workload-insight is enabled.") + + if err := flagSet.MarkHidden("workload-insight-forward-merge-threshold-mb"); err != nil { + return err + } + + flagSet.StringP("workload-insight-output-file", "", "", "The file path where the workload insights will be written. If not specified, insights will be written to stdout") + + if err := flagSet.MarkHidden("workload-insight-output-file"); err != nil { return err } - flagSet.IntP("write-global-max-blocks", "", -1, "Specifies the maximum number of blocks to be used by all files for streaming writes. The value should be >= 2 or -1 (for infinite blocks).") + flagSet.IntP("write-block-size-mb", "", 32, "Specifies the block size for streaming writes. The value should be more than 0.") - if err := flagSet.MarkHidden("write-global-max-blocks"); err != nil { + if err := flagSet.MarkHidden("write-block-size-mb"); err != nil { return err } - flagSet.IntP("write-max-blocks-per-file", "", -1, "Specifies the maximum number of blocks to be used by a single file for streaming writes. The value should be >= 2 or -1 (for infinite blocks).") + flagSet.IntP("write-global-max-blocks", "", 4, "Specifies the maximum number of blocks available for streaming writes across all files. The value should be >= 0 or -1 (for infinite blocks). A value of 0 disables streaming writes.") + + flagSet.IntP("write-max-blocks-per-file", "", 1, "Specifies the maximum number of blocks to be used by a single file for streaming writes. The value should be >= 1 or -1 (for infinite blocks).") if err := flagSet.MarkHidden("write-max-blocks-per-file"); err != nil { return err @@ -584,6 +1460,14 @@ func BindFlags(v *viper.Viper, flagSet *pflag.FlagSet) error { return err } + if err := v.BindPFlag("gcs-retries.chunk-retry-deadline-secs", flagSet.Lookup("chunk-retry-deadline-secs")); err != nil { + return err + } + + if err := v.BindPFlag("gcs-retries.chunk-transfer-timeout-secs", flagSet.Lookup("chunk-transfer-timeout-secs")); err != nil { + return err + } + if err := v.BindPFlag("gcs-connection.client-protocol", flagSet.Lookup("client-protocol")); err != nil { return err } @@ -592,6 +1476,38 @@ func BindFlags(v *viper.Viper, flagSet *pflag.FlagSet) error { return err } + if err := v.BindPFlag("cloud-profiler.allocated-heap", flagSet.Lookup("cloud-profiler-allocated-heap")); err != nil { + return err + } + + if err := v.BindPFlag("cloud-profiler.cpu", flagSet.Lookup("cloud-profiler-cpu")); err != nil { + return err + } + + if err := v.BindPFlag("cloud-profiler.goroutines", flagSet.Lookup("cloud-profiler-goroutines")); err != nil { + return err + } + + if err := v.BindPFlag("cloud-profiler.heap", flagSet.Lookup("cloud-profiler-heap")); err != nil { + return err + } + + if err := v.BindPFlag("cloud-profiler.label", flagSet.Lookup("cloud-profiler-label")); err != nil { + return err + } + + if err := v.BindPFlag("cloud-profiler.mutex", flagSet.Lookup("cloud-profiler-mutex")); err != nil { + return err + } + + if err := v.BindPFlag("cloud-profiler.service-name", flagSet.Lookup("cloud-profiler-service-name")); err != nil { + return err + } + + if err := v.BindPFlag("file-system.congestion-threshold", flagSet.Lookup("congestion-threshold")); err != nil { + return err + } + if err := v.BindPFlag("write.create-empty-file", flagSet.Lookup("create-empty-file")); err != nil { return err } @@ -620,23 +1536,87 @@ func BindFlags(v *viper.Viper, flagSet *pflag.FlagSet) error { return err } + if err := v.BindPFlag("disable-autoconfig", flagSet.Lookup("disable-autoconfig")); err != nil { + return err + } + + if err := v.BindPFlag("disable-list-access-check", flagSet.Lookup("disable-list-access-check")); err != nil { + return err + } + if err := v.BindPFlag("file-system.disable-parallel-dirops", flagSet.Lookup("disable-parallel-dirops")); err != nil { return err } + if err := v.BindPFlag("dummy-io.per-mb-latency", flagSet.Lookup("dummy-io-per-mb-latency")); err != nil { + return err + } + + if err := v.BindPFlag("dummy-io.reader-latency", flagSet.Lookup("dummy-io-reader-latency")); err != nil { + return err + } + + if err := v.BindPFlag("enable-atomic-rename-object", flagSet.Lookup("enable-atomic-rename-object")); err != nil { + return err + } + + if err := v.BindPFlag("read.enable-buffered-read", flagSet.Lookup("enable-buffered-read")); err != nil { + return err + } + + if err := v.BindPFlag("cloud-profiler.enabled", flagSet.Lookup("enable-cloud-profiler")); err != nil { + return err + } + + if err := v.BindPFlag("dummy-io.enable", flagSet.Lookup("enable-dummy-io")); err != nil { + return err + } + if err := v.BindPFlag("list.enable-empty-managed-folders", flagSet.Lookup("enable-empty-managed-folders")); err != nil { return err } + if err := v.BindPFlag("file-cache.enable-experimental-shared-chunk-cache", flagSet.Lookup("enable-experimental-shared-chunk-cache")); err != nil { + return err + } + + if err := v.BindPFlag("enable-google-lib-auth", flagSet.Lookup("enable-google-lib-auth")); err != nil { + return err + } + if err := v.BindPFlag("enable-hns", flagSet.Lookup("enable-hns")); err != nil { return err } + if err := v.BindPFlag("gcs-connection.enable-http-dns-cache", flagSet.Lookup("enable-http-dns-cache")); err != nil { + return err + } + + if err := v.BindPFlag("file-system.enable-kernel-reader", flagSet.Lookup("enable-kernel-reader")); err != nil { + return err + } + + if err := v.BindPFlag("metadata-cache.enable-metadata-prefetch", flagSet.Lookup("enable-metadata-prefetch")); err != nil { + return err + } + + if err := v.BindPFlag("gcs-retries.enable-mount-retries", flagSet.Lookup("enable-mount-retries")); err != nil { + return err + } + + if err := v.BindPFlag("enable-new-reader", flagSet.Lookup("enable-new-reader")); err != nil { + return err + } + if err := v.BindPFlag("metadata-cache.enable-nonexistent-type-cache", flagSet.Lookup("enable-nonexistent-type-cache")); err != nil { return err } - if err := v.BindPFlag("metrics.enable-otel", flagSet.Lookup("enable-otel")); err != nil { + if err := v.BindPFlag("write.enable-rapid-appends", flagSet.Lookup("enable-rapid-appends")); err != nil { + return err + } + + if err := v.BindPFlag("write.enable-rapid-writes", flagSet.Lookup("enable-rapid-writes")); err != nil { return err } @@ -644,11 +1624,39 @@ func BindFlags(v *viper.Viper, flagSet *pflag.FlagSet) error { return err } + if err := v.BindPFlag("enable-standard-symlinks", flagSet.Lookup("enable-standard-symlinks")); err != nil { + return err + } + + if err := v.BindPFlag("write.enable-streaming-writes", flagSet.Lookup("enable-streaming-writes")); err != nil { + return err + } + + if err := v.BindPFlag("enable-type-cache-deprecation", flagSet.Lookup("enable-type-cache-deprecation")); err != nil { + return err + } + + if err := v.BindPFlag("enable-unsupported-path-support", flagSet.Lookup("enable-unsupported-path-support")); err != nil { + return err + } + + if err := v.BindPFlag("file-system.experimental-enable-dentry-cache", flagSet.Lookup("experimental-enable-dentry-cache")); err != nil { + return err + } + + if err := v.BindPFlag("metrics.experimental-enable-grpc-metrics", flagSet.Lookup("experimental-enable-grpc-metrics")); err != nil { + return err + } + if err := v.BindPFlag("gcs-connection.experimental-enable-json-read", flagSet.Lookup("experimental-enable-json-read")); err != nil { return err } - if err := v.BindPFlag("write.experimental-enable-streaming-writes", flagSet.Lookup("experimental-enable-streaming-writes")); err != nil { + if err := v.BindPFlag("file-system.experimental-enable-pirlo", flagSet.Lookup("experimental-enable-pirlo")); err != nil { + return err + } + + if err := v.BindPFlag("file-system.experimental-enable-readdirplus", flagSet.Lookup("experimental-enable-readdirplus")); err != nil { return err } @@ -656,19 +1664,19 @@ func BindFlags(v *viper.Viper, flagSet *pflag.FlagSet) error { return err } - if err := v.BindPFlag("metadata-cache.experimental-metadata-prefetch-on-mount", flagSet.Lookup("experimental-metadata-prefetch-on-mount")); err != nil { + if err := v.BindPFlag("gcs-connection.experimental-local-socket-address", flagSet.Lookup("experimental-local-socket-address")); err != nil { return err } - if err := v.BindPFlag("monitoring.experimental-opentelemetry-collector-address", flagSet.Lookup("experimental-opentelemetry-collector-address")); err != nil { + if err := v.BindPFlag("metadata-cache.experimental-metadata-prefetch-on-mount", flagSet.Lookup("experimental-metadata-prefetch-on-mount")); err != nil { return err } - if err := v.BindPFlag("monitoring.experimental-tracing-mode", flagSet.Lookup("experimental-tracing-mode")); err != nil { + if err := v.BindPFlag("gcs-retries.experimental-nonrapid-folder-api-stall-retry", flagSet.Lookup("experimental-nonrapid-folder-api-stall-retry")); err != nil { return err } - if err := v.BindPFlag("monitoring.experimental-tracing-sampling-ratio", flagSet.Lookup("experimental-tracing-sampling-ratio")); err != nil { + if err := v.BindPFlag("file-system.experimental-o-direct", flagSet.Lookup("experimental-o-direct")); err != nil { return err } @@ -692,6 +1700,26 @@ func BindFlags(v *viper.Viper, flagSet *pflag.FlagSet) error { return err } + if err := v.BindPFlag("file-cache.exclude-regex", flagSet.Lookup("file-cache-exclude-regex")); err != nil { + return err + } + + if err := v.BindPFlag("file-cache.experimental-disable-size-calculation-fix", flagSet.Lookup("file-cache-experimental-disable-size-calculation-fix")); err != nil { + return err + } + + if err := v.BindPFlag("file-cache.experimental-enable-chunk-cache", flagSet.Lookup("file-cache-experimental-enable-chunk-cache")); err != nil { + return err + } + + if err := v.BindPFlag("file-cache.experimental-parallel-downloads-default-on", flagSet.Lookup("file-cache-experimental-parallel-downloads-default-on")); err != nil { + return err + } + + if err := v.BindPFlag("file-cache.include-regex", flagSet.Lookup("file-cache-include-regex")); err != nil { + return err + } + if err := v.BindPFlag("file-cache.max-parallel-downloads", flagSet.Lookup("file-cache-max-parallel-downloads")); err != nil { return err } @@ -704,6 +1732,10 @@ func BindFlags(v *viper.Viper, flagSet *pflag.FlagSet) error { return err } + if err := v.BindPFlag("file-cache.shared-cache-chunk-size-mb", flagSet.Lookup("file-cache-shared-cache-chunk-size-mb")); err != nil { + return err + } + if err := v.BindPFlag("file-cache.write-buffer-size", flagSet.Lookup("file-cache-write-buffer-size")); err != nil { return err } @@ -712,6 +1744,10 @@ func BindFlags(v *viper.Viper, flagSet *pflag.FlagSet) error { return err } + if err := v.BindPFlag("write.finalize-file-on-close", flagSet.Lookup("finalize-file-on-close")); err != nil { + return err + } + if err := v.BindPFlag("foreground", flagSet.Lookup("foreground")); err != nil { return err } @@ -720,7 +1756,7 @@ func BindFlags(v *viper.Viper, flagSet *pflag.FlagSet) error { return err } - if err := v.BindPFlag("file-system.handle-sigterm", flagSet.Lookup("handle-sigterm")); err != nil { + if err := v.BindPFlag("gcs-connection.grpc-path-strategy", flagSet.Lookup("grpc-path-strategy")); err != nil { return err } @@ -736,10 +1772,18 @@ func BindFlags(v *viper.Viper, flagSet *pflag.FlagSet) error { return err } + if err := v.BindPFlag("file-system.inactive-mrd-cache-size", flagSet.Lookup("inactive-mrd-cache-size")); err != nil { + return err + } + if err := v.BindPFlag("file-system.kernel-list-cache-ttl-secs", flagSet.Lookup("kernel-list-cache-ttl-secs")); err != nil { return err } + if err := v.BindPFlag("file-system.kernel-params-file", flagSet.Lookup("kernel-params-file")); err != nil { + return err + } + if err := v.BindPFlag("gcs-auth.key-file", flagSet.Lookup("key-file")); err != nil { return err } @@ -776,6 +1820,14 @@ func BindFlags(v *viper.Viper, flagSet *pflag.FlagSet) error { return err } + if err := v.BindPFlag("machine-type", flagSet.Lookup("machine-type")); err != nil { + return err + } + + if err := v.BindPFlag("file-system.max-background", flagSet.Lookup("max-background")); err != nil { + return err + } + if err := v.BindPFlag("gcs-connection.max-conns-per-host", flagSet.Lookup("max-conns-per-host")); err != nil { return err } @@ -784,6 +1836,10 @@ func BindFlags(v *viper.Viper, flagSet *pflag.FlagSet) error { return err } + if err := v.BindPFlag("file-system.max-read-ahead-kb", flagSet.Lookup("max-read-ahead-kb")); err != nil { + return err + } + if err := v.BindPFlag("gcs-retries.max-retry-attempts", flagSet.Lookup("max-retry-attempts")); err != nil { return err } @@ -792,10 +1848,38 @@ func BindFlags(v *viper.Viper, flagSet *pflag.FlagSet) error { return err } + if err := v.BindPFlag("metadata-cache.negative-ttl-secs", flagSet.Lookup("metadata-cache-negative-ttl-secs")); err != nil { + return err + } + if err := v.BindPFlag("metadata-cache.ttl-secs", flagSet.Lookup("metadata-cache-ttl-secs")); err != nil { return err } + if err := v.BindPFlag("metadata-cache.metadata-prefetch-entries-limit", flagSet.Lookup("metadata-prefetch-entries-limit")); err != nil { + return err + } + + if err := v.BindPFlag("metadata-cache.metadata-prefetch-max-workers", flagSet.Lookup("metadata-prefetch-max-workers")); err != nil { + return err + } + + if err := v.BindPFlag("metrics.buffer-size", flagSet.Lookup("metrics-buffer-size")); err != nil { + return err + } + + if err := v.BindPFlag("metrics.use-new-names", flagSet.Lookup("metrics-use-new-names")); err != nil { + return err + } + + if err := v.BindPFlag("metrics.workers", flagSet.Lookup("metrics-workers")); err != nil { + return err + } + + if err := v.BindPFlag("mrd.pool-size", flagSet.Lookup("mrd-pool-size")); err != nil { + return err + } + if err := v.BindPFlag("file-system.fuse-options", flagSet.Lookup("o")); err != nil { return err } @@ -804,7 +1888,7 @@ func BindFlags(v *viper.Viper, flagSet *pflag.FlagSet) error { return err } - if err := v.BindPFlag("file-system.precondition-errors", flagSet.Lookup("precondition-errors")); err != nil { + if err := v.BindPFlag("profile", flagSet.Lookup("profile")); err != nil { return err } @@ -812,6 +1896,30 @@ func BindFlags(v *viper.Viper, flagSet *pflag.FlagSet) error { return err } + if err := v.BindPFlag("read.block-size-mb", flagSet.Lookup("read-block-size-mb")); err != nil { + return err + } + + if err := v.BindPFlag("read.global-max-blocks", flagSet.Lookup("read-global-max-blocks")); err != nil { + return err + } + + if err := v.BindPFlag("read.inactive-stream-timeout", flagSet.Lookup("read-inactive-stream-timeout")); err != nil { + return err + } + + if err := v.BindPFlag("read.max-blocks-per-handle", flagSet.Lookup("read-max-blocks-per-handle")); err != nil { + return err + } + + if err := v.BindPFlag("read.min-blocks-per-handle", flagSet.Lookup("read-min-blocks-per-handle")); err != nil { + return err + } + + if err := v.BindPFlag("read.random-seek-threshold", flagSet.Lookup("read-random-seek-threshold")); err != nil { + return err + } + if err := v.BindPFlag("gcs-retries.read-stall.initial-req-timeout", flagSet.Lookup("read-stall-initial-req-timeout")); err != nil { return err } @@ -832,6 +1940,10 @@ func BindFlags(v *viper.Viper, flagSet *pflag.FlagSet) error { return err } + if err := v.BindPFlag("read.start-blocks-per-handle", flagSet.Lookup("read-start-blocks-per-handle")); err != nil { + return err + } + if err := v.BindPFlag("file-system.rename-dir-limit", flagSet.Lookup("rename-dir-limit")); err != nil { return err } @@ -872,6 +1984,18 @@ func BindFlags(v *viper.Viper, flagSet *pflag.FlagSet) error { return err } + if err := v.BindPFlag("trace.exporters", flagSet.Lookup("trace-exporters")); err != nil { + return err + } + + if err := v.BindPFlag("trace.project-id", flagSet.Lookup("trace-project-id")); err != nil { + return err + } + + if err := v.BindPFlag("trace.sampling-ratio", flagSet.Lookup("trace-sampling-ratio")); err != nil { + return err + } + if err := v.BindPFlag("metadata-cache.type-cache-max-size-mb", flagSet.Lookup("type-cache-max-size-mb")); err != nil { return err } @@ -884,6 +2008,22 @@ func BindFlags(v *viper.Viper, flagSet *pflag.FlagSet) error { return err } + if err := v.BindPFlag("workload-insight.visualize", flagSet.Lookup("visualize-workload-insight")); err != nil { + return err + } + + if err := v.BindPFlag("logging.wire-log", flagSet.Lookup("wire-log")); err != nil { + return err + } + + if err := v.BindPFlag("workload-insight.forward-merge-threshold-mb", flagSet.Lookup("workload-insight-forward-merge-threshold-mb")); err != nil { + return err + } + + if err := v.BindPFlag("workload-insight.output-file", flagSet.Lookup("workload-insight-output-file")); err != nil { + return err + } + if err := v.BindPFlag("write.block-size-mb", flagSet.Lookup("write-block-size-mb")); err != nil { return err } diff --git a/cfg/config_test.go b/cfg/config_test.go new file mode 100644 index 0000000000..ef0c3ec96f --- /dev/null +++ b/cfg/config_test.go @@ -0,0 +1,1224 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// GENERATED CODE - DO NOT EDIT MANUALLY. + +package cfg + +import ( + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" +) + +func TestApplyOptimizations(t *testing.T) { + // Tests for file-system.congestion-threshold + t.Run("file-system.congestion-threshold", func(t *testing.T) { + testCases := []struct { + name string + config Config + userSetFlags map[string]any + input *OptimizationInput + expectOptimized bool + expectedValue any + }{ + { + name: "user_set", + config: Config{}, + userSetFlags: map[string]any{ + "file-system.congestion-threshold": 98765, + "machine-type": "a2-megagpu-16g", + }, + input: &OptimizationInput{BucketType: BucketTypeZonal}, + expectOptimized: false, + expectedValue: int64(98765), + }, + { + name: "no_optimization", + config: Config{Profile: "non_existent_profile"}, + userSetFlags: map[string]any{ + "machine-type": "low-end-machine", + }, + input: nil, + expectOptimized: false, + expectedValue: 0, + }, + { + name: "bucket_type_zonal", + config: Config{Profile: ""}, + userSetFlags: map[string]any{}, + input: &OptimizationInput{BucketType: BucketTypeZonal}, + expectOptimized: true, + expectedValue: DefaultCongestionThreshold(), + }, + { + name: "bucket_type_pirlo", + config: Config{Profile: ""}, + userSetFlags: map[string]any{}, + input: &OptimizationInput{BucketType: BucketTypePirlo}, + expectOptimized: true, + expectedValue: DefaultCongestionThreshold(), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // We need a copy of the config for each test case. + c := tc.config + // Set the default or non-default value on the config object. + if tc.name == "user_set" { + c.FileSystem.CongestionThreshold = tc.expectedValue.(int64) + } else { + c.FileSystem.CongestionThreshold = 0 + } + + v := viper.New() + for key, val := range tc.userSetFlags { + v.Set(key, val) + } + + optimizedFlags := c.ApplyOptimizations(v, tc.input) + + if tc.expectOptimized { + assert.Contains(t, optimizedFlags, "file-system.congestion-threshold") + } else { + assert.NotContains(t, optimizedFlags, "file-system.congestion-threshold") + } + // Use EqualValues to handle the int vs int64 type mismatch for default values. + assert.EqualValues(t, tc.expectedValue, c.FileSystem.CongestionThreshold) + }) + } + }) + // Tests for file-system.enable-kernel-reader + t.Run("file-system.enable-kernel-reader", func(t *testing.T) { + testCases := []struct { + name string + config Config + userSetFlags map[string]any + input *OptimizationInput + expectOptimized bool + expectedValue any + }{ + { + name: "user_set", + config: Config{}, + userSetFlags: map[string]any{ + "file-system.enable-kernel-reader": true, + "machine-type": "a2-megagpu-16g", + }, + input: &OptimizationInput{BucketType: BucketTypeZonal}, + expectOptimized: false, + expectedValue: true, + }, + { + name: "no_optimization", + config: Config{Profile: "non_existent_profile"}, + userSetFlags: map[string]any{ + "machine-type": "low-end-machine", + }, + input: nil, + expectOptimized: false, + expectedValue: false, + }, + { + name: "bucket_type_zonal", + config: Config{Profile: ""}, + userSetFlags: map[string]any{}, + input: &OptimizationInput{BucketType: BucketTypeZonal}, + expectOptimized: true, + expectedValue: true, + }, + { + name: "bucket_type_pirlo", + config: Config{Profile: ""}, + userSetFlags: map[string]any{}, + input: &OptimizationInput{BucketType: BucketTypePirlo}, + expectOptimized: true, + expectedValue: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // We need a copy of the config for each test case. + c := tc.config + // Set the default or non-default value on the config object. + if tc.name == "user_set" { + c.FileSystem.EnableKernelReader = tc.expectedValue.(bool) + } else { + c.FileSystem.EnableKernelReader = false + } + + v := viper.New() + for key, val := range tc.userSetFlags { + v.Set(key, val) + } + + optimizedFlags := c.ApplyOptimizations(v, tc.input) + + if tc.expectOptimized { + assert.Contains(t, optimizedFlags, "file-system.enable-kernel-reader") + } else { + assert.NotContains(t, optimizedFlags, "file-system.enable-kernel-reader") + } + // Use EqualValues to handle the int vs int64 type mismatch for default values. + assert.EqualValues(t, tc.expectedValue, c.FileSystem.EnableKernelReader) + }) + } + }) + // Tests for file-cache.cache-file-for-range-read + t.Run("file-cache.cache-file-for-range-read", func(t *testing.T) { + testCases := []struct { + name string + config Config + userSetFlags map[string]any + input *OptimizationInput + expectOptimized bool + expectedValue any + }{ + { + name: "user_set", + config: Config{ + Profile: "aiml-serving", + }, + userSetFlags: map[string]any{ + "file-cache.cache-file-for-range-read": true, + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: false, + expectedValue: true, + }, + { + name: "no_optimization", + config: Config{Profile: "non_existent_profile"}, + userSetFlags: map[string]any{ + "machine-type": "low-end-machine", + }, + input: nil, + expectOptimized: false, + expectedValue: false, + }, + { + name: "profile_aiml-serving", + config: Config{Profile: "aiml-serving"}, + userSetFlags: map[string]any{}, + input: nil, + expectOptimized: true, + expectedValue: true, + }, + { + name: "profile_aiml-checkpointing", + config: Config{Profile: "aiml-checkpointing"}, + userSetFlags: map[string]any{}, + input: nil, + expectOptimized: true, + expectedValue: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // We need a copy of the config for each test case. + c := tc.config + // Set the default or non-default value on the config object. + if tc.name == "user_set" { + c.FileCache.CacheFileForRangeRead = tc.expectedValue.(bool) + } else { + c.FileCache.CacheFileForRangeRead = false + } + + v := viper.New() + for key, val := range tc.userSetFlags { + v.Set(key, val) + } + + optimizedFlags := c.ApplyOptimizations(v, tc.input) + + if tc.expectOptimized { + assert.Contains(t, optimizedFlags, "file-cache.cache-file-for-range-read") + } else { + assert.NotContains(t, optimizedFlags, "file-cache.cache-file-for-range-read") + } + // Use EqualValues to handle the int vs int64 type mismatch for default values. + assert.EqualValues(t, tc.expectedValue, c.FileCache.CacheFileForRangeRead) + }) + } + }) + // Tests for write.finalize-file-on-close + t.Run("write.finalize-file-on-close", func(t *testing.T) { + testCases := []struct { + name string + config Config + userSetFlags map[string]any + input *OptimizationInput + expectOptimized bool + expectedValue any + }{ + { + name: "user_set", + config: Config{}, + userSetFlags: map[string]any{ + "write.finalize-file-on-close": true, + "machine-type": "a2-megagpu-16g", + }, + input: &OptimizationInput{BucketType: BucketTypeZonal}, + expectOptimized: false, + expectedValue: true, + }, + { + name: "no_optimization", + config: Config{Profile: "non_existent_profile"}, + userSetFlags: map[string]any{ + "machine-type": "low-end-machine", + }, + input: nil, + expectOptimized: false, + expectedValue: false, + }, + { + name: "bucket_type_zonal", + config: Config{Profile: ""}, + userSetFlags: map[string]any{}, + input: &OptimizationInput{BucketType: BucketTypeZonal}, + expectOptimized: false, + expectedValue: false, + }, + { + name: "bucket_type_pirlo", + config: Config{Profile: ""}, + userSetFlags: map[string]any{}, + input: &OptimizationInput{BucketType: BucketTypePirlo}, + expectOptimized: true, + expectedValue: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // We need a copy of the config for each test case. + c := tc.config + // Set the default or non-default value on the config object. + if tc.name == "user_set" { + c.Write.FinalizeFileOnClose = tc.expectedValue.(bool) + } else { + c.Write.FinalizeFileOnClose = false + } + + v := viper.New() + for key, val := range tc.userSetFlags { + v.Set(key, val) + } + + optimizedFlags := c.ApplyOptimizations(v, tc.input) + + if tc.expectOptimized { + assert.Contains(t, optimizedFlags, "write.finalize-file-on-close") + } else { + assert.NotContains(t, optimizedFlags, "write.finalize-file-on-close") + } + // Use EqualValues to handle the int vs int64 type mismatch for default values. + assert.EqualValues(t, tc.expectedValue, c.Write.FinalizeFileOnClose) + }) + } + }) + // Tests for implicit-dirs + t.Run("implicit-dirs", func(t *testing.T) { + testCases := []struct { + name string + config Config + userSetFlags map[string]any + input *OptimizationInput + expectOptimized bool + expectedValue any + }{ + { + name: "user_set", + config: Config{ + Profile: "aiml-training", + }, + userSetFlags: map[string]any{ + "implicit-dirs": true, + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: false, + expectedValue: true, + }, + { + name: "no_optimization", + config: Config{Profile: "non_existent_profile"}, + userSetFlags: map[string]any{ + "machine-type": "low-end-machine", + }, + input: nil, + expectOptimized: false, + expectedValue: false, + }, + { + name: "profile_aiml-training", + config: Config{Profile: "aiml-training"}, + userSetFlags: map[string]any{}, + input: nil, + expectOptimized: true, + expectedValue: true, + }, + { + name: "profile_aiml-serving", + config: Config{Profile: "aiml-serving"}, + userSetFlags: map[string]any{}, + input: nil, + expectOptimized: true, + expectedValue: true, + }, + { + name: "profile_aiml-checkpointing", + config: Config{Profile: "aiml-checkpointing"}, + userSetFlags: map[string]any{}, + input: nil, + expectOptimized: true, + expectedValue: true, + }, + { + name: "machine_group_high-performance", + config: Config{Profile: ""}, + userSetFlags: map[string]any{ + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: true, + expectedValue: true, + }, + { + name: "profile_overrides_machine_type", + config: Config{Profile: "aiml-training"}, + userSetFlags: map[string]any{ + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: true, + expectedValue: true, + }, { + name: "fallback_to_machine_type_with_non_existent_profile", + config: Config{Profile: "non_existent_profile"}, + userSetFlags: map[string]any{ + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: true, + expectedValue: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // We need a copy of the config for each test case. + c := tc.config + // Set the default or non-default value on the config object. + if tc.name == "user_set" { + c.ImplicitDirs = tc.expectedValue.(bool) + } else { + c.ImplicitDirs = false + } + + v := viper.New() + for key, val := range tc.userSetFlags { + v.Set(key, val) + } + + optimizedFlags := c.ApplyOptimizations(v, tc.input) + + if tc.expectOptimized { + assert.Contains(t, optimizedFlags, "implicit-dirs") + } else { + assert.NotContains(t, optimizedFlags, "implicit-dirs") + } + // Use EqualValues to handle the int vs int64 type mismatch for default values. + assert.EqualValues(t, tc.expectedValue, c.ImplicitDirs) + }) + } + }) + // Tests for file-system.kernel-list-cache-ttl-secs + t.Run("file-system.kernel-list-cache-ttl-secs", func(t *testing.T) { + testCases := []struct { + name string + config Config + userSetFlags map[string]any + input *OptimizationInput + expectOptimized bool + expectedValue any + }{ + { + name: "user_set", + config: Config{ + Profile: "aiml-serving", + }, + userSetFlags: map[string]any{ + "file-system.kernel-list-cache-ttl-secs": 98765, + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: false, + expectedValue: int64(98765), + }, + { + name: "no_optimization", + config: Config{Profile: "non_existent_profile"}, + userSetFlags: map[string]any{ + "machine-type": "low-end-machine", + }, + input: nil, + expectOptimized: false, + expectedValue: 0, + }, + { + name: "profile_aiml-serving", + config: Config{Profile: "aiml-serving"}, + userSetFlags: map[string]any{}, + input: nil, + expectOptimized: true, + expectedValue: -1, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // We need a copy of the config for each test case. + c := tc.config + // Set the default or non-default value on the config object. + if tc.name == "user_set" { + c.FileSystem.KernelListCacheTtlSecs = tc.expectedValue.(int64) + } else { + c.FileSystem.KernelListCacheTtlSecs = 0 + } + + v := viper.New() + for key, val := range tc.userSetFlags { + v.Set(key, val) + } + + optimizedFlags := c.ApplyOptimizations(v, tc.input) + + if tc.expectOptimized { + assert.Contains(t, optimizedFlags, "file-system.kernel-list-cache-ttl-secs") + } else { + assert.NotContains(t, optimizedFlags, "file-system.kernel-list-cache-ttl-secs") + } + // Use EqualValues to handle the int vs int64 type mismatch for default values. + assert.EqualValues(t, tc.expectedValue, c.FileSystem.KernelListCacheTtlSecs) + }) + } + }) + // Tests for file-system.max-background + t.Run("file-system.max-background", func(t *testing.T) { + testCases := []struct { + name string + config Config + userSetFlags map[string]any + input *OptimizationInput + expectOptimized bool + expectedValue any + }{ + { + name: "user_set", + config: Config{}, + userSetFlags: map[string]any{ + "file-system.max-background": 98765, + "machine-type": "a2-megagpu-16g", + }, + input: &OptimizationInput{BucketType: BucketTypeZonal}, + expectOptimized: false, + expectedValue: int64(98765), + }, + { + name: "no_optimization", + config: Config{Profile: "non_existent_profile"}, + userSetFlags: map[string]any{ + "machine-type": "low-end-machine", + }, + input: nil, + expectOptimized: false, + expectedValue: 0, + }, + { + name: "bucket_type_zonal", + config: Config{Profile: ""}, + userSetFlags: map[string]any{}, + input: &OptimizationInput{BucketType: BucketTypeZonal}, + expectOptimized: true, + expectedValue: DefaultMaxBackground(), + }, + { + name: "bucket_type_pirlo", + config: Config{Profile: ""}, + userSetFlags: map[string]any{}, + input: &OptimizationInput{BucketType: BucketTypePirlo}, + expectOptimized: true, + expectedValue: DefaultMaxBackground(), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // We need a copy of the config for each test case. + c := tc.config + // Set the default or non-default value on the config object. + if tc.name == "user_set" { + c.FileSystem.MaxBackground = tc.expectedValue.(int64) + } else { + c.FileSystem.MaxBackground = 0 + } + + v := viper.New() + for key, val := range tc.userSetFlags { + v.Set(key, val) + } + + optimizedFlags := c.ApplyOptimizations(v, tc.input) + + if tc.expectOptimized { + assert.Contains(t, optimizedFlags, "file-system.max-background") + } else { + assert.NotContains(t, optimizedFlags, "file-system.max-background") + } + // Use EqualValues to handle the int vs int64 type mismatch for default values. + assert.EqualValues(t, tc.expectedValue, c.FileSystem.MaxBackground) + }) + } + }) + // Tests for file-system.max-read-ahead-kb + t.Run("file-system.max-read-ahead-kb", func(t *testing.T) { + testCases := []struct { + name string + config Config + userSetFlags map[string]any + input *OptimizationInput + expectOptimized bool + expectedValue any + }{ + { + name: "user_set", + config: Config{}, + userSetFlags: map[string]any{ + "file-system.max-read-ahead-kb": 98765, + "machine-type": "a2-megagpu-16g", + }, + input: &OptimizationInput{BucketType: BucketTypeZonal}, + expectOptimized: false, + expectedValue: int64(98765), + }, + { + name: "no_optimization", + config: Config{Profile: "non_existent_profile"}, + userSetFlags: map[string]any{ + "machine-type": "low-end-machine", + }, + input: nil, + expectOptimized: false, + expectedValue: 0, + }, + { + name: "bucket_type_zonal", + config: Config{Profile: ""}, + userSetFlags: map[string]any{}, + input: &OptimizationInput{BucketType: BucketTypeZonal}, + expectOptimized: true, + expectedValue: 16384, + }, + { + name: "bucket_type_pirlo", + config: Config{Profile: ""}, + userSetFlags: map[string]any{}, + input: &OptimizationInput{BucketType: BucketTypePirlo}, + expectOptimized: true, + expectedValue: 16384, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // We need a copy of the config for each test case. + c := tc.config + // Set the default or non-default value on the config object. + if tc.name == "user_set" { + c.FileSystem.MaxReadAheadKb = tc.expectedValue.(int64) + } else { + c.FileSystem.MaxReadAheadKb = 0 + } + + v := viper.New() + for key, val := range tc.userSetFlags { + v.Set(key, val) + } + + optimizedFlags := c.ApplyOptimizations(v, tc.input) + + if tc.expectOptimized { + assert.Contains(t, optimizedFlags, "file-system.max-read-ahead-kb") + } else { + assert.NotContains(t, optimizedFlags, "file-system.max-read-ahead-kb") + } + // Use EqualValues to handle the int vs int64 type mismatch for default values. + assert.EqualValues(t, tc.expectedValue, c.FileSystem.MaxReadAheadKb) + }) + } + }) + // Tests for metadata-cache.negative-ttl-secs + t.Run("metadata-cache.negative-ttl-secs", func(t *testing.T) { + testCases := []struct { + name string + config Config + userSetFlags map[string]any + input *OptimizationInput + expectOptimized bool + expectedValue any + }{ + { + name: "user_set", + config: Config{ + Profile: "aiml-training", + }, + userSetFlags: map[string]any{ + "metadata-cache.negative-ttl-secs": 98765, + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: false, + expectedValue: int64(98765), + }, + { + name: "no_optimization", + config: Config{Profile: "non_existent_profile"}, + userSetFlags: map[string]any{ + "machine-type": "low-end-machine", + }, + input: nil, + expectOptimized: false, + expectedValue: 5, + }, + { + name: "profile_aiml-training", + config: Config{Profile: "aiml-training"}, + userSetFlags: map[string]any{}, + input: nil, + expectOptimized: true, + expectedValue: 0, + }, + { + name: "profile_aiml-serving", + config: Config{Profile: "aiml-serving"}, + userSetFlags: map[string]any{}, + input: nil, + expectOptimized: true, + expectedValue: 0, + }, + { + name: "profile_aiml-checkpointing", + config: Config{Profile: "aiml-checkpointing"}, + userSetFlags: map[string]any{}, + input: nil, + expectOptimized: true, + expectedValue: 0, + }, + { + name: "machine_group_high-performance", + config: Config{Profile: ""}, + userSetFlags: map[string]any{ + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: true, + expectedValue: 0, + }, + { + name: "profile_overrides_machine_type", + config: Config{Profile: "aiml-training"}, + userSetFlags: map[string]any{ + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: true, + expectedValue: 0, + }, { + name: "fallback_to_machine_type_with_non_existent_profile", + config: Config{Profile: "non_existent_profile"}, + userSetFlags: map[string]any{ + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: true, + expectedValue: 0, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // We need a copy of the config for each test case. + c := tc.config + // Set the default or non-default value on the config object. + if tc.name == "user_set" { + c.MetadataCache.NegativeTtlSecs = tc.expectedValue.(int64) + } else { + c.MetadataCache.NegativeTtlSecs = 5 + } + + v := viper.New() + for key, val := range tc.userSetFlags { + v.Set(key, val) + } + + optimizedFlags := c.ApplyOptimizations(v, tc.input) + + if tc.expectOptimized { + assert.Contains(t, optimizedFlags, "metadata-cache.negative-ttl-secs") + } else { + assert.NotContains(t, optimizedFlags, "metadata-cache.negative-ttl-secs") + } + // Use EqualValues to handle the int vs int64 type mismatch for default values. + assert.EqualValues(t, tc.expectedValue, c.MetadataCache.NegativeTtlSecs) + }) + } + }) + // Tests for metadata-cache.ttl-secs + t.Run("metadata-cache.ttl-secs", func(t *testing.T) { + testCases := []struct { + name string + config Config + userSetFlags map[string]any + input *OptimizationInput + expectOptimized bool + expectedValue any + }{ + { + name: "user_set", + config: Config{ + Profile: "aiml-training", + }, + userSetFlags: map[string]any{ + "metadata-cache.ttl-secs": 98765, + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: false, + expectedValue: int64(98765), + }, + { + name: "no_optimization", + config: Config{Profile: "non_existent_profile"}, + userSetFlags: map[string]any{ + "machine-type": "low-end-machine", + }, + input: nil, + expectOptimized: false, + expectedValue: 60, + }, + { + name: "profile_aiml-training", + config: Config{Profile: "aiml-training"}, + userSetFlags: map[string]any{}, + input: nil, + expectOptimized: true, + expectedValue: -1, + }, + { + name: "profile_aiml-serving", + config: Config{Profile: "aiml-serving"}, + userSetFlags: map[string]any{}, + input: nil, + expectOptimized: true, + expectedValue: -1, + }, + { + name: "profile_aiml-checkpointing", + config: Config{Profile: "aiml-checkpointing"}, + userSetFlags: map[string]any{}, + input: nil, + expectOptimized: true, + expectedValue: -1, + }, + { + name: "machine_group_high-performance", + config: Config{Profile: ""}, + userSetFlags: map[string]any{ + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: true, + expectedValue: -1, + }, + { + name: "profile_overrides_machine_type", + config: Config{Profile: "aiml-training"}, + userSetFlags: map[string]any{ + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: true, + expectedValue: -1, + }, { + name: "fallback_to_machine_type_with_non_existent_profile", + config: Config{Profile: "non_existent_profile"}, + userSetFlags: map[string]any{ + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: true, + expectedValue: -1, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // We need a copy of the config for each test case. + c := tc.config + // Set the default or non-default value on the config object. + if tc.name == "user_set" { + c.MetadataCache.TtlSecs = tc.expectedValue.(int64) + } else { + c.MetadataCache.TtlSecs = 60 + } + + v := viper.New() + for key, val := range tc.userSetFlags { + v.Set(key, val) + } + + optimizedFlags := c.ApplyOptimizations(v, tc.input) + + if tc.expectOptimized { + assert.Contains(t, optimizedFlags, "metadata-cache.ttl-secs") + } else { + assert.NotContains(t, optimizedFlags, "metadata-cache.ttl-secs") + } + // Use EqualValues to handle the int vs int64 type mismatch for default values. + assert.EqualValues(t, tc.expectedValue, c.MetadataCache.TtlSecs) + }) + } + }) + // Tests for file-system.rename-dir-limit + t.Run("file-system.rename-dir-limit", func(t *testing.T) { + testCases := []struct { + name string + config Config + userSetFlags map[string]any + input *OptimizationInput + expectOptimized bool + expectedValue any + }{ + { + name: "user_set", + config: Config{ + Profile: "aiml-checkpointing", + }, + userSetFlags: map[string]any{ + "file-system.rename-dir-limit": 98765, + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: false, + expectedValue: int64(98765), + }, + { + name: "no_optimization", + config: Config{Profile: "non_existent_profile"}, + userSetFlags: map[string]any{ + "machine-type": "low-end-machine", + }, + input: nil, + expectOptimized: false, + expectedValue: 0, + }, + { + name: "profile_aiml-checkpointing", + config: Config{Profile: "aiml-checkpointing"}, + userSetFlags: map[string]any{}, + input: nil, + expectOptimized: true, + expectedValue: 200000, + }, + { + name: "machine_group_high-performance", + config: Config{Profile: ""}, + userSetFlags: map[string]any{ + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: true, + expectedValue: 200000, + }, + { + name: "profile_overrides_machine_type", + config: Config{Profile: "aiml-checkpointing"}, + userSetFlags: map[string]any{ + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: true, + expectedValue: 200000, + }, { + name: "fallback_to_machine_type_with_non_existent_profile", + config: Config{Profile: "non_existent_profile"}, + userSetFlags: map[string]any{ + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: true, + expectedValue: 200000, + }, { + name: "fallback_to_machine_type_when_aiml-training_is_unrelated", + config: Config{Profile: "aiml-training"}, + userSetFlags: map[string]any{ + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: true, + expectedValue: 200000, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // We need a copy of the config for each test case. + c := tc.config + // Set the default or non-default value on the config object. + if tc.name == "user_set" { + c.FileSystem.RenameDirLimit = tc.expectedValue.(int64) + } else { + c.FileSystem.RenameDirLimit = 0 + } + + v := viper.New() + for key, val := range tc.userSetFlags { + v.Set(key, val) + } + + optimizedFlags := c.ApplyOptimizations(v, tc.input) + + if tc.expectOptimized { + assert.Contains(t, optimizedFlags, "file-system.rename-dir-limit") + } else { + assert.NotContains(t, optimizedFlags, "file-system.rename-dir-limit") + } + // Use EqualValues to handle the int vs int64 type mismatch for default values. + assert.EqualValues(t, tc.expectedValue, c.FileSystem.RenameDirLimit) + }) + } + }) + // Tests for metadata-cache.stat-cache-max-size-mb + t.Run("metadata-cache.stat-cache-max-size-mb", func(t *testing.T) { + testCases := []struct { + name string + config Config + userSetFlags map[string]any + input *OptimizationInput + expectOptimized bool + expectedValue any + }{ + { + name: "user_set", + config: Config{ + Profile: "aiml-training", + }, + userSetFlags: map[string]any{ + "metadata-cache.stat-cache-max-size-mb": 98765, + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: false, + expectedValue: int64(98765), + }, + { + name: "no_optimization", + config: Config{Profile: "non_existent_profile"}, + userSetFlags: map[string]any{ + "machine-type": "low-end-machine", + }, + input: nil, + expectOptimized: false, + expectedValue: 34, + }, + { + name: "profile_aiml-training", + config: Config{Profile: "aiml-training"}, + userSetFlags: map[string]any{}, + input: nil, + expectOptimized: true, + expectedValue: -1, + }, + { + name: "profile_aiml-serving", + config: Config{Profile: "aiml-serving"}, + userSetFlags: map[string]any{}, + input: nil, + expectOptimized: true, + expectedValue: -1, + }, + { + name: "profile_aiml-checkpointing", + config: Config{Profile: "aiml-checkpointing"}, + userSetFlags: map[string]any{}, + input: nil, + expectOptimized: true, + expectedValue: -1, + }, + { + name: "machine_group_high-performance", + config: Config{Profile: ""}, + userSetFlags: map[string]any{ + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: true, + expectedValue: 1024, + }, + { + name: "profile_overrides_machine_type", + config: Config{Profile: "aiml-training"}, + userSetFlags: map[string]any{ + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: true, + expectedValue: -1, + }, { + name: "fallback_to_machine_type_with_non_existent_profile", + config: Config{Profile: "non_existent_profile"}, + userSetFlags: map[string]any{ + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: true, + expectedValue: 1024, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // We need a copy of the config for each test case. + c := tc.config + // Set the default or non-default value on the config object. + if tc.name == "user_set" { + c.MetadataCache.StatCacheMaxSizeMb = tc.expectedValue.(int64) + } else { + c.MetadataCache.StatCacheMaxSizeMb = 34 + } + + v := viper.New() + for key, val := range tc.userSetFlags { + v.Set(key, val) + } + + optimizedFlags := c.ApplyOptimizations(v, tc.input) + + if tc.expectOptimized { + assert.Contains(t, optimizedFlags, "metadata-cache.stat-cache-max-size-mb") + } else { + assert.NotContains(t, optimizedFlags, "metadata-cache.stat-cache-max-size-mb") + } + // Use EqualValues to handle the int vs int64 type mismatch for default values. + assert.EqualValues(t, tc.expectedValue, c.MetadataCache.StatCacheMaxSizeMb) + }) + } + }) + // Tests for write.global-max-blocks + t.Run("write.global-max-blocks", func(t *testing.T) { + testCases := []struct { + name string + config Config + userSetFlags map[string]any + input *OptimizationInput + expectOptimized bool + expectedValue any + }{ + { + name: "user_set", + config: Config{}, + userSetFlags: map[string]any{ + "write.global-max-blocks": 98765, + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: false, + expectedValue: int64(98765), + }, + { + name: "no_optimization", + config: Config{Profile: "non_existent_profile"}, + userSetFlags: map[string]any{ + "machine-type": "low-end-machine", + }, + input: nil, + expectOptimized: false, + expectedValue: 4, + }, + { + name: "machine_group_high-performance", + config: Config{Profile: ""}, + userSetFlags: map[string]any{ + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: true, + expectedValue: 1600, + }, { + name: "fallback_to_machine_type_with_non_existent_profile", + config: Config{Profile: "non_existent_profile"}, + userSetFlags: map[string]any{ + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: true, + expectedValue: 1600, + }, { + name: "fallback_to_machine_type_when_aiml-training_is_unrelated", + config: Config{Profile: "aiml-training"}, + userSetFlags: map[string]any{ + "machine-type": "a2-megagpu-16g", + }, + input: nil, + expectOptimized: true, + expectedValue: 1600, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // We need a copy of the config for each test case. + c := tc.config + // Set the default or non-default value on the config object. + if tc.name == "user_set" { + c.Write.GlobalMaxBlocks = tc.expectedValue.(int64) + } else { + c.Write.GlobalMaxBlocks = 4 + } + + v := viper.New() + for key, val := range tc.userSetFlags { + v.Set(key, val) + } + + optimizedFlags := c.ApplyOptimizations(v, tc.input) + + if tc.expectOptimized { + assert.Contains(t, optimizedFlags, "write.global-max-blocks") + } else { + assert.NotContains(t, optimizedFlags, "write.global-max-blocks") + } + // Use EqualValues to handle the int vs int64 type mismatch for default values. + assert.EqualValues(t, tc.expectedValue, c.Write.GlobalMaxBlocks) + }) + } + }) +} diff --git a/cfg/config_util.go b/cfg/config_util.go index 1d4c740faa..193b206b97 100644 --- a/cfg/config_util.go +++ b/cfg/config_util.go @@ -17,9 +17,26 @@ package cfg import ( "fmt" "runtime" + "strings" "time" ) +const ( + // maxBackgroundLimit configures the upper limit for max-background kernel fuse + // setting. This is more than sufficient to saturate the 200 Gbps network bandwidth + // on a single VM. Revise the numbers if you plan to support higher bandwidth VMs. + maxBackgroundLimit = 192 +) + +func DefaultMaxBackground() int { + return min(max(12, 2*runtime.NumCPU()), maxBackgroundLimit) +} + +func DefaultCongestionThreshold() int { + // 75 % of DefaultMaxBackground + return (3 * DefaultMaxBackground()) / 4 +} + func DefaultMaxParallelDownloads() int { return max(16, 2*runtime.NumCPU()) } @@ -34,7 +51,7 @@ func IsParallelDownloadsEnabled(mountConfig *Config) bool { // IsTracingEnabled returns true if tracing is enabled. func IsTracingEnabled(mountConfig *Config) bool { - return mountConfig.Monitoring.ExperimentalTracingMode != "" + return len(mountConfig.Trace.Exporters) > 0 && mountConfig.Trace.SamplingRatio > 0 } // ListCacheTTLSecsToDuration converts TTL in seconds to time.Duration. @@ -50,3 +67,30 @@ func ListCacheTTLSecsToDuration(secs int64) time.Duration { return time.Duration(secs * int64(time.Second)) } + +// IsMetricsEnabled returns true if metrics are enabled. +func IsMetricsEnabled(c *MetricsConfig) bool { + return c.CloudMetricsExportIntervalSecs > 0 || c.PrometheusPort > 0 +} + +// IsGKEEnvironment returns true for /dev/fd/N mountpoints. +func IsGKEEnvironment(mountPoint string) bool { + return strings.HasPrefix(mountPoint, "/dev/fd/") +} + +// GetBucketType converts BucketType boolean flags to a BucketType enum. +// The priority order is: Zonal > Pirlo > Hierarchical > Flat. +// This is used to determine bucket-specific optimizations for kernel configs. +// TODO (b/472597952): Make BucketType in bucket_handle.go as enum +func GetBucketType(hierarchical, zonal, pirlo bool) BucketType { + if zonal { + return BucketTypeZonal + } + if pirlo { + return BucketTypePirlo + } + if hierarchical { + return BucketTypeHierarchical + } + return BucketTypeFlat +} diff --git a/cfg/config_util_test.go b/cfg/config_util_test.go index 13089dd0b6..a0d2f5ebd3 100644 --- a/cfg/config_util_test.go +++ b/cfg/config_util_test.go @@ -21,6 +21,16 @@ import ( "github.com/stretchr/testify/assert" ) +func Test_DefaultMaxBackground(t *testing.T) { + assert.GreaterOrEqual(t, DefaultMaxBackground(), 12) + assert.LessOrEqual(t, DefaultMaxBackground(), maxBackgroundLimit) +} + +func Test_DefaultCongestionThreshold(t *testing.T) { + assert.GreaterOrEqual(t, DefaultCongestionThreshold(), 9) + assert.LessOrEqual(t, DefaultCongestionThreshold(), 144) // 75% of maxBackgroundLimit +} + func Test_DefaultMaxParallelDownloads(t *testing.T) { assert.GreaterOrEqual(t, DefaultMaxParallelDownloads(), 16) } @@ -162,22 +172,112 @@ func Test_ListCacheTtlSecsToDuration_InvalidCall(t *testing.T) { func TestIsTracingEnabled(t *testing.T) { t.Parallel() var testCases = []struct { - testName string - traceMode string - expected bool + testName string + traceMode []string + traceSamplingRatio float64 + expected bool }{ - {"empty", "", false}, - {"not_empty", "gcptrace", true}, + {"empty_non_zero_sampling_ratio", []string{}, 0.5, false}, + {"non_empty_non_zero_sampling_ratio", []string{"gcpexporter"}, 0.4, true}, + {"non_empty_zero_sampling_ratio", []string{"gcpexporter"}, 0.0, false}, + {"empty_zero_samping_ratio", []string{}, 0.0, false}, } for _, tc := range testCases { - tc := tc t.Run(tc.testName, func(t *testing.T) { t.Parallel() - assert.Equal(t, tc.expected, IsTracingEnabled(&Config{Monitoring: MonitoringConfig{ - ExperimentalTracingMode: tc.traceMode, + assert.Equal(t, tc.expected, IsTracingEnabled(&Config{Trace: TraceConfig{ + Exporters: tc.traceMode, + SamplingRatio: tc.traceSamplingRatio, }})) }) } } + +func TestIsMetricsEnabled(t *testing.T) { + t.Parallel() + var testCases = []struct { + testName string + m *MetricsConfig + enabled bool + }{ + {"cloud_metrics_export_interval_set", &MetricsConfig{CloudMetricsExportIntervalSecs: 100}, true}, + {"prom_port_set", &MetricsConfig{PrometheusPort: 10000}, true}, + {"none_set", &MetricsConfig{CloudMetricsExportIntervalSecs: 0, PrometheusPort: 0}, false}, + } + + for _, tc := range testCases { + t.Run(tc.testName, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tc.enabled, IsMetricsEnabled(tc.m)) + }) + } +} + +func TestIsGKEEnvironment(t *testing.T) { + t.Parallel() + var testCases = []struct { + testName string + mountPoint string + expected bool + }{ + {"non-GKE", "/usr/local/mount-folder", false}, + {"GKE mountpoint", "/dev/fd/", true}, + {"GKE /dev/fd/N", "/dev/fd/8", true}, + } + + for _, tc := range testCases { + t.Run(tc.testName, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tc.expected, IsGKEEnvironment(tc.mountPoint)) + }) + } +} + +func TestGetBucketType(t *testing.T) { + tests := []struct { + name string + hierarchical bool + zonal bool + pirlo bool + expected BucketType + }{ + { + name: "Zonal and Hierarchical (zonal takes priority)", + hierarchical: true, + zonal: true, + expected: BucketTypeZonal, + }, + { + name: "Pirlo and Hierarchical (pirlo takes priority)", + hierarchical: true, + pirlo: true, + expected: BucketTypePirlo, + }, + { + name: "Hierarchical bucket", + hierarchical: true, + zonal: false, + expected: BucketTypeHierarchical, + }, + { + name: "Flat bucket", + hierarchical: false, + zonal: false, + expected: BucketTypeFlat, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := GetBucketType(tt.hierarchical, tt.zonal, tt.pirlo) + if result != tt.expected { + t.Errorf("GetBucketType(%v, %v, %v) = %v; want %v", + tt.hierarchical, tt.zonal, tt.pirlo, result, tt.expected) + } + }) + } +} diff --git a/cfg/constants.go b/cfg/constants.go index 7a1b2bed74..9d1fdf5273 100644 --- a/cfg/constants.go +++ b/cfg/constants.go @@ -18,7 +18,7 @@ import ( "math" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" ) const ( @@ -32,6 +32,13 @@ const ( OFF string = "OFF" ) +const ( + // Logging-format constants + logFormatText = "text" + logFormatJSON = "json" + defaultLogFormat = logFormatJSON +) + const ( // ExperimentalMetadataPrefetchOnMountDisabled is the mode without metadata-prefetch. ExperimentalMetadataPrefetchOnMountDisabled = "disabled" @@ -67,19 +74,36 @@ const ( // meant for two purposes. // 1. for conversion from stat-cache-capacity to stat-cache-max-size-mb. // 2. internal testing. - AverageSizeOfPositiveStatCacheEntry uint64 = 1400 + // Note: When adding 'X' bytes to the heap,it is expected the Resident Set Size + // which is closer to the actual memory usage increases by roughly '2X' + // due to overheads like fragmentation and alignment. + // Thus, incase an attribute of size x has been added to stat cache entry, update + // the average size here by 2x. + AverageSizeOfPositiveStatCacheEntry uint64 = 1464 // AverageSizeOfNegativeStatCacheEntry is the assumed size of each negative stat-cache-entry, // meant for two purposes. // 1. for conversion from stat-cache-capacity to stat-cache-max-size-mb. // 2. internal testing. - AverageSizeOfNegativeStatCacheEntry uint64 = 240 + AverageSizeOfNegativeStatCacheEntry uint64 = 256 // MetadataCacheTTLConfigKey is the Viper configuration key for the metadata //cache's time-to-live (TTL) in seconds. - MetadataCacheTTLConfigKey = "metadata-cache.ttl-secs" + MetadataCacheTTLConfigKey = "metadata-cache.ttl-secs" + MetadataCacheStatCacheTTLConfigKey = "metadata-cache.deprecated-stat-cache-ttl" + MetadataCacheTypeCacheTTLConfigKey = "metadata-cache.deprecated-type-cache-ttl" + MetadataCacheStatCacheCapacityConfigKey = "metadata-cache.deprecated-stat-cache-capacity" + MetadataNegativeCacheTTLConfigKey = "metadata-cache.negative-ttl-secs" // StatCacheMaxSizeConfigKey is the Viper configuration key for the maximum //size of the metadata stat cache in megabytes. - StatCacheMaxSizeConfigKey = "metadata-cache.stat-cache-max-size-mb" - maxSupportedStatCacheMaxSizeMB = util.MaxMiBsInUint64 + StatCacheMaxSizeConfigKey = "metadata-cache.stat-cache-max-size-mb" + // FileCacheParallelDownloadsConfigKey is the Viper configuration key for the + //parallel-downloads enablement. + FileCacheParallelDownloadsConfigKey = "file-cache.enable-parallel-downloads" + maxSupportedStatCacheMaxSizeMB = util.MaxMiBsInUint64 ) +// CacheUtilMinimumAlignSizeForWriting is the minimum buffer size used for memory-aligned +// writes, ensuring all writes are a multiple of this value to optimize for +// underlying storage. This may result in padding with null data if the content +// size is not a multiple of CacheUtilMinimumAlignSizeForWriting. +const CacheUtilMinimumAlignSizeForWriting = 4096 const ConfigFileFlagName = "config-file" diff --git a/cfg/decode_hook.go b/cfg/decode_hook.go index 898edb6d41..1bbfe39cbd 100644 --- a/cfg/decode_hook.go +++ b/cfg/decode_hook.go @@ -15,7 +15,7 @@ package cfg import ( - "github.com/mitchellh/mapstructure" + "github.com/go-viper/mapstructure/v2" ) // DecodeHook will be called by Viper while constructing the config object. diff --git a/cfg/decode_hook_test.go b/cfg/decode_hook_test.go index f00d67e66f..76dda62e66 100644 --- a/cfg/decode_hook_test.go +++ b/cfg/decode_hook_test.go @@ -241,15 +241,17 @@ func TestParsingSuccess(t *testing.T) { func TestParsingError(t *testing.T) { type TestConfig struct { - OctalParam Octal - LogSeverityParam LogSeverity - ProtocolParam Protocol + OctalParam Octal + LogSeverityParam LogSeverity + ProtocolParam Protocol + DirectPathStrategyParam DirectPathStrategy } declareFlags := func() *flag.FlagSet { fs := flag.NewFlagSet("test", flag.ExitOnError) fs.String("octalParam", "0", "") fs.String("logSeverityParam", "INFO", "") fs.String("protocolParam", "http1", "") + fs.String("directPathStrategyParam", "direct-path-with-fallback", "") return fs } bindFlags := func(fs *flag.FlagSet) *viper.Viper { @@ -258,6 +260,7 @@ func TestParsingError(t *testing.T) { bindFlag(t, v, "OctalParam", fs.Lookup("octalParam")) bindFlag(t, v, "LogSeverityParam", fs.Lookup("logSeverityParam")) bindFlag(t, v, "ProtocolParam", fs.Lookup("protocolParam")) + bindFlag(t, v, "DirectPathStrategyParam", fs.Lookup("directPathStrategyParam")) return v } tests := []struct { @@ -273,13 +276,18 @@ func TestParsingError(t *testing.T) { { name: "LogSeverity", args: []string{"--logSeverityParam=abc"}, - errMsg: "invalid logseverity value: abc. It can only assume values in the list: [TRACE DEBUG INFO WARNING ERROR OFF]", + errMsg: "invalid log severity level: abc. Must be one of [TRACE, DEBUG, INFO, WARNING, ERROR, OFF]", }, { name: "Protocol", args: []string{"--protocolParam=pqr"}, errMsg: "invalid protocol value: pqr. It can only accept values in the list: [http1 http2 grpc]", }, + { + name: "DirectPathStrategy", + args: []string{"--directPathStrategyParam=invalid-strategy"}, + errMsg: "invalid direct-path strategy value: invalid-strategy. It can only accept values in the list: [direct-path-only direct-path-with-fallback]", + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { diff --git a/cfg/optimize.go b/cfg/optimize.go new file mode 100644 index 0000000000..f346db7133 --- /dev/null +++ b/cfg/optimize.go @@ -0,0 +1,216 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +package cfg + +import ( + "fmt" + "io" + "net/http" + "slices" + "strings" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg/shared" + "github.com/spf13/viper" +) + +//////////////////////////////////////////////////////////////////////// +// Constants +//////////////////////////////////////////////////////////////////////// + +const ( + maxRetries = 2 + httpTimeout = 50 * time.Millisecond + machineTypeFlg = "machine-type" +) + +//////////////////////////////////////////////////////////////////////// +// Types +//////////////////////////////////////////////////////////////////////// + +// OptimizationResult holds the outcome of an optimization check, including the +// new value and the reason for the change. +type OptimizationResult struct { + // FinalValue is the value after applying all the optimizations. This will be the same as the original value if optimizations didn't change anything. + FinalValue any `yaml:"final_value" json:"final_value"` + // If value is optimized, then this will contain the description of what optimization caused the change, e.g. "profile aiml-training", or "machine-type a3-highgpu-8g" etc. + OptimizationReason string `yaml:"optimization_reason" json:"optimization_reason"` + // Optimized true indicates that the value was changed by optimization (either machine-type based, or profile-based). + Optimized bool `yaml:"-" json:"-"` // Field hidden from YAML and JSON to avoid it in logs. +} + +//////////////////////////////////////////////////////////////////////// +// Variables +//////////////////////////////////////////////////////////////////////// + +var ( + // metadataEndpoints are the endpoints to try for fetching metadata. + // Use an array to make provision for https endpoint in the future: https://cloud.google.com/compute/docs/metadata/querying-metadata#metadata_server_endpoints + metadataEndpoints = []string{ + "http://metadata.google.internal/computeMetadata/v1/instance/machine-type", + } +) + +//////////////////////////////////////////////////////////////////////// +// Helper Functions +//////////////////////////////////////////////////////////////////////// + +// getMetadata fetches metadata from a given endpoint. +func getMetadata(client *http.Client, endpoint string) ([]byte, error) { + req, err := http.NewRequest(http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request for %s: %w", endpoint, err) + } + req.Header.Add("Metadata-Flavor", "Google") + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("request to %s failed: %w", endpoint, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("request to %s returned non-OK status: %d", endpoint, resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body from %s: %w", endpoint, err) + } + + return body, nil +} + +// getMachineType fetches the machine type, checking user-provided configuration +// first (from CLI flags or config file), and falling back to the metadata server. +func getMachineType(v *viper.Viper) (string, error) { + // Precedence: CLI flag > Config file > Metadata server. + // 1. Check if the machine-type flag is set by the user (via CLI flag or config file). + if v.IsSet(machineTypeFlg) { + if currentMachineType := v.GetString(machineTypeFlg); currentMachineType != "" { + return currentMachineType, nil + } + } + // 2. Get machine-type from metadata server. + client := http.Client{Timeout: httpTimeout} + for range maxRetries { + for _, endpoint := range metadataEndpoints { + body, err := getMetadata(&client, endpoint) + if err != nil { + continue + } + + currentMachineType := string(body) + parts := strings.Split(currentMachineType, "/") + return parts[len(parts)-1], nil + } + } + + return "", fmt.Errorf("failed to get machine type from any metadata endpoint after retries") +} + +func isFlagPresent(flags []string, flag string) bool { + return slices.Contains(flags, flag) +} + +// getOptimizedValue contains the generic logic to determine the optimized value for a flag. +func getOptimizedValue( + rules *shared.OptimizationRules, + currentValue any, + profileName string, + machineType string, + input *OptimizationInput, + machineTypeToGroupMap map[string]string, +) OptimizationResult { + // Precedence: Profile -> Machine-type -> Bucket-type -> Default + // Assuming Machine and Bucket optimizations are applied on mutually exclusive flags, so + // precedence doesn't matter between them. + + // 1. If a profile with the given name is active and has optimization defined for it, then it takes precedence. + for _, p := range rules.Profiles { + if p.Name == profileName { + return OptimizationResult{ + FinalValue: p.Value, + OptimizationReason: fmt.Sprintf("profile %q", profileName), + Optimized: true, + } + } + } + + // 2. Only if no profile is set, check for a machine-based optimization. + if group, ok := machineTypeToGroupMap[machineType]; ok { + for _, mbo := range rules.MachineBasedOptimization { + if mbo.Group == group { + return OptimizationResult{ + FinalValue: mbo.Value, + OptimizationReason: fmt.Sprintf("machine-type group %q", group), + Optimized: true, + } + } + } + } + + // 3. If no profile is set, and no machine-type optimization applies, then check for bucket-type optimization. + if input != nil && input.BucketType.IsValid() { + for _, bto := range rules.BucketTypeOptimization { + if BucketType(bto.BucketType) == input.BucketType { + return OptimizationResult{ + FinalValue: bto.Value, + OptimizationReason: fmt.Sprintf("bucket-type %q", input.BucketType), + Optimized: true, + } + } + } + } + + // 4. If no optimization is found, return the original value. + return OptimizationResult{ + FinalValue: currentValue, + Optimized: false, + } +} + +// CreateHierarchicalOptimizedFlags converts a flat map with dot-separated keys +// into a nested map structure. +// It returns an error if a key prefix conflict is detected. +func CreateHierarchicalOptimizedFlags(flatMap map[string]OptimizationResult) (map[string]any, error) { + nestedMap := make(map[string]any) + + for key, value := range flatMap { + parts := strings.Split(key, ".") + currentLevel := nestedMap + + // Traverse the path and create intermediate maps. + for i, part := range parts[:len(parts)-1] { + // Intermediate part, ensure the next level map exists + if existingVal, exists := currentLevel[part]; exists { + if _, isMap := existingVal.(map[string]any); !isMap { + return nil, fmt.Errorf("key conflict: %q is both a path and a terminal key", strings.Join(parts[0:i+1], ".")) + } + currentLevel = existingVal.(map[string]any) + } else { + newLevel := make(map[string]any) + currentLevel[part] = newLevel + currentLevel = newLevel + } + } + + // Set the value at the final key. + lastKey := parts[len(parts)-1] + if _, exists := currentLevel[lastKey]; exists { + return nil, fmt.Errorf("key conflict: %q is both a path and a terminal key", key) + } + currentLevel[lastKey] = value + } + return nestedMap, nil +} diff --git a/cfg/optimize_test.go b/cfg/optimize_test.go new file mode 100644 index 0000000000..fe7656199c --- /dev/null +++ b/cfg/optimize_test.go @@ -0,0 +1,414 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +package cfg + +import ( + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func defaultConfig() Config { + return Config{MetadataCache: MetadataCacheConfig{NegativeTtlSecs: 5, TtlSecs: 60, StatCacheMaxSizeMb: 33, TypeCacheMaxSizeMb: 4}, ImplicitDirs: false, FileSystem: FileSystemConfig{RenameDirLimit: 0}, Write: WriteConfig{EnableStreamingWrites: true}} +} + +// Helper function to create a test server. +func createTestServer(t *testing.T, handler http.HandlerFunc) *httptest.Server { + t.Helper() + server := httptest.NewServer(handler) + return server +} + +// Helper function to close a test server. +func closeTestServer(t *testing.T, server *httptest.Server) { + t.Helper() + server.Close() +} + +// Helper function to reset metadataEndpoints. +func resetMetadataEndpoints(t *testing.T) { + t.Helper() + metadataEndpoints = []string{ + "http://metadata.google.internal/computeMetadata/v1/instance/machine-type", + } +} + +// Helper function to detect if a given flag is present in the map of optimized flags. +func isFlagPresentInOptimizationResults(optimizationResults map[string]OptimizationResult, flag string) bool { + _, ok := optimizationResults[flag] + return ok +} + +func TestGetMachineType_Failure(t *testing.T) { + resetMetadataEndpoints(t) + // Create a test server that returns a non-200 status code. + server := createTestServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + }) + defer closeTestServer(t, server) + // Override metadataEndpoints for testing. + metadataEndpoints = []string{server.URL} + + _, err := getMachineType(viper.New()) + + assert.Error(t, err) +} + +func TestGetMachineType_InputPrecedenceOrder(t *testing.T) { + tests := []struct { + name string + userSetFlags map[string]string + expectedMachineType string + }{ + { + name: "User_config_set_overrides_metadata", + userSetFlags: map[string]string{ + "machine-type": "test-machine-type", + }, + expectedMachineType: "test-machine-type", + }, + { + name: "User_config_not_set_falls_back_to_metadata", + userSetFlags: map[string]string{}, + expectedMachineType: "n1-standard-1", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + resetMetadataEndpoints(t) + // Create a test server that returns a machine type. + server := createTestServer(t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "zones/us-central1-a/machineTypes/n1-standard-1") + }) + defer closeTestServer(t, server) + // Override metadataEndpoints for testing. + metadataEndpoints = []string{server.URL} + v := viper.New() + for key, val := range tc.userSetFlags { + v.Set(key, val) + } + + machineType, err := getMachineType(v) + + require.NoError(t, err) + assert.Equal(t, tc.expectedMachineType, machineType) + }) + } +} + +func TestGetMachineType_QuotaError(t *testing.T) { + resetMetadataEndpoints(t) + // Create a test server that returns a quota error. + retryCount := 0 + server := createTestServer(t, func(w http.ResponseWriter, r *http.Request) { + retryCount++ + if retryCount < maxRetries { + w.WriteHeader(http.StatusTooManyRequests) + } else { + fmt.Fprint(w, "zones/us-central1-a/machineTypes/n1-standard-1") + } + }) + defer closeTestServer(t, server) + // Override metadataEndpoints for testing. + metadataEndpoints = []string{server.URL} + + machineType, err := getMachineType(viper.New()) + + require.NoError(t, err) + assert.Equal(t, "n1-standard-1", machineType) +} + +func TestApplyOptimizations_DisableAutoConfig(t *testing.T) { + resetMetadataEndpoints(t) + // Create a test server that returns a matching machine type. + server := createTestServer(t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "zones/us-central1-a/machineTypes/a3-highgpu-8g") + }) + defer closeTestServer(t, server) + // Override metadataEndpoints for testing. + metadataEndpoints = []string{server.URL} + cfg := defaultConfig() + cfg.DisableAutoconfig = true + + optimizedFlags := cfg.ApplyOptimizations(viper.New(), nil) + + require.Empty(t, optimizedFlags) + assert.EqualValues(t, 5, cfg.MetadataCache.NegativeTtlSecs) + assert.EqualValues(t, 60, cfg.MetadataCache.TtlSecs) + assert.EqualValues(t, 33, cfg.MetadataCache.StatCacheMaxSizeMb) + assert.EqualValues(t, 4, cfg.MetadataCache.TypeCacheMaxSizeMb) + assert.False(t, cfg.ImplicitDirs) + assert.EqualValues(t, 0, cfg.FileSystem.RenameDirLimit) +} + +func TestApplyOptimizations_MatchingMachineType(t *testing.T) { + resetMetadataEndpoints(t) + // Create a test server that returns a matching machine type. + server := createTestServer(t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "zones/us-central1-a/machineTypes/a3-highgpu-8g") + }) + defer closeTestServer(t, server) + // Override metadataEndpoints for testing. + metadataEndpoints = []string{server.URL} + cfg := defaultConfig() + + optimizedFlags := cfg.ApplyOptimizations(viper.New(), nil) + + assert.NotEmpty(t, optimizedFlags) + assert.EqualValues(t, 0, cfg.MetadataCache.NegativeTtlSecs) + assert.EqualValues(t, -1, cfg.MetadataCache.TtlSecs) + assert.EqualValues(t, 1024, cfg.MetadataCache.StatCacheMaxSizeMb) + assert.True(t, cfg.ImplicitDirs) + assert.EqualValues(t, 200000, cfg.FileSystem.RenameDirLimit) +} + +func TestApplyOptimizations_NonMatchingMachineType(t *testing.T) { + resetMetadataEndpoints(t) + // Create a test server that returns a non-matching machine type. + server := createTestServer(t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "zones/us-central1-a/machineTypes/n1-standard-1") + }) + defer closeTestServer(t, server) + // Override metadataEndpoints for testing. + metadataEndpoints = []string{server.URL} + cfg := defaultConfig() + + optimizedFlags := cfg.ApplyOptimizations(viper.New(), nil) + + assert.Empty(t, optimizedFlags) + assert.EqualValues(t, 5, cfg.MetadataCache.NegativeTtlSecs) + assert.EqualValues(t, 60, cfg.MetadataCache.TtlSecs) + assert.EqualValues(t, 33, cfg.MetadataCache.StatCacheMaxSizeMb) + assert.EqualValues(t, 4, cfg.MetadataCache.TypeCacheMaxSizeMb) + assert.False(t, cfg.ImplicitDirs) + assert.EqualValues(t, 0, cfg.FileSystem.RenameDirLimit) +} + +func TestApplyOptimizations_UserSetFlag(t *testing.T) { + resetMetadataEndpoints(t) + // Create a test server that returns a matching machine type. + server := createTestServer(t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "zones/us-central1-a/machineTypes/a3-highgpu-8g") + }) + defer closeTestServer(t, server) + // Override metadataEndpoints for testing. + metadataEndpoints = []string{server.URL} + cfg := defaultConfig() + v := viper.New() + v.Set("file-system.rename-dir-limit", 10000) + // Simulate setting config value by user + cfg.FileSystem.RenameDirLimit = 10000 + + optimizedFlags := cfg.ApplyOptimizations(v, nil) + + assert.NotEmpty(t, optimizedFlags) + assert.EqualValues(t, 0, cfg.MetadataCache.NegativeTtlSecs) + assert.EqualValues(t, -1, cfg.MetadataCache.TtlSecs) + assert.EqualValues(t, 1024, cfg.MetadataCache.StatCacheMaxSizeMb) + assert.True(t, cfg.ImplicitDirs) + assert.EqualValues(t, 10000, cfg.FileSystem.RenameDirLimit) +} + +func TestApplyOptimizations_GetMachineTypeError(t *testing.T) { + resetMetadataEndpoints(t) + // Create a test server that returns an error. + server := createTestServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + }) + defer closeTestServer(t, server) + // Override metadataEndpoints for testing. + metadataEndpoints = []string{server.URL} + cfg := defaultConfig() + + optimizedFlags := cfg.ApplyOptimizations(viper.New(), nil) + + assert.Empty(t, optimizedFlags) + assert.EqualValues(t, 5, cfg.MetadataCache.NegativeTtlSecs) + assert.EqualValues(t, 60, cfg.MetadataCache.TtlSecs) + assert.EqualValues(t, 33, cfg.MetadataCache.StatCacheMaxSizeMb) + assert.EqualValues(t, 4, cfg.MetadataCache.TypeCacheMaxSizeMb) + assert.False(t, cfg.ImplicitDirs) + assert.EqualValues(t, 0, cfg.FileSystem.RenameDirLimit) +} + +func TestApplyOptimizations_NoError(t *testing.T) { + resetMetadataEndpoints(t) + // Create a test server that returns a matching machine type. + server := createTestServer(t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "zones/us-central1-a/machineTypes/a3-highgpu-8g") + }) + defer closeTestServer(t, server) + // Override metadataEndpoints for testing. + metadataEndpoints = []string{server.URL} + cfg := defaultConfig() + + optimizedFlags := cfg.ApplyOptimizations(viper.New(), nil) + + assert.NotEmpty(t, optimizedFlags) +} + +func TestApplyOptimizations_Success(t *testing.T) { + resetMetadataEndpoints(t) + // Create a test server that returns a matching machine type. + server := createTestServer(t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "zones/us-central1-a/machineTypes/a3-highgpu-8g") + }) + defer closeTestServer(t, server) + // Override metadataEndpoints for testing. + metadataEndpoints = []string{server.URL} + cfg := defaultConfig() + + optimizedFlags := cfg.ApplyOptimizations(viper.New(), nil) + + assert.True(t, isFlagPresentInOptimizationResults(optimizedFlags, "write.global-max-blocks")) + assert.EqualValues(t, 1600, cfg.Write.GlobalMaxBlocks) + assert.True(t, isFlagPresentInOptimizationResults(optimizedFlags, "metadata-cache.negative-ttl-secs")) + assert.EqualValues(t, 0, cfg.MetadataCache.NegativeTtlSecs) + assert.EqualValues(t, -1, cfg.MetadataCache.TtlSecs) + assert.EqualValues(t, 1024, cfg.MetadataCache.StatCacheMaxSizeMb) + assert.True(t, cfg.ImplicitDirs) + assert.EqualValues(t, 200000, cfg.FileSystem.RenameDirLimit) +} + +func TestCreateHierarchicalOptimizedFlags_Positive(t *testing.T) { + testCases := []struct { + name string + inputMap map[string]OptimizationResult + expected map[string]any + }{ + { + name: "Empty map", + inputMap: map[string]OptimizationResult{}, + expected: map[string]any{}, + }, + { + name: "Flat keys", + inputMap: map[string]OptimizationResult{ + "key1": {FinalValue: "value1", OptimizationReason: "reason1"}, + "key2": {FinalValue: 123, OptimizationReason: "reason2"}, + }, + expected: map[string]any{ + "key1": OptimizationResult{FinalValue: "value1", OptimizationReason: "reason1"}, + "key2": OptimizationResult{FinalValue: 123, OptimizationReason: "reason2"}, + }, + }, + { + name: "Single level nesting", + inputMap: map[string]OptimizationResult{ + "a.b": {FinalValue: "valueAB", OptimizationReason: "reasonAB"}, + "a.c": {FinalValue: "valueAC", OptimizationReason: "reasonAC"}, + }, + expected: map[string]any{ + "a": map[string]any{ + "b": OptimizationResult{FinalValue: "valueAB", OptimizationReason: "reasonAB"}, + "c": OptimizationResult{FinalValue: "valueAC", OptimizationReason: "reasonAC"}, + }, + }, + }, + { + name: "Multi-level nesting", + inputMap: map[string]OptimizationResult{ + "a.b.c": {FinalValue: "valueABC", OptimizationReason: "reasonABC"}, + "a.b.d": {FinalValue: "valueABD", OptimizationReason: "reasonABD"}, + "x.y.z": {FinalValue: true, OptimizationReason: "reasonXYZ"}, + }, + expected: map[string]any{ + "a": map[string]any{ + "b": map[string]any{ + "c": OptimizationResult{FinalValue: "valueABC", OptimizationReason: "reasonABC"}, + "d": OptimizationResult{FinalValue: "valueABD", OptimizationReason: "reasonABD"}, + }, + }, + "x": map[string]any{ + "y": map[string]any{ + "z": OptimizationResult{FinalValue: true, OptimizationReason: "reasonXYZ"}, + }, + }, + }, + }, + { + name: "No conflict complex keys", + inputMap: map[string]OptimizationResult{ + "metadata-cache.ttl-secs": {FinalValue: int64(-1), OptimizationReason: "reasonTTL"}, + "metadata-cache.stat-cache-max-size-mb": {FinalValue: int64(1024), OptimizationReason: "reasonStat"}, + "file-cache.cache-file-for-range-read": {FinalValue: true, OptimizationReason: "reasonFileCache"}, + }, + expected: map[string]any{ + "metadata-cache": map[string]any{ + "ttl-secs": OptimizationResult{FinalValue: int64(-1), OptimizationReason: "reasonTTL"}, + "stat-cache-max-size-mb": OptimizationResult{FinalValue: int64(1024), OptimizationReason: "reasonStat"}, + }, + "file-cache": map[string]any{ + "cache-file-for-range-read": OptimizationResult{FinalValue: true, OptimizationReason: "reasonFileCache"}, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := CreateHierarchicalOptimizedFlags(tc.inputMap) + + assert.NoError(t, err) + if !reflect.DeepEqual(tc.expected, got) { + t.Errorf("CreateHierarchicalOptimizedFlags() = %v, want %v", got, tc.expected) + } + }) + } +} + +func TestCreateHierarchicalOptimizedFlags_Negative(t *testing.T) { + testCases := []struct { + name string + inputMap map[string]OptimizationResult + }{ + { + name: "Conflict: Prefix as terminal key first", + inputMap: map[string]OptimizationResult{ + "a.b": {FinalValue: "valAB", OptimizationReason: "rAB"}, + "a.b.d": {FinalValue: "valABD", OptimizationReason: "rABD"}, + }, + }, + { + name: "Conflict: Path key first", + inputMap: map[string]OptimizationResult{ + "a.b.d": {FinalValue: "valABD", OptimizationReason: "rABD"}, + "a.b": {FinalValue: "valAB", OptimizationReason: "rAB"}, + }, + }, + { + name: "Conflict: Deeper nesting", + inputMap: map[string]OptimizationResult{ + "a.b.c": {FinalValue: "valABC", OptimizationReason: "rABC"}, + "a.b.c.d": {FinalValue: "valABCD", OptimizationReason: "rABCD"}, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := CreateHierarchicalOptimizedFlags(tc.inputMap) + + assert.Error(t, err) + assert.Nil(t, got) + assert.Contains(t, err.Error(), "key conflict") + }) + } +} diff --git a/cfg/params.yaml b/cfg/params.yaml index 2771cb9775..cd2daafd7f 100644 --- a/cfg/params.yaml +++ b/cfg/params.yaml @@ -32,658 +32,1320 @@ # ###################################### DOCUMENTATION ENDS ###################### -- config-path: "app-name" - flag-name: "app-name" - type: "string" - usage: "The application name of this mount." - default: "" - -- config-path: "cache-dir" - flag-name: "cache-dir" - type: "resolvedPath" - usage: "Enables file-caching. Specifies the directory to use for file-cache." - -- config-path: "debug.exit-on-invariant-violation" - flag-name: "debug_invariants" - type: "bool" - usage: "Exit when internal invariants are violated." - default: false - -- config-path: "debug.fuse" - flag-name: "debug_fuse" - type: "bool" - usage: "Enables debug logs." - default: false - deprecated: true - deprecation-warning: "Please set log-severity to TRACE instead." - -- config-path: "debug.gcs" - flag-name: "debug_gcs" - type: "bool" - usage: "Enables debug logs." - default: false - deprecated: true - deprecation-warning: "Please set log-severity to TRACE instead." - -- config-path: "debug.log-mutex" - flag-name: "debug_mutex" - type: "bool" - usage: "Print debug messages when a mutex is held too long." - default: false - -- config-path: "enable-hns" - flag-name: "enable-hns" - type: "bool" - usage: "Enables support for HNS buckets" - default: true - hide-flag: true - -- config-path: "file-cache.cache-file-for-range-read" - flag-name: "file-cache-cache-file-for-range-read" - type: "bool" - usage: "Whether to cache file for range reads." - default: false - -- config-path: "file-cache.download-chunk-size-mb" - flag-name: "file-cache-download-chunk-size-mb" - type: "int" - usage: "Size of chunks in MiB that each concurrent request downloads." - default: "50" - -- config-path: "file-cache.enable-crc" - flag-name: "file-cache-enable-crc" - type: "bool" - usage: "Performs CRC to ensure that file is correctly downloaded into cache." - default: false - hide-flag: true - -- config-path: "file-cache.enable-o-direct" - flag-name: "file-cache-enable-o-direct" - type: "bool" - usage: "Whether to use O_DIRECT while writing to file-cache in case of parallel downloads." - default: "false" - hide-flag: true - -- config-path: "file-cache.enable-parallel-downloads" - flag-name: "file-cache-enable-parallel-downloads" - type: "bool" - usage: "Enable parallel downloads." - default: false - -- config-path: "file-cache.max-parallel-downloads" - flag-name: "file-cache-max-parallel-downloads" - type: "int" - usage: "Sets an uber limit of number of concurrent file download requests that are made across all files." - default: "DefaultMaxParallelDownloads()" - -- config-path: "file-cache.max-size-mb" - flag-name: "file-cache-max-size-mb" - type: "int" - usage: "Maximum size of the file-cache in MiBs" - default: "-1" - -- config-path: "file-cache.parallel-downloads-per-file" - flag-name: "file-cache-parallel-downloads-per-file" - type: "int" - usage: "Number of concurrent download requests per file." - default: "16" - -- config-path: "file-cache.write-buffer-size" - flag-name: "file-cache-write-buffer-size" - type: "int" - usage: "Size of in-memory buffer that is used per goroutine in parallel downloads while writing to file-cache." - default: "4194304" # 4MiB - hide-flag: true - -- config-path: "file-system.dir-mode" - flag-name: "dir-mode" - type: "octal" - usage: "Permissions bits for directories, in octal." - default: "0755" - -- config-path: "file-system.disable-parallel-dirops" - flag-name: "disable-parallel-dirops" - type: "bool" - usage: "Specifies whether to allow parallel dir operations (lookups and readers)" - default: false - hide-flag: true - -- config-path: "file-system.file-mode" - flag-name: "file-mode" - type: "octal" - usage: "Permissions bits for files, in octal." - default: "0644" - -- config-path: "file-system.fuse-options" - flag-name: "o" - type: "[]string" - usage: "Additional system-specific mount options. Multiple options can be passed as comma separated. For readonly, use --o ro" - -- config-path: "file-system.gid" - flag-name: "gid" - type: "int" - default: -1 - usage: "GID owner of all inodes." - -- config-path: "file-system.handle-sigterm" - flag-name: "handle-sigterm" - type: "bool" - usage: >- - Instructs gcsfuse to handle SIGTERM to gracefully shutdown - default: true - hide-flag: true - -- config-path: "file-system.ignore-interrupts" - flag-name: "ignore-interrupts" - type: "bool" - usage: >- - Instructs gcsfuse to ignore system interrupt signals (like SIGINT, triggered - by Ctrl+C). This prevents those signals from immediately terminating gcsfuse - inflight operations. (default: true) - default: true - -- config-path: "file-system.kernel-list-cache-ttl-secs" - flag-name: "kernel-list-cache-ttl-secs" - type: "int" - usage: >- - How long the directory listing (output of ls <dir>) should be cached in the - kernel page cache. If a particular directory cache entry is kept by kernel - for longer than TTL, then it will be sent for invalidation by gcsfuse on - next opendir (comes in the start, as part of next listing) call. 0 means no - caching. Use -1 to cache for lifetime (no ttl). Negative value other than -1 - will throw error. - default: "0" - -- config-path: "file-system.precondition-errors" - flag-name: "precondition-errors" - type: "bool" - usage: >- - Throw Stale NFS file handle error in case the object being synced or read - from is modified by some other concurrent process. This helps prevent - silent data loss or data corruption. - hide-flag: true - default: true - -- config-path: "file-system.rename-dir-limit" - flag-name: "rename-dir-limit" - type: "int" - usage: "Allow rename a directory containing fewer descendants than this limit." - default: "0" - -- config-path: "file-system.temp-dir" - flag-name: "temp-dir" - type: "resolvedPath" - usage: >- - Path to the temporary directory where writes are staged prior to upload to - Cloud Storage. (default: system default, likely /tmp) - default: "" - -- config-path: "file-system.uid" - flag-name: "uid" - type: "int" - default: -1 - usage: "UID owner of all inodes." - -- flag-name: "foreground" - config-path: "foreground" - type: "bool" - usage: "Stay in the foreground after mounting." - default: false - -- config-path: "gcs-auth.anonymous-access" - flag-name: "anonymous-access" - type: "bool" - usage: "Authentication is enabled by default. This flag disables authentication" - default: false - -- config-path: "gcs-auth.key-file" - flag-name: "key-file" - type: "resolvedPath" - usage: "Absolute path to JSON key file for use with GCS. (The default is none, Google application default credentials used)" - -- config-path: "gcs-auth.reuse-token-from-url" - flag-name: "reuse-token-from-url" - type: "bool" - usage: "If false, the token acquired from token-url is not reused." - default: "true" - -- config-path: "gcs-auth.token-url" - flag-name: "token-url" - type: "string" - usage: "A url for getting an access token when the key-file is absent." - default: "" - -- config-path: "gcs-connection.billing-project" - flag-name: "billing-project" - type: "string" - usage: >- - Project to use for billing when accessing a bucket enabled with "Requester - Pays". (The default is none) - default: "" - -- config-path: "gcs-connection.client-protocol" - flag-name: "client-protocol" - type: "protocol" - usage: >- - The protocol used for communicating with the GCS backend. - Value can be 'http1' (HTTP/1.1), 'http2' (HTTP/2) or 'grpc'. - default: "http1" - -- config-path: "gcs-connection.custom-endpoint" - flag-name: "custom-endpoint" - type: "string" - usage: >- - Specifies an alternative custom endpoint for fetching data. Should only be - used for testing. The custom endpoint must support the equivalent resources - and operations as the GCS JSON endpoint, - https://storage.googleapis.com/storage/v1. If a custom endpoint is not - specified, GCSFuse uses the global GCS JSON API endpoint, - https://storage.googleapis.com/storage/v1. - default: "" - - -- config-path: "gcs-connection.experimental-enable-json-read" - flag-name: "experimental-enable-json-read" - type: "bool" - usage: >- - By default, GCSFuse uses the GCS XML API to get and read objects. When this - flag is specified, GCSFuse uses the GCS JSON API instead." - default: false - deprecated: true - deprecation-warning: "Experimental flag: could be dropped even in a minor release." - -- config-path: "gcs-connection.grpc-conn-pool-size" - flag-name: "experimental-grpc-conn-pool-size" - type: "int" - usage: "The number of gRPC channel in grpc client." - default: "1" - deprecated: true - deprecation-warning: "Experimental flag: can be removed in a minor release." - -- config-path: "gcs-connection.http-client-timeout" - flag-name: "http-client-timeout" - type: "duration" - usage: >- - The time duration that http client will wait to get response from the - server. The default value 0 indicates no timeout. - default: "0s" - -- config-path: "gcs-connection.limit-bytes-per-sec" - flag-name: "limit-bytes-per-sec" - type: "float64" - usage: "Bandwidth limit for reading data, measured over a 30-second window. (use -1 for no limit)" - default: "-1" - -- config-path: "gcs-connection.limit-ops-per-sec" - flag-name: "limit-ops-per-sec" - type: "float64" - usage: "Operations per second limit, measured over a 30-second window (use -1 for no limit)" - default: "-1" - -- config-path: "gcs-connection.max-conns-per-host" - flag-name: "max-conns-per-host" - type: "int" - usage: >- - The max number of TCP connections allowed per server. This is effective when - client-protocol is set to 'http1'. The default value 0 indicates no limit on - TCP connections (limited by the machine specifications). - default: "0" - -- config-path: "gcs-connection.max-idle-conns-per-host" - flag-name: "max-idle-conns-per-host" - type: "int" - usage: "The number of maximum idle connections allowed per server." - default: "100" - -- config-path: "gcs-connection.sequential-read-size-mb" - flag-name: "sequential-read-size-mb" - type: "int" - usage: "File chunk size to read from GCS in one call. Need to specify the value in MB. ChunkSize less than 1MB is not supported" - default: "200" - -- config-path: "gcs-retries.max-retry-attempts" - flag-name: "max-retry-attempts" - type: "int" - usage: >- - It sets a limit on the number of times an operation will be retried if it - fails, preventing endless retry loops. The default value 0 indicates no limit. - default: "0" - -- config-path: "gcs-retries.max-retry-sleep" - flag-name: "max-retry-sleep" - type: "duration" - usage: >- - The maximum duration allowed to sleep in a retry loop with exponential - backoff for failed requests to GCS backend. Once the backoff duration - exceeds this limit, the retry continues with this specified maximum value. - default: "30s" - -- config-path: "gcs-retries.multiplier" - flag-name: "retry-multiplier" - type: "float64" - usage: Param for exponential backoff algorithm, which is used to increase waiting time b/w two consecutive retries. - default: 2 - -- config-path: "gcs-retries.read-stall.enable" - flag-name: "enable-read-stall-retry" - type: "bool" - usage: >- - To turn on/off retries for stalled read requests. This is based on a timeout - that changes depending on how long similar requests took in the past. - default: false - hide-flag: true - -- config-path: "gcs-retries.read-stall.initial-req-timeout" - flag-name: "read-stall-initial-req-timeout" - type: "duration" - usage: Initial value of the read-request dynamic timeout. - default: 20s - hide-flag: true - -- config-path: "gcs-retries.read-stall.max-req-timeout" - flag-name: "read-stall-max-req-timeout" - type: "duration" - usage: Upper bound of the read-request dynamic timeout. - default: 20m - hide-flag: true - -- config-path: "gcs-retries.read-stall.min-req-timeout" - flag-name: "read-stall-min-req-timeout" - type: "duration" - usage: Lower bound of the read request dynamic timeout. - default: 1500ms - hide-flag: true - -- config-path: "gcs-retries.read-stall.req-increase-rate" - flag-name: "read-stall-req-increase-rate" - type: "float64" - usage: Determines how many increase calls it takes for dynamic timeout to double. - default: 15 - hide-flag: true - -- config-path: "gcs-retries.read-stall.req-target-percentile" - flag-name: "read-stall-req-target-percentile" - type: "float64" - usage: Retry the request which take more than p(targetPercentile * 100) of past similar request. - default: 0.99 - hide-flag: true - -- config-path: "implicit-dirs" - flag-name: "implicit-dirs" - type: "bool" - usage: "Implicitly define directories based on content. See files and directories in docs/semantics for more information" - default: false - -- config-path: "list.enable-empty-managed-folders" - flag-name: "enable-empty-managed-folders" - type: "bool" - usage: >- - This handles the corner case in listing managed folders. - There are two corner cases (a) empty managed folder (b) nested managed folder which doesn't contain any descendent as object. - This flag always works in conjunction with --implicit-dirs flag. - (a) If only ImplicitDirectories is true, all managed folders are listed other than above two mentioned cases. - (b) If both ImplicitDirectories and EnableEmptyManagedFolders are true, then all the managed folders are listed including the above-mentioned corner case. - (c) If ImplicitDirectories is false then no managed folders are listed irrespective of enable-empty-managed-folders flag. - default: false - hide-flag: true - -- config-path: "logging.file-path" - flag-name: "log-file" - type: "resolvedPath" - usage: >- - The file for storing logs that can be parsed by fluentd. When not provided, - plain text logs are printed to stdout when Cloud Storage FUSE is run - in the foreground, or to syslog when Cloud Storage FUSE is run in the - background. - -- config-path: "logging.format" - flag-name: "log-format" - type: "string" - usage: "The format of the log file: 'text' or 'json'." - default: "json" - -- config-path: "logging.log-rotate.backup-file-count" - flag-name: "log-rotate-backup-file-count" - type: "int" - usage: >- - The maximum number of backup log files to retain after they have been - rotated. The default value is 10. When value is set to 0, all backup files are - retained. - default: "10" - -- config-path: "logging.log-rotate.compress" - flag-name: "log-rotate-compress" - type: "bool" - usage: "Controls whether the rotated log files should be compressed using gzip." - default: "true" - -- config-path: "logging.log-rotate.max-file-size-mb" - flag-name: "log-rotate-max-file-size-mb" - type: "int" - usage: "The maximum size in megabytes that a log file can reach before it is rotated." - default: "512" - -- config-path: "logging.severity" - flag-name: "log-severity" - type: "logSeverity" - usage: "Specifies the logging severity expressed as one of [trace, debug, info, warning, error, off]" - default: "info" - -- config-path: "metadata-cache.deprecated-stat-cache-capacity" - flag-name: "stat-cache-capacity" - type: "int" - usage: >- - How many entries can the stat-cache hold (impacts memory consumption). This - flag has been deprecated (starting v2.0) and in favor of - stat-cache-max-size-mb. For now, the value of stat-cache-capacity will be - translated to the next higher corresponding value of stat-cache-max-size-mb - (assuming stat-cache entry-size ~= 1640 bytes, including 1400 for positive - entry and 240 for corresponding negative entry), if stat-cache-max-size-mb - is not set." - deprecated: true - deprecation-warning: "Please use --stat-cache-max-size-mb instead." - default: "20460" - -- config-path: "metadata-cache.deprecated-stat-cache-ttl" - flag-name: "stat-cache-ttl" - type: "duration" - usage: >- - How long to cache StatObject results and inode attributes. This flag - has been deprecated (starting v2.0) in favor of metadata-cache-ttl-secs. For - now, the minimum of stat-cache-ttl and type-cache-ttl values, rounded up to - the next higher multiple of a second is used as ttl for both stat-cache and - type-cache, when metadata-cache-ttl-secs is not set. - default: "60s" - deprecated: true - deprecation-warning: >- - This flag has been deprecated (starting v2.0) in favor of - metadata-cache-ttl-secs. - -- config-path: "metadata-cache.deprecated-type-cache-ttl" - flag-name: "type-cache-ttl" - type: "duration" - usage: >- - Usage: How long to cache StatObject results and inode attributes. This flag - has been deprecated (starting v2.0) in favor of metadata-cache-ttl-secs. For - now, the minimum of stat-cache-ttl and type-cache-ttl values, rounded up to - the next higher multiple of a second is used as ttl for both stat-cache and - type-cache, when metadata-cache-ttl-secs is not set. - default: "60s" - deprecated: true - deprecation-warning: >- - This flag has been deprecated (starting v2.0) in favor of - metadata-cache-ttl-secs. - -- config-path: "metadata-cache.enable-nonexistent-type-cache" - flag-name: "enable-nonexistent-type-cache" - type: "bool" - usage: >- - Once set, if an inode is not found in GCS, a type cache entry with type - NonexistentType will be created. This also means new file/dir created might - not be seen. For example, if this flag is set, and metadata-cache-ttl-secs - is set, then if we create the same file/node in the meantime using the same - mount, since we are not refreshing the cache, it will still return nil. - default: false - -- config-path: "metadata-cache.experimental-metadata-prefetch-on-mount" - flag-name: "experimental-metadata-prefetch-on-mount" - type: "string" - usage: >- - Experimental: This indicates whether or not to prefetch the metadata - (prefilling of metadata caches and creation of inodes) of the mounted bucket - at the time of mounting the bucket. Supported values: "disabled", "sync" and - "async". Any other values will return error on mounting. This is applicable - only to static mounting, and not to dynamic mounting. - default: "disabled" - deprecated: true - deprecation-warning: "Experimental flag: could be removed even in a minor release." - -- config-path: "metadata-cache.stat-cache-max-size-mb" - flag-name: "stat-cache-max-size-mb" - type: "int" - usage: >- - The maximum size of stat-cache in MiBs. It can also be set to -1 for - no-size-limit, 0 for no cache. Values below -1 are not supported. - default: "32" - -- config-path: "metadata-cache.ttl-secs" - flag-name: "metadata-cache-ttl-secs" - type: "int" - usage: >- - The ttl value in seconds to be used for expiring items in metadata-cache. It - can be set to -1 for no-ttl, 0 for no cache and > 0 for ttl-controlled - metadata-cache. Any value set below -1 will throw an error. - default: "60" - -- config-path: "metadata-cache.type-cache-max-size-mb" - flag-name: "type-cache-max-size-mb" - type: "int" - usage: "Max size of type-cache maps which are maintained at a per-directory level." - default: "4" - -- config-path: "metrics.cloud-metrics-export-interval-secs" - flag-name: "cloud-metrics-export-interval-secs" - type: "int" - usage: "Specifies the interval at which the metrics are uploaded to cloud monitoring" - default: 0 - -- config-path: "metrics.enable-otel" - flag-name: "enable-otel" - type: "bool" - usage: "Specifies whether to use OpenTelemetry for capturing and exporting metrics. If false, use OpenCensus." - default: false - hide-flag: true - -- config-path: "metrics.prometheus-port" - flag-name: "prometheus-port" - type: "int" - usage: "Expose Prometheus metrics endpoint on this port and a path of /metrics." - default: "0" - hide-flag: true - -- config-path: "metrics.stackdriver-export-interval" - flag-name: "stackdriver-export-interval" - type: "duration" - usage: >- - Export metrics to stackdriver with this interval. The default value 0 - indicates no exporting. - default: "0s" - deprecated: true - deprecation-warning: "Please use --cloud-metrics-export-interval-secs instead." - -- config-path: "monitoring.experimental-opentelemetry-collector-address" - flag-name: "experimental-opentelemetry-collector-address" - type: "string" - usage: "Experimental: Export metrics to the OpenTelemetry collector at this address." - default: "" - deprecated: true - deprecation-warning: "Experimental flag: could be dropped even in a minor release." - -- config-path: "monitoring.experimental-tracing-mode" - flag-name: "experimental-tracing-mode" - type: "string" - usage: "Experimental: specify tracing mode" - default: "" - hide-flag: true - -- config-path: "monitoring.experimental-tracing-sampling-ratio" - flag-name: "experimental-tracing-sampling-ratio" - type: "float64" - usage: "Experimental: Trace sampling ratio" - default: 0 - hide-flag: true - -- config-path: "only-dir" - flag-name: "only-dir" - type: "string" - usage: "Mount only a specific directory within the bucket. See docs/mounting for more information" - default: "" - -- config-path: "write.block-size-mb" - flag-name: "write-block-size-mb" - type: "int" - usage: >- - Specifies the block size for streaming writes. The value should be more - than 0. - default: 64 #TODO: revisit default value after perf testing. - hide-flag: true - -- config-path: "write.create-empty-file" - flag-name: "create-empty-file" - type: "bool" - usage: "For a new file, it creates an empty file in Cloud Storage bucket as a - hold." - default: false - -- config-path: "write.experimental-enable-streaming-writes" - flag-name: "experimental-enable-streaming-writes" - type: "bool" - usage: "Enables streaming uploads during write file operation." - default: false - hide-flag: true - -- config-path: "write.global-max-blocks" - flag-name: "write-global-max-blocks" - type: "int" - usage: >- - Specifies the maximum number of blocks to be used by all files for - streaming writes. The value should be >= 2 or -1 (for infinite blocks). - default: -1 #TODO: revisit default value after perf testing. - hide-flag: true - -- config-path: "write.max-blocks-per-file" - flag-name: "write-max-blocks-per-file" - type: "int" - usage: >- - Specifies the maximum number of blocks to be used by a single file for - streaming writes. The value should be >= 2 or -1 (for infinite blocks). - default: -1 #TODO: revisit default value after perf testing. - hide-flag: true - -- flag-name: "debug_fs" - type: "bool" - usage: "This flag is unused." - default: false - deprecated: true - deprecation-warning: "This flag is currently unused." - -- flag-name: "debug_fuse_errors" - type: "bool" - usage: "This flag is currently unused." - default: "true" - deprecated: true - deprecation-warning: "This flag is currently unused." - -- flag-name: "debug_http" - type: "bool" - usage: "This flag is currently unused." - default: false - deprecated: true - deprecation-warning: "This flag is currently unused." - -- flag-name: "max-retry-duration" - type: "duration" - usage: "This is currently unused." - default: "0s" - deprecated: true - deprecation-warning: "This is currently unused." +# Keep this section at the top of the file. +machine-type-groups: + high-performance: + - "a2-megagpu-16g" + - "a2-ultragpu-8g" + - "a3-edgegpu-8g" + - "a3-highgpu-8g" + - "a3-megagpu-8g" + - "a3-ultragpu-8g" + - "a4-highgpu-8g" + - "a4-highgpu-8g-lowmem" + - "a4-highgpu-8g-nolssd" + - "a4x-highgpu-4g" + - "a4x-highgpu-4g-nolssd" + - "a4x-maxgpu-4g-metal" + - "ct5l-hightpu-8t" + - "ct5lp-hightpu-8t" + - "ct5p-hightpu-4t" + - "ct5p-hightpu-4t-tpu" + - "ct6e-standard-4t" + - "ct6e-standard-4t-tpu" + - "ct6e-standard-8t" + - "ct6e-standard-8t-tpu" + - "tpu7x-standard-4t" + - "tpu7x-standard-4t-tpu" + - "tpu7x-ultranet-4t" + - "tpu7x-ultranet-4t-tpu" + # Add other machine types here as needed. + # Add other groups here as needed + +params: + - config-path: "app-name" + flag-name: "app-name" + type: "string" + usage: "The application name of this mount." + default: "" + + - config-path: "cache-dir" + flag-name: "cache-dir" + type: "resolvedPath" + usage: "Enables file-caching. Specifies the directory to use for file-cache." + + - config-path: "cloud-profiler.allocated-heap" + flag-name: "cloud-profiler-allocated-heap" + type: "bool" + usage: "Enables allocated heap (HeapProfileAllocs) profiling. This only works when --enable-cloud-profiler is set to true." + default: true + hide-flag: true + + - config-path: "cloud-profiler.cpu" + flag-name: "cloud-profiler-cpu" + type: "bool" + usage: "Enables cpu profiling. This only works when --enable-cloud-profiler is set to true." + default: true + hide-flag: true + + - config-path: "cloud-profiler.enabled" + flag-name: "enable-cloud-profiler" + type: "bool" + usage: "Enables cloud-profiler, by default disabled." + default: false + hide-flag: true + + - config-path: "cloud-profiler.goroutines" + flag-name: "cloud-profiler-goroutines" + type: "bool" + usage: "Enables goroutines cloud-profiler. This only works when --enable-cloud-profiler is set to true." + default: false + hide-flag: true + + - config-path: "cloud-profiler.heap" + flag-name: "cloud-profiler-heap" + type: "bool" + usage: "Enables heap cloud-profiler. This only works when --enable-cloud-profiler is set to true." + default: true + hide-flag: true + + - config-path: "cloud-profiler.label" + flag-name: "cloud-profiler-label" + type: "string" + usage: >- + Allow setting a profile label to uniquely identify and compare cloud-profiler data with other profiles. + This only works when --enable-cloud-profiler is set to true. + default: "gcsfuse-0.0.0" + hide-flag: true + + - config-path: "cloud-profiler.mutex" + flag-name: "cloud-profiler-mutex" + type: "bool" + usage: "Enables mutex cloud-profiler. This only works when --enable-cloud-profiler is set to true." + default: false + hide-flag: true + + - config-path: "cloud-profiler.service-name" + flag-name: "cloud-profiler-service-name" + type: "string" + usage: "The service name for cloud-profiler. This only works when --enable-cloud-profiler is set to true." + default: "gcsfuse" + hide-flag: true + + - config-path: "debug.exit-on-invariant-violation" + flag-name: "debug_invariants" + type: "bool" + usage: "Exit when internal invariants are violated." + default: false + + - config-path: "debug.fuse" + flag-name: "debug_fuse" + type: "bool" + usage: "Enables debug logs." + default: false + deprecated: true + deprecation-warning: "Please set log-severity to TRACE instead." + + - config-path: "debug.gcs" + flag-name: "debug_gcs" + type: "bool" + usage: "Enables debug logs." + default: false + deprecated: true + deprecation-warning: "Please set log-severity to TRACE instead." + + - config-path: "debug.log-mutex" + flag-name: "debug_mutex" + type: "bool" + usage: "Print debug messages when a mutex is held too long." + default: false + + - config-path: "disable-autoconfig" + flag-name: "disable-autoconfig" + type: "bool" + usage: "Disable optimizing configuration automatically for a machine" + default: false + hide-flag: true + + - config-path: "disable-list-access-check" + flag-name: "disable-list-access-check" + type: "bool" + usage: "Disables the list object based access check during mount operation" + default: true + hide-flag: true + + - config-path: "dummy-io.enable" + flag-name: "enable-dummy-io" + type: "bool" + usage: >- + Enable dummy I/O mode for testing purposes. In this mode all reads + and writes are simulated and no actual data is transferred to or from + Cloud Storage. All the metadata operations like object listing and stats + are real. + default: false + hide-flag: true + + - config-path: "dummy-io.per-mb-latency" + flag-name: "dummy-io-per-mb-latency" + type: "duration" + usage: >- + Simulates reading from the reader latency in dummy I/O mode. This value + is only used when dummy I/O mode is enabled. + default: "0s" + hide-flag: true + + - config-path: "dummy-io.reader-latency" + flag-name: "dummy-io-reader-latency" + type: "duration" + usage: >- + Simulates reader creation latency in dummy I/O mode. This value + is only used when dummy I/O mode is enabled. + default: "0s" + hide-flag: true + + - config-path: "enable-atomic-rename-object" + flag-name: "enable-atomic-rename-object" + type: "bool" + usage: "Enables support for atomic rename object operation on HNS bucket." + default: true + hide-flag: true + + - flag-name: "enable-google-lib-auth" + config-path: "enable-google-lib-auth" + type: "bool" + usage: "Enable google library authentication method to fetch the credentials" + default: true + hide-flag: true + + - config-path: "enable-hns" + flag-name: "enable-hns" + type: "bool" + usage: "Enables support for HNS buckets" + default: true + hide-flag: true + + - config-path: "enable-new-reader" + flag-name: "enable-new-reader" + type: "bool" + usage: "Enables support for new reader implementation." + default: true + hide-flag: true + + - config-path: "enable-standard-symlinks" + flag-name: "enable-standard-symlinks" + type: "bool" + usage: >- + Enables the creation and reading of symbolic links using the standard GCS representation. + When enabled, new symlinks created via GCSFuse mount ensure compatibility with other + GCS clients like Storage Transfer Service (STS). + default: true + hide-flag: true + + - config-path: "enable-type-cache-deprecation" + flag-name: "enable-type-cache-deprecation" + type: "bool" + usage: "Enables support to deprecate type cache." + default: true + hide-flag: true + + - config-path: "enable-unsupported-path-support" + flag-name: "enable-unsupported-path-support" + type: "bool" + usage: >- + Enables support for file system paths with unsupported GCS names (e.g., names containing '//' or starting with /). + When set, GCSFuse will ignore these objects during listing and copying operations. + For rename and delete operations, the flag allows the action to proceed for all specified objects, including those with unsupported names. + default: true + hide-flag: true + + - config-path: "file-cache.cache-file-for-range-read" + flag-name: "file-cache-cache-file-for-range-read" + type: "bool" + usage: "Whether to cache file for range reads." + default: false + optimizations: + profiles: + - name: "aiml-serving" + value: true + - name: "aiml-checkpointing" + value: true + + - config-path: "file-cache.download-chunk-size-mb" + flag-name: "file-cache-download-chunk-size-mb" + type: "int" + usage: "Size of chunks in MiB that each concurrent request downloads." + default: "200" + + - config-path: "file-cache.enable-crc" + flag-name: "file-cache-enable-crc" + type: "bool" + usage: "Performs CRC to ensure that file is correctly downloaded into cache. No op for rapid storage." + default: false + hide-flag: true + + - config-path: "file-cache.enable-experimental-shared-chunk-cache" + flag-name: "enable-experimental-shared-chunk-cache" + type: "bool" + usage: >- + [EXPERIMENTAL] Enable chunk-based shared cache that allows multiple + gcsfuse mount instances to safely share the same cache directory (e.g., on NFS). + Uses fixed size chunks with atomic operations (write + rename) to download chunk + file without locks. Ideal for distributed environments where multiple nodes need + to share cached GCS data. + default: false + hide-flag: false + + - config-path: "file-cache.enable-o-direct" + flag-name: "file-cache-enable-o-direct" + type: "bool" + usage: "Whether to use O_DIRECT while writing to file-cache in case of parallel downloads." + default: "false" + hide-flag: true + + - config-path: "file-cache.enable-parallel-downloads" + flag-name: "file-cache-enable-parallel-downloads" + type: "bool" + usage: "Enable parallel downloads." + default: false + + - config-path: "file-cache.exclude-regex" + flag-name: "file-cache-exclude-regex" + type: "string" + usage: "Exclude file paths (in the format bucket_name/object_key) specified by this regex from file caching." + default: "" + + - config-path: "file-cache.experimental-disable-size-calculation-fix" + flag-name: "file-cache-experimental-disable-size-calculation-fix" + type: "bool" + usage: "Disable the fix in calculation of disk-utilization of file-cache." + default: false + hide-flag: true + + - config-path: "file-cache.experimental-enable-chunk-cache" + flag-name: "file-cache-experimental-enable-chunk-cache" + type: "bool" + usage: "Enable chunk cache mode for random I/O optimization that downloads only requested blocks." + default: false + hide-flag: true + + - config-path: "file-cache.experimental-parallel-downloads-default-on" + flag-name: "file-cache-experimental-parallel-downloads-default-on" + type: "bool" + usage: "Enable parallel downloads by default on experimental basis." + default: true + hide-flag: true + + - config-path: "file-cache.include-regex" + flag-name: "file-cache-include-regex" + type: "string" + usage: "Include file paths (in the format bucket_name/object_key) specified by this regex for file caching." + default: "" + + - config-path: "file-cache.max-parallel-downloads" + flag-name: "file-cache-max-parallel-downloads" + type: "int" + usage: "Sets an uber limit of number of concurrent file download requests that are made across all files." + default: "DefaultMaxParallelDownloads()" + + - config-path: "file-cache.max-size-mb" + flag-name: "file-cache-max-size-mb" + type: "int" + usage: "Maximum size of the file-cache in MiBs" + default: "-1" + + - config-path: "file-cache.parallel-downloads-per-file" + flag-name: "file-cache-parallel-downloads-per-file" + type: "int" + usage: "Number of concurrent download requests per file." + default: "16" + + - config-path: "file-cache.shared-cache-chunk-size-mb" + flag-name: "file-cache-shared-cache-chunk-size-mb" + type: "int" + usage: "Chunk size in MiBs for shared chunk cache. Each chunk is downloaded on-demand." + default: "8" + hide-flag: true + + - config-path: "file-cache.write-buffer-size" + flag-name: "file-cache-write-buffer-size" + type: "int" + usage: "Size of in-memory buffer that is used per goroutine in parallel downloads while writing to file-cache." + default: "4194304" # 4MiB + hide-flag: true + + - config-path: "file-system.congestion-threshold" + flag-name: "congestion-threshold" + type: "int" + usage: >- + Sets the congestion threshold for background requests. When the number of + outstanding requests exceeds this threshold, the kernel may start blocking + new requests. 0 means system default (typically 75% of max-background; 9). + default: "0" + hide-flag: true + optimizations: + bucket-type-optimization: + - bucket-type: "zonal" + value: "DefaultCongestionThreshold()" + - bucket-type: "pirlo" + value: "DefaultCongestionThreshold()" + + - config-path: "file-system.dir-mode" + flag-name: "dir-mode" + type: "octal" + usage: "Permissions bits for directories, in octal." + default: "0755" + + - config-path: "file-system.disable-parallel-dirops" + flag-name: "disable-parallel-dirops" + type: "bool" + usage: "Specifies whether to allow parallel dir operations (lookups and readers)" + default: false + hide-flag: true + + - config-path: "file-system.enable-kernel-reader" + flag-name: "enable-kernel-reader" + type: "bool" + usage: "Enables kernel reader, disables prefetching gcsfuse side and relies on kernel read-ahead and page-cache." + default: false + hide-flag: true + optimizations: + bucket-type-optimization: + - bucket-type: "zonal" + value: true + - bucket-type: "pirlo" + value: true + + - config-path: "file-system.experimental-enable-dentry-cache" + flag-name: "experimental-enable-dentry-cache" + type: "bool" + usage: >- + When enabled, it sets the Dentry cache entry timeout same as metadata-cache-ttl. + This enables kernel to use cached entry to map the file paths to inodes, + instead of making LookUpInode calls to GCSFuse. + default: false + hide-flag: true + + - config-path: "file-system.experimental-enable-pirlo" + flag-name: "experimental-enable-pirlo" + type: "bool" + usage: "Enables support for pirlo." + default: false + hide-flag: true + + - config-path: "file-system.experimental-enable-readdirplus" + flag-name: "experimental-enable-readdirplus" + type: "bool" + usage: "Enables ReadDirPlus capability" + default: false + hide-flag: true + + - config-path: "file-system.experimental-o-direct" + flag-name: "experimental-o-direct" + type: "bool" + usage: >- + Experimental: Bypasses the kernel's page cache for file reads and writes. When enabled, + all I/O operations are sent directly to the GCSFuse process. + default: false + hide-flag: true + + - config-path: "file-system.file-mode" + flag-name: "file-mode" + type: "octal" + usage: "Permissions bits for files, in octal." + default: "0644" + + - config-path: "file-system.fuse-options" + flag-name: "o" + type: "[]string" + usage: "Additional system-specific mount options. Multiple options can be passed as comma separated. For readonly, use --o ro" + + - config-path: "file-system.gid" + flag-name: "gid" + type: "int" + default: -1 + usage: "GID owner of all inodes." + + - config-path: "file-system.ignore-interrupts" + flag-name: "ignore-interrupts" + type: "bool" + usage: >- + Instructs gcsfuse to ignore system interrupt signals (like SIGINT, triggered + by Ctrl+C). This prevents those signals from immediately terminating gcsfuse + inflight operations. + default: true + + - config-path: "file-system.inactive-mrd-cache-size" + flag-name: "inactive-mrd-cache-size" + type: "int" + usage: >- + Sets the cache-size of inactive (no open file) MRD instances. When this limit + is exceeded, the least recently inactive MRD instances will be closed. Set to + 0 to disable the cache, which will keep all the inactive MRD instances open forever. + default: "1000" + hide-flag: true + + - config-path: "file-system.kernel-list-cache-ttl-secs" + flag-name: "kernel-list-cache-ttl-secs" + type: "int" + usage: >- + How long the directory listing (output of ls <dir>) should be cached in the + kernel page cache. If a particular directory cache entry is kept by kernel + for longer than TTL, then it will be sent for invalidation by gcsfuse on + next opendir (comes in the start, as part of next listing) call. 0 means no + caching. Use -1 to cache for lifetime (no ttl). Negative value other than -1 + will throw error. + default: "0" + optimizations: + profiles: + - name: "aiml-serving" + value: -1 + + - config-path: "file-system.kernel-params-file" + flag-name: "kernel-params-file" + type: "resolvedPath" + usage: >- + File path used to communicate various kernel parameters to CSI Driver in GKE environment. + hide-flag: true + + - config-path: "file-system.max-background" + flag-name: "max-background" + type: "int" + usage: >- + Sets the maximum number of outstanding background requests (e.g., request corresponding to + kernel readahead, writeback cache etc.) that the kernel will send to the FUSE daemon. 0 means + system default + (typically 12). + default: "0" + hide-flag: true + optimizations: + bucket-type-optimization: + - bucket-type: "zonal" + value: "DefaultMaxBackground()" + - bucket-type: "pirlo" + value: "DefaultMaxBackground()" + + - config-path: "file-system.max-read-ahead-kb" + flag-name: "max-read-ahead-kb" + type: "int" + usage: >- + Sets max kernel-read-ahead for the mount in KiB. 0 means system default. + Requires sudo permission to set this value, otherwise the value will be ignored + and system default will be used. + default: "0" + hide-flag: true + optimizations: + bucket-type-optimization: + - bucket-type: "zonal" + value: 16384 # 16 MiB + - bucket-type: "pirlo" + value: 16384 # 16 MiB + + - config-path: "file-system.rename-dir-limit" + flag-name: "rename-dir-limit" + type: "int" + usage: "Allow rename a directory containing fewer descendants than this limit." + default: "0" + optimizations: + machine-based-optimization: + - group: "high-performance" + value: 200000 + profiles: + - name: "aiml-checkpointing" + value: 200000 + + - config-path: "file-system.temp-dir" + flag-name: "temp-dir" + type: "resolvedPath" + usage: >- + Path to the temporary directory where writes are staged prior to upload to + Cloud Storage. (default: system default, likely /tmp) + default: "" + + - config-path: "file-system.uid" + flag-name: "uid" + type: "int" + default: -1 + usage: "UID owner of all inodes." + + - flag-name: "foreground" + config-path: "foreground" + type: "bool" + usage: "Stay in the foreground after mounting." + default: false + + - config-path: "gcs-auth.anonymous-access" + flag-name: "anonymous-access" + type: "bool" + usage: "This flag disables authentication." + default: false + + - config-path: "gcs-auth.key-file" + flag-name: "key-file" + type: "resolvedPath" + usage: "Absolute path to JSON key file for use with GCS. If this flag is left unset, Google application default credentials are used." + + - config-path: "gcs-auth.reuse-token-from-url" + flag-name: "reuse-token-from-url" + type: "bool" + usage: "If false, the token acquired from token-url is not reused." + default: "true" + + - config-path: "gcs-auth.token-url" + flag-name: "token-url" + type: "string" + usage: "A url for getting an access token when the key-file is absent." + default: "" + + - config-path: "gcs-connection.billing-project" + flag-name: "billing-project" + type: "string" + usage: >- + Project to use for billing when accessing a bucket enabled with "Requester + Pays". + default: "" + + - config-path: "gcs-connection.client-protocol" + flag-name: "client-protocol" + type: "protocol" + usage: >- + The protocol used for communicating with the GCS backend. + Value can be 'http1' (HTTP/1.1), 'http2' (HTTP/2) or 'grpc'. + default: "http1" + + - config-path: "gcs-connection.custom-endpoint" + flag-name: "custom-endpoint" + type: "string" + usage: >- + To specify a custom storage endpoint, ensure it supports the same resources as the default storage.googleapis.com:443 and includes the port number. + default: "" + + - config-path: "gcs-connection.enable-http-dns-cache" + flag-name: "enable-http-dns-cache" + type: "bool" + usage: "Enables DNS cache for HTTP/1 connections" + default: true + hide-flag: true + + - config-path: "gcs-connection.experimental-enable-json-read" + flag-name: "experimental-enable-json-read" + type: "bool" + usage: >- + By default, GCSFuse uses the GCS XML API to get and read objects. When this + flag is specified, GCSFuse uses the GCS JSON API instead." + default: false + deprecated: true + deprecation-warning: "Experimental flag: could be dropped even in a minor release." + + - config-path: "gcs-connection.experimental-local-socket-address" + flag-name: "experimental-local-socket-address" + type: "string" + usage: "The local socket address to bind to. This is useful in multi-NIC scenarios. This is an experimental flag." + default: "" + hide-flag: true + + - config-path: "gcs-connection.grpc-conn-pool-size" + flag-name: "experimental-grpc-conn-pool-size" + type: "int" + usage: "The number of gRPC channel in grpc client." + default: "1" + deprecated: true + deprecation-warning: "Experimental flag: can be removed in a minor release." + + - config-path: "gcs-connection.grpc-path-strategy" + flag-name: "grpc-path-strategy" + type: "directPathStrategy" + usage: >- + Strategy for DirectPath connectivity when client-protocol=grpc. + Options: 'direct-path-only' (fail if unavailable), 'direct-path-with-fallback' (always fallback to HTTP/1 when direct path is not available). + default: "direct-path-with-fallback" + hide-flag: true + + - config-path: "gcs-connection.http-client-timeout" + flag-name: "http-client-timeout" + type: "duration" + usage: >- + The time duration that http client will wait to get response from the + server. A value of 0 indicates no timeout. + default: "0s" + + - config-path: "gcs-connection.limit-bytes-per-sec" + flag-name: "limit-bytes-per-sec" + type: "float64" + usage: "Bandwidth limit for reading data, measured over a 30-second window. (use -1 for no limit)" + default: "-1" + + - config-path: "gcs-connection.limit-ops-per-sec" + flag-name: "limit-ops-per-sec" + type: "float64" + usage: "Operations per second limit, measured over a 30-second window (use -1 for no limit)" + default: "-1" + + - config-path: "gcs-connection.max-conns-per-host" + flag-name: "max-conns-per-host" + type: "int" + usage: >- + The max number of TCP connections allowed per server. This is effective when + client-protocol is set to 'http1'. A value of 0 indicates no limit on + TCP connections (limited by the machine specifications). + default: "0" + + - config-path: "gcs-connection.max-idle-conns-per-host" + flag-name: "max-idle-conns-per-host" + type: "int" + usage: "The number of maximum idle connections allowed per server." + default: "100" + + - config-path: "gcs-connection.sequential-read-size-mb" + flag-name: "sequential-read-size-mb" + type: "int" + usage: "File chunk size to read from GCS in one call. Need to specify the value in MB. ChunkSize less than 1MB is not supported" + default: "200" + + - config-path: "gcs-retries.chunk-retry-deadline-secs" + flag-name: "chunk-retry-deadline-secs" + type: "int" + usage: >- + We send larger file uploads in 16 MiB (Legacy Writes) or 32MiB (Streaming Writes) chunks. This flag controls the overall duration + that GCSFuse would keep retrying for a single chunk upload completion. 0 means infinity duration for chunk retries. + default: "120" + hide-flag: true + + - config-path: "gcs-retries.chunk-transfer-timeout-secs" + flag-name: "chunk-transfer-timeout-secs" + type: "int" + usage: >- + We send larger file uploads in 16 MiB (Legacy Writes) or 32MiB (Streaming Writes) chunks. This flag controls the duration + that the HTTP client will wait for a response after making a request to upload a chunk. + As an example, a value of 10 indicates that the client will wait 10 seconds for upload completion; + otherwise, it cancels the request and retries for that chunk till chunk retry deadline duration. 0 means no timeout. + default: "10" + hide-flag: true + + - config-path: "gcs-retries.enable-mount-retries" + flag-name: "enable-mount-retries" + type: "bool" + usage: >- + If true, enables retry logic in GCSFuse during the mount sequence + for additional errors (such as metadata server readiness delays, IAM propagation + delays, and temporary bucket non-existence). Intended specifically for the + GKE GCSFuse CSI Driver. + default: false + hide-flag: true + + - config-path: "gcs-retries.experimental-nonrapid-folder-api-stall-retry" + flag-name: "experimental-nonrapid-folder-api-stall-retry" + type: "bool" + usage: "Enables stall-retry-fix for folder APIs for non-rapid buckets." + default: false + hide-flag: true + + - config-path: "gcs-retries.max-retry-attempts" + flag-name: "max-retry-attempts" + type: "int" + usage: >- + It sets a limit on the total number of attempts (including the initial call) made for an operation if it fails, + preventing endless retry loops. For example, a value of 5 means up to 5 total attempts (1 initial call plus 4 retries). + A value of 0 indicates unlimited attempts. + default: "0" + + - config-path: "gcs-retries.max-retry-sleep" + flag-name: "max-retry-sleep" + type: "duration" + usage: >- + The maximum backoff sleep duration allowed between retry attempts. + Once the exponential backoff exceeds this limit, subsequent retries will use this constant sleep value. + default: "30s" + + - config-path: "gcs-retries.multiplier" + flag-name: "retry-multiplier" + type: "float64" + usage: >- + The multiplier factor by which the retry backoff duration increases after each failed attempt. + For example, a multiplier of 2.0 doubles the backoff sleep duration for each subsequent retry. + default: 2 + + - config-path: "gcs-retries.read-stall.enable" + flag-name: "enable-read-stall-retry" + type: "bool" + usage: >- + To turn on/off retries for stalled read requests. This is based on a timeout + that changes depending on how long similar requests took in the past. + default: true + hide-flag: true + + - config-path: "gcs-retries.read-stall.initial-req-timeout" + flag-name: "read-stall-initial-req-timeout" + type: "duration" + usage: Initial value of the read-request dynamic timeout. + default: 20s + hide-flag: true + + - config-path: "gcs-retries.read-stall.max-req-timeout" + flag-name: "read-stall-max-req-timeout" + type: "duration" + usage: Upper bound of the read-request dynamic timeout. + default: 20m + hide-flag: true + + - config-path: "gcs-retries.read-stall.min-req-timeout" + flag-name: "read-stall-min-req-timeout" + type: "duration" + usage: Lower bound of the read request dynamic timeout. + default: 1500ms + hide-flag: true + + - config-path: "gcs-retries.read-stall.req-increase-rate" + flag-name: "read-stall-req-increase-rate" + type: "float64" + usage: Determines how many increase calls it takes for dynamic timeout to double. + default: 15 + hide-flag: true + + - config-path: "gcs-retries.read-stall.req-target-percentile" + flag-name: "read-stall-req-target-percentile" + type: "float64" + usage: Retry the request which take more than p(targetPercentile * 100) of past similar request. + default: 0.99 + hide-flag: true + + - config-path: "implicit-dirs" + flag-name: "implicit-dirs" + type: "bool" + usage: "Implicitly define directories based on content. See files and directories in docs/semantics for more information" + default: false + optimizations: + machine-based-optimization: + - group: "high-performance" + value: true + profiles: + - name: "aiml-training" + value: true + - name: "aiml-serving" + value: true + - name: "aiml-checkpointing" + value: true + + - config-path: "list.enable-empty-managed-folders" + flag-name: "enable-empty-managed-folders" + type: "bool" + usage: >- + This handles the corner case in listing managed folders. + There are two corner cases (a) empty managed folder (b) nested managed folder which doesn't contain any descendent as object. + This flag always works in conjunction with --implicit-dirs flag. + (a) If only ImplicitDirectories is true, all managed folders are listed other than above two mentioned cases. + (b) If both ImplicitDirectories and EnableEmptyManagedFolders are true, then all the managed folders are listed including the above-mentioned corner case. + (c) If ImplicitDirectories is false then no managed folders are listed irrespective of enable-empty-managed-folders flag. + default: false + hide-flag: true + + - config-path: "logging.file-path" + flag-name: "log-file" + type: "resolvedPath" + usage: >- + The file for storing logs that can be parsed by fluentd. When not provided, + plain text logs are printed to stdout when Cloud Storage FUSE is run + in the foreground, or to syslog when Cloud Storage FUSE is run in the + background. + + - config-path: "logging.format" + flag-name: "log-format" + type: "string" + usage: "The format of the log file: 'text' or 'json'." + default: "json" + + - config-path: "logging.log-rotate.backup-file-count" + flag-name: "log-rotate-backup-file-count" + type: "int" + usage: >- + The maximum number of backup log files to retain after they have been + rotated. A value of 0 indicates all backup files are retained. + default: "10" + + - config-path: "logging.log-rotate.compress" + flag-name: "log-rotate-compress" + type: "bool" + usage: "Controls whether the rotated log files should be compressed using gzip." + default: "true" + + - config-path: "logging.log-rotate.max-file-size-mb" + flag-name: "log-rotate-max-file-size-mb" + type: "int" + usage: "The maximum size in megabytes that a log file can reach before it is rotated." + default: "512" + + - config-path: "logging.severity" + flag-name: "log-severity" + type: "logSeverity" + usage: "Specifies the logging severity expressed as one of [trace, debug, info, warning, error, off]" + default: "info" + + - config-path: "logging.wire-log" + flag-name: "wire-log" + type: "resolvedPath" + usage: >- + The file name of the wire log. When specified, GCSFuse will serialize + each FUSE operation as a JSON object and append it to this file. + hide-flag: true + + - config-path: "machine-type" + flag-name: "machine-type" + type: "string" + usage: "Type of the machine on which gcsfuse is being run e.g. a3-highgpu-4g" + default: "" + hide-flag: true + + - config-path: "metadata-cache.deprecated-stat-cache-capacity" + flag-name: "stat-cache-capacity" + type: "int" + usage: >- + How many entries can the stat-cache hold (impacts memory consumption). This + flag has been deprecated (starting v2.0) and in favor of + stat-cache-max-size-mb. For now, the value of stat-cache-capacity will be + translated to the next higher corresponding value of stat-cache-max-size-mb + (assuming stat-cache entry-size ~= 1720 bytes, including 1464 for positive + entry and 256 for corresponding negative entry), if stat-cache-max-size-mb + is not set." + deprecated: true + deprecation-warning: "Please use --stat-cache-max-size-mb instead." + default: "20460" + + - config-path: "metadata-cache.deprecated-stat-cache-ttl" + flag-name: "stat-cache-ttl" + type: "duration" + usage: >- + How long to cache StatObject results and inode attributes. This flag + has been deprecated (starting v2.0) in favor of metadata-cache-ttl-secs. For + now, the minimum of stat-cache-ttl and type-cache-ttl values, rounded up to + the next higher multiple of a second is used as ttl for both stat-cache and + type-cache, when metadata-cache-ttl-secs is not set. + default: "60s" + deprecated: true + deprecation-warning: >- + This flag has been deprecated (starting v2.0) in favor of + metadata-cache-ttl-secs. + + - config-path: "metadata-cache.deprecated-type-cache-ttl" + flag-name: "type-cache-ttl" + type: "duration" + usage: >- + Usage: How long to cache StatObject results and inode attributes. This flag + has been deprecated (starting v2.0) in favor of metadata-cache-ttl-secs. For + now, the minimum of stat-cache-ttl and type-cache-ttl values, rounded up to + the next higher multiple of a second is used as ttl for both stat-cache and + type-cache, when metadata-cache-ttl-secs is not set. + default: "60s" + deprecated: true + deprecation-warning: >- + This flag has been deprecated (starting v2.0) in favor of + metadata-cache-ttl-secs. + + - config-path: "metadata-cache.enable-metadata-prefetch" + flag-name: "enable-metadata-prefetch" + type: "bool" + usage: >- + Enables background prefetching of object metadata when a directory is first opened. + This reduces latency for subsequent file lookups by pre-filling the metadata cache. + default: false + deprecated: false + hide-flag: false + + - config-path: "metadata-cache.enable-nonexistent-type-cache" + flag-name: "enable-nonexistent-type-cache" + type: "bool" + usage: >- + Once set, if an inode is not found in GCS, a type cache entry with type + NonexistentType will be created. This also means new file/dir created might + not be seen. For example, if this flag is set, and metadata-cache-ttl-secs + is set, then if we create the same file/node in the meantime using the same + mount, since we are not refreshing the cache, it will still return nil. + This flag has been deprecated in favour of a single unified flag metadata-cache-negative-ttl-secs. + default: false + + - config-path: "metadata-cache.experimental-metadata-prefetch-on-mount" + flag-name: "experimental-metadata-prefetch-on-mount" + type: "string" + usage: >- + Experimental: This indicates whether or not to prefetch the metadata + (prefilling of metadata caches and creation of inodes) of the mounted bucket + at the time of mounting the bucket. Supported values: "disabled", "sync" and + "async". Any other values will return error on mounting. This is applicable + only to static mounting, and not to dynamic mounting. + default: "disabled" + deprecated: true + deprecation-warning: "Experimental flag: could be removed even in a minor release." + + - config-path: "metadata-cache.metadata-prefetch-entries-limit" + flag-name: "metadata-prefetch-entries-limit" + type: "int" + usage: > + The maximum number of metadata entries (files and directories) to prefetch + into the cache upon a prefetch trigger. Since a single GCS List call is capped + at 5000 results, values higher than 5000 will trigger multiple sequential GCS + List calls per directory. + default: "5000" + deprecated: false + hide-flag: false + + - config-path: "metadata-cache.metadata-prefetch-max-workers" + flag-name: "metadata-prefetch-max-workers" + type: "int" + usage: > + The maximum number of concurrent goroutines (workers) allowed to perform + metadata prefetching across all directories. + default: "10" + deprecated: false + hide-flag: false + + - config-path: "metadata-cache.negative-ttl-secs" + flag-name: "metadata-cache-negative-ttl-secs" + type: "int" + usage: >- + The negative-ttl-secs value in seconds to be used for expiring negative entries in metadata-cache. It + can be set to -1 for no-ttl, 0 for no cache and > 0 for ttl-controlled + negative entries in metadata-cache. Any value set below -1 will throw an error. + default: "5" + optimizations: + machine-based-optimization: + - group: "high-performance" + value: 0 + profiles: + - name: "aiml-training" + value: 0 + - name: "aiml-serving" + value: 0 + - name: "aiml-checkpointing" + value: 0 + + - config-path: "metadata-cache.stat-cache-max-size-mb" + flag-name: "stat-cache-max-size-mb" + type: "int" + usage: >- + The maximum size of stat-cache in MiBs. It can also be set to -1 for + no-size-limit, 0 for no cache. Values below -1 are not supported. + default: "34" + optimizations: + machine-based-optimization: + - group: "high-performance" + value: 1024 + profiles: + - name: "aiml-training" + value: -1 + - name: "aiml-serving" + value: -1 + - name: "aiml-checkpointing" + value: -1 + + - config-path: "metadata-cache.ttl-secs" + flag-name: "metadata-cache-ttl-secs" + type: "int" + usage: >- + The ttl value in seconds to be used for expiring items in metadata-cache. It + can be set to -1 for no-ttl, 0 for no cache and > 0 for ttl-controlled + metadata-cache. Any value set below -1 will throw an error. + default: "60" + optimizations: + machine-based-optimization: + - group: "high-performance" + value: -1 + profiles: + - name: "aiml-training" + value: -1 + - name: "aiml-serving" + value: -1 + - name: "aiml-checkpointing" + value: -1 + + - config-path: "metadata-cache.type-cache-max-size-mb" + flag-name: "type-cache-max-size-mb" + type: "int" + usage: "Max size of type-cache maps which are maintained at a per-directory level. This flag has been deprecated in favour of a single unified flag stat-cache-max-size-mb." + default: "4" + + - config-path: "metrics.buffer-size" + flag-name: "metrics-buffer-size" + type: "int" + usage: "The maximum number of histogram metric updates in the queue." + default: "256" + hide-flag: true + + - config-path: "metrics.cloud-metrics-export-interval-secs" + flag-name: "cloud-metrics-export-interval-secs" + type: "int" + usage: "Specifies the interval at which the metrics are uploaded to cloud monitoring" + default: 0 + + - config-path: "metrics.experimental-enable-grpc-metrics" + flag-name: "experimental-enable-grpc-metrics" + type: "bool" + usage: "Enables support for gRPC metrics" + default: true + hide-flag: true + + - config-path: "metrics.prometheus-port" + flag-name: "prometheus-port" + type: "int" + usage: "Expose Prometheus metrics endpoint on this port and a path of /metrics." + default: "0" + + - config-path: "metrics.stackdriver-export-interval" + flag-name: "stackdriver-export-interval" + type: "duration" + usage: >- + Export metrics to stackdriver with this interval. A value of 0 + indicates no exporting. + default: "0s" + deprecated: true + deprecation-warning: "Please use --cloud-metrics-export-interval-secs instead." + + - config-path: "metrics.use-new-names" + flag-name: "metrics-use-new-names" + type: "bool" + usage: "Use the new metric names." + default: false + hide-flag: true + + - config-path: "metrics.workers" + flag-name: "metrics-workers" + type: "int" + usage: "The number of workers that update histogram metrics concurrently." + default: "3" + hide-flag: true + + - config-path: "mrd.pool-size" + flag-name: "mrd-pool-size" + type: "int" + usage: >- + Specifies the MRD pool size to be used for zonal buckets. The value should be more than 0. + default: "4" + hide-flag: true + + - config-path: "only-dir" + flag-name: "only-dir" + type: "string" + usage: "Mount only a specific directory within the bucket. See docs/mounting for more information" + default: "" + + - config-path: "profile" + flag-name: "profile" + type: "string" + usage: "The name of the profile to apply. e.g. aiml-training, aiml-serving, aiml-checkpointing" + default: "" + + - config-path: "read.block-size-mb" + flag-name: "read-block-size-mb" + type: "int" + usage: >- + Specifies the block size for buffered reads. The value should be more than + 0. This is used to read data in chunks from GCS. + default: 16 + hide-flag: true + + - config-path: "read.enable-buffered-read" + flag-name: "enable-buffered-read" + type: "bool" + usage: >- + When enabled, read starts using buffer to prefetch (asynchronous and in parallel) + data from GCS. This improves performance for large file sequential reads. + Note: Enabling this flag can increase the memory usage significantly. + default: false + + - config-path: "read.global-max-blocks" + flag-name: "read-global-max-blocks" + type: "int" + usage: >- + Specifies the maximum number of blocks available for buffered reads across all file-handles. + The value should be >= 0 or -1 (for infinite blocks). + A value of 0 disables buffered reads. + default: 40 + + - config-path: "read.inactive-stream-timeout" + flag-name: "read-inactive-stream-timeout" + type: "duration" + usage: >- + Duration of inactivity after which an open GCS read stream is automatically closed. + This helps conserve resources when a file handle remains open without active Read calls. + A value of '0s' disables this timeout. + default: "10s" + hide-flag: true + + - config-path: "read.max-blocks-per-handle" + flag-name: "read-max-blocks-per-handle" + type: "int" + usage: >- + Specifies the maximum number of blocks to be used by a single file handle for + buffered reads. The value should be >= 0 or -1 (for infinite blocks). + A value of 0 disables buffered reads. + default: 20 + hide-flag: true + + - config-path: "read.min-blocks-per-handle" + flag-name: "read-min-blocks-per-handle" + type: "int" + usage: >- + Specifies the minimum number of blocks required by a file-handle to start + reading via buffered reads. The value should be >= 1 or "read-max-blocks-per-handle". + default: 4 + hide-flag: true + + - config-path: "read.random-seek-threshold" + flag-name: "read-random-seek-threshold" + type: "int" + usage: >- + Specifies the random seek threshold to switch to another reader when random reads are detected. + default: 3 + hide-flag: true + + - config-path: "read.start-blocks-per-handle" + flag-name: "read-start-blocks-per-handle" + type: "int" + usage: >- + Specifies the number of blocks to be prefetched on the first read. + default: 1 + hide-flag: true + + - config-path: "trace.exporters" + flag-name: "trace-exporters" + type: "[]string" + usage: "Specify comma separated value of the exporters where traces are exported to. Supported values: stdout(writes traces to stdout), gcpexporter(exports traces to google cloud trace)" + default: '"gcpexporter"' + hide-flag: true + + - config-path: "trace.project-id" + flag-name: "trace-project-id" + type: "string" + usage: "Specify the GCP project id to which traces will be exported. When unset, a project id will be inferred as per the default credential detection process" + default: "" + hide-flag: true + + - config-path: "trace.sampling-ratio" + flag-name: "trace-sampling-ratio" + type: "float64" + usage: "Specifies the fraction of traces to export, ranging from 0.0 to 1.0. Setting a value greater than 0 enables tracing; 1.0 exports all traces, while 0.0 (default) disables them. Use this to balance the number of traces exported with the tradeoff of higher perf and cost impact." + default: 0 + + - config-path: "workload-insight.forward-merge-threshold-mb" + flag-name: "workload-insight-forward-merge-threshold-mb" + type: "int" + usage: >- + The threshold in MB for merging forward sequential reads for workload + insights visualization.Reads within this threshold will be merged into + a single read operation. Applicable only when --visualize-workload-insight + is enabled. + default: 0 + hide-flag: true + + - config-path: "workload-insight.output-file" + flag-name: "workload-insight-output-file" + type: "string" + usage: >- + The file path where the workload insights will be written. + If not specified, insights will be written to stdout + default: "" + hide-flag: true + + - config-path: "workload-insight.visualize" + flag-name: "visualize-workload-insight" + type: "bool" + usage: >- + A flag to enable workload visualization. When enabled, workload insights + will include visualizations to help understand access patterns. Insights + will be written to the file specified by --workload-insight-output-file. + default: false + hide-flag: true + + - config-path: "write.block-size-mb" + flag-name: "write-block-size-mb" + type: "int" + usage: >- + Specifies the block size for streaming writes. The value should be more + than 0. + default: 32 + hide-flag: true + + - config-path: "write.create-empty-file" + flag-name: "create-empty-file" + type: "bool" + usage: "For a new file, it creates an empty file in Cloud Storage bucket as a hold." + default: false + hide-flag: true + + - config-path: "write.enable-rapid-appends" + flag-name: "enable-rapid-appends" + type: "bool" + usage: "Enables support for appends to unfinalized object using streaming writes" + default: true + + - config-path: "write.enable-rapid-writes" + flag-name: "enable-rapid-writes" + type: "bool" + usage: "For pirlo, toggles between using STANDARD class and RAPID class for writes." + default: false + + - config-path: "write.enable-streaming-writes" + flag-name: "enable-streaming-writes" + type: "bool" + usage: "Enables streaming uploads during write file operation." + default: true + + - config-path: "write.finalize-file-on-close" + flag-name: "finalize-file-on-close" + type: "bool" + usage: "Finalizes the files on close for Rapid storage. Appends will be slower on finalized files." + default: false + hide-flag: true + optimizations: + bucket-type-optimization: + - bucket-type: "zonal" + value: false + - bucket-type: "pirlo" + value: true + + - config-path: "write.global-max-blocks" + flag-name: "write-global-max-blocks" + type: "int" + usage: >- + Specifies the maximum number of blocks available for streaming writes across all files. + The value should be >= 0 or -1 (for infinite blocks). + A value of 0 disables streaming writes. + default: 4 + optimizations: + machine-based-optimization: + - group: "high-performance" + value: 1600 + + - config-path: "write.max-blocks-per-file" + flag-name: "write-max-blocks-per-file" + type: "int" + usage: >- + Specifies the maximum number of blocks to be used by a single file for + streaming writes. The value should be >= 1 or -1 (for infinite blocks). + default: 1 + hide-flag: true + + - flag-name: "debug_fs" + type: "bool" + usage: "This flag is unused." + default: false + deprecated: true + deprecation-warning: "This flag is currently unused." + + - flag-name: "debug_fuse_errors" + type: "bool" + usage: "This flag is currently unused." + default: "true" + deprecated: true + deprecation-warning: "This flag is currently unused." + + - flag-name: "debug_http" + type: "bool" + usage: "This flag is currently unused." + default: false + deprecated: true + deprecation-warning: "This flag is currently unused." + + - flag-name: "max-retry-duration" + type: "duration" + usage: "This is currently unused." + default: "0s" + deprecated: true + deprecation-warning: "This is currently unused." diff --git a/cfg/rationalize.go b/cfg/rationalize.go index 00b1765979..f23452e3a3 100644 --- a/cfg/rationalize.go +++ b/cfg/rationalize.go @@ -15,19 +15,16 @@ package cfg import ( + "log" "math" "net/url" + "slices" + "strings" - "github.com/googlecloudplatform/gcsfuse/v2/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/spf13/viper" ) -// isSet interface is abstraction over the IsSet() method of viper, specially -// added to keep rationalize method simple. IsSet will be used to resolve -// conflicting deprecated flags and new configs. -type isSet interface { - IsSet(string) bool -} - func decodeURL(u string) (string, error) { // TODO: check if we can replace url.Parse with url.ParseRequestURI. decodedURL, err := url.Parse(u) @@ -37,39 +34,67 @@ func decodeURL(u string) (string, error) { return decodedURL.String(), nil } -// resolveMetadataCacheTTL returns the ttl to be used for stat/type cache based -// on the user flags/configs. -func resolveMetadataCacheTTL(v isSet, c *MetadataCacheConfig) { - // If metadata-cache:ttl-secs has been set, then it overrides both - // stat-cache-ttl and type-cache-tll. +// resolveMetadataCacheConfig calculates the ttl to be used for stat/type cache based +// on the user flags/configs or machine type based optimizations. +func resolveMetadataCacheConfig(v *viper.Viper, c *MetadataCacheConfig, optimizedFlags []string) { + optimizationAppliedToNegativeCacheTTL := isFlagPresent(optimizedFlags, MetadataNegativeCacheTTLConfigKey) + + if v.IsSet(MetadataNegativeCacheTTLConfigKey) || optimizationAppliedToNegativeCacheTTL { + if c.NegativeTtlSecs == -1 { + c.NegativeTtlSecs = maxSupportedTTLInSeconds + } + } + // Order of precedence for setting TTL seconds + // 1. If metadata-cache:ttl-secs has been set, then it has highest precedence + // 2. If metadata-cache:stat-cache-ttl or metadata-cache:type-cache-ttl has been set or no optimization applied, then it has second highest precedence + // 3. Optimization is applied (implicit) and take care of special case of -1 which can occur even in defaults + optimizationAppliedToMetadataCacheTTL := isFlagPresent(optimizedFlags, MetadataCacheTTLConfigKey) if v.IsSet(MetadataCacheTTLConfigKey) { if c.TtlSecs == -1 { c.TtlSecs = maxSupportedTTLInSeconds } - return + } else if (v.IsSet(MetadataCacheStatCacheTTLConfigKey) || v.IsSet(MetadataCacheTypeCacheTTLConfigKey)) || (!optimizationAppliedToMetadataCacheTTL) { + c.TtlSecs = int64(math.Ceil(math.Min(c.DeprecatedStatCacheTtl.Seconds(), c.DeprecatedTypeCacheTtl.Seconds()))) + } else if c.TtlSecs == -1 { + c.TtlSecs = maxSupportedTTLInSeconds + } + + if c.MetadataPrefetchMaxWorkers == -1 { + c.MetadataPrefetchMaxWorkers = math.MaxInt64 + } + + if c.MetadataPrefetchEntriesLimit == -1 { + c.MetadataPrefetchEntriesLimit = math.MaxInt64 } - // Else, use deprecated stat/type cache ttl to resolve metadataCacheTTL. - c.TtlSecs = int64(math.Ceil(math.Min(c.DeprecatedStatCacheTtl.Seconds(), c.DeprecatedTypeCacheTtl.Seconds()))) } -// resolveStatCacheMaxSizeMB returns the stat-cache size in MiBs based on the -// user old and new flags/configs. -func resolveStatCacheMaxSizeMB(v isSet, c *MetadataCacheConfig) { - // If metadata-cache:stat-cache-size-mb has been set, then it overrides - // stat-cache-capacity. +// resolveStatCacheMaxSizeMB calculates the stat-cache size in MiBs based on the +// machine-type default override, user's old and new flags/configs. +func resolveStatCacheMaxSizeMB(v *viper.Viper, c *MetadataCacheConfig, optimizedFlags []string) { + // Local function to calculate size based on deprecated capacity. + calculateSizeFromCapacity := func(capacity int64) int64 { + avgTotalStatCacheEntrySize := AverageSizeOfPositiveStatCacheEntry + AverageSizeOfNegativeStatCacheEntry + return int64(util.BytesToHigherMiBs(uint64(capacity) * avgTotalStatCacheEntrySize)) + } + + // Order of precedence for setting stat cache size + // 1. If metadata-cache:stat-cache-size-mb is set it has the highest precedence + // 2. If stat-cache-capacity is set or optimization is not applied then use it to calculate stat cache size + // 3. Else handle special case of -1 for both optimized or possible default value + optimizationAppliedToStatCacheMaxSize := isFlagPresent(optimizedFlags, StatCacheMaxSizeConfigKey) if v.IsSet(StatCacheMaxSizeConfigKey) { if c.StatCacheMaxSizeMb == -1 { c.StatCacheMaxSizeMb = int64(maxSupportedStatCacheMaxSizeMB) } - return + } else if v.IsSet(MetadataCacheStatCacheCapacityConfigKey) || (!optimizationAppliedToStatCacheMaxSize) { + c.StatCacheMaxSizeMb = calculateSizeFromCapacity(c.DeprecatedStatCacheCapacity) + } else if c.StatCacheMaxSizeMb == -1 { + c.StatCacheMaxSizeMb = int64(maxSupportedStatCacheMaxSizeMB) } - // Else, use deprecated stat-cache-capacity to resolve StatCacheMaxSizeMb. - avgTotalStatCacheEntrySize := AverageSizeOfPositiveStatCacheEntry + AverageSizeOfNegativeStatCacheEntry - c.StatCacheMaxSizeMb = int64(util.BytesToHigherMiBs(uint64(c.DeprecatedStatCacheCapacity) * avgTotalStatCacheEntrySize)) } func resolveStreamingWriteConfig(w *WriteConfig) { - if w.ExperimentalEnableStreamingWrites { + if w.EnableStreamingWrites { w.CreateEmptyFile = false } @@ -78,11 +103,9 @@ func resolveStreamingWriteConfig(w *WriteConfig) { } if w.MaxBlocksPerFile == -1 { - w.MaxBlocksPerFile = math.MaxInt64 - } - - if w.GlobalMaxBlocks < w.MaxBlocksPerFile { - w.MaxBlocksPerFile = w.GlobalMaxBlocks + // Setting a reasonable value here because if enough heap space is not + // available, make channel results in panic. + w.MaxBlocksPerFile = math.MaxInt16 } } @@ -92,8 +115,59 @@ func resolveCloudMetricsUploadIntervalSecs(m *MetricsConfig) { } } +func resolveParallelDownloadsValue(v *viper.Viper, fc *FileCacheConfig, c *Config) { + // Parallel downloads should be default ON when file cache is enabled, in case + // it is explicitly set by the user, use that value. + if IsFileCacheEnabled(c) && !v.IsSet(FileCacheParallelDownloadsConfigKey) { + fc.EnableParallelDownloads = true + } +} + +func resolveFileCacheAndBufferedReadConflict(v *viper.Viper, c *Config) { + if IsFileCacheEnabled(c) && c.Read.EnableBufferedRead { + // Log a warning only if the user has explicitly enabled buffered-read. + // The default value for enable-buffered-read is true, so we don't want to + // log a warning for the default case. + if v.IsSet("read.enable-buffered-read") { + log.Printf("Warning: File Cache and Buffered Read features are mutually exclusive. Disabling Buffered Read in favor of File Cache.") + } + c.Read.EnableBufferedRead = false + } +} + +func resolveReadConfig(r *ReadConfig) { + if r.GlobalMaxBlocks == -1 { + r.GlobalMaxBlocks = math.MaxInt32 + } +} + +func resolveLoggingConfig(config *Config) { + if config.Debug.Fuse || config.Debug.Gcs || config.Debug.LogMutex { + config.Logging.Severity = "TRACE" + } + + configLogFormat := config.Logging.Format // capture initial value for error reporting + config.Logging.Format = strings.ToLower(config.Logging.Format) + if !slices.Contains([]string{logFormatText, logFormatJSON}, config.Logging.Format) { + log.Printf("Unsupported log format provided: %s. Defaulting to %s log format.", configLogFormat, defaultLogFormat) + config.Logging.Format = defaultLogFormat // defaulting to json format + } +} + +func resolveTraceConfig(t *TraceConfig) { + for i, s := range t.Exporters { + t.Exporters[i] = strings.ToLower(strings.TrimSpace(s)) + } +} + +func resolveGCSRetriesConfig(c *GcsRetriesConfig) { + if c.MaxRetryAttempts == 0 { + c.MaxRetryAttempts = math.MaxInt + } +} + // Rationalize updates the config fields based on the values of other fields. -func Rationalize(v isSet, c *Config) error { +func Rationalize(v *viper.Viper, c *Config, optimizedFlags []string) error { var err error if c.GcsConnection.CustomEndpoint, err = decodeURL(c.GcsConnection.CustomEndpoint); err != nil { return err @@ -103,14 +177,16 @@ func Rationalize(v isSet, c *Config) error { return err } - if c.Debug.Fuse || c.Debug.Gcs || c.Debug.LogMutex { - c.Logging.Severity = "TRACE" - } - + resolveLoggingConfig(c) + resolveTraceConfig(&c.Trace) + resolveReadConfig(&c.Read) resolveStreamingWriteConfig(&c.Write) - resolveMetadataCacheTTL(v, &c.MetadataCache) - resolveStatCacheMaxSizeMB(v, &c.MetadataCache) + resolveMetadataCacheConfig(v, &c.MetadataCache, optimizedFlags) + resolveStatCacheMaxSizeMB(v, &c.MetadataCache, optimizedFlags) resolveCloudMetricsUploadIntervalSecs(&c.Metrics) + resolveParallelDownloadsValue(v, &c.FileCache, c) + resolveFileCacheAndBufferedReadConflict(v, c) + resolveGCSRetriesConfig(&c.GcsRetries) return nil } diff --git a/cfg/rationalize_test.go b/cfg/rationalize_test.go index 4b6b6dd686..5aa92fc226 100644 --- a/cfg/rationalize_test.go +++ b/cfg/rationalize_test.go @@ -15,19 +15,19 @@ package cfg import ( + "bytes" + "log" "math" + "os" + "strings" "testing" "time" + "github.com/spf13/viper" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -type mockIsSet struct{} - -func (*mockIsSet) IsSet(flag string) bool { - return false -} - func TestRationalizeCustomEndpointSuccessful(t *testing.T) { testCases := []struct { name string @@ -56,11 +56,82 @@ func TestRationalizeCustomEndpointSuccessful(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - actualErr := Rationalize(&mockIsSet{}, tc.config) + actualErr := Rationalize(viper.New(), tc.config, []string{}) - if assert.NoError(t, actualErr) { - assert.Equal(t, tc.expectedCustomEndpoint, tc.config.GcsConnection.CustomEndpoint) - } + require.NoError(t, actualErr) + assert.Equal(t, tc.expectedCustomEndpoint, tc.config.GcsConnection.CustomEndpoint) + }) + } +} + +func TestRationalize_GcsRetriesConfig(t *testing.T) { + testCases := []struct { + name string + config *Config + expectedMaxRetryAttempts int + }{ + { + name: "max-retry-attempts is 0", + config: &Config{ + GcsRetries: GcsRetriesConfig{ + MaxRetryAttempts: 0, + }, + }, + expectedMaxRetryAttempts: math.MaxInt, + }, + { + name: "max-retry-attempts is not 0", + config: &Config{ + GcsRetries: GcsRetriesConfig{ + MaxRetryAttempts: 10, + }, + }, + expectedMaxRetryAttempts: 10, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := Rationalize(viper.New(), tc.config, []string{}) + + require.NoError(t, err) + assert.Equal(t, tc.expectedMaxRetryAttempts, int(tc.config.GcsRetries.MaxRetryAttempts)) + }) + } +} + +func TestRationalize_ReadConfig(t *testing.T) { + testCases := []struct { + name string + config *Config + expectedGlobalMaxBlocks int64 + }{ + { + name: "global-max-blocks is -1", + config: &Config{ + Read: ReadConfig{ + GlobalMaxBlocks: -1, + }, + }, + expectedGlobalMaxBlocks: math.MaxInt32, + }, + { + name: "global-max-blocks is not -1", + config: &Config{ + Read: ReadConfig{ + GlobalMaxBlocks: 100, + }, + }, + expectedGlobalMaxBlocks: 100, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := Rationalize(viper.New(), tc.config, []string{}) + + require.NoError(t, err) + assert.Equal(t, tc.expectedGlobalMaxBlocks, tc.config.Read.GlobalMaxBlocks) }) } } @@ -82,7 +153,7 @@ func TestRationalizeCustomEndpointUnsuccessful(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - assert.Error(t, Rationalize(&mockIsSet{}, tc.config)) + assert.Error(t, Rationalize(viper.New(), tc.config, []string{})) }) } } @@ -150,11 +221,10 @@ func TestLoggingSeverityRationalization(t *testing.T) { }, } - err := Rationalize(&mockIsSet{}, &c) + err := Rationalize(viper.New(), &c, []string{}) - if assert.NoError(t, err) { - assert.Equal(t, tc.expected, c.Logging.Severity) - } + require.NoError(t, err) + assert.Equal(t, tc.expected, c.Logging.Severity) } } @@ -186,11 +256,10 @@ func TestRationalize_TokenURLSuccessful(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - actualErr := Rationalize(&mockIsSet{}, tc.config) + actualErr := Rationalize(viper.New(), tc.config, []string{}) - if assert.NoError(t, actualErr) { - assert.Equal(t, tc.expectedTokenURL, tc.config.GcsAuth.TokenUrl) - } + require.NoError(t, actualErr) + assert.Equal(t, tc.expectedTokenURL, tc.config.GcsAuth.TokenUrl) }) } } @@ -212,35 +281,29 @@ func TestRationalize_TokenURLUnsuccessful(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - assert.Error(t, Rationalize(&mockIsSet{}, tc.config)) + assert.Error(t, Rationalize(viper.New(), tc.config, []string{})) }) } } -// Implement the isSet interface -type flagSet map[string]bool - -func (f flagSet) IsSet(key string) bool { - return f[key] -} - func TestRationalizeMetadataCache(t *testing.T) { testCases := []struct { - name string - flags flagSet - config *Config - expectedTTLSecs int64 - expectedStatCacheSize int64 + name string + userSetFlags map[string]any + config *Config + expectedTTLSecs int64 + expectedNegativeTTLSecs int64 + expectedStatCacheSize int64 }{ { name: "new_ttl_flag_set", - flags: flagSet{"metadata-cache.ttl-secs": true}, + userSetFlags: map[string]any{"metadata-cache.ttl-secs": 30}, config: &Config{MetadataCache: MetadataCacheConfig{TtlSecs: 30}}, expectedTTLSecs: 30, }, { - name: "old_ttl_flags_set", - flags: flagSet{"metadata-cache.deprecated-stat-cache-ttl": true, "metadata-cache.deprecated-type-cache-ttl": true}, + name: "old_ttl_flags_set", + userSetFlags: map[string]any{"metadata-cache.deprecated-stat-cache-ttl": 10 * time.Second, "metadata-cache.deprecated-type-cache-ttl": 5 * time.Second}, config: &Config{ MetadataCache: MetadataCacheConfig{ DeprecatedStatCacheTtl: 10 * time.Second, @@ -251,30 +314,30 @@ func TestRationalizeMetadataCache(t *testing.T) { }, { name: "new_stat-cache-size-mb_flag_set", - flags: flagSet{"metadata-cache.stat-cache-max-size-mb": true}, + userSetFlags: map[string]any{"metadata-cache.stat-cache-max-size-mb": 0}, config: &Config{MetadataCache: MetadataCacheConfig{StatCacheMaxSizeMb: 0}}, expectedTTLSecs: 0, // Assuming no change to TtlSecs in this function expectedStatCacheSize: 0, // Should remain unchanged }, { name: "old_stat-cache-capacity_flag_set", - flags: flagSet{"metadata-cache.deprecated-stat-cache-capacity": true}, + userSetFlags: map[string]any{"metadata-cache.deprecated-stat-cache-capacity": 1000}, config: &Config{MetadataCache: MetadataCacheConfig{DeprecatedStatCacheCapacity: 1000}}, expectedTTLSecs: 0, expectedStatCacheSize: 2, }, { name: "no_relevant_flags_set", - flags: flagSet{}, + userSetFlags: map[string]any{}, config: &Config{MetadataCache: MetadataCacheConfig{DeprecatedStatCacheCapacity: 50}}, expectedTTLSecs: 0, expectedStatCacheSize: 1, }, { name: "both_new_and_old_flags_set", - flags: flagSet{ - "metadata-cache.stat-cache-max-size-mb": true, - "stat-cache-capacity": true, + userSetFlags: map[string]any{ + "metadata-cache.stat-cache-max-size-mb": 100, + "stat-cache-capacity": 50, }, config: &Config{ MetadataCache: MetadataCacheConfig{ @@ -286,25 +349,131 @@ func TestRationalizeMetadataCache(t *testing.T) { expectedStatCacheSize: 100, }, { - name: "ttl_and_stat_cache_size_set_to_-1", - flags: flagSet{"metadata-cache.ttl-secs": true, "metadata-cache.stat-cache-max-size-mb": true}, + name: "ttl_and_stat_cache_size_set_to_-1", + userSetFlags: map[string]any{"metadata-cache.ttl-secs": -1, "metadata-cache.stat-cache-max-size-mb": -1}, + config: &Config{ + MetadataCache: MetadataCacheConfig{ + TtlSecs: -1, + NegativeTtlSecs: -1, + StatCacheMaxSizeMb: -1, + }, + }, + expectedTTLSecs: math.MaxInt64 / int64(time.Second), // Max supported ttl in seconds. + expectedNegativeTTLSecs: math.MaxInt64 / int64(time.Second), // Max supported ttl in seconds. + expectedStatCacheSize: math.MaxUint64 >> 20, // Max supported cache size in MiB. + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + v := viper.New() + for key, val := range tc.userSetFlags { + v.Set(key, val) + } + + err := Rationalize(v, tc.config, []string{}) + + require.NoError(t, err) + assert.Equal(t, tc.expectedTTLSecs, tc.config.MetadataCache.TtlSecs) + assert.Equal(t, tc.expectedStatCacheSize, tc.config.MetadataCache.StatCacheMaxSizeMb) + }) + } +} + +func TestRationalizeMetadataCacheWithOptimization(t *testing.T) { + testCases := []struct { + name string + userSetFlags map[string]any + config *Config + expectedTTLSecs int64 + expectedNegativeTTLSecs int64 + expectedStatCacheSize int64 + }{ + { + name: "negative_ttl_flag_set", + userSetFlags: map[string]any{"metadata-cache.negative-ttl-secs": 44}, + config: &Config{MetadataCache: MetadataCacheConfig{NegativeTtlSecs: 44}}, + expectedNegativeTTLSecs: 44, + }, + { + name: "new_ttl_flag_set", + userSetFlags: map[string]any{"metadata-cache.ttl-secs": 30}, + config: &Config{MetadataCache: MetadataCacheConfig{TtlSecs: 30}}, + expectedTTLSecs: 30, + }, + { + name: "old_ttl_flags_set", + userSetFlags: map[string]any{"metadata-cache.deprecated-stat-cache-ttl": 10 * time.Second, "metadata-cache.deprecated-type-cache-ttl": 5 * time.Second}, + config: &Config{ + MetadataCache: MetadataCacheConfig{ + DeprecatedStatCacheTtl: 10 * time.Second, + DeprecatedTypeCacheTtl: 5 * time.Second, + }, + }, + expectedTTLSecs: 5, + }, + { + name: "new_and_old_ttl_flags_set", + userSetFlags: map[string]any{"metadata-cache.ttl-secs": 30, "metadata-cache.deprecated-stat-cache-ttl": 10 * time.Second, "metadata-cache.deprecated-type-cache-ttl": 5 * time.Second}, + config: &Config{ + MetadataCache: MetadataCacheConfig{ + TtlSecs: 30, + DeprecatedStatCacheTtl: 10 * time.Second, + DeprecatedTypeCacheTtl: 5 * time.Second, + }, + }, + expectedTTLSecs: 30, + }, + { + name: "new_stat-cache-size-mb_flag_set", + userSetFlags: map[string]any{"metadata-cache.stat-cache-max-size-mb": 100}, + config: &Config{MetadataCache: MetadataCacheConfig{StatCacheMaxSizeMb: 100}}, + expectedTTLSecs: 0, // Assuming no change to TtlSecs in this function + expectedStatCacheSize: 100, + }, + { + name: "old_stat-cache-capacity_flag_set", + userSetFlags: map[string]any{"metadata-cache.deprecated-stat-cache-capacity": 1000}, + config: &Config{MetadataCache: MetadataCacheConfig{DeprecatedStatCacheCapacity: 1000}}, + expectedTTLSecs: 0, + expectedStatCacheSize: 2, + }, + { + name: "new_and_old_stat-cache-capacity_flag_set", + userSetFlags: map[string]any{"metadata-cache.stat-cache-max-size-mb": 100, "metadata-cache.deprecated-stat-cache-capacity": 1000}, + config: &Config{MetadataCache: MetadataCacheConfig{StatCacheMaxSizeMb: 100, DeprecatedStatCacheCapacity: 1000}}, + expectedTTLSecs: 0, + expectedStatCacheSize: 100, + }, + { + name: "ttl_and_stat_cache_size_set_to_-1", + userSetFlags: map[string]any{"metadata-cache.ttl-secs": -1, "metadata-cache.stat-cache-max-size-mb": -1}, config: &Config{ MetadataCache: MetadataCacheConfig{ TtlSecs: -1, + NegativeTtlSecs: -1, StatCacheMaxSizeMb: -1, }, }, - expectedTTLSecs: math.MaxInt64 / int64(time.Second), // Max supported ttl in seconds. - expectedStatCacheSize: math.MaxUint64 >> 20, // Max supported cache size in MiB. + expectedTTLSecs: math.MaxInt64 / int64(time.Second), // Max supported ttl in seconds. + expectedNegativeTTLSecs: math.MaxInt64 / int64(time.Second), // Max supported ttl in seconds. + expectedStatCacheSize: math.MaxUint64 >> 20, // Max supported cache size in MiB. }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - if assert.NoError(t, Rationalize(tc.flags, tc.config)) { - assert.Equal(t, tc.expectedTTLSecs, tc.config.MetadataCache.TtlSecs) - assert.Equal(t, tc.expectedStatCacheSize, tc.config.MetadataCache.StatCacheMaxSizeMb) + v := viper.New() + for key, val := range tc.userSetFlags { + v.Set(key, val) } + + err := Rationalize(v, tc.config, []string{"metadata-cache.negative-ttl-secs", "metadata-cache.ttl-secs", "metadata-cache.stat-cache-max-size-mb", "metadata-cache.deprecated-stat-cache-capacity", "metadata-cache.deprecated-stat-cache-ttl", "metadata-cache.deprecated-type-cache-ttl"}) + + require.NoError(t, err) + assert.Equal(t, tc.expectedTTLSecs, tc.config.MetadataCache.TtlSecs) + assert.Equal(t, tc.expectedNegativeTTLSecs, tc.config.MetadataCache.NegativeTtlSecs) + assert.Equal(t, tc.expectedStatCacheSize, tc.config.MetadataCache.StatCacheMaxSizeMb) }) } } @@ -315,59 +484,63 @@ func TestRationalize_WriteConfig(t *testing.T) { config *Config expectedCreateEmptyFile bool expectedMaxBlocksPerFile int64 + expectedBlockSizeMB int64 }{ { name: "valid_config_streaming_writes_enabled", config: &Config{ Write: WriteConfig{ - BlockSizeMb: 10, - CreateEmptyFile: true, - ExperimentalEnableStreamingWrites: true, - GlobalMaxBlocks: -1, - MaxBlocksPerFile: -1, + BlockSizeMb: 10, + CreateEmptyFile: true, + EnableStreamingWrites: true, + GlobalMaxBlocks: -1, + MaxBlocksPerFile: -1, }, }, expectedCreateEmptyFile: false, - expectedMaxBlocksPerFile: math.MaxInt64, + expectedMaxBlocksPerFile: math.MaxInt16, + expectedBlockSizeMB: 10, }, { name: "valid_config_global_max_blocks_less_than_blocks_per_file", config: &Config{ Write: WriteConfig{ - BlockSizeMb: 10, - CreateEmptyFile: true, - ExperimentalEnableStreamingWrites: true, - GlobalMaxBlocks: 10, - MaxBlocksPerFile: 20, + BlockSizeMb: 5, + CreateEmptyFile: true, + EnableStreamingWrites: true, + GlobalMaxBlocks: 10, + MaxBlocksPerFile: 20, }, }, expectedCreateEmptyFile: false, - expectedMaxBlocksPerFile: 10, + expectedMaxBlocksPerFile: 20, + expectedBlockSizeMB: 5, }, { name: "valid_config_global_max_blocks_more_than_blocks_per_file", config: &Config{ Write: WriteConfig{ - BlockSizeMb: 10, - CreateEmptyFile: true, - ExperimentalEnableStreamingWrites: true, - GlobalMaxBlocks: 20, - MaxBlocksPerFile: 10, + BlockSizeMb: 64, + CreateEmptyFile: true, + EnableStreamingWrites: true, + GlobalMaxBlocks: 20, + MaxBlocksPerFile: 10, }, }, expectedCreateEmptyFile: false, expectedMaxBlocksPerFile: 10, + expectedBlockSizeMB: 64, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - actualErr := Rationalize(&mockIsSet{}, tc.config) + actualErr := Rationalize(viper.New(), tc.config, []string{}) - if assert.NoError(t, actualErr) { - assert.Equal(t, tc.expectedCreateEmptyFile, tc.config.Write.CreateEmptyFile) - assert.Equal(t, tc.expectedMaxBlocksPerFile, tc.config.Write.MaxBlocksPerFile) - } + require.NoError(t, actualErr) + assert.Equal(t, tc.expectedCreateEmptyFile, tc.config.Write.CreateEmptyFile) + assert.Equal(t, tc.expectedMaxBlocksPerFile, tc.config.Write.MaxBlocksPerFile) + assert.Equal(t, tc.expectedBlockSizeMB, tc.config.Write.BlockSizeMb) }) } } @@ -412,13 +585,319 @@ func TestRationalizeMetricsConfig(t *testing.T) { } for _, tc := range testCases { - tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() - if assert.NoError(t, Rationalize(&mockIsSet{}, tc.config)) { - assert.Equal(t, tc.expected, tc.config.Metrics.CloudMetricsExportIntervalSecs) + err := Rationalize(viper.New(), tc.config, []string{}) + + require.NoError(t, err) + assert.Equal(t, tc.expected, tc.config.Metrics.CloudMetricsExportIntervalSecs) + }) + } +} + +func TestRationalizeTraceConfig(t *testing.T) { + t.Parallel() + testCases := []struct { + name string + config *Config + expected []string + }{ + { + name: "all_params_filled", + config: &Config{ + Trace: TraceConfig{ + Exporters: []string{"stdout"}, + ProjectId: "test-gcp-project", + SamplingRatio: 0.2, + }, + }, + expected: []string{"stdout"}, + }, + { + name: "missing_project_id", + config: &Config{ + Trace: TraceConfig{ + Exporters: []string{"stdout"}, + ProjectId: "test-gcp-project", + SamplingRatio: 0.2, + }, + }, + expected: []string{"stdout"}, + }, + { + name: "multiple_tracing_modes", + config: &Config{ + Trace: TraceConfig{ + Exporters: []string{"stdout ", " gcpexporter "}, + ProjectId: "test-gcp-project", + SamplingRatio: 0.2, + }, + }, + expected: []string{"stdout", "gcpexporter"}, + }, + { + name: "multiple_tracing_modes", + config: &Config{ + Trace: TraceConfig{ + Exporters: []string{"STDout ", " GcPExpoRter "}, + ProjectId: "test-gcp-project", + SamplingRatio: 0.2, + }, + }, + expected: []string{"stdout", "gcpexporter"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + err := Rationalize(viper.New(), tc.config, []string{}) + + require.NoError(t, err) + assert.Equal(t, tc.expected, tc.config.Trace.Exporters) + }) + } +} + +func TestRationalize_ParallelDownloadsConfig(t *testing.T) { + testCases := []struct { + name string + userSetFlags map[string]any + config *Config + expectedParallelDownloads bool + }{ + { + name: "valid_config_file_cache_enabled", + config: &Config{ + CacheDir: ResolvedPath("/some-path"), + FileCache: FileCacheConfig{ + MaxSizeMb: 500, + }, + }, + expectedParallelDownloads: true, + }, + { + name: "valid_config_file_cache_disabled", + config: &Config{}, + expectedParallelDownloads: false, + }, + { + name: "valid_config_cache_dir_not_set_and_max_size_mb_set", + config: &Config{ + FileCache: FileCacheConfig{ + MaxSizeMb: 500, + }, + }, + expectedParallelDownloads: false, + }, + { + name: "valid_config_parallel_download_explicit_false", + // flagset here is representing viper config, value true is not actual value of the flag + // it just means flag is SET by the user. + userSetFlags: map[string]any{"file-cache.enable-parallel-downloads": true}, + config: &Config{ + CacheDir: ResolvedPath("/some-path"), + FileCache: FileCacheConfig{ + MaxSizeMb: 500, + }, + }, + expectedParallelDownloads: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + v := viper.New() + for key, val := range tc.userSetFlags { + v.Set(key, val) + } + + err := Rationalize(v, tc.config, []string{}) + + require.NoError(t, err) + assert.Equal(t, tc.expectedParallelDownloads, tc.config.FileCache.EnableParallelDownloads) + }) + } +} + +func TestRationalize_FileCacheAndBufferedReadConflict(t *testing.T) { + testCases := []struct { + name string + userSetFlags map[string]any + config *Config + expectedEnableBufferedRead bool + expectWarning bool + }{ + { + name: "file cache and buffered read enabled (user set)", + userSetFlags: map[string]any{"read.enable-buffered-read": true}, + config: &Config{ + CacheDir: "/some/path", + FileCache: FileCacheConfig{ + MaxSizeMb: -1, + }, + Read: ReadConfig{ + EnableBufferedRead: true, + }, + }, + expectedEnableBufferedRead: false, + expectWarning: true, + }, + { + name: "file cache enabled, buffered read enabled (default)", + userSetFlags: map[string]any{}, + config: &Config{ + CacheDir: "/some/path", + FileCache: FileCacheConfig{ + MaxSizeMb: -1, + }, + Read: ReadConfig{ + EnableBufferedRead: true, + }, + }, + expectedEnableBufferedRead: false, + expectWarning: false, + }, + { + name: "file cache disabled, buffered read enabled", + userSetFlags: map[string]any{"read.enable-buffered-read": true}, + config: &Config{ + Read: ReadConfig{ + EnableBufferedRead: true, + }, + }, + expectedEnableBufferedRead: true, + expectWarning: false, + }, + { + name: "both disabled", + userSetFlags: map[string]any{}, + config: &Config{}, + expectedEnableBufferedRead: false, + expectWarning: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Capture log output. + var buf bytes.Buffer + log.SetOutput(&buf) + // Restore original logger output after test. + defer log.SetOutput(os.Stderr) + + v := viper.New() + for key, val := range tc.userSetFlags { + v.Set(key, val) + } + + err := Rationalize(v, tc.config, []string{}) + + require.NoError(t, err) + assert.Equal(t, tc.expectedEnableBufferedRead, tc.config.Read.EnableBufferedRead) + logOutput := buf.String() + if tc.expectWarning { + assert.True(t, strings.Contains(logOutput, "Warning: File Cache and Buffered Read features are mutually exclusive. Disabling Buffered Read in favor of File Cache.")) + } else { + assert.False(t, strings.Contains(logOutput, "Warning: File Cache and Buffered Read features are mutually exclusive. Disabling Buffered Read in favor of File Cache.")) } }) } } + +func TestResolveLoggingConfig(t *testing.T) { + testCases := []struct { + name string + config *Config + expectedLogFormat string + }{ + { + name: "valid_log_format_json", + config: &Config{ + Logging: LoggingConfig{ + Format: "json", + }, + }, + expectedLogFormat: "json", + }, + { + name: "valid_log_format_text", + config: &Config{ + Logging: LoggingConfig{ + Format: "text", + }, + }, + expectedLogFormat: "text", + }, + { + name: "valid_case_insensitive_log_format", + config: &Config{ + Logging: LoggingConfig{ + Format: "TEXT", + }, + }, + expectedLogFormat: "text", + }, + { + name: "invalid_log_format", + config: &Config{ + Logging: LoggingConfig{ + Format: "INVALID", + }, + }, + expectedLogFormat: "json", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + resolveLoggingConfig(tc.config) + + assert.Equal(t, tc.expectedLogFormat, tc.config.Logging.Format) + }) + } +} + +func TestRationalize_MetadataCacheConfig(t *testing.T) { + testCases := []struct { + name string + config *Config + expectedMetadataPrefetchCount int64 + expectedConcurrentMetadataPrefetches int64 + }{ + { + name: "valid_config_metadata_prefetch_count_set_to_-1", + config: &Config{ + MetadataCache: MetadataCacheConfig{ + MetadataPrefetchEntriesLimit: -1, + MetadataPrefetchMaxWorkers: 5, + }, + }, + expectedMetadataPrefetchCount: math.MaxInt64, + expectedConcurrentMetadataPrefetches: 5, + }, + { + name: "valid_config_concurrent_prefetches_set_to_-1", + config: &Config{ + MetadataCache: MetadataCacheConfig{ + MetadataPrefetchEntriesLimit: 8, + MetadataPrefetchMaxWorkers: -1, + }, + }, + expectedMetadataPrefetchCount: 8, + expectedConcurrentMetadataPrefetches: math.MaxInt64, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualErr := Rationalize(viper.New(), tc.config, []string{}) + + require.NoError(t, actualErr) + assert.Equal(t, tc.expectedConcurrentMetadataPrefetches, tc.config.MetadataCache.MetadataPrefetchMaxWorkers) + assert.Equal(t, tc.expectedMetadataPrefetchCount, tc.config.MetadataCache.MetadataPrefetchEntriesLimit) + }) + } +} diff --git a/cfg/shared/types.go b/cfg/shared/types.go new file mode 100644 index 0000000000..49313df28f --- /dev/null +++ b/cfg/shared/types.go @@ -0,0 +1,40 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shared + +// ProfileOptimization holds the rules for a single performance profile. +type ProfileOptimization struct { + Name string `yaml:"name"` + Value any `yaml:"value"` +} + +// MachineBasedOptimization defines a machine-group-based optimization. +type MachineBasedOptimization struct { + Group string `yaml:"group"` + Value any `yaml:"value"` +} + +// BucketTypeOptimization defines a bucket-type-based optimization. +type BucketTypeOptimization struct { + BucketType string `yaml:"bucket-type"` + Value any `yaml:"value"` +} + +// OptimizationRules holds all defined optimizations for a single flag. +type OptimizationRules struct { + MachineBasedOptimization []MachineBasedOptimization `yaml:"machine-based-optimization"` + BucketTypeOptimization []BucketTypeOptimization `yaml:"bucket-type-optimization"` + Profiles []ProfileOptimization `yaml:"profiles"` +} diff --git a/cfg/types.go b/cfg/types.go index 918e476496..d9b334be91 100644 --- a/cfg/types.go +++ b/cfg/types.go @@ -20,7 +20,7 @@ import ( "strconv" "strings" - "github.com/googlecloudplatform/gcsfuse/v2/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" ) // Octal is the datatype for params such as file-mode and dir-mode which accept a base-8 value. @@ -59,21 +59,71 @@ func (p *Protocol) UnmarshalText(text []byte) error { return nil } +// DirectPathStrategy specifies how to handle DirectPath connectivity failures. +type DirectPathStrategy string + +const ( + // DirectPathOnly fails if DirectPath is unavailable (no fallback). + DirectPathOnly DirectPathStrategy = "direct-path-only" + // DirectPathWithFallback falls back to HTTP/1 if DirectPath is unavailable. + DirectPathWithFallback DirectPathStrategy = "direct-path-with-fallback" +) + +func (d *DirectPathStrategy) UnmarshalText(text []byte) error { + strategy := DirectPathStrategy(strings.ToLower(string(text))) + switch strategy { + case DirectPathOnly, DirectPathWithFallback: + *d = strategy + return nil + default: + validValues := []string{string(DirectPathOnly), string(DirectPathWithFallback)} + return fmt.Errorf("invalid direct-path strategy value: %s. It can only accept values in the list: %v", string(text), validValues) + } +} + // LogSeverity represents the logging severity and can accept the following values // "TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "OFF" type LogSeverity string +// Constants for all supported log severities. +const ( + TraceLogSeverity LogSeverity = "TRACE" + DebugLogSeverity LogSeverity = "DEBUG" + InfoLogSeverity LogSeverity = "INFO" + WarningLogSeverity LogSeverity = "WARNING" + ErrorLogSeverity LogSeverity = "ERROR" + OffLogSeverity LogSeverity = "OFF" +) + +// severityRanking maps each level to an integer for validation and comparison. +var severityRanking = map[LogSeverity]int{ + TraceLogSeverity: 0, + DebugLogSeverity: 1, + InfoLogSeverity: 2, + WarningLogSeverity: 3, + ErrorLogSeverity: 4, + OffLogSeverity: 5, +} + func (l *LogSeverity) UnmarshalText(text []byte) error { - textStr := string(text) - level := strings.ToUpper(textStr) - v := []string{"TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "OFF"} - if !slices.Contains(v, level) { - return fmt.Errorf("invalid logseverity value: %s. It can only assume values in the list: %v", textStr, v) + level := LogSeverity(strings.ToUpper(string(text))) + if _, ok := severityRanking[level]; !ok { + return fmt.Errorf("invalid log severity level: %s. Must be one of [TRACE, DEBUG, INFO, WARNING, ERROR, OFF]", text) } - *l = LogSeverity(level) + *l = level return nil } +// Rank returns the integer representation of the severity rank. +// Returns -1 if the severity is unknown. +func (l LogSeverity) Rank() int { + if rank, ok := severityRanking[l]; ok { + return rank + } + // This case should ideally not be reached as LogSeverity configs are validated before mounting. + return -1 +} + // ResolvedPath represents a file-path which is an absolute path and is resolved // based on the value of GCSFUSE_PARENT_PROCESS_DIR env var. type ResolvedPath string @@ -86,3 +136,34 @@ func (p *ResolvedPath) UnmarshalText(text []byte) error { *p = ResolvedPath(path) return nil } + +// OptimizationInput provides runtime context for applying optimizations. +// This struct can be extended in the future to support additional optimization +// dimensions such as region, storage class, or project-specific settings. +type OptimizationInput struct { + // BucketType specifies the GCS bucket type. + // An empty string means no bucket-type-based optimization should be applied. + BucketType BucketType +} + +// BucketType represents the type of GCS bucket. +type BucketType string + +const ( + // BucketTypeZonal represents a zonal bucket with single-zone storage. + BucketTypeZonal BucketType = "zonal" + + // BucketTypePirlo represents a pirlo bucket. + BucketTypePirlo BucketType = "pirlo" + + // BucketTypeHierarchical represents a bucket with hierarchical namespace enabled. + BucketTypeHierarchical BucketType = "hierarchical" + + // BucketTypeFlat represents a flat (regional or multi-regional) bucket. + BucketTypeFlat BucketType = "flat" +) + +// IsValid returns true if the BucketType is one of the defined valid types. +func (bt BucketType) IsValid() bool { + return bt == BucketTypeZonal || bt == BucketTypePirlo || bt == BucketTypeHierarchical || bt == BucketTypeFlat +} diff --git a/cfg/validate.go b/cfg/validate.go index 360ef264e6..bf17496946 100644 --- a/cfg/validate.go +++ b/cfg/validate.go @@ -15,10 +15,16 @@ package cfg import ( + "errors" "fmt" "math" + "regexp" + "slices" + "strings" + "time" - cacheutil "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/spf13/viper" ) const ( @@ -27,6 +33,9 @@ const ( ParallelDownloadsPerFileInvalidValueError = "the value of parallel-downloads-per-file for file-cache can't be less than 1" DownloadChunkSizeMBInvalidValueError = "the value of download-chunk-size-mb for file-cache can't be less than 1" MaxParallelDownloadsCantBeZeroError = "the value of max-parallel-downloads for file-cache must not be 0 when enable-parallel-downloads is true" + ProfileAIMLTraining = "aiml-training" + ProfileAIMLServing = "aiml-serving" + ProfileAIMLCheckpointing = "aiml-checkpointing" ) func isValidLogRotateConfig(config *LogRotateLoggingConfig) error { @@ -44,29 +53,44 @@ func isValidURL(u string) error { return err } +func isValidParallelDownloadConfig(config *Config) error { + if config.FileCache.EnableParallelDownloads { + if !IsFileCacheEnabled(config) { + return errors.New("file cache should be enabled for parallel download support") + } + if config.FileCache.MaxParallelDownloads == 0 { + return errors.New("the value of max-parallel-downloads for file-cache must not be 0 when enable-parallel-downloads is true") + } + if config.FileCache.WriteBufferSize < CacheUtilMinimumAlignSizeForWriting { + return errors.New("the value of write-buffer-size for file-cache can't be less than 4096") + } + if (config.FileCache.WriteBufferSize % CacheUtilMinimumAlignSizeForWriting) != 0 { + return errors.New("the value of write-buffer-size for file-cache should be in multiple of 4096") + } + } + + return nil +} + func isValidFileCacheConfig(config *FileCacheConfig) error { if config.MaxSizeMb < -1 { - return fmt.Errorf(FileCacheMaxSizeMBInvalidValueError) + return errors.New(FileCacheMaxSizeMBInvalidValueError) } if config.MaxParallelDownloads < -1 { - return fmt.Errorf(MaxParallelDownloadsInvalidValueError) - } - if config.EnableParallelDownloads { - if config.MaxParallelDownloads == 0 { - return fmt.Errorf("the value of max-parallel-downloads for file-cache must not be 0 when enable-parallel-downloads is true") - } - if config.WriteBufferSize < cacheutil.MinimumAlignSizeForWriting { - return fmt.Errorf("the value of write-buffer-size for file-cache can't be less than 4096") - } - if (config.WriteBufferSize % cacheutil.MinimumAlignSizeForWriting) != 0 { - return fmt.Errorf("the value of write-buffer-size for file-cache should be in multiple of 4096") - } + return errors.New(MaxParallelDownloadsInvalidValueError) } if config.ParallelDownloadsPerFile < 1 { - return fmt.Errorf(ParallelDownloadsPerFileInvalidValueError) + return errors.New(ParallelDownloadsPerFileInvalidValueError) } if config.DownloadChunkSizeMb < 1 { - return fmt.Errorf(DownloadChunkSizeMBInvalidValueError) + return errors.New(DownloadChunkSizeMBInvalidValueError) + } + if _, err := regexp.Compile(config.ExcludeRegex); err != nil { + return fmt.Errorf("invalid regex value %q provided for exclude-regex", config.ExcludeRegex) + } + + if _, err := regexp.Compile(config.IncludeRegex); err != nil { + return fmt.Errorf("invalid regex value %q provided for include-regex", config.IncludeRegex) } return nil @@ -108,7 +132,7 @@ func isValidKernelListCacheTTL(TTLSecs int64) error { return nil } -func isValidMetadataCache(v isSet, c *MetadataCacheConfig) error { +func isValidMetadataCache(v *viper.Viper, c *MetadataCacheConfig) error { // Validate ttl-secs. if v.IsSet(MetadataCacheTTLConfigKey) { if c.TtlSecs < -1 { @@ -119,6 +143,16 @@ func isValidMetadataCache(v isSet, c *MetadataCacheConfig) error { } } + // Validate negative-ttl-secs. + if v.IsSet(MetadataNegativeCacheTTLConfigKey) { + if c.NegativeTtlSecs < -1 { + return fmt.Errorf("the value of negative-ttl-secs for metadata-cache can't be less than -1") + } + if c.NegativeTtlSecs > maxSupportedTTLInSeconds { + return fmt.Errorf("the value of negative-ttl-secs in metadata-cache is too high to be supported. Max is 9223372036") + } + } + // Validate type-cache-max-size-mb. if c.TypeCacheMaxSizeMb < -1 { return fmt.Errorf("the value of type-cache-max-size-mb for metadata-cache can't be less than -1") @@ -139,22 +173,31 @@ func isValidMetadataCache(v isSet, c *MetadataCacheConfig) error { return fmt.Errorf("invalid value of stat-cache-capacity (%v), can't be less than 0", c.DeprecatedStatCacheCapacity) } + // Validate prefetch configs. + if c.MetadataPrefetchMaxWorkers < -1 { + return fmt.Errorf("invalid value of metadata-cache.metadata-prefetch-max-workers: %d; should be >=0 or -1 (for infinite)", c.MetadataPrefetchMaxWorkers) + } + + if c.MetadataPrefetchEntriesLimit < -1 { + return fmt.Errorf("invalid value of metadata-cache.metadata-prefetch-entries-limit: %d; should be >=0 or -1 (for infinite)", c.MetadataPrefetchEntriesLimit) + } + return nil } func isValidWriteStreamingConfig(wc *WriteConfig) error { - if !wc.ExperimentalEnableStreamingWrites { + if !wc.EnableStreamingWrites { return nil } - if wc.BlockSizeMb <= 0 { - return fmt.Errorf("invalid value of write-block-size-mb; can't be less than 1") + if wc.BlockSizeMb <= 0 || wc.BlockSizeMb > util.MaxMiBsInInt64 { + return fmt.Errorf("invalid value of write-block-size-mb; can't be less than 1 or more than %d", util.MaxMiBsInInt64) } - if !(wc.MaxBlocksPerFile == -1 || wc.MaxBlocksPerFile >= 2) { - return fmt.Errorf("invalid value of write-max-blocks-per-file: %d; should be >=2 or -1 (for infinite)", wc.MaxBlocksPerFile) + if !(wc.MaxBlocksPerFile == -1 || wc.MaxBlocksPerFile >= 1) { + return fmt.Errorf("invalid value of write-max-blocks-per-file: %d; should be >=1 or -1 (for infinite)", wc.MaxBlocksPerFile) } - if !(wc.GlobalMaxBlocks == -1 || wc.GlobalMaxBlocks >= 2) { - return fmt.Errorf("invalid value of write-global-max-blocks: %d; should be >=2 or -1 (for infinite)", wc.GlobalMaxBlocks) + if wc.GlobalMaxBlocks < -1 { + return fmt.Errorf("invalid value of write-global-max-blocks: %d; should be >=0 or -1 (for infinite)", wc.GlobalMaxBlocks) } return nil } @@ -182,11 +225,101 @@ func isValidMetricsConfig(m *MetricsConfig) error { if m.PrometheusPort > maxPortNumber { return fmt.Errorf("prometheus-port must not be higher than the maximum allowed port number: %d but received: %d instead", maxPortNumber, m.PrometheusPort) } + if m.Workers < 1 { + return fmt.Errorf("number of metrics workers cannot be less than 1") + } + if m.BufferSize < 1 { + return fmt.Errorf("metrics buffer size cannot be less than 1") + } + return nil +} + +func isValidTraceConfig(t *TraceConfig) error { + validExporters := []string{"stdout", "gcpexporter"} + + if len(t.Exporters) == 0 { + return nil + } + + if t.SamplingRatio > 1 || t.SamplingRatio < 0 { + return fmt.Errorf("invalid tracing sampling ratio: %f, tracing sampling ratio should be in the range [0.0, 1.0]", t.SamplingRatio) + } + + for _, e := range t.Exporters { + if !slices.Contains(validExporters, strings.TrimSpace(strings.ToLower(e))) { + return fmt.Errorf("encountered invalid/unsupported tracing mode: %s", e) + } + } + + return nil +} + +func isValidChunkRetryDeadlineForRetriesConfig(chunkRetryDeadlineSecs int64) error { + if chunkRetryDeadlineSecs < 0 || chunkRetryDeadlineSecs > maxSupportedTTLInSeconds { + return fmt.Errorf("invalid value for chunk-retry-deadline-secs: %d; should be >= 0 (0 for infinite)", chunkRetryDeadlineSecs) + } + return nil +} + +func isValidChunkTransferTimeoutForRetriesConfig(chunkTransferTimeoutSecs int64) error { + if chunkTransferTimeoutSecs < 0 || chunkTransferTimeoutSecs > maxSupportedTTLInSeconds { + return fmt.Errorf("invalid value for chunk-transfer-timeout-secs: %d; should be >= 0 (0 for infinite)", chunkTransferTimeoutSecs) + } + return nil +} + +func isValidBufferedReadConfig(rc *ReadConfig) error { + if !rc.EnableBufferedRead { + return nil + } + + if rc.BlockSizeMb <= 0 || rc.BlockSizeMb > util.MaxMiBsInInt64 { + return fmt.Errorf("invalid value of read-block-size-mb; can't be less than 1 or more than %d", util.MaxMiBsInInt64) + } + + if rc.GlobalMaxBlocks < -1 { + return fmt.Errorf("invalid value of read-global-max-blocks: %d; should be >=0 or -1 (for infinite)", rc.GlobalMaxBlocks) + } + + if rc.StartBlocksPerHandle < 1 && rc.StartBlocksPerHandle != -1 { + return fmt.Errorf("invalid value of read-start-blocks-per-handle: %d; should be >=1 or -1 (for infinite)", rc.StartBlocksPerHandle) + } + + if rc.MaxBlocksPerHandle < 1 && rc.MaxBlocksPerHandle != -1 { + return fmt.Errorf("invalid value of read-max-blocks-per-handle: %d; should be >=1 or -1 (for infinite)", rc.MaxBlocksPerHandle) + } + + if rc.MinBlocksPerHandle < 1 || (rc.MaxBlocksPerHandle != -1 && rc.MinBlocksPerHandle > rc.MaxBlocksPerHandle) { + return fmt.Errorf("invalid value of read-min-blocks-per-handle: %d; should be >=1 or less than or equal to read-max-blocks-per-handle: %d", rc.MinBlocksPerHandle, rc.MaxBlocksPerHandle) + } + + return nil +} + +func isValidMRDConfig(mrdConfig *MrdConfig) error { + if mrdConfig.PoolSize < 1 { + return fmt.Errorf("invalid value of mrd-pool-size: %d; should be >=1", mrdConfig.PoolSize) + } + return nil +} + +func isValidOptimizationProfile(config *Config) error { + if config.Profile == "" { + return nil + } + + switch config.Profile { + case ProfileAIMLServing, ProfileAIMLCheckpointing, ProfileAIMLTraining: + // Supported profiles. + default: + return fmt.Errorf("Unknown profile: %q", config.Profile) + } + return nil } // ValidateConfig returns a non-nil error if the config is invalid. -func ValidateConfig(v isSet, config *Config) error { +func ValidateConfig(v *viper.Viper, config *Config) error { var err error if err = isValidLogRotateConfig(&config.Logging.LogRotate); err != nil { @@ -229,9 +362,79 @@ func ValidateConfig(v isSet, config *Config) error { return fmt.Errorf("error parsing read-stall-gcs-retries config: %w", err) } + if err = isValidChunkRetryDeadlineForRetriesConfig(config.GcsRetries.ChunkRetryDeadlineSecs); err != nil { + return fmt.Errorf("error parsing chunk-retry-deadline-secs config: %w", err) + } + + if err = isValidChunkTransferTimeoutForRetriesConfig(config.GcsRetries.ChunkTransferTimeoutSecs); err != nil { + return fmt.Errorf("error parsing chunk-transfer-timeout-secs config: %w", err) + } + + if v.IsSet("gcs-retries.max-retry-attempts") { + if err = isValidMaxRetryAttempts(config.GcsRetries.MaxRetryAttempts); err != nil { + return fmt.Errorf("error parsing max-retry-attempts config: %w", err) + } + } + + if v.IsSet("gcs-retries.multiplier") { + if err = isValidMultiplier(config.GcsRetries.Multiplier); err != nil { + return fmt.Errorf("error parsing retry-multiplier config: %w", err) + } + } + + if v.IsSet("gcs-retries.max-retry-sleep") { + if err = isValidMaxRetrySleep(config.GcsRetries.MaxRetrySleep); err != nil { + return fmt.Errorf("error parsing max-retry-sleep config: %w", err) + } + } + if err = isValidMetricsConfig(&config.Metrics); err != nil { return fmt.Errorf("error parsing metrics config: %w", err) } + if err = isValidTraceConfig(&config.Trace); err != nil { + return fmt.Errorf("error parsing monitoring config: %w", err) + } + + if err = isValidParallelDownloadConfig(config); err != nil { + return fmt.Errorf("error parsing parallel download config: %w", err) + } + + if err = isValidBufferedReadConfig(&config.Read); err != nil { + return fmt.Errorf("error parsing buffered read config: %w", err) + } + + if err = isValidMRDConfig(&config.Mrd); err != nil { + return fmt.Errorf("error parsing mrd config: %w", err) + } + + if err = isValidOptimizationProfile(config); err != nil { + return fmt.Errorf("error parsing optimize profile config: %w", err) + } + + return nil +} + +func isValidMaxRetryAttempts(maxRetryAttempts int64) error { + if maxRetryAttempts < 0 { + return fmt.Errorf("invalid value for max-retry-attempts: %d; should be >= 0 (0 for unlimited)", maxRetryAttempts) + } + if maxRetryAttempts > math.MaxInt { + return fmt.Errorf("invalid value for max-retry-attempts: %d; exceeds maximum supported value (%d)", maxRetryAttempts, math.MaxInt) + } + return nil +} + +func isValidMultiplier(multiplier float64) error { + if multiplier < 1.0 { + return fmt.Errorf("invalid value for retry-multiplier: %f; should be >= 1.0", multiplier) + } + return nil +} + +func isValidMaxRetrySleep(maxRetrySleep time.Duration) error { + if maxRetrySleep < 0 { + return fmt.Errorf("invalid value for max-retry-sleep: %v; should be >= 0", maxRetrySleep) + } return nil } diff --git a/cfg/validate_test.go b/cfg/validate_test.go index ff7b1d0fc2..8e4b0d3b41 100644 --- a/cfg/validate_test.go +++ b/cfg/validate_test.go @@ -15,9 +15,13 @@ package cfg import ( + "math" + "math/bits" "testing" "time" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/spf13/viper" "github.com/stretchr/testify/assert" ) @@ -44,6 +48,20 @@ func validFileCacheConfig(t *testing.T) FileCacheConfig { } } +func validFileCacheConfigWithExcludeRegex(t *testing.T, r string) FileCacheConfig { + t.Helper() + cfg := validFileCacheConfig(t) + cfg.ExcludeRegex = r + return cfg +} + +func validFileCacheConfigWithIncludeRegex(t *testing.T, r string) FileCacheConfig { + t.Helper() + cfg := validFileCacheConfig(t) + cfg.IncludeRegex = r + return cfg +} + func TestValidateConfigSuccessful(t *testing.T) { testCases := []struct { name string @@ -61,6 +79,13 @@ func TestValidateConfigSuccessful(t *testing.T) { MetadataCache: MetadataCacheConfig{ ExperimentalMetadataPrefetchOnMount: "disabled", }, + Metrics: MetricsConfig{ + Workers: 3, + BufferSize: 256, + }, + Mrd: MrdConfig{ + PoolSize: 4, + }, }, }, { @@ -75,6 +100,13 @@ func TestValidateConfigSuccessful(t *testing.T) { MetadataCache: MetadataCacheConfig{ ExperimentalMetadataPrefetchOnMount: "disabled", }, + Metrics: MetricsConfig{ + Workers: 3, + BufferSize: 256, + }, + Mrd: MrdConfig{ + PoolSize: 4, + }, }, }, { @@ -88,6 +120,13 @@ func TestValidateConfigSuccessful(t *testing.T) { GcsConnection: GcsConnectionConfig{ SequentialReadSizeMb: 200, }, + Metrics: MetricsConfig{ + Workers: 3, + BufferSize: 256, + }, + Mrd: MrdConfig{ + PoolSize: 4, + }, }, }, { @@ -101,6 +140,13 @@ func TestValidateConfigSuccessful(t *testing.T) { GcsConnection: GcsConnectionConfig{ SequentialReadSizeMb: 200, }, + Metrics: MetricsConfig{ + Workers: 3, + BufferSize: 256, + }, + Mrd: MrdConfig{ + PoolSize: 4, + }, }, }, { @@ -114,6 +160,13 @@ func TestValidateConfigSuccessful(t *testing.T) { GcsConnection: GcsConnectionConfig{ SequentialReadSizeMb: 200, }, + Metrics: MetricsConfig{ + Workers: 3, + BufferSize: 256, + }, + Mrd: MrdConfig{ + PoolSize: 4, + }, }, }, { @@ -127,6 +180,13 @@ func TestValidateConfigSuccessful(t *testing.T) { MetadataCache: MetadataCacheConfig{ ExperimentalMetadataPrefetchOnMount: "sync", }, + Metrics: MetricsConfig{ + Workers: 3, + BufferSize: 256, + }, + Mrd: MrdConfig{ + PoolSize: 4, + }, }, }, { @@ -140,6 +200,13 @@ func TestValidateConfigSuccessful(t *testing.T) { MetadataCache: MetadataCacheConfig{ ExperimentalMetadataPrefetchOnMount: "sync", }, + Metrics: MetricsConfig{ + Workers: 3, + BufferSize: 256, + }, + Mrd: MrdConfig{ + PoolSize: 4, + }, }, }, { @@ -153,14 +220,188 @@ func TestValidateConfigSuccessful(t *testing.T) { MetadataCache: MetadataCacheConfig{ ExperimentalMetadataPrefetchOnMount: "sync", }, + Metrics: MetricsConfig{ + Workers: 3, + BufferSize: 256, + }, + FileSystem: FileSystemConfig{KernelListCacheTtlSecs: 30}, + Mrd: MrdConfig{ + PoolSize: 4, + }, + }, + }, + { + name: "valid_parallel_download_config_with_file_cache_enabled", + config: &Config{ + Logging: LoggingConfig{LogRotate: validLogRotateConfig()}, + CacheDir: "/some/valid/path", + FileCache: FileCacheConfig{ + DownloadChunkSizeMb: 50, + EnableParallelDownloads: true, + MaxParallelDownloads: 4, + ParallelDownloadsPerFile: 16, + MaxSizeMb: -1, + WriteBufferSize: 4 * 1024 * 1024, + }, + GcsConnection: GcsConnectionConfig{ + CustomEndpoint: "https://bing.com/search?q=dotnet", + SequentialReadSizeMb: 200, + }, + MetadataCache: MetadataCacheConfig{ + ExperimentalMetadataPrefetchOnMount: "disabled", + }, + Metrics: MetricsConfig{ + Workers: 3, + BufferSize: 256, + }, + Mrd: MrdConfig{ + PoolSize: 4, + }, + }, + }, + { + name: "valid_file_cache_exclude_config", + config: &Config{ + Logging: LoggingConfig{LogRotate: validLogRotateConfig()}, + CacheDir: "/some/valid/path", + FileCache: validFileCacheConfigWithExcludeRegex(t, ".*"), + GcsConnection: GcsConnectionConfig{ + CustomEndpoint: "https://bing.com/search?q=dotnet", + SequentialReadSizeMb: 200, + }, + MetadataCache: MetadataCacheConfig{ + ExperimentalMetadataPrefetchOnMount: "disabled", + }, + Metrics: MetricsConfig{ + Workers: 3, + BufferSize: 256, + }, + Mrd: MrdConfig{ + PoolSize: 4, + }, + }, + }, + { + name: "valid_file_cache_include_config", + config: &Config{ + Logging: LoggingConfig{LogRotate: validLogRotateConfig()}, + CacheDir: "/some/valid/path", + FileCache: validFileCacheConfigWithIncludeRegex(t, ".*"), + GcsConnection: GcsConnectionConfig{ + CustomEndpoint: "https://bing.com/search?q=dotnet", + SequentialReadSizeMb: 200, + }, + MetadataCache: MetadataCacheConfig{ + ExperimentalMetadataPrefetchOnMount: "disabled", + }, + Metrics: MetricsConfig{ + Workers: 3, + BufferSize: 256, + }, + Mrd: MrdConfig{ + PoolSize: 4, + }, + }, + }, + { + name: "valid_chunk_retry_deadline_secs", + config: &Config{ + Logging: LoggingConfig{LogRotate: validLogRotateConfig()}, + FileCache: validFileCacheConfig(t), + GcsConnection: GcsConnectionConfig{ + SequentialReadSizeMb: 10, + }, + MetadataCache: MetadataCacheConfig{ + ExperimentalMetadataPrefetchOnMount: "sync", + }, + Metrics: MetricsConfig{ + Workers: 3, + BufferSize: 256, + }, + FileSystem: FileSystemConfig{KernelListCacheTtlSecs: 30}, + GcsRetries: GcsRetriesConfig{ChunkRetryDeadlineSecs: 60}, + Mrd: MrdConfig{ + PoolSize: 4, + }, + }, + }, + { + name: "valid_chunk_transfer_timeout_secs", + config: &Config{ + Logging: LoggingConfig{LogRotate: validLogRotateConfig()}, + FileCache: validFileCacheConfig(t), + GcsConnection: GcsConnectionConfig{ + SequentialReadSizeMb: 10, + }, + MetadataCache: MetadataCacheConfig{ + ExperimentalMetadataPrefetchOnMount: "sync", + }, + Metrics: MetricsConfig{ + Workers: 3, + BufferSize: 256, + }, + FileSystem: FileSystemConfig{KernelListCacheTtlSecs: 30}, + GcsRetries: GcsRetriesConfig{ChunkTransferTimeoutSecs: 15}, + Mrd: MrdConfig{ + PoolSize: 4, + }, + }, + }, + { + name: "valid_max_retry_attempts_and_multiplier", + config: &Config{ + Logging: LoggingConfig{LogRotate: validLogRotateConfig()}, + FileCache: validFileCacheConfig(t), + GcsConnection: GcsConnectionConfig{ + SequentialReadSizeMb: 10, + }, + MetadataCache: MetadataCacheConfig{ + ExperimentalMetadataPrefetchOnMount: "sync", + }, + Metrics: MetricsConfig{ + Workers: 3, + BufferSize: 256, + }, + FileSystem: FileSystemConfig{KernelListCacheTtlSecs: 30}, + GcsRetries: GcsRetriesConfig{ + MaxRetryAttempts: 3, + Multiplier: 2.0, + MaxRetrySleep: 30 * time.Second, + }, + Mrd: MrdConfig{ + PoolSize: 4, + }, + }, + }, + { + name: "valid_zero_max_retry_sleep", + config: &Config{ + Logging: LoggingConfig{LogRotate: validLogRotateConfig()}, + FileCache: validFileCacheConfig(t), + GcsConnection: GcsConnectionConfig{ + SequentialReadSizeMb: 10, + }, + MetadataCache: MetadataCacheConfig{ + ExperimentalMetadataPrefetchOnMount: "sync", + }, + Metrics: MetricsConfig{ + Workers: 3, + BufferSize: 256, + }, FileSystem: FileSystemConfig{KernelListCacheTtlSecs: 30}, + GcsRetries: GcsRetriesConfig{ + MaxRetrySleep: 0 * time.Second, + }, + Mrd: MrdConfig{ + PoolSize: 4, + }, }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - actualErr := ValidateConfig(&mockIsSet{}, tc.config) + actualErr := ValidateConfig(viper.New(), tc.config) assert.NoError(t, actualErr) }) @@ -326,11 +567,132 @@ func TestValidateConfig_ErrorScenarios(t *testing.T) { }, }, }, + { + //TODO: Remove this test as check is also removed when parallel download is default ON + name: "parallel_download_config_without_file_cache_enabled", + config: &Config{ + Logging: LoggingConfig{LogRotate: validLogRotateConfig()}, + FileCache: FileCacheConfig{ + DownloadChunkSizeMb: 50, + EnableParallelDownloads: true, + MaxParallelDownloads: 4, + ParallelDownloadsPerFile: 16, + WriteBufferSize: 4 * 1024 * 1024, + }, + GcsConnection: GcsConnectionConfig{ + CustomEndpoint: "https://bing.com/search?q=dotnet", + SequentialReadSizeMb: 200, + }, + MetadataCache: MetadataCacheConfig{ + ExperimentalMetadataPrefetchOnMount: "disabled", + }, + }, + }, + { + name: "file_cache_exclude_regex", + config: &Config{ + Logging: LoggingConfig{LogRotate: validLogRotateConfig()}, + FileCache: validFileCacheConfigWithExcludeRegex(t, "["), + }, + }, + { + name: "file_cache_include_regex", + config: &Config{ + Logging: LoggingConfig{LogRotate: validLogRotateConfig()}, + FileCache: validFileCacheConfigWithIncludeRegex(t, "["), + }, + }, + { + name: "chunk_retry_deadline_secs_in_negative", + config: &Config{ + Logging: LoggingConfig{LogRotate: validLogRotateConfig()}, + FileCache: validFileCacheConfig(t), + MetadataCache: MetadataCacheConfig{ + ExperimentalMetadataPrefetchOnMount: "sync", + }, + GcsRetries: GcsRetriesConfig{ + ChunkRetryDeadlineSecs: -10, + }, + }, + }, + { + name: "chunk_transfer_timeout_in_negative", + config: &Config{ + Logging: LoggingConfig{LogRotate: validLogRotateConfig()}, + FileCache: validFileCacheConfig(t), + MetadataCache: MetadataCacheConfig{ + ExperimentalMetadataPrefetchOnMount: "sync", + }, + GcsRetries: GcsRetriesConfig{ + ChunkTransferTimeoutSecs: -5, + }, + }, + }, + { + name: "Invalid experimental-concurrent-metadata-prefetches", + config: &Config{ + Logging: LoggingConfig{LogRotate: validLogRotateConfig()}, + MetadataCache: MetadataCacheConfig{ + MetadataPrefetchMaxWorkers: -4, + }, + GcsConnection: GcsConnectionConfig{ + SequentialReadSizeMb: 200, + }, + }, + }, + { + name: "Invalid experimental-metadata-prefetch-count", + config: &Config{ + Logging: LoggingConfig{LogRotate: validLogRotateConfig()}, + MetadataCache: MetadataCacheConfig{ + MetadataPrefetchEntriesLimit: -2, + }, + GcsConnection: GcsConnectionConfig{ + SequentialReadSizeMb: 200, + }, + }, + }, + { + name: "Invalid negative max-retry-attempts", + config: &Config{ + Logging: LoggingConfig{LogRotate: validLogRotateConfig()}, + GcsRetries: GcsRetriesConfig{ + MaxRetryAttempts: -3, + }, + }, + }, + { + name: "Invalid too large max-retry-attempts", + config: &Config{ + Logging: LoggingConfig{LogRotate: validLogRotateConfig()}, + GcsRetries: GcsRetriesConfig{ + MaxRetryAttempts: math.MaxInt64, + }, + }, + }, + { + name: "Invalid multiplier less than 1.0", + config: &Config{ + Logging: LoggingConfig{LogRotate: validLogRotateConfig()}, + GcsRetries: GcsRetriesConfig{ + Multiplier: 0.8, + }, + }, + }, + { + name: "Invalid negative max-retry-sleep", + config: &Config{ + Logging: LoggingConfig{LogRotate: validLogRotateConfig()}, + GcsRetries: GcsRetriesConfig{ + MaxRetrySleep: -10 * time.Second, + }, + }, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - assert.Error(t, ValidateConfig(&mockIsSet{}, tc.config)) + assert.Error(t, ValidateConfig(viper.New(), tc.config)) }) } } @@ -375,60 +737,46 @@ func Test_isValidWriteStreamingConfig_ErrorScenarios(t *testing.T) { writeConfig WriteConfig }{ {"zero_block_size", WriteConfig{ - BlockSizeMb: 0, - CreateEmptyFile: false, - ExperimentalEnableStreamingWrites: true, - GlobalMaxBlocks: -1, - MaxBlocksPerFile: -1, + BlockSizeMb: 0, + CreateEmptyFile: false, + EnableStreamingWrites: true, + GlobalMaxBlocks: -1, + MaxBlocksPerFile: -1, + }}, + {"very_large_block_size", WriteConfig{ + BlockSizeMb: util.MaxMiBsInInt64 + 1, + CreateEmptyFile: false, + EnableStreamingWrites: true, + GlobalMaxBlocks: -1, + MaxBlocksPerFile: -1, }}, {"negative_block_size", WriteConfig{ - BlockSizeMb: -1, - CreateEmptyFile: false, - ExperimentalEnableStreamingWrites: true, - GlobalMaxBlocks: -1, - MaxBlocksPerFile: -1, + BlockSizeMb: -1, + CreateEmptyFile: false, + EnableStreamingWrites: true, + GlobalMaxBlocks: -1, + MaxBlocksPerFile: -1, }}, {"-2_global_max_blocks", WriteConfig{ - BlockSizeMb: 10, - CreateEmptyFile: false, - ExperimentalEnableStreamingWrites: true, - GlobalMaxBlocks: -2, - MaxBlocksPerFile: -1, - }}, - {"0_global_max_blocks", WriteConfig{ - BlockSizeMb: 10, - CreateEmptyFile: false, - ExperimentalEnableStreamingWrites: true, - GlobalMaxBlocks: 0, - MaxBlocksPerFile: -1, - }}, - {"1_global_max_blocks", WriteConfig{ - BlockSizeMb: 10, - CreateEmptyFile: false, - ExperimentalEnableStreamingWrites: true, - GlobalMaxBlocks: 1, - MaxBlocksPerFile: -1, + BlockSizeMb: 10, + CreateEmptyFile: false, + EnableStreamingWrites: true, + GlobalMaxBlocks: -2, + MaxBlocksPerFile: -1, }}, {"-2_max_blocks_per_file", WriteConfig{ - BlockSizeMb: 10, - CreateEmptyFile: false, - ExperimentalEnableStreamingWrites: true, - GlobalMaxBlocks: 20, - MaxBlocksPerFile: -2, + BlockSizeMb: 10, + CreateEmptyFile: false, + EnableStreamingWrites: true, + GlobalMaxBlocks: 20, + MaxBlocksPerFile: -2, }}, {"0_max_blocks_per_file", WriteConfig{ - BlockSizeMb: 10, - CreateEmptyFile: false, - ExperimentalEnableStreamingWrites: true, - GlobalMaxBlocks: 20, - MaxBlocksPerFile: 0, - }}, - {"1_max_blocks_per_file", WriteConfig{ - BlockSizeMb: 10, - CreateEmptyFile: false, - ExperimentalEnableStreamingWrites: true, - GlobalMaxBlocks: 20, - MaxBlocksPerFile: 1, + BlockSizeMb: 10, + CreateEmptyFile: false, + EnableStreamingWrites: true, + GlobalMaxBlocks: 20, + MaxBlocksPerFile: 0, }}, } @@ -439,45 +787,215 @@ func Test_isValidWriteStreamingConfig_ErrorScenarios(t *testing.T) { } } +func Test_isValidBufferedReadConfig_ErrorScenarios(t *testing.T) { + var testCases = []struct { + testName string + read ReadConfig + }{ + {"negative_block_size", ReadConfig{ + BlockSizeMb: -1, + EnableBufferedRead: true, + GlobalMaxBlocks: -1, + MaxBlocksPerHandle: -1, + StartBlocksPerHandle: 1, + MinBlocksPerHandle: 4, + }}, + {"zero_block_size", ReadConfig{ + BlockSizeMb: 0, + EnableBufferedRead: true, + GlobalMaxBlocks: -1, + MaxBlocksPerHandle: -1, + StartBlocksPerHandle: 1, + MinBlocksPerHandle: 4, + }}, + {"negative_global_max_blocks", ReadConfig{ + BlockSizeMb: 16, + EnableBufferedRead: true, + GlobalMaxBlocks: -2, + MaxBlocksPerHandle: -1, + StartBlocksPerHandle: 1, + MinBlocksPerHandle: 4, + }}, + {"negative_max_blocks_per_handle", ReadConfig{ + BlockSizeMb: 16, + EnableBufferedRead: true, + GlobalMaxBlocks: -1, + MaxBlocksPerHandle: -2, + StartBlocksPerHandle: 1, + MinBlocksPerHandle: 4, + }}, + {"negative_min_blocks_per_handle", ReadConfig{ + BlockSizeMb: 16, + EnableBufferedRead: true, + GlobalMaxBlocks: -1, + MaxBlocksPerHandle: -1, + StartBlocksPerHandle: 1, + MinBlocksPerHandle: -4, + }}, + {"zero_min_blocks_per_handle", ReadConfig{ + BlockSizeMb: 16, + EnableBufferedRead: true, + GlobalMaxBlocks: -1, + MaxBlocksPerHandle: -1, + StartBlocksPerHandle: 1, + MinBlocksPerHandle: 0, + }}, + } + + for _, tc := range testCases { + t.Run(tc.testName, func(t *testing.T) { + assert.Error(t, isValidBufferedReadConfig(&tc.read)) + }) + } +} + +func Test_isValidMRDConfig(t *testing.T) { + testCases := []struct { + name string + mrdConfig MrdConfig + wantErr bool + }{ + { + name: "valid_pool_size", + mrdConfig: MrdConfig{ + PoolSize: 10, + }, + wantErr: false, + }, + { + name: "invalid_pool_size_zero", + mrdConfig: MrdConfig{ + PoolSize: 0, + }, + wantErr: true, + }, + { + name: "invalid_pool_size_negative", + mrdConfig: MrdConfig{ + PoolSize: -1, + }, + wantErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := isValidMRDConfig(&tc.mrdConfig) + if tc.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func Test_isValidBufferedReadConfig_ValidScenarios(t *testing.T) { + var testCases = []struct { + testName string + read ReadConfig + }{ + {"valid_config_1", ReadConfig{ + BlockSizeMb: 16, + EnableBufferedRead: true, + GlobalMaxBlocks: -1, + MaxBlocksPerHandle: -1, + StartBlocksPerHandle: 1, + MinBlocksPerHandle: 1, + }}, + {"valid_config_2", ReadConfig{ + BlockSizeMb: 16, + EnableBufferedRead: true, + GlobalMaxBlocks: 10, + MaxBlocksPerHandle: -1, + StartBlocksPerHandle: 1, + MinBlocksPerHandle: 4, + }}, + {"valid_config_3", ReadConfig{ + BlockSizeMb: 16, + EnableBufferedRead: true, + GlobalMaxBlocks: 10, + MaxBlocksPerHandle: 5, + StartBlocksPerHandle: 1, + MinBlocksPerHandle: 5, + }}, + {"valid_config_4", ReadConfig{ + BlockSizeMb: 16, + EnableBufferedRead: false, + GlobalMaxBlocks: 10, + MaxBlocksPerHandle: 5, + StartBlocksPerHandle: 10, + MinBlocksPerHandle: 3, + }}, + } + + for _, tc := range testCases { + t.Run(tc.testName, func(t *testing.T) { + assert.NoError(t, isValidBufferedReadConfig(&tc.read)) + }) + } +} + func Test_isValidWriteStreamingConfig_SuccessScenarios(t *testing.T) { var testCases = []struct { testName string writeConfig WriteConfig }{ {"streaming_writes_disabled", WriteConfig{ - BlockSizeMb: -1, - CreateEmptyFile: false, - ExperimentalEnableStreamingWrites: false, - GlobalMaxBlocks: -10, - MaxBlocksPerFile: -10, + BlockSizeMb: -1, + CreateEmptyFile: false, + EnableStreamingWrites: false, + GlobalMaxBlocks: -10, + MaxBlocksPerFile: -10, }}, {"valid_write_config_1", WriteConfig{ - BlockSizeMb: 1, - CreateEmptyFile: false, - ExperimentalEnableStreamingWrites: true, - GlobalMaxBlocks: -1, - MaxBlocksPerFile: -1, + BlockSizeMb: 1, + CreateEmptyFile: false, + EnableStreamingWrites: true, + GlobalMaxBlocks: -1, + MaxBlocksPerFile: -1, }}, {"valid_write_config_2", WriteConfig{ - BlockSizeMb: 10, - CreateEmptyFile: false, - ExperimentalEnableStreamingWrites: true, - GlobalMaxBlocks: 20, - MaxBlocksPerFile: -1, + BlockSizeMb: 10, + CreateEmptyFile: false, + EnableStreamingWrites: true, + GlobalMaxBlocks: 20, + MaxBlocksPerFile: -1, }}, {"valid_write_config_3", WriteConfig{ - BlockSizeMb: 10, - CreateEmptyFile: false, - ExperimentalEnableStreamingWrites: true, - GlobalMaxBlocks: 20, - MaxBlocksPerFile: 20, + BlockSizeMb: 10, + CreateEmptyFile: false, + EnableStreamingWrites: true, + GlobalMaxBlocks: 20, + MaxBlocksPerFile: 20, }}, {"valid_write_config_4", WriteConfig{ - BlockSizeMb: 10, - CreateEmptyFile: false, - ExperimentalEnableStreamingWrites: true, - GlobalMaxBlocks: 40, - MaxBlocksPerFile: 20, + BlockSizeMb: 10, + CreateEmptyFile: false, + EnableStreamingWrites: true, + GlobalMaxBlocks: 40, + MaxBlocksPerFile: 20, + }}, + {"0_global_max_blocks", WriteConfig{ + BlockSizeMb: 10, + CreateEmptyFile: false, + EnableStreamingWrites: true, + GlobalMaxBlocks: 0, + MaxBlocksPerFile: -1, + }}, + {"1_global_max_blocks", WriteConfig{ + BlockSizeMb: 10, + CreateEmptyFile: false, + EnableStreamingWrites: true, + GlobalMaxBlocks: 1, + MaxBlocksPerFile: -1, + }}, + {"1_max_blocks_per_file", WriteConfig{ + BlockSizeMb: 10, + CreateEmptyFile: false, + EnableStreamingWrites: true, + GlobalMaxBlocks: 20, + MaxBlocksPerFile: 1, }}, } @@ -499,6 +1017,13 @@ func validConfig(t *testing.T) Config { MetadataCache: MetadataCacheConfig{ ExperimentalMetadataPrefetchOnMount: "disabled", }, + Metrics: MetricsConfig{ + Workers: 1, + BufferSize: 1, + }, + Mrd: MrdConfig{ + PoolSize: 4, + }, } } @@ -522,6 +1047,8 @@ func TestValidateMetrics(t *testing.T) { name: "neg_cloud_metrics_export_interval", metricsConfig: MetricsConfig{ CloudMetricsExportIntervalSecs: -1, + Workers: 10, + BufferSize: 100, }, wantErr: false, }, @@ -529,6 +1056,8 @@ func TestValidateMetrics(t *testing.T) { name: "neg_stackdriver_export_interval", metricsConfig: MetricsConfig{ StackdriverExportInterval: -1 * time.Second, + Workers: 10, + BufferSize: 100, }, wantErr: false, }, @@ -536,6 +1065,8 @@ func TestValidateMetrics(t *testing.T) { name: "neg_cloud_metrics_export_interval", metricsConfig: MetricsConfig{ CloudMetricsExportIntervalSecs: 10, + Workers: 10, + BufferSize: 100, }, wantErr: false, }, @@ -550,6 +1081,8 @@ func TestValidateMetrics(t *testing.T) { name: "valid_prom_port", metricsConfig: MetricsConfig{ PrometheusPort: 5550, + Workers: 10, + BufferSize: 100, }, wantErr: false, }, @@ -557,6 +1090,8 @@ func TestValidateMetrics(t *testing.T) { name: "prom_disabled_0", metricsConfig: MetricsConfig{ PrometheusPort: 0, + Workers: 10, + BufferSize: 100, }, wantErr: false, }, @@ -564,24 +1099,369 @@ func TestValidateMetrics(t *testing.T) { name: "prom_disabled_less_than_0", metricsConfig: MetricsConfig{ PrometheusPort: -21, + Workers: 10, + BufferSize: 100, + }, + wantErr: false, + }, + { + name: "metrics_workers_less_than_1", + metricsConfig: MetricsConfig{ + Workers: 0, + }, + wantErr: true, + }, + { + name: "metrics_buffer_size_less_than_1", + metricsConfig: MetricsConfig{ + BufferSize: 0, + }, + wantErr: true, + }, + { + name: "valid_workers_and_buffer_size", + metricsConfig: MetricsConfig{ + Workers: 10, + BufferSize: 100, }, wantErr: false, }, } for _, tc := range testCases { - tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() c := validConfig(t) c.Metrics = tc.metricsConfig - err := ValidateConfig(&mockIsSet{}, &c) + err := ValidateConfig(viper.New(), &c) + + if tc.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestValidateMonitoringSuccessScenarios(t *testing.T) { + t.Parallel() + testCases := []struct { + name string + TraceConfig TraceConfig + }{ + + { + name: "verify_tracing_modes_exact_match", + TraceConfig: TraceConfig{ + Exporters: []string{"stdout", "gcpexporter"}, + SamplingRatio: 0.2, + }, + }, + { + name: "verify_tracing_modes_unnecessary_space_match", + TraceConfig: TraceConfig{ + Exporters: []string{"stdout", " gcpexporter"}, + SamplingRatio: 0.4, + }, + }, + { + name: "verify_tracing_modes_case_insensitive", + TraceConfig: TraceConfig{ + Exporters: []string{"STDout", " GcpExporTer"}, + SamplingRatio: 0.7, + }, + }, + { + name: "verify_complete_tracing_config_success_case", + TraceConfig: TraceConfig{ + Exporters: []string{"gcpexporter"}, + ProjectId: "test-gcloud-project", + SamplingRatio: 0.3, + }, + }, + { + name: "invalid_tracing_sampling_ratio_success_empty_mode", + TraceConfig: TraceConfig{ + Exporters: []string{}, + ProjectId: "test-gcloud-project", + SamplingRatio: 1.4, + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + c := validConfig(t) + c.Trace = tc.TraceConfig + + err := ValidateConfig(viper.New(), &c) + + assert.NoError(t, err) + }) + } +} + +func TestValidateMonitoringFailureScenarios(t *testing.T) { + t.Parallel() + testCases := []struct { + name string + TraceConfig TraceConfig + }{ + { + name: "invalid_tracing_mode_failure", + TraceConfig: TraceConfig{ + Exporters: []string{"stdout", " random_export"}, + }, + }, + { + name: "invalid_tracing_sampling_ratio_failure", + TraceConfig: TraceConfig{ + Exporters: []string{"stdout"}, + ProjectId: "test-gcloud-project", + SamplingRatio: 1.4, + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + c := validConfig(t) + c.Trace = tc.TraceConfig + + err := ValidateConfig(viper.New(), &c) + + assert.Error(t, err) + }) + } +} + +func TestValidateLogSeverityRanks(t *testing.T) { + t.Parallel() + testCases := []struct { + logSev string + wantLogSevRank int + wantLogSev LogSeverity + wantErr bool + }{ + { + logSev: "off", + wantLogSevRank: 5, + wantLogSev: OffLogSeverity, + }, + { + logSev: "error", + wantLogSevRank: 4, + wantLogSev: ErrorLogSeverity, + }, + { + logSev: "warning", + wantLogSevRank: 3, + wantLogSev: WarningLogSeverity, + }, + { + logSev: "info", + wantLogSevRank: 2, + wantLogSev: InfoLogSeverity, + }, + { + logSev: "debug", + wantLogSevRank: 1, + wantLogSev: DebugLogSeverity, + }, + { + logSev: "trace", + wantLogSevRank: 0, + wantLogSev: TraceLogSeverity, + }, + { + logSev: "invalid", + wantLogSevRank: -1, + wantErr: true, + }, + } + for _, tc := range testCases { + t.Run(tc.logSev, func(t *testing.T) { + t.Parallel() + level := LogSeverity(tc.logSev) + + err := level.UnmarshalText([]byte(tc.logSev)) if tc.wantErr { assert.Error(t, err) + assert.Equal(t, -1, level.Rank()) } else { assert.NoError(t, err) + assert.Equal(t, tc.wantLogSev.Rank(), level.Rank()) } }) } } + +func TestValidateProfile(t *testing.T) { + t.Parallel() + testCases := []struct { + name string + profile string + wantErr bool + }{ + { + name: "empty_profile", + profile: "", + wantErr: false, + }, { + name: "profile_training", + profile: ProfileAIMLTraining, + wantErr: false, + }, { + name: "profile_serving", + profile: ProfileAIMLServing, + wantErr: false, + }, { + name: "profile_checkpointing", + profile: ProfileAIMLCheckpointing, + wantErr: false, + }, { + name: "unsupported_profile", + profile: "unsupported-profile", + wantErr: true, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + c := validConfig(t) + c.Profile = tc.profile + + err := ValidateConfig(viper.New(), &c) + + if tc.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func Test_isValidMaxRetryAttempts_ValidScenarios(t *testing.T) { + testCases := []struct { + name string + maxRetryAttempts int64 + }{ + {"valid_attempts_zero", 0}, + {"valid_attempts_positive", 5}, + {"valid_attempts_max_int", int64(math.MaxInt)}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := isValidMaxRetryAttempts(tc.maxRetryAttempts) + + assert.NoError(t, err) + }) + } +} + +func Test_isValidMaxRetryAttempts_ErrorScenarios(t *testing.T) { + testCases := []struct { + name string + maxRetryAttempts int64 + }{ + {"invalid_attempts_negative", -3}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := isValidMaxRetryAttempts(tc.maxRetryAttempts) + + assert.Error(t, err) + }) + } +} + +func Test_isValidMaxRetryAttempts_ExceedsMaxInt_ErrorScenario(t *testing.T) { + // math.MaxInt is math.MaxInt64 on 64-bit systems and math.MaxInt32 on 32-bit systems. + // We can only test the "exceeds math.MaxInt" case on 32-bit systems because on 64-bit systems, + // incrementing math.MaxInt overflows int64. + if is64Bit := bits.UintSize == 64; is64Bit { + t.Skip("Skipping on 64-bit systems because math.MaxInt exceeds the capacity of int64 when incremented.") + } + + maxIntVal := int64(math.MaxInt) + err := isValidMaxRetryAttempts(maxIntVal + 1) + + assert.Error(t, err) +} + +func Test_isValidMultiplier_ValidScenarios(t *testing.T) { + testCases := []struct { + name string + multiplier float64 + }{ + {"valid_multiplier_standard", 2.0}, + {"valid_multiplier_minimum", 1.0}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := isValidMultiplier(tc.multiplier) + + assert.NoError(t, err) + }) + } +} + +func Test_isValidMultiplier_ErrorScenarios(t *testing.T) { + testCases := []struct { + name string + multiplier float64 + }{ + {"invalid_multiplier_too_low", 0.8}, + {"invalid_multiplier_negative", -1.5}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := isValidMultiplier(tc.multiplier) + + assert.Error(t, err) + }) + } +} + +func Test_isValidMaxRetrySleep_ValidScenarios(t *testing.T) { + testCases := []struct { + name string + maxRetrySleep time.Duration + }{ + {"valid_sleep_zero", 0 * time.Second}, + {"valid_sleep_seconds", 30 * time.Second}, + {"valid_sleep_milliseconds", 500 * time.Millisecond}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := isValidMaxRetrySleep(tc.maxRetrySleep) + + assert.NoError(t, err) + }) + } +} + +func Test_isValidMaxRetrySleep_ErrorScenarios(t *testing.T) { + testCases := []struct { + name string + maxRetrySleep time.Duration + }{ + {"invalid_sleep_negative", -10 * time.Second}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := isValidMaxRetrySleep(tc.maxRetrySleep) + + assert.Error(t, err) + }) + } +} diff --git a/cmd/config_rationalization_test.go b/cmd/config_rationalization_test.go index 7b96351a2b..610d1de868 100644 --- a/cmd/config_rationalization_test.go +++ b/cmd/config_rationalization_test.go @@ -33,13 +33,13 @@ func TestRationalizeMetadataCache(t *testing.T) { name: "new_ttl_flag_set", args: []string{"--metadata-cache-ttl-secs=30"}, expectedTTLSecs: 30, - expectedStatCacheSize: 32, // default. + expectedStatCacheSize: 34, // default. }, { name: "old_ttl_flags_set", args: []string{"--stat-cache-ttl=10s", "--type-cache-ttl=5s"}, expectedTTLSecs: 5, - expectedStatCacheSize: 32, // default. + expectedStatCacheSize: 34, // default. }, { name: "new_stat-cache-size-mb_flag_set", @@ -57,7 +57,7 @@ func TestRationalizeMetadataCache(t *testing.T) { name: "no_relevant_flags_set", args: []string{""}, expectedTTLSecs: 60, // default. - expectedStatCacheSize: 32, //default. + expectedStatCacheSize: 34, //default. }, { name: "both_new_and_old_flags_set", diff --git a/cmd/config_validation_test.go b/cmd/config_validation_test.go index 00c7ebab12..3b525ed4d9 100644 --- a/cmd/config_validation_test.go +++ b/cmd/config_validation_test.go @@ -23,7 +23,7 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -31,8 +31,8 @@ import ( func getConfigObject(t *testing.T, args []string) (*cfg.Config, error) { t.Helper() var c *cfg.Config - cmd, err := NewRootCmd(func(config *cfg.Config, _, _ string) error { - c = config + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + c = mountInfo.config return nil }) require.Nil(t, err) @@ -54,15 +54,18 @@ func getConfigObjectWithConfigFile(t *testing.T, configFilePath string) (*cfg.Co func defaultFileCacheConfig(t *testing.T) cfg.FileCacheConfig { t.Helper() return cfg.FileCacheConfig{ - CacheFileForRangeRead: false, - DownloadChunkSizeMb: 50, - EnableCrc: false, - EnableParallelDownloads: false, - MaxParallelDownloads: int64(max(16, 2*runtime.NumCPU())), - MaxSizeMb: -1, - ParallelDownloadsPerFile: 16, - WriteBufferSize: 4 * 1024 * 1024, - EnableODirect: false, + CacheFileForRangeRead: false, + DownloadChunkSizeMb: 200, + EnableCrc: false, + EnableParallelDownloads: false, + ExperimentalParallelDownloadsDefaultOn: true, + MaxParallelDownloads: int64(max(16, 2*runtime.NumCPU())), + MaxSizeMb: -1, + ParallelDownloadsPerFile: 16, + SharedCacheChunkSizeMb: 8, + WriteBufferSize: 4 * 1024 * 1024, + EnableODirect: false, + ExperimentalDisableSizeCalculationFix: false, } } @@ -117,6 +120,16 @@ func TestValidateConfigFile(t *testing.T) { configFile: "testdata/invalid_log_rotate_config_2.yaml", wantErr: true, }, + { + name: "invalid_profile", + configFile: "testdata/invalid_profile.yaml", + wantErr: true, + }, + { + name: "valid_profile", + configFile: "testdata/valid_profile.yaml", + wantErr: false, + }, } for _, tc := range testCases { @@ -148,6 +161,11 @@ func TestValidateCliFlag(t *testing.T) { args: []string{"--log-rotate-backup-file-count=0"}, wantErr: false, }, + { + name: "valid optimize-flag", + args: []string{"--profile=" + cfg.ProfileAIMLTraining}, + wantErr: false, + }, { name: "invalid log severity", args: []string{"--log-severity=critical"}, @@ -163,6 +181,11 @@ func TestValidateCliFlag(t *testing.T) { args: []string{"--log-rotate-backup-file-count=-1"}, wantErr: true, }, + { + name: "invalid optimize-flag", + args: []string{"--profile=unknown-profile"}, + wantErr: true, + }, } for _, tc := range testCases { @@ -189,11 +212,13 @@ func TestValidateConfigFile_WriteConfig(t *testing.T) { configFile: "testdata/empty_file.yaml", expectedConfig: &cfg.Config{ Write: cfg.WriteConfig{ - CreateEmptyFile: false, - BlockSizeMb: 64, - ExperimentalEnableStreamingWrites: false, - GlobalMaxBlocks: math.MaxInt64, - MaxBlocksPerFile: math.MaxInt64}, + CreateEmptyFile: false, + BlockSizeMb: 32, + EnableStreamingWrites: true, + GlobalMaxBlocks: 4, + MaxBlocksPerFile: 1, + EnableRapidAppends: true, + }, }, }, { @@ -201,11 +226,11 @@ func TestValidateConfigFile_WriteConfig(t *testing.T) { configFile: "testdata/valid_config.yaml", expectedConfig: &cfg.Config{ Write: cfg.WriteConfig{ - CreateEmptyFile: false, // changed due to enabled streaming writes. - BlockSizeMb: 10, - ExperimentalEnableStreamingWrites: true, - GlobalMaxBlocks: 20, - MaxBlocksPerFile: 2, + CreateEmptyFile: false, // changed due to enabled streaming writes. + BlockSizeMb: 10, + EnableStreamingWrites: true, + GlobalMaxBlocks: 20, + MaxBlocksPerFile: 2, }, }, }, @@ -222,6 +247,57 @@ func TestValidateConfigFile_WriteConfig(t *testing.T) { } } +func TestValidateConfigFile_ReadConfig(t *testing.T) { + testCases := []struct { + name string + configFile string + expectedConfig *cfg.Config + }{ + { + name: "Empty config file [default values].", + configFile: "testdata/empty_file.yaml", + expectedConfig: &cfg.Config{ + Read: cfg.ReadConfig{ + InactiveStreamTimeout: 10 * time.Second, + BlockSizeMb: 16, + EnableBufferedRead: false, + GlobalMaxBlocks: 40, + MaxBlocksPerHandle: 20, + StartBlocksPerHandle: 1, + MinBlocksPerHandle: 4, + RandomSeekThreshold: 3, + }, + }, + }, + { + name: "Valid config file.", + configFile: "testdata/valid_config.yaml", + expectedConfig: &cfg.Config{ + Read: cfg.ReadConfig{ + InactiveStreamTimeout: 10 * time.Second, + BlockSizeMb: 8, + EnableBufferedRead: true, + MaxBlocksPerHandle: 20, + GlobalMaxBlocks: 20, + StartBlocksPerHandle: 4, + MinBlocksPerHandle: 2, + RandomSeekThreshold: 10, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gotConfig, err := getConfigObjectWithConfigFile(t, tc.configFile) + + if assert.NoError(t, err) { + assert.EqualValues(t, tc.expectedConfig.Read, gotConfig.Read) + } + }) + } +} + func TestValidateConfigFile_InvalidConfigThrowsError(t *testing.T) { testCases := []struct { name string @@ -301,11 +377,11 @@ func TestValidateConfigFile_InvalidConfigThrowsError(t *testing.T) { }, { name: "small_global_max_blocks", - configFile: "testdata/write_config/invalid_write_config_due_to_small_global_max_blocks.yaml", + configFile: "testdata/write_config/invalid_write_config_due_to_invalid_global_max_blocks.yaml", }, { name: "small_max_blocks_per_file", - configFile: "testdata/write_config/invalid_write_config_due_to_small_max_blocks_per_file.yaml", + configFile: "testdata/write_config/invalid_write_config_due_to_zero_max_blocks_per_file.yaml", }, { name: "negative req_increase_rate", @@ -352,15 +428,18 @@ func TestValidateConfigFile_FileCacheConfigSuccessful(t *testing.T) { configFile: "testdata/valid_config.yaml", expectedConfig: &cfg.Config{ FileCache: cfg.FileCacheConfig{ - CacheFileForRangeRead: true, - DownloadChunkSizeMb: 300, - EnableCrc: true, - EnableParallelDownloads: true, - MaxParallelDownloads: 200, - MaxSizeMb: 40, - ParallelDownloadsPerFile: 10, - WriteBufferSize: 8192, - EnableODirect: true, + CacheFileForRangeRead: true, + DownloadChunkSizeMb: 300, + EnableCrc: true, + EnableParallelDownloads: false, + MaxParallelDownloads: 200, + MaxSizeMb: 40, + ParallelDownloadsPerFile: 10, + SharedCacheChunkSizeMb: 8, + WriteBufferSize: 8192, + EnableODirect: true, + ExperimentalParallelDownloadsDefaultOn: true, + ExperimentalDisableSizeCalculationFix: true, }, }, }, @@ -449,6 +528,7 @@ func TestValidateConfigFile_GCSConnectionConfigSuccessful(t *testing.T) { ClientProtocol: "http1", CustomEndpoint: "", ExperimentalEnableJsonRead: false, + GrpcPathStrategy: "direct-path-with-fallback", GrpcConnPoolSize: 1, HttpClientTimeout: 0, LimitBytesPerSec: -1, @@ -456,6 +536,7 @@ func TestValidateConfigFile_GCSConnectionConfigSuccessful(t *testing.T) { MaxConnsPerHost: 0, MaxIdleConnsPerHost: 100, SequentialReadSizeMb: 200, + EnableHttpDnsCache: true, }, }, }, @@ -468,6 +549,7 @@ func TestValidateConfigFile_GCSConnectionConfigSuccessful(t *testing.T) { ClientProtocol: "http2", CustomEndpoint: "www.abc.com", ExperimentalEnableJsonRead: true, + GrpcPathStrategy: "direct-path-only", GrpcConnPoolSize: 200, HttpClientTimeout: 400 * time.Second, LimitBytesPerSec: 20, @@ -475,6 +557,7 @@ func TestValidateConfigFile_GCSConnectionConfigSuccessful(t *testing.T) { MaxConnsPerHost: 400, MaxIdleConnsPerHost: 20, SequentialReadSizeMb: 450, + EnableHttpDnsCache: true, }, }, }, @@ -512,11 +595,11 @@ func TestValidateConfigFile_FileSystemConfigSuccessful(t *testing.T) { Gid: -1, IgnoreInterrupts: true, KernelListCacheTtlSecs: 0, + InactiveMrdCacheSize: 1000, RenameDirLimit: 0, TempDir: "", - PreconditionErrors: true, Uid: -1, - HandleSigterm: true, + MaxReadAheadKb: 0, }, }, }, @@ -532,11 +615,11 @@ func TestValidateConfigFile_FileSystemConfigSuccessful(t *testing.T) { Gid: -1, IgnoreInterrupts: true, KernelListCacheTtlSecs: 0, + InactiveMrdCacheSize: 1000, RenameDirLimit: 0, TempDir: "", - PreconditionErrors: true, Uid: -1, - HandleSigterm: true, + MaxReadAheadKb: 0, }, }, }, @@ -552,11 +635,14 @@ func TestValidateConfigFile_FileSystemConfigSuccessful(t *testing.T) { Gid: 7, IgnoreInterrupts: false, KernelListCacheTtlSecs: 300, + InactiveMrdCacheSize: 1000, RenameDirLimit: 10, TempDir: cfg.ResolvedPath(path.Join(hd, "temp")), - PreconditionErrors: false, Uid: 8, - HandleSigterm: true, + MaxReadAheadKb: 1024, + }, + GcsConnection: cfg.GcsConnectionConfig{ + EnableHttpDnsCache: true, }, }, }, @@ -641,6 +727,40 @@ func TestValidateConfigFile_EnableHNSConfigSuccessful(t *testing.T) { } } +func TestValidateConfigFile_DisableListAccessCheckSuccessful(t *testing.T) { + testCases := []struct { + name string + configFile string + expectedConfig *cfg.Config + }{ + { + // Test default values. + name: "empty_config_file", + configFile: "testdata/empty_file.yaml", + expectedConfig: &cfg.Config{ + DisableListAccessCheck: true, + }, + }, + { + name: "valid_config_file", + configFile: "testdata/valid_config.yaml", + expectedConfig: &cfg.Config{ + DisableListAccessCheck: false, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gotConfig, err := getConfigObjectWithConfigFile(t, tc.configFile) + + if assert.NoError(t, err) { + assert.EqualValues(t, tc.expectedConfig.DisableListAccessCheck, gotConfig.DisableListAccessCheck) + } + }) + } +} + func TestValidateConfigFile_MetadataCacheConfigSuccessful(t *testing.T) { testCases := []struct { name string @@ -657,9 +777,13 @@ func TestValidateConfigFile_MetadataCacheConfigSuccessful(t *testing.T) { DeprecatedStatCacheTtl: 60 * time.Second, DeprecatedTypeCacheTtl: 60 * time.Second, EnableNonexistentTypeCache: false, + MetadataPrefetchEntriesLimit: 5000, + MetadataPrefetchMaxWorkers: 10, + EnableMetadataPrefetch: false, ExperimentalMetadataPrefetchOnMount: "disabled", - StatCacheMaxSizeMb: 32, + StatCacheMaxSizeMb: 34, TtlSecs: 60, + NegativeTtlSecs: 5, TypeCacheMaxSizeMb: 4, }, }, @@ -673,9 +797,13 @@ func TestValidateConfigFile_MetadataCacheConfigSuccessful(t *testing.T) { DeprecatedStatCacheTtl: 30 * time.Second, DeprecatedTypeCacheTtl: 20 * time.Second, EnableNonexistentTypeCache: true, + EnableMetadataPrefetch: true, + MetadataPrefetchMaxWorkers: 5, + MetadataPrefetchEntriesLimit: 50, ExperimentalMetadataPrefetchOnMount: "sync", StatCacheMaxSizeMb: 40, TtlSecs: 100, + NegativeTtlSecs: 5, TypeCacheMaxSizeMb: 10, }, }, @@ -693,8 +821,8 @@ func TestValidateConfigFile_MetadataCacheConfigSuccessful(t *testing.T) { } } -func TestValidateConfigFile_ReadStallConfigSuccessful(t *testing.T) { - testCases := []struct { +func TestValidateConfigFile_GCSRetries(t *testing.T) { + tests := []struct { name string configFile string expectedConfig *cfg.Config @@ -705,8 +833,14 @@ func TestValidateConfigFile_ReadStallConfigSuccessful(t *testing.T) { configFile: "testdata/empty_file.yaml", expectedConfig: &cfg.Config{ GcsRetries: cfg.GcsRetriesConfig{ + ChunkRetryDeadlineSecs: 120, + ChunkTransferTimeoutSecs: 10, + EnableMountRetries: false, + MaxRetryAttempts: math.MaxInt, + MaxRetrySleep: 30 * time.Second, + Multiplier: 2, ReadStall: cfg.ReadStallGcsRetriesConfig{ - Enable: false, + Enable: true, MinReqTimeout: 1500 * time.Millisecond, MaxReqTimeout: 1200 * time.Second, InitialReqTimeout: 20 * time.Second, @@ -721,8 +855,15 @@ func TestValidateConfigFile_ReadStallConfigSuccessful(t *testing.T) { configFile: "testdata/valid_config.yaml", expectedConfig: &cfg.Config{ GcsRetries: cfg.GcsRetriesConfig{ + ExperimentalNonrapidFolderApiStallRetry: true, + ChunkRetryDeadlineSecs: 180, + ChunkTransferTimeoutSecs: 20, + EnableMountRetries: false, + MaxRetryAttempts: math.MaxInt, + MaxRetrySleep: 30 * time.Second, + Multiplier: 2, ReadStall: cfg.ReadStallGcsRetriesConfig{ - Enable: true, + Enable: false, MinReqTimeout: 10 * time.Second, MaxReqTimeout: 200 * time.Second, InitialReqTimeout: 20 * time.Second, @@ -733,13 +874,12 @@ func TestValidateConfigFile_ReadStallConfigSuccessful(t *testing.T) { }, }, } - - for _, tc := range testCases { + for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { gotConfig, err := getConfigObjectWithConfigFile(t, tc.configFile) if assert.NoError(t, err) { - assert.EqualValues(t, tc.expectedConfig.GcsRetries.ReadStall, gotConfig.GcsRetries.ReadStall) + assert.EqualValues(t, tc.expectedConfig.GcsRetries, gotConfig.GcsRetries) } }) } @@ -797,6 +937,9 @@ func TestValidateConfigFile_MetricsConfigSuccessful(t *testing.T) { StackdriverExportInterval: 0, CloudMetricsExportIntervalSecs: 0, PrometheusPort: 0, + Workers: 3, + BufferSize: 256, + ExperimentalEnableGrpcMetrics: true, }, }, { @@ -804,6 +947,9 @@ func TestValidateConfigFile_MetricsConfigSuccessful(t *testing.T) { configFile: "testdata/valid_config.yaml", expectedConfig: &cfg.MetricsConfig{ CloudMetricsExportIntervalSecs: 10, + Workers: 10, + BufferSize: 128, + ExperimentalEnableGrpcMetrics: true, }, }, } @@ -838,3 +984,36 @@ func TestValidateConfigFile_MetricsConfigInvalid(t *testing.T) { }) } } + +func TestValidateConfigFile_MachineTypeConfig(t *testing.T) { + testCases := []struct { + name string + configFile string + expectedConfig *cfg.Config + }{ + { + name: "set_machine_type_in_config_file", + configFile: "testdata/valid_config.yaml", + expectedConfig: &cfg.Config{ + MachineType: "config-file-machine-type", + }, + }, + { + name: "unset_machine_type", + configFile: "testdata/unset_machine_type.yaml", + expectedConfig: &cfg.Config{ + MachineType: "", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gotConfig, err := getConfigObjectWithConfigFile(t, tc.configFile) + + if assert.NoError(t, err) { + assert.Equal(t, tc.expectedConfig.MachineType, gotConfig.MachineType) + } + }) + } +} diff --git a/cmd/datatypes_parsing_test.go b/cmd/datatypes_parsing_test.go index 0561011f23..599e888ee0 100644 --- a/cmd/datatypes_parsing_test.go +++ b/cmd/datatypes_parsing_test.go @@ -19,7 +19,7 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -569,13 +569,27 @@ func TestCLIFlagPassing(t *testing.T) { assert.Equal(t, cfg.ResolvedPath(""), c.CacheDir) }, }, + { + name: "profile1", + args: []string{"--profile", cfg.ProfileAIMLTraining}, + testFn: func(t *testing.T, c *cfg.Config) { + assert.Equal(t, cfg.ProfileAIMLTraining, c.Profile) + }, + }, + { + name: "socketaddress1", + args: []string{"--experimental-local-socket-address", "127.0.0.1"}, + testFn: func(t *testing.T, c *cfg.Config) { + assert.Equal(t, "127.0.0.1", c.GcsConnection.ExperimentalLocalSocketAddress) + }, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var c *cfg.Config - command, err := NewRootCmd(func(config *cfg.Config, _, _ string) error { - c = config + command, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + c = mountInfo.config return nil }) require.NoError(t, err) @@ -747,8 +761,8 @@ func TestConfigPassing(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var c *cfg.Config - command, err := NewRootCmd(func(config *cfg.Config, _, _ string) error { - c = config + command, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + c = mountInfo.config return nil }) require.NoError(t, err) @@ -802,7 +816,7 @@ func TestPredefinedFlagThrowNoError(t *testing.T) { } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - command, err := NewRootCmd(func(config *cfg.Config, _, _ string) error { + command, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { return nil }) require.NoError(t, err) diff --git a/cmd/legacy_main.go b/cmd/legacy_main.go index 6c854d5561..46dc69360b 100644 --- a/cmd/legacy_main.go +++ b/cmd/legacy_main.go @@ -20,6 +20,7 @@ package cmd import ( + "errors" "fmt" "io/fs" "os" @@ -27,41 +28,48 @@ import ( "path" "path/filepath" "strings" + "time" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" "golang.org/x/sys/unix" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/common" - "github.com/googlecloudplatform/gcsfuse/v2/internal/canned" - "github.com/googlecloudplatform/gcsfuse/v2/internal/locker" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" - "github.com/googlecloudplatform/gcsfuse/v2/internal/monitor" - "github.com/googlecloudplatform/gcsfuse/v2/internal/mount" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" - "github.com/googlecloudplatform/gcsfuse/v2/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/common" + "github.com/googlecloudplatform/gcsfuse/v3/internal/canned" + "github.com/googlecloudplatform/gcsfuse/v3/internal/kernelparams" + "github.com/googlecloudplatform/gcsfuse/v3/internal/locker" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/monitor" + "github.com/googlecloudplatform/gcsfuse/v3/internal/mount" + "github.com/googlecloudplatform/gcsfuse/v3/internal/profiler" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" "github.com/jacobsa/daemonize" "github.com/jacobsa/fuse" "github.com/kardianos/osext" + "github.com/spf13/viper" "golang.org/x/net/context" ) const ( SuccessfulMountMessage = "File system has been successfully mounted." UnsuccessfulMountMessagePrefix = "Error while mounting gcsfuse" + MountSlownessMessage = "Mount slowness detected: mount time %v exceeded threshold %v" + DynamicMountFSName = "gcsfuse" + WaitTimeOnSignalReceive = 30 * time.Second + MountTimeThreshold = 8 * time.Second ) //////////////////////////////////////////////////////////////////////// // Helpers //////////////////////////////////////////////////////////////////////// -func registerTerminatingSignalHandler(mountPoint string, c *cfg.Config) { +func registerTerminatingSignalHandler(mountPoint string) { // Register for SIGINT. signalChan := make(chan os.Signal, 1) - signal.Notify(signalChan, os.Interrupt) - if c.FileSystem.HandleSigterm { - signal.Notify(signalChan, unix.SIGTERM) - } + signal.Notify(signalChan, os.Interrupt, unix.SIGTERM) // Start a goroutine that will unmount when the signal is received. go func() { @@ -74,10 +82,24 @@ func registerTerminatingSignalHandler(mountPoint string, c *cfg.Config) { case os.Interrupt: sigName = "SIGINT" } - logger.Infof("Received %s, attempting to unmount...", sigName) + //On signal receive wait in background and give 30 second for unmount to finish + //and then exit, so application is closed. + go func() { + logger.Warnf("Received %s, waiting for %s to let system gracefully unmount before killing the process", sigName, WaitTimeOnSignalReceive) + time.Sleep(WaitTimeOnSignalReceive) + logger.Warnf("killing goroutines and exit") + //Forcefully exit to 0 so that caller get success on forcefull exit also. + os.Exit(0) + }() + + logger.Warnf("Received %s, attempting to unmount...", sigName) err := fuse.Unmount(mountPoint) if err != nil { + if errors.Is(err, fuse.ErrExternallyManagedMountPoint) { + logger.Infof("Mount point %s is externally managed; gcsfuse will not unmount it.", mountPoint) + return + } logger.Errorf("Failed to unmount in response to %s: %v", sigName, err) } else { logger.Infof("Successfully unmounted in response to %s.", sigName) @@ -87,56 +109,72 @@ func registerTerminatingSignalHandler(mountPoint string, c *cfg.Config) { }() } -func getUserAgent(appName string, config string) string { +func getUserAgent(appName, config, mountInstanceID string) string { + var userAgent string gcsfuseMetadataImageType := os.Getenv("GCSFUSE_METADATA_IMAGE_TYPE") if len(gcsfuseMetadataImageType) > 0 { - userAgent := fmt.Sprintf("gcsfuse/%s %s (GPN:gcsfuse-%s) (Cfg:%s)", common.GetVersion(), appName, gcsfuseMetadataImageType, config) - return strings.Join(strings.Fields(userAgent), " ") + userAgent = fmt.Sprintf("gcsfuse/%s %s (GPN:gcsfuse-%s) (Cfg:%s)", common.GetVersion(), appName, gcsfuseMetadataImageType, config) + userAgent = strings.Join(strings.Fields(userAgent), " ") } else if len(appName) > 0 { - return fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-%s) (Cfg:%s)", common.GetVersion(), appName, config) + userAgent = fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-%s) (Cfg:%s)", common.GetVersion(), appName, config) } else { - return fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse) (Cfg:%s)", common.GetVersion(), config) + userAgent = fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse) (Cfg:%s)", common.GetVersion(), config) } + return fmt.Sprintf("%s (mount-id:%s)", userAgent, mountInstanceID) } -func getConfigForUserAgent(mountConfig *cfg.Config) string { - // Minimum configuration details created in a bitset fashion. Right now, its restricted only to File Cache Settings. - isFileCacheEnabled := "0" - if cfg.IsFileCacheEnabled(mountConfig) { - isFileCacheEnabled = "1" - } - isFileCacheForRangeReadEnabled := "0" - if mountConfig.FileCache.CacheFileForRangeRead { - isFileCacheForRangeReadEnabled = "1" +func boolToBin(b bool) string { + if b { + return "1" } - isParallelDownloadsEnabled := "0" - if cfg.IsParallelDownloadsEnabled(mountConfig) { - isParallelDownloadsEnabled = "1" + return "0" +} + +func getConfigForUserAgent(mountConfig *cfg.Config) string { + // Minimum configuration details created in a bitset fashion. + parts := []string{ + boolToBin(cfg.IsFileCacheEnabled(mountConfig)), + boolToBin(mountConfig.FileCache.CacheFileForRangeRead), + boolToBin(cfg.IsParallelDownloadsEnabled(mountConfig)), + boolToBin(mountConfig.Write.EnableStreamingWrites), + boolToBin(mountConfig.Read.EnableBufferedRead), + boolToBin(mountConfig.Profile != ""), } - return fmt.Sprintf("%s:%s:%s", isFileCacheEnabled, isFileCacheForRangeReadEnabled, isParallelDownloadsEnabled) + return strings.Join(parts, ":") } -func createStorageHandle(newConfig *cfg.Config, userAgent string) (storageHandle storage.StorageHandle, err error) { +func createStorageHandle(newConfig *cfg.Config, userAgent string, metricHandle metrics.MetricHandle, isGKE bool) (storageHandle storage.StorageHandle, err error) { storageClientConfig := storageutil.StorageClientConfig{ - ClientProtocol: newConfig.GcsConnection.ClientProtocol, - MaxConnsPerHost: int(newConfig.GcsConnection.MaxConnsPerHost), - MaxIdleConnsPerHost: int(newConfig.GcsConnection.MaxIdleConnsPerHost), - HttpClientTimeout: newConfig.GcsConnection.HttpClientTimeout, - MaxRetrySleep: newConfig.GcsRetries.MaxRetrySleep, - MaxRetryAttempts: int(newConfig.GcsRetries.MaxRetryAttempts), - RetryMultiplier: newConfig.GcsRetries.Multiplier, - UserAgent: userAgent, - CustomEndpoint: newConfig.GcsConnection.CustomEndpoint, - KeyFile: string(newConfig.GcsAuth.KeyFile), - AnonymousAccess: newConfig.GcsAuth.AnonymousAccess, - TokenUrl: newConfig.GcsAuth.TokenUrl, - ReuseTokenFromUrl: newConfig.GcsAuth.ReuseTokenFromUrl, - ExperimentalEnableJsonRead: newConfig.GcsConnection.ExperimentalEnableJsonRead, - GrpcConnPoolSize: int(newConfig.GcsConnection.GrpcConnPoolSize), - EnableHNS: newConfig.EnableHns, - ReadStallRetryConfig: newConfig.GcsRetries.ReadStall, + ClientProtocol: newConfig.GcsConnection.ClientProtocol, + MaxConnsPerHost: int(newConfig.GcsConnection.MaxConnsPerHost), + MaxIdleConnsPerHost: int(newConfig.GcsConnection.MaxIdleConnsPerHost), + HttpClientTimeout: newConfig.GcsConnection.HttpClientTimeout, + MaxRetrySleep: newConfig.GcsRetries.MaxRetrySleep, + MaxRetryAttempts: int(newConfig.GcsRetries.MaxRetryAttempts), + RetryMultiplier: newConfig.GcsRetries.Multiplier, + UserAgent: userAgent, + CustomEndpoint: newConfig.GcsConnection.CustomEndpoint, + KeyFile: string(newConfig.GcsAuth.KeyFile), + AnonymousAccess: newConfig.GcsAuth.AnonymousAccess, + TokenUrl: newConfig.GcsAuth.TokenUrl, + ReuseTokenFromUrl: newConfig.GcsAuth.ReuseTokenFromUrl, + ExperimentalNonrapidFolderApiStallRetry: newConfig.GcsRetries.ExperimentalNonrapidFolderApiStallRetry, + ExperimentalEnableJsonRead: newConfig.GcsConnection.ExperimentalEnableJsonRead, + GrpcConnPoolSize: int(newConfig.GcsConnection.GrpcConnPoolSize), + GrpcPathStrategy: newConfig.GcsConnection.GrpcPathStrategy, + EnableHNS: newConfig.EnableHns, + EnableGoogleLibAuth: newConfig.EnableGoogleLibAuth, + ReadStallRetryConfig: newConfig.GcsRetries.ReadStall, + MetricHandle: metricHandle, + ExperimentalEnablePirlo: newConfig.FileSystem.ExperimentalEnablePirlo, + TracingEnabled: cfg.IsTracingEnabled(newConfig), + EnableHTTPDNSCache: newConfig.GcsConnection.EnableHttpDnsCache, + LocalSocketAddress: newConfig.GcsConnection.ExperimentalLocalSocketAddress, + EnableGrpcMetrics: newConfig.Metrics.ExperimentalEnableGrpcMetrics, + IsGKE: isGKE, + WriteConfig: &newConfig.Write, } logger.Infof("UserAgent = %s\n", storageClientConfig.UserAgent) - storageHandle, err = storage.NewStorageHandle(context.Background(), storageClientConfig) + storageHandle, err = storage.NewStorageHandle(context.Background(), storageClientConfig, newConfig.GcsConnection.BillingProject) return } @@ -145,7 +183,7 @@ func createStorageHandle(newConfig *cfg.Config, userAgent string) (storageHandle //////////////////////////////////////////////////////////////////////// // Mount the file system according to arguments in the supplied context. -func mountWithArgs(bucketName string, mountPoint string, newConfig *cfg.Config) (mfs *fuse.MountedFileSystem, err error) { +func mountWithArgs(bucketName string, mountPoint string, newConfig *cfg.Config, metricHandle metrics.MetricHandle, traceHandle tracing.TraceHandle, viperConfig *viper.Viper) (mfs *fuse.MountedFileSystem, err error) { // Enable invariant checking if requested. if newConfig.Debug.ExitOnInvariantViolation { locker.EnableInvariantsCheck() @@ -153,6 +191,8 @@ func mountWithArgs(bucketName string, mountPoint string, newConfig *cfg.Config) if newConfig.Debug.LogMutex { locker.EnableDebugMessages() } + // Parse the mountPoint string and detect whether or not in GKE environment + isGKE := cfg.IsGKEEnvironment(mountPoint) // Grab the connection. // @@ -160,9 +200,9 @@ func mountWithArgs(bucketName string, mountPoint string, newConfig *cfg.Config) // connection. var storageHandle storage.StorageHandle if bucketName != canned.FakeBucketName { - userAgent := getUserAgent(newConfig.AppName, getConfigForUserAgent(newConfig)) + userAgent := getUserAgent(newConfig.AppName, getConfigForUserAgent(newConfig), logger.MountInstanceID(fsName(bucketName))) logger.Info("Creating Storage handle...") - storageHandle, err = createStorageHandle(newConfig, userAgent) + storageHandle, err = createStorageHandle(newConfig, userAgent, metricHandle, isGKE) if err != nil { err = fmt.Errorf("failed to create storage handle using createStorageHandle: %w", err) return @@ -176,7 +216,10 @@ func mountWithArgs(bucketName string, mountPoint string, newConfig *cfg.Config) bucketName, mountPoint, newConfig, - storageHandle) + storageHandle, + metricHandle, + traceHandle, + viperConfig) if err != nil { err = fmt.Errorf("mountWithStorageHandle: %w", err) @@ -247,15 +290,106 @@ func isDynamicMount(bucketName string) bool { return bucketName == "" || bucketName == "_" } -func Mount(newConfig *cfg.Config, bucketName, mountPoint string) (err error) { - // Ideally this call to SetLogFormat (which internally creates a new defaultLogger) +func fsName(bucketName string) string { + if isDynamicMount(bucketName) { + return DynamicMountFSName + } + return bucketName +} + +// forwardedEnvVars collects and returns all the environment +// variables which should be sent to the gcsfuse daemon +// process in case of background run. +func forwardedEnvVars() []string { + // Pass along PATH so that the daemon can find fusermount on Linux. + env := []string{ + fmt.Sprintf("PATH=%s", os.Getenv("PATH")), + } + + // Pass through the https_proxy/http_proxy environment variable, + // in case the host requires a proxy server to reach the GCS endpoint. + // https_proxy has precedence over http_proxy, in case both are set + if p, ok := os.LookupEnv("https_proxy"); ok { + env = append(env, fmt.Sprintf("https_proxy=%s", p)) + fmt.Fprintf( + os.Stdout, + "Added environment https_proxy: %s\n", + p) + } else if p, ok := os.LookupEnv("http_proxy"); ok { + env = append(env, fmt.Sprintf("http_proxy=%s", p)) + fmt.Fprintf( + os.Stdout, + "Added environment http_proxy: %s\n", + p) + } + + // Forward GOOGLE_APPLICATION_CREDENTIALS, since we document in + // mounting.md that it can be used for specifying a key file. + // Forward the no_proxy environment variable. Whenever + // using the http(s)_proxy environment variables. This should + // also be included to know for which hosts the use of proxies + // should be ignored. + // Forward GCE_METADATA_HOST, GCE_METADATA_ROOT, GCE_METADATA_IP as these are used for mocked metadata services. + // Forward GRPC_GO_LOG_VERBOSITY_LEVEL and GRPC_GO_LOG_SEVERITY_LEVEL as these are used to enable grpc debug logs. + for _, envvar := range []string{"GOOGLE_APPLICATION_CREDENTIALS", "no_proxy", "GCE_METADATA_HOST", "GCE_METADATA_ROOT", "GCE_METADATA_IP", "GRPC_GO_LOG_VERBOSITY_LEVEL", "GRPC_GO_LOG_SEVERITY_LEVEL"} { + if envval, ok := os.LookupEnv(envvar); ok { + env = append(env, fmt.Sprintf("%s=%s", envvar, envval)) + fmt.Fprintf( + os.Stdout, + "Added environment %s: %s\n", + envvar, envval) + } + } + + // Pass the parent process working directory to child process via + // environment variable. This variable will be used to resolve relative paths. + if parentProcessExecutionDir, err := os.Getwd(); err == nil { + env = append(env, fmt.Sprintf("%s=%s", util.GCSFUSE_PARENT_PROCESS_DIR, + parentProcessExecutionDir)) + } + + // Here, parent process doesn't pass the $HOME to child process implicitly, + // hence we need to pass it explicitly. + if homeDir, err := os.UserHomeDir(); err == nil { + env = append(env, fmt.Sprintf("HOME=%s", homeDir)) + } + + // This environment variable will be helpful to distinguish b/w the main + // process and daemon process. If this environment variable set that means + // programme is running as daemon process. + env = append(env, fmt.Sprintf("%s=true", logger.GCSFuseInBackgroundMode)) + + // This environment variable is used to enhance gcsfuse logging by using unique + // MountUUID to identify logs from different mounts. + // MountUUID is used here instead of the MountInstanceID for unified logic + // in callers of MountInstaceID in both background and foreground mode. + env = append(env, fmt.Sprintf("%s=%s", logger.MountUUIDEnvKey, logger.MountUUID())) + return env +} + +// logGCSFuseMountInformation logs the CLI flags, config file flags and the resolved config. +func logGCSFuseMountInformation(mountInfo *mountInfo) { + logger.Info("GCSFuse Config", "CLI Flags", mountInfo.cliFlags) + if mountInfo.configFileFlags != nil { + logger.Info("GCSFuse Config", "ConfigFile Flags", mountInfo.configFileFlags) + } + if len(mountInfo.optimizedFlags) > 0 { + logger.Info("GCSFuse Config", "Optimized Flags", mountInfo.optimizedFlags) + } + logger.Info("GCSFuse Config", "Full Config", mountInfo.config) +} + +func Mount(mountInfo *mountInfo, bucketName, mountPoint string) (err error) { + newConfig := mountInfo.config + // Ideally this call to UpdateDefaultLogger (which internally creates a + // new defaultLogger with user provided log-format and custom attribute 'fsName-MountInstanceID') // should be set as an else to the 'if flags.Foreground' check below, but currently // that means the logs generated by resolveConfigFilePaths below don't honour // the user-provided log-format. - logger.SetLogFormat(newConfig.Logging.Format) + logger.UpdateDefaultLogger(newConfig.Logging.Format, fsName(bucketName)) if newConfig.Foreground { - err = logger.InitLogFile(newConfig.Logging) + err = logger.InitLogFile(newConfig.Logging, fsName(bucketName)) if err != nil { return fmt.Errorf("init log file: %w", err) } @@ -269,7 +403,7 @@ func Mount(newConfig *cfg.Config, bucketName, mountPoint string) (err error) { // if these are already being logged into a log-file, otherwise // there will be duplicate logs for these in both places (stdout and log-file). if newConfig.Foreground || newConfig.Logging.FilePath == "" { - logger.Info("GCSFuse config", "config", newConfig) + logGCSFuseMountInformation(mountInfo) } // The following will not warn if the user explicitly passed the default value for StatCacheCapacity. @@ -282,6 +416,10 @@ func Mount(newConfig *cfg.Config, bucketName, mountPoint string) (err error) { logger.Warnf("Deprecated flag stat-cache-ttl and/or type-cache-ttl used! Please switch to config parameter 'metadata-cache: ttl-secs' .") } + if newConfig.EnableTypeCacheDeprecation && (newConfig.MetadataCache.TypeCacheMaxSizeMb != mount.DefaultTypeCacheSizeMB || newConfig.MetadataCache.EnableNonexistentTypeCache) { + logger.Warnf("Type cache is deprecated. The flags 'type-cache-max-size-mb' and 'enable-nonexistent-type-cache' will be ignored. Please use 'stat-cache-max-size-mb' and 'metadata-cache-negative-ttl-secs' instead.") + } + // If we haven't been asked to run in foreground mode, we should run a daemon // with the foreground flag set and wait for it to mount. if !newConfig.Foreground { @@ -298,61 +436,7 @@ func Mount(newConfig *cfg.Config, bucketName, mountPoint string) (err error) { args := append([]string{"--foreground"}, os.Args[1:]...) args[len(args)-1] = mountPoint - // Pass along PATH so that the daemon can find fusermount on Linux. - env := []string{ - fmt.Sprintf("PATH=%s", os.Getenv("PATH")), - } - - // Pass along GOOGLE_APPLICATION_CREDENTIALS, since we document in - // mounting.md that it can be used for specifying a key file. - if p, ok := os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS"); ok { - env = append(env, fmt.Sprintf("GOOGLE_APPLICATION_CREDENTIALS=%s", p)) - } - // Pass through the https_proxy/http_proxy environment variable, - // in case the host requires a proxy server to reach the GCS endpoint. - // https_proxy has precedence over http_proxy, in case both are set - if p, ok := os.LookupEnv("https_proxy"); ok { - env = append(env, fmt.Sprintf("https_proxy=%s", p)) - fmt.Fprintf( - os.Stdout, - "Added environment https_proxy: %s\n", - p) - } else if p, ok := os.LookupEnv("http_proxy"); ok { - env = append(env, fmt.Sprintf("http_proxy=%s", p)) - fmt.Fprintf( - os.Stdout, - "Added environment http_proxy: %s\n", - p) - } - // Pass through the no_proxy environment variable. Whenever - // using the http(s)_proxy environment variables. This should - // also be included to know for which hosts the use of proxies - // should be ignored. - if p, ok := os.LookupEnv("no_proxy"); ok { - env = append(env, fmt.Sprintf("no_proxy=%s", p)) - fmt.Fprintf( - os.Stdout, - "Added environment no_proxy: %s\n", - p) - } - - // Pass the parent process working directory to child process via - // environment variable. This variable will be used to resolve relative paths. - if parentProcessExecutionDir, err := os.Getwd(); err == nil { - env = append(env, fmt.Sprintf("%s=%s", util.GCSFUSE_PARENT_PROCESS_DIR, - parentProcessExecutionDir)) - } - - // Here, parent process doesn't pass the $HOME to child process implicitly, - // hence we need to pass it explicitly. - if homeDir, _ := os.UserHomeDir(); err == nil { - env = append(env, fmt.Sprintf("HOME=%s", homeDir)) - } - - // This environment variable will be helpful to distinguish b/w the main - // process and daemon process. If this environment variable set that means - // programme is running as daemon process. - env = append(env, fmt.Sprintf("%s=true", logger.GCSFuseInBackgroundMode)) + env := forwardedEnvVars() // logfile.stderr will capture the standard error (stderr) output of the gcsfuse background process. var stderrFile *os.File @@ -363,29 +447,47 @@ func Mount(newConfig *cfg.Config, bucketName, mountPoint string) (err error) { } } // Run. + startTime := time.Now() err = daemonize.Run(path, args, env, os.Stdout, stderrFile) if err != nil { return fmt.Errorf("daemonize.Run: %w", err) } + mountDuration := time.Since(startTime) + if mountDuration > MountTimeThreshold { + logger.Warnf(MountSlownessMessage, mountDuration, MountTimeThreshold) + } logger.Infof(SuccessfulMountMessage) return err } ctx := context.Background() var metricExporterShutdownFn common.ShutdownFn - if newConfig.Metrics.EnableOtel { - metricExporterShutdownFn = monitor.SetupOTelMetricExporters(ctx, newConfig) - } else { - metricExporterShutdownFn = monitor.SetupOpenCensusExporters(newConfig) + metricHandle := metrics.NewNoopMetrics() + if cfg.IsMetricsEnabled(&newConfig.Metrics) { + metricExporterShutdownFn = monitor.SetupOTelMetricExporters(ctx, newConfig, logger.MountInstanceID(fsName(bucketName))) + if metricHandle, err = metrics.NewOTelMetrics(ctx, int(newConfig.Metrics.Workers), int(newConfig.Metrics.BufferSize)); err != nil { + metricHandle = metrics.NewNoopMetrics() + } } - shutdownTracingFn := monitor.SetupTracing(ctx, newConfig) + shutdownTracingFn := monitor.SetupTracing(ctx, newConfig, logger.MountInstanceID(fsName(bucketName))) + traceHandle := tracing.NewNoopTracer() + if cfg.IsTracingEnabled(newConfig) { + traceHandle = tracing.NewOTELTracer() + } + shutdownFn := common.JoinShutdownFunc(metricExporterShutdownFn, shutdownTracingFn) + // No-op if profiler is disabled. + if err := profiler.SetupCloudProfiler(&newConfig.CloudProfiler); err != nil { + logger.Warnf("Failed to setup cloud profiler: %v", err) + } + // Mount, writing information about our progress to the writer that package // daemonize gives us and telling it about the outcome. var mfs *fuse.MountedFileSystem { - mfs, err = mountWithArgs(bucketName, mountPoint, newConfig) + startTime := time.Now() + mfs, err = mountWithArgs(bucketName, mountPoint, newConfig, metricHandle, traceHandle, mountInfo.viperConfig) // This utility is to absorb the error // returned by daemonize.SignalOutcome calls by simply @@ -397,6 +499,10 @@ func Mount(newConfig *cfg.Config, bucketName, mountPoint string) (err error) { } markSuccessfulMount := func() { + mountDuration := time.Since(startTime) + if mountDuration > MountTimeThreshold { + logger.Warnf(MountSlownessMessage, mountDuration, MountTimeThreshold) + } // Print the success message in the log-file/stdout depending on what the logger is set to. logger.Info(SuccessfulMountMessage) callDaemonizeSignalOutcome(nil) @@ -431,10 +537,19 @@ func Mount(newConfig *cfg.Config, bucketName, mountPoint string) (err error) { } } markSuccessfulMount() + + // Apply post mount kernel settings in non-GKE environments for non dynamic mounts when kernel reader is enabled. + if !isDynamicMount(bucketName) && !cfg.IsGKEEnvironment(mountPoint) && newConfig.FileSystem.EnableKernelReader { + kernelparams := kernelparams.NewKernelParamsManager() + kernelparams.SetReadAheadKb(int(newConfig.FileSystem.MaxReadAheadKb)) + kernelparams.SetCongestionWindowThreshold(int(newConfig.FileSystem.CongestionThreshold)) + kernelparams.SetMaxBackgroundRequests(int(newConfig.FileSystem.MaxBackground)) + kernelparams.ApplyNonGKE(mountPoint) + } } // Let the user unmount with Ctrl-C (SIGINT). - registerTerminatingSignalHandler(mfs.Dir(), newConfig) + registerTerminatingSignalHandler(mfs.Dir()) // Wait for the file system to be unmounted. if err = mfs.Join(ctx); err != nil { @@ -443,7 +558,7 @@ func Mount(newConfig *cfg.Config, bucketName, mountPoint string) (err error) { if shutdownFn != nil { if shutdownErr := shutdownFn(ctx); shutdownErr != nil { - logger.Errorf("Error while shutting down trace exporter: %v", shutdownErr) + logger.Errorf("Error while shutting down dependencies: %v", shutdownErr) } } diff --git a/cmd/legacy_main_test.go b/cmd/legacy_main_test.go index 970f1f5d37..cd8711fb67 100644 --- a/cmd/legacy_main_test.go +++ b/cmd/legacy_main_test.go @@ -20,9 +20,13 @@ import ( "strings" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/common" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/common" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -41,13 +45,13 @@ type MainTest struct { func (t *MainTest) TestCreateStorageHandle() { newConfig := &cfg.Config{ GcsConnection: cfg.GcsConnectionConfig{ClientProtocol: cfg.HTTP1}, - GcsAuth: cfg.GcsAuthConfig{KeyFile: "testdata/test_creds.json"}} + GcsAuth: cfg.GcsAuthConfig{KeyFile: "testdata/test_creds.json"}, + } - userAgent := "AppName" - storageHandle, err := createStorageHandle(newConfig, userAgent) + storageHandle, err := createStorageHandle(newConfig, "AppName", metrics.NewNoopMetrics(), false) - assert.Equal(t.T(), nil, err) - assert.NotEqual(t.T(), nil, storageHandle) + assert.Nil(t.T(), err) + assert.NotNil(t.T(), storageHandle) } func (t *MainTest) TestCreateStorageHandle_WithClientProtocolAsGRPC() { @@ -56,36 +60,49 @@ func (t *MainTest) TestCreateStorageHandle_WithClientProtocolAsGRPC() { GcsAuth: cfg.GcsAuthConfig{KeyFile: "testdata/test_creds.json"}, } - userAgent := "AppName" - storageHandle, err := createStorageHandle(newConfig, userAgent) + storageHandle, err := createStorageHandle(newConfig, "AppName", metrics.NewNoopMetrics(), false) - assert.Equal(t.T(), nil, err) - assert.NotEqual(t.T(), nil, storageHandle) + assert.Nil(t.T(), err) + assert.NotNil(t.T(), storageHandle) } -func (t *MainTest) TestGetUserAgentWhenMetadataImageTypeEnvVarIsSet() { - os.Setenv("GCSFUSE_METADATA_IMAGE_TYPE", "DLVM") - defer os.Unsetenv("GCSFUSE_METADATA_IMAGE_TYPE") +func (t *MainTest) TestCreateStorageHandle_WithClientProtocolAsGRPCIsGKE() { + newConfig := &cfg.Config{ + GcsConnection: cfg.GcsConnectionConfig{ClientProtocol: cfg.GRPC}, + GcsAuth: cfg.GcsAuthConfig{KeyFile: "testdata/test_creds.json"}, + } + storageHandle, err := createStorageHandle(newConfig, "AppName", metrics.NewNoopMetrics(), true) + + assert.Nil(t.T(), err) + assert.NotNil(t.T(), storageHandle) +} + +func (t *MainTest) TestGetUserAgentWhenMetadataImageTypeEnvVarIsSet() { + t.T().Setenv("GCSFUSE_METADATA_IMAGE_TYPE", "DLVM") mountConfig := &cfg.Config{} - userAgent := getUserAgent("AppName", getConfigForUserAgent(mountConfig)) - expectedUserAgent := strings.TrimSpace(fmt.Sprintf("gcsfuse/%s AppName (GPN:gcsfuse-DLVM) (Cfg:0:0:0)", common.GetVersion())) + userAgent := getUserAgent("AppName", getConfigForUserAgent(mountConfig), "testFS-123") + + expectedUserAgent := strings.TrimSpace(fmt.Sprintf("gcsfuse/%s AppName (GPN:gcsfuse-DLVM) (Cfg:0:0:0:0:0:0) (mount-id:testFS-123)", common.GetVersion())) assert.Equal(t.T(), expectedUserAgent, userAgent) } func (t *MainTest) TestGetUserAgentWhenMetadataImageTypeEnvVarIsNotSet() { mountConfig := &cfg.Config{} - userAgent := getUserAgent("AppName", getConfigForUserAgent(mountConfig)) - expectedUserAgent := strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:0:0:0)", common.GetVersion())) + userAgent := getUserAgent("AppName", getConfigForUserAgent(mountConfig), "testFS-123") + + expectedUserAgent := strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:0:0:0:0:0:0) (mount-id:testFS-123)", common.GetVersion())) assert.Equal(t.T(), expectedUserAgent, userAgent) } func (t *MainTest) TestGetUserAgentConfigWithNoFileCache() { mountConfig := &cfg.Config{} - userAgent := getUserAgent("AppName", getConfigForUserAgent(mountConfig)) - expectedUserAgent := strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:0:0:0)", common.GetVersion())) + + userAgent := getUserAgent("AppName", getConfigForUserAgent(mountConfig), "testFS-123") + + expectedUserAgent := strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:0:0:0:0:0:0) (mount-id:testFS-123)", common.GetVersion())) assert.Equal(t.T(), expectedUserAgent, userAgent) } @@ -103,7 +120,7 @@ func (t *MainTest) TestGetUserAgentConfig() { MaxSizeMb: 0, }, }, - expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:0:0:0)", common.GetVersion())), + expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:0:0:0:0:0:0) (mount-id:testFS-123)", common.GetVersion())), }, { name: "Config with file cache disabled where maxsize is set but cache dir is not set.", @@ -112,7 +129,7 @@ func (t *MainTest) TestGetUserAgentConfig() { MaxSizeMb: -1, }, }, - expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:0:0:0)", common.GetVersion())), + expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:0:0:0:0:0:0) (mount-id:testFS-123)", common.GetVersion())), }, { name: "Config with file cache enabled but random read disabled.", @@ -122,7 +139,7 @@ func (t *MainTest) TestGetUserAgentConfig() { MaxSizeMb: -1, }, }, - expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:1:0:0)", common.GetVersion())), + expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:1:0:0:0:0:0) (mount-id:testFS-123)", common.GetVersion())), }, { name: "Config with file cache and random read enabled.", @@ -133,7 +150,7 @@ func (t *MainTest) TestGetUserAgentConfig() { CacheFileForRangeRead: true, }, }, - expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:1:1:0)", common.GetVersion())), + expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:1:1:0:0:0:0) (mount-id:testFS-123)", common.GetVersion())), }, { name: "Config with file cache disabled and enable parallel downloads set.", @@ -144,7 +161,7 @@ func (t *MainTest) TestGetUserAgentConfig() { EnableParallelDownloads: true, }, }, - expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:0:0:0)", common.GetVersion())), + expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:0:0:0:0:0:0) (mount-id:testFS-123)", common.GetVersion())), }, { name: "Config with file cache and parallel downloads enabled.", @@ -155,7 +172,7 @@ func (t *MainTest) TestGetUserAgentConfig() { EnableParallelDownloads: true, }, }, - expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:1:0:1)", common.GetVersion())), + expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:1:0:1:0:0:0) (mount-id:testFS-123)", common.GetVersion())), }, { name: "Config with file cache, random reads and parallel downloads enabled.", @@ -167,25 +184,97 @@ func (t *MainTest) TestGetUserAgentConfig() { EnableParallelDownloads: true, }, }, - expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:1:1:1)", common.GetVersion())), + expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:1:1:1:0:0:0) (mount-id:testFS-123)", common.GetVersion())), + }, + { + name: "streaming_writes_enabled", + mountConfig: &cfg.Config{ + CacheDir: "/cache/path", + FileCache: cfg.FileCacheConfig{ + MaxSizeMb: -1, + CacheFileForRangeRead: false, + EnableParallelDownloads: true, + }, + Write: cfg.WriteConfig{EnableStreamingWrites: true}, + }, + expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:1:0:1:1:0:0) (mount-id:testFS-123)", common.GetVersion())), + }, + { + name: "streaming_writes_disabled", + mountConfig: &cfg.Config{ + CacheDir: "/cache/path", + FileCache: cfg.FileCacheConfig{ + MaxSizeMb: -1, + CacheFileForRangeRead: true, + EnableParallelDownloads: false, + }, + Write: cfg.WriteConfig{EnableStreamingWrites: false}, + }, + expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:1:1:0:0:0:0) (mount-id:testFS-123)", common.GetVersion())), + }, + { + name: "buffered_read_enabled", + mountConfig: &cfg.Config{ + Read: cfg.ReadConfig{EnableBufferedRead: true}, + }, + expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:0:0:0:0:1:0) (mount-id:testFS-123)", common.GetVersion())), + }, + { + name: "buffered_read_disabled", + mountConfig: &cfg.Config{ + Read: cfg.ReadConfig{EnableBufferedRead: false}, + }, + expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:0:0:0:0:0:0) (mount-id:testFS-123)", common.GetVersion())), + }, + { + name: "file_cache_enabled_and_buffered_read_enabled", + mountConfig: &cfg.Config{ + CacheDir: "/cache/path", + FileCache: cfg.FileCacheConfig{MaxSizeMb: -1}, + Read: cfg.ReadConfig{EnableBufferedRead: true}, + }, + // Note: getConfigForUserAgent runs before config rationalization, which + // would disable buffered-read when file-cache is enabled. + expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:1:0:0:0:1:0) (mount-id:testFS-123)", common.GetVersion())), + }, + { + name: "profile_enabled_aiml_training", + mountConfig: &cfg.Config{ + Profile: cfg.ProfileAIMLTraining, + }, + expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:0:0:0:0:0:1) (mount-id:testFS-123)", common.GetVersion())), + }, + { + name: "profile_enabled_aiml_serving", + mountConfig: &cfg.Config{ + Profile: cfg.ProfileAIMLServing, + }, + expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:0:0:0:0:0:1) (mount-id:testFS-123)", common.GetVersion())), + }, + { + name: "profile_enabled_aiml_checkpointing", + mountConfig: &cfg.Config{ + Profile: cfg.ProfileAIMLCheckpointing, + }, + expectedUserAgent: strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-AppName) (Cfg:0:0:0:0:0:1) (mount-id:testFS-123)", common.GetVersion())), }, } for _, tc := range testCases { t.T().Run(tc.name, func(t *testing.T) { - userAgent := getUserAgent("AppName", getConfigForUserAgent(tc.mountConfig)) + userAgent := getUserAgent("AppName", getConfigForUserAgent(tc.mountConfig), "testFS-123") + assert.Equal(t, tc.expectedUserAgent, userAgent) }) } } func (t *MainTest) TestGetUserAgentWhenMetadataImageTypeEnvVarSetAndAppNameNotSet() { - os.Setenv("GCSFUSE_METADATA_IMAGE_TYPE", "DLVM") - defer os.Unsetenv("GCSFUSE_METADATA_IMAGE_TYPE") - + t.T().Setenv("GCSFUSE_METADATA_IMAGE_TYPE", "DLVM") + expectedUserAgent := strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-DLVM) (Cfg:0:0:0:0:0:0) (mount-id:testFS-123)", common.GetVersion())) mountConfig := &cfg.Config{} - userAgent := getUserAgent("", getConfigForUserAgent(mountConfig)) - expectedUserAgent := strings.TrimSpace(fmt.Sprintf("gcsfuse/%s (GPN:gcsfuse-DLVM) (Cfg:0:0:0)", common.GetVersion())) + + userAgent := getUserAgent("", getConfigForUserAgent(mountConfig), "testFS-123") assert.Equal(t.T(), expectedUserAgent, userAgent) } @@ -197,11 +286,11 @@ func (t *MainTest) TestCallListRecursiveOnExistingDirectory() { t.T().Fatalf("Failed to set up test. error = %v", err) } defer os.RemoveAll(rootdir) + _, err = os.CreateTemp(rootdir, "abc-*.txt") if err != nil { t.T().Fatalf("Failed to set up test. error = %v", err) } - err = callListRecursive(rootdir) assert.Nil(t.T(), err) @@ -217,21 +306,189 @@ func (t *MainTest) TestCallListRecursiveOnNonExistingDirectory() { } func (t *MainTest) TestIsDynamicMount() { - for _, input := range []struct { + testCases := []struct { + name string bucketName string isDynamic bool }{ { + name: "Empty bucket name", bucketName: "", isDynamic: true, - }, { + }, + { + name: "Underscore bucket name", bucketName: "_", isDynamic: true, - }, { + }, + { + name: "Regular bucket name", bucketName: "abc", isDynamic: false, }, - } { - assert.Equal(t.T(), input.isDynamic, isDynamicMount(input.bucketName)) + } + + for _, tc := range testCases { + t.T().Run(tc.name, func(t *testing.T) { + isDynamic := isDynamicMount(tc.bucketName) + + assert.Equal(t, tc.isDynamic, isDynamic) + }) + } +} + +func (t *MainTest) TestFSName() { + testCases := []struct { + name string + bucketName string + fsName string + }{ + { + name: "Empty bucket name", + bucketName: "", + fsName: DynamicMountFSName, + }, + { + name: "Underscore bucket name", + bucketName: "_", + fsName: DynamicMountFSName, + }, + { + name: "Regular bucket name", + bucketName: "abc", + fsName: "abc", + }, + } + + for _, tc := range testCases { + t.T().Run(tc.name, func(t *testing.T) { + actualFSName := fsName(tc.bucketName) + + assert.Equal(t, tc.fsName, actualFSName) + }) + } +} + +func (t *MainTest) TestForwardedEnvVars_AlwaysPresent() { + // These variables are always added to the forwarded environment. + homeDir, err := os.UserHomeDir() + require.NoError(t.T(), err) + parentDir, err := os.Getwd() + require.NoError(t.T(), err) + expectedForwardedEnvVars := []string{ + "GCSFUSE_IN_BACKGROUND_MODE=true", + "GCSFUSE_MOUNT_UUID=" + logger.MountUUID(), + "PATH=" + os.Getenv("PATH"), + "HOME=" + homeDir, + util.GCSFUSE_PARENT_PROCESS_DIR + "=" + parentDir, + } + + forwardedEnvVars := forwardedEnvVars() + + assert.Subset(t.T(), forwardedEnvVars, expectedForwardedEnvVars) +} + +func (t *MainTest) TestForwardedEnvVars_Precedence() { + // This test handles cases where the presence of one env var affects another. + testCases := []struct { + name string + inputEnvVars map[string]string + expectedForwardedEnvVars []string + unexpectedForwardedEnvVarNames []string + }{ + { + name: "https_proxy is forwarded over http_proxy", + inputEnvVars: map[string]string{"https_proxy": "https-proxy-123", "http_proxy": "http-proxy-123"}, + expectedForwardedEnvVars: []string{"https_proxy=https-proxy-123"}, + unexpectedForwardedEnvVarNames: []string{"http_proxy"}, + }, + { + name: "http_proxy is forwarded when https_proxy is not set", + inputEnvVars: map[string]string{"http_proxy": "http-proxy-123"}, + expectedForwardedEnvVars: []string{"http_proxy=http-proxy-123"}, + unexpectedForwardedEnvVarNames: []string{"https_proxy"}, + }, + } + + for _, tc := range testCases { + t.T().Run(tc.name, func(t *testing.T) { + for k, v := range tc.inputEnvVars { + t.Setenv(k, v) + } + + forwardedEnvVars := forwardedEnvVars() + + assert.Subset(t, forwardedEnvVars, tc.expectedForwardedEnvVars) + // Verify that none of the unexpected variables were forwarded. + for _, forwardedVar := range forwardedEnvVars { + name, _, ok := strings.Cut(forwardedVar, "=") + require.True(t, ok, "Invalid env var format: %s", forwardedVar) + assert.NotContains(t, tc.unexpectedForwardedEnvVarNames, name, "unexpected env var %q was forwarded", name) + } + }) + } +} + +func (t *MainTest) TestForwardedEnvVars_PassedWhenSet() { + // These variables are only forwarded if they are set in the environment. + testCases := []struct { + name string + inputEnvVars map[string]string + expectedForwardedEnvVars []string + }{ + { + name: "GCE metadata env vars", + inputEnvVars: map[string]string{"GCE_METADATA_HOST": "www.metadata-host.com", "GCE_METADATA_ROOT": "metadata-root", "GCE_METADATA_IP": "99.100.101.102"}, + expectedForwardedEnvVars: []string{"GCE_METADATA_HOST=www.metadata-host.com", "GCE_METADATA_ROOT=metadata-root", "GCE_METADATA_IP=99.100.101.102"}, + }, + { + name: "GOOGLE_APPLICATION_CREDENTIALS", + inputEnvVars: map[string]string{"GOOGLE_APPLICATION_CREDENTIALS": "goog-app-cred"}, + expectedForwardedEnvVars: []string{"GOOGLE_APPLICATION_CREDENTIALS=goog-app-cred"}, + }, + { + name: "GRPC debug env vars", + inputEnvVars: map[string]string{"GRPC_GO_LOG_VERBOSITY_LEVEL": "99", "GRPC_GO_LOG_SEVERITY_LEVEL": "INFO"}, + expectedForwardedEnvVars: []string{"GRPC_GO_LOG_VERBOSITY_LEVEL=99", "GRPC_GO_LOG_SEVERITY_LEVEL=INFO"}, + }, + { + name: "no_proxy", + inputEnvVars: map[string]string{"no_proxy": "no-proxy-123"}, + expectedForwardedEnvVars: []string{"no_proxy=no-proxy-123"}, + }, + } + + for _, tc := range testCases { + t.T().Run(tc.name, func(t *testing.T) { + for k, v := range tc.inputEnvVars { + t.Setenv(k, v) + } + + forwardedEnvVars := forwardedEnvVars() + + assert.Subset(t, forwardedEnvVars, tc.expectedForwardedEnvVars) + }) + } +} + +func (t *MainTest) TestForwardedEnvVars_NotPassedWhenUnset() { + // These variables should NOT be forwarded if they are not set. + unexpectedForwardedEnvVars := []string{ + "GCE_METADATA_HOST", + "GCE_METADATA_ROOT", + "GCE_METADATA_IP", + "GOOGLE_APPLICATION_CREDENTIALS", + "GRPC_GO_LOG_VERBOSITY_LEVEL", + "GRPC_GO_LOG_SEVERITY_LEVEL", + "no_proxy", + } + + forwardedEnvVars := forwardedEnvVars() + + // Verify that none of the unexpected/unset variables were forwarded. + for _, forwardedVar := range forwardedEnvVars { + name, _, ok := strings.Cut(forwardedVar, "=") + require.True(t.T(), ok, "Invalid env var format: %s", forwardedVar) + assert.NotContains(t.T(), unexpectedForwardedEnvVars, name, "unexpected env var %q was forwarded", name) } } diff --git a/cmd/mount.go b/cmd/mount.go index 29ead6e8d3..287b793da3 100644 --- a/cmd/mount.go +++ b/cmd/mount.go @@ -19,15 +19,18 @@ import ( "os" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/mount" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/mount" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/spf13/viper" "golang.org/x/net/context" - "github.com/googlecloudplatform/gcsfuse/v2/internal/fs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" - "github.com/googlecloudplatform/gcsfuse/v2/internal/perms" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/perms" "github.com/jacobsa/fuse" "github.com/jacobsa/fuse/fsutil" "github.com/jacobsa/timeutil" @@ -40,7 +43,11 @@ func mountWithStorageHandle( bucketName string, mountPoint string, newConfig *cfg.Config, - storageHandle storage.StorageHandle) (mfs *fuse.MountedFileSystem, err error) { + storageHandle storage.StorageHandle, + metricHandle metrics.MetricHandle, + traceHandle tracing.TraceHandle, + viperConfig *viper.Viper) (mfs *fuse.MountedFileSystem, err error) { + // Sanity check: make sure the temporary directory exists and is writable // currently. This gives a better user experience than harder to debug EIO // errors when reading files in the future. @@ -91,9 +98,17 @@ be interacting with the file system.`) OpRateLimitHz: newConfig.GcsConnection.LimitOpsPerSec, StatCacheMaxSizeMB: uint64(newConfig.MetadataCache.StatCacheMaxSizeMb), StatCacheTTL: time.Duration(newConfig.MetadataCache.TtlSecs) * time.Second, - EnableMonitoring: newConfig.Metrics.StackdriverExportInterval > 0 || newConfig.Metrics.PrometheusPort != 0, + NegativeStatCacheTTL: time.Duration(newConfig.MetadataCache.NegativeTtlSecs) * time.Second, + EnableMonitoring: cfg.IsMetricsEnabled(&newConfig.Metrics), + LogSeverity: newConfig.Logging.Severity, AppendThreshold: 1 << 21, // 2 MiB, a total guess. + ChunkRetryDeadlineSecs: newConfig.GcsRetries.ChunkRetryDeadlineSecs, + ChunkTransferTimeoutSecs: newConfig.GcsRetries.ChunkTransferTimeoutSecs, TmpObjectPrefix: ".gcsfuse_tmp/", + DisableListAccessCheck: newConfig.DisableListAccessCheck, + DummyIOCfg: newConfig.DummyIo, + IsTypeCacheDeprecated: newConfig.EnableTypeCacheDeprecation, + ImplicitDir: newConfig.ImplicitDirs, } bm := gcsx.NewBucketManager(bucketCfg, storageHandle) @@ -115,6 +130,12 @@ be interacting with the file system.`) SequentialReadSizeMb: int32(newConfig.GcsConnection.SequentialReadSizeMb), EnableNonexistentTypeCache: newConfig.MetadataCache.EnableNonexistentTypeCache, NewConfig: newConfig, + ViperConfig: viperConfig, + MetricHandle: metricHandle, + TraceHandle: traceHandle, + } + if serverCfg.NewConfig.FileSystem.ExperimentalEnableDentryCache { + serverCfg.Notifier = fuse.NewNotifier() } logger.Infof("Creating a new server...\n") @@ -124,11 +145,7 @@ be interacting with the file system.`) return } - fsName := bucketName - if isDynamicMount(bucketName) { - // mounting all the buckets at once - fsName = "gcsfuse" - } + fsName := fsName(bucketName) // Mount the file system. logger.Infof("Mounting file system %q...", fsName) @@ -164,9 +181,36 @@ func getFuseMountConfig(fsName string, newConfig *cfg.Config) *fuse.MountConfig // access two files under same directory parallely, then the lookups also // happen parallely. EnableParallelDirOps: !(newConfig.FileSystem.DisableParallelDirops), + // We disable write-back cache when streaming writes are enabled. + DisableWritebackCaching: newConfig.Write.EnableStreamingWrites, + // Enables ReadDirPlus, allowing the kernel to retrieve directory entries and their + // attributes in a single operation. + EnableReaddirplus: newConfig.FileSystem.ExperimentalEnableReaddirplus, + // Enable async reads if enable-kernel-reader flag is set to true. + EnableAsyncReads: newConfig.FileSystem.EnableKernelReader, + } + + if newConfig.Logging.WireLog != "" { + wireLog, err := os.Create(string(newConfig.Logging.WireLog)) + if err == nil { + mountCfg.WireLogger = wireLog + } else { + logger.Errorf("Unable to create wire log: %v", err) + } } - mountCfg.ErrorLogger = logger.NewLegacyLogger(logger.LevelError, "fuse: ") - mountCfg.DebugLogger = logger.NewLegacyLogger(logger.LevelTrace, "fuse_debug: ") + // GCSFuse to Jacobsa Fuse Log Level mapping: + // OFF OFF + // ERROR ERROR + // WARNING ERROR + // INFO ERROR + // DEBUG ERROR + // TRACE TRACE + if newConfig.Logging.Severity.Rank() <= cfg.ErrorLogSeverity.Rank() { + mountCfg.ErrorLogger = logger.NewLegacyLogger(logger.LevelError, "fuse: ", fsName) + } + if newConfig.Logging.Severity.Rank() <= cfg.TraceLogSeverity.Rank() { + mountCfg.DebugLogger = logger.NewLegacyLogger(logger.LevelTrace, "fuse_debug: ", fsName) + } return mountCfg } diff --git a/cmd/mount_test.go b/cmd/mount_test.go index 2f9a106c54..ac74b6bb97 100644 --- a/cmd/mount_test.go +++ b/cmd/mount_test.go @@ -17,7 +17,7 @@ package cmd import ( "testing" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" "github.com/stretchr/testify/assert" ) @@ -66,3 +66,85 @@ func TestGetFuseMountConfig_MountOptionsFormattedCorrectly(t *testing.T) { assert.True(t, fuseMountCfg.EnableParallelDirOps) // Default true unless explicitly disabled } } + +func TestGetFuseMountConfig_LoggerInitializationInFuse(t *testing.T) { + testCases := []struct { + name string + gcsFuseLogLevel string + shouldInitializeTrace bool + shouldInitializeError bool + }{ + { + name: "GcsFuseOffLogLevelShouldNotInitializeAnyLogger", + gcsFuseLogLevel: "OFF", + shouldInitializeTrace: false, + shouldInitializeError: false, + }, + { + name: "GcsFuseErrorLogLevelShouldInitializeErrorLoggerOnly", + gcsFuseLogLevel: "ERROR", + shouldInitializeTrace: false, + shouldInitializeError: true, + }, + { + name: "GcsFuseDebugLogLevelShouldInitializeErrorLoggerOnly", + gcsFuseLogLevel: "DEBUG", + shouldInitializeTrace: false, + shouldInitializeError: true, + }, + { + name: "GcsFuseTraceLogLevelShouldInitializeBothLogger", + gcsFuseLogLevel: "TRACE", + shouldInitializeTrace: true, + shouldInitializeError: true, + }, + } + + fsName := "mybucket" + for _, tc := range testCases { + newConfig := &cfg.Config{ + Logging: cfg.LoggingConfig{ + Severity: cfg.LogSeverity(tc.gcsFuseLogLevel), + }, + } + + fuseMountCfg := getFuseMountConfig(fsName, newConfig) + + assert.Equal(t, tc.shouldInitializeError, fuseMountCfg.ErrorLogger != nil) + assert.Equal(t, tc.shouldInitializeTrace, fuseMountCfg.DebugLogger != nil) + } +} + +func TestGetFuseMountConfig_EnableReaddirplus(t *testing.T) { + testCases := []struct { + name string + enableReaddirplus bool + expectedValue bool + }{ + { + name: "ExperimentalEnableReaddirplusFlagFalse", + enableReaddirplus: false, + expectedValue: false, + }, + { + name: "ExperimentalEnableReaddirplusFlagTrue", + enableReaddirplus: true, + expectedValue: true, + }, + } + + fsName := "mybucket" + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + newConfig := &cfg.Config{ + FileSystem: cfg.FileSystemConfig{ + ExperimentalEnableReaddirplus: tc.enableReaddirplus, + }, + } + + fuseMountCfg := getFuseMountConfig(fsName, newConfig) + + assert.Equal(t, tc.expectedValue, fuseMountCfg.EnableReaddirplus) + }) + } +} diff --git a/cmd/root.go b/cmd/root.go index 105a6cf4a4..1709524266 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -17,28 +17,83 @@ package cmd import ( "fmt" "log" + "maps" "os" + "slices" "strings" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/common" - "github.com/googlecloudplatform/gcsfuse/v2/internal/util" - "github.com/mitchellh/mapstructure" + "github.com/go-viper/mapstructure/v2" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/common" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/spf13/viper" ) -type mountFn func(c *cfg.Config, bucketName, mountPoint string) error +type mountInfo struct { + // cliFlags are the flags passed through the command line to GCSFuse Program. + // This field is used only for logging purpose. + cliFlags map[string]string + // configFileFlags are the flags passed through the config file to GCSFuse Program. + // This field is used only for logging purpose. + configFileFlags map[string]any + // config is the final config object after merging cli and config file flags applying + // all optimisation based on machineType, Profile etc. This is the final config used for mounting GCSFuse. + config *cfg.Config + // optimizedFlags contains the flags that were optimized + // based on either machine-type or profile. + optimizedFlags map[string]any + // viperConfig is used to check if a flag was explicitly set by the user. + // This is used to determine if optimization rules should be applied. + viperConfig *viper.Viper +} + +type mountFn func(mountInfo *mountInfo, bucketName, mountPoint string) error + +// getCliFlags returns the cli flags set by the user in map[string]string format. +func getCliFlags(flagSet *pflag.FlagSet) map[string]string { + cliFlags := make(map[string]string) + flagSet.VisitAll(func(f *pflag.Flag) { + if f.Changed { + cliFlags[f.Name] = f.Value.String() + } + }) + // Do not display --foreground flag to the user in logs if user + // hasn't passed this flag and was added by GCSFuse during demonized run. + if _, ok := os.LookupEnv(logger.GCSFuseInBackgroundMode); ok { + delete(cliFlags, "foreground") + } + return cliFlags +} -// NewRootCmd accepts the mountFn that it executes with the parsed configuration -func NewRootCmd(m mountFn) (*cobra.Command, error) { +// getConfigFileFlags returns the flags set by the user in the config file. +func getConfigFileFlags(v *viper.Viper) map[string]any { + if v.ConfigFileUsed() == "" { + return nil + } + + // v.AllSettings() includes defaults, which we don't want. + // We only want what's explicitly in the config file. + // We can achieve this by creating a new Viper instance and reading the + // same config file into it without setting any defaults. + configOnlyViper := viper.New() + configOnlyViper.SetConfigFile(v.ConfigFileUsed()) + configOnlyViper.SetConfigType("yaml") + // We can ignore the error here, as the original viper instance would have already failed. + _ = configOnlyViper.ReadInConfig() + return configOnlyViper.AllSettings() +} + +// newRootCmd accepts the mountFn that it executes with the parsed configuration +func newRootCmd(m mountFn) (*cobra.Command, error) { var ( - configObj cfg.Config - cfgFile string - cfgErr error - v = viper.New() + mountInfo mountInfo + cfgFile string + viperConfig = viper.New() ) + mountInfo.config = &cfg.Config{} rootCmd := &cobra.Command{ Use: "gcsfuse [flags] bucket mount_point", Short: "Mount a specified GCS bucket or all accessible buckets locally", @@ -48,49 +103,60 @@ of Cloud Storage FUSE, see https://cloud.google.com/storage/docs/gcs-fuse.`, Version: common.GetVersion(), Args: cobra.RangeArgs(2, 3), SilenceUsage: true, - RunE: func(cmd *cobra.Command, args []string) error { - if cfgErr != nil { - return fmt.Errorf("error while parsing config: %w", cfgErr) + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + if cfgFile != "" { + resolvedCfgFile, err := util.GetResolvedPath(cfgFile) + if err != nil { + return fmt.Errorf("error while resolving config-file path[%s]: %w", cfgFile, err) + } + viperConfig.SetConfigFile(resolvedCfgFile) + viperConfig.SetConfigType("yaml") + if err := viperConfig.ReadInConfig(); err != nil { + return fmt.Errorf("error while reading the config: %w", err) + } } - bucket, mountPoint, err := populateArgs(args[1:]) + + if err := viperConfig.Unmarshal(mountInfo.config, viper.DecodeHook(cfg.DecodeHook()), func(decoderConfig *mapstructure.DecoderConfig) { + // By default, viper supports mapstructure tags for unmarshalling. Override that to support yaml tag. + decoderConfig.TagName = "yaml" + // Reject the config file if any of the fields in the YAML don't map to the struct. + decoderConfig.ErrorUnused = true + }, + ); err != nil { + return fmt.Errorf("error while unmarshalling config: %w", err) + } + if err := cfg.ValidateConfig(viperConfig, mountInfo.config); err != nil { + return fmt.Errorf("invalid config: %w", err) + } + + mountInfo.viperConfig = viperConfig + optimizedFlags := mountInfo.config.ApplyOptimizations(viperConfig, nil) + optimizedFlagNames := slices.Collect(maps.Keys(optimizedFlags)) + if err := cfg.Rationalize(viperConfig, mountInfo.config, optimizedFlagNames); err != nil { + return fmt.Errorf("error rationalizing config: %w", err) + } + mountInfo.cliFlags = getCliFlags(cmd.PersistentFlags()) + mountInfo.configFileFlags = getConfigFileFlags(viperConfig) + optimizedFlagsAsHierarchicalMap, err := cfg.CreateHierarchicalOptimizedFlags(optimizedFlags) if err != nil { - return fmt.Errorf("error occurred while extracting the bucket and mountPoint: %w", err) + logger.Errorf("GCSFuse Config: error creating hierarchical map for optimized flags: %v", err) + // Log the raw map as a fallback + optimizedFlagsAsHierarchicalMap = make(map[string]any, len(optimizedFlags)) + for flag, value := range optimizedFlags { + optimizedFlagsAsHierarchicalMap[flag] = value + } } - return m(&configObj, bucket, mountPoint) + mountInfo.optimizedFlags = optimizedFlagsAsHierarchicalMap + return nil }, - } - initConfig := func() { - if cfgFile != "" { - cfgFile, err := util.GetResolvedPath(cfgFile) + RunE: func(cmd *cobra.Command, args []string) error { + bucket, mountPoint, err := populateArgs(args[1:]) if err != nil { - cfgErr = fmt.Errorf("error while resolving config-file path[%s]: %w", cfgFile, err) - return - } - v.SetConfigFile(cfgFile) - v.SetConfigType("yaml") - if err := v.ReadInConfig(); err != nil { - cfgErr = fmt.Errorf("error while reading the config: %w", err) - return + return fmt.Errorf("error occurred while extracting the bucket and mountPoint: %w", err) } - } - - if cfgErr = v.Unmarshal(&configObj, viper.DecodeHook(cfg.DecodeHook()), func(decoderConfig *mapstructure.DecoderConfig) { - // By default, viper supports mapstructure tags for unmarshalling. Override that to support yaml tag. - decoderConfig.TagName = "yaml" - // Reject the config file if any of the fields in the YAML don't map to the struct. - decoderConfig.ErrorUnused = true + return m(&mountInfo, bucket, mountPoint) }, - ); cfgErr != nil { - return - } - if cfgErr = cfg.ValidateConfig(v, &configObj); cfgErr != nil { - return - } - if cfgErr = cfg.Rationalize(v, &configObj); cfgErr != nil { - return - } } - cobra.OnInitialize(initConfig) rootCmd.PersistentFlags().StringVar(&cfgFile, cfg.ConfigFileFlagName, "", "The path to the config file where all gcsfuse related config needs to be specified. "+ "Refer to 'https://cloud.google.com/storage/docs/gcsfuse-cli#config-file' for possible configurations.") @@ -98,7 +164,7 @@ of Cloud Storage FUSE, see https://cloud.google.com/storage/docs/gcs-fuse.`, if err := cfg.BuildFlagSet(rootCmd.PersistentFlags()); err != nil { return nil, fmt.Errorf("error while declaring flags: %w", err) } - if err := cfg.BindFlags(v, rootCmd.PersistentFlags()); err != nil { + if err := cfg.BindFlags(viperConfig, rootCmd.PersistentFlags()); err != nil { return nil, fmt.Errorf("error while binding flags: %w", err) } return rootCmd, nil @@ -147,12 +213,12 @@ func convertToPosixArgs(args []string, c *cobra.Command) []string { } var ExecuteMountCmd = func() { - rootCmd, err := NewRootCmd(Mount) + rootCmd, err := newRootCmd(Mount) if err != nil { - log.Fatalf("Error occurred while creating the root command: %v", err) + log.Fatalf("Error occurred while creating the root command on gcsfuse/%s: %v", common.GetVersion(), err) } rootCmd.SetArgs(convertToPosixArgs(os.Args, rootCmd)) if err := rootCmd.Execute(); err != nil { - log.Fatalf("Error occurred during command execution: %v", err) + log.Fatalf("Error occurred during command execution on gcsfuse/%s: %v", common.GetVersion(), err) } } diff --git a/cmd/root_test.go b/cmd/root_test.go index 0475deedb4..5c774d9868 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -23,15 +23,38 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/spf13/pflag" + "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +const testMaxSupportedTTLInSeconds = math.MaxInt64 / int64(time.Second) + +//////////////////// +// Helpers +//////////////////// + +func createTempConfigFile(t *testing.T, content string) string { + t.Helper() + f, err := os.CreateTemp(t.TempDir(), "config.yaml") + require.NoError(t, err) + _, err = f.WriteString(content) + require.NoError(t, err) + require.NoError(t, f.Close()) + return f.Name() +} + +//////////////////// +// Tests +//////////////////// + func TestDefaultMaxParallelDownloads(t *testing.T) { var actual *cfg.Config - cmd, err := NewRootCmd(func(c *cfg.Config, _, _ string) error { - actual = c + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + actual = mountInfo.config return nil }) require.Nil(t, err) @@ -72,7 +95,7 @@ func TestCobraArgsNumInRange(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - cmd, err := NewRootCmd(func(*cfg.Config, string, string) error { return nil }) + cmd, err := newRootCmd(func(*mountInfo, string, string) error { return nil }) require.Nil(t, err) cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) @@ -127,7 +150,7 @@ func TestArgsParsing_MountPoint(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var bucketName, mountPoint string - cmd, err := NewRootCmd(func(_ *cfg.Config, b string, m string) error { + cmd, err := newRootCmd(func(_ *mountInfo, b string, m string) error { bucketName = b mountPoint = m return nil @@ -176,8 +199,8 @@ func TestArgsParsing_MountOptions(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var mountOptions []string - cmd, err := NewRootCmd(func(cfg *cfg.Config, _, _ string) error { - mountOptions = cfg.FileSystem.FuseOptions + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + mountOptions = mountInfo.config.FileSystem.FuseOptions return nil }) require.Nil(t, err) @@ -192,95 +215,243 @@ func TestArgsParsing_MountOptions(t *testing.T) { } } +// Lets test for ImplicitDirs which is goverened by implicit-dirs flags +func TestArgsParsing_ImplicitDirsFlag(t *testing.T) { + tests := []struct { + name string + args []string + expectedImplicit bool + }{ + { + name: "normal", + args: []string{"gcsfuse", "--implicit-dirs", "abc", "pqr"}, + expectedImplicit: true, + }, + { + name: "default", + args: []string{"gcsfuse", "abc", "pqr"}, + expectedImplicit: false, + }, + { + name: "normal_false", + args: []string{"gcsfuse", "--implicit-dirs=false", "abc", "pqr"}, + expectedImplicit: false, + }, + { + name: "default false on high performance machine with autoconfig disabled", + args: []string{"gcsfuse", "--machine-type=a3-highgpu-8g", "--disable-autoconfig=true", "abc", "pqr"}, + expectedImplicit: false, + }, + { + name: "default true on high performance machine with autoconfig enabled", + args: []string{"gcsfuse", "--machine-type=a3-highgpu-8g", "--disable-autoconfig=false", "abc", "pqr"}, + expectedImplicit: true, + }, + { + name: "default overriden on high performance machine", + args: []string{"gcsfuse", "--machine-type=a3-highgpu-8g", "--disable-autoconfig=false", "--implicit-dirs=false", "abc", "pqr"}, + expectedImplicit: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var gotImplicit bool + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotImplicit = mountInfo.config.ImplicitDirs + return nil + }) + require.Nil(t, err) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) + + err = cmd.Execute() + + if assert.NoError(t, err) { + assert.Equal(t, tc.expectedImplicit, gotImplicit) + } + }) + } +} func TestArgsParsing_WriteConfigFlags(t *testing.T) { tests := []struct { name string args []string expectedCreateEmptyFile bool expectedEnableStreamingWrites bool + expectedEnableRapidAppends bool expectedWriteBlockSizeMB int64 expectedWriteGlobalMaxBlocks int64 expectedWriteMaxBlocksPerFile int64 + expectedEnableRapidWrites bool + expectedFinalizeFileOnClose bool }{ { - name: "Test create-empty-file flag true.", - args: []string{"gcsfuse", "--create-empty-file=true", "abc", "pqr"}, + name: "Test create-empty-file flag true works when streaming writes are explicitly disabled.", + args: []string{"gcsfuse", "--create-empty-file=true", "--enable-streaming-writes=false", "abc", "pqr"}, expectedCreateEmptyFile: true, expectedEnableStreamingWrites: false, - expectedWriteBlockSizeMB: 64, - expectedWriteGlobalMaxBlocks: math.MaxInt64, - expectedWriteMaxBlocksPerFile: math.MaxInt64, + expectedEnableRapidAppends: true, + expectedWriteBlockSizeMB: 32, + expectedWriteGlobalMaxBlocks: 4, + expectedWriteMaxBlocksPerFile: 1, }, { name: "Test create-empty-file flag false.", args: []string{"gcsfuse", "--create-empty-file=false", "abc", "pqr"}, expectedCreateEmptyFile: false, - expectedEnableStreamingWrites: false, - expectedWriteBlockSizeMB: 64, - expectedWriteGlobalMaxBlocks: math.MaxInt64, - expectedWriteMaxBlocksPerFile: math.MaxInt64, + expectedEnableStreamingWrites: true, + expectedEnableRapidAppends: true, + expectedWriteBlockSizeMB: 32, + expectedWriteGlobalMaxBlocks: 4, + expectedWriteMaxBlocksPerFile: 1, }, { name: "Test default flags.", args: []string{"gcsfuse", "abc", "pqr"}, expectedCreateEmptyFile: false, - expectedEnableStreamingWrites: false, - expectedWriteBlockSizeMB: 64, - expectedWriteGlobalMaxBlocks: math.MaxInt64, - expectedWriteMaxBlocksPerFile: math.MaxInt64, + expectedEnableStreamingWrites: true, + expectedEnableRapidAppends: true, + expectedWriteBlockSizeMB: 32, + expectedWriteGlobalMaxBlocks: 4, + expectedWriteMaxBlocksPerFile: 1, }, { name: "Test enable-streaming-writes flag true.", - args: []string{"gcsfuse", "--experimental-enable-streaming-writes", "abc", "pqr"}, + args: []string{"gcsfuse", "--enable-streaming-writes", "abc", "pqr"}, expectedCreateEmptyFile: false, expectedEnableStreamingWrites: true, - expectedWriteBlockSizeMB: 64, - expectedWriteGlobalMaxBlocks: math.MaxInt64, - expectedWriteMaxBlocksPerFile: math.MaxInt64, + expectedEnableRapidAppends: true, + expectedWriteBlockSizeMB: 32, + expectedWriteGlobalMaxBlocks: 4, + expectedWriteMaxBlocksPerFile: 1, }, { name: "Test enable-streaming-writes flag false.", - args: []string{"gcsfuse", "--experimental-enable-streaming-writes=false", "abc", "pqr"}, + args: []string{"gcsfuse", "--enable-streaming-writes=false", "abc", "pqr"}, expectedCreateEmptyFile: false, expectedEnableStreamingWrites: false, - expectedWriteBlockSizeMB: 64, - expectedWriteGlobalMaxBlocks: math.MaxInt64, - expectedWriteMaxBlocksPerFile: math.MaxInt64, + expectedEnableRapidAppends: true, + expectedWriteBlockSizeMB: 32, + expectedWriteGlobalMaxBlocks: 4, + expectedWriteMaxBlocksPerFile: 1, + }, + { + name: "Test enable-rapid-appends flag true.", + args: []string{"gcsfuse", "--enable-rapid-appends=false", "abc", "pqr"}, + expectedCreateEmptyFile: false, + expectedEnableStreamingWrites: true, + expectedEnableRapidAppends: false, + expectedWriteBlockSizeMB: 32, + expectedWriteGlobalMaxBlocks: 4, + expectedWriteMaxBlocksPerFile: 1, }, { name: "Test positive write-block-size-mb flag.", - args: []string{"gcsfuse", "--experimental-enable-streaming-writes", "--write-block-size-mb=10", "abc", "pqr"}, + args: []string{"gcsfuse", "--enable-streaming-writes", "--write-block-size-mb=10", "abc", "pqr"}, expectedCreateEmptyFile: false, expectedEnableStreamingWrites: true, + expectedEnableRapidAppends: true, expectedWriteBlockSizeMB: 10, - expectedWriteGlobalMaxBlocks: math.MaxInt64, - expectedWriteMaxBlocksPerFile: math.MaxInt64, + expectedWriteGlobalMaxBlocks: 4, + expectedWriteMaxBlocksPerFile: 1, }, { name: "Test positive write-global-max-blocks flag.", - args: []string{"gcsfuse", "--experimental-enable-streaming-writes", "--write-global-max-blocks=10", "abc", "pqr"}, + args: []string{"gcsfuse", "--enable-streaming-writes", "--write-global-max-blocks=10", "abc", "pqr"}, expectedCreateEmptyFile: false, expectedEnableStreamingWrites: true, - expectedWriteBlockSizeMB: 64, + expectedEnableRapidAppends: true, + expectedWriteBlockSizeMB: 32, expectedWriteGlobalMaxBlocks: 10, - expectedWriteMaxBlocksPerFile: math.MaxInt64, + expectedWriteMaxBlocksPerFile: 1, }, { name: "Test positive write-max-blocks-per-file flag.", - args: []string{"gcsfuse", "--experimental-enable-streaming-writes", "--write-max-blocks-per-file=10", "abc", "pqr"}, + args: []string{"gcsfuse", "--enable-streaming-writes", "--write-max-blocks-per-file=10", "abc", "pqr"}, expectedCreateEmptyFile: false, expectedEnableStreamingWrites: true, - expectedWriteBlockSizeMB: 64, - expectedWriteGlobalMaxBlocks: math.MaxInt64, + expectedEnableRapidAppends: true, + expectedWriteBlockSizeMB: 32, + expectedWriteGlobalMaxBlocks: 4, expectedWriteMaxBlocksPerFile: 10, }, + { + name: "Test high performance config values.", + args: []string{"gcsfuse", "--machine-type=a3-highgpu-8g", "--disable-autoconfig=false", "abc", "pqr"}, + expectedEnableStreamingWrites: true, + expectedEnableRapidAppends: true, + expectedWriteBlockSizeMB: 32, + expectedWriteGlobalMaxBlocks: 1600, + }, + { + name: "Test high performance config values with --write-global-max-blocks flag overriden.", + args: []string{"gcsfuse", "--write-global-max-blocks=2000", "--disable-autoconfig=false", "abc", "pqr"}, + expectedCreateEmptyFile: false, + expectedEnableStreamingWrites: true, + expectedEnableRapidAppends: true, + expectedWriteBlockSizeMB: 32, + expectedWriteGlobalMaxBlocks: 2000, + expectedWriteMaxBlocksPerFile: 1, + }, + { + name: "Test_optimization_fallback_to_machine-type_config_with_un-overridden_profile_on_high-end_machine", + args: []string{"gcsfuse", "--machine-type=a3-highgpu-8g", "--profile=" + cfg.ProfileAIMLCheckpointing, "abc", "pqr"}, + expectedCreateEmptyFile: false, + expectedEnableStreamingWrites: true, + expectedEnableRapidAppends: true, + expectedWriteBlockSizeMB: 32, + expectedWriteGlobalMaxBlocks: 1600, + expectedWriteMaxBlocksPerFile: 1, + }, + { + name: "Test_optimization_fallback_to_default_config_with_un-overridden_profile_on_low-end_machine", + args: []string{"gcsfuse", "--machine-type=low-end-machine", "--profile=" + cfg.ProfileAIMLCheckpointing, "abc", "pqr"}, + expectedCreateEmptyFile: false, + expectedEnableStreamingWrites: true, + expectedEnableRapidAppends: true, + expectedWriteBlockSizeMB: 32, + expectedWriteGlobalMaxBlocks: 4, + expectedWriteMaxBlocksPerFile: 1, + }, + { + name: "Test_optimization_overriden_by_user_config_with_profile_set_on_high-end_machine", + args: []string{"gcsfuse", "--write-global-max-blocks=200", "--machine-type=a3-highgpu-8g", "--profile=" + cfg.ProfileAIMLCheckpointing, "abc", "pqr"}, + expectedCreateEmptyFile: false, + expectedEnableStreamingWrites: true, + expectedEnableRapidAppends: true, + expectedWriteBlockSizeMB: 32, + expectedWriteGlobalMaxBlocks: 200, + expectedWriteMaxBlocksPerFile: 1, + }, + { + name: "Test_optimizationoverriden_by_user_config_with_profile_set_on_low-end_machine", + args: []string{"gcsfuse", "--write-global-max-blocks=16", "--machine-type=low-end-machine", "--profile=" + cfg.ProfileAIMLCheckpointing, "abc", "pqr"}, + expectedCreateEmptyFile: false, + expectedEnableStreamingWrites: true, + expectedEnableRapidAppends: true, + expectedWriteBlockSizeMB: 32, + expectedWriteGlobalMaxBlocks: 16, + expectedWriteMaxBlocksPerFile: 1, + }, + { + name: "Test enable-rapid-writes and finalize-file-on-close flags.", + args: []string{"gcsfuse", "--enable-rapid-writes=true", "--finalize-file-on-close=true", "abc", "pqr"}, + expectedCreateEmptyFile: false, + expectedEnableStreamingWrites: true, + expectedEnableRapidAppends: true, + expectedWriteBlockSizeMB: 32, + expectedWriteGlobalMaxBlocks: 4, + expectedWriteMaxBlocksPerFile: 1, + expectedEnableRapidWrites: true, + expectedFinalizeFileOnClose: true, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var wc cfg.WriteConfig - cmd, err := NewRootCmd(func(cfg *cfg.Config, _, _ string) error { - wc = cfg.Write + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + wc = mountInfo.config.Write return nil }) require.Nil(t, err) @@ -290,9 +461,119 @@ func TestArgsParsing_WriteConfigFlags(t *testing.T) { if assert.NoError(t, err) { assert.Equal(t, tc.expectedCreateEmptyFile, wc.CreateEmptyFile) - assert.Equal(t, tc.expectedEnableStreamingWrites, wc.ExperimentalEnableStreamingWrites) + assert.Equal(t, tc.expectedEnableStreamingWrites, wc.EnableStreamingWrites) assert.Equal(t, tc.expectedWriteBlockSizeMB, wc.BlockSizeMb) assert.Equal(t, tc.expectedWriteGlobalMaxBlocks, wc.GlobalMaxBlocks) + assert.Equal(t, tc.expectedEnableRapidAppends, wc.EnableRapidAppends) + assert.Equal(t, tc.expectedEnableRapidWrites, wc.EnableRapidWrites) + assert.Equal(t, tc.expectedFinalizeFileOnClose, wc.FinalizeFileOnClose) + } + }) + } +} + +func TestArgsParsing_ReadConfigFlags(t *testing.T) { + tests := []struct { + name string + args []string + expectedReadBlockSizeMB int64 + expectedReadGlobalMaxBlocks int64 + expectedReadMaxBlocksPerHandle int64 + expectedReadStartBlocksPerHandle int64 + expectedReadMinBlocksPerHandle int64 + }{ + { + name: "Test default flags.", + args: []string{"gcsfuse", "abc", "pqr"}, + expectedReadBlockSizeMB: 16, + expectedReadGlobalMaxBlocks: 40, + expectedReadMaxBlocksPerHandle: 20, + expectedReadStartBlocksPerHandle: 1, + expectedReadMinBlocksPerHandle: 4, + }, + { + name: "Test enable buffered read flag true.", + args: []string{"gcsfuse", "--enable-buffered-read", "abc", "pqr"}, + expectedReadBlockSizeMB: 16, + expectedReadGlobalMaxBlocks: 40, + expectedReadMaxBlocksPerHandle: 20, + expectedReadStartBlocksPerHandle: 1, + expectedReadMinBlocksPerHandle: 4, + }, + { + name: "Test enable buffered read flag false.", + args: []string{"gcsfuse", "--enable-buffered-read=false", "abc", "pqr"}, + expectedReadBlockSizeMB: 16, + expectedReadGlobalMaxBlocks: 40, + expectedReadMaxBlocksPerHandle: 20, + expectedReadStartBlocksPerHandle: 1, + expectedReadMinBlocksPerHandle: 4, + }, + { + name: "Test positive read-block-size-mb flag.", + args: []string{"gcsfuse", "--read-block-size-mb=10", "abc", "pqr"}, + expectedReadBlockSizeMB: 10, + expectedReadGlobalMaxBlocks: 40, + expectedReadMaxBlocksPerHandle: 20, + expectedReadStartBlocksPerHandle: 1, + expectedReadMinBlocksPerHandle: 4, + }, + { + name: "Test positive read-global-max-blocks flag.", + args: []string{"gcsfuse", "--read-global-max-blocks=10", "abc", "pqr"}, + expectedReadBlockSizeMB: 16, + expectedReadGlobalMaxBlocks: 10, + expectedReadMaxBlocksPerHandle: 20, + expectedReadStartBlocksPerHandle: 1, + expectedReadMinBlocksPerHandle: 4, + }, + { + name: "Test positive read-max-blocks-per-handle flag.", + args: []string{"gcsfuse", "--read-max-blocks-per-handle=10", "abc", "pqr"}, + expectedReadBlockSizeMB: 16, + expectedReadGlobalMaxBlocks: 40, + expectedReadMaxBlocksPerHandle: 10, + expectedReadStartBlocksPerHandle: 1, + expectedReadMinBlocksPerHandle: 4, + }, + { + name: "Test positive read-start-blocks-per-handle flag.", + args: []string{"gcsfuse", "--read-start-blocks-per-handle=10", "abc", "pqr"}, + expectedReadBlockSizeMB: 16, + expectedReadGlobalMaxBlocks: 40, + expectedReadMaxBlocksPerHandle: 20, + expectedReadStartBlocksPerHandle: 10, + expectedReadMinBlocksPerHandle: 4, + }, + { + name: "Test positive read-min-blocks-per-handle flag.", + args: []string{"gcsfuse", "--read-min-blocks-per-handle=10", "abc", "pqr"}, + expectedReadBlockSizeMB: 16, + expectedReadGlobalMaxBlocks: 40, + expectedReadMaxBlocksPerHandle: 20, + expectedReadStartBlocksPerHandle: 1, + expectedReadMinBlocksPerHandle: 10, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var rc cfg.ReadConfig + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + rc = mountInfo.config.Read + return nil + }) + require.Nil(t, err) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) + + err = cmd.Execute() + + if assert.NoError(t, err) { + assert.Equal(t, tc.expectedReadBlockSizeMB, rc.BlockSizeMb) + assert.Equal(t, tc.expectedReadGlobalMaxBlocks, rc.GlobalMaxBlocks) + assert.Equal(t, tc.expectedReadMaxBlocksPerHandle, rc.MaxBlocksPerHandle) + assert.Equal(t, tc.expectedReadStartBlocksPerHandle, rc.StartBlocksPerHandle) + assert.Equal(t, tc.expectedReadMinBlocksPerHandle, rc.MinBlocksPerHandle) } }) } @@ -305,36 +586,47 @@ func TestArgsParsing_FileCacheFlags(t *testing.T) { expectedConfig *cfg.Config }{ { - name: "Test file cache flags.", - args: []string{"gcsfuse", "--file-cache-cache-file-for-range-read", "--file-cache-download-chunk-size-mb=20", "--file-cache-enable-crc", "--file-cache-enable-parallel-downloads", "--file-cache-max-parallel-downloads=40", "--file-cache-max-size-mb=100", "--file-cache-parallel-downloads-per-file=2", "--file-cache-enable-o-direct=false", "abc", "pqr"}, + name: "file_cache_flags", + args: []string{"gcsfuse", "--file-cache-cache-file-for-range-read", "--file-cache-download-chunk-size-mb=20", "--file-cache-enable-crc", "--cache-dir=/some/valid/dir", "--file-cache-exclude-regex=.*", "--file-cache-include-regex=.*", "--file-cache-enable-parallel-downloads", "--file-cache-max-parallel-downloads=40", "--file-cache-max-size-mb=100", "--file-cache-parallel-downloads-per-file=2", "--file-cache-enable-o-direct=false", "--file-cache-experimental-disable-size-calculation-fix=true", "abc", "pqr"}, expectedConfig: &cfg.Config{ + CacheDir: "/some/valid/dir", FileCache: cfg.FileCacheConfig{ - CacheFileForRangeRead: true, - DownloadChunkSizeMb: 20, - EnableCrc: true, - EnableParallelDownloads: true, - MaxParallelDownloads: 40, - MaxSizeMb: 100, - ParallelDownloadsPerFile: 2, - WriteBufferSize: 4 * 1024 * 1024, - EnableODirect: false, + CacheFileForRangeRead: true, + DownloadChunkSizeMb: 20, + EnableCrc: true, + EnableParallelDownloads: true, + ExcludeRegex: ".*", + IncludeRegex: ".*", + ExperimentalParallelDownloadsDefaultOn: true, + MaxParallelDownloads: 40, + MaxSizeMb: 100, + ParallelDownloadsPerFile: 2, + SharedCacheChunkSizeMb: 8, + WriteBufferSize: 4 * 1024 * 1024, + EnableODirect: false, + ExperimentalDisableSizeCalculationFix: true, }, }, }, { - name: "Test default file cache flags.", + name: "default_file_cache_flags", args: []string{"gcsfuse", "abc", "pqr"}, expectedConfig: &cfg.Config{ FileCache: cfg.FileCacheConfig{ - CacheFileForRangeRead: false, - DownloadChunkSizeMb: 50, - EnableCrc: false, - EnableParallelDownloads: false, - MaxParallelDownloads: int64(max(16, 2*runtime.NumCPU())), - MaxSizeMb: -1, - ParallelDownloadsPerFile: 16, - WriteBufferSize: 4 * 1024 * 1024, - EnableODirect: false, + CacheFileForRangeRead: false, + DownloadChunkSizeMb: 200, + EnableCrc: false, + EnableParallelDownloads: false, + ExcludeRegex: "", + IncludeRegex: "", + ExperimentalParallelDownloadsDefaultOn: true, + MaxParallelDownloads: int64(max(16, 2*runtime.NumCPU())), + MaxSizeMb: -1, + ParallelDownloadsPerFile: 16, + SharedCacheChunkSizeMb: 8, + WriteBufferSize: 4 * 1024 * 1024, + EnableODirect: false, + ExperimentalDisableSizeCalculationFix: false, }, }, }, @@ -343,8 +635,8 @@ func TestArgsParsing_FileCacheFlags(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var gotConfig *cfg.Config - cmd, err := NewRootCmd(func(cfg *cfg.Config, _, _ string) error { - gotConfig = cfg + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotConfig = mountInfo.config return nil }) require.Nil(t, err) @@ -395,8 +687,8 @@ func TestArgParsing_ExperimentalMetadataPrefetchFlag(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var experimentalMetadataPrefetch string - cmd, err := NewRootCmd(func(cfg *cfg.Config, _, _ string) error { - experimentalMetadataPrefetch = cfg.MetadataCache.ExperimentalMetadataPrefetchOnMount + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + experimentalMetadataPrefetch = mountInfo.config.MetadataCache.ExperimentalMetadataPrefetchOnMount return nil }) require.Nil(t, err) @@ -428,7 +720,7 @@ func TestArgParsing_ExperimentalMetadataPrefetchFlag_Failed(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - cmd, err := NewRootCmd(func(cfg *cfg.Config, _, _ string) error { + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { return nil }) require.Nil(t, err) @@ -478,8 +770,8 @@ func TestArgsParsing_GCSAuthFlags(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var gotConfig *cfg.Config - cmd, err := NewRootCmd(func(cfg *cfg.Config, _, _ string) error { - gotConfig = cfg + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotConfig = mountInfo.config return nil }) require.Nil(t, err) @@ -516,7 +808,7 @@ func TestArgsParsing_GCSAuthFlagsThrowsError(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - cmd, err := NewRootCmd(func(cfg *cfg.Config, _, _ string) error { + cmd, err := newRootCmd(func(_ *mountInfo, _, _ string) error { return nil }) require.Nil(t, err) @@ -535,13 +827,14 @@ func TestArgsParsing_GCSConnectionFlags(t *testing.T) { }{ { name: "Test gcs connection flags.", - args: []string{"gcsfuse", "--billing-project=abc", "--client-protocol=http2", "--custom-endpoint=www.abc.com", "--experimental-enable-json-read", "--experimental-grpc-conn-pool-size=20", "--http-client-timeout=20s", "--limit-bytes-per-sec=30", "--limit-ops-per-sec=10", "--max-conns-per-host=1000", "--max-idle-conns-per-host=20", "--sequential-read-size-mb=70", "abc", "pqr"}, + args: []string{"gcsfuse", "--billing-project=abc", "--client-protocol=http2", "--custom-endpoint=www.abc.com", "--experimental-enable-json-read", "--experimental-grpc-conn-pool-size=20", "--http-client-timeout=20s", "--limit-bytes-per-sec=30", "--limit-ops-per-sec=10", "--max-conns-per-host=1000", "--max-idle-conns-per-host=20", "--sequential-read-size-mb=70", "abc", "pqr", "--grpc-path-strategy=direct-path-only"}, expectedConfig: &cfg.Config{ GcsConnection: cfg.GcsConnectionConfig{ BillingProject: "abc", ClientProtocol: "http2", CustomEndpoint: "www.abc.com", ExperimentalEnableJsonRead: true, + GrpcPathStrategy: "direct-path-only", GrpcConnPoolSize: 20, HttpClientTimeout: 20 * time.Second, LimitBytesPerSec: 30, @@ -549,6 +842,7 @@ func TestArgsParsing_GCSConnectionFlags(t *testing.T) { MaxConnsPerHost: 1000, MaxIdleConnsPerHost: 20, SequentialReadSizeMb: 70, + EnableHttpDnsCache: true, }, }, }, @@ -561,6 +855,28 @@ func TestArgsParsing_GCSConnectionFlags(t *testing.T) { ClientProtocol: "http1", CustomEndpoint: "", ExperimentalEnableJsonRead: false, + GrpcPathStrategy: "direct-path-with-fallback", + GrpcConnPoolSize: 1, + HttpClientTimeout: 0, + LimitBytesPerSec: -1, + LimitOpsPerSec: -1, + MaxConnsPerHost: 0, + MaxIdleConnsPerHost: 100, + SequentialReadSizeMb: 200, + EnableHttpDnsCache: true, + }, + }, + }, + { + name: "test_dns_cache_disabled", + args: []string{"gcsfuse", "--enable-http-dns-cache=false", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + GcsConnection: cfg.GcsConnectionConfig{ + BillingProject: "", + ClientProtocol: "http1", + CustomEndpoint: "", + ExperimentalEnableJsonRead: false, + GrpcPathStrategy: "direct-path-with-fallback", GrpcConnPoolSize: 1, HttpClientTimeout: 0, LimitBytesPerSec: -1, @@ -568,6 +884,7 @@ func TestArgsParsing_GCSConnectionFlags(t *testing.T) { MaxConnsPerHost: 0, MaxIdleConnsPerHost: 100, SequentialReadSizeMb: 200, + EnableHttpDnsCache: false, }, }, }, @@ -576,8 +893,8 @@ func TestArgsParsing_GCSConnectionFlags(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var gotConfig *cfg.Config - cmd, err := NewRootCmd(func(cfg *cfg.Config, _, _ string) error { - gotConfig = cfg + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotConfig = mountInfo.config return nil }) require.Nil(t, err) @@ -620,7 +937,7 @@ func TestArgsParsing_GCSConnectionFlagsThrowsError(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - cmd, err := NewRootCmd(func(cfg *cfg.Config, _, _ string) error { + cmd, err := newRootCmd(func(_ *mountInfo, _, _ string) error { return nil }) require.Nil(t, err) @@ -632,30 +949,53 @@ func TestArgsParsing_GCSConnectionFlagsThrowsError(t *testing.T) { } func TestArgsParsing_FileSystemFlags(t *testing.T) { + expectedDefaultFileSystemConfig := cfg.FileSystemConfig{ + DirMode: 0755, + DisableParallelDirops: false, + ExperimentalEnableDentryCache: false, + ExperimentalEnableReaddirplus: false, + FileMode: 0644, + FuseOptions: []string{}, + Gid: -1, + IgnoreInterrupts: true, + KernelListCacheTtlSecs: 0, + InactiveMrdCacheSize: 1000, + RenameDirLimit: 0, + TempDir: "", + ExperimentalODirect: false, + Uid: -1, + } + expectedAIMLCheckpointingFileSystemConfig := expectedDefaultFileSystemConfig + expectedAIMLCheckpointingFileSystemConfig.RenameDirLimit = 200000 + expectedAIMLTrainingFileSystemConfig := expectedDefaultFileSystemConfig + hd, err := os.UserHomeDir() require.NoError(t, err) tests := []struct { - name string - args []string - expectedConfig *cfg.Config + name string + args []string + expectedConfig *cfg.Config + checkMachineType bool }{ { name: "normal", - args: []string{"gcsfuse", "--dir-mode=0777", "--disable-parallel-dirops", "--file-mode=0666", "--o", "ro", "--gid=7", "--ignore-interrupts=false", "--kernel-list-cache-ttl-secs=300", "--rename-dir-limit=10", "--temp-dir=~/temp", "--uid=8", "--precondition-errors=false", "abc", "pqr"}, + args: []string{"gcsfuse", "--dir-mode=0777", "--disable-parallel-dirops", "--experimental-enable-dentry-cache", "--experimental-enable-readdirplus", "--file-mode=0666", "--o", "ro", "--gid=7", "--ignore-interrupts=false", "--kernel-list-cache-ttl-secs=300", "--rename-dir-limit=10", "--temp-dir=~/temp", "--uid=8", "abc", "pqr"}, expectedConfig: &cfg.Config{ FileSystem: cfg.FileSystemConfig{ - DirMode: 0777, - DisableParallelDirops: true, - FileMode: 0666, - FuseOptions: []string{"ro"}, - Gid: 7, - IgnoreInterrupts: false, - KernelListCacheTtlSecs: 300, - RenameDirLimit: 10, - TempDir: cfg.ResolvedPath(path.Join(hd, "temp")), - PreconditionErrors: false, - Uid: 8, - HandleSigterm: true, + DirMode: 0777, + DisableParallelDirops: true, + ExperimentalEnableDentryCache: true, + ExperimentalEnableReaddirplus: true, + FileMode: 0666, + FuseOptions: []string{"ro"}, + Gid: 7, + IgnoreInterrupts: false, + KernelListCacheTtlSecs: 300, + InactiveMrdCacheSize: 1000, + RenameDirLimit: 10, + TempDir: cfg.ResolvedPath(path.Join(hd, "temp")), + ExperimentalODirect: false, + Uid: 8, }, }, }, @@ -664,48 +1004,376 @@ func TestArgsParsing_FileSystemFlags(t *testing.T) { args: []string{"gcsfuse", "--dir-mode=777", "--file-mode=666", "abc", "pqr"}, expectedConfig: &cfg.Config{ FileSystem: cfg.FileSystemConfig{ - DirMode: 0777, - DisableParallelDirops: false, - FileMode: 0666, - FuseOptions: []string{}, - Gid: -1, - IgnoreInterrupts: true, - KernelListCacheTtlSecs: 0, - RenameDirLimit: 0, - TempDir: "", - PreconditionErrors: true, - Uid: -1, - HandleSigterm: true, + DirMode: 0777, + DisableParallelDirops: false, + ExperimentalEnableDentryCache: false, + ExperimentalEnableReaddirplus: false, + FileMode: 0666, + FuseOptions: []string{}, + Gid: -1, + IgnoreInterrupts: true, + KernelListCacheTtlSecs: 0, + InactiveMrdCacheSize: 1000, + RenameDirLimit: 0, + TempDir: "", + ExperimentalODirect: false, + Uid: -1, + }, + }, + }, + { + name: "high performance defaults with rename dir options with autoconfig enabled", + args: []string{"gcsfuse", "--dir-mode=777", "--machine-type=a3-highgpu-8g", "--disable-autoconfig=false", "--file-mode=666", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: cfg.FileSystemConfig{ + DirMode: 0777, + DisableParallelDirops: false, + ExperimentalEnableDentryCache: false, + ExperimentalEnableReaddirplus: false, + FileMode: 0666, + FuseOptions: []string{}, + Gid: -1, + IgnoreInterrupts: true, + KernelListCacheTtlSecs: 0, + InactiveMrdCacheSize: 1000, + RenameDirLimit: 200000, + TempDir: "", + ExperimentalODirect: false, + Uid: -1, + }, + MachineType: "a3-highgpu-8g", + }, + checkMachineType: true, + }, + { + name: "high performance defaults with rename dir options with autoconfig disabled", + args: []string{"gcsfuse", "--dir-mode=777", "--machine-type=a3-highgpu-8g", "--disable-autoconfig=true", "--file-mode=666", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: cfg.FileSystemConfig{ + DirMode: 0777, + DisableParallelDirops: false, + ExperimentalEnableDentryCache: false, + ExperimentalEnableReaddirplus: false, + FileMode: 0666, + FuseOptions: []string{}, + Gid: -1, + IgnoreInterrupts: true, + KernelListCacheTtlSecs: 0, + InactiveMrdCacheSize: 1000, + RenameDirLimit: 0, + TempDir: "", + ExperimentalODirect: false, + Uid: -1, + }, + MachineType: "a3-highgpu-8g", + }, + checkMachineType: true, + }, + { + name: "high performance defaults with overriden rename dir options", + args: []string{"gcsfuse", "--dir-mode=777", "--machine-type=a3-highgpu-8g", "--disable-autoconfig=false", "--rename-dir-limit=15000", "--file-mode=666", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: cfg.FileSystemConfig{ + DirMode: 0777, + DisableParallelDirops: false, + ExperimentalEnableDentryCache: false, + ExperimentalEnableReaddirplus: false, + FileMode: 0666, + FuseOptions: []string{}, + Gid: -1, + IgnoreInterrupts: true, + KernelListCacheTtlSecs: 0, + InactiveMrdCacheSize: 1000, + RenameDirLimit: 15000, + TempDir: "", + ExperimentalODirect: false, + Uid: -1, }, + MachineType: "a3-highgpu-8g", + }, + checkMachineType: true, + }, + { + name: "profile_checkpointing_with_low_machine_type", + args: []string{"gcsfuse", "--profile=" + cfg.ProfileAIMLCheckpointing, "--machine-type=machine-type-1", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: expectedAIMLCheckpointingFileSystemConfig, + Profile: cfg.ProfileAIMLCheckpointing, + MachineType: "machine-type-1", + }, + checkMachineType: true, + }, + { + name: "profile_checkpointing_with_high_machine_type", + args: []string{"gcsfuse", "--profile=" + cfg.ProfileAIMLCheckpointing, "--machine-type=a3-highgpu-8g", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: expectedAIMLCheckpointingFileSystemConfig, + Profile: cfg.ProfileAIMLCheckpointing, + MachineType: "a3-highgpu-8g", + }, + checkMachineType: true, + }, + { + name: "profile_training_with_machine_type", + args: []string{"gcsfuse", "--profile=" + cfg.ProfileAIMLTraining, "--machine-type=machine-type-1", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: expectedAIMLTrainingFileSystemConfig, + Profile: cfg.ProfileAIMLTraining, + MachineType: "machine-type-1", + }, + checkMachineType: true, + }, + { + name: "profile_checkpointing_without_machine_type", + args: []string{"gcsfuse", "--profile=" + cfg.ProfileAIMLCheckpointing, "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: expectedAIMLCheckpointingFileSystemConfig, + Profile: cfg.ProfileAIMLCheckpointing, + }, + }, + { + name: "machine_type_without_profile", + args: []string{"gcsfuse", "--machine-type=machine-type-1", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: expectedDefaultFileSystemConfig, + MachineType: "machine-type-1", }, }, { name: "default", args: []string{"gcsfuse", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: expectedDefaultFileSystemConfig, + MachineType: "", + Profile: "", + }, + }, + { + name: "Test file system o-direct flag enabled.", + args: []string{"gcsfuse", "--experimental-o-direct", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: cfg.FileSystemConfig{ + DirMode: 0755, + FileMode: 0644, + FuseOptions: []string{}, + Gid: -1, + IgnoreInterrupts: true, + InactiveMrdCacheSize: 1000, + ExperimentalODirect: true, + Uid: -1, + }, + }, + }, + { + name: "Test file system experimental-enable-pirlo flag enabled.", + args: []string{"gcsfuse", "--experimental-enable-pirlo", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: cfg.FileSystemConfig{ + DirMode: 0755, + FileMode: 0644, + FuseOptions: []string{}, + Gid: -1, + IgnoreInterrupts: true, + InactiveMrdCacheSize: 1000, + Uid: -1, + ExperimentalEnablePirlo: true, + }, + }, + }, + { + name: "Test file system max-read-ahead-kb flag enabled.", + args: []string{"gcsfuse", "--max-read-ahead-kb=1024", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: cfg.FileSystemConfig{ + DirMode: 0755, + FileMode: 0644, + FuseOptions: []string{}, + Gid: -1, + IgnoreInterrupts: true, + InactiveMrdCacheSize: 1000, + ExperimentalODirect: false, + Uid: -1, + MaxReadAheadKb: 1024, + }, + }, + }, + { + name: "Test file system max-read-ahead-kb flag disabled.", + args: []string{"gcsfuse", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: cfg.FileSystemConfig{ + DirMode: 0755, + FileMode: 0644, + FuseOptions: []string{}, + Gid: -1, + IgnoreInterrupts: true, + InactiveMrdCacheSize: 1000, + Uid: -1, + MaxReadAheadKb: 0, + }, + }, + }, + { + name: "Test file system max-background flag enabled.", + args: []string{"gcsfuse", "--max-background=512", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: cfg.FileSystemConfig{ + DirMode: 0755, + FileMode: 0644, + FuseOptions: []string{}, + Gid: -1, + IgnoreInterrupts: true, + InactiveMrdCacheSize: 1000, + ExperimentalODirect: false, + Uid: -1, + MaxBackground: 512, + }, + }, + }, + { + name: "Test file system max-background flag disabled.", + args: []string{"gcsfuse", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: cfg.FileSystemConfig{ + DirMode: 0755, + FileMode: 0644, + FuseOptions: []string{}, + Gid: -1, + IgnoreInterrupts: true, + InactiveMrdCacheSize: 1000, + ExperimentalODirect: false, + Uid: -1, + MaxBackground: 0, + }, + }, + }, + { + name: "Test file system congestion-threshold flag enabled.", + args: []string{"gcsfuse", "--congestion-threshold=256", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: cfg.FileSystemConfig{ + DirMode: 0755, + FileMode: 0644, + FuseOptions: []string{}, + Gid: -1, + IgnoreInterrupts: true, + InactiveMrdCacheSize: 1000, + ExperimentalODirect: false, + Uid: -1, + CongestionThreshold: 256, + }, + }, + }, + { + name: "Test file system congestion-threshold flag disabled.", + args: []string{"gcsfuse", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: cfg.FileSystemConfig{ + DirMode: 0755, + FileMode: 0644, + FuseOptions: []string{}, + Gid: -1, + IgnoreInterrupts: true, + InactiveMrdCacheSize: 1000, + ExperimentalODirect: false, + Uid: -1, + CongestionThreshold: 0, + }, + }, + }, + { + name: "Test file system enable-kernel-reader flag enabled.", + args: []string{"gcsfuse", "--enable-kernel-reader", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: cfg.FileSystemConfig{ + DirMode: 0755, + FileMode: 0644, + FuseOptions: []string{}, + Gid: -1, + IgnoreInterrupts: true, + InactiveMrdCacheSize: 1000, + ExperimentalODirect: false, + Uid: -1, + EnableKernelReader: true, + }, + }, + }, + { + name: "Test file system enable-kernel-reader flag disabled.", + args: []string{"gcsfuse", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: cfg.FileSystemConfig{ + DirMode: 0755, + FileMode: 0644, + FuseOptions: []string{}, + Gid: -1, + IgnoreInterrupts: true, + InactiveMrdCacheSize: 1000, + ExperimentalODirect: false, + Uid: -1, + EnableKernelReader: false, + }, + }, + }, + { + name: "Test file system kernel-params-file flag.", + args: []string{"gcsfuse", "--kernel-params-file=/tmp/params", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: cfg.FileSystemConfig{ + DirMode: 0755, + FileMode: 0644, + FuseOptions: []string{}, + Gid: -1, + IgnoreInterrupts: true, + InactiveMrdCacheSize: 1000, + ExperimentalODirect: false, + Uid: -1, + KernelParamsFile: "/tmp/params", + }, + }, + }, + { + name: "Test file system kernel-params-file from config file.", + args: []string{"gcsfuse", "--config-file", createTempConfigFile(t, "file-system:\n kernel-params-file: /tmp/config_params"), "abc", "pqr"}, expectedConfig: &cfg.Config{ FileSystem: cfg.FileSystemConfig{ - DirMode: 0755, - DisableParallelDirops: false, - FileMode: 0644, - FuseOptions: []string{}, - Gid: -1, - IgnoreInterrupts: true, - KernelListCacheTtlSecs: 0, - RenameDirLimit: 0, - TempDir: "", - PreconditionErrors: true, - Uid: -1, - HandleSigterm: true, + DirMode: 0755, + FileMode: 0644, + FuseOptions: []string{}, + Gid: -1, + IgnoreInterrupts: true, + InactiveMrdCacheSize: 1000, + ExperimentalODirect: false, + Uid: -1, + KernelParamsFile: "/tmp/config_params", }, }, }, + { + name: "cli_flag_overrides_config_file", + args: []string{"gcsfuse", "--config-file", createTempConfigFile(t, "machine-type: config-file-type"), "--machine-type=cli-type", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: expectedDefaultFileSystemConfig, + MachineType: "cli-type", + }, + checkMachineType: true, + }, + { + name: "config_file_overrides_metadata_server", + args: []string{"gcsfuse", "--config-file", createTempConfigFile(t, "machine-type: config-file-type"), "abc", "pqr"}, + expectedConfig: &cfg.Config{ + FileSystem: expectedDefaultFileSystemConfig, + MachineType: "config-file-type", + }, + checkMachineType: true, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var gotConfig *cfg.Config - cmd, err := NewRootCmd(func(cfg *cfg.Config, _, _ string) error { - gotConfig = cfg + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotConfig = mountInfo.config return nil }) require.Nil(t, err) @@ -715,6 +1383,10 @@ func TestArgsParsing_FileSystemFlags(t *testing.T) { if assert.NoError(t, err) { assert.Equal(t, tc.expectedConfig.FileSystem, gotConfig.FileSystem) + if tc.checkMachineType { + assert.Equal(t, tc.expectedConfig.MachineType, gotConfig.MachineType) + } + assert.Equal(t, tc.expectedConfig.Profile, gotConfig.Profile) } }) } @@ -745,11 +1417,19 @@ func TestArgsParsing_FileSystemFlagsThrowsError(t *testing.T) { name: "invalid_disable_parallel_dirops", args: []string{"gcsfuse", "--disable-parallel-dirops=abc", "abc", "pqr"}, }, + { + name: "invalid_experimental_enable_readdirplus", + args: []string{"gcsfuse", "--experimental-enable-readdirplus=abc", "abc", "pqr"}, + }, + { + name: "invalid_experimental_enable_dentry_cache", + args: []string{"gcsfuse", "--experimental-enable-dentry-cache=abc", "abc", "pqr"}, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - cmd, err := NewRootCmd(func(cfg *cfg.Config, _, _ string) error { + cmd, err := newRootCmd(func(_ *mountInfo, _, _ string) error { return nil }) require.Nil(t, err) @@ -785,8 +1465,8 @@ func TestArgsParsing_ListFlags(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var gotConfig *cfg.Config - cmd, err := NewRootCmd(func(cfg *cfg.Config, _, _ string) error { - gotConfig = cfg + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotConfig = mountInfo.config return nil }) require.Nil(t, err) @@ -822,8 +1502,8 @@ func TestArgsParsing_EnableHNSFlags(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var gotEnableHNS bool - cmd, err := NewRootCmd(func(cfg *cfg.Config, _, _ string) error { - gotEnableHNS = cfg.EnableHns + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotEnableHNS = mountInfo.config.EnableHns return nil }) require.Nil(t, err) @@ -838,56 +1518,103 @@ func TestArgsParsing_EnableHNSFlags(t *testing.T) { } } -func TestArgsParsing_MetricsFlags(t *testing.T) { +func TestArgsParsing_EnableTypeCacheDeprecationFlags(t *testing.T) { tests := []struct { - name string - args []string - expected *cfg.MetricsConfig + name string + args []string + expectedEnableTypeCacheDeprecation bool }{ { - name: "default", - args: []string{"gcsfuse", "abc", "pqr"}, - expected: &cfg.MetricsConfig{ - EnableOtel: false, - }, - }, - { - name: "enable_otel_normal", - args: []string{"gcsfuse", "--enable-otel", "abc", "pqr"}, - expected: &cfg.MetricsConfig{ - EnableOtel: true, - }, + name: "explicitly_disabled", + args: []string{"gcsfuse", "--enable-type-cache-deprecation=false", "abc", "pqr"}, + expectedEnableTypeCacheDeprecation: false, }, { - name: "enable_otel_false", - args: []string{"gcsfuse", "--enable-otel=false", "abc", "pqr"}, - expected: &cfg.MetricsConfig{ - EnableOtel: false, - }, + name: "default", + args: []string{"gcsfuse", "abc", "pqr"}, + expectedEnableTypeCacheDeprecation: true, }, - { - name: "enable_otel_false", - args: []string{"gcsfuse", "--enable-otel=true", "abc", "pqr"}, - expected: &cfg.MetricsConfig{ - EnableOtel: true, - }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var gotEnableTypeCacheDeprecation bool + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotEnableTypeCacheDeprecation = mountInfo.config.EnableTypeCacheDeprecation + return nil + }) + require.Nil(t, err) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) + + err = cmd.Execute() + + if assert.NoError(t, err) { + assert.Equal(t, tc.expectedEnableTypeCacheDeprecation, gotEnableTypeCacheDeprecation) + } + }) + } +} + +func TestArgsParsing_EnableUnsupportedPathSupport(t *testing.T) { + tests := []struct { + name string + args []string + expectedUnsupportedPathSupport bool + }{ + { + name: "normal", + args: []string{"gcsfuse", "--enable-unsupported-path-support=false", "abc", "pqr"}, + expectedUnsupportedPathSupport: false, }, { - name: "cloud-metrics-export-interval-secs-positive", - args: []string{"gcsfuse", "--cloud-metrics-export-interval-secs=10", "abc", "pqr"}, - expected: &cfg.MetricsConfig{CloudMetricsExportIntervalSecs: 10}, + name: "default", + args: []string{"gcsfuse", "abc", "pqr"}, + expectedUnsupportedPathSupport: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var gotUnsupportedPathSupport bool + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotUnsupportedPathSupport = mountInfo.config.EnableUnsupportedPathSupport + return nil + }) + require.Nil(t, err) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) + + err = cmd.Execute() + + if assert.NoError(t, err) { + assert.Equal(t, tc.expectedUnsupportedPathSupport, gotUnsupportedPathSupport) + } + }) + } +} + +func TestArgsParsing_EnableStandardSymlinks(t *testing.T) { + tests := []struct { + name string + args []string + expectedEnableStandardSymlinks bool + }{ + { + name: "default", + args: []string{"gcsfuse", "abc", "pqr"}, + expectedEnableStandardSymlinks: true, }, { - name: "stackdriver-export-interval-positive", - args: []string{"gcsfuse", "--stackdriver-export-interval=10h", "abc", "pqr"}, - expected: &cfg.MetricsConfig{CloudMetricsExportIntervalSecs: 10 * 3600, StackdriverExportInterval: time.Duration(10) * time.Hour}, + name: "normal", + args: []string{"gcsfuse", "--enable-standard-symlinks=false", "abc", "pqr"}, + expectedEnableStandardSymlinks: false, }, } + for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - var gotConfig *cfg.Config - cmd, err := NewRootCmd(func(cfg *cfg.Config, _, _ string) error { - gotConfig = cfg + var gotEnableStandardSymlinks bool + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotEnableStandardSymlinks = mountInfo.config.EnableStandardSymlinks return nil }) require.Nil(t, err) @@ -896,55 +1623,276 @@ func TestArgsParsing_MetricsFlags(t *testing.T) { err = cmd.Execute() if assert.NoError(t, err) { - assert.Equal(t, tc.expected, &gotConfig.Metrics) + assert.Equal(t, tc.expectedEnableStandardSymlinks, gotEnableStandardSymlinks) } }) } } -func TestArgsParsing_MetricsViewConfig(t *testing.T) { +func TestArgsParsing_EnableGoogleLibAuthFlag(t *testing.T) { + tests := []struct { + name string + args []string + expectedEnableGoogleLibAuth bool + }{ + { + name: "default", + args: []string{"gcsfuse", "abc", "pqr"}, + expectedEnableGoogleLibAuth: true, + }, + { + name: "normal", + args: []string{"gcsfuse", "--enable-google-lib-auth=false", "abc", "pqr"}, + expectedEnableGoogleLibAuth: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var gotEnableGoogleLibAuth bool + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotEnableGoogleLibAuth = mountInfo.config.EnableGoogleLibAuth + return nil + }) + require.Nil(t, err) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) + + err = cmd.Execute() + + if assert.NoError(t, err) { + assert.Equal(t, tc.expectedEnableGoogleLibAuth, gotEnableGoogleLibAuth) + } + }) + } +} + +func TestArgsParsing_EnableAtomicRenameObjectFlag(t *testing.T) { + tests := []struct { + name string + args []string + expectedEnableAtomicRenameObject bool + }{ + { + name: "default", + args: []string{"gcsfuse", "abc", "pqr"}, + expectedEnableAtomicRenameObject: true, + }, + { + name: "normal", + args: []string{"gcsfuse", "--enable-atomic-rename-object=false", "abc", "pqr"}, + expectedEnableAtomicRenameObject: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var gotEnableAtomicRenameObject bool + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotEnableAtomicRenameObject = mountInfo.config.EnableAtomicRenameObject + return nil + }) + require.Nil(t, err) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) + + err = cmd.Execute() + + if assert.NoError(t, err) { + assert.Equal(t, tc.expectedEnableAtomicRenameObject, gotEnableAtomicRenameObject) + } + }) + } +} + +func TestArgsParsing_DisableListAccessCheckFlag(t *testing.T) { + tests := []struct { + name string + args []string + expectedDisableListAccessCheck bool + }{ + { + name: "default", + args: []string{"gcsfuse", "abc", "pqr"}, + expectedDisableListAccessCheck: true, + }, + { + name: "normal", + args: []string{"gcsfuse", "--disable-list-access-check=false", "abc", "pqr"}, + expectedDisableListAccessCheck: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var gotDisableListAccessCheck bool + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotDisableListAccessCheck = mountInfo.config.DisableListAccessCheck + return nil + }) + require.Nil(t, err) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) + + err = cmd.Execute() + + if assert.NoError(t, err) { + assert.Equal(t, tc.expectedDisableListAccessCheck, gotDisableListAccessCheck) + } + }) + } +} + +func TestArgsParsing_EnableNewReaderFlag(t *testing.T) { + tests := []struct { + name string + args []string + expectedEnableNewReader bool + }{ + { + name: "default", + args: []string{"gcsfuse", "abc", "pqr"}, + expectedEnableNewReader: true, + }, + { + name: "normal", + args: []string{"gcsfuse", "--enable-new-reader=false", "abc", "pqr"}, + expectedEnableNewReader: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var gotEnableNewReader bool + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotEnableNewReader = mountInfo.config.EnableNewReader + return nil + }) + require.Nil(t, err) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) + + err = cmd.Execute() + + require.NoError(t, err) + assert.Equal(t, tc.expectedEnableNewReader, gotEnableNewReader) + }) + } +} + +func TestArgsParsing_MetricsFlags(t *testing.T) { tests := []struct { name string - cfgFile string + args []string expected *cfg.MetricsConfig }{ { - name: "default", - cfgFile: "empty.yml", + name: "cloud-metrics-export-interval-secs-positive", + args: []string{"gcsfuse", "--cloud-metrics-export-interval-secs=10", "abc", "pqr"}, + expected: &cfg.MetricsConfig{ + CloudMetricsExportIntervalSecs: 10, + Workers: 3, + BufferSize: 256, + ExperimentalEnableGrpcMetrics: true, + }, + }, + { + name: "stackdriver-export-interval-positive", + args: []string{"gcsfuse", "--stackdriver-export-interval=10h", "abc", "pqr"}, expected: &cfg.MetricsConfig{ - EnableOtel: false, + CloudMetricsExportIntervalSecs: 10 * 3600, + StackdriverExportInterval: time.Duration(10) * time.Hour, + Workers: 3, + BufferSize: 256, + ExperimentalEnableGrpcMetrics: true, }, }, { - name: "enable_otel_true", - cfgFile: "enable_otel_true.yml", + name: "use_new_metric_names", + args: []string{"gcsfuse", "--metrics-use-new-names=true", "abc", "pqr"}, expected: &cfg.MetricsConfig{ - EnableOtel: true, + UseNewNames: true, + Workers: 3, + BufferSize: 256, + ExperimentalEnableGrpcMetrics: true, }, }, { - name: "enable_otel_false", - cfgFile: "enable_otel_false.yml", + name: "metrics_workers_non_default", + args: []string{"gcsfuse", "--metrics-workers=10", "abc", "pqr"}, expected: &cfg.MetricsConfig{ - EnableOtel: false, + Workers: 10, + BufferSize: 256, + ExperimentalEnableGrpcMetrics: true, }, }, + { + name: "metrics_buffer_size_non_default", + args: []string{"gcsfuse", "--metrics-buffer-size=1024", "abc", "pqr"}, + expected: &cfg.MetricsConfig{ + Workers: 3, + BufferSize: 1024, + ExperimentalEnableGrpcMetrics: true, + }, + }, + { + name: "enable_grpc_metrics_non_default", + args: []string{"gcsfuse", "--experimental-enable-grpc-metrics=false", "abc", "pqr"}, + expected: &cfg.MetricsConfig{ + Workers: 3, + BufferSize: 256, + ExperimentalEnableGrpcMetrics: false, + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var gotConfig *cfg.Config + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotConfig = mountInfo.config + return nil + }) + require.Nil(t, err) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) + + err = cmd.Execute() + + if assert.NoError(t, err) { + assert.Equal(t, tc.expected, &gotConfig.Metrics) + } + }) + } +} + +func TestArgsParsing_MetricsViewConfig(t *testing.T) { + tests := []struct { + name string + cfgFile string + expected *cfg.MetricsConfig + }{ + { + name: "default", + cfgFile: "empty.yml", + expected: &cfg.MetricsConfig{Workers: 3, BufferSize: 256, ExperimentalEnableGrpcMetrics: true}, + }, { name: "cloud-metrics-export-interval-secs-positive", cfgFile: "metrics_export_interval_positive.yml", - expected: &cfg.MetricsConfig{CloudMetricsExportIntervalSecs: 100}, + expected: &cfg.MetricsConfig{CloudMetricsExportIntervalSecs: 100, Workers: 3, BufferSize: 256, ExperimentalEnableGrpcMetrics: true}, }, { - name: "stackdriver-export-interval-positive", - cfgFile: "stackdriver_export_interval_positive.yml", - expected: &cfg.MetricsConfig{CloudMetricsExportIntervalSecs: 12 * 3600, StackdriverExportInterval: 12 * time.Hour}, + name: "stackdriver-export-interval-positive", + cfgFile: "stackdriver_export_interval_positive.yml", + expected: &cfg.MetricsConfig{ + CloudMetricsExportIntervalSecs: 12 * 3600, + StackdriverExportInterval: 12 * time.Hour, + Workers: 3, + BufferSize: 256, + ExperimentalEnableGrpcMetrics: true, + }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var gotConfig *cfg.Config - cmd, err := NewRootCmd(func(cfg *cfg.Config, _, _ string) error { - gotConfig = cfg + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotConfig = mountInfo.config return nil }) require.Nil(t, err) @@ -959,6 +1907,42 @@ func TestArgsParsing_MetricsViewConfig(t *testing.T) { } } +func TestArgsParsingTraceConfig(t *testing.T) { + tests := []struct { + name string + cfgFile string + expected *cfg.TraceConfig + }{ + { + name: "default", + cfgFile: "empty.yml", + expected: &cfg.TraceConfig{Exporters: []string{"gcpexporter"}, SamplingRatio: 0, ProjectId: ""}, + }, + { + name: "sanitize_trace_exporters.yml", + cfgFile: "sanitize_trace_exporters.yml", + expected: &cfg.TraceConfig{Exporters: []string{"gcpexporter", "stdout"}, SamplingRatio: 0.5, ProjectId: "gcp-sample-test"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var gotConfig *cfg.Config + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotConfig = mountInfo.config + return nil + }) + require.Nil(t, err) + cmd.SetArgs(convertToPosixArgs([]string{"gcsfuse", fmt.Sprintf("--config-file=testdata/monitoring_config/%s", tc.cfgFile), "abc", "pqr"}, cmd)) + + err = cmd.Execute() + + require.NoError(t, err) + assert.Equal(t, tc.expected, &gotConfig.Trace) + }) + } +} + func TestArgsParsing_MetadataCacheFlags(t *testing.T) { tests := []struct { name string @@ -967,16 +1951,20 @@ func TestArgsParsing_MetadataCacheFlags(t *testing.T) { }{ { name: "normal", - args: []string{"gcsfuse", "--stat-cache-capacity=2000", "--stat-cache-ttl=2m", "--type-cache-ttl=1m20s", "--enable-nonexistent-type-cache", "--experimental-metadata-prefetch-on-mount=async", "--stat-cache-max-size-mb=15", "--metadata-cache-ttl-secs=25", "--type-cache-max-size-mb=30", "abc", "pqr"}, + args: []string{"gcsfuse", "--stat-cache-capacity=2000", "--stat-cache-ttl=2m", "--type-cache-ttl=1m20s", "--enable-nonexistent-type-cache", "--experimental-metadata-prefetch-on-mount=async", "--metadata-prefetch-max-workers=3", "--enable-metadata-prefetch=true", "--metadata-prefetch-entries-limit=500", "--stat-cache-max-size-mb=15", "--metadata-cache-ttl-secs=25", "--metadata-cache-negative-ttl-secs=20", "--type-cache-max-size-mb=30", "abc", "pqr"}, expectedConfig: &cfg.Config{ MetadataCache: cfg.MetadataCacheConfig{ DeprecatedStatCacheCapacity: 2000, DeprecatedStatCacheTtl: 2 * time.Minute, DeprecatedTypeCacheTtl: 80 * time.Second, EnableNonexistentTypeCache: true, + MetadataPrefetchMaxWorkers: 3, + EnableMetadataPrefetch: true, + MetadataPrefetchEntriesLimit: 500, ExperimentalMetadataPrefetchOnMount: "async", StatCacheMaxSizeMb: 15, TtlSecs: 25, + NegativeTtlSecs: 20, TypeCacheMaxSizeMb: 30, }, }, @@ -991,19 +1979,103 @@ func TestArgsParsing_MetadataCacheFlags(t *testing.T) { DeprecatedTypeCacheTtl: 60 * time.Second, EnableNonexistentTypeCache: false, ExperimentalMetadataPrefetchOnMount: "disabled", - StatCacheMaxSizeMb: 32, + MetadataPrefetchMaxWorkers: 10, + EnableMetadataPrefetch: false, + MetadataPrefetchEntriesLimit: 5000, + StatCacheMaxSizeMb: 34, + TtlSecs: 60, + NegativeTtlSecs: 5, + TypeCacheMaxSizeMb: 4, + }, + }, + }, + { + name: "high_performance_default_config_values_with_autoconfig_disabled", + args: []string{"gcsfuse", "--machine-type=a3-highgpu-8g", "--disable-autoconfig=true", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + MetadataCache: cfg.MetadataCacheConfig{ + DeprecatedStatCacheCapacity: 20460, + DeprecatedStatCacheTtl: 60 * time.Second, + DeprecatedTypeCacheTtl: 60 * time.Second, + EnableNonexistentTypeCache: false, + ExperimentalMetadataPrefetchOnMount: "disabled", + MetadataPrefetchMaxWorkers: 10, + EnableMetadataPrefetch: false, + MetadataPrefetchEntriesLimit: 5000, + StatCacheMaxSizeMb: 34, TtlSecs: 60, + NegativeTtlSecs: 5, + TypeCacheMaxSizeMb: 4, + }, + }, + }, + { + name: "high_performance_default_config_values_with_autoconfig_enabled", + args: []string{"gcsfuse", "--machine-type=a3-highgpu-8g", "--disable-autoconfig=false", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + MetadataCache: cfg.MetadataCacheConfig{ + DeprecatedStatCacheCapacity: 20460, + DeprecatedStatCacheTtl: 60 * time.Second, + DeprecatedTypeCacheTtl: 60 * time.Second, + EnableNonexistentTypeCache: false, + ExperimentalMetadataPrefetchOnMount: "disabled", + MetadataPrefetchMaxWorkers: 10, + EnableMetadataPrefetch: false, + MetadataPrefetchEntriesLimit: 5000, + StatCacheMaxSizeMb: 1024, + TtlSecs: 9223372036, + NegativeTtlSecs: 0, TypeCacheMaxSizeMb: 4, }, }, }, + { + name: "high_performance_default_config_values_obey_customer_flags", + args: []string{"gcsfuse", "--machine-type=a3-highgpu-8g", "--disable-autoconfig=false", "--enable-metadata-prefetch", "--stat-cache-capacity=2000", "--stat-cache-ttl=2m", "--type-cache-ttl=1m20s", "--enable-nonexistent-type-cache", "--experimental-metadata-prefetch-on-mount=async", "--stat-cache-max-size-mb=15", "--metadata-cache-ttl-secs=25", "--metadata-cache-negative-ttl-secs=20", "--type-cache-max-size-mb=30", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + MetadataCache: cfg.MetadataCacheConfig{ + DeprecatedStatCacheCapacity: 2000, + DeprecatedStatCacheTtl: 2 * time.Minute, + DeprecatedTypeCacheTtl: 80 * time.Second, + EnableNonexistentTypeCache: true, + MetadataPrefetchMaxWorkers: 10, + EnableMetadataPrefetch: true, + MetadataPrefetchEntriesLimit: 5000, + ExperimentalMetadataPrefetchOnMount: "async", + StatCacheMaxSizeMb: 15, + TtlSecs: 25, + NegativeTtlSecs: 20, + TypeCacheMaxSizeMb: 30, + }, + }, + }, + { + name: "high_performance_default_config_values_use_deprecated_flags", + args: []string{"gcsfuse", "--machine-type=a3-highgpu-8g", "--disable-autoconfig=false", "--stat-cache-capacity=2000", "--stat-cache-ttl=2m", "--type-cache-ttl=4m", "--enable-nonexistent-type-cache", "--experimental-metadata-prefetch-on-mount=async", "--metadata-cache-negative-ttl-secs=20", "--type-cache-max-size-mb=30", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + MetadataCache: cfg.MetadataCacheConfig{ + DeprecatedStatCacheCapacity: 2000, + DeprecatedStatCacheTtl: 2 * time.Minute, + DeprecatedTypeCacheTtl: 4 * time.Minute, + EnableNonexistentTypeCache: true, + ExperimentalMetadataPrefetchOnMount: "async", + MetadataPrefetchMaxWorkers: 10, + EnableMetadataPrefetch: false, + MetadataPrefetchEntriesLimit: 5000, + StatCacheMaxSizeMb: 4, + TtlSecs: 120, + NegativeTtlSecs: 20, + TypeCacheMaxSizeMb: 30, + }, + }, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var gotConfig *cfg.Config - cmd, err := NewRootCmd(func(cfg *cfg.Config, _, _ string) error { - gotConfig = cfg + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotConfig = mountInfo.config return nil }) require.Nil(t, err) @@ -1017,3 +2089,823 @@ func TestArgsParsing_MetadataCacheFlags(t *testing.T) { }) } } + +func TestArgParsing_GCSRetries(t *testing.T) { + tests := []struct { + name string + args []string + expectedConfig *cfg.Config + }{ + { + name: "Test with non default chunkTransferTimeout", + args: []string{"gcsfuse", "--chunk-transfer-timeout-secs=30", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + GcsRetries: cfg.GcsRetriesConfig{ + ChunkRetryDeadlineSecs: 120, + ChunkTransferTimeoutSecs: 30, + EnableMountRetries: false, + MaxRetryAttempts: math.MaxInt, + MaxRetrySleep: 30 * time.Second, + Multiplier: 2, + ReadStall: cfg.ReadStallGcsRetriesConfig{ + Enable: true, + InitialReqTimeout: 20 * time.Second, + MinReqTimeout: 1500 * time.Millisecond, + MaxReqTimeout: 1200 * time.Second, + ReqIncreaseRate: 15, + ReqTargetPercentile: 0.99, + }, + }, + }, + }, + { + name: "Test with non default chunkRetryDeadline", + args: []string{"gcsfuse", "--chunk-retry-deadline-secs=360", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + GcsRetries: cfg.GcsRetriesConfig{ + ChunkRetryDeadlineSecs: 360, + ChunkTransferTimeoutSecs: 10, + EnableMountRetries: false, + MaxRetryAttempts: math.MaxInt, + MaxRetrySleep: 30 * time.Second, + Multiplier: 2, + ReadStall: cfg.ReadStallGcsRetriesConfig{ + Enable: true, + InitialReqTimeout: 20 * time.Second, + MinReqTimeout: 1500 * time.Millisecond, + MaxReqTimeout: 1200 * time.Second, + ReqIncreaseRate: 15, + ReqTargetPercentile: 0.99, + }, + }, + }, + }, + { + name: "Test_with_non_default_experimental-nonrapid-folder-api-stall-retry", + args: []string{"gcsfuse", "--experimental-nonrapid-folder-api-stall-retry=true", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + GcsRetries: cfg.GcsRetriesConfig{ + ExperimentalNonrapidFolderApiStallRetry: true, + ChunkRetryDeadlineSecs: 120, + ChunkTransferTimeoutSecs: 10, + EnableMountRetries: false, + MaxRetryAttempts: math.MaxInt, + MaxRetrySleep: 30 * time.Second, + Multiplier: 2, + ReadStall: cfg.ReadStallGcsRetriesConfig{ + Enable: true, + InitialReqTimeout: 20 * time.Second, + MinReqTimeout: 1500 * time.Millisecond, + MaxReqTimeout: 1200 * time.Second, + ReqIncreaseRate: 15, + ReqTargetPercentile: 0.99, + }, + }, + }, + }, + { + name: "Test with enable-mount-retries explicitly true", + args: []string{"gcsfuse", "--enable-mount-retries=true", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + GcsRetries: cfg.GcsRetriesConfig{ + ChunkRetryDeadlineSecs: 120, + ChunkTransferTimeoutSecs: 10, + EnableMountRetries: true, + MaxRetryAttempts: math.MaxInt, + MaxRetrySleep: 30 * time.Second, + Multiplier: 2, + ReadStall: cfg.ReadStallGcsRetriesConfig{ + Enable: true, + InitialReqTimeout: 20 * time.Second, + MinReqTimeout: 1500 * time.Millisecond, + MaxReqTimeout: 1200 * time.Second, + ReqIncreaseRate: 15, + ReqTargetPercentile: 0.99, + }, + }, + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var gotConfig *cfg.Config + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotConfig = mountInfo.config + return nil + }) + require.Nil(t, err) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) + + err = cmd.Execute() + + if assert.NoError(t, err) { + assert.Equal(t, tc.expectedConfig.GcsRetries, gotConfig.GcsRetries) + } + }) + } +} + +func TestArgsParsing_ProfilerFlags(t *testing.T) { + tests := []struct { + name string + args []string + expectedConfig cfg.CloudProfilerConfig + }{ + { + name: "Default profiler config (disabled)", + args: []string{"gcsfuse", "bucket", "mountpoint"}, + expectedConfig: cfg.CloudProfilerConfig{ + Enabled: false, // Profiler is disabled by default + Label: "gcsfuse-0.0.0", + Mutex: false, // Default for --cloud-profiler-mutex + Cpu: true, // Default for --cloud-profiler-cpu + AllocatedHeap: true, // Default for --cloud-profiler-allocated-heap + Heap: true, // Default for --cloud-profiler-heap + Goroutines: false, // Default for --cloud-profiler-goroutines + ServiceName: "gcsfuse", + }, + }, + { + name: "Profiler enabled, sub-profilers default", + args: []string{"gcsfuse", "--enable-cloud-profiler", "bucket", "mountpoint"}, + expectedConfig: cfg.CloudProfilerConfig{ + Enabled: true, + Label: "gcsfuse-0.0.0", + Mutex: false, + Cpu: true, + AllocatedHeap: true, + Heap: true, + Goroutines: false, + ServiceName: "gcsfuse", + }, + }, + { + name: "Profiler enabled, all sub-profilers explicitly true and label set", + args: []string{"gcsfuse", "--enable-cloud-profiler", "--cloud-profiler-label=v1.0.0", "--cloud-profiler-mutex=true", "--cloud-profiler-cpu=true", "--cloud-profiler-allocated-heap=true", "--cloud-profiler-heap=true", "--cloud-profiler-goroutines=true", "bucket", "mountpoint"}, + expectedConfig: cfg.CloudProfilerConfig{ + Enabled: true, + Label: "v1.0.0", + Mutex: true, + Cpu: true, + AllocatedHeap: true, + Heap: true, + Goroutines: true, + ServiceName: "gcsfuse", + }, + }, + { + name: "Profiler enabled, all sub-profilers explicitly false", + args: []string{"gcsfuse", "--enable-cloud-profiler", "--cloud-profiler-mutex=false", "--cloud-profiler-cpu=false", "--cloud-profiler-allocated-heap=false", "--cloud-profiler-heap=false", "--cloud-profiler-goroutines=false", "bucket", "mountpoint"}, + expectedConfig: cfg.CloudProfilerConfig{ + Enabled: true, + Label: "gcsfuse-0.0.0", + Mutex: false, + Cpu: false, + AllocatedHeap: false, + Heap: false, + Goroutines: false, + ServiceName: "gcsfuse", + }, + }, + { + name: "Profiler explicitly disabled, some sub-profiler flags set", + args: []string{"gcsfuse", "--enable-cloud-profiler=false", "--cloud-profiler-mutex=true", "--cloud-profiler-cpu=false", "bucket", "mountpoint"}, + expectedConfig: cfg.CloudProfilerConfig{ + Enabled: false, // Master switch is off + Label: "gcsfuse-0.0.0", + Mutex: true, // Flag was parsed + Cpu: false, // Flag was parsed + AllocatedHeap: true, // Default for its flag + Heap: true, // Default for its flag + Goroutines: false, // Default for its flag + ServiceName: "gcsfuse", + }, + }, + { + name: "Profiler enabled, custom service name set", + args: []string{"gcsfuse", "--enable-cloud-profiler", "--cloud-profiler-service-name=custom_service", "bucket", "mountpoint"}, + expectedConfig: cfg.CloudProfilerConfig{ + Enabled: true, + Label: "gcsfuse-0.0.0", + Mutex: false, + Cpu: true, + AllocatedHeap: true, + Heap: true, + Goroutines: false, + ServiceName: "custom_service", + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var gotProfilerConfig cfg.CloudProfilerConfig + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotProfilerConfig = mountInfo.config.CloudProfiler + return nil + }) + require.Nil(t, err) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) + + err = cmd.Execute() + + if assert.NoError(t, err) { + assert.Equal(t, tc.expectedConfig, gotProfilerConfig) + } + }) + } +} + +func TestArgsParsing_ReadInactiveTimeoutConfig(t *testing.T) { + tests := []struct { + name string + cfgFile string + expectedTimeout time.Duration + }{ + { + name: "default", + cfgFile: "empty.yaml", + expectedTimeout: 10 * time.Second, + }, + { + name: "override_default", + cfgFile: "override.yaml", + expectedTimeout: 30 * time.Second, + }, + { + name: "override_with_grpc", + cfgFile: "override_with_grpc.yaml", + expectedTimeout: 30 * time.Second, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var gotConfig *cfg.Config + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotConfig = mountInfo.config + return nil + }) + require.Nil(t, err) + cmd.SetArgs(convertToPosixArgs([]string{"gcsfuse", fmt.Sprintf("--config-file=testdata/read_config/%s", tc.cfgFile), "abc", "pqr"}, cmd)) + + err = cmd.Execute() + + require.NoError(t, err) + assert.Equal(t, tc.expectedTimeout, gotConfig.Read.InactiveStreamTimeout) + }) + } +} + +func TestArgsParsing_WorkloadInsightFlags(t *testing.T) { + tests := []struct { + name string + args []string + expectedConfig *cfg.Config + }{ + { + name: "default", + args: []string{"gcsfuse", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + WorkloadInsight: cfg.WorkloadInsightConfig{ + Visualize: false, + OutputFile: "", + ForwardMergeThresholdMb: 0, + }, + }, + }, + { + name: "visual with output file", + args: []string{"gcsfuse", "--visualize-workload-insight=true", "--workload-insight-output-file=/tmp/insight.html", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + WorkloadInsight: cfg.WorkloadInsightConfig{ + Visualize: true, + OutputFile: "/tmp/insight.html", + }, + }, + }, + { + name: "visual without output file", + args: []string{"gcsfuse", "--visualize-workload-insight=true", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + WorkloadInsight: cfg.WorkloadInsightConfig{ + Visualize: true, + OutputFile: "", + ForwardMergeThresholdMb: 0, + }, + }, + }, + { + name: "visual with forward merge threshold", + args: []string{"gcsfuse", "--visualize-workload-insight=true", "--workload-insight-forward-merge-threshold-mb=50", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + WorkloadInsight: cfg.WorkloadInsightConfig{ + Visualize: true, + OutputFile: "", + ForwardMergeThresholdMb: 50, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var gotConfig *cfg.Config + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotConfig = mountInfo.config + return nil + }) + require.Nil(t, err) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) + + err = cmd.Execute() + + require.NoError(t, err) + assert.Equal(t, tc.expectedConfig.WorkloadInsight, gotConfig.WorkloadInsight) + }) + } +} + +func TestArgsParsing_WorkloadInsightConfigFile(t *testing.T) { + tests := []struct { + name string + cfgFile string + expectedVisualize bool + expectedOutputFile string + }{ + { + name: "default", + cfgFile: "empty.yaml", + expectedVisualize: false, + expectedOutputFile: "", + }, + { + name: "visual with output file", + cfgFile: "visual_with_output_file.yaml", + expectedVisualize: true, + expectedOutputFile: "/tmp/insight.html", + }, + { + name: "visual without output file", + cfgFile: "visual_without_output_file.yaml", + expectedVisualize: true, + expectedOutputFile: "", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var gotConfig *cfg.Config + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotConfig = mountInfo.config + return nil + }) + require.Nil(t, err) + cmd.SetArgs(convertToPosixArgs([]string{"gcsfuse", fmt.Sprintf("--config-file=testdata/workload_insight_config/%s", tc.cfgFile), "abc", "pqr"}, cmd)) + + err = cmd.Execute() + + require.NoError(t, err) + assert.Equal(t, tc.expectedVisualize, gotConfig.WorkloadInsight.Visualize) + assert.Equal(t, tc.expectedOutputFile, gotConfig.WorkloadInsight.OutputFile) + }) + } +} + +func TestMountInfoPopulation(t *testing.T) { + testCases := []struct { + name string + cliArgs []string + configFilePath string + validateMountInfo func(t *testing.T, mi *mountInfo) + expectedResolvedGid int64 + expectedResolvedAppName string + }{ + { + name: "CLI flags only", + cliArgs: []string{"--app-name=cli-app", "--foreground", "--gid=1001"}, + validateMountInfo: func(t *testing.T, mi *mountInfo) { + assert.Contains(t, mi.cliFlags, "app-name") + assert.Equal(t, "cli-app", mi.cliFlags["app-name"]) + assert.Contains(t, mi.cliFlags, "foreground") + assert.Equal(t, "true", mi.cliFlags["foreground"]) + assert.Contains(t, mi.cliFlags, "gid") + assert.Equal(t, "1001", mi.cliFlags["gid"]) + assert.Empty(t, mi.configFileFlags) + }, + expectedResolvedGid: 1001, + expectedResolvedAppName: "cli-app", + }, + { + name: "Config file only", + configFilePath: "testdata/mount_info_population/config_file_only.yaml", + validateMountInfo: func(t *testing.T, mi *mountInfo) { + assert.NotContains(t, mi.cliFlags, "app-name") + assert.NotContains(t, mi.cliFlags, "gid") + assert.Contains(t, mi.configFileFlags, "app-name") + assert.Equal(t, "config-app", mi.configFileFlags["app-name"]) + fsFlags, ok := mi.configFileFlags["file-system"].(map[string]any) + require.True(t, ok) + assert.Equal(t, 1002, fsFlags["gid"]) + }, + expectedResolvedGid: 1002, + expectedResolvedAppName: "config-app", + }, + { + name: "CLI flags override config file", + cliArgs: []string{"--app-name=cli-app-override", "--gid=1003"}, + configFilePath: "testdata/mount_info_population/cli_override_config.yaml", + validateMountInfo: func(t *testing.T, mi *mountInfo) { + // Check CLI flags + assert.Equal(t, "cli-app-override", mi.cliFlags["app-name"]) + assert.Equal(t, "1003", mi.cliFlags["gid"]) + + // Check config file flags + assert.Equal(t, "config-app", mi.configFileFlags["app-name"]) + fsFlags, ok := mi.configFileFlags["file-system"].(map[string]any) + require.True(t, ok) + assert.Equal(t, 1002, fsFlags["gid"]) + logFlags, ok := mi.configFileFlags["logging"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "error", logFlags["severity"]) + }, + expectedResolvedGid: 1003, // CLI overrides config + expectedResolvedAppName: "cli-app-override", // CLI overrides config + }, + { + name: "Defaults when no flags or config", + validateMountInfo: func(t *testing.T, mi *mountInfo) { + assert.Empty(t, mi.cliFlags) + assert.Empty(t, mi.configFileFlags) + }, + expectedResolvedGid: -1, // Default value + expectedResolvedAppName: "", // Default value + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var capturedMountInfo *mountInfo + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + capturedMountInfo = mountInfo + return nil + }) + require.NoError(t, err) + + args := []string{"gcsfuse"} + if tc.configFilePath != "" { + // Use the provided config file path from testdata. + args = append(args, "--config-file", tc.configFilePath) + } + args = append(args, tc.cliArgs...) + args = append(args, "my-bucket", "/mnt/gcs") + cmd.SetArgs(convertToPosixArgs(args, cmd)) + + require.NoError(t, cmd.Execute()) + + require.NotNil(t, capturedMountInfo) + tc.validateMountInfo(t, capturedMountInfo) + assert.Equal(t, tc.expectedResolvedGid, capturedMountInfo.config.FileSystem.Gid) + assert.Equal(t, tc.expectedResolvedAppName, capturedMountInfo.config.AppName) + }) + } +} + +func TestGetCliFlags(t *testing.T) { + testCases := []struct { + name string + setupFlags func(t *testing.T, fs *pflag.FlagSet) + backgroundMode bool + expectedCliFlags map[string]string + unexpectedCliFlag string + }{ + { + name: "No flags set", + setupFlags: func(t *testing.T, fs *pflag.FlagSet) {}, + backgroundMode: false, + expectedCliFlags: map[string]string{}, + }, + { + name: "Some flags set", + setupFlags: func(t *testing.T, fs *pflag.FlagSet) { + fs.String("app-name", "", "") + require.NoError(t, fs.Set("app-name", "test-app")) + }, + backgroundMode: false, + expectedCliFlags: map[string]string{ + "app-name": "test-app", + }, + }, + { + name: "Foreground flag set in foreground mode", + setupFlags: func(t *testing.T, fs *pflag.FlagSet) { + fs.Bool("foreground", false, "") + require.NoError(t, fs.Set("foreground", "true")) + }, + backgroundMode: false, + expectedCliFlags: map[string]string{"foreground": "true"}, + }, + { + name: "Foreground flag set in background mode", + setupFlags: func(t *testing.T, fs *pflag.FlagSet) { + fs.Bool("foreground", false, "") + require.NoError(t, fs.Set("foreground", "true")) + }, + backgroundMode: true, + expectedCliFlags: map[string]string{}, + unexpectedCliFlag: "foreground", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if tc.backgroundMode { + t.Setenv(logger.GCSFuseInBackgroundMode, "true") + } + flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) + tc.setupFlags(t, flagSet) + + cliFlags := getCliFlags(flagSet) + + assert.Equal(t, tc.expectedCliFlags, cliFlags) + if tc.unexpectedCliFlag != "" { + _, ok := cliFlags[tc.unexpectedCliFlag] + assert.False(t, ok, "unexpected flag %q found", tc.unexpectedCliFlag) + } + }) + } +} + +func TestGetConfigFileFlags(t *testing.T) { + testCases := []struct { + name string + defaults map[string]any + filePath string + noFile bool + expected map[string]any + expectNil bool + }{ + { + name: "No config file", + noFile: true, + expectNil: true, + }, + { + name: "Empty config file", + defaults: map[string]any{"key1": "default"}, + filePath: "testdata/get_config_file_flags/empty.yaml", + expected: map[string]any{}, + }, + { + name: "Default values are ignored", + defaults: map[string]any{"default_key": "default_value"}, + filePath: "testdata/get_config_file_flags/simple_values.yaml", + expected: map[string]any{"key1": "value1", "key2": 123}, + }, + { + name: "Config file with nested values", + filePath: "testdata/get_config_file_flags/nested_values.yaml", + expected: map[string]any{ + "logging": map[string]any{ + "file-path": "/var/log/gcsfuse.log", + "format": "json", + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + inputViper := viper.New() + + if !tc.noFile { + // Set defaults on the input viper to simulate the real scenario + for key, value := range tc.defaults { + inputViper.SetDefault(key, value) + } + + // Configure viper to use the testdata file and read it + inputViper.SetConfigFile(tc.filePath) + inputViper.SetConfigType("yaml") // Ensure inputViper also knows the config type + require.NoError(t, inputViper.ReadInConfig()) + } + + got := getConfigFileFlags(inputViper) + + if tc.expectNil { + assert.Nil(t, got) + } else { + assert.Equal(t, tc.expected, got) + } + }) + } +} + +func TestArgsParsing_DummyIOFlags(t *testing.T) { + tests := []struct { + name string + args []string + expectedConfig *cfg.Config + }{ + { + name: "default", + args: []string{"gcsfuse", "abc", "pqr"}, + expectedConfig: &cfg.Config{ + DummyIo: cfg.DummyIoConfig{ + Enable: false, + ReaderLatency: 0, + PerMbLatency: 0, + }, + }, + }, + { + name: "normal", + args: []string{"gcsfuse", "--dummy-io-reader-latency=150ms", "--dummy-io-per-mb-latency=20ms", "--enable-dummy-io", "pqr"}, + expectedConfig: &cfg.Config{ + DummyIo: cfg.DummyIoConfig{ + Enable: true, + ReaderLatency: 150 * time.Millisecond, + PerMbLatency: 20 * time.Millisecond, + }, + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var gotConfig *cfg.Config + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotConfig = mountInfo.config + return nil + }) + require.Nil(t, err) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) + + err = cmd.Execute() + + assert.NoError(t, err) + assert.Equal(t, tc.expectedConfig.DummyIo, gotConfig.DummyIo) + }) + } +} + +func TestArgsParsing_DummyIOConfigFile(t *testing.T) { + tests := []struct { + name string + cfgFile string + expectedConfig *cfg.Config + }{ + { + name: "default", + cfgFile: "empty_file.yaml", + expectedConfig: &cfg.Config{ + DummyIo: cfg.DummyIoConfig{ + Enable: false, + ReaderLatency: 0, + PerMbLatency: 0, + }, + }, + }, + { + name: "normal", + cfgFile: "valid_config.yaml", + expectedConfig: &cfg.Config{ + DummyIo: cfg.DummyIoConfig{ + Enable: true, + ReaderLatency: 150 * time.Millisecond, + PerMbLatency: 20 * time.Millisecond, + }, + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var gotConfig *cfg.Config + cmd, err := newRootCmd(func(mountInfo *mountInfo, _, _ string) error { + gotConfig = mountInfo.config + return nil + }) + require.Nil(t, err) + cmd.SetArgs(convertToPosixArgs([]string{"gcsfuse", fmt.Sprintf("--config-file=testdata/%s", tc.cfgFile), "abc", "pqr"}, cmd)) + + err = cmd.Execute() + + assert.NoError(t, err) + assert.Equal(t, tc.expectedConfig.DummyIo, gotConfig.DummyIo) + }) + } +} + +func TestArgParsing_ConfigFileOverridesFlagOptimizations(t *testing.T) { + testCases := []struct { + name string + configContent string + args []string + validate func(*testing.T, *mountInfo) + }{ + { + name: "machine_type_optimization_respects_config_file", + configContent: `write: + global-max-blocks: 123 +`, + args: []string{"--machine-type=a3-highgpu-8g"}, + validate: func(t *testing.T, mi *mountInfo) { + assert.Equal(t, int64(123), mi.config.Write.GlobalMaxBlocks, "Should respect config file value 123, not optimize to 1600") + assert.True(t, mi.viperConfig.IsSet("write.global-max-blocks"), "ViperConfig.IsSet should be true for write.global-max-blocks") + assert.True(t, mi.config.ImplicitDirs, "Should optimize implicit-dirs to true based on machine-type") + assert.False(t, mi.viperConfig.IsSet("implicit-dirs")) + }, + }, + { + name: "profile_optimization_respects_config_file", + configContent: `implicit-dirs: false +`, + args: []string{"--profile=" + cfg.ProfileAIMLTraining}, + validate: func(t *testing.T, mi *mountInfo) { + assert.False(t, mi.config.ImplicitDirs, "Should respect config file value false, not optimize to true") + assert.True(t, mi.viperConfig.IsSet("implicit-dirs"), "ViperConfig.IsSet should be true for implicit-dirs") + assert.Equal(t, int64(testMaxSupportedTTLInSeconds), mi.config.MetadataCache.TtlSecs, "Should optimize metadata-cache.ttl-secs to -1 based on profile") + assert.False(t, mi.viperConfig.IsSet("metadata-cache.ttl-secs")) + }, + }, + { + name: "machine_type_in_config_file_respects_other_config_file_configurations", + configContent: ` +machine-type: a3-highgpu-8g +write: + global-max-blocks: 123`, + validate: func(t *testing.T, mi *mountInfo) { + assert.Equal(t, int64(123), mi.config.Write.GlobalMaxBlocks, "Should respect config file value 123, not optimize to 1600") + assert.True(t, mi.viperConfig.IsSet("write.global-max-blocks"), "ViperConfig.IsSet should be true for write.global-max-blocks") + assert.True(t, mi.config.ImplicitDirs, "Should optimize implicit-dirs to true based on machine-type") + assert.False(t, mi.viperConfig.IsSet("implicit-dirs")) + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + configFile := createTempConfigFile(t, tc.configContent) + defer os.Remove(configFile) + var capturedMountInfo *mountInfo + cmd, err := newRootCmd(func(mi *mountInfo, _, _ string) error { + capturedMountInfo = mi + return nil + }) + require.NoError(t, err) + cmdArgs := append([]string{"gcsfuse", "--config-file=" + configFile}, tc.args...) + cmdArgs = append(cmdArgs, "bucket", "mountpoint") + cmd.SetArgs(convertToPosixArgs(cmdArgs, cmd)) + + err = cmd.Execute() + + require.NoError(t, err) + require.NotNil(t, capturedMountInfo) + tc.validate(t, capturedMountInfo) + }) + } +} + +func TestArgParsing_CliFlagsOverridesFlagOptimizations(t *testing.T) { + testCases := []struct { + name string + args []string + validate func(*testing.T, *mountInfo) + }{ + { + name: "machine_type_optimization_respects_cli_flag", + args: []string{"--machine-type=a3-highgpu-8g", "--write-global-max-blocks=123"}, + validate: func(t *testing.T, mi *mountInfo) { + assert.Equal(t, int64(123), mi.config.Write.GlobalMaxBlocks, "Should respect CLI value 123, not optimize to 1600") + assert.True(t, mi.viperConfig.IsSet("write.global-max-blocks"), "ViperConfig.IsSet should be true for write.global-max-blocks") + assert.True(t, mi.config.ImplicitDirs, "Should optimize implicit-dirs to true based on machine-type") + assert.False(t, mi.viperConfig.IsSet("implicit-dirs")) + }, + }, + { + name: "profile_optimization_respects_cli_flag", + args: []string{"--profile=" + cfg.ProfileAIMLTraining, "--implicit-dirs=false"}, + validate: func(t *testing.T, mi *mountInfo) { + assert.False(t, mi.config.ImplicitDirs, "Should respect CLI value false, not optimize to true") + assert.True(t, mi.viperConfig.IsSet("implicit-dirs"), "ViperConfig.IsSet should be true for implicit-dirs") + assert.Equal(t, int64(testMaxSupportedTTLInSeconds), mi.config.MetadataCache.TtlSecs, "Should optimize metadata-cache.ttl-secs to -1 based on profile") + assert.False(t, mi.viperConfig.IsSet("metadata-cache.ttl-secs")) + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var capturedMountInfo *mountInfo + cmd, err := newRootCmd(func(mi *mountInfo, _, _ string) error { + capturedMountInfo = mi + return nil + }) + require.NoError(t, err) + cmdArgs := append([]string{"gcsfuse"}, tc.args...) + cmdArgs = append(cmdArgs, "bucket", "mountpoint") + cmd.SetArgs(convertToPosixArgs(cmdArgs, cmd)) + + err = cmd.Execute() + + require.NoError(t, err) + require.NotNil(t, capturedMountInfo) + tc.validate(t, capturedMountInfo) + }) + } +} diff --git a/cmd/testdata/get_config_file_flags/empty.yaml b/cmd/testdata/get_config_file_flags/empty.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/testdata/get_config_file_flags/nested_values.yaml b/cmd/testdata/get_config_file_flags/nested_values.yaml new file mode 100644 index 0000000000..a55c299424 --- /dev/null +++ b/cmd/testdata/get_config_file_flags/nested_values.yaml @@ -0,0 +1,4 @@ +logging: + file-path: /var/log/gcsfuse.log + format: json + diff --git a/cmd/testdata/get_config_file_flags/simple_values.yaml b/cmd/testdata/get_config_file_flags/simple_values.yaml new file mode 100644 index 0000000000..1d4dd1a62a --- /dev/null +++ b/cmd/testdata/get_config_file_flags/simple_values.yaml @@ -0,0 +1,2 @@ +key1: value1 +key2: 123 diff --git a/cmd/testdata/invalid_profile.yaml b/cmd/testdata/invalid_profile.yaml new file mode 100644 index 0000000000..9021e373f6 --- /dev/null +++ b/cmd/testdata/invalid_profile.yaml @@ -0,0 +1,2 @@ +profile: unknown-profile + diff --git a/cmd/testdata/metrics_config/enable_otel_false.yml b/cmd/testdata/metrics_config/enable_otel_false.yml deleted file mode 100644 index 60a9ec6010..0000000000 --- a/cmd/testdata/metrics_config/enable_otel_false.yml +++ /dev/null @@ -1,2 +0,0 @@ -metrics: - enable-otel: false diff --git a/cmd/testdata/metrics_config/enable_otel_true.yml b/cmd/testdata/metrics_config/enable_otel_true.yml deleted file mode 100644 index e5b1b20cce..0000000000 --- a/cmd/testdata/metrics_config/enable_otel_true.yml +++ /dev/null @@ -1,2 +0,0 @@ -metrics: - enable-otel: true diff --git a/cmd/testdata/monitoring_config/empty.yml b/cmd/testdata/monitoring_config/empty.yml new file mode 100644 index 0000000000..0b8d0596b9 --- /dev/null +++ b/cmd/testdata/monitoring_config/empty.yml @@ -0,0 +1 @@ +# To verify that the defaults are getting parsed correctly diff --git a/cmd/testdata/monitoring_config/sanitize_trace_exporters.yml b/cmd/testdata/monitoring_config/sanitize_trace_exporters.yml new file mode 100644 index 0000000000..582c7d5607 --- /dev/null +++ b/cmd/testdata/monitoring_config/sanitize_trace_exporters.yml @@ -0,0 +1,4 @@ +trace: + exporters: GcPexPorTer, STdOut + sampling-ratio: 0.5 + project-id: gcp-sample-test diff --git a/cmd/testdata/mount_info_population/cli_override_config.yaml b/cmd/testdata/mount_info_population/cli_override_config.yaml new file mode 100644 index 0000000000..12a350fdfe --- /dev/null +++ b/cmd/testdata/mount_info_population/cli_override_config.yaml @@ -0,0 +1,5 @@ +app-name: config-app +file-system: + gid: 1002 +logging: + severity: error diff --git a/cmd/testdata/mount_info_population/config_file_only.yaml b/cmd/testdata/mount_info_population/config_file_only.yaml new file mode 100644 index 0000000000..5b8f1793fd --- /dev/null +++ b/cmd/testdata/mount_info_population/config_file_only.yaml @@ -0,0 +1,3 @@ +app-name: config-app +file-system: + gid: 1002 diff --git a/cmd/testdata/read_config/empty.yaml b/cmd/testdata/read_config/empty.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/testdata/read_config/override.yaml b/cmd/testdata/read_config/override.yaml new file mode 100644 index 0000000000..43b0e019b1 --- /dev/null +++ b/cmd/testdata/read_config/override.yaml @@ -0,0 +1,2 @@ +read: + inactive-stream-timeout: 30s diff --git a/cmd/testdata/read_config/override_with_grpc.yaml b/cmd/testdata/read_config/override_with_grpc.yaml new file mode 100644 index 0000000000..d3f4518720 --- /dev/null +++ b/cmd/testdata/read_config/override_with_grpc.yaml @@ -0,0 +1,4 @@ +read: + inactive-stream-timeout: 30s +gcs-connection: + client-protocol: grpc diff --git a/cmd/testdata/unset_machine_type.yaml b/cmd/testdata/unset_machine_type.yaml new file mode 100644 index 0000000000..2c4c0104c5 --- /dev/null +++ b/cmd/testdata/unset_machine_type.yaml @@ -0,0 +1 @@ +disable-autoconfig: true diff --git a/cmd/testdata/valid_config.yaml b/cmd/testdata/valid_config.yaml index 6cb23064ea..4be604beec 100644 --- a/cmd/testdata/valid_config.yaml +++ b/cmd/testdata/valid_config.yaml @@ -1,20 +1,31 @@ app-name: hello +read: + inactive-stream-timeout: 10s + enable-buffered-read: true + global-max-blocks: 20 + block-size-mb: 8 + start-blocks-per-handle: 4 + max-blocks-per-handle: 20 + min-blocks-per-handle: 2 + random-seek-threshold: 10 write: create-empty-file: true - experimental-enable-streaming-writes: true + enable-streaming-writes: true global-max-blocks: 20 block-size-mb: 10 max-blocks-per-file: 2 + enable-rapid-appends: false file-cache: cache-file-for-range-read: true download-chunk-size-mb: 300 enable-crc: true - enable-parallel-downloads: true + enable-parallel-downloads: false max-parallel-downloads: 200 max-size-mb: 40 parallel-downloads-per-file: 10 write-buffer-size: 8192 enable-o-direct: true + experimental-disable-size-calculation-fix: true gcs-auth: anonymous-access: true key-file: "~/key.file" @@ -26,6 +37,7 @@ gcs-connection: custom-endpoint: www.abc.com experimental-enable-json-read: true grpc-conn-pool-size: 200 + grpc-path-strategy: direct-path-only http-client-timeout: 400s limit-bytes-per-sec: 20 limit-ops-per-sec: 30 @@ -33,8 +45,11 @@ gcs-connection: max-idle-conns-per-host: 20 sequential-read-size-mb: 450 gcs-retries: + experimental-nonrapid-folder-api-stall-retry: true + chunk-retry-deadline-secs: 180 + chunk-transfer-timeout-secs: 20 read-stall: - enable: true + enable: false min-req-timeout: 10s max-req-timeout: 200s initial-req-timeout: 20s @@ -51,15 +66,20 @@ file-system: kernel-list-cache-ttl-secs: 300 rename-dir-limit: 10 temp-dir: ~/temp - precondition-errors: false + max-read-ahead-kb: 1024 list: enable-empty-managed-folders: true enable-hns: false +enable-atomic-rename-object: false +disable-list-access-check: false metadata-cache: deprecated-stat-cache-capacity: 200 deprecated-stat-cache-ttl: 30s deprecated-type-cache-ttl: 20s enable-nonexistent-type-cache: true + metadata-prefetch-max-workers: 5 + enable-metadata-prefetch: true + metadata-prefetch-entries-limit: 50 experimental-metadata-prefetch-on-mount: sync stat-cache-max-size-mb: 40 ttl-secs: 100 @@ -67,3 +87,10 @@ metadata-cache: metrics: cloud-metrics-export-interval-secs: 10 + workers: 10 + buffer-size: 128 +machine-type: "config-file-machine-type" +dummy-io: + enable: true + reader-latency: 150ms + per-mb-latency: 20ms diff --git a/cmd/testdata/valid_profile.yaml b/cmd/testdata/valid_profile.yaml new file mode 100644 index 0000000000..aae1011ca5 --- /dev/null +++ b/cmd/testdata/valid_profile.yaml @@ -0,0 +1,2 @@ +profile: aiml-training + diff --git a/cmd/testdata/workload_insight_config/empty.yaml b/cmd/testdata/workload_insight_config/empty.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/testdata/workload_insight_config/visual_with_output_file.yaml b/cmd/testdata/workload_insight_config/visual_with_output_file.yaml new file mode 100644 index 0000000000..acc5676f52 --- /dev/null +++ b/cmd/testdata/workload_insight_config/visual_with_output_file.yaml @@ -0,0 +1,3 @@ +workload-insight: + visualize: true + output-file: /tmp/insight.html diff --git a/cmd/testdata/workload_insight_config/visual_without_output_file.yaml b/cmd/testdata/workload_insight_config/visual_without_output_file.yaml new file mode 100644 index 0000000000..5ecdd7c4a7 --- /dev/null +++ b/cmd/testdata/workload_insight_config/visual_without_output_file.yaml @@ -0,0 +1,2 @@ +workload-insight: + visualize: true diff --git a/cmd/testdata/write_config/invalid_write_config_due_to_0_block_size.yaml b/cmd/testdata/write_config/invalid_write_config_due_to_0_block_size.yaml index 0e0bf5418d..15570c2398 100644 --- a/cmd/testdata/write_config/invalid_write_config_due_to_0_block_size.yaml +++ b/cmd/testdata/write_config/invalid_write_config_due_to_0_block_size.yaml @@ -1,4 +1,4 @@ write: create-empty-file: true - experimental-enable-streaming-writes: true + enable-streaming-writes: true block-size-mb: 0 diff --git a/cmd/testdata/write_config/invalid_write_config_due_to_small_global_max_blocks.yaml b/cmd/testdata/write_config/invalid_write_config_due_to_invalid_global_max_blocks.yaml similarity index 53% rename from cmd/testdata/write_config/invalid_write_config_due_to_small_global_max_blocks.yaml rename to cmd/testdata/write_config/invalid_write_config_due_to_invalid_global_max_blocks.yaml index 80ae7444d3..c9c7722387 100644 --- a/cmd/testdata/write_config/invalid_write_config_due_to_small_global_max_blocks.yaml +++ b/cmd/testdata/write_config/invalid_write_config_due_to_invalid_global_max_blocks.yaml @@ -1,6 +1,6 @@ write: create-empty-file: true - experimental-enable-streaming-writes: true - global-max-blocks: 0 + enable-streaming-writes: true + global-max-blocks: -2 block-size-mb: 10 max-blocks-per-file: 2 diff --git a/cmd/testdata/write_config/invalid_write_config_due_to_small_max_blocks_per_file.yaml b/cmd/testdata/write_config/invalid_write_config_due_to_zero_max_blocks_per_file.yaml similarity index 52% rename from cmd/testdata/write_config/invalid_write_config_due_to_small_max_blocks_per_file.yaml rename to cmd/testdata/write_config/invalid_write_config_due_to_zero_max_blocks_per_file.yaml index 54e96462df..d6b6605ef5 100644 --- a/cmd/testdata/write_config/invalid_write_config_due_to_small_max_blocks_per_file.yaml +++ b/cmd/testdata/write_config/invalid_write_config_due_to_zero_max_blocks_per_file.yaml @@ -1,6 +1,6 @@ write: create-empty-file: true - experimental-enable-streaming-writes: true + enable-streaming-writes: true global-max-blocks: 20 block-size-mb: 10 - max-blocks-per-file: 1 + max-blocks-per-file: 0 diff --git a/common/queue.go b/common/queue.go new file mode 100644 index 0000000000..d29e2f6cb6 --- /dev/null +++ b/common/queue.go @@ -0,0 +1,102 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +// Queue is a generic interface for a queue data structure. +type Queue[T any] interface { + // IsEmpty checks if the queue is empty. + IsEmpty() bool + + // Peek returns the front of the queue without removing it. + // Panics if the queue is empty. + Peek() T + + // Push puts an item on the end of the queue. + Push(value T) + + // Pop removes and returns the front item from the queue. + // Panics if the queue is empty. + Pop() T + + // Len returns the number of items in the queue. + Len() int +} + +// node represents a node in the queue. +type node[T any] struct { + value T + next *node[T] +} + +// linkedListQueue is a linked list implementation of a queue. +type linkedListQueue[T any] struct { + start, end *node[T] + size int +} + +// NewQueue creates a new empty queue. +func NewLinkedListQueue[T any]() Queue[T] { + return &linkedListQueue[T]{} +} + +// IsEmpty returns true if the queue is empty. +func (q *linkedListQueue[T]) IsEmpty() bool { + return q.size == 0 +} + +// Peek returns the front of the queue without removing it. +// Panics if the queue is empty. +func (q *linkedListQueue[T]) Peek() T { + if q.size == 0 { + panic("Peek called on an empty queue.") + } + return q.start.value +} + +// Push puts an item on the end of the queue. +func (q *linkedListQueue[T]) Push(value T) { + n := &node[T]{value, nil} + if q.size == 0 { + q.start = n + q.end = n + } else { + q.end.next = n + q.end = n + } + q.size++ +} + +// Pop removes and returns the front item from the queue. +// Panics if the queue is empty. +func (q *linkedListQueue[T]) Pop() T { + if q.size == 0 { + panic("Pop called on an empty queue.") + } + + n := q.start + if q.size == 1 { + q.start = nil + q.end = nil + } else { + q.start = q.start.next + } + q.size-- + return n.value +} + +// Len returns the number of items in the queue. +func (q *linkedListQueue[T]) Len() int { + return q.size +} diff --git a/common/queue_test.go b/common/queue_test.go new file mode 100644 index 0000000000..6b38f824a2 --- /dev/null +++ b/common/queue_test.go @@ -0,0 +1,134 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewLinkedListQueue(t *testing.T) { + q := NewLinkedListQueue[int]() + + assert.NotNil(t, q, "NewLinkedListQueue() should return a non-nil queue.") + assert.True(t, q.IsEmpty(), "A new queue should be empty.") + assert.Equal(t, 0, q.Len(), "A new queue should have a size of 0.") +} + +func TestLinkedListQueue_Push(t *testing.T) { + q := NewLinkedListQueue[int]() + + q.Push(4) + q.Push(5) + + assert.Equal(t, 4, q.Peek()) + assert.False(t, q.IsEmpty()) +} + +func TestLinkedListQueue_SinglePop(t *testing.T) { + q := NewLinkedListQueue[int]() + q.Push(4) + q.Push(5) + require.Equal(t, 4, q.Peek()) + require.False(t, q.IsEmpty()) + + val := q.Pop() + + assert.Equal(t, 4, val) + assert.Equal(t, 5, q.Peek()) +} + +func TestLinkedListQueue_MultiplePops(t *testing.T) { + q := NewLinkedListQueue[int]() + q.Push(4) + q.Push(5) + require.Equal(t, 4, q.Peek()) + require.False(t, q.IsEmpty()) + val := q.Pop() + require.Equal(t, 4, val) + require.Equal(t, 5, q.Peek()) + + val = q.Pop() + + assert.Equal(t, 5, val) + assert.True(t, q.IsEmpty()) +} + +func TestLinkedListQueue_PopEmptyQueue(t *testing.T) { + assert.Panics(t, func() { + NewLinkedListQueue[int]().Pop() + }, "Pop should panic when called on an empty queue.") +} + +func TestLinkedListQueue_Peek(t *testing.T) { + q := NewLinkedListQueue[int]() + q.Push(4) + require.Equal(t, 1, q.Len()) + + val := q.Peek() + + assert.Equal(t, 4, val) + assert.Equal(t, 1, q.Len()) // Length should remain unchanged. + assert.False(t, q.IsEmpty()) +} + +func TestLinkedListQueue_PeekEmptyQueue(t *testing.T) { + assert.Panics(t, func() { + NewLinkedListQueue[int]().Peek() + }, "Peek should panic when called on an empty queue.") +} + +func TestLinkedListQueue_IsEmptyTrue(t *testing.T) { + q := NewLinkedListQueue[int]() + q.Push(4) + q.Pop() + + assert.True(t, q.IsEmpty()) +} + +func TestLinkedListQueue_IsEmptyFalse(t *testing.T) { + q := NewLinkedListQueue[int]() + q.Push(4) + + assert.False(t, q.IsEmpty()) +} + +func TestLinkedListQueue_Len(t *testing.T) { + q := NewLinkedListQueue[int]() + assert.Equal(t, 0, q.Len()) + + q.Push(4) + assert.Equal(t, 1, q.Len()) + + q.Push(5) + assert.Equal(t, 2, q.Len()) + + q.Push(6) + assert.Equal(t, 3, q.Len()) + + val := q.Pop() + assert.Equal(t, 4, val) + assert.Equal(t, 2, q.Len()) + + val = q.Pop() + assert.Equal(t, 5, val) + assert.Equal(t, 1, q.Len()) + + val = q.Pop() + assert.Equal(t, 6, val) + assert.Equal(t, 0, q.Len()) +} diff --git a/common/telemetry_test.go b/common/telemetry_test.go deleted file mode 100644 index bbe02015e0..0000000000 --- a/common/telemetry_test.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package common - -import ( - "context" - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestJoinShutdownFunc(t *testing.T) { - t.Parallel() - tests := []struct { - name string - fns []ShutdownFn - expectedErrs []string - }{ - { - name: "normal", - fns: []ShutdownFn{func(_ context.Context) error { return nil }}, - expectedErrs: nil, - }, - { - name: "one_err", - fns: []ShutdownFn{func(_ context.Context) error { return fmt.Errorf("err") }}, - expectedErrs: []string{"err"}, - }, - { - name: "two_err", - fns: []ShutdownFn{ - func(_ context.Context) error { return fmt.Errorf("err1") }, - func(_ context.Context) error { return fmt.Errorf("err2") }, - }, - expectedErrs: []string{"err1", "err2"}, - }, - { - name: "two_err_one_normal", - fns: []ShutdownFn{ - func(_ context.Context) error { return fmt.Errorf("err1") }, - func(_ context.Context) error { return nil }, - func(_ context.Context) error { return fmt.Errorf("err2") }, - }, - expectedErrs: []string{"err1", "err2"}, - }, - { - name: "nil", - fns: []ShutdownFn{ - func(_ context.Context) error { return fmt.Errorf("err1") }, - nil, - func(_ context.Context) error { return fmt.Errorf("err2") }, - }, - expectedErrs: []string{"err1", "err2"}, - }, - } - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - err := JoinShutdownFunc(tc.fns...)(context.Background()) - - if len(tc.expectedErrs) == 0 { - assert.NoError(t, err) - } else { - require.Error(t, err) - for _, e := range tc.expectedErrs { - assert.ErrorContains(t, err, e) - } - } - }) - } -} diff --git a/common/util.go b/common/util.go new file mode 100644 index 0000000000..0845670556 --- /dev/null +++ b/common/util.go @@ -0,0 +1,122 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "os/exec" + "regexp" + "strings" +) + +type ShutdownFn func(ctx context.Context) error + +// JoinShutdownFunc combines the provided shutdown functions into a single function. +func JoinShutdownFunc(shutdownFns ...ShutdownFn) ShutdownFn { + return func(ctx context.Context) error { + var err error + for _, fn := range shutdownFns { + if fn == nil { + continue + } + err = errors.Join(err, fn(ctx)) + } + return err + } +} + +// GetKernelVersion returns the kernel version. +func GetKernelVersion() (string, error) { + cmd := exec.Command("uname", "-r") + out, err := cmd.Output() + if err != nil { + return "", err + } + kernelVersion := strings.TrimSpace(string(out)) + return kernelVersion, nil +} + +// kernelVersion is just a wrapper over GetKernelVersion. This +// allows us to mock it in the unit test of ShouldSkipKernelListCacheTest. +var kernelVersionToTest = func() (string, error) { + return GetKernelVersion() +} + +// IsKLCacheEvictionUnSupported returns true if Kernel List Cache Eviction is not supported +// for the current linux version. +// In case of any non-nil error it returns false. +func IsKLCacheEvictionUnSupported() (bool, error) { + UnsupportedKernelVersions := []string{`^6\.9\.\d+`, `^6\.10\.\d+`, `^6\.11\.\d+`, `^6\.12\.\d+`} + + kernelVersion, err := kernelVersionToTest() + if err != nil { + return false, err + } + + for i := range UnsupportedKernelVersions { + matched, err := regexp.MatchString(UnsupportedKernelVersions[i], kernelVersion) + if err != nil { + return false, err + } + if matched { + return true, nil + } + } + + return false, nil +} + +func CloseFile(file *os.File) { + if err := file.Close(); err != nil { + log.Fatalf("error in closing: %v", err) + } +} + +func WriteFile(fileName string, content string) (err error) { + f, err := os.OpenFile(fileName, os.O_RDWR, 0600) + if err != nil { + err = fmt.Errorf("open file for write at start: %v", err) + return + } + + // Closing file at the end. + defer CloseFile(f) + + _, err = f.WriteAt([]byte(content), 0) + + return +} + +func ReadFile(filePath string) (content []byte, err error) { + f, err := os.OpenFile(filePath, os.O_RDONLY, 0600) + if err != nil { + err = fmt.Errorf("error in the opening the file %v", err) + return + } + + // Closing file at the end. + defer CloseFile(f) + + content, err = os.ReadFile(f.Name()) + if err != nil { + err = fmt.Errorf("ReadAll: %v", err) + return + } + return +} diff --git a/common/util_test.go b/common/util_test.go new file mode 100644 index 0000000000..6a2763a95c --- /dev/null +++ b/common/util_test.go @@ -0,0 +1,193 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package common + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestIsKLCacheEvictionUnSupported(t *testing.T) { + testCases := []struct { + name string + mockKernelVersion string + expectedSkip bool + }{ + { + name: "Cloudtop_Supported", + mockKernelVersion: "4.19.0-17-amd64", + expectedSkip: false, + }, + { + name: "Cloudtop_Unsupported", + mockKernelVersion: "6.10.11-1rodete2-amd64", + expectedSkip: true, + }, + { + name: "GCP_Supported", + mockKernelVersion: "6.8.0-1020-gcp", + expectedSkip: false, + }, + { + name: "GCP_Unsupported_6.9.x", + mockKernelVersion: "6.9.0-1020-gcp", + expectedSkip: true, + }, + { + name: "GCP_Unsupported_6.10.x", + mockKernelVersion: "6.10.0-1020-gcp", + expectedSkip: true, + }, + { + name: "GCP_Unsupported_6.11.x", + mockKernelVersion: "6.11.0-1020-gcp", + expectedSkip: true, + }, + { + name: "GCP_Unsupported_6.12.x", + mockKernelVersion: "6.12.0-1020-gcp", + expectedSkip: true, + }, + { + name: "Amd64_Unsupported_6.10.x", + mockKernelVersion: "6.10.0-1-amd64", + expectedSkip: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + originalKernelVersion := kernelVersionToTest + kernelVersionToTest = func() (string, error) { return tc.mockKernelVersion, nil } + defer func() { kernelVersionToTest = originalKernelVersion }() + + skip, err := IsKLCacheEvictionUnSupported() + require.NoError(t, err) + assert.Equal(t, tc.expectedSkip, skip) + }) + } +} + +func TestJoinShutdownFunc(t *testing.T) { + t.Parallel() + tests := []struct { + name string + fns []ShutdownFn + expectedErrs []string + }{ + { + name: "normal", + fns: []ShutdownFn{func(_ context.Context) error { return nil }}, + expectedErrs: nil, + }, + { + name: "one_err", + fns: []ShutdownFn{func(_ context.Context) error { return fmt.Errorf("err") }}, + expectedErrs: []string{"err"}, + }, + { + name: "two_err", + fns: []ShutdownFn{ + func(_ context.Context) error { return fmt.Errorf("err1") }, + func(_ context.Context) error { return fmt.Errorf("err2") }, + }, + expectedErrs: []string{"err1", "err2"}, + }, + { + name: "two_err_one_normal", + fns: []ShutdownFn{ + func(_ context.Context) error { return fmt.Errorf("err1") }, + func(_ context.Context) error { return nil }, + func(_ context.Context) error { return fmt.Errorf("err2") }, + }, + expectedErrs: []string{"err1", "err2"}, + }, + { + name: "nil", + fns: []ShutdownFn{ + func(_ context.Context) error { return fmt.Errorf("err1") }, + nil, + func(_ context.Context) error { return fmt.Errorf("err2") }, + }, + expectedErrs: []string{"err1", "err2"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + err := JoinShutdownFunc(tc.fns...)(context.Background()) + + if len(tc.expectedErrs) == 0 { + assert.NoError(t, err) + } else { + require.Error(t, err) + for _, e := range tc.expectedErrs { + assert.ErrorContains(t, err, e) + } + } + }) + } +} + +func TestCloseFile(t *testing.T) { + // Setup + f, err := os.CreateTemp("", "testFile-*") + require.NoError(t, err) + + // Close file and assert + assert.NotPanics(t, func() { CloseFile(f) }) +} + +func TestWriteFile(t *testing.T) { + // Setup + tmpFile, err := os.CreateTemp("", "testFile-*") + require.NoError(t, err) + filePath := tmpFile.Name() + defer os.Remove(filePath) + require.NoError(t, tmpFile.Close()) + + // Call WriteFile + err = WriteFile(filePath, "content") + + // Assertions + assert.NoError(t, err) + data, err := ReadFile(filePath) + require.NoError(t, err) + assert.Equal(t, "content", string(data)) +} + +func TestReadFile(t *testing.T) { + // Setup + file, err := os.CreateTemp("", "testFile-*") + require.NoError(t, err) + fileName := file.Name() + defer os.Remove(fileName) + _, err = file.WriteString("content") + require.NoError(t, err) + require.NoError(t, file.Close()) + + // Call ReadFile + content, err := ReadFile(fileName) + + // Assertions + assert.NoError(t, err) + assert.Equal(t, "content", string(content)) +} diff --git a/common/version.go b/common/version.go index 76e5817714..b1c6594d7e 100644 --- a/common/version.go +++ b/common/version.go @@ -19,7 +19,7 @@ import ( "runtime" ) -// Set with `-ldflags -X github.com/googlecloudplatform/gcsfuse/v2/common.gcsfuseVersion=1.2.3` +// Set with `-ldflags -X github.com/googlecloudplatform/gcsfuse/v3/common.gcsfuseVersion=1.2.3` // by tools/build_gcsfuse. If not defined, we use "unknown" in getVersion. var gcsfuseVersion string diff --git a/csi_driver_build.yml b/csi_driver_build.yml new file mode 100644 index 0000000000..b7e6161645 --- /dev/null +++ b/csi_driver_build.yml @@ -0,0 +1,134 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +substitutions: + _GOLANG_VERSION: '0.0.0' #Makefile will pass _GOLANG_VERSION parameter to override this value. + _GCSFUSE_VERSION: 'v4' + _CSI_VERSION: 'main' + _BUILD_ARM: 'false' + _USER: 'cloudbuild' + _IMAGE_PREFIX: 'build' + _STAGINGVERSION: '' + +steps: +# --- Build GCSFuse --- +# This step clones the GCSFuse CSI Driver repository. +- name: 'gcr.io/cloud-builders/git' + id: 'clone-csi-driver' # ID is used in waitFor below. + waitFor: ['-'] # This step can run concurrently with other steps that have waitFor: ['-'] + args: ['clone', '--branch', '${_CSI_VERSION}', '--depth', '1', 'https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver.git', 'csi-driver-src'] + +# This step builds the GCSFuse binary for each platform specified in the _PLATFORMS substitution. +# The binaries are placed in the /workspace/gcsfuse-artifacts directory. +- name: 'golang:${_GOLANG_VERSION}' + id: 'build-gcsfuse' # ID is used in waitFor below. + waitFor: ['-'] # This step can run concurrently with other steps that have waitFor: ['-'] + entrypoint: 'bash' + args: + - '-c' + - | + set -e + ARCHS="amd64" + if [[ "${_BUILD_ARM}" == "true" ]]; then + ARCHS="$$ARCHS arm64" + fi + + for arch in $$ARCHS; do + echo "Building GCSFuse for linux/$arch..." + GOOS=linux go run tools/build_gcsfuse/main.go --arch=$$arch . . "${_GCSFUSE_VERSION}" + mkdir -p "/workspace/gcsfuse-artifacts/linux/$$arch" + mv "bin/gcsfuse" "/workspace/gcsfuse-artifacts/linux/$$arch/gcsfuse" + echo "Cleaning up bin and sbin directories..." + rm -rf bin sbin + done + +# --- Store GCSFuse binaries in GCS --- +# The GCSFuse CSI Driver build process needs to fetch the GCSFuse binaries from a GCS bucket. +# This step creates a temporary GCS bucket for this purpose. +- name: 'gcr.io/cloud-builders/gcloud' + id: 'create-gcs-bucket' # ID is used in waitFor below. + waitFor: ['-'] # This step can run concurrently with other steps that have waitFor: ['-'] + entrypoint: 'bash' + args: + - '-c' + - | + set -e + BUILD_ID_VAL="${BUILD_ID}" + BUCKET_NAME="gs://${PROJECT_ID}-csi-$${BUILD_ID_VAL:0:8}" + gcloud storage buckets create $$BUCKET_NAME + echo $$BUCKET_NAME > /workspace/bucket_name + +# This step uploads the GCSFuse binaries to the GCS bucket. +- name: 'gcr.io/cloud-builders/gcloud' + id: 'upload-gcsfuse-binaries' + waitFor: ['create-gcs-bucket', 'build-gcsfuse'] + entrypoint: 'bash' + args: + - '-c' + - | + set -e + BUCKET_NAME=$$(cat /workspace/bucket_name) + gcloud storage cp -r /workspace/gcsfuse-artifacts/* $$BUCKET_NAME/ + +# --- Build and Push GCSFuse CSI Driver Image --- + +# Step 1: Build and Push GCSFuse CSI Driver Image +# This step builds the GCSFuse CSI Driver image and pushes it to the registry. +# It uses the docker builder and installs gcloud to get the credential helper. +- name: 'gcr.io/cloud-builders/docker' + id: 'build-and-push-csi-driver' + waitFor: ['upload-gcsfuse-binaries', 'clone-csi-driver'] + dir: 'csi-driver-src' + entrypoint: 'bash' + args: + - '-c' + - | + set -e + # --- Install gcloud --- + apt-get update && apt-get install -y apt-transport-https ca-certificates gnupg curl + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee /etc/apt/sources.list.d/google-cloud-sdk.list + apt-get update && apt-get install -y google-cloud-cli kubectl + + # --- Authenticate Docker --- + gcloud auth configure-docker gcr.io --quiet + + # --- Download gcsfuse binaries --- + GCSFUSE_PATH=$$(cat /workspace/bucket_name) + if [[ -n "${_STAGINGVERSION}" ]]; then + STAGINGVERSION="${_STAGINGVERSION}" + else + STAGINGVERSION=${BUILD_ID} + STAGINGVERSION=${_IMAGE_PREFIX}-$${STAGINGVERSION:0:8} + fi + REGISTRY="gcr.io/${PROJECT_ID}/${_USER}-gcsfuse-csi" + + make build-image-and-push-multi-arch REGISTRY=$${REGISTRY} GCSFUSE_PATH=$${GCSFUSE_PATH} STAGINGVERSION=$${STAGINGVERSION} BUILD_ARM=${_BUILD_ARM} + +# This step cleans up the GCS bucket. +- name: 'gcr.io/cloud-builders/gcloud' + id: 'cleanup-gcs-bucket' + waitFor: ['build-and-push-csi-driver'] + entrypoint: 'bash' + args: + - '-c' + - | + BUCKET_NAME=$$(cat /workspace/bucket_name) + gcloud storage rm --recursive $$BUCKET_NAME + rm /workspace/bucket_name + + +options: + # Using a more powerful machine is recommended for multi-platform builds. + machineType: 'E2_HIGHCPU_32' diff --git a/docs/benchmarks.md b/docs/benchmarks.md index eeacaa614a..3835d4d6e4 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -1,174 +1,228 @@ # GCSFuse Performance Benchmarks [FIO](https://fio.readthedocs.io/en/latest/) is used to perform load tests on -GCSFuse. Below tables shows performance metrics of GCSFuse for different +GCSFuse. The tables below show performance metrics of GCSFuse for different workloads for the given test setup: ## Test setup: -* Infra: GCP VM -* VM Type: n2-standard-96 -* OS: ubuntu-20.04 -* VM Bandwidth: 100Gbps -* VM location: us-west1-b -* Disk Type: SSD persistent disk -* GCS Bucket location: us-west1 -* Framework: FIO -* GCSFuse version: 2.2.0 - -## Reads - -### FIO spec - - ``` - [global] - ioengine=sync - direct=1 - fadvise_hint=0 - verify=0 - iodepth=64 - invalidate=1 - ramp_time=10s - runtime=60s - time_based=1 - thread=1 - openfiles=1 - group_reporting=1 - allrandrepeat=1 - # Change this to randread to test random reads. - rw=read - # Update the block size value from the table for different experiments. - bs=1M - # Update the file size value from table(file size) for different experiments. - filesize=10M - # Change the test directory (1mb) for different experiments. The directory must exist within the mounted directory. - directory=/mnt/1mb - filename_format=$jobname.$jobnum.$filenum - [experiment] - stonewall - # Number of threads - numjobs=128 +* Infra: GCP VM [C4-standard-192](https://cloud.google.com/compute/docs/general-purpose-machines#c4_series) +* Network: [Tier_1](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) Networking enabled on VM providing 200 Gbps egress bandwidth. +* OS Version: [Ubuntu 22.04 LTS](https://cloud.google.com/compute/docs/images/os-details#notable-difference-ubuntu) +* Image Family: [ubuntu-2204-lts](https://cloud.google.com/compute/docs/images/os-details#notable-difference-ubuntu) +* Disk Type: [Hyperdisk Balanced](https://cloud.google.com/compute/docs/disks/hd-types/hyperdisk-balanced) +* VM Region: us-south1 +* GCS Bucket ([HNS enabled](https://cloud.google.com/storage/docs/hns-overview)) Region: us-south1 +* Framework: fio (version 3.39) +* GCSFuse version: [v3.4.3](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v3.4.3) + +## Fio workloads + +<!-- Benchmarks start --> +--- + +### Sequential Reads +| File Size | BlockSize | NumJobs | NRFiles | **Avg Bandwidth (GB/s)** | **Avg IOPS (K)** | **Avg Latency (msec)** | +| :--- | :--- | ---: | ---: | ---: | ---: | ---: | +| 128 KiB | 128 KiB | 192 | 30 | 1.30 | 9.95 | 0.10 | +| 256 KiB | 128 KiB | 192 | 30 | 2.54 | 19.38 | 0.05 | +| 1 MiB | 1 MiB | 192 | 30 | 6.20 | 5.92 | 0.17 | +| 5 MiB | 1 MiB | 192 | 20 | 12.39 | 11.82 | 0.08 | +| 10 MiB | 1 MiB | 192 | 20 | 14.49 | 13.82 | 0.07 | +| 50 MiB | 1 MiB | 192 | 20 | 13.81 | 13.17 | 0.08 | +| 100 MiB | 1 MiB | 144 | 10 | 13.43 | 12.81 | 0.08 | +| 200 MiB | 1 MiB | 144 | 10 | 13.26 | 12.65 | 0.08 | +| 1 GiB | 1 MiB | 144 | 10 | 14.20 | 13.54 | 0.07 | + +<details> + <summary>GCSFuse Mount Option and fio configuration</summary> + +##### GCSFuse Mount Options +See more details about each option [here](https://cloud.google.com/storage/docs/cloud-storage-fuse/cli-options#options). +```bash +--implicit-dirs +--metadata-cache-ttl-secs=-1 ``` +##### Fio templated configuration +```ini +[global] +ioengine=libaio +direct=1 +fadvise_hint=0 +iodepth=64 +invalidate=1 +thread=1 +openfiles=1 +group_reporting=1 +create_serialize=0 +allrandrepeat=0 +file_service_type=random +rw=read +filename_format=$jobname.$jobnum.$filenum.size-${FILESIZE} + +[seq_read] +directory=${DIR} +filesize=${FILESIZE} +bs=${BS} +numjobs=${NUMJOBS} +nrfiles=${NRFILES} +``` +</details> + +--- + +### Random Reads +| File Size | BlockSize | NumJobs | NRFiles | **Avg Bandwidth (GB/s)** | **Avg IOPS (K)** | **Avg Latency (msec)** | +| :--- | :--- | ---: | ---: | ---: | ---: | ---: | +| 256 KiB | 128 KiB | 192 | 30 | 1.59 | 12.14 | 0.08 | +| 5 MiB | 1 MiB | 192 | 20 | 5.01 | 4.78 | 0.21 | +| 10 MiB | 1 MiB | 192 | 20 | 4.20 | 4.00 | 0.25 | +| 50 MiB | 1 MiB | 192 | 20 | 4.42 | 4.22 | 0.24 | +| 100 MiB | 1 MiB | 192 | 10 | 4.45 | 4.25 | 0.24 | +| 200 MiB | 1 MiB | 192 | 10 | 4.21 | 4.01 | 0.25 | +| 1 GiB | 1 MiB | 192 | 10 | 4.11 | 3.92 | 0.26 | + +<details> + <summary>GCSFuse Mount Option and fio configuration</summary> + +##### GCSFuse Mount Options +See more details about each option [here](https://cloud.google.com/storage/docs/cloud-storage-fuse/cli-options#options). +```bash +--implicit-dirs +--metadata-cache-ttl-secs=-1 +``` +##### Fio templated configuration +```ini +[global] +ioengine=libaio +direct=1 +fadvise_hint=0 +iodepth=64 +invalidate=1 +thread=1 +openfiles=1 +group_reporting=1 +create_serialize=0 +allrandrepeat=0 +file_service_type=random +rw=randread +filename_format=$jobname.$jobnum.$filenum.size-${FILESIZE} + +[rand_read] +directory=${DIR} +filesize=${FILESIZE} +bs=${BS} +numjobs=${NUMJOBS} +nrfiles=${NRFILES} +``` +</details> + +--- + +### Sequential Writes +| File Size | BlockSize | NumJobs | NRFiles | **Avg Bandwidth (GB/s)** | **Avg IOPS (K)** | **Avg Latency (msec)** | +| :--- | :--- | ---: | ---: | ---: | ---: | ---: | +| 256 KiB | 16 KiB | 96 | 30 | 0.17 | 10.43 | 0.10 | +| 1 MiB | 1 MiB | 96 | 30 | 0.53 | 0.50 | 1.98 | +| 50 MiB | 1 MiB | 96 | 30 | 3.58 | 3.42 | 0.29 | +| 100 MiB | 1 MiB | 96 | 20 | 4.06 | 3.87 | 0.26 | +| 500 MiB | 1 MiB | 96 | 20 | 4.57 | 4.36 | 0.23 | +| 1 GiB | 1 MiB | 96 | 10 | 4.62 | 4.41 | 0.23 | + +<details> + <summary>GCSFuse Mount Option and fio configuration</summary> + +##### GCSFuse Mount Options +See more details about each option [here](https://cloud.google.com/storage/docs/cloud-storage-fuse/cli-options#options). +```bash +--implicit-dirs +--metadata-cache-ttl-secs=-1 +--write-global-max-blocks=-1 +``` +##### Fio templated configuration +```ini +[global] +ioengine=libaio +direct=1 +fadvise_hint=0 +iodepth=64 +verify=0 +invalidate=1 +file_append=0 +create_on_open=1 +end_fsync=1 +thread=1 +openfiles=1 +group_reporting=1 +allrandrepeat=1 +filename_format=$jobname.$jobnum.$filenum.size-${FILESIZE} +rw=write + +[write_seq] +directory=${DIR} +filesize=${FILESIZE} +bs=${BS} +numjobs=${NUMJOBS} +nrfiles=${NRFILES} +``` +</details> + +> [!NOTE] +> Edits and appends to existing files are handled by first downloading the entire file. After changes are made locally, the entire file is uploaded again on `close` or `sync`. This results in performance similar to a full file read followed by a full file write. + +> [!NOTE] +> The bandwidth observed during benchmark runs can vary by up to 10%. This variation is expected and is due to normal fluctuations in Google Cloud Storage. To obtain more statistically reliable results, we recommend running the benchmarks multiple times. The numbers published above are the average of 5 runs. -### Results - -#### Sequential Reads - -| File Size | BlockSize | Bandwidth in (MiB/sec) | Avg Latency (msec) | IOPs | -|-----------|-----------|------------------------|--------------------|----------| -| 128KB | 128K | 862 | 18.54 | 6898.27 | -| 256KB | 128K | 1548 | 10.325 | 12386.03 | -| 1MB | 1M | 5108 | 24.99 | 5113.21 | -| 5MB | 1M | 7282 | 17.505 | 7308.51 | -| 10MB | 1M | 7946 | 16.092 | 7946.63 | -| 50MB | 1M | 7810 | 16.356 | 7818.17 | -| 100MB | 1M | 7839 | 16.295 | 7840.17 | -| 200MB | 1M | 7879 | 16.217 | 7884.45 | -| 1GB | 1M | 7911 | 16.162 | 7910.19 | - -#### Random Reads - -| File Size | BlockSize | Bandwidth in MiB/sec | Avg Latency (msec) | IOPs | -|-----------|-----------|----------------------|--------------------|----------| -| 256KB | 128K | 1264 | 12.648 | 10109.62 | -| 5MB | 1M | 4367 | 29.129 | 4449.03 | -| 10MB | 1M | 3810 | 33.496 | 3825.54 | -| 50MB | 1M | 4370 | 29.185 | 4426.73 | -| 100MB | 1M | 3504 | 36.421 | 3505.01 | -| 200MB | 1M | 3048 | 41.919 | 3044.43 | -| 1GB | 1M | 2120 | 60.246 | 2114.33 | - -## Writes - -### FIO spec - - ``` - [global] - ioengine=sync - direct=1 - fadvise_hint=0 - verify=0 - iodepth=64 - invalidate=1 - time_based=0 - file_append=0 - # By default fio creates all files first and then starts writing to them. This option is to disable that behavior. - create_on_open=1 - thread=1 - openfiles=1 - group_reporting=1 - allrandrepeat=1 - # Every file is written only once. Set nrfiles per thread in such a way that the test runs for 1-2 min. - # This will vary based on file size. Change the value from table to get provided results. - nrfiles=2 - filename_format=$jobname.$jobnum.$filenum - # Change this to randwrite to test random writes. - rw=write - # Update the block size value from the table for different - bs=1M - # Update the file size value from table(file size) for different experiments. - filesize=1G - [experiment] - stonewall - # Change the test directory (1mb) for different experiments. The directory must exist within the mounted directory. - directory=gcs/1gb - numjobs=112 - ``` - -**Note:** Benchmarking is done by writing out new files to GCS. Performance -numbers will be different for edits/appends to existing files. - -### Results - -#### Sequential Write - -| File Size | BlockSize | nrfiles | Bandwidth in MiB/sec | IOPS(avg) | Avg Latency (msec) | Network Send Traffic (GiB/s) | -|-----------|-----------|---------|----------------------|-----------|--------------------|------------------------------| -| 256KB | 16K | 30 | 212 | 14976.95 | 3.206 | 0.027 | -| 1MB | 1M | 30 | 772 | 794.32 | 1.150 | 0.036 | -| 50MB | 1M | 20 | 3611 | 5948.63 | 8.929 | 1.33 | -| 100MB | 1M | 10 | 3577 | 4672.64 | 1.911 | 1.41 | -| 1GB | 1M | 2 | 1766 | 2121.66 | 49.114 | 1.77 | - -#### Random Write - -Random writes and sequential write performance will generally be the same, as -all writes are first staged to a local temporary directory before being written -to GCS on close/fsync. +--- +<!-- Benchmarks end --> ## Steps to benchmark GCSFuse performance +> [!IMPORTANT] +> GCSFuse performance may differ based on region of VM and GCS Bucket region and GCSFuse version in use. To reproduce above benchmark please use the exact testing infra setup mentioned above. Use new GCS Bucket for each fio run which ensures for sequential write objects are not being overwritten. + 1. [Create](https://cloud.google.com/compute/docs/instances/create-start-instance#publicimage) - a GCP VM instance. + a GCP VM instance 2. [Connect](https://cloud.google.com/compute/docs/instances/connecting-to-instance) - to the VM instance. + to the VM instance 3. Install FIO. - ``` + ```bash sudo apt-get update sudo apt-get install fio ``` -5. [Install GCSFuse](https://cloud.google.com/storage/docs/gcsfuse-install). -6. Create a directory on the VM and then mount the gcs bucket to that directory. +4. [Install GCSFuse](https://cloud.google.com/storage/docs/gcsfuse-install) +5. [Create GCS Bucket](https://cloud.google.com/storage/docs/creating-buckets) +6. Create a directory on the VM and then mount the gcs bucket to that directory with the mount options provided in benchmark result section. - ``` + ```bash mkdir <path-to-mount-point> - gcsfuse <bucket-name> <path-to-mount-point> + gcsfuse <mount options> <bucket-name> <path-to-mount-point> ``` -7. Create a FIO job spec file. - The FIO content referred to above. Please read the details about the FIO - specification - [here](https://fio.readthedocs.io/en/latest/). - ``` +7. Create a fio job file with the templated fio configuration content provided in benchmark result section. + ```bash + # Copy content of fio configuration to this file. vi samplejobspec.fio ``` -8. Run the FIO test using following command. - - ``` - fio samplejobspec.fio +8. Run the FIO tool using following command. + + ```bash + # See the values of these variables from the respective benchmark result table. + DIR=<path-to-mount-point> \ + NUMJOBS="" \ + BS="" \ + FILESIZE="" \ + NRFILES="" fio samplejobspec.fio + + # Example command for last row of sequential write benchmark result table. + DIR=<path-to-mount-point> \ + NUMJOBS="96" \ + BS="1M" \ + FILESIZE="1G" \ + NRFILES="10" fio samplejobspec.fio ``` -9. Metrics will be displayed on the terminal after test is completed. \ No newline at end of file +9. Metrics will be displayed on the terminal after test is completed. diff --git a/docs/client_and_retries.md b/docs/client_and_retries.md new file mode 100644 index 0000000000..82bde92260 --- /dev/null +++ b/docs/client_and_retries.md @@ -0,0 +1,206 @@ +# GCSFuse Network Clients and Retry Strategies + +## Overview + +GCSFuse uses multiple network clients to communicate with Google Cloud Storage (GCS). Each client has specific retry strategies tailored to its use case and the operations it performs. + +## Network Clients + +### 1. HTTP Storage Clients (HTTP/1.1 and HTTP/2) + +**Purpose:** Primary storage operations using HTTP protocol + +**Protocols:** +- **HTTP/1.1** (`cfg.HTTP1`): Default protocol +- **HTTP/2** (`cfg.HTTP2`): Alternative protocol for better multiplexing + +**Configuration Options:** +- `MaxConnsPerHost`: The max number of TCP connections allowed per host. This is effective when client-protocol is set to 'http1'. A value of 0 indicates no limit on TCP connections (limited by the machine specifications). (Default: 0) +- `MaxIdleConnsPerHost`: The number of maximum idle connections allowed per host. (Default: 100) +- `HttpClientTimeout`: The time duration that http client will wait to get response from the server. A value of 0 indicates no timeout. (Default: 0s) +- `ExperimentalEnableJsonRead`: By default, GCSFuse uses the GCS XML API to read objects. When this flag is specified, GCSFuse uses the GCS JSON API instead. (Default: false) +- `ReadStallRetryConfig`: To turn on/off retries for stalled read requests. This is based on a timeout that changes depending on how long similar requests took in the past. (Default: true) + +### 2. gRPC Storage Client (Standard) + +**Purpose:** Storage operations using gRPC protocol for better performance and features. +**Configuration Options:** +- `GrpcConnPoolSize`: The number of gRPC channel in grpc client. (Default: 1) +- `EnableGrpcMetrics`: Enables support for gRPC metrics. (Default: false) + +**When Used:** +- When `--client-protocol=grpc` is set +- First tries DirectPath, defaulting to CloudPath as a secondary failover. + +### 3. gRPC Storage Client with Bidi Configuration + +**Purpose:** gRPC client optimized for rapid buckets with bidirectional read streaming + +**When Used:** +- Automatically used for rapid buckets regardless of `--client-protocol` setting. + +### 4. Storage Control Client + +**Purpose:** Handles HNS folder operations and utilizes GetStorageLayout to determine the bucket type using default retry logic. + +**Operations:** +- GetStorageLayout +- CreateFolder +- DeleteFolder +- GetFolder +- RenameFolder + +**When Used:** +- For HNS buckets folder operations. + +--- + +## Retry Strategies + +### Standard Retry Configuration + +**Applied To:** +- All HTTP storage clients +- All gRPC storage clients + +**Parameters:** +```go +Max Backoff: 30 seconds (Configurable via `--max-retry-sleep`) +Multiplier: 2 (Configurable via `--retry-multiplier`) +Max Attempts: 0 (unlimited) (Configurable via `--max-retry-attempts`) +Policy: storage.RetryAlways +``` + +--- + +### GCSFuse-Level Control Client Retry with Stall Detection + +Implemented [custom retry logic](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/internal/storage/storageutil/retry.go) for Folder APIs to mitigate stall issues and improve request reliability. + +**Applied To:** +- GetStorageLayout calls (all buckets) +- All control client operations (rapid buckets) + +**Default Parameters:** +```go +Retry Deadline: 30 seconds +Total Budget: 5 minutes +Initial Backoff: 1 second +``` + +**Features:** +- Time-bound retry approach +- Exponential backoff with jitter +- Stall detection with deadline per attempt +- Retries on timeout and retryable errors + +--- + +### Read Stall Retry Configuration (HTTP Only) + +**Configuration:** `ReadStallRetryConfig` in config + +**Applied To:** +- HTTP storage clients when `ReadStallRetryConfig.Enable = true` + +**Default Parameters:** +```go +Min Timeout: 1.5 seconds (Configurable via `--read-stall-min-req-timeout`) +Target Percentile: 0.99 (Configurable via `--read-stall-req-target-percentile`) +Initial Timeout: 20 seconds (Configurable via `--read-stall-initial-req-timeout`) +Max Timeout: 20 minutes (Configurable via `--read-stall-max-req-timeout`) +Increase Rate: 15 (Configurable via `--read-stall-req-increase-rate`) +``` + +**Purpose:** +- Handle stalled read operations +- Dynamic timeout adjustment based on request performance + +### Write Stall Retry Configuration (HTTP Only) + +**Default Parameters:** +```go +Chunk Transfer Timeout: 10 seconds (Configurable via `--chunk-transfer-timeout-secs`) +``` + +**Purpose:** +- Detect and retry (without exponential backoff) stalled chunk write operations within 10 seconds for resumable uploads. + +--- + +## Client Selection Decision Tree + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Bucket Access Request │ +└────────────────────────────┬────────────────────────────────┘ + │ + ▼ + ┌────────────────────┐ + │ Operation Type? │ + └──────────┬─────────┘ + │ + ┌──────────────┴──────────────┐ + │ │ + Storage Ops Control Ops + (Read/Write/List/Stat) (GetStorageLayout/Folders) + │ │ + ▼ ▼ + ┌───────────────┐ ┌─────────────────────┐ + │ Lookup Bucket │ │ Storage Control │ + │ Type │ │ Client │ + └───────┬───────┘ └──────────┬──────────┘ + │ │ + ▼ ▼ + Is rapid? ─── YES ──▶ gRPC Client │ + │ with Bidi │ + │ Config │ + NO │ + │ │ + ▼ │ + ┌─────────────────────┐ │ + │ Client Protocol? │ │ + └──────────┬──────────┘ │ + │ │ + ┌────────┴────────┬─────────────┐ │ + │ │ │ │ + HTTP1/2 GRPC OTHER │ + │ │ │ │ + │ │ │ │ + ▼ ▼ ▼ ▼ + │ │ Error │ + │ │ │ + │ Standard gRPC Client │ + │ │ + ▼ │ +HTTP Storage Client │ + │ + ▼ + ┌─────────────────────────┐ + │ Enable HNS? │ + └────────┬────────────────┘ + │ + ┌────┴────┐ + │ │ + YES NO + │ │ + │ └──▶ No Control Client + │ + ▼ + ┌─────────────────┐ + │ Bucket Type? │ + └────────┬────────┘ + │ + ┌────┴────┐ + │ │ + Rapid Non-rapid + │ │ + │ │ + ▼ ▼ + GAX + GCSFuse GAX Retries + Retries for Folders + (All APIs) + GCSFuse for + GetStorageLayout +``` + +--- diff --git a/docs/dev_guide.md b/docs/dev_guide.md index 9548fae122..9271cbbff3 100644 --- a/docs/dev_guide.md +++ b/docs/dev_guide.md @@ -175,7 +175,7 @@ write end-to-end tests for GCSFuse. GODEBUG=asyncpreemptoff=1 CGO_ENABLED=0 go test ./tools/integration_tests/$TEST_PACKAGE_NAME/... -p 1 -short --integrationTest -v --testbucket=$TEST_BUCKET_NAME --timeout=60m -run $TEST_NAME ``` 4. **Run all tests as pre-submit:** Existing GCSFuse end-to-end tests can be run - as a pre-submit check by adding the `execute-integration-tests` label to your + as a pre-submit check by adding the `execute-integration-tests` and `kokoro:run` label to your pull request. Ask one of your assigned code reviewers to apply this label, which will trigger the tests. Your reviewer will share any test failure details on the pull request. @@ -184,3 +184,35 @@ write end-to-end tests for GCSFuse. feature or have questions about scenarios to test, please feel free to open a [discussion thread](https://github.com/GoogleCloudPlatform/gcsfuse/discussions) with GCSFuse team. We're here to help! + +## Dummy I/O Mode for Performance Testing + +Dummy I/O mode simulates read operations without transferring data from Cloud Storage, allowing you to isolate and measure GCSFuse overhead independent of network latency. + +**Note:** +- Currently supports read operations only. +- Hidden feature for developers/performance engineers. +- Metadata operations (list, stat) remain real. +- Reads return zeros without fetching data. + +### Use Cases + +**✅ When to use:** +- Micro-optimizations in GCSFuse read flow +- Isolate performance from network latency +- Profile GCSFuse CPU/memory usage without network noise +- Benchmark different kernel configurations + +**❌ When NOT to use:** +- Real-world performance testing (network latency is critical) +- Data correctness validation +- Production workloads + +### Configuration + +```bash +--enable-dummy-io +--dummy-io-reader-latency=150ms # Simulates reader creation latency +--dummy-io-per-mb-latency=20ms # Simulates per-MB read from stream latency +``` + diff --git a/docs/known-issues.md b/docs/known-issues.md new file mode 100644 index 0000000000..fde78d54d5 --- /dev/null +++ b/docs/known-issues.md @@ -0,0 +1,24 @@ +# Known Issues + +This document lists known issues and bugs in GCSFuse, their impact, and the releases in which they were fixed. + +## Active Issues + +| Issue | Impact | Reference | +|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| :--- | +| `Error: GPG check FAILED` while installing GCSFuse on newer OS with strict cryptographic policies | This interrupts GCSFuse installation on certain OS (e.g., Rocky Linux 10, Red Hat Enterprise Linux 10). | [#3874](https://github.com/GoogleCloudPlatform/gcsfuse/issues/3874) | +| Object rename operations may fail or stall on the application side, resulting in 5xx errors in the GCSFuse logs. This issue is observed only under high QPS (greater than 1000 queries per second). | **Workaround:** To fix this issue, disable the atomic rename API by passing the flag `--enable-atomic-rename-object=false` when mounting GCSFuse. || + +## Resolved Issues + +| Issue | Affected Versions | Fixed in | Reference | +|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------|:---------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------| +| GCSFuse can lead to premature EOF/incorrect data reads on [A4X](https://docs.cloud.google.com/compute/docs/accelerator-optimized-machines#a4x-machine-type) and [A4X Max](https://docs.cloud.google.com/compute/docs/accelerator-optimized-machines#a4x-max-metal-machine-type) machines that feature a kernel with 64KiB page-size when setting a high [kernel-read-ahead](https://docs.cloud.google.com/storage/docs/cloud-storage-fuse/performance#increase-read-ahead-size) value.<br>**Workaround:** To fix this issue: Do not set the read-ahead | \[All versions up to v3.7.1\] | v3.7.2 || +| Writing to a file fails with an Input/Output error on the application side, accompanied by 503 errors in the GCSFuse logs. This occurs when streaming writes are enabled. This issue was caused by stalls during write operations. It has been resolved in GCSFuse v3.3.0. Users should upgrade to v3.3.0 or later. **Workaround:** Disable streaming writes (`--enable-streaming-writes=false`) only if user can't upgrade. This flag reliably prevents the error only when staging writes uses fast media type like, SSD, tmpfs (specified using`--temp-dir`). | \[v3.0.* - v3.2.*\] | v3.3.0 || +| Metrics: Input/output error when metrics are enabled. Applications may receive input/output errors from GCSFuse mounts when metrics are enabled. | v2.11.* | v2.12.0 | [#3870](https://github.com/GoogleCloudPlatform/gcsfuse/issues/3870) | +| Metrics: Incorrect gcs/reader_count and gcs/download_bytes_count metrics. | \[v2.5.0, v3.4.*] | v3.5.1 | [#3895](https://github.com/GoogleCloudPlatform/gcsfuse/pull/3895) | +| Metrics: Points must be written in order. One or more of the points specified had an older start time than the most recent point. Happens when multiple GCSFuse mounts are present on the same machine. | \[v2.5.0, v3.4.*\] | v3.5.1 | [#3895](https://github.com/GoogleCloudPlatform/gcsfuse/pull/3923) | +| Metrics: One or more points were written more frequently than the maximum sampling period configured for the metric. Happens when multiple GCSFuse mounts are present on the same machine. | \[v2.5.0, v3.4.*\] | v3.5.1 | [#3895](https://github.com/GoogleCloudPlatform/gcsfuse/pull/3923) | +| GCSFuse does not use machine-type passed from GKE CSI Driver.<br>**Impact**: Optimized GCSFuse configs for high-performance machines will not be applied when using the GCSFuse GKE CSI driver.<br>**Workarounds**: When using the GKE CSI driver,<br>1. Pass the machine type explicitly through `machine-type=<MACHINE_TYPE>` in `mountOptions` in `volumeAttributes`, Or<br>2. Enable host-network in the pod configuration by inserting `hostNetwork: true` in `spec` in pod-configuration and `hostNetworkPodKSA: "true"` in `volumeAttributes`. | v3.4.*, v3.5.1, v3.5.2 | v3.5.3 | [#3799](https://github.com/GoogleCloudPlatform/gcsfuse/pull/3799) [#4083](https://github.com/GoogleCloudPlatform/gcsfuse/issues/4083) | +| User-defined values in the configuration file were ignored due to automatic optimizations performed by GCSFuse for different machine types and profiles. | [v3.4.0 - v3.5.6] | GCSFuse v3.6.0, GKE 1.35.0-gke.1972000 | [#4271](https://github.com/GoogleCloudPlatform/gcsfuse/issues/4271) | +| Listing: GCSFuse returning stale listing data even after expiry of --kernel-list-cache-ttl-secs flag. This flag doesn't work on linux with kernel version from 6.9.x to 6.12.x. Not specific to GCSFuse version. | Kernel version 6.9.x to 6.12.x | Kernel v6.13.0+ | [#2792](https://github.com/GoogleCloudPlatform/gcsfuse/issues/2792) | diff --git a/docs/metrics.md b/docs/metrics.md index c15a596e25..ace049a1f0 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -1,151 +1,3 @@ # GCSFuse Metrics -GCSFuse supports exporting custom metrics to Google cloud monitoring. -Metrics are collected using OpenCensus and exported via Stackdriver exporter. -As of today, GCSFuse exports following metrics related to filesystem and -gcs calls. - -## File system metrics: -* **fs/ops_count:** Cumulative number of operations processed by file system. It allows -grouping by op_type to get counts for individual operations. -* **fs/ops_error_count:** Cumulative number of errors generated by file system operations. -This metric can be grouped by op_type, error and error_category. -Each error is mapped to an error_category in a many-to-one relationship. -* **fs/ops_latency:** Cumulative distribution of file system operation latencies. We -can group by op_type. - -## GCS metrics -* **gcs/download_bytes_count:** Cumulative number of bytes downloaded from GCS along -with read type. Read type specifies sequential or random or parallel read. -* **gcs/read_bytes_count:** Cumulative number of bytes read from GCS objects. This -is different from download_bytes_count. For eg: we might download x number of -bytes from GCS but read only <x bytes. -* **gcs/reader_count:** Cumulative number of GCS object readers opened or closed. We -can group the data by IO Method type i.e., opened or closed. -* **gcs/request_count:** Cumulative number of GCS requests processed. -* **gcs/request_latencies:** Cumulative distribution of the GCS request latencies. -* **gcs/read_count:** Specifies the count of gcs reads made along with read type. -Read type specifies sequential or random read. - -Note: Both request_count and request_latencies allows grouping by gcs method type. - -## File cache metrics -* **file_cache/read_bytes_count:** The cumulative number of bytes read from file -cache along with read type - Sequential/Random. -* **file_cache/read_latencies:** The cumulative distribution of the file cache read -latencies along with cache hit - true/false. -* **file_cache/read_count:** Specifies the number of read requests made via file cache -along with type - Sequential/Random and cache hit - true/false. - - -# Usage - -## Stackdriver exporter - -1. We need to set **stackdriver-export-interval** flag to enable exporting metrics to -Google cloud monitoring. The value of this flag represents the interval with -which data will be exported. - - Example command for exporting metrics every 60sec: -```angular2html - gcsfuse --stackdriver-export-interval=60s <bucket_name> <directory_name> -``` -2. Cloud monitoring api has to be [enabled](https://cloud.google.com/monitoring/api/enable-api) -on the Google cloud project. -3. Service account with which the GCSFuse is running should have -**monitoring.metricsDescriptors.create** permission. -4. Install the [Ops agent](https://cloud.google.com/monitoring/agent/ops-agent/install-index) on the VM. -5. For viewing the metrics: - 1. In the Google cloud console, go to **Metrics Explorer** page within **Monitoring**. - 2. In the toolbar, select the **Explorer** tab. - 3. Select the **configuration** tab. - 4. Expand the **Select a metric** menu. All the GCSFuse metrics will be under - **Global > Custom > metric name** - 5. Example graph for fs/ops_count -![fs/ops_count](https://user-images.githubusercontent.com/101323867/188802087-6423f4f1-2aa6-4501-8db6-3d1997986f68.png) - -## Prometheus metrics - -1. Specify Prometheus port via field `metrics:prometheus-port` in configuration file, or `--prometheus-port` cli flag. The Prometheus metrics endpoint will be exposed on this port and a path of `/metrics`. - -For example, use the following configuration file: - -```yaml -metrics: - prometheus-port: 8080 -``` - -Or, use the cli flag: - -```bash -gcsfuse --prometheus-port 8080 <bucket_name> <directory_name> -``` - -2. Run the following command to validate the Prometheus metrics endpoint is available. - -```bash -curl http://localhost:8080/metrics -``` - -The output is similar to the following: - -```text -# HELP file_cache_read_bytes_count The cumulative number of bytes read from file cache along with read type - Sequential/Random -# TYPE file_cache_read_bytes_count counter -file_cache_read_bytes_count{read_type="Random"} 0 -file_cache_read_bytes_count{read_type="Sequential"} 80 -# HELP file_cache_read_count Specifies the number of read requests made via file cache along with type - Sequential/Random and cache hit - true/false -# TYPE file_cache_read_count counter -file_cache_read_count{cache_hit="false",read_type="Random"} 215 -file_cache_read_count{cache_hit="false",read_type="Sequential"} 5 -# HELP file_cache_read_latencies The cumulative distribution of the file cache read latencies along with cache hit - true/false -# TYPE file_cache_read_latencies histogram -file_cache_read_latencies_bucket{cache_hit="false",le="1"} 215 -file_cache_read_latencies_bucket{cache_hit="false",le="2"} 216 -file_cache_read_latencies_bucket{cache_hit="false",le="3"} 216 -file_cache_read_latencies_bucket{cache_hit="false",le="4"} 216 -file_cache_read_latencies_bucket{cache_hit="false",le="5"} 216 -... -file_cache_read_latencies_sum{cache_hit="false"} 483.62783500000023 -file_cache_read_latencies_count{cache_hit="false"} 220 -# HELP fs_ops_count The cumulative number of ops processed by the file system. -# TYPE fs_ops_count counter -fs_ops_count{fs_op="FlushFile"} 9 -fs_ops_count{fs_op="GetInodeAttributes"} 91 -fs_ops_count{fs_op="LookUpInode"} 584 -fs_ops_count{fs_op="OpenDir"} 122 -fs_ops_count{fs_op="OpenFile"} 9 -fs_ops_count{fs_op="ReadDir"} 184 -fs_ops_count{fs_op="ReadFile"} 220 -fs_ops_count{fs_op="ReleaseDirHandle"} 122 -fs_ops_count{fs_op="ReleaseFileHandle"} 9 -fs_ops_count{fs_op="StatFS"} 10 -# HELP fs_ops_error_count The cumulative number of errors generated by file system operations -# TYPE fs_ops_error_count counter -fs_ops_error_count{fs_error="function not implemented",fs_error_category="function not implemented",fs_op="GetXattr"} 1 -fs_ops_error_count{fs_error="function not implemented",fs_error_category="function not implemented",fs_op="ListXattr"} 1 -fs_ops_error_count{fs_error="interrupted system call",fs_error_category="interrupt errors",fs_op="LookUpInode"} 58 -fs_ops_error_count{fs_error="no such file or directory",fs_error_category="no such file or directory",fs_op="LookUpInode"} 6 -# HELP fs_ops_latency The cumulative distribution of file system operation latencies -# TYPE fs_ops_latency histogram -fs_ops_latency_bucket{fs_op="FlushFile",le="1"} 9 -fs_ops_latency_bucket{fs_op="FlushFile",le="2"} 9 -fs_ops_latency_bucket{fs_op="FlushFile",le="3"} 9 -fs_ops_latency_bucket{fs_op="FlushFile",le="4"} 9 -fs_ops_latency_bucket{fs_op="FlushFile",le="5"} 9 -... -fs_ops_latency_sum{fs_op="FlushFile"} 0.28800000000000003 -fs_ops_latency_count{fs_op="FlushFile"} 9 -# HELP gcs_download_bytes_count The cumulative number of bytes downloaded from GCS along with type - Sequential/Random -# TYPE gcs_download_bytes_count counter -gcs_download_bytes_count{read_type="Sequential"} 2.0971528e+08 -# HELP gcs_read_count Specifies the number of gcs reads made along with type - Sequential/Random -# TYPE gcs_read_count counter -gcs_read_count{read_type="Sequential"} 5 -``` - -3. Follow [Prometheus documentation](https://prometheus.io/docs/introduction/first_steps/#configuring-prometheus) -to specify the target Prometheus metric endpoint under the `scrape_configs` section in the Prometheus configuration file. - -## References: -* More details around adding custom metrics using OpenCensus can be found [here](https://cloud.google.com/monitoring/custom-metrics/open-census) \ No newline at end of file +For instructions on how to enable and use Cloud Storage FUSE metrics, refer to the metrics guide at <https://cloud.google.com/storage/docs/cloud-storage-fuse/metrics>. diff --git a/docs/semantics.md b/docs/semantics.md index 9b31d24268..e8159f1d4c 100644 --- a/docs/semantics.md +++ b/docs/semantics.md @@ -1,25 +1,120 @@ # Read/Writes -**Reads** +## Reads -Cloud Storage FUSE makes API calls to Cloud Storage to read an object directly, without downloading it to a local directory. A TCP connection is established, in which the entire object, or just portions as specified by the application/operating system via an offset, can be read back. - -Files that have not been modified are read portion by portion on demand. Cloud Storage FUSE uses a heuristic to detect when a file is being read sequentially, and will issue fewer, larger read requests to Cloud Storage in this case, increasing performance. +### Default Reads -**Writes** +Cloud Storage FUSE makes API calls to Cloud Storage to read an object directly, without downloading it to a local directory. A TCP connection is established, in which the entire object, or just portions as specified by the application/operating system via an offset, can be read back. -For modifications to existing file objects, Cloud Storage FUSE downloads the entire -backing object's contents from Cloud Storage. The contents are stored in a local -temporary file (temp-file for short) whose location is controlled by the flag ```--temp-dir```. Later, -when the file is closed or fsync'd, Cloud Storage FUSE writes the contents of -the local file back to Cloud Storage as a new object generation, and deletes temp-file. Modifying even -a single bit of an object results in the full re-upload of the object. The +Files that have not been modified are read portion by portion on demand. Cloud Storage FUSE uses a heuristic to detect when a file is being read sequentially, and will issue fewer, larger read requests to Cloud Storage in this case, increasing performance. + +### Buffered Reads + +Buffered Read feature accelerates large sequential reads by asynchronously prefetching data into in-memory buffers and serving subsequent reads from in-memory buffer instead of making network calls. + +The feature is **disabled by default** and can be enabled using: +- Command-line flag: `--enable-buffered-read` +- Config file: `read:enable-buffered-read: true` + +**Note:** Buffered reads are designed to operate exclusively when the file cache is disabled. If both features are enabled, the file cache takes precedence and buffered reads will be ignored. + +**Best Use Cases:** +- Applications reading large files (> 100MB) with sequential access patterns. Users need to ensure enough buffer memory is configured to allow for higher parallelism (default buffer settings allow up to 2 concurrent threads). If you need to increase this, you can increase `--read-global-max-blocks`. + +**Performance Gains:** +- Can provide 2-6x improvement in sequential read throughput. +- Most effective for files larger than 100 MB. + +**Memory Usage:** +- **Per file handle:** Up to 320 MB (20 × 16MB memory blocks) while reading. +- **Global limit:** Controlled by `--read-global-max-blocks` flag or `read:global-max-blocks` config (default: 40 blocks). By default 640 MB (40 × 16MB) across all the file handles. + +**Important:** Please Consider available system memory when enabling buffered reads or adjusting `--read-global-max-blocks` to prevent out-of-memory (OOM) issues. + +**CPU Usage:** The CPU overhead is typically proportional to the performance gains achieved. + +**Random Reads:** Workloads with frequent random reads may fall back to default reads. However, GCSFuse monitors the read pattern and will automatically switch back to buffered reads if the pattern becomes sequential again. + +## Writes + +Starting with v3.0, streaming writes is the default write path. For more details, see the `With Streaming Writes` +section below. You can revert to the previous default write path (staging writes to a temporary file on disk) using the +`--enable-streaming-writes=false` flag or `write:enable-streaming-writes: false` in the config file. + +### Streaming Writes - Default Write Path + +Starting with version 2.9.1, and becoming the default in v3.0.0, GCSFuse supports streaming-writes, which is a new write +path that uploads data directly to Google Cloud Storage (GCS) as it's written without fully staging the file in the +temp-dir. This reduces both latency and disk space usage, making it particularly beneficial for large, sequential writes +such as checkpoints. Streaming writes can be enabled using `--enable-streaming-writes` flag or +`write:enable-streaming-writes:true` in the config file (Default starting GCSFuse v3.0.0). + +**Memory Usage:** Each file opened for streaming writes will consume +approximately 96MiB of RAM during the upload process. This memory is released +when the file handle is closed. This should be considered when planning resource +allocation for applications using streaming writes. + +Memory usage can be controlled using the `--write-global-max-blocks` flag or `write:global-max-blocks` config. The +default value is 4 for low-spec machines and 1600 for high-spec machines. One block is used per file, which means that +on low-spec machines, writes will automatically fall back to legacy staged writes if more than 4 files are concurrently +opened for streaming writes. + +#### Note on Streaming Writes: + +- **New files, Sequential Writes:** Streaming writes are designed for sequential + writes to a new file only. Modifying existing files, or doing out-of-order + writes (whether from the same file handle or concurrent writes from multiple + file handles) will cause GCSFuse to automatically revert to the existing write + path of staging writes to a temporary file on disk. An informational log + message will be emitted when this fallback occurs. + +- **Concurrent Writes to the Same File:** While concurrent writes to the same + file are possible, they are not the primary use case for this initial phase of + streaming writes. If a (rare, often server-related) error occurs during + concurrent writes, all file handles must be closed before any future writes + can resume. This phase of streaming writes is optimized for single-stream + writes to new files, such as for AI/ML checkpointing. + +- **File System Semantics Change:** + - **FSync operation does not finalize the object:** When streaming writes + are enabled, the fsync operation will not finalize the object on GCS. + Instead, the object will be finalized only when the file is closed. + Only finalized objects are visible to the end user. This is a key + difference from the default non-streaming-writes behavior and should be considered when + using streaming writes. Relying on fsync for data durability with + streaming writes enabled is not recommended. Data is guaranteed to be + on GCS only after the file is closed. + - **Rename Operation Syncs the File:** Rename operation on a file undergoing + writes via streaming writes will be finalized and then renamed. This means + that any follow up writes will automatically revert to the existing + behavior of staging writes to a temporary file on disk. + - **Read Operations During Write:** Reads are now supported on files that are being + written to with streaming writes. However, performing a read operation will finalize the object on GCS. Any + subsequent write operations to that file will then automatically revert to legacy staged writes. Applications should + generally avoid reading from a file while it is being written to using streaming writes, as this will prematurely + finalize the object. + - **Truncate During Writes:** If a file is truncated downwards using truncate() or ftruncate() while streaming + writes + are in progress, the file on GCS is finalized, and any subsequent writes revert to legacy staged writes. + +### Staged Writes - Legacy Write Path + +Files are written locally as a temporary file (temp-file for short) whose +location is controlled by the flag `--temp-dir`. Upon closing or fsyncing +the file, the file is then written to your Cloud Storage bucket and the +temp-file is deleted. + +For modifications to existing files, Cloud Storage FUSE downloads the +entire +backing object's contents from Cloud Storage, storing them in the same temporary +directory as mentioned above. When the file is closed or fsync'd, Cloud Storage +FUSE writes the contents of the local file back to Cloud Storage as a new object +generation, and deletes +temp-file. Modifying even a single bit of an object results in the full +re-upload of the object. The exception is if an append is done to the end of a file, where the original file is at least 2MB, then only the appended content is uploaded. -For new objects, objects are first written to the same temporary directory as -mentioned above. Upon closing or fsyncing the file, the file is then written to -your Cloud Storage bucket. As new and modified files are fully staged in the local temporary directory until they are written out to Cloud Storage, you must ensure that there is enough free space available to handle staged content @@ -27,25 +122,29 @@ when writing large files. #### Notes -- Prior to version 1.2.0, you will notice that an empty file is created in the - Cloud Storage bucket as a hold. Upon closing or fsyncing the file, the file - is written to your Cloud Storage bucket, with the existing empty file now - reflecting the accurate file size and content. Starting with version 1.2, - the default behavior is to not create this zero-byte file, which increases - write performance. If needed, it can be re-enabled by setting the - `create-empty-file: true` configuration in the config file. -- If the application never sends fsync for a file, it will leave behind its - temp-file (in temp-dir), which will not be cleared until the user unmounts - the bucket. As an example, if you are writing a large file, and temp-dir - does not have enough free space available, then you will get 'out of space' - error. Then the temp-file will not be deleted until you do an fsync for that - file, or unmount the bucket. +- Prior to version 1.2.0, you will notice that an empty file is created in the + Cloud Storage bucket as a hold. Upon closing or fsyncing the file, the file + is written to your Cloud Storage bucket, with the existing empty file now + reflecting the accurate file size and content. Starting with version 1.2, + the default behavior is to not create this zero-byte file, which increases + write performance. If needed, it can be re-enabled by setting the + `create-empty-file: true` configuration in the config file. +- If the application never sends fsync for a file, it will leave behind its + temp-file (in temp-dir), which will not be cleared until the user unmounts + the bucket. As an example, if you are writing a large file, and temp-dir + does not have enough free space available, then you will get 'out of space' + error. Then the temp-file will not be deleted until you do an fsync for that + file, or unmount the bucket. + +___ -**Concurrency** +# Concurrency -Multiple readers can access the same or different objects from the same bucket without issue. Multiple writers can also write to different objects in the same bucket without issue. However, there is no concurrency control for multiple writers to the same file. When multiple writers try to replace a file, the last write wins and all previous writes are lost - there is no merging, version control, or user notification of the subsequent overwrite. Therefore, for data integrity it is recommended that multiple sources do not modify the same object. +Multiple readers can access the same or different objects within a bucket without issue. Likewise, multiple writers can modify different objects in the same bucket simultaneously without any issue. Concurrent writes to the same gcs object are supported from the same mount and behave similar to native file system. -**Write/read consistency** +However, when different mounts try to write to the same object, the flush from first mount wins. Other mounts that have not updated their local file descriptors after the object is modified will encounter a ```syscall.ESTALE``` error when attempting to save their edits due to precondition checks. Therefore, to ensure data is consistently written, it is strongly recommended that multiple sources do not modify the same object. + +### Write/Read consistency Cloud Storage by nature is [strongly consistent](https://cloud.google.com/storage/docs/consistency). Cloud Storage FUSE offers close-to-open and fsync-to-open consistency. Once a file is closed, consistency is guaranteed in the following open and read immediately. @@ -53,7 +152,20 @@ Close and fsync create a new generation of the object before returning, as long Examples: - Machine A opens a file and writes then successfully closes or syncs it, and the file was not concurrently unlinked from the point of view of A. Machine B then opens the file after machine A finishes closing or syncing. Machine B will observe a version of the file at least as new as the one created by machine A. -- Machine A and B both open the same file, which contains the text ‘ABC’. Machine A modifies the file to ‘ABC-123’ and closes/syncs the file which gets written back to Cloud Storage. After, Machine B, which still has the file open, instead modifies the file to ‘ABC-XYZ’, and saves and closes the file. As the last writer wins, the current state of the file will read ‘ABC-XYZ’. +- Machine A and B both open the same file, which contains the text ‘ABC’. Machine A modifies the file to ‘ABC-123’ and closes/syncs the file which gets written back to Cloud Storage. Afterward, Machine B, which still has the file open, modifies its local copy to ‘ABC-XYZ’, then tries to save and close the file. Since the first writer wins, the final state of the file in the cloud storage will be 'ABC-123'. Consequently, Machine B's file descriptor will receive an ESTALE error. + +### Stale File Handle Errors + +To ensure consistency, Cloud Storage FUSE returns a ```syscall.ESTALE``` error when an application tries to access stale data. This can occur in the following circumstances: + +- **Concurrent Writes**: When multiple mounts have the same file open for writing, and one mount modifies and syncs the file, other mounts with open file descriptors will encounter this error when attempting to sync or close the file. +- **Read During Modification**: When an application is reading a file through a GCSFuse mount, and the same object is modified on GCS (by deleting, renaming, or changing its content or metadata), the GCSFuse reader will encounter this error. This is because GCSFuse detects that the file it was accessing has changed. +- **File Renaming During Write**: When an application is writing to a file through a GCSFuse mount, and the same object is renamed on Google Cloud Storage (via same or different GCSFuse mount or through another interface), the writer will encounter this error when syncing or closing the file. +- **File Deletion During Write**: When an application is writing to a file through a GCSFuse mount, and the same object is deleted on Google Cloud Storage (via different GCSFuse mount or through another interface), the writer will encounter this error when syncing or closing the file. + +These changes in Cloud Storage FUSE prioritize data integrity and provide users with clear indications of potential conflicts, preventing silent data loss and ensuring a more robust and reliable experience. + +___ # Caching @@ -64,7 +176,7 @@ The default behavior is appropriate, and brings significant performance benefits **Important**: The rest of this document assumes that caching is disabled (by setting ```--stat-cache-ttl 0``` and ```--type-cache-ttl 0``` or ```metadata-cache:ttl-secs: 0```). This is not the default. If you want the consistency guarantees discussed in this document, you must use these options to disable caching. -**Stat caching** +## Stat caching The cost of the consistency guarantees discussed in the rest of this document is that Cloud Storage FUSE must frequently send stat object requests to Cloud Storage in order to get the freshest possible answer for the kernel when it asks about a particular name or inode, which happens frequently. This can make what appear to the user to be simple operations, like ```ls -l```, take quite a long time. @@ -99,12 +211,14 @@ The behavior of stat cache is controlled by the following flags/config parameter Positive and negative stat results will be cached for the specified amount of time. -Warning: Using stat caching breaks the consistency guarantees discussed in this document. It is safe only in the following situations: -- The mounted bucket is never modified. -- The mounted bucket is only modified on a single machine, via a single Cloud Storage FUSE mount. -- The mounted bucket is modified by multiple actors, but the user is confident that they don't need the guarantees discussed in this document. +Warnings: +- Using stat caching breaks the consistency guarantees discussed in this document. It is safe only in the following situations: + - The mounted bucket is never modified. + - The mounted bucket is only modified on a single machine, via a single Cloud Storage FUSE mount. + - The mounted bucket is modified by multiple actors, but the user is confident that they don't need the guarantees discussed in this document. +- On high performance machines GCSFuse sets TTL to infinite by default ([refer](https://cloud.google.com/storage/docs/cloud-storage-fuse/caching#cache-invalidation)). Please override it manually if your workload requires consistency guarantees. -**Type caching** +## Type caching Because Cloud Storage does not forbid an object named ```foo``` from existing next to an object named ```foo/``` (see the Name conflicts section), when Cloud Storage FUSE is asked to look up the name "foo" it must stat both objects. @@ -127,7 +241,7 @@ The behavior of type cache is controlled by the following flags/config parameter - The mounted bucket is never modified. - The type (file or directory) for any given path never changes. -**File caching** +## File caching The Cloud Storage FUSE file cache feature is a client-based read cache that lets repeat file reads to be served from a faster local cache storage media of your choice. @@ -163,7 +277,7 @@ Additional file cache [behavior](https://cloud.google.com/storage/docs/gcsfuse-c - If a Cloud Storage FUSE client modifies a cached file or its metadata, then the file is immediately invalidated and consistency is ensured in the following read by the same client. However, if different clients access the same file or its metadata, and its entries are cached, then the cached version of the file or metadata is read and not the updated version until the file is invalidated by that specific client's TTL setting. -**Kernel List Cache** +## Kernel List Cache As the name suggests, the Cloud Storage FUSE kernel-list-cache is used to cache the directory listing (output of `ls`) in kernel page-cache. It significantly improves the workload which involves repeated listing. For multi node/mount-point scenario, this is recommended to be used only for read only workloads, e.g. for Serving and Training workloads. @@ -176,6 +290,7 @@ By default, the list cache is disabled. It can be enabled by configuring the `-- * The kernel-list-cache is kept within the kernel's page-cache. Consequently, this functionality depends upon the availability of page-cache memory on the system. This contrasts with the stat and type caches, which are retained in user memory as part of Cloud Storage Fuse daemon. * The kernel's list cache is maintained on a per-directory level, resulting in either all list entries being retained in the page cache or none at all. * The creation, renaming, or deletion of new files or folders causes the eviction of the page-cache of their immediate parent directory, but not of all ancestral directories. +* The ttl-based eviction doesn't work with kernel versions 6.9.x to 6.12.x ([details](https://github.com/GoogleCloudPlatform/gcsfuse/issues/2792)), because of a [bug](https://lore.kernel.org/linux-fsdevel/CAEW=TRr7CYb4LtsvQPLj-zx5Y+EYBmGfM24SuzwyDoGVNoKm7w@mail.gmail.com/) in kernel-fuse driver, which is [fixed](https://github.com/torvalds/linux/commit/03f275adb8fbd7b4ebe96a1ad5044d8e602692dc) in 6.13.x. Although, eviction because of creation, renaming, or deletion of a file or folders from the same mount works as expected. **Consistency** * Kernel List cache ensures consistency within the mount. That means, creation, deletion or rename of files/folder within a directory evicts the kernel list cache of the directory. @@ -189,6 +304,8 @@ By default, the list cache is disabled. It can be enabled by configuring the `-- For now, for backward compatibility, both are accepted, and the minimum of the two, rounded to the next higher multiple of a second, is used as TTL for both stat-cache and type-cache, when ```metadata-cache: ttl-secs``` is not set. 1. Both stat-cache and type-cache internally use the same TTL. +___ + # Files and Directories As Cloud Storage FUSE is a way to mount a bucket as a local filesystem, and directories are essential to filesystems, Cloud Storage FUSE presents directories logically using ```/``` prefixes. Cloud Storage object names map directly to file paths using the separator '/'. Object names ending in a slash represent a directory, and all other object names represent a file. Directories are by default not implicitly defined; they exist only if a matching object ending in a slash exists. @@ -219,9 +336,9 @@ Even though A/, A/B/, and C/ are directories in the filesystem, a 0-byte object The above example was based on greenfield deployments which assumes starting fresh, where the directories are created from Cloud Storage FUSE. If a user unmounts this Cloud Storage FUSE bucket, and then re-mounts it to a different path, the user will see the directory structure correctly in the filesystem because it was originally created by Cloud Storage FUSE. -However, if a user already has objects with prefixes to simulate a directory structure in their buckets that did not originate from Cloud Storage FUSE, and mounts the bucket using Cloud Storage FUSE, the directories and objects under the directories will not be visible until a user manually creates the directory, with the same name, using mkdir on the local instance. This is because with Cloud Storage FUSE, directories are by default not implicitly defined; they exist only if a matching object ending in a slash exists. These backing objects for directories are special 0-byte objects which are placeholders for directories. Note that these can also be created via the WebUI and Cloud Storage SDKs, but not by the Cloud Storage CLI tools such as gcloud or gsutil. +However, if a user already has objects with prefixes to simulate a directory structure in their buckets that did not originate from Cloud Storage FUSE, and mounts the bucket using Cloud Storage FUSE, the directories and objects under the directories will not be visible until a user manually creates the directory, with the same name, using mkdir on the local instance. This is because with Cloud Storage FUSE, directories are by default not implicitly defined; they exist only if a matching object ending in a slash exists. These backing objects for directories are special 0-byte objects which are placeholders for directories. Note that these can also be created via the WebUI and Cloud Storage SDKs, but not by the Cloud Storage CLI tools such as gcloud. -If a user has the following objects in their Cloud Storage buckets, for example created by uploading a local directory using `gsutil cp -r` command. +If a user has the following objects in their Cloud Storage buckets, for example created by uploading a local directory using `gcloud storage cp -r` command. - A/1.txt - A/B/2.txt - C/3.txt @@ -254,17 +371,21 @@ Cloud Storage Fuse also offers seamless support for buckets with hierarchical na HNS-enabled buckets offer several advantages over standard buckets when used with cloud storage fuse: -- HNS-enabled buckets eliminates the need for --implicit-dirs flag. HNS buckets inherently understand directories, so gcsfuse does not need to simulate directories using placeholder objects ( 0-byte objects ending with '/' ). Users will see consistent directory listings with or without the flag. +- HNS-enabled buckets eliminate the need for --implicit-dirs flag. HNS buckets inherently understand directories, so gcsfuse does not need to simulate directories using placeholder objects ( 0-byte objects ending with '/' ). Users will see consistent directory listings with or without the flag. - In HNS buckets, renaming a folder and its child folders is an atomic operation, meaning all associated resources—including objects and managed folders—are renamed in a single step. This ensures data consistency and significantly improves operation performance. - HNS buckets treat folders as first-class entities, closely aligning with traditional file system semantics. Commands like mkdir now directly create folder resources within the bucket, unlike with traditional buckets where directories were simulated using prefixes and 0-byte objects. - List object calls ([BucketHandle.Objects](https://cloud.google.com/storage/docs/json_api/v1/objects/list)), are replaced with [get folder](https://cloud.google.com/storage/docs/json_api/v1/folders/getfoldermetadata) calls, resulting in quicker response times and fewer overall list calls for every lookup operation. +___ + # Generations With each record in Cloud Storage is stored object and metadata [generation numbers](https://cloud.google.com/storage/docs/generations-preconditions). These provide a total order on requests to modify an object's contents and metadata, compatible with causality. So if insert operation A happens before insert operation B, then the generation number resulting from A will be less than that resulting from B. In the discussion below, the term "generation" refers to both object generation and meta-generation numbers from Cloud Storage. In other words, what we call "generation" is a pair ```(G, M)``` of Cloud Storage object generation number ```G``` and associated meta-generation number ```M```. +___ + # File inodes As in any file system, file inodes in a Cloud Storage FUSE file system logically contain file contents and metadata. A file inode is initialized with a particular generation of a particular object within Cloud Storage (the "source generation"), and its contents are initially exactly the contents and metadata of that generation. @@ -318,6 +439,8 @@ Cloud Storage FUSE sets the following pieces of Cloud Storage object metadata fo - contentType is set to Cloud Storage's best guess as to the MIME type of the file, based on its file extension. - The custom metadata key gcsfuse_mtime is set to track mtime, as discussed above. +___ + # Directory Inodes Cloud Storage FUSE directory inodes exist simply to satisfy the kernel and export a way to look up child inodes. Unlike file inodes: @@ -339,9 +462,21 @@ Cloud Storage FUSE makes similar calls while deleting a directory: it lists obje Note that by definition, implicit directories cannot be empty. +___ + # Symlink inodes +Prior to GCSFuse v3.9.0, symlinks were represented by empty Cloud Storage objects. These objects included a custom metadata key, ```gcsfuse_symlink_target```, whose value specified the symlink's target. + +From GCSFuse v3.9.0 onwards, the standard GCS representation for symbolic links is adopted. This means symlinks are now non-zero sized GCS objects. They feature the custom metadata key ```goog-reserved-file-is-symlink``` with a value of ```true```, and the symlink's target path is stored directly within the object's content.In addition, to ensure that older GCSFuse versions can still read these symlinks, the legacy `gcsfuse_symlink_target` metadata key is also populated with the target path. In other respects they work like a file inode, including receiving the same permissions. **Please note that GCSFuse remains backward compatible with symlinks created by versions prior to v3.9.0.** + + -Cloud Storage FUSE represents symlinks with empty Cloud Storage objects that contain the custom metadata key ```gcsfuse_symlink_target```, with the value giving the target of a symlink. In other respects they work like a file inode, including receiving the same permissions. +**Note** + +While GCSFuse supports symlinks that point to paths external to the mount point, it should be avoided as it could lead to broken links and security issues. + + +___ # Permissions and ownership @@ -357,6 +492,8 @@ The fuse kernel layer itself restricts file system access to the mounting user ( This can be overridden by setting ```-o allow_other``` to allow other users to access the file system. However, there may be [security implications](https://github.com/torvalds/linux/blob/a33f32244d8550da8b4a26e277ce07d5c6d158b5/Documentation/filesystems/fuse.txt#L218-L310). +___ + # Non-standard filesystem behaviors See [Key Differences from a POSIX filesystem](https://cloud.google.com/storage/docs/gcs-fuse#expandable-1) @@ -383,15 +520,12 @@ Traditional file systems do not allow multiple directory entries with the same n Instead, when a conflicting pair of foo and ```foo/``` objects both exist, it appears in the Cloud Storage FUSE file system as if there is a directory named foo and a file or symlink named ```foo\n``` (i.e. foo followed by U+000A, line feed). This is what will appear when the parent's directory entries are read, and Cloud Storage FUSE will respond to requests to look up the inode named ```foo\n``` by returning the file inode. ```\n``` in particular is chosen because it is not legal in Cloud Storage object names, and therefore is not ambiguous. -### Unsupported object names +### Unsupported Path names + +- Due to limitations in the Linux filesystem, path segments such as `//`, `/./`, or `/../` are not supported locally, even though they are valid object names in GCS. From v3.6.0 onwards, GCSFuse handles these objects gracefully: -Objects in GCS with double slashes '//' as a name or -prefix are not supported in GCSfuse. Accessing a directory with such -named files will cause an 'input/output error', as the Linux -filesystem does not support files or directories named with a '/'. -The most common example of this is an object called, for example -'A//C.txt' where 'A' indicates a directory and 'C.txt' indicates a -file, and is missing directory 'B/' between 'A/' and 'C.txt'. + 1. Listing: To prevent system errors or crashes, these unsupported objects are hidden from file listings. + 2. Rename/Delete: Directory-level operations still apply to all contained objects, ensuring that unsupported objects are not accidentally left behind. ## Memory-mapped files @@ -418,4 +552,3 @@ Not all of the usual file system features are supported. Most prominently: - File and directory permissions and ownership cannot be changed. See the permissions section above. - Modification times are not tracked for any inodes except for files. - No other times besides modification time are tracked. For example, ctime and atime are not tracked (but will be set to something reasonable). Requests to change them will appear to succeed, but the results are unspecified. - diff --git a/docs/tracing.md b/docs/tracing.md new file mode 100644 index 0000000000..fc9ff6dc12 --- /dev/null +++ b/docs/tracing.md @@ -0,0 +1,164 @@ +## GCSFuse Tracing + +### Introduction + +GCSFuse traces each of its FUSE file system operations. It further traces both the standard HTTP/1 and gRPC client calls inside every FUSE file system operation. GCSFuse supports exporting the traces locally to the process output stream and also to Google Cloud Trace. For information on permissions, requirements for exporting traces to Google Cloud Trace, and basic tracing concepts, refer to the [Google Cloud Trace Docs](https://docs.cloud.google.com/trace/docs/overview). + +### Enabling tracing + +To enable tracing in GCSFuse and visualize the data as a waterfall/gantt chart in the Google Cloud Trace Explorer, add the required configuration to your GCSFuse config-file as shown below: + +``` +trace: + sampling-ratio: 0.1 +``` + +OR + +GCSFuse command line option below: + +`--trace-sampling-ratio=0.1` Sets the **sampling rate**. This means **10%** (1 out of 10) of all traces will be exported. This helps manage cost and overhead in high-traffic applications. + +### Accessing and Viewing Trace Exports + +For better visualization of the traces exported in GCSFuse you can access the Trace Explorer on Google Cloud Console using the following link to find the traces. + +[Trace Explorer Link](https://console.cloud.google.com/traces/explorer) + +To restrict the exported traces in the filter input to only the current running instance of the GCSFuse mount, use the following attribute filter: + +``` +Key = service.instance.id +Value = <your-unique-mount-id> +``` + +You can find your unique mount instance ID in any GCSFuse log line from the beginning of the mount, for example: mount-id=<your-unique-mount-id>. + +The trace explorer displays the spans generated by GCSFuse, which you can filter by attributes like the GCSFuse instance ID or the host's machine type. This allows you to isolate traces for a specific mount and gain performance insights, such as comparing performance across different machine types. + +![](https://github.com/user-attachments/assets/26baaa4c-083f-486e-8db3-6f2f38cb2ecc) + +Trace Explorer view filtering by mount instance ID + +#### Filtering Traces in Trace Explorer + +We can filter by several attributes and also filter only specific spans to get a timeline view of all the calls attributed to a single trace. Once you get a trace ID, you can also search spans using a unique trace ID. + +![](https://github.com/user-attachments/assets/4956965e-1b7c-4bce-b6d0-18def4fc239e) + +Waterfall/Gantt chart view of a single trace + +### Interpreting GCSFuse Spans + +Common Spans recorded and what each of them signifies + +| Span Name (as visible in trace explorer) | Description (of what the underlying span traces) | +| :---- | :---- | +| **FUSE Operations** | | +| **fs.stat_fs** | Retrieves file system-wide statistics, such as total blocks, free blocks, and block size. | +| **fs.inode.lookup** | Look up a directory entry by name within a parent directory to find the corresponding inode. This is fundamental for path resolution. | +| **fs.inode.get_attributes** | Retrieves the attributes of an inode, such as its size, permissions, and modification times. | +| **fs.inode.set_attributes** | Modifies the attributes of an inode, for example, changing its size (**truncate**), permissions (**chmod**), or owner (**chown**). | +| **fs.inode.forget** | Informs the file system that the kernel is no longer referencing a particular inode, allowing the file system to reclaim resources associated with it. | +| **fs.batch_forget** | A batch version of **ForgetInode** that allows the kernel to inform the file system about multiple inodes that are no longer in use. | +| **fs.dir.mk** | Creates a new directory. | +| **fs.mknode** | Creates a new file system node, which can be a regular file, a device file, or a named pipe. | +| **fs.file.create** | Creates and opens a new regular file. | +| **fs.link.create** | Creates a **hard link** to an existing file. | +| **fs.symlink.create** | Creates a **symbolic link**. | +| **fs.rename** | Renames a file or directory, potentially moving it to a different directory. | +| **fs.dir.rm** | Removes an empty directory. | +| **fs.unlink** | Removes a file (deletes a name from the file system). If that name was the last link to a file and no processes have the file open, the file is deleted and the space it was using is made available for reuse. | +| **fs.dir.open** | Open a directory for reading its contents. | +| **fs.dir.read** | Reads entries from an open directory. | +| **fs.dir.read_plus** | Similar to **ReadDir**, but it can also return the attributes of the entries, which can be more efficient than calling **LookUpInode** and **GetInodeAttributes** for each entry. | +| **fs.dir.release_handle** | Releases an open directory handle, called when a process is done reading a directory. | +| **fs.file.open** | Open a file for reading or writing. | +| **fs.file.read** | Reads data from an open file. | +| **fs.file.write** | Writes data to an open file. | +| **fs.file.sync** | Requests that any cached data for an open file be written to the underlying storage. | +| **fs.file.flush** | Called when a file handle is being closed. This is an opportunity to flush any cached data. | +| **fs.file.release_handle** | Releases an open file handle, called when a process closes a file. | +| **fs.symlink.read** | Reads the target of a symbolic link. | +| **GCS Operations (gRPC)** | | +| **google.storage.v2.Storage/ListObjects** | The gRPC call to list a collection of objects (like a directory listing) within a Google Cloud Storage bucket. | +| **cloud.google.com/go/storage.grpcStorageClient.ObjectsListCall** | The gRPC client-side function call within the Go library that initiates the object listing operation. | +| **google.storage.v2.Storage/ReadObject** | The gRPC call to stream the content (data) of a specific GCS object. | +| **google.storage.v2.Storage/GetObject** | The gRPC call to retrieve the **metadata/attributes** (not the data content) of a single GCS object. | +| **google.storage.control.v2.StorageControl/GetFolder** | A specific gRPC call to retrieve metadata for a GCS Folder (using the Storage Control API). | +| **Read flow traces** | | +| **buffered_reader** | Indicates the read is served by the in-memory buffered reader, which prefetches data from GCS. | +| **file_cache_reader** | Indicates the read is served from the on-disk file cache. | +| **gcs_reader** | Indicates the read is served by making a direct request to GCS. | +| **file.cache.read** | Tracks read operations specifically from the local file cache. | +| **file.cache.write** | Tracks write or population operations into the local file cache. | +| **GCS Operations (HTTP)** | | +| **HTTP GET** | An entire end-to-end trace for a client's GET request. | +| **HTTP POST** | An entire end-to-end trace for a client's POST request. | +| **cloud.google.com/go/storage.httpStorageClient.ObjectsListCall** | A specific operation to list objects within a Google Cloud Storage (GCS) bucket. | +| **cloud.google.com/go/storage.Object.Attrs** | A specific operation to get the metadata/attributes of a single GCS object. | +| **Low-level HTTP Transport** | | +| **http.dns** | Time spent resolving the domain name to an IP address. | +| **http.getconn** | Time spent waiting for an idle connection from the connection pool or establishing a new one. | +| **http.tls** | Time spent performing the TLS/SSL handshake (key exchange and certificate verification). | +| **http.headers** | Time spent waiting for the first byte of the response headers after sending the request. | +| **http.send** | Time spent sending the entire request (headers and body) to the server. | +| **http.receive** | Time spent receiving the entire response body from the server. | + +### Differentiating gRPC from HTTP Spans + +You can differentiate gRPC traces and spans from traditional HTTP ones based on two key characteristics: **Naming Convention** and **Trace Content/Attributes**. + +#### Naming Convention + +| Trace Type | Naming Pattern | Example | +| :---- | :---- | :---- | +| **gRPC** | Uses the full **Service/Method** format, often starting with the API version and service name. | google.storage.v2.Storage/ListObjects | +| **HTTP** | Uses the **HTTP method** or lower-level network phases. | HTTP GET, http.dns, http.send | + +#### Trace Content and Attributes + +The attributes (tags) attached to the span clearly indicate the protocol: + +* **gRPC Spans** will contain OpenTelemetry attributes starting with `rpc.`: + * `rpc.system`: Typically `"grpc"` + * `rpc.method`: The full method name (e.g., `ListObjects`) + * `rpc.grpc.status_code`: The gRPC status code (e.g., `0` for OK) +* **HTTP Spans** will contain attributes starting with `http.`: + * `http.method`: The HTTP verb (`GET`, `POST`) + * `http.url`: The full resource URL + * `http.status_code`: The three-digit HTTP status code (e.g., `200`, `404`) + +### Best practices + +#### Controlling Trace Volume with Sampling + +Trace sampling is a critical mechanism for managing the operational overhead and costs associated with tracing in high-throughput environments. + +**Sampling Ratio:** The `trace-sampling-ratio` flag controls the fraction of GCSFuse operations that are traced and exported. + +This ratio is a floating-point number between 0.0 (no traces exported) and 1.0 (all traces exported). + +**Crucially:** Once a root trace is selected for export by the sampling mechanism, all of its associated spans (sub-operations) are guaranteed to be fully captured. This ensures that the exported trace is complete and useful for analysis. + +| Sampling Ratio | Effect | +| :---- | :---- | +| **1.0** | Exports **100%** of all GCSFuse operations (Highest detail, highest cost/overhead). | +| **0.1** | Exports **10%** of all GCSFuse operations (Good for production monitoring, balances detail and cost). | +| **0.01** | Exports **1%** of all GCSFuse operations (Used for high-volume traffic/low-cost scenarios). | + +### Analyzing Latency with Span Groups + +When analyzing a trace, the total time taken for an operation is the sum of time spent in various components. By grouping spans, you can identify where the majority of the time is being spent. This helps pinpoint whether a latency bottleneck is within GCSFuse, the GCS client library, the network, or the GCS service itself. + +Here is a breakdown of span groups and what they represent: + +| Span Group | Span Name Examples | What it Measures | +| :--- | :--- | :--- | +| **GCSFuse Kernel/FUSE Layer** | `fs.inode.lookup`, `fs.file.read` | The time taken to serve the kernel request by GCSFuse. This is the top-level span for any file system operation. | +| **GCSFuse Internal Caching & Buffering** | `buffered_reader`, `file_cache_reader`, `file.cache.read` | Time spent in GCSFuse's internal mechanisms like read-ahead buffering or file content caching. High latency here might indicate cache misses or inefficient buffering. | +| **GCS Client Library (Go)** | `cloud.google.com/go/storage...` | The time spent within the official Google Cloud Storage Go client library. This includes logic for preparing and parsing requests/responses. | +| **GCS API Calls (Network)** | `google.storage.v2...`, `HTTP GET`/`POST` | The end-to-end time for a network call to the GCS API, as measured from the client side. This includes network latency, time spent processing on the GCS server, and data transfer time. If this is high, the bottleneck is likely network-related or on the GCS service side. | +| **Low-Level Network (HTTP only)** | `http.dns`, `http.tls`, `http.getconn` | For HTTP-based traffic, these spans break down the network time into its constituent parts: DNS lookup, TLS handshake, and connection acquisition. High latency in these spans points to specific network or infrastructure issues. | + +By examining the waterfall view of a trace, you can visually see how these spans nest and which ones contribute most to the overall latency of the parent FUSE operation. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 291a208356..278ecfb27b 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -7,13 +7,17 @@ discusses potential solutions to the same. Most of the common mount point issues are around permissions on both local mount point and the Cloud Storage bucket. It is highly recommended to retry with --foreground --log-severity=TRACE flags which would provide much more detailed logs to understand the errors better and possibly provide a solution. -### Mount successful but files not visible +### Mount successful but files are not visible -Try mounting the gcsfuse with --implicit-dir flag. Read the [semantics](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md) to know the reasoning. +Try mounting the gcsfuse with `--implicit-dirs` flag. Read the [semantics](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#files-and-directories) to know the reasoning. ### Mount failed with fusermount3 exit status 1 -It comes when the bucket is already mounted in a folder, and we try to mount it again. You need to unmount first and then remount. +- It can come when the bucket is already mounted in a folder, and we try to mount it again. You need to unmount first and then remount. +- It can also happen if you're trying to mount the bucket on a directory that has read-only permissions. Please provide write permissions to the directory and try mounting it again. You can use the below command to grant write permissions. + ``` + chmod 755 mount_point + ``` ### version GLIBC_x.yz not found @@ -27,6 +31,12 @@ Run ```gcloud auth application-default login``` command to fetch default credent It’s a generic error, but the most probable culprit is the bucket not having the right permission for Cloud Storage FUSE to operate on. Ref - [here](https://stackoverflow.com/questions/36382704/gcsfuse-input-output-error) +### Stale File Handle Error (ESTALE) + +This error occurs when GCSFuse detects that a file has been modified or deleted on GCS by another process or mount since it was opened. This is a data integrity feature that prioritizes providing clear indications of potential conflicts, preventing silent data loss. + +To avoid this, it's best to prevent multiple sources from modifying the same object simultaneously. For a detailed explanation of the scenarios that can cause this error, please refer to the [Stale File Handle Errors](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#stale-file-handle-errors) section in our semantics documentation. + ### Generic NO_PUBKEY Error - while installing Cloud Storage FUSE on ubuntu 22.04 It happens while running - ```sudo apt-get update``` - working on installing Cloud Storage FUSE. You just have to add the pubkey you get in the error using the below command: ```sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys <PUBKEY> ``` And then try running ```sudo apt-get update``` @@ -62,10 +72,6 @@ Only specific OS distributions are currently supported, learn more about [Instal Pass [_netdev option](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/mounting.md#persisting-a-mount) in fstab entry (reference issue [here](https://github.com/GoogleCloudPlatform/gcsfuse/issues/1043)). With this option, mount will be attempted on reboot only when network is connected. -### Cloud Storage FUSE get stuck when using it to concurrently work with a large number of opened files (reference issue [here](https://github.com/GoogleCloudPlatform/gcsfuse/issues/1043)) - -This happens when gcsfuse is mounted with http1 client (default) and the application using gcsfuse tries to keep more than value of `--max-conns-per-host` number of files opened. You can try (a) Passing a value higher than the number of files you want to keep open to `--max-conns-per-host` flag. (b) Adding some timeout for http client connections using `--http-client-timeout` flag. - ### Permission Denied error. Please refer [here](https://cloud.google.com/storage/docs/gcsfuse-mount#authenticate_by_using_a_service_account) to know more about permissions @@ -73,11 +79,11 @@ Please refer [here](https://cloud.google.com/storage/docs/gcsfuse-mount#authenti **Solution** - depending upon the use-case, you can choose one of the following options. * If you are explicitly authenticating for a specific service account by providing say a key-file, then make sure that the service account has appropriate IAM role for the operation e.g. roles/storage.objectAdmin, roles/storage.objectUser * If you are using the default service account i.e. not specifying a key-file, then ensure that - * The VM's service account has got the required IAM roles for the operation e.g. roles/storage.objectUser to allow read-write access. - * The VM's scope has been appropriately set. You can set the scope to storage-full to give the VM full-access to the cloud-storage buckets. For this: - * Turn-off the instance - * Change the VM's scope either by using the GCP console or by executing `gcloud beta compute instances set-scopes INSTANCE_NAME --scopes=storage-full` - * Start the instance + * The VM's service account has got the required IAM roles for the operation e.g. roles/storage.objectUser to allow read-write access. + * The VM's scope has been appropriately set. You can set the scope to storage-full to give the VM full-access to the cloud-storage buckets. For this: + * Turn-off the instance + * Change the VM's scope either by using the GCP console or by executing `gcloud beta compute instances set-scopes INSTANCE_NAME --scopes=storage-full` + * Start the instance ### Bad gateway error while installing/upgrading GCSFuse: `Err: http://packages.cloud.google.com/apt gcsfuse-focal/main amd64 gcsfuse amd64 1.2.0`<br/>`502 Bad Gateway [IP: xxx.xxx.xx.xxx 80]` @@ -101,9 +107,15 @@ Unable to mount with the following error `daemonize.Run: readFromProcess: sub-pr Fuse package is not installed. It may throw this error. Run the following command to install fuse <br/> <code> sudo apt-get install fuse </code> for Debian/Ubuntu <br/> <code> sudo yum install fuse </code> for RHEL/CentOs/Rocky <br/> -### ls: reading directory \<mountpath>/\<path>: Input/output error +### Encountered unsupported prefixes during listing, or ls: reading directory \<mountpath>/\<path>: Input/output error +Unable to list unsupported objects in a mounted bucket, i.e. objects which have names with `//` in them or have names starting with `/` e.g. `gs://<bucket>//A` or `gs://<bucket>/A//B` etc. Such objects can be listed by the following command: `gcloud storage ls gs://<bucket>/**//**`. + +The `Input/output error` has been suppressed from v3.6.0 onwards. In older versions, this error can be mitigated in one of the following two ways. + +* Move/Rename such objects e.g. for objects of names like `A//B`, use `gcloud storage mv gs://<bucket>/A//* gs://<bucket>/A/`), Or +* Delete such objects e.g. for objects of names like `A//B`, use `gcloud storage rm gs://<bucket>/A//**`. -Find out if you have any object(s) with name/prefix having `//` in it or starting with `/`, in the mounted GCS bucket (use `gsutil ls gs://<bucket>/<path>` to find out). If yes, move/rename such objects to name/prefix not having `//` or starting with `/` (e.g. for object/prefix `A//B`, use `gsutil -m mv -r gs://<bucket>/A//* gs://<bucket>/A/`), or delete it (use `gsutil -m rm -r A//`). Refer [semantics](semantics.md#unsupported-object-names) for more details. +Refer [semantics](semantics.md#unsupported-object-names) for more details. ### Experiencing hang while executing "ls" on a directory containing large number of files/directories. @@ -113,7 +125,7 @@ By default `ls` does listing but sometimes additionally does `stat` for each lis Both these errors are expected and part of GCSFuse standard operating procedure. More details [here](https://github.com/GoogleCloudPlatform/gcsfuse/discussions/2300). -### GCSFuse logs showing errors for StatObject NotFoundError +### GCSFuse logs showing errors for StatObject NotFoundError `StatObject(\"<object_name>") (<time>ms): gcs.NotFoundError: storage: object doesn't exist"`. This is an expected error. Please refer to **NOT_FOUND GetObjectMetadata** section [here](https://github.com/GoogleCloudPlatform/gcsfuse/discussions/2300#discussioncomment-10261838). @@ -135,7 +147,131 @@ It is possible customer is seeing the error "transport endpoint is not connected **Additional troubleshooting steps:** - Try to unmount and mount the mount-point using the command: - `umount -f /<mount point>` && `mount /<mount point>` + `umount -f /<mount point>` && `mount /<mount point>` - Try restarting/rebooting the VM Instance. If it's running on GKE, the issue could be caused by an Out-of-Memory (OOM) error. Consider increasing the memory allocated to the GKE sidecar container. For more info refer [here](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/blob/main/docs/known-issues.md#implications-of-the-sidecar-container-design). + +### GCSFuse crashes with `fatal error: sync: unlock of unlocked mutex` or `Panic: Inode 'a/' cannot have child file ''` + +**Solution:** This happens when the mounting bucket contains an object with suffix `/\n` like, `gs://gcs-bkt/a/\n` +You need to find such objects and replace them with any other valid gcs object names. - [How](https://github.com/GoogleCloudPlatform/gcsfuse/discussions/2894)? + +### Mount fails with 'can't create with 0 workers' when using buffered read + +When using buffered reads (`--enable-buffered-read`) with `--read-global-max-blocks` set to `-1`, GCSFuse versions `v3.3.0` and `v3.4.0` fail to mount with an error similar to this: + +``` +Error: ... failed to create worker pool for buffered read: staticWorkerPool: can't create with 0 workers, priority: 0, normal: 0 +``` + +This is a known issue and is fixed in later versions. + +**Workaround:** If you are using an affected version, avoid setting `--read-global-max-blocks` to `-1`. Instead, set it to a large positive integer (e.g., `2147483647`) to approximate an infinite limit while avoiding the bug. + +### OSError [ErrNo 28] No space left on device + +The Writes in GCSFuse are staged locally before they are uploaded to GCS buckets. It takes up disk space equivalent to the size of the files that are being uploaded concurrently and deleted locally once they are uploaded. During this time, since the disk is used, this error may come up. + +The path can be configured by using the mount flag [--temp-dir](https://cloud.google.com/storage/docs/cloud-storage-fuse/cli-options) to a path which has the disk space if available. By default, it takes the `/tmp` directory of the machine. (sometimes may be limited depending on the machine ). + +Alternatively, from [GCSFuse version 2.9.1](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v2.9.1) onwards, writes can be configured with streaming writes feature ( which doesnt involve staging the file locally ) with the help of `--enable-streaming-writes` flag + +### Permission Denied When Accessing a Mounted File or Directory + +By default, GCSFuse assigns file-mode 0644 and dir-mode 0755 for mounted files and directories. As a result, other users (such as third-party clients or the root user) may not have the necessary permissions to access the mounted file system. To resolve this issue, you can modify the permissions using the following options: + +- **Adjust File and Directory Permissions:** + Use the `--file-mode` and `--dir-mode` flags to set the appropriate file and directory permissions when mounting. +- **Allow Access for Other Users:** + To allow users other than the mounting user to access the bucket, use the `-o allow_other` flag during the mount process. Additionally, for this flag to function, the `user_allow_other` option must be enabled in the `/etc/fuse.conf` file, or the gcsfuse command must be run as the root user. + +**Note:** Be aware that allowing access to other users can introduce potential [security risks](https://github.com/torvalds/linux/blob/a33f32244d8550da8b4a26e277ce07d5c6d158b5/Documentation/filesystems/fuse.txt#L218-L310). Therefore, it should be done with caution. + +- **Set User and Group IDs:** + Use the `--uid` and `--gid` flags to specify the correct user and group IDs for access. + +Please note that GCSFuse does not support using `chmod` or similar commands to manage file access. For more detailed information, refer to the [Permissions and Ownership](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#permissions-and-ownership). + + +### Cloning GitHub repository inside mounted bucket is extremely slow +Ensure your GCS bucket mount configuration does not include `o=sync` or `o=dirsync`. \ +During a Git clone, Git doesn’t just fetch object data, it builds out the entire .git directory structure, including initializing config, refs, hooks, and other internals. As part of this setup: + +- Git repeatedly writes and updates .git/config, especially when setting remotes, branches, and defaults. +- Each update uses Git’s lock-write-rename-delete pattern to ensure consistency. + +While using the mount configuration `o=sync,o=dirsync`, all modifications to the config file incur a network call due to enforced synchronous writes, resulting in +performance bottleneck. \ +**Note** : There is no impact of disabling this mount configuration on the user workflow, since we avoid flushing data to GCS on sync( happens multiple times during the course of a Git clone ) , but only on close(), thus ensuring data persistence. + +### Increased CPU Utilization with File Cache after upgrade to version 2.12.0 +Starting with [version 2.12.0](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v2.12.0), you might observe a slight increase in CPU utilization when the file cache is enabled. This occurs because GCSFuse uses parallel threads to download data to the read cache. While this dramatically improves read performance, it may consume slightly more CPU than in previous versions. + +If this increased CPU usage negatively impacts your workload's performance, you can disable this behavior by setting the `file-cache:enable-parallel-downloads` configuration option to `false`. + +### Potential Stat Consistency Issues on high-performance machines with Default TTL +Starting with [version 3.0.0](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v3.0.0), On high-performance machines - gcsfuse will default to infinite stat cache TTL ([refer](https://cloud.google.com/storage/docs/cloud-storage-fuse/automated-configurations)), potentially causing stale file/directory information if the bucket is modified externally. If strict consistency is needed, manually set a finite TTL (e.g., --stat-cache-ttl 1m) to ensure metadata reflects recent changes. Consult [semantics](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md) doc for more details. + +### User-defined configuration values in YAML are being ignored + +**Issue:** Specific settings defined in your YAML configuration file (such as `metadata-cache.ttl-secs` or `write.global-max-blocks`) do not seem to take effect. + +**Cause:** GCSFuse automatically identifies the machine type it is running on and applies performance optimizations accordingly. In versions `v3.4.0` through `v3.5.6`, these optimizations (and those triggered by `--profile`) incorrectly overrode user-provided values in the configuration file because the logic only checked if the flag was explicitly set via the command line. + +**Solution:** +* **Upgrade:** Update to GCSFuse **v3.6.0**, GKE **1.35.0-gke.1972000** or later. This version ensures that optimizations honor user-defined values provided in configuration files. +* **Workaround:** For affected versions, pass the configuration values as **CLI flags** (e.g., `--metadata-cache-ttl-secs=60`). Values provided via CLI flags take the highest precedence and will correctly override automatic optimizations. + * **GKE:** Pass the flags in the `mountOptions` field or `gke-gcsfuse/mount-options` annotation without the leading dashes (e.g., `metadata-cache-ttl-secs=60`). + +### Writes still using legacy staged writes even though streaming writes are enabled. +If you observe that GCSFuse is still utilizing staged writes despite streaming writes being enabled, several factors could be at play. + +- **Concurrent Streaming Write Limit Reached:** When the number of concurrent streaming writes exceeds the configured limit (`--write-global-max-blocks`), GCSFuse automatically uses legacy staged writes for concurrent file writes above this limit. You will see an warnining log message when this happens, similar to: + > File <var>file_name</var> will use legacy staged writes because concurrent streaming write limit (set by --write-global-max-blocks) has been reached. + + This is not an error, but a fallback mechanism to manage memory usage. If your system has sufficient memory, you can increase the number of allowed concurrent streaming writes by adjusting the `--write-global-max-blocks` flag to prevent this warning. For memory usage refer [write semantics](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#writes) doc for more details. + +- **Unsupported Streaming Write Operations:** Streaming writes only work for sequential writes to new or empty files. GCSFuse will automatically revert to legacy staged writes for the following scenarios: + - **Modifying existing files (non-zero size):** Writing to a file that is not empty will cause that file to use legacy staged writes. You will see an informational log message similar to: + > Existing file <var>file_name</var> of size <var>size</var> bytes (non-zero) will use legacy staged writes. + + - **Performing out-of-order writes:** Streaming writes require data to be written sequentially. If a write occurs at an unexpected offset, GCSFuse will finalize the currently written sequential data and switch to legacy staged writes. You will see an informational log message similar to: + > Out of order write detected. File <var>file_name</var> will now use legacy staged writes. + + - **Reading from a file while writes are in progress:** Performing a read on a file that is being actively written to using streaming writes will finalizes the object on GCS. Subsequent writes to that file will use the legacy staged writes. + + - **Truncating a file downwards while streaming writes are in progress:** If a file is truncated to a smaller size while being written via streaming writes, the object is finalized on GCS, and subsequent writes will use the legacy staged writes. You will see an informational log message similar to: + > Out of order write detected. File <var>file_name</var> will now use legacy staged writes. + +### Issues related to the gcs-fuse-csi-driver +The `gcs-fuse-csi-driver` serves as the orchestration layer that manages the gke-gcsfuse-sidecar which hosts GCSFuse in Google Kubernetes Engine (GKE) environments. This driver is maintained in a separate repository. Consequently, any issues regarding the gcs-fuse-csi-driver should be reported in its dedicated GitHub repository: https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver + +### Errors for unsupported file system operations +This is an expected error for file operations unsupported in GCSFUSE file system. Currently, GCSFuse does not support +the following operations: +- **Fallocate:** Used for pre-allocating disk space for a file so that disk space does not run out before writing. This + is usually not implemented for Cloud based FUSE products as the disk space running out is not a concern there. +- **SetXattr, ListXattr, GetXattr, RemoveXattr:** GCSFuse doesn't support extended-attributes (x-attrs) operations. + Extended attributes provide a way to associate additional metadata or information with files and directories beyond + the standard attributes like file size, modification time, etc. This is not usually implemented for Cloud based Fuse + products. +- **CreateLink:** Creates a hard link (a directory entry that associates a name with a file). GCSFuse doesn't support + hardlinks. +- **BatchForget:** This is a performance optimization for batch-forgetting inodes. When this is unimplemented, + filesystem instead utilizes individual ForgetInode calls. + +### Installation error: The repository does not have a Release file +The full error log would be something like: `Error: The repository 'https://packages.cloud.google.com/apt gcsfuse-<abc> Release' does not have a Release file.` + +This occurs when the gcsfuse package corresponding to OS version returned by `lsb_release` (say x) is not in the [list of supported OS versions](https://cloud.google.com/storage/docs/cloud-storage-fuse/overview#frameworks-os-architectures) . + +**Workaround**: Install GCSFuse for the closest supported OS version (say y), by running `export GCSFUSE_REPO="gcsfuse-y"` and retrying installation. An example of this is in https://github.com/GoogleCloudPlatform/gcsfuse/issues/3779, with x=`trixie` (for debian-13), and y=`bookworm` (for debian-12). + +### fuse: writeMessage: no such file or directory error + +This error indicates that the GCSFuse process attempted to write a response (such as an "OK" status for a pending I/O request) back to the kernel, but the underlying communication channel— the FUSE file descriptor— was already closed. + +The primary cause is typically a concurrent or premature `unmount` syscall that tears down the mount point while active I/O is still being finalized. When the mount is unmounted, the kernel closes the FUSE device file descriptor, causing subsequent attempts by GCSFuse to send responses back to the kernel to fail with "no such file or directory". + +To fix the issue, ensure that unmount operations are only initiated after all application processes have completely finished using the filesystem. diff --git a/go.mod b/go.mod index fd6803c1bd..5be269f7f9 100644 --- a/go.mod +++ b/go.mod @@ -1,22 +1,23 @@ -module github.com/googlecloudplatform/gcsfuse/v2 +module github.com/googlecloudplatform/gcsfuse/v3 -go 1.23.3 +go 1.26.3 require ( - cloud.google.com/go/compute/metadata v0.5.2 - cloud.google.com/go/iam v1.2.2 - cloud.google.com/go/secretmanager v1.14.2 - cloud.google.com/go/storage v1.47.0 - contrib.go.opencensus.io/exporter/ocagent v0.7.0 - contrib.go.opencensus.io/exporter/prometheus v0.4.2 - contrib.go.opencensus.io/exporter/stackdriver v0.13.14 - github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 - github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.25.0 - github.com/fsouza/fake-gcs-server v1.50.2 + cloud.google.com/go/auth v0.20.0 + cloud.google.com/go/auth/oauth2adapt v0.2.8 + cloud.google.com/go/compute/metadata v0.9.0 + cloud.google.com/go/iam v1.11.0 + cloud.google.com/go/profiler v0.6.0 + cloud.google.com/go/secretmanager v1.20.0 + cloud.google.com/go/storage v1.62.2 + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.56.0 + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.32.0 + github.com/fsouza/fake-gcs-server v1.54.0 + github.com/go-viper/mapstructure/v2 v2.5.0 github.com/google/uuid v1.6.0 - github.com/googleapis/gax-go/v2 v2.14.0 + github.com/googleapis/gax-go/v2 v2.22.0 github.com/jacobsa/daemonize v0.0.0-20240917082746-f35568b6c3ec - github.com/jacobsa/fuse v0.0.0-20240607092844-7285af0d05b0 + github.com/jacobsa/fuse v0.0.0-20260302145937-f1ba38d60fdf github.com/jacobsa/oglematchers v0.0.0-20150720000706-141901ea67cd github.com/jacobsa/oglemock v0.0.0-20150831005832-e94d794d06ff github.com/jacobsa/ogletest v0.0.0-20170503003838-80d50a735a11 @@ -24,99 +25,85 @@ require ( github.com/jacobsa/syncutil v0.0.0-20180201203307-228ac8e5a6c3 github.com/jacobsa/timeutil v0.0.0-20170205232429-577e5acbbcf6 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 - github.com/mitchellh/mapstructure v1.5.0 - github.com/prometheus/client_golang v1.20.5 - github.com/spf13/cobra v1.8.1 - github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.19.0 - github.com/stretchr/testify v1.10.0 + github.com/ncruces/go-dns v1.3.3 + github.com/pkg/xattr v0.4.12 + github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_model v0.6.2 + github.com/prometheus/common v0.67.5 + github.com/spf13/cobra v1.10.2 + github.com/spf13/pflag v1.0.10 + github.com/spf13/viper v1.21.0 + github.com/stretchr/testify v1.11.1 go.opencensus.io v0.24.0 - go.opentelemetry.io/contrib/detectors/gcp v1.32.0 - go.opentelemetry.io/otel v1.32.0 - go.opentelemetry.io/otel/exporters/prometheus v0.54.0 - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.32.0 - go.opentelemetry.io/otel/sdk v1.32.0 - go.opentelemetry.io/otel/sdk/metric v1.32.0 - go.opentelemetry.io/otel/trace v1.32.0 - golang.org/x/net v0.31.0 - golang.org/x/oauth2 v0.24.0 - golang.org/x/sync v0.9.0 - golang.org/x/sys v0.27.0 - golang.org/x/text v0.20.0 - golang.org/x/time v0.8.0 - google.golang.org/api v0.209.0 - google.golang.org/grpc v1.68.0 - google.golang.org/protobuf v1.35.2 + go.opentelemetry.io/contrib/detectors/gcp v1.43.0 + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 + go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.68.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 + go.opentelemetry.io/otel v1.43.0 + go.opentelemetry.io/otel/exporters/prometheus v0.65.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0 + go.opentelemetry.io/otel/metric v1.43.0 + go.opentelemetry.io/otel/sdk v1.43.0 + go.opentelemetry.io/otel/sdk/metric v1.43.0 + go.opentelemetry.io/otel/trace v1.43.0 + golang.org/x/net v0.54.0 + golang.org/x/oauth2 v0.36.0 + golang.org/x/sync v0.20.0 + golang.org/x/sys v0.44.0 + golang.org/x/text v0.37.0 + golang.org/x/time v0.15.0 + google.golang.org/api v0.280.0 + google.golang.org/grpc v1.81.1 + google.golang.org/protobuf v1.36.11 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 ) require ( - cel.dev/expr v0.16.1 // indirect - cloud.google.com/go v0.116.0 // indirect - cloud.google.com/go/auth v0.10.2 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.5 // indirect - cloud.google.com/go/longrunning v0.6.2 // indirect - cloud.google.com/go/monitoring v1.21.2 // indirect - cloud.google.com/go/pubsub v1.45.1 // indirect - cloud.google.com/go/trace v1.11.2 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.49.0 // indirect - github.com/aws/aws-sdk-go v1.44.217 // indirect + cel.dev/expr v0.25.1 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/longrunning v0.9.0 // indirect + cloud.google.com/go/monitoring v1.24.3 // indirect + cloud.google.com/go/pubsub/v2 v2.4.0 // indirect + cloud.google.com/go/trace v1.11.7 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.56.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/envoyproxy/go-control-plane v0.13.0 // indirect - github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-kit/log v0.2.1 // indirect - github.com/go-logfmt/logfmt v0.5.1 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/google/renameio/v2 v2.0.0 // indirect - github.com/google/s2a-go v0.1.8 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect + github.com/google/renameio/v2 v2.0.2 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect github.com/gorilla/handlers v1.5.2 // indirect github.com/gorilla/mux v1.8.1 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/klauspost/compress v1.17.11 // indirect - github.com/magiconair/properties v1.8.7 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pelletier/go-toml/v2 v2.2.2 // indirect - github.com/pkg/xattr v0.4.10 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.60.1 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - github.com/prometheus/prometheus v0.35.0 // indirect - github.com/prometheus/statsd_exporter v0.22.7 // indirect - github.com/rogpeppe/go-internal v1.12.0 // indirect - github.com/sagikazarmark/locafero v0.4.0 // indirect - github.com/sagikazarmark/slog-shim v0.1.0 // indirect - github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.11.0 // indirect - github.com/spf13/cast v1.6.0 // indirect - github.com/stretchr/objx v0.5.2 // indirect + github.com/prometheus/otlptranslator v1.0.0 // indirect + github.com/prometheus/procfs v0.20.1 // indirect + github.com/sagikazarmark/locafero v0.12.0 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect + github.com/stretchr/objx v0.5.3 // indirect github.com/subosito/gotenv v1.6.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 // indirect - go.opentelemetry.io/otel/metric v1.32.0 // indirect - go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.29.0 // indirect - golang.org/x/exp v0.0.0-20240530194437-404ba88c7ed0 // indirect - google.golang.org/genproto v0.0.0-20241113202542-65e8d215514f // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241113202542-65e8d215514f // indirect - google.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a // indirect - gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.yaml.in/yaml/v2 v2.4.4 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.51.0 // indirect + google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260511170946-3700d4141b60 // indirect ) diff --git a/go.sum b/go.sum index 723579c8b4..e6897f8e23 100644 --- a/go.sum +++ b/go.sum @@ -1,756 +1,141 @@ -bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= -bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM= -cel.dev/expr v0.16.1 h1:NR0+oFYzR1CqLFhTAqg3ql59G9VfN8fKq1TCHJ6gq1g= -cel.dev/expr v0.16.1/go.mod h1:AsGA5zb3WruAEQeQng1RZdGEXmBj0jvMWh6l5SnNuC8= +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= -cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= -cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= -cloud.google.com/go/auth v0.10.2 h1:oKF7rgBfSHdp/kuhXtqU/tNDr0mZqhYbEh+6SiqzkKo= -cloud.google.com/go/auth v0.10.2/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI= -cloud.google.com/go/auth/oauth2adapt v0.2.5 h1:2p29+dePqsCHPP1bqDJcKj4qxRyYCcbzKpFyKGt3MTk= -cloud.google.com/go/auth/oauth2adapt v0.2.5/go.mod h1:AlmsELtlEBnaNTL7jCj8VQFLy6mbZv0s4Q7NGBeQ5E8= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= -cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= -cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo= -cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/iam v1.2.2 h1:ozUSofHUGf/F4tCNy/mu9tHLTaxZFLOUiKzjcgWHGIA= -cloud.google.com/go/iam v1.2.2/go.mod h1:0Ys8ccaZHdI1dEUilwzqng/6ps2YB6vRsjIe00/+6JY= -cloud.google.com/go/kms v1.20.1 h1:og29Wv59uf2FVaZlesaiDAqHFzHaoUyHI3HYp9VUHVg= -cloud.google.com/go/kms v1.20.1/go.mod h1:LywpNiVCvzYNJWS9JUcGJSVTNSwPwi0vBAotzDqn2nc= -cloud.google.com/go/logging v1.12.0 h1:ex1igYcGFd4S/RZWOCU51StlIEuey5bjqwH9ZYjHibk= -cloud.google.com/go/logging v1.12.0/go.mod h1:wwYBt5HlYP1InnrtYI0wtwttpVU1rifnMT7RejksUAM= -cloud.google.com/go/longrunning v0.6.2 h1:xjDfh1pQcWPEvnfjZmwjKQEcHnpz6lHjfy7Fo0MK+hc= -cloud.google.com/go/longrunning v0.6.2/go.mod h1:k/vIs83RN4bE3YCswdXC5PFfWVILjm3hpEUlSko4PiI= -cloud.google.com/go/monitoring v1.21.2 h1:FChwVtClH19E7pJ+e0xUhJPGksctZNVOk2UhMmblmdU= -cloud.google.com/go/monitoring v1.21.2/go.mod h1:hS3pXvaG8KgWTSz+dAdyzPrGUYmi2Q+WFX8g2hqVEZU= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/pubsub v1.45.1 h1:ZC/UzYcrmK12THWn1P72z+Pnp2vu/zCZRXyhAfP1hJY= -cloud.google.com/go/pubsub v1.45.1/go.mod h1:3bn7fTmzZFwaUjllitv1WlsNMkqBgGUb3UdMhI54eCc= -cloud.google.com/go/secretmanager v1.14.2 h1:2XscWCfy//l/qF96YE18/oUaNJynAx749Jg3u0CjQr8= -cloud.google.com/go/secretmanager v1.14.2/go.mod h1:Q18wAPMM6RXLC/zVpWTlqq2IBSbbm7pKBlM3lCKsmjw= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.47.0 h1:ajqgt30fnOMmLfWfu1PWcb+V9Dxz6n+9WKjdNg5R4HM= -cloud.google.com/go/storage v1.47.0/go.mod h1:Ks0vP374w0PW6jOUameJbapbQKXqkjGd/OJRp2fb9IQ= -cloud.google.com/go/trace v1.11.2 h1:4ZmaBdL8Ng/ajrgKqY5jfvzqMXbrDcBsUGXOT9aqTtI= -cloud.google.com/go/trace v1.11.2/go.mod h1:bn7OwXd4pd5rFuAnTrzBuoZ4ax2XQeG3qNgYmfCy0Io= -contrib.go.opencensus.io/exporter/ocagent v0.7.0 h1:BEfdCTXfMV30tLZD8c9n64V/tIZX5+9sXiuFLnrr1k8= -contrib.go.opencensus.io/exporter/ocagent v0.7.0/go.mod h1:IshRmMJBhDfFj5Y67nVhMYTTIze91RUeT73ipWKs/GY= -contrib.go.opencensus.io/exporter/prometheus v0.4.2 h1:sqfsYl5GIY/L570iT+l93ehxaWJs2/OwXtiWwew3oAg= -contrib.go.opencensus.io/exporter/prometheus v0.4.2/go.mod h1:dvEHbiKmgvbr5pjaF9fpw1KeYcjrnC1J8B+JKjsZyRQ= -contrib.go.opencensus.io/exporter/stackdriver v0.13.14 h1:zBakwHardp9Jcb8sQHcHpXy/0+JIb1M8KjigCJzx7+4= -contrib.go.opencensus.io/exporter/stackdriver v0.13.14/go.mod h1:5pSSGY0Bhuk7waTHuDf4aQ8D2DrhgETRo9fy6k3Xlzc= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg= -github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v63.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= -github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest v0.11.25/go.mod h1:7l8ybrIdUmGqZMTD0sRtAr8NvbHjfofbf8RSP2q7w7U= -github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= -github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= -github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= -github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= -github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= +cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.11.0 h1:KieQ9Pb+LLPak1O3Rv3GgCxhnmkYf7Xyh0P5HfF1jFM= +cloud.google.com/go/iam v1.11.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= +cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA= +cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak= +cloud.google.com/go/longrunning v0.9.0 h1:0EzbDEGsAvOZNbqXopgniY0w0a1phvu5IdUFq8grmqY= +cloud.google.com/go/longrunning v0.9.0/go.mod h1:pkTz846W7bF4o2SzdWJ40Hu0Re+UoNT6Q5t+igIcb8E= +cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= +cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= +cloud.google.com/go/profiler v0.6.0 h1:Vwxqgnp8CQwBNcKjO8luwLCh7qblEkcZCtMUvhU9Yik= +cloud.google.com/go/profiler v0.6.0/go.mod h1:cJV7Qfj0o9PAC7q/xQTkM6qn2FO9So3TFk4P5O5yLis= +cloud.google.com/go/pubsub/v2 v2.4.0 h1:oMKNiBQpXImRWnHYla9uSU66ZzByZwBSCJOEs/pTKVg= +cloud.google.com/go/pubsub/v2 v2.4.0/go.mod h1:2lS/XQKq5qtOMs6kHBK+WX1ytUC36kLl2ig3zqsGUx8= +cloud.google.com/go/secretmanager v1.20.0 h1:GjE3NoyFXo7ipRPy26PMmg4oRX1Ra8fswH45r16rWV0= +cloud.google.com/go/secretmanager v1.20.0/go.mod h1:9OmSuOeiiUicANglrbdKWSnT3gYkRcXuUQDk7dDW0zU= +cloud.google.com/go/storage v1.62.2 h1:WgR4U9n7bIzXkkVnwPKKE8bkaKUNsHG+0MAAlh9DGU4= +cloud.google.com/go/storage v1.62.2/go.mod h1:cpYz/kRVZ+UQAF1uHeea10/9ewcRbxGoGNKsS9daSXA= +cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U= +cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 h1:3c8yed4lgqTt+oTQ+JNMDo+F4xprBf+O/il4ZC0nRLw= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0/go.mod h1:obipzmGjfSjam60XLwGfqUkJsfiheAl+TUjG+4yzyPM= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 h1:UQ0AhxogsIRZDkElkblfnwjc3IaltCm2HUMvezQaL7s= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1/go.mod h1:jyqM3eLpJ3IbIFDTKVz2rF9T/xWGW0rIriGwnz8l9Tk= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.25.0 h1:4PoDbd/9/06IpwLGxSfvfNoEr9urvfkrN6mmJangGCg= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.25.0/go.mod h1:EycllQ1gupHbjqbcmfCr/H6FKSGSmEUONJ2ivb86qeY= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.49.0 h1:jJKWl98inONJAr/IZrdFQUWcwUO95DLY1XMD1ZIut+g= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.49.0/go.mod h1:l2fIqmwB+FKSfvn3bAD/0i+AXAxhIZjTK2svT/mgUXs= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.49.0 h1:GYUJLfvd++4DMuMhCFLgLXvFwofIxh/qOwoGuS/LTew= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.49.0/go.mod h1:wRbFgBQUVm1YXrvWKofAEmq9HNJTDphbAaJSSX01KUI= -github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= -github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= -github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= -github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= -github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= -github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= -github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= -github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= -github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= -github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= -github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= -github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= -github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= -github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= -github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= -github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.3.3/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= -github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.43.11/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go v1.44.217 h1:FcWC56MRl+k756aH3qeMQTylSdeJ58WN0iFz3fkyRz0= -github.com/aws/aws-sdk-go v1.44.217/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= -github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 h1:rIkQfkCOVKc1OiRCNcSDD8ml5RJlZbH/Xsq7lbpynwc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0/go.mod h1:RD2SsorTmYhF6HkTmDw7KmPYQk8OBYwTkuasChwv7R4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.56.0 h1:O2sXMyJh8b7devAGdE+163xtRurt0RVpB6DIzX5vGfg= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.56.0/go.mod h1:hEpiGU18xf70qb3jbTcIggWAiEfX/cOIVc2OTe4OegA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.32.0 h1:ftVmySBwuOJafpEJnnZvco+iV3p6Lokgu2sd89/qY7M= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.32.0/go.mod h1:nikqFGPI5OGwEsdxXzd3f58sB3tzkjqpqwYOV/S1rmo= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.56.0 h1:ZIT85vKP7LBS84XJ0WdJ3dPOX3iz4j3c0+lpajGQMyo= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.56.0/go.mod h1:rqP9UEhOXv9WhQ7Gjz+G5y/pf8+BJZW5/Ts0AhE0PwE= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.56.0 h1:0YP0+/ixwu+Uqeu/FGiBZNQ19huiUxxiPXIc9WsLKuQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.56.0/go.mod h1:6ZZMQhZKDvUvkJw2rc+oDP90tMMzuU/J+5HG1ZmPOmE= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= -github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= -github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= -github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= -github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= -github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= -github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= -github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= -github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= -github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= -github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= -github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= -github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= -github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= -github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= -github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 h1:QVw89YDxXxEe+l8gU8ETbOasdwEV+avkR75ZzsVV9WI= -github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= -github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= -github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= -github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= -github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= -github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E= -github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= -github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= -github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= -github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= -github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= -github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= -github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= -github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= -github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= -github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= -github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= -github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= -github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= -github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= -github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= -github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= -github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= -github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= -github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= -github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s= -github.com/containerd/containerd v1.6.1/go.mod h1:1nJz5xCZPusx6jJU8Frfct988y0NpumIq9ODB0kLtoE= -github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= -github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= -github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= -github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= -github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk= -github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= -github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= -github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= -github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= -github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= -github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= -github.com/containerd/go-cni v1.1.0/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= -github.com/containerd/go-cni v1.1.3/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= -github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= -github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= -github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= -github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= -github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= -github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= -github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= -github.com/containerd/imgcrypt v1.1.3/go.mod h1:/TPA1GIDXMzbj01yd8pIbQiLdQxed5ue1wb8bP7PQu4= -github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= -github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= -github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= -github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM= -github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= -github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= -github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= -github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= -github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= -github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= -github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= -github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= -github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= -github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= -github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw= -github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y= -github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= -github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= -github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= -github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y= -github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= -github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= -github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNBDZcxSOplJT5ico8/FLE= -github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= -github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= -github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= -github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= -github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= -github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= -github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= -github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= -github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= -github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= -github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= -github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/digitalocean/godo v1.78.0/go.mod h1:GBmu8MkjZmNARE7IXRPmkbbnocNN8+uBm0xbEVw2LCs= -github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= -github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v20.10.14+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= -github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= -github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= -github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= -github.com/envoyproxy/go-control-plane v0.13.0 h1:HzkeUz1Knt+3bK+8LG1bxOO/jzWZmdxpwC51i202les= -github.com/envoyproxy/go-control-plane v0.13.0/go.mod h1:GRaKG3dwvFoTg4nj7aXdZnvMg4d7nvT/wl9WgVXn3Q8= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= -github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM= -github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4= -github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/fsouza/fake-gcs-server v1.50.2 h1:ulrS1pavCOCbMZfN5ZPgBRMFWclON9xDsuLBniXtQoE= -github.com/fsouza/fake-gcs-server v1.50.2/go.mod h1:VU6Zgei4647KuT4XER8WHv5Hcj2NIySndyG8gfvwckA= -github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= -github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= -github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= -github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fsouza/fake-gcs-server v1.54.0 h1:DGO4EkFVbtP/A5Ha+CAHHx+Xa6O6LeskMB4hQ1wBE48= +github.com/fsouza/fake-gcs-server v1.54.0/go.mod h1:ryXYE4debQs8GjOxwaOAwFRwM4Cvs6S+NKPPgdVJe6g= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= -github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= -github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g= -github.com/go-openapi/runtime v0.23.1/go.mod h1:AKurw9fNre+h3ELZfk6ILsfvPN+bvvlaU/M9q/r9hpk= -github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= -github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg= -github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= -github.com/go-openapi/strfmt v0.21.2/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= -github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/validate v0.21.0/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= -github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= -github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= -github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= -github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= -github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= -github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= -github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= -github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= -github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= -github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= -github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= -github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= -github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= -github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= -github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= -github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= -github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= -github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= -github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= -github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= -github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= -github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= -github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY= -github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U= +github.com/golang/mock v1.7.0-rc.1/go.mod h1:s42URUywIqd+OcERslBJvOjepvNymP31m3q8d/GkuRs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20220318212150-b2ab0324ddda/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg= -github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4= -github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= -github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg= +github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/renameio/v2 v2.0.2 h1:qKZs+tfn+arruZZhQ7TKC/ergJunuJicWS6gLDt/dGw= +github.com/google/renameio/v2 v2.0.2/go.mod h1:OX+G6WHHpHq3NVj7cAOleLOwJfcQ1s3uUJQCrr78SWo= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= -github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/googleapis/gax-go/v2 v2.14.0 h1:f+jMrjBPl+DL9nI4IQzLUxMq7XrAqFYB7hBPqMNIe8o= -github.com/googleapis/gax-go/v2 v2.14.0/go.mod h1:lhBCnjdLrWRaPvLWhmc8IS24m9mr07qSYnHncrgo+zk= -github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= -github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gophercloud/gophercloud v0.24.0/go.mod h1:Q8fZtyi5zZxPS/j9aj3sSxtvj41AdQMDwyo1myduD5c= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/googleapis/enterprise-certificate-proxy v0.3.15 h1:xolVQTEXusUcAA5UgtyRLjelpFFHWlPQ4XfWGc7MBas= +github.com/googleapis/enterprise-certificate-proxy v0.3.15/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= +github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4= +github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE= github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w= -github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grafana/regexp v0.0.0-20220304095617-2e8d9baf4ac2/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.14.6/go.mod h1:zdiPV4Yse/1gnckTHtghG4GkDEdKCRJduHpTxT3/jcw= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 h1:gDLXvp5S9izjldquuoAhDzccbskOL6tDC5jMSyx3zxE= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2/go.mod h1:7pdNwVWBBHGiCxa9lAszqCJMbfTISJ7oMftp8+UGV08= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/api v1.12.0/go.mod h1:6pVBMo0ebnYdt2S3H87XhekM/HHrUoTD2XXb/VrZVy0= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= -github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v0.12.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.2.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= -github.com/hetznercloud/hcloud-go v1.33.1/go.mod h1:XX/TQub3ge0yWR2yHWmnDVIrB+MQbda1pHxkUmDlUME= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ= -github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= -github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw= github.com/jacobsa/daemonize v0.0.0-20240917082746-f35568b6c3ec h1:xsRGrfdnjvJtEMD2ouh8gOGIeDF9LrgXjo+9Q69RVzI= github.com/jacobsa/daemonize v0.0.0-20240917082746-f35568b6c3ec/go.mod h1:Ip4fOwzCrnDVuluHBd7FXIMb7SHOKfkt9/UDrYSZvqI= -github.com/jacobsa/fuse v0.0.0-20240607092844-7285af0d05b0 h1:IWVMQZZvWN+9FeRwWnZAINYNrsr3yyCWI2BcddQBDvk= -github.com/jacobsa/fuse v0.0.0-20240607092844-7285af0d05b0/go.mod h1:JYi9iIxdYNgxmMgLwtSHO/hmVnP2kfX1oc+mtx+XWLA= +github.com/jacobsa/fuse v0.0.0-20260302145937-f1ba38d60fdf h1:1FpPcJSf6jjJGvIltaLwJCpbFCMI9bVUCAAxUSxqWnY= +github.com/jacobsa/fuse v0.0.0-20260302145937-f1ba38d60fdf/go.mod h1:fcpw1yk/suvFhB8rT9P+pst+NLboWsBLky9csooKjPc= github.com/jacobsa/oglematchers v0.0.0-20150720000706-141901ea67cd h1:9GCSedGjMcLZCrusBZuo4tyKLpKUPenUUqi34AkuFmA= github.com/jacobsa/oglematchers v0.0.0-20150720000706-141901ea67cd/go.mod h1:TlmyIZDpGmwRoTWiakdr+HA1Tukze6C6XbRVidYq02M= github.com/jacobsa/oglemock v0.0.0-20150831005832-e94d794d06ff h1:2xRHTvkpJ5zJmglXLRqHiZQNjUoOkhUyhTAhEQvPAWw= @@ -763,1115 +148,187 @@ github.com/jacobsa/syncutil v0.0.0-20180201203307-228ac8e5a6c3 h1:+gHfvQxomE6fI4 github.com/jacobsa/syncutil v0.0.0-20180201203307-228ac8e5a6c3/go.mod h1:mPvulh9VKXvo+yOlrD4VYOOYuLdZJ36wa/5QIrtXvWs= github.com/jacobsa/timeutil v0.0.0-20170205232429-577e5acbbcf6 h1:XKHJmHcgU9glxk3eLPiRZT5VFSHJitVTnMj/EgIoXC4= github.com/jacobsa/timeutil v0.0.0-20170205232429-577e5acbbcf6/go.mod h1:JEWKD6V8xETMW+DEv+IQVz++f8Cn8O/X0HPeDY3qNis= -github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= -github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= -github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/kolo/xmlrpc v0.0.0-20201022064351-38db28db192b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/linode/linodego v1.4.0/go.mod h1:PVsRxSlOiJyvG4/scTszpmZDTdgS+to3X6eS8pRrWI8= -github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= -github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= -github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= -github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= -github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= -github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= -github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.78 h1:LqW2zy52fxnI4gg8C2oZviTaKHcBV36scS+RzJnxUFs= -github.com/minio/minio-go/v7 v7.0.78/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0= -github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= -github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= -github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= -github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= -github.com/moby/sys/signal v0.6.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= -github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= -github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs= -github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= -github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= -github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0= +github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= -github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= -github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= -github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.2-0.20211117181255-693428a734f5/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= -github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= -github.com/opencontainers/runc v1.1.0/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= -github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= -github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= -github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= -github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= -github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= -github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= -github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA= -github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= +github.com/ncruces/go-dns v1.3.3 h1:59OV7XoJrTCoUMZjWRVs4GOjtntMTZqiQ5Mn+BT13hk= +github.com/ncruces/go-dns v1.3.3/go.mod h1:tuzixNY8PY/M7yUzcvRbUaeLs3ifIdydpi5H2bfRU+s= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pkg/xattr v0.4.12 h1:rRTkSyFNTRElv6pkA3zpjHpQ90p/OdHQC1GmGh1aTjM= +github.com/pkg/xattr v0.4.12/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/prometheus/alertmanager v0.24.0/go.mod h1:r6fy/D7FRuZh5YbnX6J3MBY0eI4Pb5yPYS7/bPSXXqI= -github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= -github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= -github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= -github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.34.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= -github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc= -github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= -github.com/prometheus/common/assets v0.1.0/go.mod h1:D17UVUE12bHbim7HzwUvtqm6gwBEaDQ0F+hIGbFbccI= -github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI= -github.com/prometheus/exporter-toolkit v0.7.1/go.mod h1:ZUBIj498ePooX9t/2xtDjeQYwvRpiPP2lh5u4iblj2g= -github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/prometheus/prometheus v0.35.0 h1:N93oX6BrJ2iP3UuE2Uz4Lt+5BkUpaFer3L9CbADzesc= -github.com/prometheus/prometheus v0.35.0/go.mod h1:7HaLx5kEPKJ0GDgbODG0fZgXbQ8K/XjZNJXQmbmgQlY= -github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= -github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= +github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= +github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEoIwkU+A6qos= +github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM= +github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= +github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= -github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= -github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= -github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= -github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= -github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= -github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= -github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= -github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= -github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= -github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= -github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= -github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= -github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= -github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= -github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= -github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4= +github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= +github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= +github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= -github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= -github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= -github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= -github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= -github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= -go.einride.tech/aip v0.68.0 h1:4seM66oLzTpz50u4K1zlJyOXQ3tCzcJN7I22tKkjipw= -go.einride.tech/aip v0.68.0/go.mod h1:7y9FF8VtPWqpxuAxl0KQWqaULxW4zFIesD6zF5RIHHg= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= -go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= -go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= -go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= -go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= -go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= -go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= -go.mongodb.org/mongo-driver v1.8.3/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= -go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +go.einride.tech/aip v0.79.0 h1:19zdPlZzlUvxOA8syAFw4LkdJdXepzyTl6gt9XEeqdU= +go.einride.tech/aip v0.79.0/go.mod h1:E8+wdTApA70odnpFzJgsGogHozC2JCIhFJBKPr8bVig= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= -go.opentelemetry.io/contrib/detectors/gcp v1.32.0 h1:P78qWqkLSShicHmAzfECaTgvslqHxblNE9j62Ws1NK8= -go.opentelemetry.io/contrib/detectors/gcp v1.32.0/go.mod h1:TVqo0Sda4Cv8gCIixd7LuLwW4EylumVWfhjZJjDD4DU= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0/go.mod h1:vEhqr0m4eTc+DWxfsXoXue2GBgV2uUwVznkGIHW/e5w= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.31.0/go.mod h1:PFmBsWbldL1kiWZk9+0LBZz2brhByaGsvp6pRICMlPE= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 h1:ZIg3ZT/aQ7AfKqdwp7ECpOK6vHqquXXuyTjIO8ZdmPs= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0/go.mod h1:DQAwmETtZV00skUwgD6+0U89g80NKsJE3DCKeLLPQMI= -go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= -go.opentelemetry.io/otel v1.6.0/go.mod h1:bfJD2DZVw0LBxghOTlgnlI0CV3hLDu9XF/QKOUXMTQQ= -go.opentelemetry.io/otel v1.6.1/go.mod h1:blzUabWHkX6LJewxvadmzafgh/wnvBSDBdOuwkAtrWQ= -go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= -go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= -go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.6.1/go.mod h1:NEu79Xo32iVb+0gVNV8PMd7GoWqnyDXRlj04yFjqz40= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.6.1/go.mod h1:YJ/JbY5ag/tSQFXzH3mtDmHqzF3aFn3DI/aB1n7pt4w= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1:keUU7UfnwWTWpJ+FWnyqmogPa82nuU5VUANFq49hlMY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.6.1/go.mod h1:UJJXJj0rltNIemDMwkOJyggsvyMG9QHfJeFH0HS5JjM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.6.1/go.mod h1:DAKwdo06hFLc0U88O10x4xnb5sc7dDRDqRuiN+io8JE= -go.opentelemetry.io/otel/exporters/prometheus v0.54.0 h1:rFwzp68QMgtzu9PgP3jm9XaMICI6TsofWWPcBDKwlsU= -go.opentelemetry.io/otel/exporters/prometheus v0.54.0/go.mod h1:QyjcV9qDP6VeK5qPyKETvNjmaaEc7+gqjh4SS0ZYzDU= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0 h1:WDdP9acbMYjbKIyJUhTvtzj601sVJOqgWdUxSdR/Ysc= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.32.0 h1:cC2yDI3IQd0Udsux7Qmq8ToKAx1XCilTQECZ0KDZyTw= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.32.0/go.mod h1:2PD5Ex6z8CFzDbTdOlwyNIUywRr1DN0ospafJM1wJ+s= -go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/metric v0.28.0/go.mod h1:TrzsfQAmQaB1PDcdhBauLMk7nyyg9hm+GoQq/ekE9Iw= -go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= -go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= -go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= -go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs= -go.opentelemetry.io/otel/sdk v1.6.1/go.mod h1:IVYrddmFZ+eJqu2k38qD3WezFR2pymCzm8tdxyh3R4E= -go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4= -go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU= -go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= -go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= -go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= -go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= -go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= -go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= -go.opentelemetry.io/otel/trace v1.6.0/go.mod h1:qs7BrU5cZ8dXQHBGxHMOxwME/27YH2qEp4/+tZLLwJE= -go.opentelemetry.io/otel/trace v1.6.1/go.mod h1:RkFRM1m0puWIq10oxImnGEduNBzxiN7TXluRBtE+5j0= -go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= -go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ= -go.opentelemetry.io/proto/otlp v0.12.1/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmfElKI4d/Myu2OxDFU= +go.opentelemetry.io/contrib/detectors/gcp v1.43.0/go.mod h1:RyaZMFY7yi1kAs45S6mbFGz8O8rqB0dTY14uzvG4LCs= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 h1:0Qx7VGBacMm9ZENQ7TnNObTYI4ShC+lHI16seduaxZo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0/go.mod h1:Sje3i3MjSPKTSPvVWCaL8ugBzJwik3u4smCjUeuupqg= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.68.0 h1:cuXaPAfIoJKsYjBjPSb2nKZEmgM43zVr25l37IxhKME= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.68.0/go.mod h1:BuzhPofpCzlDi/Q/Xjg54M4/3oWqqyDe2Zeq7A2I0QE= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/exporters/prometheus v0.65.0 h1:jOveH/b4lU9HT7y+Gfamf18BqlOuz2PWEvs8yM7Q6XE= +go.opentelemetry.io/otel/exporters/prometheus v0.65.0/go.mod h1:i1P8pcumauPtUI4YNopea1dhzEMuEqWP1xoUZDylLHo= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0 h1:TC+BewnDpeiAmcscXbGMfxkO+mwYUwE/VySwvw88PfA= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0/go.mod h1:J/ZyF4vfPwsSr9xJSPyQ4LqtcTPULFR64KwTikGLe+A= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0 h1:mS47AX77OtFfKG4vtp+84kuGSFZHTyxtXIN269vChY0= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0/go.mod h1:PJnsC41lAGncJlPUniSwM81gc80GkgWJWr3cu2nKEtU= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= -golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240530194437-404ba88c7ed0 h1:Mi0bCswbz+9cXmwFAdxoo5GPFMKONUpua6iUdtQS7lk= -golang.org/x/exp v0.0.0-20240530194437-404ba88c7ed0/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= -golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= -golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= +golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= -golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= -golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= -golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= -golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.25.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= -google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= -google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= -google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= -google.golang.org/api v0.209.0 h1:Ja2OXNlyRlWCWu8o+GgI4yUn/wz9h/5ZfFbKz+dQX+w= -google.golang.org/api v0.209.0/go.mod h1:I53S168Yr/PNDNMi5yPnDc0/LGRZO6o7PoEbl/HY3CM= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.280.0 h1:F4OfEHZhZh6a7uTufJAXXVd/2TQ8EjM4vZH+jX/vFYk= +google.golang.org/api v0.280.0/go.mod h1:oGKmPZRDoD3vdkf6MA7F4VNkR1rxCiuaPSkhsf3EolU= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= -google.golang.org/genproto v0.0.0-20241113202542-65e8d215514f h1:zDoHYmMzMacIdjNe+P2XiTmPsLawi/pCbSPfxt6lTfw= -google.golang.org/genproto v0.0.0-20241113202542-65e8d215514f/go.mod h1:Q5m6g8b5KaFFzsQFIGdJkSJDGeJiybVenoYFMMa3ohI= -google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 h1:M0KvPgPmDZHPlbRbaNU1APr28TvwvvdUPlSv7PUvy8g= -google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28/go.mod h1:dguCy7UOdZhTvLzDyt15+rOrawrpM4q7DD9dQ1P11P4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241113202542-65e8d215514f h1:C1QccEa9kUwvMgEUORqQD9S17QesQijxjZ84sO82mfo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241113202542-65e8d215514f/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= -google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260511170946-3700d4141b60 h1:seT2EwLWM78plQ7wcDfuWBc/4FAEAXDDiaSol4ku4qo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260511170946-3700d4141b60/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.68.0 h1:aHQeeJbo8zAkAa3pRzrVjZlbz6uSfeOXlJNQM0RAbz0= -google.golang.org/grpc v1.68.0/go.mod h1:fmSPC5AsjSBCK54MyHRx48kpOti1/jRfOlwEWywNjWA= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= -google.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a h1:UIpYSuWdWHSzjwcAFRLjKcPXFZVVLXGEM23W+NWqipw= -google.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a/go.mod h1:9i1T9n4ZinTUZGgzENMi8MDDgbGC5mqTS75JAv6xN3A= +google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1880,130 +337,16 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= -google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/telebot.v3 v3.0.0/go.mod h1:7rExV8/0mDDNu9epSrDm/8j22KLaActH1Tbee6YjzWg= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= -k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= -k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= -k8s.io/api v0.22.5/go.mod h1:mEhXyLaSD1qTOf40rRiKXkc+2iCem09rWLlFwhCEiAs= -k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8= -k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= -k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/apimachinery v0.22.5/go.mod h1:xziclGKwuuJ2RM5/rSFQSYAj0zdbci3DH8kj+WvyN0U= -k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= -k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= -k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= -k8s.io/apiserver v0.22.5/go.mod h1:s2WbtgZAkTKt679sYtSudEQrTGWUSQAPe6MupLnlmaQ= -k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= -k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= -k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= -k8s.io/client-go v0.22.5/go.mod h1:cs6yf/61q2T1SdQL5Rdcjg9J1ElXSwbjSrW2vFImM4Y= -k8s.io/client-go v0.23.5/go.mod h1:flkeinTO1CirYgzMPRWxUCnV0G4Fbu2vLhYCObnt/r4= -k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= -k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= -k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= -k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= -k8s.io/component-base v0.22.5/go.mod h1:VK3I+TjuF9eaa+Ln67dKxhGar5ynVbwnGrUiNF4MqCI= -k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= -k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= -k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= -k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= -k8s.io/cri-api v0.23.1/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= -sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/internal/auth/auth.go b/internal/auth/auth.go index d124dfce9e..1445652413 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -24,7 +24,7 @@ import ( storagev1 "google.golang.org/api/storage/v1" ) -const universeDomainDefault = "googleapis.com" +const UniverseDomainDefault = "googleapis.com" func getUniverseDomain(ctx context.Context, contents []byte, scope string) (string, error) { creds, err := google.CredentialsFromJSON(ctx, contents, scope) @@ -70,7 +70,7 @@ func newTokenSourceFromPath(ctx context.Context, path string, scope string) (oau // For non-GDU universe domains, token exchange is impossible and services // must support self-signed JWTs with scopes. // Override the token source to use self-signed JWT. - if domain != universeDomainDefault { + if domain != UniverseDomainDefault { // Create self signed JWT access token. ts, err = google.JWTAccessTokenSourceWithScope(contents, scope) if err != nil { diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go index 08dbc524a8..a9d94a2f6b 100644 --- a/internal/auth/auth_test.go +++ b/internal/auth/auth_test.go @@ -49,7 +49,7 @@ func (t *AuthTest) TestGetUniverseDomainForGoogle() { domain, err := getUniverseDomain(context.Background(), contents, storagev1.DevstorageFullControlScope) assert.NoError(t.T(), err) - assert.Equal(t.T(), universeDomainDefault, domain) + assert.Equal(t.T(), UniverseDomainDefault, domain) } func (t *AuthTest) TestGetUniverseDomainForTPC() { diff --git a/internal/auth/credentials.go b/internal/auth/credentials.go new file mode 100644 index 0000000000..504bde6a33 --- /dev/null +++ b/internal/auth/credentials.go @@ -0,0 +1,62 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package auth + +import ( + "fmt" + + "cloud.google.com/go/auth" + "cloud.google.com/go/auth/credentials" + "cloud.google.com/go/storage" +) + +const scope = storage.ScopeFullControl + +// getCredentials is a private helper that takes a custom DetectDefault function. +func getCredentials(keyFile string, detectCredential func(*credentials.DetectOptions) (*auth.Credentials, error)) (*auth.Credentials, error) { + opts := &credentials.DetectOptions{ + CredentialsFile: keyFile, + Scopes: []string{scope}, + } + + creds, err := detectCredential(opts) + if err != nil { + return nil, fmt.Errorf("failed to detect credentials: %w", err) + } + + return creds, nil +} + +// GetCredentials detects default Google Cloud credentials. +// +// It prioritizes a service account key file if `keyFile` is provided. If `keyFile` is +// empty, it attempts to detect Application Default Credentials (ADC) and checks +// the metadata server for credentials. +// +// The function requests storage.ScopeFullControl to ensure the most comprehensive +// permissions for GCS. This allows subsequent operations using these credentials to have full read, +// write, and administrative control over GCS resources. +// +// Args: +// +// keyFile: Path to a service account key file. Pass an empty string to use ADC. +// +// Returns: +// +// *auth.Credentials: Discovered authentication credentials. +// error: An error if credential detection fails. +func GetCredentials(keyFile string) (*auth.Credentials, error) { + return getCredentials(keyFile, credentials.DetectDefault) +} diff --git a/internal/auth/credentials_test.go b/internal/auth/credentials_test.go new file mode 100644 index 0000000000..d60ff514a7 --- /dev/null +++ b/internal/auth/credentials_test.go @@ -0,0 +1,96 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package auth + +import ( + "fmt" + "testing" + + "cloud.google.com/go/auth" + "cloud.google.com/go/auth/credentials" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +//////////////////////////////////////////////////////////////////////// +// Mock +//////////////////////////////////////////////////////////////////////// + +type MockDetectCredentials struct { + mock.Mock +} + +func (m *MockDetectCredentials) DetectDefault(opts *credentials.DetectOptions) (*auth.Credentials, error) { + args := m.Called(opts) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(*auth.Credentials), args.Error(1) +} + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func Test_getCredentials_Success(t *testing.T) { + tests := []struct { + name string + keyFile string + }{ + { + name: "valid key file", + keyFile: "/path/to/key.json", + }, + { + name: "empty key file", + keyFile: "", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + mockDetector := new(MockDetectCredentials) + opts := &credentials.DetectOptions{ + CredentialsFile: tc.keyFile, + Scopes: []string{scope}, + } + mockDetector.On("DetectDefault", opts).Return(&auth.Credentials{}, nil).Once() + + creds, err := getCredentials(tc.keyFile, mockDetector.DetectDefault) + + assert.NoError(t, err) + assert.NotNil(t, creds) + mockDetector.AssertExpectations(t) + }) + } +} + +func Test_getCredentials_Error(t *testing.T) { + mockDetector := new(MockDetectCredentials) + keyFile := "/path/to/key.json" + expectedErr := fmt.Errorf("simulated detection error") + opts := &credentials.DetectOptions{ + CredentialsFile: keyFile, + Scopes: []string{scope}, + } + mockDetector.On("DetectDefault", opts).Return(nil, expectedErr).Once() + + creds, err := getCredentials(keyFile, mockDetector.DetectDefault) + + assert.Error(t, err) + assert.Nil(t, creds) + assert.ErrorIs(t, err, expectedErr) + mockDetector.AssertExpectations(t) +} diff --git a/internal/auth/proxy.go b/internal/auth/token_source.go similarity index 86% rename from internal/auth/proxy.go rename to internal/auth/token_source.go index 9818bad28e..837c89d495 100644 --- a/internal/auth/proxy.go +++ b/internal/auth/token_source.go @@ -36,7 +36,7 @@ func newProxyTokenSource( u, err := url.Parse(endpoint) if err != nil { err = fmt.Errorf("newProxyTokenSource cannot parse endpoint %s: %w", endpoint, err) - return + return nil, err } client := &http.Client{} @@ -71,13 +71,16 @@ func (ts proxyTokenSource) Token() (token *oauth2.Token, err error) { resp, err := ts.client.Get(ts.endpoint) if err != nil { err = fmt.Errorf("proxyTokenSource cannot fetch token: %w", err) - return + return nil, err } + defer func() { + _ = resp.Body.Close() + }() body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) if err != nil { err = fmt.Errorf("proxyTokenSource cannot load body: %w", err) - return + return nil, err } if c := resp.StatusCode; c < 200 || c >= 300 { @@ -85,15 +88,19 @@ func (ts proxyTokenSource) Token() (token *oauth2.Token, err error) { Response: resp, Body: body, } - return + return nil, err } token = &oauth2.Token{} err = json.Unmarshal(body, token) if err != nil { err = fmt.Errorf("proxyTokenSource cannot decode body: %w", err) - return + return nil, err } - return + return token, nil +} + +func NewTokenSourceFromURL(ctx context.Context, tokenUrl string, reuseTokenFromUrl bool) (tokenSrc oauth2.TokenSource, err error) { + return newProxyTokenSource(ctx, tokenUrl, reuseTokenFromUrl) } diff --git a/internal/auth/token_source_test.go b/internal/auth/token_source_test.go new file mode 100644 index 0000000000..df649ce99a --- /dev/null +++ b/internal/auth/token_source_test.go @@ -0,0 +1,88 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package auth + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/oauth2" +) + +func Test_NewTokenSourceFromURL_Success(t *testing.T) { + // Create fake token server. + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + token := oauth2.Token{ + AccessToken: "test-access-token", + TokenType: "Bearer", + } + require.NoError(t, json.NewEncoder(w).Encode(token)) + })) + defer server.Close() + + ts, err := NewTokenSourceFromURL(context.Background(), server.URL, false) + + assert.NoError(t, err) + assert.NotNil(t, ts) + // Fetch token + token, err := ts.Token() + assert.NoError(t, err) + assert.Equal(t, "test-access-token", token.AccessToken) +} + +func Test_NewTokenSourceFromURL_InvalidURL(t *testing.T) { + ts, err := NewTokenSourceFromURL(context.Background(), ":", false) // invalid URL + + assert.Error(t, err) + assert.Nil(t, ts) +} + +func TestProxyTokenSource_TokenFetch_ServerError(t *testing.T) { + // Simulate HTTP 500 error. + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "server error", http.StatusInternalServerError) + })) + defer server.Close() + ts, err := NewTokenSourceFromURL(context.Background(), server.URL, false) + require.NoError(t, err) + + token, err := ts.Token() + + assert.Error(t, err) + assert.Contains(t, err.Error(), "server error") + assert.Nil(t, token) +} + +func TestProxyTokenSource_TokenFetch_InvalidJSON(t *testing.T) { + // Simulate invalid JSON. + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, err := w.Write([]byte("not-json")) + require.NoError(t, err) + })) + defer server.Close() + ts, err := NewTokenSourceFromURL(context.Background(), server.URL, false) + require.NoError(t, err) + + token, err := ts.Token() + + assert.Error(t, err) + assert.Contains(t, err.Error(), "decode body") + assert.Nil(t, token) +} diff --git a/internal/block/block.go b/internal/block/block.go index 72a14f0ac5..6bce0785f6 100644 --- a/internal/block/block.go +++ b/internal/block/block.go @@ -15,7 +15,6 @@ package block import ( - "bytes" "fmt" "io" "syscall" @@ -23,61 +22,109 @@ import ( // Block represents the buffer which holds the data. type Block interface { - // Reuse resets the blocks for reuse. - Reuse() + // ReadSeeker is the interface that groups the basic Read and Seek methods. + // It is used to read data from the block. + io.ReadSeeker + + // Writer provides a way to write data to the block. + io.Writer + + // GenBlock defines reuse and deallocation of the block. + GenBlock // Size provides the current data size of the block. The capacity of the block // can be >= data_size. Size() int64 - // Write writes the given data to block. - Write(bytes []byte) error - - // Reader interface helps in copying the data directly to storage.writer - // while uploading to GCS. - Reader() io.Reader + // Cap returns the capacity of the block, kind of block-size. + Cap() int64 - Deallocate() error -} - -// TODO: check if we need offset or just storing end is sufficient. We might need -// for handling ordered writes. It will be decided after ordered writes design. -type offset struct { - start, end int64 + // Write writes the given data to block. + Write(bytes []byte) (n int, err error) } type memoryBlock struct { - Block buffer []byte - offset offset + + // readSeek is used to track the position for reading data. + readSeek int64 } func (m *memoryBlock) Reuse() { - clear(m.buffer) - - m.offset.end = 0 - m.offset.start = 0 + m.readSeek = 0 + m.buffer = m.buffer[:0] } func (m *memoryBlock) Size() int64 { - return m.offset.end - m.offset.start + return int64(len(m.buffer)) +} + +func (m *memoryBlock) Cap() int64 { + return int64(cap(m.buffer)) } -func (m *memoryBlock) Write(bytes []byte) error { - if m.Size()+int64(len(bytes)) > int64(cap(m.buffer)) { - return fmt.Errorf("received data more than capacity of the block") + +// Read reads data from the block into the provided byte slice. +// Please make sure to call Seek before calling Read if you want to read from a specific position. +func (m *memoryBlock) Read(bytes []byte) (int, error) { + if m.readSeek < 0 { + return 0, fmt.Errorf("readSeek %d is less than start offset 0", m.readSeek) } - n := copy(m.buffer[m.offset.end:], bytes) - if n != len(bytes) { - return fmt.Errorf("error in copying the data to block. Expected %d, got %d", len(bytes), n) + if m.readSeek >= int64(len(m.buffer)) { + return 0, io.EOF } - m.offset.end += int64(len(bytes)) - return nil + n := copy(bytes, m.buffer[m.readSeek:]) + m.readSeek += int64(n) + + // If readSeek is beyond the end of the block, return EOF early. + if m.readSeek >= int64(len(m.buffer)) { + return n, io.EOF + } + + return n, nil } -func (m *memoryBlock) Reader() io.Reader { - return bytes.NewReader(m.buffer[0:m.offset.end]) +// Seek sets the readSeek position in the block. +// It returns the new readSeek position and an error if any. +// The whence argument specifies how the offset should be interpreted: +// - io.SeekStart: offset is relative to the start of the block. +// - io.SeekCurrent: offset is relative to the current readSeek position. +// - io.SeekEnd: offset is relative to the end of the block. +// +// It returns an error if the whence value is invalid or if the new +// readSeek position is out of bounds. +func (m *memoryBlock) Seek(offset int64, whence int) (int64, error) { + newReadSeek := m.readSeek + switch whence { + case io.SeekStart: + newReadSeek = offset + case io.SeekCurrent: + newReadSeek += offset + case io.SeekEnd: + newReadSeek = int64(len(m.buffer)) + offset + default: + return 0, fmt.Errorf("invalid whence value: %d", whence) + } + + if newReadSeek < 0 || newReadSeek > int64(len(m.buffer)) { + return 0, fmt.Errorf("new readSeek position %d is out of bounds", newReadSeek) + } + + m.readSeek = newReadSeek + return m.readSeek, nil +} + +func (m *memoryBlock) Write(bytes []byte) (int, error) { + if len(bytes) > cap(m.buffer)-len(m.buffer) { + return 0, fmt.Errorf("received data more than capacity of the block") + } + + currentLen := len(m.buffer) + m.buffer = m.buffer[:currentLen+len(bytes)] + n := copy(m.buffer[currentLen:], bytes) + + return n, nil } func (m *memoryBlock) Deallocate() error { @@ -85,7 +132,7 @@ func (m *memoryBlock) Deallocate() error { return fmt.Errorf("invalid buffer") } - err := syscall.Munmap(m.buffer) + err := syscall.Munmap(m.buffer[:cap(m.buffer)]) m.buffer = nil if err != nil { // if we get here, there is likely memory corruption. @@ -104,8 +151,7 @@ func createBlock(blockSize int64) (Block, error) { } mb := memoryBlock{ - buffer: addr, - offset: offset{0, 0}, + buffer: addr[:0], } return &mb, nil } diff --git a/internal/block/block_pool.go b/internal/block/block_pool.go index 1a94ff9494..4e1de2feed 100644 --- a/internal/block/block_pool.go +++ b/internal/block/block_pool.go @@ -15,15 +15,35 @@ package block import ( + "errors" "fmt" "golang.org/x/sync/semaphore" ) -// BlockPool handles the creation of blocks as per the user configuration. -type BlockPool struct { +var CantAllocateAnyBlockError error = errors.New("cant allocate any block as global max blocks limit is reached") + +type GenBlock interface { + // Reuse resets the block for reuse. + Reuse() + + // Deallocate releases the resources held by the block. + Deallocate() error +} + +// GenBlockPool is a generic block pool for managing blocks that implement the GenBlock interface. +// It offers methods to get blocks, return blocks to the free pool, and clear the free pool. +// This implementation is NOT thread-safe - concurrent access from multiple goroutines requires external synchronization. +// +// Block allocation is controlled by maxBlocks (per-pool limit) and a global semaphore (cross-pool limit). +// When the global limit is reached, Get() will block until blocks become available, while TryGet() +// returns an error immediately to avoid blocking. +// +// The pool supports reserving blocks at creation time - these reserved blocks hold semaphore permits +// and can only be released when clearing the pool with releaseReservedBlocks=true. +type GenBlockPool[T GenBlock] struct { // Channel holding free blocks. - freeBlocksCh chan Block + freeBlocksCh chan T // Size of each block this pool holds. blockSize int64 @@ -34,31 +54,50 @@ type BlockPool struct { // Total number of blocks created so far. totalBlocks int64 + // Number of blocks reserved at the time of block pool creation. + reservedBlocks int64 + // Semaphore used to limit the total number of blocks created across // different files. globalMaxBlocksSem *semaphore.Weighted + + // createBlockFunc is a function that creates a new block of type T + createBlockFunc func(blockSize int64) (T, error) } -// NewBlockPool creates the blockPool based on the user configuration. -func NewBlockPool(blockSize int64, maxBlocks int64, globalMaxBlocksSem *semaphore.Weighted) (bp *BlockPool, err error) { +// NewGenBlockPool creates the blockPool based on the user configuration. +func NewGenBlockPool[T GenBlock](blockSize int64, maxBlocks int64, reservedBlocks int64, globalMaxBlocksSem *semaphore.Weighted, createBlockFunc func(blockSize int64) (T, error)) (bp *GenBlockPool[T], err error) { if blockSize <= 0 || maxBlocks <= 0 { err = fmt.Errorf("invalid configuration provided for blockPool, blocksize: %d, maxBlocks: %d", blockSize, maxBlocks) return } - bp = &BlockPool{ - freeBlocksCh: make(chan Block, maxBlocks), + if reservedBlocks < 0 || reservedBlocks > maxBlocks { + err = fmt.Errorf("invalid reserved blocks count: %d, it should be between 0 and maxBlocks: %d", reservedBlocks, maxBlocks) + return + } + + semAcquired := globalMaxBlocksSem.TryAcquire(reservedBlocks) + if !semAcquired { + return nil, CantAllocateAnyBlockError + } + + return &GenBlockPool[T]{ + freeBlocksCh: make(chan T, maxBlocks), blockSize: blockSize, maxBlocks: maxBlocks, + reservedBlocks: reservedBlocks, totalBlocks: 0, globalMaxBlocksSem: globalMaxBlocksSem, - } - return + createBlockFunc: createBlockFunc, + }, nil } // Get returns a block. It returns an existing block if it's ready for reuse or // creates a new one if required. -func (bp *BlockPool) Get() (Block, error) { +// Not thread-safe, calling from multiple goroutines may lead memory leaks because +// of race conditions. +func (bp *GenBlockPool[T]) Get() (T, error) { for { select { case b := <-bp.freeBlocksCh: @@ -67,39 +106,79 @@ func (bp *BlockPool) Get() (Block, error) { return b, nil default: - // No lock is required here since blockPool is per file and all write - // calls to a single file are serialized because of inode.lock(). - if bp.totalBlocks < bp.maxBlocks { - freeSlotsAvailable := bp.globalMaxBlocksSem.TryAcquire(1) - // We are allowed to create one block per file irrespective of free slots. - if bp.totalBlocks > 0 && !freeSlotsAvailable { - continue - } - - b, err := createBlock(bp.blockSize) + if bp.canAllocateBlock() { + b, err := bp.createBlockFunc(bp.blockSize) if err != nil { - return nil, err + var zero T + return zero, err } bp.totalBlocks++ return b, nil + } + } + } +} +// TryGet returns a block if available, or an error if no blocks can be allocated. +// It returns an existing block if it's ready for reuse or creates a new one if required. +// Not thread-safe, calling from multiple goroutines may lead to memory leaks because +// of race conditions. +func (bp *GenBlockPool[T]) TryGet() (T, error) { + select { + case b := <-bp.freeBlocksCh: + // Reset the block for reuse. + b.Reuse() + return b, nil + + default: + if bp.canAllocateBlock() { + b, err := bp.createBlockFunc(bp.blockSize) + if err != nil { + var zero T + return zero, err } + + bp.totalBlocks++ + return b, nil } + var zero T + return zero, CantAllocateAnyBlockError } } -// FreeBlocksChannel returns the freeBlocksCh being used by the block pool. -func (bp *BlockPool) FreeBlocksChannel() chan Block { - return bp.freeBlocksCh +// canAllocateBlock checks if a new block can be allocated. +func (bp *GenBlockPool[T]) canAllocateBlock() bool { + // If max blocks limit is reached, then no more blocks can be allocated. + if bp.totalBlocks >= bp.maxBlocks { + return false + } + + // Always allow allocation upto reserved number of blocks. + if bp.totalBlocks < bp.reservedBlocks { + return true + } + + // Otherwise, check if we can acquire a semaphore. + semAcquired := bp.globalMaxBlocksSem.TryAcquire(1) + return semAcquired +} + +// Release puts the block back into the free blocks channel for reuse. +func (bp *GenBlockPool[T]) Release(b T) { + select { + case bp.freeBlocksCh <- b: + default: + panic("Block pool's free blocks channel is full, this should never happen") + } } // BlockSize returns the block size used by the blockPool. -func (bp *BlockPool) BlockSize() int64 { +func (bp *GenBlockPool[T]) BlockSize() int64 { return bp.blockSize } -func (bp *BlockPool) ClearFreeBlockChannel() error { +func (bp *GenBlockPool[T]) ClearFreeBlockChannel(releaseReservedBlocks bool) error { for { select { case b := <-bp.freeBlocksCh: @@ -108,11 +187,34 @@ func (bp *BlockPool) ClearFreeBlockChannel() error { // if we get here, there is likely memory corruption. return fmt.Errorf("munmap error: %v", err) } + // Release semaphore for all but the reserved blocks. + if bp.totalBlocks > bp.reservedBlocks { + bp.globalMaxBlocksSem.Release(1) + } bp.totalBlocks-- - bp.globalMaxBlocksSem.Release(1) default: - // Return if there are no more blocks on the channel. + // We are here, it means there are no more blocks in the free blocks channel. + // Release semaphore for the released blocks iff releaseReservedBlocks is true. + if releaseReservedBlocks { + bp.globalMaxBlocksSem.Release(bp.reservedBlocks) + } return nil } } } + +// TotalFreeBlocks returns the total number of free blocks available in the pool. +// This is useful for testing and debugging purposes. +func (bp *GenBlockPool[T]) TotalFreeBlocks() int { + return len(bp.freeBlocksCh) +} + +// NewBlockPool creates GenBlockPool for block.Block interface. +func NewBlockPool(blockSize int64, maxBlocks int64, reservedBlocks int64, globalMaxBlocksSem *semaphore.Weighted) (bp *GenBlockPool[Block], err error) { + return NewGenBlockPool(blockSize, maxBlocks, reservedBlocks, globalMaxBlocksSem, createBlock) +} + +// NewPrefetchBlockPool creates GenBlockPool for block.PrefetchBlock interface. +func NewPrefetchBlockPool(blockSize int64, maxBlocks int64, reservedBlocks int64, globalMaxBlocksSem *semaphore.Weighted) (bp *GenBlockPool[PrefetchBlock], err error) { + return NewGenBlockPool(blockSize, maxBlocks, reservedBlocks, globalMaxBlocksSem, createPrefetchBlock) +} diff --git a/internal/block/block_pool_test.go b/internal/block/block_pool_test.go index 36470f98d6..25ab8b7437 100644 --- a/internal/block/block_pool_test.go +++ b/internal/block/block_pool_test.go @@ -16,7 +16,6 @@ package block import ( "fmt" - "io" "testing" "time" @@ -37,7 +36,7 @@ func TestBlockPoolTestSuite(t *testing.T) { } func (t *BlockPoolTest) TestInitBlockPool() { - bp, err := NewBlockPool(1024, 10, semaphore.NewWeighted(10)) + bp, err := NewGenBlockPool(1024, 10, 0, semaphore.NewWeighted(10), createBlock) require.Nil(t.T(), err) require.NotNil(t.T(), bp) @@ -47,47 +46,81 @@ func (t *BlockPoolTest) TestInitBlockPool() { } func (t *BlockPoolTest) TestInitBlockPoolForZeroBlockSize() { - _, err := NewBlockPool(0, 10, semaphore.NewWeighted(10)) + _, err := NewGenBlockPool(0, 10, 0, semaphore.NewWeighted(10), createBlock) require.NotNil(t.T(), err) assert.Equal(t.T(), fmt.Errorf(invalidConfigError, 0, 10), err) } func (t *BlockPoolTest) TestInitBlockPoolForNegativeBlockSize() { - _, err := NewBlockPool(-1, 10, semaphore.NewWeighted(10)) + _, err := NewGenBlockPool(-1, 10, 0, semaphore.NewWeighted(10), createBlock) require.NotNil(t.T(), err) assert.Equal(t.T(), fmt.Errorf(invalidConfigError, -1, 10), err) } func (t *BlockPoolTest) TestInitBlockPoolForZeroMaxBlocks() { - _, err := NewBlockPool(10, 0, semaphore.NewWeighted(10)) + _, err := NewGenBlockPool(10, 0, 0, semaphore.NewWeighted(10), createBlock) require.NotNil(t.T(), err) assert.Equal(t.T(), fmt.Errorf(invalidConfigError, 10, 0), err) } func (t *BlockPoolTest) TestInitBlockPoolForNegativeMaxBlocks() { - _, err := NewBlockPool(10, -1, semaphore.NewWeighted(10)) + _, err := NewGenBlockPool(10, -1, 0, semaphore.NewWeighted(10), createBlock) require.NotNil(t.T(), err) assert.Equal(t.T(), fmt.Errorf(invalidConfigError, 10, -1), err) } // Represents when block is available on the freeBlocksCh. -func (t *BlockPoolTest) TestGetWhenBlockIsAvailableForReuse() { - bp, err := NewBlockPool(1024, 10, semaphore.NewWeighted(10)) +func (t *BlockPoolTest) TestTryGetWhenBlockIsAvailableForReuse() { + bp, err := NewGenBlockPool(1024, 10, 0, semaphore.NewWeighted(10), createBlock) require.Nil(t.T(), err) // Creating a block with some data and send it to blockCh. b, err := createBlock(2) require.Nil(t.T(), err) - content := []byte("hi") - err = b.Write(content) + bp.freeBlocksCh <- b + // Setting totalBlocks same as maxBlocks to ensure no new blocks are created. + bp.totalBlocks = 10 + + block, err := bp.TryGet() + + require.Nil(t.T(), err) + require.NotNil(t.T(), block) + // This ensures the block is reset. + assert.Equal(t.T(), int64(0), block.Size()) +} + +func (t *BlockPoolTest) TestTryGetWhenTotalBlocksIsLessThanThanMaxBlocks() { + bp, err := NewGenBlockPool(1024, 10, 0, semaphore.NewWeighted(10), createBlock) + require.Nil(t.T(), err) + + block, err := bp.TryGet() + + require.Nil(t.T(), err) + require.NotNil(t.T(), block) + assert.Equal(t.T(), int64(0), block.Size()) +} + +func (t *BlockPoolTest) TestTryGetToCreateLargeBlock() { + // Creating block of size 1TB + bp, err := NewGenBlockPool(1024*1024*1024*1024, 10, 0, semaphore.NewWeighted(10), createBlock) + require.Nil(t.T(), err) + + _, err = bp.TryGet() + + require.NotNil(t.T(), err) + assert.Equal(t.T(), "mmap error: cannot allocate memory", err.Error()) +} + +// Represents when block is available on the freeBlocksCh. +func (t *BlockPoolTest) TestGetWhenBlockIsAvailableForReuse() { + bp, err := NewGenBlockPool(1024, 10, 0, semaphore.NewWeighted(10), createBlock) + require.Nil(t.T(), err) + // Creating a block with some data and send it to blockCh. + b, err := createBlock(2) require.Nil(t.T(), err) - // Validating the content of the block - output, err := io.ReadAll(b.Reader()) - assert.Nil(t.T(), err) - assert.Equal(t.T(), content, output) bp.freeBlocksCh <- b // Setting totalBlocks same as maxBlocks to ensure no new blocks are created. bp.totalBlocks = 10 @@ -101,7 +134,7 @@ func (t *BlockPoolTest) TestGetWhenBlockIsAvailableForReuse() { } func (t *BlockPoolTest) TestGetWhenTotalBlocksIsLessThanThanMaxBlocks() { - bp, err := NewBlockPool(1024, 10, semaphore.NewWeighted(10)) + bp, err := NewGenBlockPool(1024, 10, 0, semaphore.NewWeighted(10), createBlock) require.Nil(t.T(), err) block, err := bp.Get() @@ -113,7 +146,7 @@ func (t *BlockPoolTest) TestGetWhenTotalBlocksIsLessThanThanMaxBlocks() { func (t *BlockPoolTest) TestCreateBlockWithLargeSize() { // Creating block of size 1TB - bp, err := NewBlockPool(1024*1024*1024*1024, 10, semaphore.NewWeighted(10)) + bp, err := NewGenBlockPool(1024*1024*1024*1024, 10, 0, semaphore.NewWeighted(10), createBlock) require.Nil(t.T(), err) _, err = bp.Get() @@ -123,79 +156,256 @@ func (t *BlockPoolTest) TestCreateBlockWithLargeSize() { } func (t *BlockPoolTest) TestBlockSize() { - bp, err := NewBlockPool(1024, 10, semaphore.NewWeighted(10)) + bp, err := NewGenBlockPool(1024, 10, 0, semaphore.NewWeighted(10), createBlock) require.Nil(t.T(), err) require.Equal(t.T(), int64(1024), bp.BlockSize()) } -func (t *BlockPoolTest) TestClearFreeBlockChannel() { - bp, err := NewBlockPool(1024, 10, semaphore.NewWeighted(3)) - require.Nil(t.T(), err) - b1, err := bp.Get() - require.Nil(t.T(), err) - require.NotNil(t.T(), b1) - b2, err := bp.Get() - require.Nil(t.T(), err) - require.NotNil(t.T(), b2) - b3, err := bp.Get() +func (t *BlockPoolTest) TestClearFreeBlockChannelWithReleaseReservedBlocksTrue() { + tests := []struct { + name string + reservedBlocks int64 + performGetBlock int + }{ + { + name: "with_0_reserved_blocks", + reservedBlocks: 0, + }, + { + name: "with_1_reserved_blocks", + reservedBlocks: 1, + }, + { + name: "with_2_reserved_blocks", + reservedBlocks: 2, + }, + { + name: "with_3_reserved_blocks", + reservedBlocks: 3, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func() { + bp, err := NewGenBlockPool(1024, 4, tt.reservedBlocks, semaphore.NewWeighted(4), createBlock) + require.Nil(t.T(), err) + blocks := make([]Block, 4) + for i := range 4 { + blocks[i] = t.validateGetBlockIsNotBlocked(bp) + } + // Adding all blocks to freeBlocksCh + for i := range 4 { + bp.freeBlocksCh <- blocks[i] + } + require.Equal(t.T(), int64(4), bp.totalBlocks) + + err = bp.ClearFreeBlockChannel(true) + + require.Nil(t.T(), err) + require.EqualValues(t.T(), 0, bp.totalBlocks) + for i := range 4 { + require.Nil(t.T(), blocks[i].(*memoryBlock).buffer) + } + // All 4 semaphore slots should be available to acquire. + require.True(t.T(), bp.globalMaxBlocksSem.TryAcquire(4)) + require.False(t.T(), bp.globalMaxBlocksSem.TryAcquire(1)) + }) + } +} + +func (t *BlockPoolTest) TestClearFreeBlockChannelWithReleaseReservedBlocksFalse() { + tests := []struct { + name string + releaseReservedBlocks bool + reservedBlocks int64 + possibleSemaphoreSlots int64 + }{ + { + name: "with_0_reserved_blocks", + reservedBlocks: 0, + possibleSemaphoreSlots: 4, + }, + { + name: "with_1_reserved_blocks", + reservedBlocks: 1, + possibleSemaphoreSlots: 3, // 4 - 1 reserved + }, + { + name: "with_2_reserved_blocks", + reservedBlocks: 2, + possibleSemaphoreSlots: 2, // 4 - 2 reserved + }, + { + name: "all_4_reserved_blocks", + reservedBlocks: 4, + possibleSemaphoreSlots: 0, // 4 - 4 reserved + }, + } + + for _, tt := range tests { + t.Run(tt.name, func() { + bp, err := NewGenBlockPool(1024, 4, tt.reservedBlocks, semaphore.NewWeighted(4), createBlock) + require.Nil(t.T(), err) + blocks := make([]Block, 4) + for i := range 4 { + blocks[i] = t.validateGetBlockIsNotBlocked(bp) + } + // Adding all blocks to freeBlocksCh + for i := range 4 { + bp.freeBlocksCh <- blocks[i] + } + require.Equal(t.T(), int64(4), bp.totalBlocks) + + err = bp.ClearFreeBlockChannel(false) + + require.Nil(t.T(), err) + require.EqualValues(t.T(), 0, bp.totalBlocks) + for i := range 4 { + require.Nil(t.T(), blocks[i].(*memoryBlock).buffer) + } + // Only reserved blocks semaphore slots should be available to acquire. + require.True(t.T(), bp.globalMaxBlocksSem.TryAcquire(tt.possibleSemaphoreSlots)) + require.False(t.T(), bp.globalMaxBlocksSem.TryAcquire(1)) + }) + } +} + +func (t *BlockPoolTest) TestClearFreeBlockChannelWhenTotalBlocksIsZero() { + bp, err := NewGenBlockPool(1024, 10, 0, semaphore.NewWeighted(1), createBlock) require.Nil(t.T(), err) - require.NotNil(t.T(), b3) - // Adding 2 blocks to freeBlocksCh - bp.freeBlocksCh <- b1 - bp.freeBlocksCh <- b2 - require.Equal(t.T(), int64(3), bp.totalBlocks) + require.Equal(t.T(), int64(0), bp.totalBlocks) - err = bp.ClearFreeBlockChannel() + err = bp.ClearFreeBlockChannel(true) require.Nil(t.T(), err) - require.Equal(t.T(), int64(1), bp.totalBlocks) - require.Nil(t.T(), b1.(*memoryBlock).buffer) - require.Nil(t.T(), b2.(*memoryBlock).buffer) - require.NotNil(t.T(), b3.(*memoryBlock).buffer) + require.Equal(t.T(), int64(0), bp.totalBlocks) // Check if semaphore is released correctly. - require.True(t.T(), bp.globalMaxBlocksSem.TryAcquire(2)) + require.True(t.T(), bp.globalMaxBlocksSem.TryAcquire(1)) require.False(t.T(), bp.globalMaxBlocksSem.TryAcquire(1)) } -func (t *BlockPoolTest) TestGetWhenGlobalMaxBlocksIsZero() { - bp, err := NewBlockPool(1024, 10, semaphore.NewWeighted(0)) - require.Nil(t.T(), err) +func (t *BlockPoolTest) TestBlockPoolCreationAcquiresGlobalSem() { + globalBlocksSem := semaphore.NewWeighted(1) - // First block is allowed even with globalMaxBlocks being zero. + bp, err := NewGenBlockPool(1024, 3, 1, globalBlocksSem, createBlock) + + require.Nil(t.T(), err) + // Validate that semaphore got acquired. + acquired := globalBlocksSem.TryAcquire(1) + assert.False(t.T(), acquired) + // Validate that 1st block can be created as it was reserved. b1, err := bp.Get() require.Nil(t.T(), err) require.NotNil(t.T(), b1) - // We shouldn't be allowed to create another block. - t.validateGetBlockIsBlocked(bp) + // Validate that adding block to freeBlocksCh and clearing it up releases the semaphore + bp.freeBlocksCh <- b1 + require.Equal(t.T(), int64(1), bp.totalBlocks) + err = bp.ClearFreeBlockChannel(true) + require.Nil(t.T(), err) + require.Equal(t.T(), int64(0), bp.totalBlocks) + require.Nil(t.T(), b1.(*memoryBlock).buffer) + // Validate that semaphore can be acquired now. + acquired = globalBlocksSem.TryAcquire(1) + assert.True(t.T(), acquired) } -func (t *BlockPoolTest) TestGetWhenTotalBlocksEqualToGlobalBlocks() { - bp, err := NewBlockPool(1024, 10, semaphore.NewWeighted(2)) +func (t *BlockPoolTest) TestClearFreeBlockChannelWithMultipleBlockPools() { + globalMaxBlocksSem := semaphore.NewWeighted(3) + bp1, err := NewGenBlockPool(1024, 3, 1, globalMaxBlocksSem, createBlock) require.Nil(t.T(), err) + bp2, err := NewGenBlockPool(1024, 3, 1, globalMaxBlocksSem, createBlock) + require.Nil(t.T(), err) + // Create 2 blocks in bp1. + b1 := t.validateGetBlockIsNotBlocked(bp1) + b2 := t.validateGetBlockIsNotBlocked(bp1) + require.Equal(t.T(), int64(2), bp1.totalBlocks) + // Create 1 block in bp2. + b3 := t.validateGetBlockIsNotBlocked(bp2) + require.Equal(t.T(), int64(1), bp2.totalBlocks) + // Freeing up bp1. + bp1.freeBlocksCh <- b1 + bp1.freeBlocksCh <- b2 + err = bp1.ClearFreeBlockChannel(true) + require.Nil(t.T(), err) + require.Nil(t.T(), b1.(*memoryBlock).buffer) + require.Nil(t.T(), b2.(*memoryBlock).buffer) - // Create 1st block - b1, err := bp.Get() + // After bp1 is freed up, 1 more block can be created in bp2. + b4 := t.validateGetBlockIsNotBlocked(bp2) + require.Equal(t.T(), int64(2), bp2.totalBlocks) + + // Freeing up bp2. + bp2.freeBlocksCh <- b3 + bp2.freeBlocksCh <- b4 + err = bp2.ClearFreeBlockChannel(true) require.Nil(t.T(), err) - require.NotNil(t.T(), b1) - // Create 2nd block - b2, err := bp.Get() + require.Nil(t.T(), b3.(*memoryBlock).buffer) + require.Nil(t.T(), b4.(*memoryBlock).buffer) +} + +func (t *BlockPoolTest) TestBlockPoolCreationFailsWhenGlobalMaxBlocksIsZero() { + bp, err := NewGenBlockPool(1024, 10, 1, semaphore.NewWeighted(0), createBlock) + + require.Error(t.T(), err) + assert.Nil(t.T(), bp) + assert.ErrorContains(t.T(), err, CantAllocateAnyBlockError.Error()) +} + +func (t *BlockPoolTest) TestTryGetWhenLimitedByGlobalBlocks() { + bp, err := NewGenBlockPool(1024, 10, 1, semaphore.NewWeighted(2), createBlock) require.Nil(t.T(), err) + // 2 blocks can be created. + b1, err1 := bp.TryGet() + require.Nil(t.T(), err1) + require.NotNil(t.T(), b1) + b2, err2 := bp.TryGet() + require.Nil(t.T(), err2) require.NotNil(t.T(), b2) + + b3, err3 := bp.TryGet() + + require.Nil(t.T(), b3) + require.NotNil(t.T(), err3) + require.ErrorIs(t.T(), err3, CantAllocateAnyBlockError) + require.Equal(t.T(), int64(2), bp.totalBlocks) +} + +func (t *BlockPoolTest) TestTryGetWhenTotalBlocksEqualToMaxBlocks() { + bp, err := NewGenBlockPool(1024, 10, 0, semaphore.NewWeighted(10), createBlock) + require.Nil(t.T(), err) + bp.totalBlocks = 10 + + b, err := bp.TryGet() + + require.NotNil(t.T(), err) + assert.Equal(t.T(), CantAllocateAnyBlockError, err) + require.Nil(t.T(), b) +} + +func (t *BlockPoolTest) TestGetWhenLimitedByGlobalBlocks() { + bp, err := NewGenBlockPool(1024, 10, 1, semaphore.NewWeighted(2), createBlock) + require.Nil(t.T(), err) + + // 2 blocks can be created. + for range 2 { + _ = t.validateGetBlockIsNotBlocked(bp) + } require.Equal(t.T(), int64(2), bp.totalBlocks) t.validateGetBlockIsBlocked(bp) } func (t *BlockPoolTest) TestGetWhenTotalBlocksEqualToMaxBlocks() { - bp, err := NewBlockPool(1024, 10, semaphore.NewWeighted(2)) + bp, err := NewGenBlockPool(1024, 10, 0, semaphore.NewWeighted(10), createBlock) require.Nil(t.T(), err) bp.totalBlocks = 10 t.validateGetBlockIsBlocked(bp) } -func (t *BlockPoolTest) validateGetBlockIsBlocked(bp *BlockPool) { +func (t *BlockPoolTest) validateGetBlockIsBlocked(bp *GenBlockPool[Block]) { + t.T().Helper() done := make(chan bool, 1) go func() { b, err := bp.Get() @@ -211,14 +421,215 @@ func (t *BlockPoolTest) validateGetBlockIsBlocked(bp *BlockPool) { } } -func (t *BlockPoolTest) TestBlockPool_FreeBlocksChannel() { - freeBlocksCh := make(chan Block) - bp := &BlockPool{ - freeBlocksCh: freeBlocksCh, +func (t *BlockPoolTest) validateGetBlockIsNotBlocked(bp *GenBlockPool[Block]) Block { + t.T().Helper() + done := make(chan Block, 1) + go func() { + b, err := bp.Get() + require.Nil(t.T(), err) + require.NotNil(t.T(), b) + done <- b + }() + + select { + case block := <-done: + return block + case <-time.After(1 * time.Second): + assert.FailNow(t.T(), "Not able to get/create a block") + return nil + } +} + +func (t *BlockPoolTest) TestCanAllocateBlock() { + tests := []struct { + name string + maxBlocks int64 + totalBlocks int64 + reservedBlocks int64 + globalSem *semaphore.Weighted + expected bool + }{ + { + name: "max_blocks_reached", + maxBlocks: 10, + totalBlocks: 10, + reservedBlocks: 0, + globalSem: semaphore.NewWeighted(0), + expected: false, + }, + { + name: "first_block", + maxBlocks: 10, + totalBlocks: 0, + reservedBlocks: 0, + globalSem: semaphore.NewWeighted(1), + expected: true, + }, + { + name: "semaphore_acquirable", + maxBlocks: 10, + totalBlocks: 5, + reservedBlocks: 0, + globalSem: semaphore.NewWeighted(1), + expected: true, + }, + { + name: "semaphore_not_acquirable", + maxBlocks: 10, + totalBlocks: 5, + reservedBlocks: 0, + globalSem: semaphore.NewWeighted(0), + expected: false, + }, + { + name: "equal_max_blocks_and_total_blocks_0", + maxBlocks: 0, + totalBlocks: 0, + reservedBlocks: 0, + globalSem: semaphore.NewWeighted(0), + expected: false, + }, + { + name: "total_blocks_more_than_max_blocks", + maxBlocks: 0, + totalBlocks: 1, + reservedBlocks: 0, + globalSem: semaphore.NewWeighted(0), + expected: false, + }, + { + name: "reserved_blocks_equal_to_max_blocks", + maxBlocks: 10, + totalBlocks: 0, + reservedBlocks: 10, + globalSem: semaphore.NewWeighted(10), + expected: true, + }, + { + name: "reserved_blocks_less_than_max_blocks", + maxBlocks: 10, + totalBlocks: 0, + reservedBlocks: 5, + globalSem: semaphore.NewWeighted(10), + expected: true, + }, + { + name: "reserved_blocks_equal_to_total_blocks", + maxBlocks: 10, + totalBlocks: 5, + reservedBlocks: 5, + globalSem: semaphore.NewWeighted(0), + expected: false, + }, + { + name: "reserved_blocks_less_than_total_blocks", + maxBlocks: 10, + totalBlocks: 6, + reservedBlocks: 5, + globalSem: semaphore.NewWeighted(0), + expected: false, + }, + { + name: "reserved_blocks_more_than_total_blocks", + maxBlocks: 10, + totalBlocks: 4, + reservedBlocks: 5, + globalSem: semaphore.NewWeighted(0), + expected: false, + }, } - ch := bp.FreeBlocksChannel() + for _, tt := range tests { + t.Run(tt.name, func() { + bp := &GenBlockPool[Block]{ + maxBlocks: tt.maxBlocks, + totalBlocks: tt.totalBlocks, + globalMaxBlocksSem: tt.globalSem, + } - assert.NotNil(t.T(), ch) - assert.Equal(t.T(), freeBlocksCh, ch) + got := bp.canAllocateBlock() + + assert.Equal(t.T(), tt.expected, got) + }) + } +} + +func (t *BlockPoolTest) TestBlockPoolCreationWithReservedBlocksSuccess() { + tests := []struct { + name string + reservedBlocks int64 + maxBlocks int64 + }{ + { + name: "zero_reserved_blocks", + reservedBlocks: 0, + maxBlocks: 10, + }, + { + name: "one_reserved_block", + reservedBlocks: 1, + maxBlocks: 10, + }, + { + name: "two_reserved_blocks", + reservedBlocks: 2, + maxBlocks: 10, + }, + { + name: "max_blocks_equal_to_reserved_blocks", + reservedBlocks: 10, + maxBlocks: 10, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func() { + bp, err := NewGenBlockPool(1024, 10, tt.reservedBlocks, semaphore.NewWeighted(20), createBlock) + + require.NoError(t.T(), err) + require.NotNil(t.T(), bp) + }) + } +} + +func (t *BlockPoolTest) TestBlockPoolCreationWithReservedBlocksFailure() { + tests := []struct { + name string + reservedBlocks int64 + maxBlocks int64 + globalMaxBlocks int64 + expectedErrMsg string + }{ + { + name: "reserved_blocks_greater_than_max_blocks", + reservedBlocks: 11, + maxBlocks: 10, + globalMaxBlocks: 20, + expectedErrMsg: "invalid reserved blocks count: 11, it should be between 0 and maxBlocks: 10", + }, + { + name: "negative_reserved_blocks", + reservedBlocks: -1, + maxBlocks: 10, + globalMaxBlocks: 20, + expectedErrMsg: "invalid reserved blocks count: -1, it should be between 0 and maxBlocks: 10", + }, + { + name: "reserved_blocks_greater_than_global_max_blocks", + reservedBlocks: 7, + maxBlocks: 7, + globalMaxBlocks: 6, + expectedErrMsg: "cant allocate any block as global max blocks limit is reached", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func() { + bp, err := NewGenBlockPool(1024, 10, tt.reservedBlocks, semaphore.NewWeighted(tt.globalMaxBlocks), createBlock) + + require.Error(t.T(), err) + assert.Nil(t.T(), bp) + assert.EqualError(t.T(), err, tt.expectedErrMsg) + }) + } } diff --git a/internal/block/block_test.go b/internal/block/block_test.go index 4ac5ea5d28..f1eb5d780d 100644 --- a/internal/block/block_test.go +++ b/internal/block/block_test.go @@ -15,6 +15,8 @@ package block import ( + "errors" + "fmt" "io" "testing" @@ -37,10 +39,12 @@ func (testSuite *MemoryBlockTest) TestMemoryBlockWrite() { mb, err := createBlock(12) require.Nil(testSuite.T(), err) content := []byte("hi") - err = mb.Write(content) + n, err := mb.Write(content) assert.Nil(testSuite.T(), err) - output, err := io.ReadAll(mb.Reader()) + assert.Equal(testSuite.T(), len(content), n) + assert.Equal(testSuite.T(), int64(0), mb.(*memoryBlock).readSeek) + output, err := io.ReadAll(mb) assert.Nil(testSuite.T(), err) assert.Equal(testSuite.T(), content, output) assert.Equal(testSuite.T(), int64(2), mb.Size()) @@ -50,21 +54,25 @@ func (testSuite *MemoryBlockTest) TestMemoryBlockWriteWithDataGreaterThanCapacit mb, err := createBlock(1) require.Nil(testSuite.T(), err) content := []byte("hi") - err = mb.Write(content) + n, err := mb.Write(content) assert.NotNil(testSuite.T(), err) + assert.Equal(testSuite.T(), 0, n) assert.EqualError(testSuite.T(), err, outOfCapacityError) } func (testSuite *MemoryBlockTest) TestMemoryBlockWriteWithMultipleWrites() { mb, err := createBlock(12) require.Nil(testSuite.T(), err) - err = mb.Write([]byte("hi")) - assert.Nil(testSuite.T(), err) - err = mb.Write([]byte("hello")) - assert.Nil(testSuite.T(), err) + n, err := mb.Write([]byte("hi")) + require.Nil(testSuite.T(), err) + require.Equal(testSuite.T(), 2, n) + n, err = mb.Write([]byte("hello")) + require.Nil(testSuite.T(), err) + require.Equal(testSuite.T(), 5, n) - output, err := io.ReadAll(mb.Reader()) + assert.Equal(testSuite.T(), int64(0), mb.(*memoryBlock).readSeek) + output, err := io.ReadAll(mb) assert.Nil(testSuite.T(), err) assert.Equal(testSuite.T(), []byte("hihello"), output) assert.Equal(testSuite.T(), int64(7), mb.Size()) @@ -74,11 +82,13 @@ func (testSuite *MemoryBlockTest) TestMemoryBlockWriteWith2ndWriteBeyondCapacity mb, err := createBlock(2) require.Nil(testSuite.T(), err) content := []byte("hi") - err = mb.Write(content) - assert.Nil(testSuite.T(), err) - err = mb.Write(content) + n, err := mb.Write(content) + require.Nil(testSuite.T(), err) + require.Equal(testSuite.T(), 2, n) + n, err = mb.Write(content) assert.NotNil(testSuite.T(), err) + assert.Equal(testSuite.T(), 0, n) assert.EqualError(testSuite.T(), err, outOfCapacityError) } @@ -86,16 +96,19 @@ func (testSuite *MemoryBlockTest) TestMemoryBlockReuse() { mb, err := createBlock(12) require.Nil(testSuite.T(), err) content := []byte("hi") - err = mb.Write(content) - assert.Nil(testSuite.T(), err) - output, err := io.ReadAll(mb.Reader()) - assert.Nil(testSuite.T(), err) - assert.Equal(testSuite.T(), content, output) - assert.Equal(testSuite.T(), int64(2), mb.Size()) + n, err := mb.Write(content) + require.Nil(testSuite.T(), err) + require.Equal(testSuite.T(), 2, n) + require.Equal(testSuite.T(), int64(0), mb.(*memoryBlock).readSeek) + output, err := io.ReadAll(mb) + require.Nil(testSuite.T(), err) + require.Equal(testSuite.T(), content, output) + require.Equal(testSuite.T(), int64(2), mb.Size()) mb.Reuse() - output, err = io.ReadAll(mb.Reader()) + assert.Equal(testSuite.T(), int64(0), mb.(*memoryBlock).readSeek) + output, err = io.ReadAll(mb) assert.Nil(testSuite.T(), err) assert.Empty(testSuite.T(), output) assert.Equal(testSuite.T(), int64(0), mb.Size()) @@ -114,7 +127,8 @@ func (testSuite *MemoryBlockTest) TestMemoryBlockReaderForEmptyBlock() { mb, err := createBlock(12) require.Nil(testSuite.T(), err) - output, err := io.ReadAll(mb.Reader()) + assert.Equal(testSuite.T(), int64(0), mb.(*memoryBlock).readSeek) + output, err := io.ReadAll(mb) assert.Nil(testSuite.T(), err) assert.Empty(testSuite.T(), output) assert.Equal(testSuite.T(), int64(0), mb.Size()) @@ -124,15 +138,184 @@ func (testSuite *MemoryBlockTest) TestMemoryBlockDeAllocate() { mb, err := createBlock(12) require.Nil(testSuite.T(), err) content := []byte("hi") - err = mb.Write(content) + n, err := mb.Write(content) require.Nil(testSuite.T(), err) - output, err := io.ReadAll(mb.Reader()) + require.Equal(testSuite.T(), 2, n) + require.Equal(testSuite.T(), int64(0), mb.(*memoryBlock).readSeek) + output, err := io.ReadAll(mb) require.Nil(testSuite.T(), err) require.Equal(testSuite.T(), content, output) require.Equal(testSuite.T(), int64(2), mb.Size()) err = mb.Deallocate() + assert.Nil(testSuite.T(), err) + assert.Nil(testSuite.T(), mb.(*memoryBlock).buffer) +} + +func (testSuite *MemoryBlockTest) TestMemoryBlockCap() { + mb, err := createBlock(12) + require.Nil(testSuite.T(), err) + + assert.Equal(testSuite.T(), int64(12), mb.Cap()) +} + +func (testSuite *MemoryBlockTest) TestMemoryBlockCapAfterWrite() { + mb, err := createBlock(12) + require.Nil(testSuite.T(), err) + content := []byte("hi") + n, err := mb.Write(content) + require.Nil(testSuite.T(), err) + require.Equal(testSuite.T(), 2, n) + + assert.Equal(testSuite.T(), int64(12), mb.Cap()) +} + +func (testSuite *MemoryBlockTest) TestMemoryBlockReadSuccess() { + mb, err := createBlock(12) + require.Nil(testSuite.T(), err) + content := []byte("hello world") + n, err := mb.Write(content) + require.Nil(testSuite.T(), err) + require.Equal(testSuite.T(), len(content), n) + readBuffer := make([]byte, 5) + + n, err = mb.Read(readBuffer) + + require.Nil(testSuite.T(), err) + require.Equal(testSuite.T(), 5, n) + assert.Equal(testSuite.T(), "hello", string(readBuffer)) +} + +func (testSuite *MemoryBlockTest) TestMemoryBlockReadWithReadBufferMoreThanBlockSize() { + mb, err := createBlock(12) + require.Nil(testSuite.T(), err) + content := []byte("hello world") + n, err := mb.Write(content) + require.Nil(testSuite.T(), err) + require.Equal(testSuite.T(), len(content), n) + readBuffer := make([]byte, 20) + + n, err = mb.Read(readBuffer) + + require.Error(testSuite.T(), io.EOF, err) + require.Equal(testSuite.T(), 11, n) // Read should return all bytes written. +} + +func (testSuite *MemoryBlockTest) TestMemoryBlockReadSeekBeyondEnd() { + mb, err := createBlock(12) + require.Nil(testSuite.T(), err) + content := []byte("hello world") + n, err := mb.Write(content) + require.Nil(testSuite.T(), err) + require.Equal(testSuite.T(), len(content), n) + readBuffer := make([]byte, 12) + mb.(*memoryBlock).readSeek = 13 // Set readSeek to a position beyond the end of the block. + + n, err = mb.Read(readBuffer) + + require.Equal(testSuite.T(), io.EOF, err) + require.Equal(testSuite.T(), 0, n) +} + +func (testSuite *MemoryBlockTest) TestMemoryBlockReadFailure() { + mb, err := createBlock(12) + require.Nil(testSuite.T(), err) + readBuffer := make([]byte, 5) + mb.(*memoryBlock).readSeek = -1 // Simulate an invalid readSeek position. + require.Nil(testSuite.T(), err) + + n, err := mb.Read(readBuffer) + + require.Equal(testSuite.T(), errors.New("readSeek -1 is less than start offset 0"), err) + require.Equal(testSuite.T(), 0, n) // Read should return 0 bytes read. +} + +func (testSuite *MemoryBlockTest) TestMemoryBlockSeek() { + mb, err := createBlock(12) + require.Nil(testSuite.T(), err) + content := []byte("hello world") + n, err := mb.Write(content) + require.Nil(testSuite.T(), err) + require.Equal(testSuite.T(), len(content), n) + + tests := []struct { + whence int + offset int64 + expectedOutput string + expectedOffset int64 + }{ + {io.SeekStart, 0, "hello", 0}, // After this, readSeek = 5 + {io.SeekCurrent, 1, "world", 6}, // After this readSeek = 11 + {io.SeekEnd, -6, " worl", 5}, + } + + for _, tt := range tests { + testSuite.T().Run(fmt.Sprintf("whence=%d, offset=%d, expectedOutput:%s", tt.whence, tt.offset, tt.expectedOutput), func(t *testing.T) { + offset, err := mb.Seek(tt.offset, tt.whence) + + require.Nil(t, err) + require.Equal(t, tt.expectedOffset, offset) + readBuffer := make([]byte, 5) + n, err = mb.Read(readBuffer) + require.Condition(t, func() bool { + return err == nil || errors.Is(err, io.EOF) + }, "Read err can be nil or io.EOF") + require.Equal(t, 5, n) + assert.Equal(t, tt.expectedOutput, string(readBuffer)) + }) + } +} + +func (testSuite *MemoryBlockTest) TestMemoryBlockSeekInvalidWhence() { + mb, err := createBlock(12) + require.Nil(testSuite.T(), err) + _, err = mb.Write([]byte("hello")) + require.Nil(testSuite.T(), err) + + _, err = mb.Seek(0, 4) + + assert.ErrorContains(testSuite.T(), err, "invalid whence value") +} + +func (testSuite *MemoryBlockTest) TestMemoryBlockSeekOutOfBounds() { + mb, err := createBlock(12) + require.Nil(testSuite.T(), err) + _, err = mb.Write([]byte("hello")) + require.Nil(testSuite.T(), err) + + tests := []struct { + name string + offset int64 + whence int + }{ + {"SeekStartNegative", -1, io.SeekStart}, + {"SeekStartBeyondEnd", 6, io.SeekStart}, + {"SeekCurrentNegative", -1, io.SeekCurrent}, + {"SeekCurrentBeyondEnd", 6, io.SeekCurrent}, + {"SeekEndNegative", -6, io.SeekEnd}, + {"SeekEndBeyondEnd", 1, io.SeekEnd}, + } + + for _, tt := range tests { + testSuite.T().Run(tt.name, func(t *testing.T) { + // Reset readSeek to 0 before each test case + _, _ = mb.Seek(0, io.SeekStart) + + _, err := mb.Seek(tt.offset, tt.whence) + + assert.ErrorContains(t, err, "out of bounds") + }) + } +} + +func (testSuite *MemoryBlockTest) TestMemoryBlockDoubleDeallocate() { + mb, err := createBlock(12) require.Nil(testSuite.T(), err) - require.Nil(testSuite.T(), mb.(*memoryBlock).buffer) + err = mb.Deallocate() + require.Nil(testSuite.T(), err) + + err = mb.Deallocate() + + assert.ErrorContains(testSuite.T(), err, "invalid buffer") } diff --git a/internal/block/prefetch_block.go b/internal/block/prefetch_block.go new file mode 100644 index 0000000000..bb45ccc92c --- /dev/null +++ b/internal/block/prefetch_block.go @@ -0,0 +1,265 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package block + +import ( + "context" + "fmt" + "io" + "sync/atomic" + "syscall" +) + +// BlockStatus represents the status of a block. +// It contains the state of the block and an error +// that may have occurred during the block's operation. +type BlockStatus struct { + State BlockState + Err error +} + +// BlockState represents the state of the block. +type BlockState int + +const ( + BlockStateInProgress BlockState = iota // Download of this block is in progress + BlockStateDownloaded // Download of this block is complete + BlockStateDownloadFailed // Download of this block has failed +) + +type PrefetchBlock interface { + Block + + // Follows io.ReaderAt interface. + // Here, off is relative to the start of the block. + ReadAt(p []byte, off int64) (n int, err error) + + // ReadAtSlice provides a way to read data from the block by returning a + // slice of the underlying buffer. The returned slice must not be modified. + // The offset is relative to the start of the block. + ReadAtSlice(off int64, size int) (p []byte, err error) + + // AbsStartOff returns the absolute start offset of the block. + // Panics if the absolute start offset is not set. + AbsStartOff() int64 + + // SetAbsStartOff sets the absolute start offset of the block. + // This should be called only once just after getting the block from the pool. + // It returns an error if the startOff is negative or if it is already set. + // TODO(princer): check if a way to set it as part of constructor. + SetAbsStartOff(startOff int64) error + + // AwaitReady waits for the block to be ready to consume. + // It returns the status of the block and an error if any. + AwaitReady(ctx context.Context) (BlockStatus, error) + + // NotifyReady is used by producer to mark the block as ready to consume. + // The value indicates the status of the block: + // - BlockStatusDownloaded: Download of this block is complete. + // - BlockStatusDownloadFailed: Download of this block has failed. + NotifyReady(val BlockStatus) + + // IncRef increments the reference count of the block. + IncRef() + + // DecRef decrements the reference count of the block. It returns true if + // the reference count reaches 0, otherwise false. Panics if the reference + // count becomes negative. + DecRef() bool + + // RefCount returns the current reference count of the block. + RefCount() int32 +} + +type prefetchMemoryBlock struct { + memoryBlock + + // Indicates if block is in progress, downloaded, download failed or download cancelled. + status BlockStatus + + // notification is a channel that notifies when the block is ready to consume. + notification chan BlockStatus + + // Stores the absolute start offset of the block-segment in the file. + absStartOff int64 + + // refCount tracks the number of active references to the block. + refCount atomic.Int32 +} + +func (pmb *prefetchMemoryBlock) Reuse() { + pmb.memoryBlock.Reuse() + + pmb.notification = make(chan BlockStatus, 1) + pmb.status = BlockStatus{State: BlockStateInProgress} + pmb.absStartOff = -1 + pmb.refCount.Store(0) +} + +// createPrefetchBlock creates a new PrefetchBlock. +func createPrefetchBlock(blockSize int64) (PrefetchBlock, error) { + prot, flags := syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_ANON|syscall.MAP_PRIVATE + addr, err := syscall.Mmap(-1, 0, int(blockSize), prot, flags) + if err != nil { + return nil, fmt.Errorf("createPrefetchBlock: Mmap: %w", err) + } + + mb := memoryBlock{ + buffer: addr[:0], + } + + pmb := prefetchMemoryBlock{ + memoryBlock: mb, + status: BlockStatus{State: BlockStateInProgress}, + notification: make(chan BlockStatus, 1), + absStartOff: -1, + } + + return &pmb, nil +} + +// ReadAt reads data from the block at the specified offset. +// The offset is relative to the start of the block. +// It returns the number of bytes read and an error if any. +func (pmb *prefetchMemoryBlock) ReadAt(p []byte, off int64) (n int, err error) { + if off < 0 || off >= pmb.Size() { + return 0, fmt.Errorf("prefetchMemoryBlock.ReadAt: offset %d is out of bounds for block size %d", off, pmb.Size()) + } + + n = copy(p, pmb.buffer[off:]) + + if n < len(p) { + return n, io.EOF + } + return n, nil +} + +// ReadAtSlice returns a slice of the underlying buffer starting at the given +// offset, which is relative to the start of the block. This allows for reading +// data without an additional copy. The returned slice must not be modified by +// the caller. +// +// If the requested size exceeds the available data from the offset, it returns +// a slice of the available data and an io.EOF error. If the offset is out of +// bounds, it returns an error. +func (pmb *prefetchMemoryBlock) ReadAtSlice(off int64, size int) ([]byte, error) { + if off < 0 || off >= pmb.Size() { + return nil, fmt.Errorf("prefetchMemoryBlock.ReadAtSlice: offset %d is out of bounds for block size %d", off, pmb.Size()) + } + + dataEnd := off + int64(size) + if dataEnd > int64(len(pmb.buffer)) { + dataEnd = int64(len(pmb.buffer)) + return pmb.buffer[off:dataEnd], io.EOF + } + + return pmb.buffer[off:dataEnd], nil +} + +func (pmb *prefetchMemoryBlock) AbsStartOff() int64 { + if pmb.absStartOff < 0 { + panic("AbsStartOff is not set, it should be set before calling this method.") + } + return pmb.absStartOff +} + +func (pmb *prefetchMemoryBlock) SetAbsStartOff(startOff int64) error { + if startOff < 0 { + return fmt.Errorf("SetAbsStartOff: negative startOff %d is not allowed", startOff) + } + + // If absStartOff is already set, then return an error. + if pmb.absStartOff >= 0 { + return fmt.Errorf("SetAbsStartOff: absStartOff is already set, re-setting is not allowed") + } + + pmb.absStartOff = startOff + return nil +} + +// AwaitReady waits for the block to be ready to consume. +// It returns the status of the block and an error if any. +func (pmb *prefetchMemoryBlock) AwaitReady(ctx context.Context) (BlockStatus, error) { + select { + case val, ok := <-pmb.notification: + if !ok { + return pmb.status, nil + } + + // First Save the last status for subsequent AwaitReady calls, and + // then close the notification channel which allows to read the last status + // without blocking. + // This is safe because NotifyReady is expected to be called only once. + pmb.status = val + close(pmb.notification) + + return pmb.status, nil + case <-ctx.Done(): + return BlockStatus{State: BlockStateInProgress}, ctx.Err() + } +} + +// NotifyReady is used by the producer to mark the block as ready to consume. +// This should be called only once to notify the consumer. +// If called multiple times, it will panic - either because of writing to the +// closed channel or blocking due to writing over full notification channel. +func (pmb *prefetchMemoryBlock) NotifyReady(val BlockStatus) { + select { + case pmb.notification <- val: + default: + panic("Expected to notify only once, but got multiple notifications.") + } +} + +func (pmb *prefetchMemoryBlock) IncRef() { + pmb.refCount.Add(1) +} + +func (pmb *prefetchMemoryBlock) DecRef() bool { + newRefCount := pmb.refCount.Add(-1) + if newRefCount < 0 { + panic("DecRef called more times than IncRef, resulting in a negative refCount.") + } + return newRefCount == 0 +} + +func (pmb *prefetchMemoryBlock) RefCount() int32 { + return pmb.refCount.Load() +} + +// ReadFrom implements io.ReaderFrom on prefetch blocks +// to efficiently read from reader avoiding an intermediate buffer. +func (pmb *prefetchMemoryBlock) ReadFrom(r io.Reader) (n int64, err error) { + var bytesRead int + for { + // return if buffer is full. + if len(pmb.buffer) == cap(pmb.buffer) { + return n, nil + } + + // Read into the remaining capacity of the buffer. + bytesRead, err = r.Read(pmb.buffer[len(pmb.buffer):cap(pmb.buffer)]) + if bytesRead > 0 { + pmb.buffer = pmb.buffer[:len(pmb.buffer)+bytesRead] + n += int64(bytesRead) + } + if err != nil { + if err == io.EOF { + return n, nil // End of reader + } + return n, err + } + } +} diff --git a/internal/block/prefetch_block_test.go b/internal/block/prefetch_block_test.go new file mode 100644 index 0000000000..53625d0421 --- /dev/null +++ b/internal/block/prefetch_block_test.go @@ -0,0 +1,469 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package block + +import ( + "bytes" + "context" + "io" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +// const outOfCapacityError string = "received data more than capacity of the block" + +type PrefetchMemoryBlockTest struct { + MemoryBlockTest +} + +func TestPrefetchMemoryBlockTestSuite(t *testing.T) { + suite.Run(t, new(PrefetchMemoryBlockTest)) +} + +func (testSuite *PrefetchMemoryBlockTest) TestPrefetchMemoryBlockReuse() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + content := []byte("hi") + n, err := pmb.Write(content) + require.Nil(testSuite.T(), err) + require.Equal(testSuite.T(), 2, n) + output, err := io.ReadAll(pmb) + require.Nil(testSuite.T(), err) + require.Equal(testSuite.T(), content, output) + require.Equal(testSuite.T(), int64(2), pmb.Size()) + err = pmb.SetAbsStartOff(23) + require.Nil(testSuite.T(), err) + pmb.IncRef() + assert.Equal(testSuite.T(), int32(1), pmb.RefCount()) + require.Nil(testSuite.T(), err) + + pmb.Reuse() + + assert.Equal(testSuite.T(), int64(0), pmb.(*prefetchMemoryBlock).readSeek) + output, err = io.ReadAll(pmb) + assert.Nil(testSuite.T(), err) + assert.Empty(testSuite.T(), output) + assert.Equal(testSuite.T(), int64(0), pmb.Size()) + assert.Panics(testSuite.T(), func() { + _ = pmb.AbsStartOff() + }) + assert.Equal(testSuite.T(), int32(0), pmb.RefCount()) +} + +func (testSuite *PrefetchMemoryBlockTest) TestPrefetchMemoryBlockReadAtSuccess() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + content := []byte("hello world") + _, err = pmb.Write(content) + require.Nil(testSuite.T(), err) + readBuffer := make([]byte, 5) + + n, err := pmb.ReadAt(readBuffer, 6) // Read "world" + + assert.Nil(testSuite.T(), err) + assert.Equal(testSuite.T(), 5, n) + assert.Equal(testSuite.T(), []byte("world"), readBuffer) +} + +func (testSuite *PrefetchMemoryBlockTest) TestPrefetchMemoryBlockReadAtBeyondBlockSize() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + content := []byte("hello world") + _, err = pmb.Write(content) + require.Nil(testSuite.T(), err) + readBuffer := make([]byte, 5) + + n, err := pmb.ReadAt(readBuffer, 15) // Read beyond the block size + + assert.NotNil(testSuite.T(), err) + assert.NotErrorIs(testSuite.T(), err, io.EOF) + assert.Equal(testSuite.T(), 0, n) +} + +func (testSuite *PrefetchMemoryBlockTest) TestPrefetchMemoryBlockReadAtWithNegativeOffset() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + content := []byte("hello world") + _, err = pmb.Write(content) + require.Nil(testSuite.T(), err) + readBuffer := make([]byte, 5) + + n, err := pmb.ReadAt(readBuffer, -1) // Negative offset + + assert.NotNil(testSuite.T(), err) + assert.NotErrorIs(testSuite.T(), err, io.EOF) + assert.Equal(testSuite.T(), 0, n) +} + +func (testSuite *PrefetchMemoryBlockTest) TestPrefetchMemoryBlockReadAtEOF() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + content := []byte("hello world") + _, err = pmb.Write(content) + require.Nil(testSuite.T(), err) + readBuffer := make([]byte, 15) + + n, err := pmb.ReadAt(readBuffer, 6) // Read "world" + + assert.Equal(testSuite.T(), io.EOF, err) + assert.Equal(testSuite.T(), 5, n) + assert.Equal(testSuite.T(), []byte("world"), readBuffer[0:n]) +} + +func (testSuite *PrefetchMemoryBlockTest) TestPrefetchMemoryBlockReadAtSliceSuccess() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + content := []byte("hello world") + _, err = pmb.Write(content) + require.Nil(testSuite.T(), err) + + slice, err := pmb.ReadAtSlice(6, 5) // Read "world" + + assert.NoError(testSuite.T(), err) + assert.Equal(testSuite.T(), []byte("world"), slice) +} + +func (testSuite *PrefetchMemoryBlockTest) TestPrefetchMemoryBlockReadAtSliceEOF() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + content := []byte("hello world") + _, err = pmb.Write(content) + require.Nil(testSuite.T(), err) + + slice, err := pmb.ReadAtSlice(6, 15) // Read "world" and beyond + + assert.Equal(testSuite.T(), io.EOF, err) + assert.Equal(testSuite.T(), []byte("world"), slice) +} + +func (testSuite *PrefetchMemoryBlockTest) TestPrefetchMemoryBlockReadAtSliceWithNegativeOffset() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + _, err = pmb.Write([]byte("hello world")) + require.Nil(testSuite.T(), err) + + _, err = pmb.ReadAtSlice(-1, 5) + + assert.Error(testSuite.T(), err) +} + +func (testSuite *PrefetchMemoryBlockTest) TestPrefetchMemoryBlockReadAtSliceWithOffsetOutOfBounds() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + _, err = pmb.Write([]byte("hello")) + require.Nil(testSuite.T(), err) + + _, err = pmb.ReadAtSlice(5, 1) // Offset is equal to size, which is out of bounds + + assert.Error(testSuite.T(), err) +} + +func (testSuite *PrefetchMemoryBlockTest) TestPrefetchMemoryBlockAbsStartOffsetPanicsOnEmptyBlock() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + + // The absolute start offset should be -1 initially. + assert.Panics(testSuite.T(), func() { + _ = pmb.AbsStartOff() + }) +} + +func (testSuite *PrefetchMemoryBlockTest) TestPrefetchMemoryBlockAbsStartOffsetValid() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + + // Set the absolute start offset to a valid value. + pmb.(*prefetchMemoryBlock).absStartOff = 100 + + // The absolute start offset should return the set value. + assert.Equal(testSuite.T(), int64(100), pmb.AbsStartOff()) +} + +func (testSuite *PrefetchMemoryBlockTest) TestPrefetchMemoryBlockSetAbsStartOffsetInvalid() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + + err = pmb.SetAbsStartOff(-23) + + assert.Error(testSuite.T(), err) +} + +func (testSuite *PrefetchMemoryBlockTest) TestPrefetchMemoryBlockSetAbsStartOffsetValid() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + + err = pmb.SetAbsStartOff(23) + + assert.NoError(testSuite.T(), err) + assert.Equal(testSuite.T(), int64(23), pmb.AbsStartOff()) +} + +func (testSuite *PrefetchMemoryBlockTest) TestPrefetchMemoryBlockSetAbsStartOffsetTwiceInvalid() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + err = pmb.SetAbsStartOff(23) + require.Nil(testSuite.T(), err) + + err = pmb.SetAbsStartOff(42) + + assert.Error(testSuite.T(), err) +} + +func (testSuite *PrefetchMemoryBlockTest) TestAwaitReadyWaitIfNotNotify() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + ctx, cancel := context.WithTimeout(testSuite.T().Context(), 100*time.Millisecond) + defer cancel() + + _, err = pmb.AwaitReady(ctx) + + assert.NotNil(testSuite.T(), err) + assert.EqualError(testSuite.T(), context.DeadlineExceeded, err.Error()) +} + +func (testSuite *PrefetchMemoryBlockTest) TestAwaitReadyReturnsErrorOnContextCancellation() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + ctx, cancel := context.WithCancel(testSuite.T().Context()) + cancel() // Cancel the context immediately + + _, err = pmb.AwaitReady(ctx) + + require.NotNil(testSuite.T(), err) + assert.EqualError(testSuite.T(), context.Canceled, err.Error()) +} + +func (testSuite *PrefetchMemoryBlockTest) TestAwaitReadyNotifyVariants() { + tests := []struct { + name string + notifyStatus BlockStatus + wantStatus BlockStatus + }{ + { + name: "AfterNotifySuccess", + notifyStatus: BlockStatus{State: BlockStateDownloaded}, + wantStatus: BlockStatus{State: BlockStateDownloaded}, + }, + { + name: "AfterNotifyError", + notifyStatus: BlockStatus{State: BlockStateDownloadFailed}, + wantStatus: BlockStatus{State: BlockStateDownloadFailed}, + }, + } + + for _, tt := range tests { + testSuite.T().Run(tt.name, func(t *testing.T) { + pmb, err := createPrefetchBlock(12) + require.Nil(t, err) + go func() { + time.Sleep(time.Millisecond) + pmb.NotifyReady(tt.notifyStatus) + }() + + status, err := pmb.AwaitReady(t.Context()) + + require.Nil(t, err) + assert.Equal(t, tt.wantStatus, status) + }) + } +} + +func (testSuite *PrefetchMemoryBlockTest) TestTwoNotifyReadyWithoutAwaitReady() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + + pmb.NotifyReady(BlockStatus{State: BlockStateDownloaded}) + // 2nd notify will lead to panic since it is not allowed to notify a block more than once. + assert.Panics(testSuite.T(), func() { + pmb.NotifyReady(BlockStatus{State: BlockStateDownloaded}) + }) +} + +func (testSuite *PrefetchMemoryBlockTest) TestNotifyReadyAfterAwaitReady() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + go func() { + pmb.NotifyReady(BlockStatus{State: BlockStateDownloaded}) + }() + status, err := pmb.AwaitReady(testSuite.T().Context()) + require.Nil(testSuite.T(), err) + assert.Equal(testSuite.T(), BlockStatus{State: BlockStateDownloaded}, status) + + // 2nd notify will lead to panic since channel is closed after first await ready. + assert.Panics(testSuite.T(), func() { + pmb.NotifyReady(BlockStatus{State: BlockStateDownloaded}) + }) +} + +func (testSuite *PrefetchMemoryBlockTest) TestSingleNotifyAndMultipleAwaitReady() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + go func() { + pmb.NotifyReady(BlockStatus{State: BlockStateDownloaded}) + }() + var wg sync.WaitGroup + wg.Add(5) + + // Multiple goroutines waiting for the same block to be ready. + // They should all receive the same status once the block is notified. + for range 5 { + go func() { + defer wg.Done() + + status, err := pmb.AwaitReady(testSuite.T().Context()) + + require.Nil(testSuite.T(), err) + assert.Equal(testSuite.T(), BlockStatus{State: BlockStateDownloaded}, status) + }() + } + wg.Wait() +} + +func (testSuite *PrefetchMemoryBlockTest) TestPrefetchMemoryBlockIncRef() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + + pmb.IncRef() + + assert.Equal(testSuite.T(), int32(1), pmb.RefCount()) +} + +func (testSuite *PrefetchMemoryBlockTest) TestPrefetchMemoryBlockDecRef() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + pmb.IncRef() + pmb.IncRef() + + isZero := pmb.DecRef() + + assert.False(testSuite.T(), isZero) + assert.Equal(testSuite.T(), int32(1), pmb.RefCount()) + + isZero = pmb.DecRef() + + assert.True(testSuite.T(), isZero) + assert.Equal(testSuite.T(), int32(0), pmb.RefCount()) +} + +func (testSuite *PrefetchMemoryBlockTest) TestPrefetchMemoryBlockDecRefPanics() { + pmb, err := createPrefetchBlock(12) + require.Nil(testSuite.T(), err) + + assert.PanicsWithValue(testSuite.T(), "DecRef called more times than IncRef, resulting in a negative refCount.", func() { + pmb.DecRef() + }) +} + +// errorReader is a reader that returns an error. +type errorReader struct { + err error +} + +func (r *errorReader) Read(p []byte) (n int, err error) { + return 0, r.err +} + +func (testSuite *PrefetchMemoryBlockTest) TestReadFrom_ReaderHasLessDataThanBufferCapacity() { + pmb, err := createPrefetchBlock(10) + require.NoError(testSuite.T(), err) + content := "hello" + + n, err := pmb.(*prefetchMemoryBlock).ReadFrom(bytes.NewReader([]byte(content))) + + assert.NoError(testSuite.T(), err) + assert.Equal(testSuite.T(), int64(len(content)), n) + assert.Equal(testSuite.T(), []byte(content), pmb.(*prefetchMemoryBlock).buffer) +} + +func (testSuite *PrefetchMemoryBlockTest) TestReadFrom_ReaderHasMoreDataThanBufferCapacity() { + pmb, err := createPrefetchBlock(5) + require.NoError(testSuite.T(), err) + content := "helloworld" + + _, err = pmb.(*prefetchMemoryBlock).ReadFrom(bytes.NewReader([]byte(content))) + + assert.NoError(testSuite.T(), err) + assert.Equal(testSuite.T(), []byte(content[:5]), pmb.(*prefetchMemoryBlock).buffer) +} + +func (testSuite *PrefetchMemoryBlockTest) TestReadFrom_ReaderIsEmpty() { + pmb, err := createPrefetchBlock(10) + require.NoError(testSuite.T(), err) + + n, err := pmb.(*prefetchMemoryBlock).ReadFrom(bytes.NewReader([]byte{})) + + assert.NoError(testSuite.T(), err) + assert.Equal(testSuite.T(), int64(0), n) + assert.Equal(testSuite.T(), []byte{}, pmb.(*prefetchMemoryBlock).buffer) +} + +func (testSuite *PrefetchMemoryBlockTest) TestReadFrom_ReaderReturnsError() { + pmb, err := createPrefetchBlock(10) + require.NoError(testSuite.T(), err) + + n, err := pmb.(*prefetchMemoryBlock).ReadFrom(&errorReader{err: assert.AnError}) + + assert.ErrorIs(testSuite.T(), err, assert.AnError) + assert.Equal(testSuite.T(), int64(0), n) + assert.Equal(testSuite.T(), []byte{}, pmb.(*prefetchMemoryBlock).buffer) +} + +func (testSuite *PrefetchMemoryBlockTest) TestReadFrom_BufferIsAlreadyFull() { + pmb, err := createPrefetchBlock(5) + require.NoError(testSuite.T(), err) + initialContent := "abcde" + _, err = pmb.Write([]byte(initialContent)) + require.NoError(testSuite.T(), err) + readerContent := "fgh" + + _, err = pmb.(*prefetchMemoryBlock).ReadFrom(bytes.NewReader([]byte(readerContent))) + + assert.NoError(testSuite.T(), err) + assert.Equal(testSuite.T(), []byte(initialContent), pmb.(*prefetchMemoryBlock).buffer) +} + +func (testSuite *PrefetchMemoryBlockTest) TestReadFrom_BufferIsPartiallyFull() { + pmb, err := createPrefetchBlock(10) + require.NoError(testSuite.T(), err) + initialContent := "abcde" + _, err = pmb.Write([]byte(initialContent)) + require.NoError(testSuite.T(), err) + readerContent := "fghij" + + n, err := pmb.(*prefetchMemoryBlock).ReadFrom(bytes.NewReader([]byte(readerContent))) + + assert.NoError(testSuite.T(), err) + assert.Equal(testSuite.T(), int64(len(readerContent)), n) + assert.Equal(testSuite.T(), []byte(initialContent+readerContent), pmb.(*prefetchMemoryBlock).buffer) +} + +func (testSuite *PrefetchMemoryBlockTest) TestReadFrom_BufferIsPartiallyFullAndReaderHasMoreData() { + pmb, err := createPrefetchBlock(10) + require.NoError(testSuite.T(), err) + initialContent := "abc" + _, err = pmb.Write([]byte(initialContent)) + require.NoError(testSuite.T(), err) + readerContent := "defghijkl" + + _, err = pmb.(*prefetchMemoryBlock).ReadFrom(bytes.NewReader([]byte(readerContent))) + + assert.NoError(testSuite.T(), err) + assert.Equal(testSuite.T(), []byte(initialContent+readerContent[:7]), pmb.(*prefetchMemoryBlock).buffer) +} diff --git a/internal/bufferedread/block_queue_entry.go b/internal/bufferedread/block_queue_entry.go new file mode 100644 index 0000000000..e798bdf352 --- /dev/null +++ b/internal/bufferedread/block_queue_entry.go @@ -0,0 +1,51 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufferedread + +import ( + "context" + "errors" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/block" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" +) + +// blockQueueEntry holds a data block with a function +// to cancel its in-flight download. +type blockQueueEntry struct { + block block.PrefetchBlock + cancel context.CancelFunc + + // wasEvicted is true if the block has been removed from the block queue but + // still has outstanding references. + wasEvicted bool +} + +// cancelAndWait cancels the download context for the entry and waits for the +// download goroutine to finish. It logs a warning if the download terminates +// with an error other than context.Canceled. +func (bqe *blockQueueEntry) cancelAndWait() { + bqe.cancel() + // We wait for the block's worker goroutine to finish. We expect its + // status to contain a context.Canceled error because we just called cancel. + status, err := bqe.block.AwaitReady(context.Background()) + if err != nil { + logger.Warnf("cancelAndWait: AwaitReady for block starting at %d failed: %v", + bqe.block.AbsStartOff(), err) + } else if status.Err != nil && !errors.Is(status.Err, context.Canceled) { + logger.Warnf("cancelAndWait: block starting at %d terminated with an unexpected error: %v", + bqe.block.AbsStartOff(), status.Err) + } +} diff --git a/internal/bufferedread/buffered_reader.go b/internal/bufferedread/buffered_reader.go new file mode 100644 index 0000000000..f0388ac3a7 --- /dev/null +++ b/internal/bufferedread/buffered_reader.go @@ -0,0 +1,665 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufferedread + +import ( + "context" + "errors" + "fmt" + "io" + "sync" + "time" + + "github.com/google/uuid" + "github.com/googlecloudplatform/gcsfuse/v3/common" + "github.com/googlecloudplatform/gcsfuse/v3/internal/block" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/workerpool" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/jacobsa/fuse/fuseops" + "golang.org/x/sync/semaphore" +) + +// ErrPrefetchBlockNotAvailable is returned when a block cannot be +// acquired from the pool for prefetching. This can be used by callers to +// implement a fallback mechanism, e.g. falling back to another reader. +var ErrPrefetchBlockNotAvailable = errors.New("block for prefetching not available") + +type BufferedReadConfig struct { + MaxPrefetchBlockCnt int64 // Maximum number of blocks that can be prefetched. + PrefetchBlockSizeBytes int64 // Size of each block to be prefetched. + InitialPrefetchBlockCnt int64 // Number of blocks to prefetch initially. + MinBlocksPerHandle int64 // Minimum number of blocks available in block-pool to start buffered-read. + RandomSeekThreshold int64 // Seek count threshold to switch another reader +} + +const ( + defaultPrefetchMultiplier = 2 +) + +type BufferedReader struct { + gcsx.Reader + object *gcs.MinObject + bucket gcs.Bucket + config *BufferedReadConfig + + // nextBlockIndexToPrefetch is the index of the next block to be + // prefetched. + nextBlockIndexToPrefetch int64 + + // randomSeekCount is the number of random seeks performed. This is used to + // detect if the read pattern is random and fall back to another reader. + randomSeekCount int64 + + // numPrefetchBlocks is the number of blocks to prefetch in the next + // prefetching operation. + numPrefetchBlocks int64 + + metricHandle metrics.MetricHandle + + traceHandle tracing.TraceHandle + + // handleID is the file handle id, used for logging. + handleID fuseops.HandleID + + readHandle []byte // For zonal bucket. + + ctx context.Context + cancelFunc context.CancelFunc + + prefetchMultiplier int64 // Multiplier for number of blocks to prefetch. + + randomReadsThreshold int64 // Number of random reads after which the reader falls back to another reader. + + // `mu` synchronizes access to the buffered reader's shared state. + // All shared variables, such as the block pool and queue, require this lock before any operation. + mu sync.Mutex + + // GUARDED by (mu) + workerPool workerpool.WorkerPool + + // blockQueue is the core of the prefetching pipeline, holding blocks that are + // either downloaded or in the process of being downloaded. + // GUARDED by (mu) + blockQueue common.Queue[*blockQueueEntry] + + // blockPool is a pool of blocks that can be reused for prefetching. + // It is used to avoid allocating new blocks for each prefetch operation. + // The pool is initialized with a maximum number of blocks that can be + // prefetched at a time, and it allows for efficient reuse of blocks. + // The pool is also responsible for managing the global limit on the number + // of blocks that can be allocated across all BufferedReader instances. + // GUARDED by (mu) + blockPool *block.GenBlockPool[block.PrefetchBlock] + + // A WaitGroup to synchronize the destruction of the reader with any ongoing + // FUSE read callback goroutines. This ensures that all callbacks for + // in-flight data slices have completed before the reader is fully torn down. + inflightCallbackWg sync.WaitGroup + + // readTypeClassifier tracks the read access pattern (e.g., sequential, random) + // to optimize read strategies. It is shared across different reader layers. + readTypeClassifier *gcsx.ReadTypeClassifier +} + +// BufferedReaderOptions holds the dependencies for a BufferedReader. +type BufferedReaderOptions struct { + Object *gcs.MinObject + Bucket gcs.Bucket + Config *BufferedReadConfig + GlobalMaxBlocksSem *semaphore.Weighted + WorkerPool workerpool.WorkerPool + MetricHandle metrics.MetricHandle + TraceHandle tracing.TraceHandle + ReadTypeClassifier *gcsx.ReadTypeClassifier + HandleID fuseops.HandleID +} + +// NewBufferedReader returns a new bufferedReader instance. +func NewBufferedReader(opts *BufferedReaderOptions) (*BufferedReader, error) { + if opts.Config.PrefetchBlockSizeBytes <= 0 { + return nil, fmt.Errorf("NewBufferedReader: PrefetchBlockSizeBytes must be positive, but is %d", opts.Config.PrefetchBlockSizeBytes) + } + // To optimize resource usage, reserve only the number of blocks required for + // the file, capped by the configured minimum. + blocksInFile := (int64(opts.Object.Size) + opts.Config.PrefetchBlockSizeBytes - 1) / opts.Config.PrefetchBlockSizeBytes + numBlocksToReserve := min(blocksInFile, opts.Config.MinBlocksPerHandle) + blockpool, err := block.NewPrefetchBlockPool(opts.Config.PrefetchBlockSizeBytes, opts.Config.MaxPrefetchBlockCnt, numBlocksToReserve, opts.GlobalMaxBlocksSem) + if err != nil { + if errors.Is(err, block.CantAllocateAnyBlockError) { + opts.MetricHandle.BufferedReadFallbackTriggerCount(1, "insufficient_memory") + } + return nil, fmt.Errorf("NewBufferedReader: creating block-pool: %w", err) + } + + reader := &BufferedReader{ + object: opts.Object, + bucket: opts.Bucket, + config: opts.Config, + nextBlockIndexToPrefetch: 0, + randomSeekCount: 0, + numPrefetchBlocks: opts.Config.InitialPrefetchBlockCnt, + blockQueue: common.NewLinkedListQueue[*blockQueueEntry](), + blockPool: blockpool, + workerPool: opts.WorkerPool, + metricHandle: opts.MetricHandle, + traceHandle: opts.TraceHandle, + handleID: opts.HandleID, + prefetchMultiplier: defaultPrefetchMultiplier, + randomReadsThreshold: opts.Config.RandomSeekThreshold, + readTypeClassifier: opts.ReadTypeClassifier, + } + + reader.ctx, reader.cancelFunc = context.WithCancel(context.Background()) + return reader, nil +} + +func (p *BufferedReader) ReaderName() string { + return "buffered_reader" +} + +// handleRandomRead detects and handles random read patterns. A read is considered +// random if the requested offset is outside the currently prefetched window. +// If the number of detected random reads exceeds a configured threshold, it +// returns a gcsx.FallbackToAnotherReader error to signal that another reader +// should be used. If the read pattern changes back to sequential, it resets +// the reader state to resume buffered reading. +// LOCKS_REQUIRED(p.mu) +func (p *BufferedReader) handleRandomRead(offset int64) error { + // Exit early if we have already decided to fall back to another reader. + // This avoids re-evaluating the read pattern on every call when the random + // read threshold has been met. + if p.randomSeekCount > p.randomReadsThreshold { + if p.readTypeClassifier.IsReadSequential() { + logger.Tracef("Restarting buffered reader due to sequential read pattern detected for object %q, handle %d", p.object.Name, p.handleID) + p.resetBufferedReaderState() + return nil + } + return gcsx.FallbackToAnotherReader + } + + if !p.isRandomSeek(offset) { + return nil + } + + p.randomSeekCount++ + + // When a random seek is detected, the prefetched blocks in the queue become + // irrelevant. We must clear the queue, cancel any ongoing downloads, and + // release the blocks back to the pool. + for !p.blockQueue.IsEmpty() { + entry := p.blockQueue.Pop() + entry.cancelAndWait() + p.releaseOrMarkEvicted(entry) + } + + if p.randomSeekCount > p.randomReadsThreshold { + // If the read pattern becomes sequential again, reset the state to resume buffered reading. + if p.readTypeClassifier.IsReadSequential() { + logger.Tracef("Restarting buffered reader due to sequential read pattern detected for object %q, handle %d", p.object.Name, p.handleID) + p.resetBufferedReaderState() + return nil + } + logger.Tracef("Fallback to another reader for object %q, handle %d. Random seek count %d exceeded threshold %d and read pattern is not sequential.", p.object.Name, p.handleID, p.randomSeekCount, p.randomReadsThreshold) + p.metricHandle.BufferedReadFallbackTriggerCount(1, "random_read_detected") + return gcsx.FallbackToAnotherReader + } + + return nil +} + +// isRandomSeek checks if the read for the given offset is random or not. +// LOCKS_REQUIRED(p.mu) +func (p *BufferedReader) isRandomSeek(offset int64) bool { + if p.blockQueue.IsEmpty() { + return offset != 0 + } + + start := p.blockQueue.Peek().block.AbsStartOff() + end := start + int64(p.blockQueue.Len())*p.config.PrefetchBlockSizeBytes + if offset < start || offset >= end { + return true + } + + return false +} + +// prepareQueueForOffset cleans the head of the block queue by discarding any +// blocks that are no longer relevant for the given read offset. This occurs on +// seeks (both forward and backward) that land outside the current block. +// For each discarded block, its download is cancelled, and it is returned to +// the block pool. +// LOCKS_REQUIRED(p.mu) +func (p *BufferedReader) prepareQueueForOffset(offset int64) { + for !p.blockQueue.IsEmpty() { + entry := p.blockQueue.Peek() + block := entry.block + blockStart := block.AbsStartOff() + blockEnd := blockStart + block.Cap() + + if offset < blockStart || offset >= blockEnd { + // Offset is either before or beyond this block – discard. + p.blockQueue.Pop() + entry.cancelAndWait() + p.releaseOrMarkEvicted(entry) + } else { + break + } + } +} + +// ReadAt reads data from the GCS object into the provided buffer starting at +// the given offset. It implements the gcsx.Reader interface. +// +// The read is satisfied by reading from in-memory blocks that are prefetched +// in the background. The core logic is as follows: +// 1. Detect if the read pattern is random. If so, and if the random read +// threshold is exceeded, it returns a FallbackToAnotherReader error. +// 2. Prepare the internal block queue by discarding any stale blocks from the +// head of the queue that are before the requested offset. +// 3. If the queue becomes empty (e.g., on a fresh read or a large seek), it +// initiates a "fresh start" to prefetch blocks starting from the current +// offset. +// 4. It then enters a loop to fill the destination buffer: +// a. It waits for the block at the head of the queue to be downloaded. +// b. If the download failed or was cancelled, it returns an appropriate error. +// c. If successful, it copies data from the downloaded block into the buffer. +// d. If a block is fully consumed, it is removed from the queue, and a new +// prefetch operation is triggered to keep the pipeline full. +// 5. The loop continues until the buffer is full, the end of the file is +// reached, or an error occurs. +// +// LOCKS_EXCLUDED(p.mu) +func (p *BufferedReader) ReadAt(ctx context.Context, req *gcsx.ReadRequest) (resp gcsx.ReadResponse, err error) { + reqID := uuid.New() + start := time.Now() + readOffset := req.Offset + blockIdx := readOffset / p.config.PrefetchBlockSizeBytes + var bytesRead int + + logger.Tracef("%.13v <- ReadAt(%s:/%s, %d, %d, %d, %d)", reqID, p.bucket.Name(), p.object.Name, p.handleID, readOffset, len(req.Buffer), blockIdx) + + if readOffset >= int64(p.object.Size) { + err = io.EOF + return resp, err + } + + if len(req.Buffer) == 0 { + return resp, nil + } + + p.mu.Lock() + defer p.mu.Unlock() + + var dataSlices [][]byte + var entriesToCallback []*blockQueueEntry + defer func() { + dur := time.Since(start) + p.metricHandle.BufferedReadReadLatency(ctx, dur) + p.metricHandle.GcsReadBytesCount(int64(bytesRead)) + + if err == nil || errors.Is(err, io.EOF) { + logger.Tracef("%.13v -> ReadAt(): Ok(%v)", reqID, dur) + // Setting the return response. + resp.Data = dataSlices + resp.Callback = func() { p.callback(entriesToCallback) } + resp.Size = bytesRead + } else if errors.Is(err, gcsx.FallbackToAnotherReader) { + // When falling back, we must immediately release the blocks we've acquired references to. + p.releaseInflightBlocks(entriesToCallback) + resp = gcsx.ReadResponse{} + } + }() + + if err = p.handleRandomRead(readOffset); err != nil { + err = fmt.Errorf("BufferedReader.ReadAt: handleRandomRead: %w", err) + return + } + + prefetchTriggered := false + for bytesRead < len(req.Buffer) { + p.prepareQueueForOffset(readOffset) + + if p.blockQueue.IsEmpty() { + if err = p.freshStart(readOffset); err != nil { + logger.Warnf("Fallback to another reader for object %q, handle %d, due to freshStart failure: %v", p.object.Name, p.handleID, err) + p.metricHandle.BufferedReadFallbackTriggerCount(1, "insufficient_memory") + err = gcsx.FallbackToAnotherReader + return + } + prefetchTriggered = true + } + + entry := p.blockQueue.Peek() + blk := entry.block + + status, waitErr := blk.AwaitReady(ctx) + if waitErr != nil { + err = fmt.Errorf("BufferedReader.ReadAt: AwaitReady: %w", waitErr) + break + } + + if status.State != block.BlockStateDownloaded { + p.blockQueue.Pop() + p.blockPool.Release(blk) + entry.cancel() + + switch status.State { + case block.BlockStateDownloadFailed: + err = fmt.Errorf("BufferedReader.ReadAt: download failed: %w", status.Err) + default: + err = fmt.Errorf("BufferedReader.ReadAt: unexpected block state: %d", status.State) + } + break + } + + relOff := readOffset - blk.AbsStartOff() + bytesToRead := len(req.Buffer) - bytesRead + dataSlice, readErr := blk.ReadAtSlice(relOff, bytesToRead) + sliceLen := len(dataSlice) + bytesRead += sliceLen + readOffset += int64(sliceLen) + + if readErr != nil && !errors.Is(readErr, io.EOF) { + err = fmt.Errorf("BufferedReader.ReadAt: block.ReadAt: %w", readErr) + break + } + + if sliceLen > 0 { + dataSlices = append(dataSlices, dataSlice) + p.inflightCallbackWg.Add(1) + blk.IncRef() + entriesToCallback = append(entriesToCallback, entry) + } + + if readOffset >= int64(p.object.Size) { + break + } + + if readOffset >= blk.AbsStartOff()+blk.Size() { + entry := p.blockQueue.Pop() + p.releaseOrMarkEvicted(entry) + + if !prefetchTriggered { + prefetchTriggered = true + if pfErr := p.prefetch(); pfErr != nil { + logger.Warnf("BufferedReader.ReadAt: while prefetching: %v", pfErr) + } + } + } + } + + return +} + +// releaseInflightBlocks immediately invokes the callback for a list of block +// entries and waits for them to complete. This is used when a read operation +// must fall back to another reader, ensuring that any blocks referenced during +// the failed attempt are properly released before proceeding. +// LOCKS_REQUIRED(p.mu) +func (p *BufferedReader) releaseInflightBlocks(entries []*blockQueueEntry) { + p.mu.Unlock() + defer p.mu.Lock() + + p.callback(entries) +} + +// callback is called when the FUSE library is finished with buffer slices that +// were returned directly from blocks. It decrements the reference count for each +// associated block and releases it back to the pool if the count drops to zero +// and it was previously marked for eviction. +func (p *BufferedReader) callback(entries []*blockQueueEntry) { + defer func() { + for range entries { + p.inflightCallbackWg.Done() + } + }() + p.mu.Lock() + defer p.mu.Unlock() + for _, entry := range entries { + if entry.block.DecRef() && entry.wasEvicted { + if p.blockPool != nil { + p.blockPool.Release(entry.block) + } + } + } +} + +// prefetch schedules the next set of blocks for prefetching starting from +// the nextBlockIndexToPrefetch. +// LOCKS_REQUIRED(p.mu) +func (p *BufferedReader) prefetch() error { + // Determine the number of blocks to prefetch in this cycle, respecting the + // MaxPrefetchBlockCnt and the number of blocks remaining in the file. + availableSlots := p.config.MaxPrefetchBlockCnt - int64(p.blockQueue.Len()) + if availableSlots <= 0 { + return nil + } + totalBlockCount := (int64(p.object.Size) + p.config.PrefetchBlockSizeBytes - 1) / p.config.PrefetchBlockSizeBytes + remainingBlocksInFile := totalBlockCount - p.nextBlockIndexToPrefetch + blockCountToPrefetch := min(min(p.numPrefetchBlocks, availableSlots), remainingBlocksInFile) + if blockCountToPrefetch <= 0 { + return nil + } + + allBlocksScheduledSuccessfully := true + for range blockCountToPrefetch { + if err := p.scheduleNextBlock(false); err != nil { + if errors.Is(err, ErrPrefetchBlockNotAvailable) { + // This is not a critical error for a background prefetch. We just stop + // trying to prefetch more in this cycle. The specific reason has + // already been logged by scheduleNextBlock. + allBlocksScheduledSuccessfully = false + break // Stop prefetching more blocks. + } + return fmt.Errorf("prefetch: scheduling block index %d: %w", p.nextBlockIndexToPrefetch, err) + } + } + + // Only increase the prefetch window size if we successfully scheduled all the + // intended blocks. This is a more conservative approach that prevents the + // window from growing aggressively if block pool is consistently under pressure. + if allBlocksScheduledSuccessfully { + // Set the size for the next multiplicative prefetch. + p.numPrefetchBlocks *= p.prefetchMultiplier + + // Cap the prefetch window size for the next cycle at the configured + // maximum to prevent unbounded growth. + if p.numPrefetchBlocks > p.config.MaxPrefetchBlockCnt { + p.numPrefetchBlocks = p.config.MaxPrefetchBlockCnt + } + } + return nil +} + +// freshStart resets the prefetching state and schedules the initial set of +// blocks starting from the given offset. +// LOCKS_REQUIRED(p.mu) +func (p *BufferedReader) freshStart(currentOffset int64) error { + blockIndex := currentOffset / p.config.PrefetchBlockSizeBytes + p.nextBlockIndexToPrefetch = blockIndex + + // Determine the number of blocks for the initial prefetch. + p.numPrefetchBlocks = min(p.config.InitialPrefetchBlockCnt, p.config.MaxPrefetchBlockCnt) + + // Schedule the first block as urgent. + if err := p.scheduleNextBlock(true); err != nil { + return fmt.Errorf("freshStart: scheduling first block: %w", err) + } + + // Prefetch the initial blocks. + if err := p.prefetch(); err != nil { + // A failure during the initial prefetch is not fatal, as the first block + // has already been scheduled. Log the error and continue. + logger.Warnf("freshStart: initial prefetch: %v", err) + } + return nil +} + +// scheduleNextBlock schedules the next block for prefetch. +// LOCKS_REQUIRED(p.mu) +func (p *BufferedReader) scheduleNextBlock(urgent bool) error { + b, err := p.blockPool.TryGet() + if err != nil { + // Any error from TryGet (e.g., pool exhausted, mmap failure) means we + // can't get a block. For the buffered reader, this is a recoverable + // condition that should either trigger a fallback to another reader (for + // urgent reads) or be ignored (for background prefetches). + logger.Tracef("scheduleNextBlock: could not get block from pool (urgent=%t): %v", urgent, err) + return ErrPrefetchBlockNotAvailable + } + + if err := p.scheduleBlockWithIndex(b, p.nextBlockIndexToPrefetch, urgent); err != nil { + p.blockPool.Release(b) + return fmt.Errorf("scheduleNextBlock: %w", err) + } + p.nextBlockIndexToPrefetch++ + return nil +} + +// scheduleBlockWithIndex schedules a block with a specific index. +// LOCKS_REQUIRED(p.mu) +func (p *BufferedReader) scheduleBlockWithIndex(b block.PrefetchBlock, blockIndex int64, urgent bool) error { + startOffset := blockIndex * p.config.PrefetchBlockSizeBytes + if err := b.SetAbsStartOff(startOffset); err != nil { + return fmt.Errorf("scheduleBlockWithIndex: setting start offset: %w", err) + } + + ctx, cancel := context.WithCancel(p.ctx) + task := &downloadTask{ + ctx: ctx, + object: p.object, + bucket: p.bucket, + block: b, + readHandle: p.readHandle, + metricHandle: p.metricHandle, + } + + logger.Tracef("Scheduling block: (%s, %d, %t).", p.object.Name, blockIndex, urgent) + p.blockQueue.Push(&blockQueueEntry{ + block: b, + cancel: cancel, + }) + p.workerPool.Schedule(urgent, task) + return nil +} + +// LOCKS_EXCLUDED(p.mu) +func (p *BufferedReader) Destroy() { + p.mu.Lock() + for !p.blockQueue.IsEmpty() { + bqe := p.blockQueue.Pop() + bqe.cancelAndWait() + p.releaseOrMarkEvicted(bqe) + } + p.mu.Unlock() + + // Wait for any remaining operations where data slices were returned directly + // to complete, with a timeout to prevent indefinite blocking. Their Done + // callbacks will handle the final release of those blocks. + done := make(chan struct{}) + go func() { + defer close(done) + p.inflightCallbackWg.Wait() + }() + + timer := time.NewTimer(10 * time.Second) + defer timer.Stop() + + select { + case <-done: + // Wait completed successfully. + case <-timer.C: + // If this timeout is reached, it implies that the callback was not called + // within 10 seconds, which is highly unexpected. In this scenario, we + // proceed with destruction, meaning the in-flight blocks will not be + // returned to the pool for deallocation. This results in a memory leak + // (as the blocks are never released back to the pool), but it is + // considered an acceptable tradeoff given the rarity of this condition. + logger.Warnf("BufferedReader.Destroy: timed out waiting for outstanding data slice references to be released.") + } + + // After the wait, no new read calls can arrive. This is because the kernel + // prevents read operations on a closed file handle, so no further locking is + // necessary for the final cleanup. + if p.cancelFunc != nil { + p.cancelFunc() + p.cancelFunc = nil + } + + p.mu.Lock() + if err := p.blockPool.ClearFreeBlockChannel(true); err != nil { + logger.Warnf("Destroy: clearing free block channel: %v", err) + } + p.blockPool = nil + p.mu.Unlock() +} + +// releaseOrMarkEvicted handles the release of a block that has been removed +// from the prefetch queue. If the block has no outstanding references (i.e., +// it has not been returned to a FUSE read), it is immediately returned to the +// block pool. Otherwise, the block is marked as evicted, and its final release +// is deferred until the last reference's callback is executed. +// LOCKS_REQUIRED(p.mu) +func (p *BufferedReader) releaseOrMarkEvicted(entry *blockQueueEntry) { + // If the block still has outstanding references, do not release it to the + // pool. Instead, mark it as evicted so the callback can release it later, + // when the reference count drops to zero. + if entry.block.RefCount() > 0 { + entry.wasEvicted = true + } else { + p.blockPool.Release(entry.block) + } +} + +// CheckInvariants checks for internal consistency of the reader. +// LOCKS_EXCLUDED(p.mu) +func (p *BufferedReader) CheckInvariants() { + p.mu.Lock() + defer p.mu.Unlock() + + // The prefetch block size must be positive. + if p.config.PrefetchBlockSizeBytes <= 0 { + panic(fmt.Sprintf("BufferedReader: PrefetchBlockSizeBytes must be positive, but is %d", p.config.PrefetchBlockSizeBytes)) + } + + // The prefetch block size must be at least 1 MiB. + if p.config.PrefetchBlockSizeBytes < util.MiB { + panic(fmt.Sprintf("BufferedReader: PrefetchBlockSizeBytes must be at least 1 MiB, but is %d", p.config.PrefetchBlockSizeBytes)) + } + + // The number of items in the blockQueue should not exceed MaxPrefetchBlockCnt. + if int64(p.blockQueue.Len()) > p.config.MaxPrefetchBlockCnt { + panic(fmt.Sprintf("BufferedReader: blockQueue length %d exceeds limit %d", p.blockQueue.Len(), p.config.MaxPrefetchBlockCnt)) + } + + // The random seek count should never exceed randomReadsThreshold. + if p.randomSeekCount > p.randomReadsThreshold { + panic(fmt.Sprintf("BufferedReader: randomSeekCount %d exceeds threshold %d", p.randomSeekCount, p.randomReadsThreshold)) + } +} + +// resetBufferedReaderState resets the internal state to restart buffered reading. +// LOCKS_REQUIRED(p.mu) +func (p *BufferedReader) resetBufferedReaderState() { + // Reset the reader state + p.randomSeekCount = 0 + p.nextBlockIndexToPrefetch = 0 + p.numPrefetchBlocks = p.config.InitialPrefetchBlockCnt +} diff --git a/internal/bufferedread/buffered_reader_test.go b/internal/bufferedread/buffered_reader_test.go new file mode 100644 index 0000000000..4ea4cec25a --- /dev/null +++ b/internal/bufferedread/buffered_reader_test.go @@ -0,0 +1,2120 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufferedread + +import ( + "bytes" + "context" + "errors" + "io" + "sync" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/block" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/workerpool" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/stretchr/testify/mock" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "golang.org/x/sync/semaphore" +) + +const ( + testMaxPrefetchBlockCnt int64 = 10 + testMinBlocksPerHandle int64 = 2 + testRandomSeekThreshold int64 = 3 + testGlobalMaxBlocks int64 = 20 + testPrefetchBlockSizeBytes int64 = 1024 + testInitialPrefetchBlockCnt int64 = 2 + oneTB int64 = 1 << 40 +) + +type BufferedReaderTest struct { + suite.Suite + ctx context.Context + object *gcs.MinObject + bucket *storage.TestifyMockBucket + globalMaxBlocksSem *semaphore.Weighted + config *BufferedReadConfig + workerPool workerpool.WorkerPool + metricHandle metrics.MetricHandle + readTypeClassifier *gcsx.ReadTypeClassifier +} + +//////////////////////////////////////////////////////////////////////// +// Helpers +//////////////////////////////////////////////////////////////////////// + +// createFakeReaderWithOffset returns a FakeReader with deterministic, non-zero content +// starting from a specific absolute offset. +func createFakeReaderWithOffset(t *testing.T, size int, startOffset int64) *fake.FakeReader { + t.Helper() + content := make([]byte, size) + for i := range content { + content[i] = byte('A' + ((int(startOffset) + i) % 26)) // A-Z repeating pattern + } + return &fake.FakeReader{ + ReadCloser: io.NopCloser(bytes.NewReader(content)), + } +} + +// assertBlockContent validates that block data matches expected pattern (A-Z loop). +func assertBlockContent(t *testing.T, blk block.PrefetchBlock, expectedOffset int64, length int) { + t.Helper() + buf := make([]byte, length) + n, err := blk.ReadAt(buf, 0) + require.NoError(t, err) + require.Equal(t, length, n) + assertBufferContent(t, buf, expectedOffset) +} + +// assertReadResponseContent iterates through the data slices in a ReadResponse +// and validates their content against the expected A-Z repeating pattern, +// starting from a given absolute offset. +func assertReadResponseContent(t *testing.T, resp gcsx.ReadResponse, expectedStartOffset int64) { + t.Helper() + var totalBytesVerified int + currentOffset := expectedStartOffset + for _, dataSlice := range resp.Data { + assertBufferContent(t, dataSlice, currentOffset) + currentOffset += int64(len(dataSlice)) + totalBytesVerified += len(dataSlice) + } + assert.Equal(t, resp.Size, totalBytesVerified, "Total bytes in resp.Data slices should match resp.Size") +} + +// assertBufferContent validates that a buffer's data matches the expected A-Z repeating pattern +// for a given absolute starting offset. +func assertBufferContent(t *testing.T, buf []byte, absStartOffset int64) { + t.Helper() + for i := range buf { + expected := byte('A' + ((int(absStartOffset) + i) % 26)) + assert.Equalf(t, expected, buf[i], "Mismatch at buffer index %d (absolute offset %d)", i, absStartOffset+int64(i)) + } +} + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func TestBufferedReaderTestSuite(t *testing.T) { + suite.Run(t, new(BufferedReaderTest)) +} + +func (t *BufferedReaderTest) SetupTest() { + t.object = &gcs.MinObject{ + Name: "test_object", + Size: 8192, + Generation: 1234567890, + } + t.bucket = new(storage.TestifyMockBucket) + t.globalMaxBlocksSem = semaphore.NewWeighted(testGlobalMaxBlocks) + t.config = &BufferedReadConfig{ + MaxPrefetchBlockCnt: testMaxPrefetchBlockCnt, + PrefetchBlockSizeBytes: testPrefetchBlockSizeBytes, + InitialPrefetchBlockCnt: testInitialPrefetchBlockCnt, + MinBlocksPerHandle: testMinBlocksPerHandle, + RandomSeekThreshold: testRandomSeekThreshold, + } + var err error + t.workerPool, err = workerpool.NewStaticWorkerPool(5, 10, 15) + require.NoError(t.T(), err, "Failed to create worker pool") + t.workerPool.Start() + t.metricHandle = metrics.NewNoopMetrics() + t.ctx = context.Background() + t.readTypeClassifier = gcsx.NewReadTypeClassifier(1, 0) +} + +func (t *BufferedReaderTest) TearDownTest() { + t.workerPool.Stop() +} + +func (t *BufferedReaderTest) TestNewBufferedReader() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err, "NewBufferedReader should not return error") + + assert.Equal(t.T(), t.object, reader.object, "object should match") + assert.Equal(t.T(), t.bucket, reader.bucket, "bucket should match") + assert.Equal(t.T(), t.config, reader.config, "config should match") + assert.Equal(t.T(), int64(0), reader.nextBlockIndexToPrefetch, "nextBlockIndexToPrefetch should be 0") + assert.Equal(t.T(), int64(0), reader.randomSeekCount, "randomSeekCount should be 0") + assert.Equal(t.T(), testInitialPrefetchBlockCnt, reader.numPrefetchBlocks, "numPrefetchBlocks should match") + assert.NotNil(t.T(), reader.blockQueue, "blockQueue should not be nil") + assert.NotNil(t.T(), reader.blockPool, "blockPool should have been created") + assert.Equal(t.T(), t.workerPool, reader.workerPool) + assert.Equal(t.T(), t.metricHandle, reader.metricHandle) + assert.NotNil(t.T(), reader.ctx) + assert.NotNil(t.T(), reader.cancelFunc) + assert.Equal(t.T(), t.readTypeClassifier, reader.readTypeClassifier) +} + +func (t *BufferedReaderTest) TestNewBufferedReaderReservesRequiredBlocks() { + testCases := []struct { + name string + objectSize uint64 + minBlocksPerHandle int64 + expectedReserved int64 + }{ + { + name: "SmallFile", + objectSize: uint64(testPrefetchBlockSizeBytes) / 2, // Requires 1 block + minBlocksPerHandle: 5, + expectedReserved: 1, + }, + { + name: "LargeFile", + objectSize: uint64(testPrefetchBlockSizeBytes) * 10, // Requires 10 blocks + minBlocksPerHandle: 5, + expectedReserved: 5, + }, + { + name: "ZeroSizeFile", + objectSize: 0, // Requires 0 blocks + minBlocksPerHandle: 5, + expectedReserved: 0, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.object.Size = tc.objectSize + t.config.MinBlocksPerHandle = tc.minBlocksPerHandle + t.globalMaxBlocksSem = semaphore.NewWeighted(testGlobalMaxBlocks) + + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + + require.NoError(t.T(), err) + require.NotNil(t.T(), reader) + // Verify that the correct number of blocks were reserved by checking the semaphore's state. + assert.True(t.T(), t.globalMaxBlocksSem.TryAcquire(testGlobalMaxBlocks-tc.expectedReserved), "Should acquire remaining permits") + assert.False(t.T(), t.globalMaxBlocksSem.TryAcquire(1), "Should not acquire more permits") + }) + } +} + +func (t *BufferedReaderTest) TestNewBufferedReaderFailsWhenPoolAllocationFails() { + t.globalMaxBlocksSem = semaphore.NewWeighted(1) + + _, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + + require.Error(t.T(), err) + assert.ErrorIs(t.T(), err, block.CantAllocateAnyBlockError) +} + +func (t *BufferedReaderTest) TestNewBufferedReaderWithMinimumBlockNotAvailableInPool() { + // Simulate no blocks available globally. + t.globalMaxBlocksSem = semaphore.NewWeighted(1) + + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + + assert.Error(t.T(), err) + assert.ErrorIs(t.T(), err, block.CantAllocateAnyBlockError) + assert.Nil(t.T(), reader, "BufferedReader should be nil on error") +} + +func (t *BufferedReaderTest) TestNewBufferedReaderWithZeroBlockSize() { + t.config.PrefetchBlockSizeBytes = 0 + + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + + assert.ErrorContains(t.T(), err, "PrefetchBlockSizeBytes must be positive") + assert.Nil(t.T(), reader, "BufferedReader should be nil on error") +} + +func (t *BufferedReaderTest) TestDestroySuccess() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err, "NewBufferedReader should not return error") + b, err := reader.blockPool.Get() + require.NoError(t.T(), err, "Failed to get block from pool") + ctx, cancel := context.WithCancel(context.Background()) + go func() { + <-ctx.Done() + b.NotifyReady(block.BlockStatus{State: block.BlockStateDownloadFailed, Err: context.Canceled}) + }() + reader.blockQueue.Push(&blockQueueEntry{ + block: b, + cancel: cancel, + }) + + reader.Destroy() + + assert.Nil(t.T(), reader.cancelFunc) + assert.True(t.T(), reader.blockQueue.IsEmpty()) + assert.Nil(t.T(), reader.blockPool) +} + +func (t *BufferedReaderTest) TestDestroyAwaitReadyError() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err, "NewBufferedReader should not return error") + b, err := reader.blockPool.Get() + require.NoError(t.T(), err, "Failed to get block from pool") + err = b.SetAbsStartOff(0) + require.NoError(t.T(), err) + reader.blockQueue.Push(&blockQueueEntry{ + block: b, + cancel: func() {}, + }) + + b.NotifyReady(block.BlockStatus{State: block.BlockStateDownloadFailed, Err: errors.New("test error")}) + reader.Destroy() + + assert.Nil(t.T(), reader.cancelFunc) + assert.True(t.T(), reader.blockQueue.IsEmpty(), "blockQueue should be empty after Destroy") + assert.Nil(t.T(), reader.blockPool) +} + +func (t *BufferedReaderTest) TestCheckInvariantsBlockQueueExceedsLimit() { + t.config.MaxPrefetchBlockCnt = 2 + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err, "NewBufferedReader should not return error") + b, err := reader.blockPool.Get() + require.NoError(t.T(), err, "Failed to get block from pool") + + // Push 3 blocks to exceed the limit of 2. + reader.blockQueue.Push(&blockQueueEntry{block: b, cancel: func() {}}) + reader.blockQueue.Push(&blockQueueEntry{block: b, cancel: func() {}}) + reader.blockQueue.Push(&blockQueueEntry{block: b, cancel: func() {}}) + + assert.Panics(t.T(), func() { reader.CheckInvariants() }) +} + +func (t *BufferedReaderTest) TestCheckInvariantsRandomSeekCountExceedsThreshold() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err, "NewBufferedReader should not return error") + + reader.randomSeekCount = reader.randomReadsThreshold + 1 + + assert.Panics(t.T(), func() { reader.CheckInvariants() }) +} + +func (t *BufferedReaderTest) TestCheckInvariantsPrefetchBlockSizeNotPositive() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err, "NewBufferedReader should not return error") + testCases := []struct { + name string + blockSize int64 + }{ + { + name: "zero block size", + blockSize: 0, + }, + { + name: "negative block size", + blockSize: -1, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func() { + reader.config.PrefetchBlockSizeBytes = tc.blockSize + + assert.Panics(t.T(), func() { reader.CheckInvariants() }, "Should panic for non-positive block size") + }) + } +} + +func (t *BufferedReaderTest) TestCheckInvariantsPrefetchBlockSizeTooSmall() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err, "NewBufferedReader should not return error") + + reader.config.PrefetchBlockSizeBytes = util.MiB - 1 + + assert.Panics(t.T(), func() { reader.CheckInvariants() }, "Should panic for block size less than 1 MiB") +} + +func (t *BufferedReaderTest) TestCheckInvariantsNoPanic() { + t.config.PrefetchBlockSizeBytes = util.MiB + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err, "NewBufferedReader should not return error") + + assert.NotPanics(t.T(), func() { reader.CheckInvariants() }) +} + +func (t *BufferedReaderTest) TestScheduleNextBlock() { + testCases := []struct { + name string + urgent bool + }{ + {name: "non-urgent", urgent: false}, + {name: "urgent", urgent: true}, + } + for _, tc := range testCases { + t.Run(tc.name, func() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + initialBlockCount := reader.blockQueue.Len() + startOffset := int64(0) + t.bucket.On("NewReaderWithReadHandle", + mock.Anything, + mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == uint64(startOffset) }), + ).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), startOffset), nil).Once() + + err = reader.scheduleNextBlock(tc.urgent) + + require.NoError(t.T(), err) + bqe := reader.blockQueue.Peek() + assert.Equal(t.T(), int64(1), reader.nextBlockIndexToPrefetch) + status, err := bqe.block.AwaitReady(t.ctx) + require.NoError(t.T(), err) + assert.Equal(t.T(), block.BlockStateDownloaded, status.State) + assert.Equal(t.T(), initialBlockCount+1, reader.blockQueue.Len()) + assert.Equal(t.T(), int64(0), bqe.block.AbsStartOff()) + assertBlockContent(t.T(), bqe.block, bqe.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + t.bucket.AssertExpectations(t.T()) + }) + } +} + +func (t *BufferedReaderTest) TestScheduleNextBlockSuccessive() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + initialBlockCount := reader.blockQueue.Len() + startOffset1 := int64(0) + t.bucket.On("NewReaderWithReadHandle", + mock.Anything, + mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == uint64(startOffset1) }), + ).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), startOffset1), nil).Once() + err = reader.scheduleNextBlock(false) + require.NoError(t.T(), err) + bqe1 := reader.blockQueue.Pop() + assert.Equal(t.T(), int64(1), reader.nextBlockIndexToPrefetch) + status1, err := bqe1.block.AwaitReady(t.ctx) + require.NoError(t.T(), err) + assert.Equal(t.T(), block.BlockStateDownloaded, status1.State) + assert.Equal(t.T(), int64(0), bqe1.block.AbsStartOff()) + assertBlockContent(t.T(), bqe1.block, bqe1.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + startOffset2 := int64(testPrefetchBlockSizeBytes) + t.bucket.On("NewReaderWithReadHandle", + mock.Anything, + mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == uint64(startOffset2) }), + ).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), startOffset2), nil).Once() + + err = reader.scheduleNextBlock(false) + + require.NoError(t.T(), err) + bqe2 := reader.blockQueue.Pop() + assert.Equal(t.T(), int64(2), reader.nextBlockIndexToPrefetch) + status2, err := bqe2.block.AwaitReady(t.ctx) + require.NoError(t.T(), err) + assert.Equal(t.T(), block.BlockStateDownloaded, status2.State) + assert.Equal(t.T(), int64(testPrefetchBlockSizeBytes), bqe2.block.AbsStartOff()) + assert.Equal(t.T(), int64(2), reader.nextBlockIndexToPrefetch) + assert.Equal(t.T(), initialBlockCount, reader.blockQueue.Len()) + assertBlockContent(t.T(), bqe2.block, bqe2.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestScheduleBlockWithIndex() { + testCases := []struct { + name string + urgent bool + blockIndex int64 + }{ + {name: "non-urgent", urgent: false, blockIndex: 5}, + {name: "urgent", urgent: true, blockIndex: 3}, + } + for _, tc := range testCases { + t.Run(tc.name, func() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + initialBlockCount := reader.blockQueue.Len() + startOffset := tc.blockIndex * reader.config.PrefetchBlockSizeBytes + t.bucket.On("NewReaderWithReadHandle", + mock.Anything, + mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == uint64(startOffset) }), + ).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), startOffset), nil).Once() + b, err := reader.blockPool.Get() + require.NoError(t.T(), err) + + err = reader.scheduleBlockWithIndex(b, tc.blockIndex, tc.urgent) + + require.NoError(t.T(), err) + bqe := reader.blockQueue.Peek() + status, err := bqe.block.AwaitReady(t.ctx) + require.NoError(t.T(), err) + assert.Equal(t.T(), block.BlockStateDownloaded, status.State) + assert.Equal(t.T(), initialBlockCount+1, reader.blockQueue.Len()) + assert.Equal(t.T(), startOffset, bqe.block.AbsStartOff()) + assertBlockContent(t.T(), bqe.block, bqe.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + t.bucket.AssertExpectations(t.T()) + }) + } +} + +func (t *BufferedReaderTest) TestFreshStart() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + currentOffset := int64(2048) // Start prefetching from offset 2048 (block 2). + // freshStart schedules 1 urgent block and 2 initial prefetch blocks, totaling 3 blocks. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 2048 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 2048), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 3072 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 3072), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 4096 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 4096), nil).Once() + + err = reader.freshStart(currentOffset) + + require.NoError(t.T(), err) + // nextBlockIndexToPrefetch should be current block index (2) + scheduled blocks (3). + assert.Equal(t.T(), int64(5), reader.nextBlockIndexToPrefetch) + // numPrefetchBlocks for the next prefetch should be initialPrefetchBlockCnt (2) * prefetchMultiplier (2). + assert.Equal(t.T(), int64(4), reader.numPrefetchBlocks) + assert.Equal(t.T(), 3, reader.blockQueue.Len()) + // Pop and verify the downloaded blocks. + bqe1 := reader.blockQueue.Pop() + assert.Equal(t.T(), int64(2048), bqe1.block.AbsStartOff()) + status1, err1 := bqe1.block.AwaitReady(t.ctx) + require.NoError(t.T(), err1) + assert.Equal(t.T(), block.BlockStateDownloaded, status1.State) + assertBlockContent(t.T(), bqe1.block, bqe1.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + bqe2 := reader.blockQueue.Pop() + assert.Equal(t.T(), int64(3072), bqe2.block.AbsStartOff()) + status2, err2 := bqe2.block.AwaitReady(t.ctx) + require.NoError(t.T(), err2) + assert.Equal(t.T(), block.BlockStateDownloaded, status2.State) + assertBlockContent(t.T(), bqe2.block, bqe2.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + bqe3 := reader.blockQueue.Pop() + assert.Equal(t.T(), int64(4096), bqe3.block.AbsStartOff()) + status3, err3 := bqe3.block.AwaitReady(t.ctx) + require.NoError(t.T(), err3) + assert.Equal(t.T(), block.BlockStateDownloaded, status3.State) + assertBlockContent(t.T(), bqe3.block, bqe3.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestFreshStartWithNonBlockAlignedOffset() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + currentOffset := int64(2500) // Start prefetching from offset 2500 (inside block 2). + // freshStart should start prefetching from block 2. It schedules 1 urgent block + // and 2 initial prefetch blocks, totaling 3 blocks. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 2048 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 2048), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 3072 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 3072), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 4096 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 4096), nil).Once() + + err = reader.freshStart(currentOffset) + + require.NoError(t.T(), err) + // nextBlockIndexToPrefetch should be current block index (2) + scheduled blocks (3). + assert.Equal(t.T(), int64(5), reader.nextBlockIndexToPrefetch) + // numPrefetchBlocks for the next prefetch should be initialPrefetchBlockCnt (2) * prefetchMultiplier (2). + assert.Equal(t.T(), int64(4), reader.numPrefetchBlocks) + assert.Equal(t.T(), 3, reader.blockQueue.Len()) + // Pop and verify the downloaded blocks. + bqe1 := reader.blockQueue.Pop() + assert.Equal(t.T(), int64(2048), bqe1.block.AbsStartOff()) + status1, err1 := bqe1.block.AwaitReady(t.ctx) + require.NoError(t.T(), err1) + assert.Equal(t.T(), block.BlockStateDownloaded, status1.State) + assertBlockContent(t.T(), bqe1.block, bqe1.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + bqe2 := reader.blockQueue.Pop() + assert.Equal(t.T(), int64(3072), bqe2.block.AbsStartOff()) + status2, err2 := bqe2.block.AwaitReady(t.ctx) + require.NoError(t.T(), err2) + assert.Equal(t.T(), block.BlockStateDownloaded, status2.State) + assertBlockContent(t.T(), bqe2.block, bqe2.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + bqe3 := reader.blockQueue.Pop() + assert.Equal(t.T(), int64(4096), bqe3.block.AbsStartOff()) + status3, err3 := bqe3.block.AwaitReady(t.ctx) + require.NoError(t.T(), err3) + assert.Equal(t.T(), block.BlockStateDownloaded, status3.State) + assertBlockContent(t.T(), bqe3.block, bqe3.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestFreshStartWhenInitialCountGreaterThanMax() { + t.config.MaxPrefetchBlockCnt = 3 + t.config.InitialPrefetchBlockCnt = 4 + t.object.Size = 4096 + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + // freshStart schedules 1 urgent block and 2 prefetch blocks (InitialPrefetchBlockCnt capped by MaxPrefetchBlockCnt). + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 0 })).Return(createFakeReaderWithOffset(t.T(), 1024, 0), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 1024 })).Return(createFakeReaderWithOffset(t.T(), 1024, 1024), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 2048 })).Return(createFakeReaderWithOffset(t.T(), 1024, 2048), nil).Once() + + err = reader.freshStart(0) + + require.NoError(t.T(), err) + // nextBlockIndexToPrefetch should be start block index (0) + scheduled blocks (3). + assert.Equal(t.T(), int64(3), reader.nextBlockIndexToPrefetch) + // numPrefetchBlocks for next prefetch should be capped at MaxPrefetchBlockCnt (3). + assert.Equal(t.T(), int64(3), reader.numPrefetchBlocks) + assert.Equal(t.T(), 3, reader.blockQueue.Len()) + // Pop and verify blocks are downloaded. + bqe1 := reader.blockQueue.Pop() + status1, err1 := bqe1.block.AwaitReady(t.ctx) + require.NoError(t.T(), err1) + assert.Equal(t.T(), block.BlockStateDownloaded, status1.State) + assertBlockContent(t.T(), bqe1.block, bqe1.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + bqe2 := reader.blockQueue.Pop() + status2, err2 := bqe2.block.AwaitReady(t.ctx) + require.NoError(t.T(), err2) + assert.Equal(t.T(), block.BlockStateDownloaded, status2.State) + assertBlockContent(t.T(), bqe2.block, bqe2.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + bqe3 := reader.blockQueue.Pop() + status3, err3 := bqe3.block.AwaitReady(t.ctx) + require.NoError(t.T(), err3) + assert.Equal(t.T(), block.BlockStateDownloaded, status3.State) + assertBlockContent(t.T(), bqe3.block, bqe3.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestFreshStartStopsAtObjectEnd() { + t.object.Size = 4000 // Object size is 3 blocks + a partial block. + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + currentOffset := int64(2048) // Start from block 2. + // freshStart schedules 1 urgent block (block 2) and 1 prefetch block (block 3 - partial). + // The object ends after block 3, so only these 2 blocks are scheduled. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 2*uint64(testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 2*testPrefetchBlockSizeBytes), nil).Once() + partialBlockSize := int(int64(t.object.Size) - (3 * testPrefetchBlockSizeBytes)) + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 3*uint64(testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), partialBlockSize, 3*testPrefetchBlockSizeBytes), nil).Once() + + err = reader.freshStart(currentOffset) + + require.NoError(t.T(), err) + // nextBlockIndexToPrefetch should be start block index (2) + scheduled blocks (2). + assert.Equal(t.T(), int64(4), reader.nextBlockIndexToPrefetch) + // numPrefetchBlocks for the next prefetch should be initialPrefetchBlockCnt (2) * prefetchMultiplier (2). + assert.Equal(t.T(), int64(4), reader.numPrefetchBlocks) + assert.Equal(t.T(), 2, reader.blockQueue.Len()) + // Verify block 2. + bqe1 := reader.blockQueue.Pop() + assert.Equal(t.T(), int64(2048), bqe1.block.AbsStartOff()) + status1, err1 := bqe1.block.AwaitReady(t.ctx) + require.NoError(t.T(), err1) + assert.Equal(t.T(), block.BlockStateDownloaded, status1.State) + assertBlockContent(t.T(), bqe1.block, bqe1.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + // Verify block 3. + bqe2 := reader.blockQueue.Pop() + assert.Equal(t.T(), int64(3072), bqe2.block.AbsStartOff()) + status2, err2 := bqe2.block.AwaitReady(t.ctx) + require.NoError(t.T(), err2) + assert.Equal(t.T(), block.BlockStateDownloaded, status2.State) + // Assert content for the partial block. + assertBlockContent(t.T(), bqe2.block, bqe2.block.AbsStartOff(), partialBlockSize) + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestPrefetch() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 0 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 0), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 1024 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 1024), nil).Once() + + err = reader.prefetch() + + require.NoError(t.T(), err) + // nextBlockIndexToPrefetch should be start block index (0) + initialPrefetchBlockCnt (2). + assert.Equal(t.T(), int64(2), reader.nextBlockIndexToPrefetch) + // numPrefetchBlocks for the next prefetch should be initialPrefetchBlockCnt (2) * prefetchMultiplier (2). + assert.Equal(t.T(), int64(4), reader.numPrefetchBlocks) + assert.Equal(t.T(), 2, reader.blockQueue.Len()) + // Wait for all downloads to complete. + bqe1 := reader.blockQueue.Pop() + status1, err1 := bqe1.block.AwaitReady(t.ctx) + require.NoError(t.T(), err1) + assert.Equal(t.T(), block.BlockStateDownloaded, status1.State) + assertBlockContent(t.T(), bqe1.block, bqe1.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + bqe2 := reader.blockQueue.Pop() + status2, err2 := bqe2.block.AwaitReady(t.ctx) + require.NoError(t.T(), err2) + assert.Equal(t.T(), block.BlockStateDownloaded, status2.State) + assertBlockContent(t.T(), bqe2.block, bqe2.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestPrefetchWithMultiplicativeIncrease() { + t.config.InitialPrefetchBlockCnt = 1 + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + // First prefetch schedules 1 block. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 0 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 0), nil).Once() + err = reader.prefetch() + require.NoError(t.T(), err) + // Wait for the first prefetch to complete and drain the queue. + bqe1 := reader.blockQueue.Pop() + status1, err1 := bqe1.block.AwaitReady(t.ctx) + require.NoError(t.T(), err1) + assert.Equal(t.T(), block.BlockStateDownloaded, status1.State) + assertBlockContent(t.T(), bqe1.block, bqe1.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + // Second prefetch should schedule 2 blocks due to multiplicative increase. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 1024 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 1024), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 2048 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 2048), nil).Once() + + err = reader.prefetch() + + require.NoError(t.T(), err) + // nextBlockIndexToPrefetch should be blocks from first prefetch (1) + blocks from second prefetch (2). + assert.Equal(t.T(), int64(3), reader.nextBlockIndexToPrefetch) + // numPrefetchBlocks for the next prefetch should be numPrefetchBlocks from previous prefetch (2) * prefetchMultiplier (2). + assert.Equal(t.T(), int64(4), reader.numPrefetchBlocks) + assert.Equal(t.T(), 2, reader.blockQueue.Len()) + // Wait for the second prefetch to complete. + bqe2 := reader.blockQueue.Pop() + status2, err2 := bqe2.block.AwaitReady(t.ctx) + require.NoError(t.T(), err2) + assert.Equal(t.T(), block.BlockStateDownloaded, status2.State) + assertBlockContent(t.T(), bqe2.block, bqe2.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + bqe3 := reader.blockQueue.Pop() + status3, err3 := bqe3.block.AwaitReady(t.ctx) + require.NoError(t.T(), err3) + assert.Equal(t.T(), block.BlockStateDownloaded, status3.State) + assertBlockContent(t.T(), bqe3.block, bqe3.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestPrefetchWhenQueueIsFull() { + t.config.MaxPrefetchBlockCnt = 2 + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + b, err := reader.blockPool.Get() + require.NoError(t.T(), err) + // Fill the block queue to its maximum capacity. + reader.blockQueue.Push(&blockQueueEntry{block: b}) + reader.blockQueue.Push(&blockQueueEntry{block: b}) + + err = reader.prefetch() + + require.NoError(t.T(), err) + // No new blocks should be prefetched, so the index remains 0. + assert.Equal(t.T(), int64(0), reader.nextBlockIndexToPrefetch) + // The queue length should remain at MaxPrefetchBlockCnt. + assert.Equal(t.T(), 2, reader.blockQueue.Len()) + // numPrefetchBlocks should remain at its default/current value (2 in this case, due to InitialPrefetchBlockCnt). + assert.Equal(t.T(), int64(2), reader.numPrefetchBlocks) +} + +func (t *BufferedReaderTest) TestPrefetchWhenQueueIsPartiallyFull() { + t.config.MaxPrefetchBlockCnt = 4 + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + b, err := reader.blockPool.Get() + require.NoError(t.T(), err) + reader.blockQueue.Push(&blockQueueEntry{block: b}) + reader.blockQueue.Push(&blockQueueEntry{block: b}) + // blockCountToPrefetch = min(numPrefetchBlocks (2), availableSlots (2)) = 2. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 0 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 0), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 1024 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 1024), nil).Once() + + err = reader.prefetch() + + require.NoError(t.T(), err) + // nextBlockIndexToPrefetch should be the number of scheduled blocks (2). + assert.Equal(t.T(), int64(2), reader.nextBlockIndexToPrefetch) + // blockQueue.Len() should be already in queue (2) + newly scheduled blocks (2). + assert.Equal(t.T(), 4, reader.blockQueue.Len()) + // numPrefetchBlocks for the next prefetch should be previous numPrefetchBlocks (2) * prefetchMultiplier (2). + assert.Equal(t.T(), int64(4), reader.numPrefetchBlocks) + // Wait for the newly scheduled downloads to complete. The old blocks are dummies. + bqe1 := reader.blockQueue.Pop() + reader.blockPool.Release(bqe1.block) + bqe2 := reader.blockQueue.Pop() + reader.blockPool.Release(bqe2.block) + bqe3 := reader.blockQueue.Pop() + status3, err3 := bqe3.block.AwaitReady(t.ctx) + require.NoError(t.T(), err3) + assert.Equal(t.T(), block.BlockStateDownloaded, status3.State) + assertBlockContent(t.T(), bqe3.block, bqe3.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + bqe4 := reader.blockQueue.Pop() + status4, err4 := bqe4.block.AwaitReady(t.ctx) + require.NoError(t.T(), err4) + assert.Equal(t.T(), block.BlockStateDownloaded, status4.State) + assertBlockContent(t.T(), bqe4.block, bqe4.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestPrefetchLimitedByAvailableSlots() { + t.config.MaxPrefetchBlockCnt = 4 + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + reader.numPrefetchBlocks = 4 + b, err := reader.blockPool.Get() + require.NoError(t.T(), err) + reader.blockQueue.Push(&blockQueueEntry{block: b}) + reader.blockQueue.Push(&blockQueueEntry{block: b}) + reader.blockQueue.Push(&blockQueueEntry{block: b}) + // blockCountToPrefetch = min(numPrefetchBlocks (4), availableSlots (1)) = 1. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 0 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 0), nil).Once() + + err = reader.prefetch() + + require.NoError(t.T(), err) + // nextBlockIndexToPrefetch should be the number of scheduled blocks (1). + assert.Equal(t.T(), int64(1), reader.nextBlockIndexToPrefetch) + // blockQueue.Len() should be already in queue (3) + newly scheduled blocks (1). + assert.Equal(t.T(), 4, reader.blockQueue.Len()) + // numPrefetchBlocks for the next prefetch should be current numPrefetchBlocks (4) * prefetchMultiplier (2) = 8, + // but capped at MaxPrefetchBlockCnt (4). + assert.Equal(t.T(), int64(4), reader.numPrefetchBlocks) + // Release dummy blocks and wait for the newly scheduled download to complete. + bqe1 := reader.blockQueue.Pop() + reader.blockPool.Release(bqe1.block) + bqe2 := reader.blockQueue.Pop() + reader.blockPool.Release(bqe2.block) + bqe3 := reader.blockQueue.Pop() + reader.blockPool.Release(bqe3.block) + bqe4 := reader.blockQueue.Pop() + status4, err4 := bqe4.block.AwaitReady(t.ctx) + require.NoError(t.T(), err4) + assert.Equal(t.T(), block.BlockStateDownloaded, status4.State) + assertBlockContent(t.T(), bqe4.block, bqe4.block.AbsStartOff(), int(testPrefetchBlockSizeBytes)) + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestPrefetchStopsWhenPoolIsExhausted() { + // Configure a small pool that will be exhausted, to test the case where + // prefetching is not possible. + t.config.MaxPrefetchBlockCnt = 4 + t.config.InitialPrefetchBlockCnt = 2 + // The global semaphore only has enough permits for the reserved blocks. + t.globalMaxBlocksSem = semaphore.NewWeighted(2) + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + // At this point, NewBufferedReader has acquired 2 permits for its reserved blocks. + // The global semaphore is now empty. + // The first prefetch() call will succeed by allocating the 2 reserved blocks. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 0 })).Return(createFakeReaderWithOffset(t.T(), 1024, 0), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 1024 })).Return(createFakeReaderWithOffset(t.T(), 1024, 1024), nil).Once() + err = reader.prefetch() + require.NoError(t.T(), err) + require.Equal(t.T(), 2, reader.blockQueue.Len()) + assert.Equal(t.T(), int64(4), reader.numPrefetchBlocks, "numPrefetchBlocks should be multiplied after successful prefetch") + // The pool has now created 2 blocks (totalBlocks=2), which is its max (maxBlocks=2). + // To simulate a state where the pool is exhausted, we drain the queue without + // releasing the blocks back to the pool's free channel. We must wait for the + // downloads to complete before proceeding. + bqe1 := reader.blockQueue.Pop() + _, _ = bqe1.block.AwaitReady(t.ctx) + bqe2 := reader.blockQueue.Pop() + _, _ = bqe2.block.AwaitReady(t.ctx) + // Now the blockQueue and freeBlocksCh are empty, but totalBlocks is at its limit. + + // The next prefetch call should attempt to schedule blocks but fail to get + // any from the exhausted pool. It should not return an error. + err = reader.prefetch() + + require.NoError(t.T(), err, "prefetch should handle block unavailability gracefully") + assert.Equal(t.T(), 0, reader.blockQueue.Len(), "No new blocks should have been scheduled") + assert.Equal(t.T(), int64(2), reader.nextBlockIndexToPrefetch, "The index should not have advanced") + assert.Equal(t.T(), int64(4), reader.numPrefetchBlocks, "numPrefetchBlocks should not increase when prefetch is incomplete") + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestReadAtOffsetBeyondEOF() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + buf := make([]byte, 10) + t.bucket.On("Name").Return("test-bucket").Maybe() // Bucket name used for logging. + + resp, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: int64(t.object.Size + 1), + }) + + assert.ErrorIs(t.T(), err, io.EOF) + assert.Zero(t.T(), resp.Size) +} + +func (t *BufferedReaderTest) TestReadAtEmptyBuffer() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + buf := make([]byte, 0) + t.bucket.On("Name").Return("test-bucket").Maybe() // Bucket name used for logging. + + resp, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: 0, + }) + + assert.NoError(t.T(), err) + assert.Zero(t.T(), resp.Size) +} + +func (t *BufferedReaderTest) TestReadAtBackwardSeekIsRandomRead() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + // Perform a read that populates the prefetch queue. + // This is a random read since offset != 0 and queue is empty. + startOffset := int64(3072) // block 3 + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == uint64(startOffset) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), startOffset), nil).Once() + t.bucket.On("Name").Return("test-bucket").Maybe() // Bucket name used for logging. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { + return r.Range.Start == uint64(startOffset+testPrefetchBlockSizeBytes) + })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), startOffset+testPrefetchBlockSizeBytes), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { + return r.Range.Start == uint64(startOffset+2*testPrefetchBlockSizeBytes) + })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), startOffset+2*testPrefetchBlockSizeBytes), nil).Once() + _, err = reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: make([]byte, 10), + Offset: startOffset, + }) + require.NoError(t.T(), err) + assert.Equal(t.T(), int64(1), reader.randomSeekCount, "First read should be counted as random.") + require.Equal(t.T(), 3, reader.blockQueue.Len(), "Queue should be populated after first read.") + // Perform a backward seek, which is another random read. + // This should clear the existing queue and start a new prefetch. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 0 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 0), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == uint64(testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), testPrefetchBlockSizeBytes), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 2*uint64(testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 2*testPrefetchBlockSizeBytes), nil).Once() + buf := make([]byte, 1024) + + resp, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: 0, + }) + + require.NoError(t.T(), err) + assert.Equal(t.T(), int(1024), resp.Size) + assert.Equal(t.T(), int64(2), reader.randomSeekCount, "Second read should be counted as random.") + assert.Equal(t.T(), 2, reader.blockQueue.Len(), "Queue should contain newly prefetched blocks.") + assertReadResponseContent(t.T(), resp, 0) + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestReadAtForwardSeekDiscardsPreviousBlocks() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + var cancelCount int + addBlockToQueue := func(offset int64) { + b, poolErr := reader.blockPool.Get() + require.NoError(t.T(), poolErr) + require.NoError(t.T(), b.SetAbsStartOff(offset)) + _, writeErr := b.Write(make([]byte, testPrefetchBlockSizeBytes)) + require.NoError(t.T(), writeErr) + b.NotifyReady(block.BlockStatus{State: block.BlockStateDownloaded}) + reader.blockQueue.Push(&blockQueueEntry{ + block: b, + cancel: func() { cancelCount++ }, + }) + } + addBlockToQueue(0) // block 0 + addBlockToQueue(1024) // block 1 + addBlockToQueue(2048) // block 2 + // Manually update the reader's state to reflect the manually added blocks. + reader.nextBlockIndexToPrefetch = 3 + require.Equal(t.T(), 3, reader.blockQueue.Len()) + // Reading block 2 should trigger a prefetch for blocks 3 and 4. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 3*uint64(testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 3*testPrefetchBlockSizeBytes), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 4*uint64(testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 4*testPrefetchBlockSizeBytes), nil).Once() + readOffset := int64(2048) + t.bucket.On("Name").Return("test-bucket").Maybe() // Bucket name used for logging. + + // Read the entire block at offset 2048 to trigger the prefetch logic. + _, err = reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: make([]byte, 1024), + Offset: readOffset, + }) + + require.NoError(t.T(), err) + assert.Equal(t.T(), 2, cancelCount, "Expected 2 blocks to be discarded") + // The queue should now contain the two newly prefetched blocks. + require.Equal(t.T(), 2, reader.blockQueue.Len(), "Queue should contain the 2 newly prefetched blocks") + // Wait for the async prefetch tasks to complete to verify the mock calls. + bqe3 := reader.blockQueue.Pop() + bqe4 := reader.blockQueue.Pop() + _, err = bqe3.block.AwaitReady(t.ctx) + require.NoError(t.T(), err, "AwaitReady for block 3 failed") + _, err = bqe4.block.AwaitReady(t.ctx) + require.NoError(t.T(), err, "AwaitReady for block 4 failed") + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestReadAtInitialDownloadFails() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + downloadError := errors.New("gcs error") + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.AnythingOfType("*gcs.ReadObjectRequest")).Return(nil, downloadError) + t.bucket.On("Name").Return("test-bucket").Maybe() // Bucket name used for logging. + buf := make([]byte, 10) + + _, err = reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: 0, + }) + + assert.ErrorContains(t.T(), err, "download failed") + assert.ErrorIs(t.T(), err, downloadError) + // After the failed read, the other prefetched blocks should also have failed. + // We wait for them to finish to avoid a race condition and to verify their state. + require.Equal(t.T(), 2, reader.blockQueue.Len()) + bqe1 := reader.blockQueue.Pop() + status1, err1 := bqe1.block.AwaitReady(t.ctx) + require.NoError(t.T(), err1) + assert.Equal(t.T(), block.BlockStateDownloadFailed, status1.State) + assert.ErrorIs(t.T(), status1.Err, downloadError) + bqe2 := reader.blockQueue.Pop() + status2, err2 := bqe2.block.AwaitReady(t.ctx) + require.NoError(t.T(), err2) + assert.Equal(t.T(), block.BlockStateDownloadFailed, status2.State) + assert.ErrorIs(t.T(), status2.Err, downloadError) + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestReadAtAwaitReadyCancelled() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + b, err := reader.blockPool.Get() + require.NoError(t.T(), err) + err = b.SetAbsStartOff(0) + require.NoError(t.T(), err) + reader.blockQueue.Push(&blockQueueEntry{block: b, cancel: func() {}}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + t.bucket.On("Name").Return("test-bucket").Maybe() // Bucket name used for logging. + + // Read with a cancelled context. + _, err = reader.ReadAt(ctx, &gcsx.ReadRequest{ + Buffer: make([]byte, 10), + Offset: 0, + }) + + assert.ErrorIs(t.T(), err, context.Canceled) +} + +func (t *BufferedReaderTest) TestReadAtBlockStateDownloadFailed() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + b, err := reader.blockPool.Get() + require.NoError(t.T(), err) + err = b.SetAbsStartOff(0) + require.NoError(t.T(), err) + downloadError := errors.New("simulated download error") + b.NotifyReady(block.BlockStatus{State: block.BlockStateDownloadFailed, Err: downloadError}) + reader.blockQueue.Push(&blockQueueEntry{block: b, cancel: func() {}}) + t.bucket.On("Name").Return("test-bucket").Maybe() // Bucket name used for logging. + + // Read from a reader where the next block has failed to download. + _, err = reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: make([]byte, 10), + Offset: 0, + }) + + assert.ErrorIs(t.T(), err, downloadError) + status, err := b.AwaitReady(t.ctx) + require.NoError(t.T(), err) + assert.Equal(t.T(), block.BlockStateDownloadFailed, status.State) + assert.ErrorIs(t.T(), status.Err, downloadError) + assert.True(t.T(), reader.blockQueue.IsEmpty()) +} + +func (t *BufferedReaderTest) TestReadAtBlockDownloadCancelled() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + b, err := reader.blockPool.Get() + require.NoError(t.T(), err) + err = b.SetAbsStartOff(0) + require.NoError(t.T(), err) + b.NotifyReady(block.BlockStatus{State: block.BlockStateDownloadFailed, Err: context.Canceled}) + reader.blockQueue.Push(&blockQueueEntry{block: b, cancel: func() {}}) + t.bucket.On("Name").Return("test-bucket").Maybe() // Bucket name used for logging. + + // Read from a reader where the next block download was cancelled. + _, err = reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: make([]byte, 10), + Offset: 0, + }) + + assert.ErrorIs(t.T(), err, context.Canceled) + status, err := b.AwaitReady(t.ctx) + require.NoError(t.T(), err) + assert.Equal(t.T(), block.BlockStateDownloadFailed, status.State) + assert.ErrorIs(t.T(), status.Err, context.Canceled) + assert.True(t.T(), reader.blockQueue.IsEmpty()) +} + +func (t *BufferedReaderTest) TestReadAtBlockStateUnexpected() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + b, err := reader.blockPool.Get() + require.NoError(t.T(), err) + err = b.SetAbsStartOff(0) + require.NoError(t.T(), err) + b.NotifyReady(block.BlockStatus{State: block.BlockStateInProgress}) + reader.blockQueue.Push(&blockQueueEntry{block: b, cancel: func() {}}) + t.bucket.On("Name").Return("test-bucket").Maybe() // Bucket name used for logging. + + // Read from a reader where the next block is in an unexpected state. + _, err = reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: make([]byte, 10), + Offset: 0, + }) + + assert.ErrorContains(t.T(), err, "unexpected block state") + status, err := b.AwaitReady(t.ctx) + require.NoError(t.T(), err) + assert.Equal(t.T(), block.BlockStateInProgress, status.State) + assert.Nil(t.T(), status.Err) + assert.True(t.T(), reader.blockQueue.IsEmpty()) +} + +func (t *BufferedReaderTest) TestReadAtFromDownloadedBlock() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + b, err := reader.blockPool.Get() + require.NoError(t.T(), err) + err = b.SetAbsStartOff(0) + require.NoError(t.T(), err) + content := []byte("abcdefghijk") + _, err = b.Write(content) + require.NoError(t.T(), err) + b.NotifyReady(block.BlockStatus{State: block.BlockStateDownloaded}) + reader.blockQueue.Push(&blockQueueEntry{block: b, cancel: func() {}}) + buf := make([]byte, 5) + t.bucket.On("Name").Return("test-bucket").Maybe() // Bucket name used for logging. + + // Read from a block that is already downloaded and in the queue. + resp, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: 0, + }) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), 5, resp.Size) + assert.Equal(t.T(), content[:5], util.ConvertReadResponseToBytes(resp.Data, resp.Size)) + assert.False(t.T(), reader.blockQueue.IsEmpty()) +} + +func (t *BufferedReaderTest) TestReadAtExactlyToEndOfFile() { + t.object.Size = uint64(testPrefetchBlockSizeBytes + 50) // 1 full block and 50 bytes + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 0 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 0), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == uint64(testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), 50, testPrefetchBlockSizeBytes), nil).Once() + buf := make([]byte, t.object.Size) + t.bucket.On("Name").Return("test-bucket").Maybe() // Bucket name used for logging. + + // Read the entire file. + resp, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: 0, + }) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), int(t.object.Size), resp.Size) + assertReadResponseContent(t.T(), resp, 0) + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestReadAtSucceedsWhenPrefetchFails() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + // Mock GCS reads where the initial read and first prefetch succeed, but the second prefetch fails. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 0 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 0), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == uint64(testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), testPrefetchBlockSizeBytes), nil).Once() + prefetchError := errors.New("prefetch failed") + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 2*uint64(testPrefetchBlockSizeBytes) })).Return(nil, prefetchError).Once() + buf := make([]byte, testPrefetchBlockSizeBytes) + t.bucket.On("Name").Return("test-bucket").Maybe() // Bucket name used for logging. + + // Read the first block. This should succeed, even though a background prefetch will fail. + resp, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: 0, + }) + + require.NoError(t.T(), err) + assert.Equal(t.T(), int(testPrefetchBlockSizeBytes), resp.Size) + assertReadResponseContent(t.T(), resp, 0) + // After reading block 0, the queue should contain the successful and failed prefetched blocks. + require.Equal(t.T(), 2, reader.blockQueue.Len()) + // Wait for background downloads to complete to prevent a race condition. + bqe1 := reader.blockQueue.Pop() + status1, err1 := bqe1.block.AwaitReady(t.ctx) + require.NoError(t.T(), err1) + assert.Equal(t.T(), block.BlockStateDownloaded, status1.State) + bqe2 := reader.blockQueue.Pop() + status2, err2 := bqe2.block.AwaitReady(t.ctx) + require.NoError(t.T(), err2) + assert.Equal(t.T(), block.BlockStateDownloadFailed, status2.State) + assert.ErrorIs(t.T(), status2.Err, prefetchError) + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestReadAtSpanningMultipleBlocks() { + // Read 2.5 blocks of data in a single ReadAt call. + readSize := 2560 + readOffset := int64(0) + t.object.Size = 3072 + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + buf := make([]byte, readSize) + // freshStart will be called, downloading block 0 (urgent) and + // prefetching blocks 1 and 2 (InitialPrefetchBlockCnt=2). + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { + return r.Range.Start == uint64(0*testPrefetchBlockSizeBytes) + })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 0*testPrefetchBlockSizeBytes), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { + return r.Range.Start == uint64(1*testPrefetchBlockSizeBytes) + })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 1*testPrefetchBlockSizeBytes), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { + return r.Range.Start == uint64(2*testPrefetchBlockSizeBytes) + })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 2*testPrefetchBlockSizeBytes), nil).Once() + t.bucket.On("Name").Return("test-bucket").Maybe() // Bucket name used for logging. + + resp, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: readOffset, + }) + + require.NoError(t.T(), err) + assert.Equal(t.T(), 2560, resp.Size) + assertReadResponseContent(t.T(), resp, readOffset) + assert.Equal(t.T(), 1, reader.blockQueue.Len(), "Block 2 should be left in the queue.") + assert.Equal(t.T(), int64(2048), reader.blockQueue.Peek().block.AbsStartOff()) + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestReadAtSequentialReadAcrossBlocks() { + t.config.InitialPrefetchBlockCnt = 1 + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + // Mock reads for all blocks that will be downloaded. + // First ReadAt(0) triggers freshStart, which downloads block 0 (urgent) and prefetches block 1. + // Second ReadAt(1024) consumes block 1 and triggers prefetch for blocks 2, 3. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { + return r.Range.Start == uint64(0*testPrefetchBlockSizeBytes) + })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 0*testPrefetchBlockSizeBytes), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { + return r.Range.Start == uint64(1*testPrefetchBlockSizeBytes) + })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 1*testPrefetchBlockSizeBytes), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { + return r.Range.Start == uint64(2*testPrefetchBlockSizeBytes) + })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 2*testPrefetchBlockSizeBytes), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { + return r.Range.Start == uint64(3*testPrefetchBlockSizeBytes) + })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 3*testPrefetchBlockSizeBytes), nil).Once() + buf1 := make([]byte, testPrefetchBlockSizeBytes) + buf2 := make([]byte, testPrefetchBlockSizeBytes) + t.bucket.On("Name").Return("test-bucket").Maybe() // Bucket name used for logging. + + // Perform two sequential reads. + resp1, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf1, + Offset: 0, + }) + require.NoError(t.T(), err) + resp2, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf2, + Offset: testPrefetchBlockSizeBytes, + }) + require.NoError(t.T(), err) + + assert.Equal(t.T(), int64(0), reader.randomSeekCount) + assertReadResponseContent(t.T(), resp1, 0) + assertReadResponseContent(t.T(), resp2, testPrefetchBlockSizeBytes) + // Wait for all background prefetches to complete before asserting mock expectations. + require.Equal(t.T(), 2, reader.blockQueue.Len()) + bqe1 := reader.blockQueue.Pop() + _, err = bqe1.block.AwaitReady(t.ctx) + require.NoError(t.T(), err) + bqe2 := reader.blockQueue.Pop() + _, err = bqe2.block.AwaitReady(t.ctx) + require.NoError(t.T(), err) + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestReadAtFallsBackAfterRandomReads() { + t.config.InitialPrefetchBlockCnt = 1 + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + reader.randomReadsThreshold = 2 + require.NoError(t.T(), err) + buf := make([]byte, 10) + // Mock GCS calls for the first random read, which will download block 5 and prefetch block 6. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 5*uint64(testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 5*testPrefetchBlockSizeBytes), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 6*uint64(testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 6*testPrefetchBlockSizeBytes), nil).Once() + t.bucket.On("Name").Return("test-bucket").Maybe() // Bucket name used for logging. + // First random read should succeed. + resp, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: 5 * testPrefetchBlockSizeBytes, + ReadInfo: reader.readTypeClassifier.GetReadInfo(5*testPrefetchBlockSizeBytes, false), + }) + require.NoError(t.T(), err, "Random read #1 should succeed") + reader.readTypeClassifier.RecordRead(5*testPrefetchBlockSizeBytes, int64(resp.Size)) + // Mock GCS calls for the second random read, which will download block 4 and prefetch block 5. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 4*uint64(testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 4*testPrefetchBlockSizeBytes), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 5*uint64(testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 5*testPrefetchBlockSizeBytes), nil).Once() + // Second random read should succeed. + resp, err = reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: 4 * testPrefetchBlockSizeBytes, + ReadInfo: reader.readTypeClassifier.GetReadInfo(4*testPrefetchBlockSizeBytes, false), + }) + require.NoError(t.T(), err, "Random read #2 should succeed") + reader.readTypeClassifier.RecordRead(4*testPrefetchBlockSizeBytes, int64(resp.Size)) + + // The third random read should exceed the threshold and trigger the fallback. + _, err = reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: 3 * testPrefetchBlockSizeBytes, + ReadInfo: reader.readTypeClassifier.GetReadInfo(3*testPrefetchBlockSizeBytes, false), + }) + + assert.ErrorIs(t.T(), err, gcsx.FallbackToAnotherReader, "Error should be FallbackToAnotherReader") + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestReadAtResumesAfterFallbackWhenReadBecomesSequential() { + // 1. Setup reader and classifier + t.config.InitialPrefetchBlockCnt = 1 + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: gcsx.NewReadTypeClassifier(1, 0), + }) + reader.randomReadsThreshold = 2 + require.NoError(t.T(), err) + buf := make([]byte, 10) + t.bucket.On("Name").Return("test-bucket").Maybe() + // Perform random reads to trigger fallback + // First random read (offset 6) + offset1 := 5 * testPrefetchBlockSizeBytes + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == uint64(offset1) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), offset1), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { + return r.Range.Start == uint64(offset1+testPrefetchBlockSizeBytes) + })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), offset1+testPrefetchBlockSizeBytes), nil).Once() + resp1, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: offset1, + ReadInfo: reader.readTypeClassifier.GetReadInfo(offset1, false), + }) + require.NoError(t.T(), err, "Random read #1 should succeed") + reader.readTypeClassifier.RecordRead(offset1, int64(resp1.Size)) + // Second random read (offset 4) + offset2 := 4 * testPrefetchBlockSizeBytes + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == uint64(offset2) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), offset2), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { + return r.Range.Start == uint64(offset2+testPrefetchBlockSizeBytes) + })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), offset2+testPrefetchBlockSizeBytes), nil).Once() + resp2, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: offset2, + ReadInfo: reader.readTypeClassifier.GetReadInfo(offset2, false), + }) + require.NoError(t.T(), err, "Random read #2 should succeed") + reader.readTypeClassifier.RecordRead(offset2, int64(resp2.Size)) + // Third random read (offset 3) - this should trigger fallback + offset3 := 3 * testPrefetchBlockSizeBytes + resp3, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: offset3, + ReadInfo: reader.readTypeClassifier.GetReadInfo(offset3, false), + }) + assert.ErrorIs(t.T(), err, gcsx.FallbackToAnotherReader, "Third random read should trigger fallback") + assert.Equal(t.T(), reader.randomReadsThreshold+1, reader.randomSeekCount) + reader.readTypeClassifier.RecordRead(offset3, int64(resp3.Size)) + // Simulate sequential reads to reset the classifier + const thirtyThreeMiB = 33 * 1024 * 1024 + reader.readTypeClassifier.RecordRead(0, thirtyThreeMiB) + _ = reader.readTypeClassifier.GetReadInfo(0, false) // Call GetReadInfo to update internal state before checking IsReadSequential + assert.True(t.T(), reader.readTypeClassifier.IsReadSequential(), "Read pattern should now be sequential") + // Perform a new read and verify the buffered reader resumes + // This read should succeed and reset the buffered reader's state. + // It will trigger a freshStart, downloading block 0 and prefetching block 1. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 0 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 0), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 1*uint64(testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 1*testPrefetchBlockSizeBytes), nil).Once() + + readBuf := make([]byte, 512) + readOffset := int64(0) + resp, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: readBuf, + Offset: readOffset, + }) + reader.readTypeClassifier.RecordRead(readOffset, int64(resp.Size)) + + require.NoError(t.T(), err, "Read should succeed after pattern becomes sequential") + assert.Equal(t.T(), 512, resp.Size) + assert.Equal(t.T(), int64(0), reader.randomSeekCount, "randomSeekCount should be reset") + assert.Equal(t.T(), int64(2), reader.nextBlockIndexToPrefetch, "Prefetching should resume from the start") + assert.Equal(t.T(), 2, reader.blockQueue.Len(), "Two blocks should be left in the queue after the read") + assertReadResponseContent(t.T(), resp, 0) + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestReadAtFallbackOnSecondBlockDownloadFailure() { + // 1. Config: 1 block prefetch, file size is 3 blocks. + t.config.InitialPrefetchBlockCnt = 1 + t.config.MaxPrefetchBlockCnt = 1 + t.config.MinBlocksPerHandle = 1 + t.object.Size = uint64(3 * testPrefetchBlockSizeBytes) + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + t.bucket.On("Name").Return("test-bucket").Maybe() + // 2. Mock GCS: First block succeeds, second fails. + // freshStart for block 0 will succeed. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 0 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 0), nil).Once() + + // 3. Act: Read spanning 2 blocks. + readSize := testPrefetchBlockSizeBytes + 10 + resp, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{Buffer: make([]byte, readSize), Offset: 0}) + + // 4. Assert: Fallback error is returned and response is empty. + assert.ErrorIs(t.T(), err, gcsx.FallbackToAnotherReader, "ReadAt should fall back when the second block download fails") + assert.Nil(t.T(), resp.Data, "Response data should be nil on fallback") + assert.Zero(t.T(), resp.Size, "Response size should be zero on fallback") + assert.Nil(t.T(), resp.Callback, "Response callback should be nil on fallback") + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestReadAtFallbackOnFreshStartFailure() { + t.config.MaxPrefetchBlockCnt = 2 + t.config.InitialPrefetchBlockCnt = 2 + t.globalMaxBlocksSem = semaphore.NewWeighted(2) + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + // Manually exhaust the pool's blocks to simulate a scenario where all blocks are in use. + _, err = reader.blockPool.Get() + require.NoError(t.T(), err) + _, err = reader.blockPool.Get() + require.NoError(t.T(), err) + t.bucket.On("Name").Return("test-bucket").Maybe() + + _, err = reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: make([]byte, 10), + Offset: 0, + }) + + assert.ErrorIs(t.T(), err, gcsx.FallbackToAnotherReader, "ReadAt should fall back when freshStart fails to get a block") +} + +func (t *BufferedReaderTest) TestReadAtFallbackOnMmapFailure() { + // Configure a huge block size that will likely cause mmap to fail. + // This simulates a non-recoverable error during block creation within the + // buffered reader, which should cause a fallback. + t.config.PrefetchBlockSizeBytes = oneTB + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + t.bucket.On("Name").Return("test-bucket").Maybe() + + _, err = reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: make([]byte, 10), + Offset: 0, + }) + + assert.ErrorIs(t.T(), err, gcsx.FallbackToAnotherReader, "ReadAt should fall back when mmap fails") +} + +func (t *BufferedReaderTest) TestReadAtExceedsObjectSize() { + objectSize := uint64(1536) // 1.5 blocks + readOffset := int64(1024) + readSize := int(1024) // Tries to read 1024 bytes, but only 512 are available. + t.object.Size = objectSize + t.object.Generation = 12345 + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + buf := make([]byte, readSize) + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { + return r.Range.Start == uint64(testPrefetchBlockSizeBytes) && r.Range.Limit == objectSize + })).Return(createFakeReaderWithOffset(t.T(), 512, testPrefetchBlockSizeBytes), nil).Once() + t.bucket.On("Name").Return("test-bucket").Maybe() // Bucket name used for logging. + + resp, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: readOffset, + }) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), 512, resp.Size) + assertReadResponseContent(t.T(), resp, readOffset) + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestReadAtSucceedsWhenBackgroundPrefetchFailsDueToGlobalSem() { + // Configure a scenario where the initial read succeeds, but the subsequent + // background prefetch fails due to an exhausted global semaphore. + t.config.MaxPrefetchBlockCnt = 3 + t.config.InitialPrefetchBlockCnt = 1 + t.globalMaxBlocksSem = semaphore.NewWeighted(2) + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 0 })).Return(createFakeReaderWithOffset(t.T(), 1024, 0), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 1024 })).Return(createFakeReaderWithOffset(t.T(), 1024, 1024), nil).Once() + t.bucket.On("Name").Return("test-bucket").Maybe() + buf := make([]byte, 1024) + + // The read should succeed. When this read consumes block 0, it will trigger + // a background prefetch for block 2, which will fail because the global + // semaphore is exhausted. This failure should not affect the foreground read. + resp, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: 0, + }) + + require.NoError(t.T(), err) + assert.Equal(t.T(), 1024, resp.Size) + assertReadResponseContent(t.T(), resp, 0) + require.Equal(t.T(), 1, reader.blockQueue.Len()) + assert.Equal(t.T(), int64(1024), reader.blockQueue.Peek().block.AbsStartOff()) + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestReadAtSucceedsWhenBackgroundPrefetchFailsOnGCSError() { + t.config.MaxPrefetchBlockCnt = 2 + t.config.InitialPrefetchBlockCnt = 2 + t.globalMaxBlocksSem = semaphore.NewWeighted(2) + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + // Mock the first block download to succeed, but the second (prefetched) block + // to fail with a GCS error. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 0 })).Return(createFakeReaderWithOffset(t.T(), 1024, 0), nil).Once() + gcsError := errors.New("simulated GCS error") + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 1024 })).Return(nil, gcsError).Once() + t.bucket.On("Name").Return("test-bucket").Maybe() + buf := make([]byte, 10) + + // The initial read should succeed because it reads from the first block, which + // was downloaded successfully. The background prefetch failure for the second + // block should not affect this call. + resp, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: 0, + }) + + require.NoError(t.T(), err) + assert.Equal(t.T(), 10, resp.Size) + assertReadResponseContent(t.T(), resp, 0) + // A subsequent attempt to read the second block (which failed to prefetch) + // should return the original GCS error. + _, err = reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: 1024, + }) + assert.ErrorIs(t.T(), err, gcsError) + assert.ErrorContains(t.T(), err, "download failed") +} + +func (t *BufferedReaderTest) TestReadAtSubsequentReadAfterFallbackAlsoFallsBack() { + t.config.InitialPrefetchBlockCnt = 1 + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + reader.randomReadsThreshold = 2 + require.NoError(t.T(), err) + buf := make([]byte, 10) + t.bucket.On("Name").Return("test-bucket").Maybe() + // Arrange mocks for the first random read. This will trigger a freshStart, + // downloading block 5 and prefetching block 6. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 5*uint64(testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 5*testPrefetchBlockSizeBytes), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 6*uint64(testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 6*testPrefetchBlockSizeBytes), nil).Once() + // First random read. This should succeed and count as the 1st random seek. + resp, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: 5 * testPrefetchBlockSizeBytes, + ReadInfo: reader.readTypeClassifier.GetReadInfo(2*testPrefetchBlockSizeBytes, false), + }) + // Check that the first random read was successful. + require.NoError(t.T(), err, "Random read #1 should succeed") + reader.readTypeClassifier.RecordRead(5*testPrefetchBlockSizeBytes, int64(resp.Size)) + assert.Equal(t.T(), int64(1), reader.randomSeekCount) + // Arrange mocks for the second random read. This will clear the queue and + // trigger a new freshStart, downloading block 4 and prefetching block 5. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 4*uint64(testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 4*testPrefetchBlockSizeBytes), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 5*uint64(testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 5*testPrefetchBlockSizeBytes), nil).Once() + // Second random read. This should also succeed and count as the 2nd random seek. + resp, err = reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: 4 * testPrefetchBlockSizeBytes, + ReadInfo: reader.readTypeClassifier.GetReadInfo(4*testPrefetchBlockSizeBytes, false), + }) + // Check that the second random read was successful. + require.NoError(t.T(), err, "Random read #2 should succeed") + reader.readTypeClassifier.RecordRead(4*testPrefetchBlockSizeBytes, int64(resp.Size)) + assert.Equal(t.T(), int64(2), reader.randomSeekCount) + // Third random read. This should exceed the threshold and trigger a fallback. + _, err = reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: 3 * testPrefetchBlockSizeBytes, + ReadInfo: reader.readTypeClassifier.GetReadInfo(3*testPrefetchBlockSizeBytes, false), + }) + // Check that the read correctly triggered a fallback. + assert.ErrorIs(t.T(), err, gcsx.FallbackToAnotherReader, "Random read #2 should trigger fallback") + assert.Equal(t.T(), int64(3), reader.randomSeekCount) + + // A subsequent read at any offset. + _, err = reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: 0, + }) + + // The reader is in a fallback state, so this read should also fall back. + assert.ErrorIs(t.T(), err, gcsx.FallbackToAnotherReader, "Subsequent read should also fallback") + assert.Equal(t.T(), int64(3), reader.randomSeekCount, "Random seek count should not change") + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestReadAtConcurrentReads() { + const ( + fileSize = 10 * util.MiB + numGoroutines = 3 + blockSize = 1 * util.MiB + readSize = 1 * util.MiB + ) + t.object.Size = fileSize + t.config.PrefetchBlockSizeBytes = blockSize + t.config.MaxPrefetchBlockCnt = 10 + t.config.InitialPrefetchBlockCnt = 2 // This will prefetch 2 blocks after the initial one. + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + // Set up mocks for all possible block reads. Because the goroutines run + // concurrently, we prepare mocks for all blocks that could be read or + // prefetched (2 blocks) and use .Maybe() to allow them to be called in + // any order. + for i := 0; i <= 8; i++ { + start := uint64(i * blockSize) + // Create content for this block using the A-Z pattern from the test helpers. + blockContent := make([]byte, blockSize) + for j := range blockContent { + blockContent[j] = byte('A' + ((int(start) + j) % 26)) + } + t.bucket.On("NewReaderWithReadHandle", + mock.Anything, + mock.MatchedBy(func(req *gcs.ReadObjectRequest) bool { + return req.Range.Start == start + }), + ).Return(&fake.FakeReader{ReadCloser: io.NopCloser(bytes.NewReader(blockContent))}, nil).Maybe() + } + t.bucket.On("Name").Return("test-bucket").Maybe() + var wg sync.WaitGroup + wg.Add(numGoroutines) + results := make([][]byte, numGoroutines) + + // Each go routine will read different range to avoid duplicate calls for same range. + // That's why we are multiplying by 3 to have offset 3 blocks apart. + var readIndex = 3 + for i := range numGoroutines { + go func(index int) { + defer wg.Done() + offset := int64(index * readIndex * readSize) + readBuf := make([]byte, readSize) + + resp, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: readBuf, + Offset: offset, + }) + + require.NoError(t.T(), err) + require.Equal(t.T(), readSize, resp.Size) + results[index] = util.ConvertReadResponseToBytes(resp.Data, resp.Size) + require.Equal(t.T(), readSize, len(results[index])) + }(i) + } + + wg.Wait() + // Verify the results from all goroutines individually. + for i, res := range results { + offset := int64(i * readIndex * readSize) + assertBufferContent(t.T(), res, offset) + } + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestDestroyWaitsForCallback() { + readSize := 512 + buf := make([]byte, readSize) + startOffset := int64(0) + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + // A read will trigger a freshStart, downloading blocks 0, 1, and 2. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == uint64(0*testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 0), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == uint64(1*testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 1*testPrefetchBlockSizeBytes), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == uint64(2*testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 2*testPrefetchBlockSizeBytes), nil).Once() + t.bucket.On("Name").Return("test-bucket").Maybe() + resp, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: startOffset, + }) + require.NoError(t.T(), err) + require.NotNil(t.T(), resp.Callback) + destroyFinished := make(chan struct{}) + destroyStarted := make(chan struct{}) + go func() { + close(destroyStarted) + reader.Destroy() + close(destroyFinished) + }() + <-destroyStarted + time.Sleep(100 * time.Millisecond) // Give Destroy() time to start waiting. + select { + case <-destroyFinished: + t.T().Fatalf("Destroy() finished prematurely before callback was called.") + default: + // This is expected. Destroy() is waiting. + } + + resp.Callback() + + select { + case <-destroyFinished: + // Success! Destroy() completed after the callback. + case <-time.After(2 * time.Second): + t.T().Fatalf("Destroy() did not complete after callback was called.") + } +} + +func (t *BufferedReaderTest) TestConcurrentReadsOnSameBlock() { + readSize := 512 + startOffset := int64(0) + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + defer reader.Destroy() + // The first read will trigger a freshStart, downloading blocks 0, 1, and 2. + // Since both reads are for block 0, only one download for that block should occur. + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == uint64(0*testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 0), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == uint64(1*testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 1*testPrefetchBlockSizeBytes), nil).Maybe() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == uint64(2*testPrefetchBlockSizeBytes) })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 2*testPrefetchBlockSizeBytes), nil).Maybe() + t.bucket.On("Name").Return("test-bucket").Maybe() + + var wg sync.WaitGroup + wg.Add(2) + var resp1, resp2 gcsx.ReadResponse + var err1, err2 error + go func() { + defer wg.Done() + resp1, err1 = reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: make([]byte, readSize), + Offset: startOffset, + }) + }() + go func() { + defer wg.Done() + resp2, err2 = reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: make([]byte, readSize), + Offset: startOffset, + }) + }() + wg.Wait() + + require.NoError(t.T(), err1) + require.NoError(t.T(), err2) + require.NotNil(t.T(), resp1.Callback) + require.NotNil(t.T(), resp2.Callback) + reader.mu.Lock() + entry := reader.blockQueue.Peek() + require.NotNil(t.T(), entry) + assert.Equal(t.T(), int32(2), entry.block.RefCount(), "RefCount should be 2 after two concurrent reads") + reader.mu.Unlock() + + resp1.Callback() + + reader.mu.Lock() + assert.Equal(t.T(), int32(1), entry.block.RefCount(), "RefCount should be 1 after first callback") + reader.mu.Unlock() + + resp2.Callback() + + reader.mu.Lock() + assert.Equal(t.T(), int32(0), entry.block.RefCount(), "RefCount should be 0 after second callback") + reader.mu.Unlock() + t.bucket.AssertExpectations(t.T()) +} + +func (t *BufferedReaderTest) TestEvictedBlockIsReleasedOnlyAfterCallback() { + reader, err := NewBufferedReader(&BufferedReaderOptions{ + Object: t.object, + Bucket: t.bucket, + Config: t.config, + GlobalMaxBlocksSem: t.globalMaxBlocksSem, + WorkerPool: t.workerPool, + MetricHandle: t.metricHandle, + ReadTypeClassifier: t.readTypeClassifier}) + require.NoError(t.T(), err) + defer reader.Destroy() + // Mock initial read and prefetch (blocks 0, 1, 2). + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 0 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 0), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 1024 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 1024), nil).Once() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 2048 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 2048), nil).Once() + t.bucket.On("Name").Return("test-bucket").Maybe() + // Mock random read and prefetch (blocks 5, 6, 7). + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 5120 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 5120), nil).Maybe() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 6144 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 6144), nil).Maybe() + t.bucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(r *gcs.ReadObjectRequest) bool { return r.Range.Start == 7168 })).Return(createFakeReaderWithOffset(t.T(), int(testPrefetchBlockSizeBytes), 7168), nil).Maybe() + resp1, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: make([]byte, 10), + Offset: 0, + }) + require.NoError(t.T(), err) + require.NotNil(t.T(), resp1.Callback) + // This random read evicts the prefetched blocks (1 and 2) from the queue. Block 0 is not evicted as it's in use by the first read. + resp2, err := reader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: make([]byte, 10), + Offset: 5 * testPrefetchBlockSizeBytes, + }) + require.NoError(t.T(), err) + assert.Equal(t.T(), 0, reader.blockPool.TotalFreeBlocks(), "Free blocks should be consumed by the new prefetch.") + + resp1.Callback() + + assert.Equal(t.T(), 1, reader.blockPool.TotalFreeBlocks(), "Evicted block should be released after its callback.") + resp2.Callback() +} diff --git a/internal/bufferedread/download_task.go b/internal/bufferedread/download_task.go new file mode 100644 index 0000000000..b3b851f91d --- /dev/null +++ b/internal/bufferedread/download_task.go @@ -0,0 +1,106 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufferedread + +import ( + "context" + "errors" + "fmt" + "io" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/block" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/gcsfuse_errors" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/workerpool" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" +) + +type downloadTask struct { + workerpool.Task + object *gcs.MinObject + bucket gcs.Bucket + metricHandle metrics.MetricHandle + + // block is the block to which the data will be downloaded. + block block.PrefetchBlock + + // ctx is the context for the download task. It is used to cancel the download. + ctx context.Context + + // Used for zonal bucket to bypass the auth & metadata checks. + readHandle []byte +} + +// Execute implements the workerpool.Task interface. It downloads the data from +// the GCS object to the block. +// After completion, it notifies the block consumer about the status of the +// download task. The status can be one of the following: +// - BlockStatusDownloaded: The download was successful. +// - BlockStatusDownloadFailed: The download failed due to an error. +func (p *downloadTask) Execute() { + startOff := p.block.AbsStartOff() + blockId := startOff / p.block.Cap() + logger.Tracef("Download: <- block (%s, %v).", p.object.Name, blockId) + stime := time.Now() + var err error + var n int64 + defer func() { + dur := time.Since(stime) + if err == nil { + logger.Tracef("Download: -> block (%s, %v) Ok(%v).", p.object.Name, blockId, dur) + p.block.NotifyReady(block.BlockStatus{State: block.BlockStateDownloaded}) + } else if errors.Is(err, context.Canceled) && p.ctx.Err() == context.Canceled { + logger.Tracef("Download: -> block (%s, %v) cancelled: %v.", p.object.Name, blockId, err) + p.block.NotifyReady(block.BlockStatus{State: block.BlockStateDownloadFailed, Err: err}) + } else { + logger.Errorf("Download: -> block (%s, %v) failed: %v.", p.object.Name, blockId, err) + p.block.NotifyReady(block.BlockStatus{State: block.BlockStateDownloadFailed, Err: err}) + } + p.metricHandle.GcsDownloadBytesCount(n, metrics.ReadTypeBufferedAttr) + }() + + start := uint64(startOff) + end := min(start+uint64(p.block.Cap()), p.object.Size) + newReader, err := p.bucket.NewReaderWithReadHandle( + p.ctx, + &gcs.ReadObjectRequest{ + Name: p.object.Name, + Generation: p.object.Generation, + Range: &gcs.ByteRange{ + Start: start, + Limit: end, + }, + ReadCompressed: p.object.HasContentEncodingGzip(), + ReadHandle: p.readHandle, + }) + if err != nil { + var notFoundError *gcs.NotFoundError + if errors.As(err, ¬FoundError) { + err = &gcsfuse_errors.FileClobberedError{Err: err, ObjectName: p.object.Name} + return + } + err = fmt.Errorf("DownloadTask.Execute: while reader-creations: %w", err) + return + } + defer newReader.Close() + + n, err = io.CopyN(p.block, newReader, int64(end-start)) + if err != nil { + err = fmt.Errorf("DownloadTask.Execute: while data-copy: %w", err) + return + } +} diff --git a/internal/bufferedread/download_task_test.go b/internal/bufferedread/download_task_test.go new file mode 100644 index 0000000000..48ac3c983c --- /dev/null +++ b/internal/bufferedread/download_task_test.go @@ -0,0 +1,310 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufferedread + +import ( + "bytes" + "context" + "errors" + "io" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/block" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/gcsfuse_errors" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + testutil "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/workerpool" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "golang.org/x/sync/semaphore" +) + +const ( + testBlockSize = 500 +) + +type DownloadTaskTestSuite struct { + workerpool.Task + suite.Suite + object *gcs.MinObject + mockBucket *storage.TestifyMockBucket + blockPool *block.GenBlockPool[block.PrefetchBlock] + metricHandle metrics.MetricHandle +} + +func TestDownloadTaskTestSuite(t *testing.T) { + suite.Run(t, new(DownloadTaskTestSuite)) +} + +func (dts *DownloadTaskTestSuite) SetupTest() { + dts.object = &gcs.MinObject{ + Name: "test-object", + Size: 1024, + Generation: 1234567890, + } + dts.mockBucket = new(storage.TestifyMockBucket) + var err error + dts.blockPool, err = block.NewPrefetchBlockPool(testBlockSize, 10, 1, semaphore.NewWeighted(100)) + dts.metricHandle = metrics.NewNoopMetrics() + require.NoError(dts.T(), err, "Failed to create block pool") +} + +func getReadCloser(content []byte) io.ReadCloser { + r := bytes.NewReader(content) + rc := io.NopCloser(r) + return rc +} + +func (dts *DownloadTaskTestSuite) TestExecuteSuccess() { + downloadBlock, err := dts.blockPool.Get() + require.Nil(dts.T(), err) + err = downloadBlock.SetAbsStartOff(0) + require.Nil(dts.T(), err) + task := &downloadTask{ + ctx: context.Background(), + object: dts.object, + bucket: dts.mockBucket, + block: downloadBlock, + readHandle: nil, + metricHandle: dts.metricHandle, + } + testContent := testutil.GenerateRandomBytes(testBlockSize) + rc := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + readObjectRequest := &gcs.ReadObjectRequest{ + Name: dts.object.Name, + Generation: dts.object.Generation, + Range: &gcs.ByteRange{ + Start: uint64(0), + Limit: uint64(testBlockSize), + }, + } + dts.mockBucket.On("NewReaderWithReadHandle", mock.Anything, readObjectRequest).Return(rc, nil).Times(1) + + task.Execute() + + assert.Equal(dts.T(), int64(len(testContent)), downloadBlock.Size()) + assert.Equal(dts.T(), int64(testBlockSize), downloadBlock.Cap()) + assert.NoError(dts.T(), err) + dts.mockBucket.AssertExpectations(dts.T()) + ctx, cancelFunc := context.WithDeadline(context.Background(), time.Now().Add(1*time.Second)) + defer cancelFunc() + status, err := downloadBlock.AwaitReady(ctx) + assert.Equal(dts.T(), block.BlockStatus{State: block.BlockStateDownloaded}, status) + assert.NoError(dts.T(), err) +} + +func (dts *DownloadTaskTestSuite) TestExecuteError() { + downloadBlock, err := dts.blockPool.Get() + require.Nil(dts.T(), err) + err = downloadBlock.SetAbsStartOff(0) + require.Nil(dts.T(), err) + task := &downloadTask{ + ctx: context.Background(), + object: dts.object, + bucket: dts.mockBucket, + block: downloadBlock, + readHandle: nil, + metricHandle: dts.metricHandle, + } + readObjectRequest := &gcs.ReadObjectRequest{ + Name: dts.object.Name, + Generation: dts.object.Generation, + Range: &gcs.ByteRange{ + Start: uint64(0), + Limit: uint64(testBlockSize), + }, + } + dts.mockBucket.On("NewReaderWithReadHandle", mock.Anything, readObjectRequest).Return(nil, errors.New("read error")).Times(1) + + task.Execute() + + dts.mockBucket.AssertExpectations(dts.T()) + ctx, cancelFunc := context.WithDeadline(context.Background(), time.Now().Add(1*time.Second)) + defer cancelFunc() + status, err := downloadBlock.AwaitReady(ctx) + assert.Equal(dts.T(), block.BlockStateDownloadFailed, status.State) + assert.NotNil(dts.T(), status.Err) + assert.NoError(dts.T(), err) +} + +func (dts *DownloadTaskTestSuite) TestExecuteContextDeadlineExceededByServerTreatedAsFailed() { + downloadBlock, err := dts.blockPool.Get() + require.Nil(dts.T(), err) + err = downloadBlock.SetAbsStartOff(0) + require.Nil(dts.T(), err) + taskCtx, taskCancelFunc := context.WithTimeout(context.Background(), 1*time.Millisecond) + defer taskCancelFunc() // Ensure the context is cancelled after the test. + task := &downloadTask{ + ctx: taskCtx, + object: dts.object, + bucket: dts.mockBucket, + block: downloadBlock, + readHandle: nil, + metricHandle: dts.metricHandle, + } + readObjectRequest := &gcs.ReadObjectRequest{ + Name: dts.object.Name, + Generation: dts.object.Generation, + Range: &gcs.ByteRange{ + Start: uint64(0), + Limit: uint64(testBlockSize), + }, + } + dts.mockBucket.On("NewReaderWithReadHandle", mock.Anything, readObjectRequest).Return(nil, context.DeadlineExceeded).Times(1) + + task.Execute() + + assert.Error(dts.T(), context.DeadlineExceeded) + dts.mockBucket.AssertExpectations(dts.T()) + ctx, cancelFunc := context.WithDeadline(context.Background(), time.Now().Add(1*time.Second)) + defer cancelFunc() + status, err := downloadBlock.AwaitReady(ctx) + assert.NoError(dts.T(), err) + assert.Equal(dts.T(), block.BlockStateDownloadFailed, status.State) + assert.NotNil(dts.T(), status.Err) +} + +func (dts *DownloadTaskTestSuite) TestExecuteContextCancelledWhileReaderCreation() { + downloadBlock, err := dts.blockPool.Get() + require.Nil(dts.T(), err) + err = downloadBlock.SetAbsStartOff(0) + require.Nil(dts.T(), err) + taskCtx, taskCancelFunc := context.WithCancel(context.TODO()) + task := &downloadTask{ + ctx: taskCtx, + object: dts.object, + bucket: dts.mockBucket, + block: downloadBlock, + readHandle: nil, + metricHandle: dts.metricHandle, + } + rc := &fake.FakeReader{ReadCloser: getReadCloser(nil)} // No content since context is cancelled + readObjectRequest := &gcs.ReadObjectRequest{ + Name: dts.object.Name, + Generation: dts.object.Generation, + Range: &gcs.ByteRange{ + Start: uint64(0), + Limit: uint64(testBlockSize), + }, + } + dts.mockBucket.On("NewReaderWithReadHandle", mock.Anything, readObjectRequest).Return(rc, context.Canceled).Times(1) + taskCancelFunc() // Ensure client side cancellation. + + task.Execute() + + assert.Error(dts.T(), context.Canceled) + dts.mockBucket.AssertExpectations(dts.T()) + ctx, cancelFunc := context.WithDeadline(context.Background(), time.Now().Add(1*time.Second)) + defer cancelFunc() + status, err := downloadBlock.AwaitReady(ctx) + assert.NoError(dts.T(), err) + assert.Equal(dts.T(), block.BlockStateDownloadFailed, status.State) + assert.ErrorIs(dts.T(), status.Err, context.Canceled) +} + +// ctxCancelledReader is a mock reader that simulates a context cancellation error while reading. +type ctxCancelledReader struct { + io.Reader + io.Closer +} + +func (r *ctxCancelledReader) Read(p []byte) (n int, err error) { + return 0, context.Canceled +} + +func (r *ctxCancelledReader) Close() error { + return nil +} + +func (dts *DownloadTaskTestSuite) TestExecuteContextCancelledWhileReadingFromReader() { + downloadBlock, err := dts.blockPool.Get() + require.Nil(dts.T(), err) + err = downloadBlock.SetAbsStartOff(0) + require.Nil(dts.T(), err) + taskCtx, taskCancelFunc := context.WithCancel(context.TODO()) + task := &downloadTask{ + ctx: taskCtx, + object: dts.object, + bucket: dts.mockBucket, + block: downloadBlock, + readHandle: nil, + metricHandle: dts.metricHandle, + } + rc := &fake.FakeReader{ReadCloser: new(ctxCancelledReader)} + readObjectRequest := &gcs.ReadObjectRequest{ + Name: dts.object.Name, + Generation: dts.object.Generation, + Range: &gcs.ByteRange{ + Start: uint64(0), + Limit: uint64(testBlockSize), + }, + } + dts.mockBucket.On("NewReaderWithReadHandle", mock.Anything, readObjectRequest).Return(rc, nil).Times(1) + taskCancelFunc() // Ensure client side cancellation. + + task.Execute() + + assert.Error(dts.T(), context.Canceled) + dts.mockBucket.AssertExpectations(dts.T()) + ctx, cancelFunc := context.WithDeadline(context.Background(), time.Now().Add(1*time.Second)) + defer cancelFunc() + status, err := downloadBlock.AwaitReady(ctx) + assert.NoError(dts.T(), err) + assert.Equal(dts.T(), block.BlockStateDownloadFailed, status.State) + assert.ErrorIs(dts.T(), status.Err, context.Canceled) +} + +func (dts *DownloadTaskTestSuite) TestExecuteClobbered() { + downloadBlock, err := dts.blockPool.Get() + require.Nil(dts.T(), err) + err = downloadBlock.SetAbsStartOff(0) + require.Nil(dts.T(), err) + task := &downloadTask{ + ctx: context.Background(), + object: dts.object, + bucket: dts.mockBucket, + block: downloadBlock, + readHandle: nil, + metricHandle: dts.metricHandle, + } + // Simulate NewReaderWithReadHandle returning a NotFoundError. + notFoundErr := &gcs.NotFoundError{Err: errors.New("object not found")} + readObjectRequest := &gcs.ReadObjectRequest{ + Name: dts.object.Name, + Generation: dts.object.Generation, + Range: &gcs.ByteRange{ + Start: uint64(0), + Limit: uint64(testBlockSize), + }, + } + dts.mockBucket.On("NewReaderWithReadHandle", mock.Anything, readObjectRequest).Return(nil, notFoundErr).Times(1) + + task.Execute() + + dts.mockBucket.AssertExpectations(dts.T()) + ctx, cancelFunc := context.WithDeadline(context.Background(), time.Now().Add(1*time.Second)) + defer cancelFunc() + status, err := downloadBlock.AwaitReady(ctx) + assert.NoError(dts.T(), err) + assert.Equal(dts.T(), block.BlockStateDownloadFailed, status.State) + var fileClobberedError *gcsfuse_errors.FileClobberedError + assert.True(dts.T(), errors.As(status.Err, &fileClobberedError)) +} diff --git a/internal/bufferedwrites/buffered_write_handler.go b/internal/bufferedwrites/buffered_write_handler.go index 7c6a370a75..1c29409c9f 100644 --- a/internal/bufferedwrites/buffered_write_handler.go +++ b/internal/bufferedwrites/buffered_write_handler.go @@ -15,29 +15,72 @@ package bufferedwrites import ( + "context" + "errors" "fmt" "math" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/block" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/block" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" "golang.org/x/sync/semaphore" ) // Note: All the write operations take inode lock in fs.go, hence we don't need any locks here // as we will get write operations serially. -// BufferedWriteHandler is responsible for filling up the buffers with the data +type BufferedWriteHandler interface { + // Write writes the given data to the buffer. It writes to an existing buffer if + // the capacity is available otherwise writes to a new buffer. + Write(ctx context.Context, data []byte, offset int64) (err error) + + // Sync uploads all the pending buffers to GCS. + // Sync returns + // 1. un-finalized object created on GCS for zonal buckets. + // 2. nil object for non-zonal buckets. + Sync(ctx context.Context) (*gcs.MinObject, error) + + // Flush finalizes the upload. + Flush(ctx context.Context) (*gcs.MinObject, error) + + // SetMtime stores the mtime with the bufferedWriteHandler. + SetMtime(mtime time.Time) + + // Truncate allows truncating the file to a larger size. + Truncate(size int64) error + + // WriteFileInfo returns the file info i.e, how much data has been buffered so far + // and the mtime. + WriteFileInfo() WriteFileInfo + + // Destroy destroys the upload handler and then free up the buffers. + Destroy() error + + // Unlink cancels the ongoing upload and free up the buffers. + Unlink() +} + +// bufferedWriteHandlerImpl is responsible for filling up the buffers with the data // as it receives and handing over to uploadHandler which uploads to GCS. -type BufferedWriteHandler struct { +type bufferedWriteHandlerImpl struct { current block.Block - blockPool *block.BlockPool + blockPool *block.GenBlockPool[block.Block] uploadHandler *UploadHandler // Total size of data buffered so far. Some part of buffered data might have - // been uploaded to GCS as well. + // been uploaded to GCS as well. Depending on the state we are in, it might or + // might not include truncatedSize. totalSize int64 // Stores the mtime value updated by kernel as part of setInodeAttributes call. mtime time.Time + // Stores the size to truncate. No action is made when truncate is called. + // Will be used as mentioned below: + // 1. During flush if totalSize != truncatedSize, additional dummy data is + // added before flush and uploaded. + // 2. If write is started after the truncate offset, dummy data is created + // as per the truncatedSize and then new data is appended to it. + truncatedSize int64 } // WriteFileInfo is used as part of serving fileInode attributes (GetInodeAttributes call). @@ -46,31 +89,80 @@ type WriteFileInfo struct { Mtime time.Time } +var ErrOutOfOrderWrite = errors.New("outOfOrder write detected") + +type CreateBWHandlerRequest struct { + Object *gcs.Object + ObjectName string + Bucket gcs.Bucket + BlockSize int64 + MaxBlocksPerFile int64 + GlobalMaxBlocksSem *semaphore.Weighted + ChunkRetryDeadlineSecs int64 + ChunkTransferTimeoutSecs int64 + TraceHandle tracing.TraceHandle +} + // NewBWHandler creates the bufferedWriteHandler struct. -func NewBWHandler(objectName string, bucket gcs.Bucket, blockSize int64, maxBlocks int64, globalMaxBlocksSem *semaphore.Weighted) (bwh *BufferedWriteHandler, err error) { - bp, err := block.NewBlockPool(blockSize, maxBlocks, globalMaxBlocksSem) +func NewBWHandler(req *CreateBWHandlerRequest) (bwh BufferedWriteHandler, err error) { + bp, err := block.NewBlockPool(req.BlockSize, req.MaxBlocksPerFile, 1, req.GlobalMaxBlocksSem) if err != nil { return } + var size int64 + if req.Object != nil { + size = int64(req.Object.Size) + } - bwh = &BufferedWriteHandler{ - current: nil, - blockPool: bp, - uploadHandler: newUploadHandler(objectName, bucket, bp.FreeBlocksChannel(), blockSize), - totalSize: 0, + bwh = &bufferedWriteHandlerImpl{ + current: nil, + blockPool: bp, + uploadHandler: newUploadHandler(&CreateUploadHandlerRequest{ + Object: req.Object, + ObjectName: req.ObjectName, + Bucket: req.Bucket, + BlockPool: bp, + MaxBlocksPerFile: req.MaxBlocksPerFile, + BlockSize: req.BlockSize, + ChunkRetryDeadlineSecs: req.ChunkRetryDeadlineSecs, + ChunkTransferTimeoutSecs: req.ChunkTransferTimeoutSecs, + TraceHandle: req.TraceHandle, + }), + totalSize: size, mtime: time.Now(), + truncatedSize: -1, } return } -// Write writes the given data to the buffer. It writes to an existing buffer if -// the capacity is available otherwise writes to a new buffer. -func (wh *BufferedWriteHandler) Write(data []byte, offset int64) (err error) { - if offset > wh.totalSize { - // TODO: Will be handled as part of ordered writes. - return fmt.Errorf("non sequential writes") +func (wh *bufferedWriteHandlerImpl) Write(ctx context.Context, data []byte, offset int64) (err error) { + // Fail early if the uploadHandler has already failed. + err = wh.uploadHandler.UploadError() + if err != nil { + return + } + // Once we write past the truncated size, any writes starting from the truncated + // offset are considered out of order. For example, if a file is truncated to 10 + // bytes, and we write 10 bytes starting from offset 5, the total size becomes 15. + // A subsequent write at offset 10 (the truncated size) will be rejected as an out of order write. + if offset != wh.totalSize && (offset != wh.truncatedSize || wh.totalSize >= wh.truncatedSize) { + logger.Errorf("BufferedWriteHandler.OutOfOrderError for object: %s, expectedOffset: %d, actualOffset: %d", + wh.uploadHandler.objectName, wh.totalSize, offset) + return ErrOutOfOrderWrite + } + + if offset == wh.truncatedSize { + // Check and update if any data filling has to be done. + err = wh.writeDataForTruncatedSize(ctx) + if err != nil { + return + } } + return wh.appendBuffer(ctx, data) +} + +func (wh *bufferedWriteHandlerImpl) appendBuffer(ctx context.Context, data []byte) (err error) { dataWritten := 0 for dataWritten < len(data) { if wh.current == nil { @@ -83,7 +175,7 @@ func (wh *BufferedWriteHandler) Write(data []byte, offset int64) (err error) { remainingBlockSize := float64(wh.blockPool.BlockSize()) - float64(wh.current.Size()) pendingDataForWrite := float64(len(data)) - float64(dataWritten) bytesToCopy := int(math.Min(remainingBlockSize, pendingDataForWrite)) - err := wh.current.Write(data[dataWritten : dataWritten+bytesToCopy]) + _, err := wh.current.Write(data[dataWritten : dataWritten+bytesToCopy]) if err != nil { return err } @@ -91,7 +183,7 @@ func (wh *BufferedWriteHandler) Write(data []byte, offset int64) (err error) { dataWritten += bytesToCopy if wh.current.Size() == wh.blockPool.BlockSize() { - err := wh.uploadHandler.Upload(wh.current) + err := wh.uploadHandler.Upload(ctx, wh.current) if err != nil { return err } @@ -100,37 +192,147 @@ func (wh *BufferedWriteHandler) Write(data []byte, offset int64) (err error) { } wh.totalSize += int64(dataWritten) + + // If the file size has surpassed the truncation point, the truncation requirement + // is fulfilled and we can safely discard the stale offset. + if wh.truncatedSize != -1 && wh.totalSize >= wh.truncatedSize { + wh.truncatedSize = -1 + } + return } -// Sync uploads all the pending full buffers to GCS. -func (wh *BufferedWriteHandler) Sync() (err error) { - // TODO: Will be added after uploadHandler changes are done. - return fmt.Errorf("not implemented") +func (wh *bufferedWriteHandlerImpl) Sync(ctx context.Context) (o *gcs.MinObject, err error) { + // Upload current block (for both regional and zonal buckets). + if wh.current != nil && wh.current.Size() != 0 { + err = wh.uploadHandler.Upload(ctx, wh.current) + if err != nil { + return nil, err + } + wh.current = nil + } + // Upload all the pending buffers. + wh.uploadHandler.AwaitBlocksUpload() + // The FlushPendingWrites method synchronizes all bytes currently residing in + // the Writer's buffer to Cloud Storage, thereby making them available for + // other operations like read. + // This functionality is exclusively supported on rapid buckets. + if wh.uploadHandler.bucket.BucketType().RapidWritesEnabled() { + o, err = wh.uploadHandler.FlushPendingWrites(ctx) + if err != nil { + return nil, err + } + if o.Size != uint64(wh.totalSize) { + return nil, fmt.Errorf("could not upload entire data, expected size %d, got %d", wh.totalSize, o.Size) + } + } + // Release memory used by buffers. + err = wh.blockPool.ClearFreeBlockChannel(false) + if err != nil { + // Only logging an error in case of resource leak as upload succeeded. + logger.Errorf("blockPool.ClearFreeBlockChannel() failed during sync: %v", err) + } + err = wh.uploadHandler.UploadError() + if err != nil { + return nil, err + } + return o, nil } // Flush finalizes the upload. -func (wh *BufferedWriteHandler) Flush() (err error) { +func (wh *bufferedWriteHandlerImpl) Flush(ctx context.Context) (*gcs.MinObject, error) { + // Fail early if upload already failed. + err := wh.uploadHandler.UploadError() + if err != nil { + return nil, err + } + + // In case it is a truncated file, upload empty blocks as required. + err = wh.writeDataForTruncatedSize(ctx) + if err != nil { + return nil, err + } + if wh.current != nil { - err := wh.uploadHandler.Upload(wh.current) + err := wh.uploadHandler.Upload(ctx, wh.current) if err != nil { - return err + return nil, err } wh.current = nil } - return wh.uploadHandler.Finalize() + + obj, err := wh.uploadHandler.Finalize(ctx) + if err != nil { + return nil, fmt.Errorf("BufferedWriteHandler.Flush(): %w", err) + } + + if obj != nil && obj.Size != uint64(wh.totalSize) { + return nil, fmt.Errorf("could not upload entire data, expected size %d, got %d", wh.totalSize, obj.Size) + } + + err = wh.blockPool.ClearFreeBlockChannel(true) + if err != nil { + // Only logging an error in case of resource leak as upload succeeded. + logger.Errorf("blockPool.ClearFreeBlockChannel() failed: %v", err) + } + + return obj, nil } -// SetMtime stores the mtime with the bufferedWriteHandler. -func (wh *BufferedWriteHandler) SetMtime(mtime time.Time) { +func (wh *bufferedWriteHandlerImpl) SetMtime(mtime time.Time) { wh.mtime = mtime } -// WriteFileInfo returns the file info i.e, how much data has been buffered so far -// and the mtime. -func (wh *BufferedWriteHandler) WriteFileInfo() WriteFileInfo { +func (wh *bufferedWriteHandlerImpl) Truncate(size int64) error { + if size < wh.totalSize { + return ErrOutOfOrderWrite + } + + wh.truncatedSize = size + return nil +} + +func (wh *bufferedWriteHandlerImpl) WriteFileInfo() WriteFileInfo { return WriteFileInfo{ - TotalSize: wh.totalSize, + TotalSize: int64(math.Max(float64(wh.totalSize), float64(wh.truncatedSize))), Mtime: wh.mtime, } } + +func (wh *bufferedWriteHandlerImpl) Destroy() error { + wh.uploadHandler.Destroy() + return wh.blockPool.ClearFreeBlockChannel(true) +} + +func (wh *bufferedWriteHandlerImpl) writeDataForTruncatedSize(ctx context.Context) error { + // If totalSize is greater than truncatedSize, that means user has + // written more data than they actually truncated in the beginning. + if wh.totalSize >= wh.truncatedSize { + return nil + } + + // Otherwise append dummy data to match truncatedSize. + diff := wh.truncatedSize - wh.totalSize + // Create 1MB of data at a time to avoid OOM + chunkSize := 1024 * 1024 + for i := 0; i < int(diff); i += chunkSize { + size := math.Min(float64(chunkSize), float64(int(diff)-i)) + err := wh.appendBuffer(ctx, make([]byte, int(size))) + if err != nil { + return err + } + } + + return nil +} + +func (wh *bufferedWriteHandlerImpl) Unlink() { + wh.uploadHandler.CancelUpload() + // Since bwh is not cleared after unlink, we will not release last block yet. + // Last block will be released when file handle for this file is closed. + err := wh.blockPool.ClearFreeBlockChannel(false) + if err != nil { + // Only logging an error in case of resource leak. + logger.Errorf("blockPool.ClearFreeBlockChannel() failed: %v", err) + } +} diff --git a/internal/bufferedwrites/buffered_write_handler_test.go b/internal/bufferedwrites/buffered_write_handler_test.go index 3ee5512548..d013e07fff 100644 --- a/internal/bufferedwrites/buffered_write_handler_test.go +++ b/internal/bufferedwrites/buffered_write_handler_test.go @@ -15,21 +15,36 @@ package bufferedwrites import ( + "context" + "errors" "strings" "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/fake" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + storagemock "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/mock" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" "github.com/jacobsa/timeutil" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "golang.org/x/sync/semaphore" ) +const ( + chunkRetryDeadlineSecs int64 = 120 + chunkTransferTimeoutSecs int64 = 10 +) + +var errUploadFailure = errors.New("error while uploading object to GCS") + type BufferedWriteTest struct { - bwh *BufferedWriteHandler + bwh BufferedWriteHandler + globalSemaphore *semaphore.Weighted suite.Suite } @@ -38,8 +53,24 @@ func TestBufferedWriteTestSuite(t *testing.T) { } func (testSuite *BufferedWriteTest) SetupTest() { - bucket := fake.NewFakeBucket(timeutil.RealClock(), "FakeBucketName", gcs.NonHierarchical) - bwh, err := NewBWHandler("testObject", bucket, 1024, 10, semaphore.NewWeighted(10)) + bucketType := gcs.BucketType{} + testSuite.setupTestWithBucketType(bucketType) +} + +func (testSuite *BufferedWriteTest) setupTestWithBucketType(bucketType gcs.BucketType) { + bucket := fake.NewFakeBucket(timeutil.RealClock(), "FakeBucketName", bucketType) + testSuite.globalSemaphore = semaphore.NewWeighted(10) + bwh, err := NewBWHandler(&CreateBWHandlerRequest{ + Object: nil, + ObjectName: "testObject", + Bucket: bucket, + BlockSize: blockSize, + MaxBlocksPerFile: 10, + GlobalMaxBlocksSem: testSuite.globalSemaphore, + ChunkRetryDeadlineSecs: chunkRetryDeadlineSecs, + ChunkTransferTimeoutSecs: chunkTransferTimeoutSecs, + TraceHandle: tracing.NewNoopTracer(), + }) require.Nil(testSuite.T(), err) testSuite.bwh = bwh } @@ -54,20 +85,22 @@ func (testSuite *BufferedWriteTest) TestSetMTime() { } func (testSuite *BufferedWriteTest) TestWrite() { - err := testSuite.bwh.Write([]byte("hi"), 0) + err := testSuite.bwh.Write(context.Background(), []byte("hi"), 0) require.Nil(testSuite.T(), err) fileInfo := testSuite.bwh.WriteFileInfo() - assert.Equal(testSuite.T(), testSuite.bwh.mtime, fileInfo.Mtime) + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + assert.Equal(testSuite.T(), bwhImpl.mtime, fileInfo.Mtime) assert.Equal(testSuite.T(), int64(2), fileInfo.TotalSize) } func (testSuite *BufferedWriteTest) TestWriteWithEmptyBuffer() { - err := testSuite.bwh.Write([]byte{}, 0) + err := testSuite.bwh.Write(context.Background(), []byte{}, 0) require.Nil(testSuite.T(), err) fileInfo := testSuite.bwh.WriteFileInfo() - assert.Equal(testSuite.T(), testSuite.bwh.mtime, fileInfo.Mtime) + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + assert.Equal(testSuite.T(), bwhImpl.mtime, fileInfo.Mtime) assert.Equal(testSuite.T(), int64(0), fileInfo.TotalSize) } @@ -75,11 +108,12 @@ func (testSuite *BufferedWriteTest) TestWriteEqualToBlockSize() { size := 1024 data := strings.Repeat("A", size) - err := testSuite.bwh.Write([]byte(data), 0) + err := testSuite.bwh.Write(context.Background(), []byte(data), 0) require.Nil(testSuite.T(), err) fileInfo := testSuite.bwh.WriteFileInfo() - assert.Equal(testSuite.T(), testSuite.bwh.mtime, fileInfo.Mtime) + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + assert.Equal(testSuite.T(), bwhImpl.mtime, fileInfo.Mtime) assert.Equal(testSuite.T(), int64(size), fileInfo.TotalSize) } @@ -87,34 +121,552 @@ func (testSuite *BufferedWriteTest) TestWriteDataSizeGreaterThanBlockSize() { size := 2000 data := strings.Repeat("A", size) - err := testSuite.bwh.Write([]byte(data), 0) + err := testSuite.bwh.Write(context.Background(), []byte(data), 0) require.Nil(testSuite.T(), err) fileInfo := testSuite.bwh.WriteFileInfo() - assert.Equal(testSuite.T(), testSuite.bwh.mtime, fileInfo.Mtime) + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + assert.Equal(testSuite.T(), bwhImpl.mtime, fileInfo.Mtime) assert.Equal(testSuite.T(), int64(size), fileInfo.TotalSize) } +func (testSuite *BufferedWriteTest) TestWriteWhenNextOffsetIsGreaterThanExpected() { + err := testSuite.bwh.Write(context.Background(), []byte("hi"), 0) + require.Nil(testSuite.T(), err) + + // Next offset should be 2, but we are calling with 5. + err = testSuite.bwh.Write(context.Background(), []byte("hello"), 5) + + require.NotNil(testSuite.T(), err) + require.Equal(testSuite.T(), err, ErrOutOfOrderWrite) + fileInfo := testSuite.bwh.WriteFileInfo() + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + assert.Equal(testSuite.T(), bwhImpl.mtime, fileInfo.Mtime) + assert.Equal(testSuite.T(), int64(2), fileInfo.TotalSize) +} + +func (testSuite *BufferedWriteTest) TestWriteWhenNextOffsetIsLessThanExpected() { + err := testSuite.bwh.Write(context.Background(), []byte("hello"), 0) + require.Nil(testSuite.T(), err) + + // Next offset should be 5, but we are calling with 2. + err = testSuite.bwh.Write(context.Background(), []byte("abcdefgh"), 2) + + require.NotNil(testSuite.T(), err) + require.Equal(testSuite.T(), err, ErrOutOfOrderWrite) + fileInfo := testSuite.bwh.WriteFileInfo() + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + assert.Equal(testSuite.T(), bwhImpl.mtime, fileInfo.Mtime) + assert.Equal(testSuite.T(), int64(5), fileInfo.TotalSize) +} + +func (testSuite *BufferedWriteTest) TestMultipleWrites() { + err := testSuite.bwh.Write(context.Background(), []byte("hello"), 0) + require.Nil(testSuite.T(), err) + + err = testSuite.bwh.Write(context.Background(), []byte("abcdefgh"), 5) + require.Nil(testSuite.T(), err) + + fileInfo := testSuite.bwh.WriteFileInfo() + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + assert.Equal(testSuite.T(), bwhImpl.mtime, fileInfo.Mtime) + assert.Equal(testSuite.T(), int64(13), fileInfo.TotalSize) +} + +func (testSuite *BufferedWriteTest) TestWriteWithSignalUploadFailureInBetween() { + err := testSuite.bwh.Write(context.Background(), []byte("hello"), 0) + require.Nil(testSuite.T(), err) + fileInfo := testSuite.bwh.WriteFileInfo() + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + assert.Equal(testSuite.T(), bwhImpl.mtime, fileInfo.Mtime) + assert.Equal(testSuite.T(), int64(5), fileInfo.TotalSize) + + // Set an error to simulate failure in uploader. + bwhImpl.uploadHandler.uploadError.Store(&errUploadFailure) + + err = testSuite.bwh.Write(context.Background(), []byte("hello"), 5) + require.Error(testSuite.T(), err) + assert.Equal(testSuite.T(), err, errUploadFailure) +} + +func (testSuite *BufferedWriteTest) TestWriteAtTruncatedOffset() { + // Truncate + err := testSuite.bwh.Truncate(2) + require.NoError(testSuite.T(), err) + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + require.Equal(testSuite.T(), int64(2), bwhImpl.truncatedSize) + + // Write at offset = truncatedSize + err = testSuite.bwh.Write(context.Background(), []byte("hello"), 2) + + require.Nil(testSuite.T(), err) + fileInfo := testSuite.bwh.WriteFileInfo() + assert.Equal(testSuite.T(), bwhImpl.mtime, fileInfo.Mtime) + assert.Equal(testSuite.T(), int64(7), fileInfo.TotalSize) +} + +func (testSuite *BufferedWriteTest) TestWriteAfterTruncateAtCurrentSize() { + err := testSuite.bwh.Write(context.Background(), []byte("hello"), 0) + require.Nil(testSuite.T(), err) + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + require.Equal(testSuite.T(), int64(5), bwhImpl.totalSize) + // Truncate + err = testSuite.bwh.Truncate(20) + require.NoError(testSuite.T(), err) + require.Equal(testSuite.T(), int64(20), bwhImpl.truncatedSize) + require.Equal(testSuite.T(), int64(20), testSuite.bwh.WriteFileInfo().TotalSize) + + // Write at offset=bwh.totalSize + err = testSuite.bwh.Write(context.Background(), []byte("abcde"), 5) + + require.Nil(testSuite.T(), err) + assert.Equal(testSuite.T(), int64(10), bwhImpl.totalSize) + assert.Equal(testSuite.T(), int64(20), testSuite.bwh.WriteFileInfo().TotalSize) +} + +func (testSuite *BufferedWriteTest) TestOutOfOrderWriteAtStaleTruncatedSize() { + err := testSuite.bwh.Write(context.Background(), []byte("hello"), 0) + require.Nil(testSuite.T(), err) + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + // Truncate to a larger size + err = testSuite.bwh.Truncate(10) + require.NoError(testSuite.T(), err) + // Write past the truncated size (from offset 5, writing 10 bytes -> totalSize = 15) + err = testSuite.bwh.Write(context.Background(), []byte("0123456789"), 5) + require.Nil(testSuite.T(), err) + require.Equal(testSuite.T(), int64(-1), bwhImpl.truncatedSize) + require.Equal(testSuite.T(), int64(15), bwhImpl.totalSize) + + // Attempt to seek backwards and write exactly at the stale truncatedSize (10) + err = testSuite.bwh.Write(context.Background(), []byte("abc"), 10) + + require.Error(testSuite.T(), err) + assert.Equal(testSuite.T(), ErrOutOfOrderWrite, err) +} + func (testSuite *BufferedWriteTest) TestFlushWithNonNilCurrentBlock() { - err := testSuite.bwh.Write([]byte("hi"), 0) - currentBlock := testSuite.bwh.current + err := testSuite.bwh.Write(context.Background(), []byte("hi"), 0) require.Nil(testSuite.T(), err) - err = testSuite.bwh.Flush() + obj, err := testSuite.bwh.Flush(context.Background()) require.NoError(testSuite.T(), err) - assert.Equal(testSuite.T(), nil, testSuite.bwh.current) - // The current block should be available on the free channel as flush triggers - // an upload before finalize. - freeCh := testSuite.bwh.blockPool.FreeBlocksChannel() - got := <-freeCh - assert.Equal(testSuite.T(), ¤tBlock, &got) + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + assert.Equal(testSuite.T(), nil, bwhImpl.current) + // Validate object. + assert.NotNil(testSuite.T(), obj) + assert.Equal(testSuite.T(), uint64(2), obj.Size) + // Validate that all blocks have been freed up. + assert.Equal(testSuite.T(), 0, bwhImpl.uploadHandler.blockPool.TotalFreeBlocks()) } func (testSuite *BufferedWriteTest) TestFlushWithNilCurrentBlock() { - require.Nil(testSuite.T(), testSuite.bwh.current) + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + require.Nil(testSuite.T(), bwhImpl.current) - err := testSuite.bwh.Flush() + obj, err := testSuite.bwh.Flush(context.Background()) assert.NoError(testSuite.T(), err) + // Validate empty object created. + assert.NotNil(testSuite.T(), obj) + assert.Equal(testSuite.T(), uint64(0), obj.Size) +} + +func (testSuite *BufferedWriteTest) TestFlushWithSignalUploadFailureDuringWrite() { + err := testSuite.bwh.Write(context.Background(), []byte("hi"), 0) + require.Nil(testSuite.T(), err) + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + + // Set an error to simulate failure in uploader. + bwhImpl.uploadHandler.uploadError.Store(&errUploadFailure) + + obj, err := testSuite.bwh.Flush(context.Background()) + require.Error(testSuite.T(), err) + assert.Equal(testSuite.T(), err, errUploadFailure) + assert.Nil(testSuite.T(), obj) +} + +func (testSuite *BufferedWriteTest) TestFlush_SizeMismatch_ReturnsError() { + testCases := []struct { + name string + bucketType gcs.BucketType + obj *gcs.Object + }{ + { + name: "non_zonal", + bucketType: gcs.BucketType{Zonal: false}, + }, + { + name: "zonal_new_file", + bucketType: gcs.BucketType{Zonal: true}, + }, + { + name: "zonal_append", + bucketType: gcs.BucketType{Zonal: true}, + obj: &gcs.Object{Name: "testObject", Size: 0}, + }, + { + name: "pirlo_new_file_rapid_writes", + bucketType: gcs.BucketType{Pirlo: gcs.PirloStateRapidWritesEnabled}, + }, + { + name: "pirlo_append_rapid_writes", + bucketType: gcs.BucketType{Pirlo: gcs.PirloStateRapidWritesEnabled}, + obj: &gcs.Object{Name: "testObject", Size: 0}, + }, + } + for _, tc := range testCases { + testSuite.Run(tc.name, func() { + mockBucket := new(storagemock.TestifyMockBucket) + mockBucket.On("BucketType").Return(tc.bucketType) + writer := &storagemock.Writer{} + writer.On("Write", mock.Anything).Return(2, nil) + if tc.bucketType.RapidWritesEnabled() && tc.obj != nil { + mockBucket.On("CreateAppendableObjectWriter", mock.Anything, mock.Anything).Return(writer, nil) + } else { + mockBucket.On("CreateObjectChunkWriter", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(writer, nil) + } + mockObj := &gcs.MinObject{Name: "testObject", Size: 0} + mockBucket.On("FinalizeUpload", mock.Anything, writer).Return(mockObj, nil) + bwh, err := NewBWHandler(&CreateBWHandlerRequest{ + Object: tc.obj, + ObjectName: "testObject", + Bucket: mockBucket, + BlockSize: blockSize, + MaxBlocksPerFile: 10, + GlobalMaxBlocksSem: testSuite.globalSemaphore, + ChunkRetryDeadlineSecs: chunkRetryDeadlineSecs, + ChunkTransferTimeoutSecs: chunkTransferTimeoutSecs, + TraceHandle: tracing.NewNoopTracer(), + }) + require.Nil(testSuite.T(), err) + err = bwh.Write(context.Background(), []byte("hi"), 0) + require.Nil(testSuite.T(), err) + + obj, err := bwh.Flush(context.Background()) + + require.Error(testSuite.T(), err) + assert.Contains(testSuite.T(), err.Error(), "could not upload entire data, expected size 2, got 0") + assert.Nil(testSuite.T(), obj) + }) + } +} + +func (testSuite *BufferedWriteTest) TestSync_SizeMismatch_ReturnsError() { + testCases := []struct { + name string + bucketType gcs.BucketType + obj *gcs.Object + }{ + { + name: "zonal_new_file", + bucketType: gcs.BucketType{Zonal: true}, + }, + { + name: "zonal_append", + bucketType: gcs.BucketType{Zonal: true}, + obj: &gcs.Object{Name: "testObject", Size: 0}, + }, + { + name: "pirlo_new_file_rapid_writes", + bucketType: gcs.BucketType{Pirlo: gcs.PirloStateRapidWritesEnabled}, + }, + { + name: "pirlo_append_rapid_writes", + bucketType: gcs.BucketType{Pirlo: gcs.PirloStateRapidWritesEnabled}, + obj: &gcs.Object{Name: "testObject", Size: 0}, + }, + } + for _, tc := range testCases { + testSuite.Run(tc.name, func() { + mockBucket := new(storagemock.TestifyMockBucket) + mockBucket.On("BucketType").Return(tc.bucketType) + writer := &storagemock.Writer{} + writer.On("Write", mock.Anything).Return(2, nil) + if tc.bucketType.RapidWritesEnabled() && tc.obj != nil { + mockBucket.On("CreateAppendableObjectWriter", mock.Anything, mock.Anything).Return(writer, nil) + } else { + mockBucket.On("CreateObjectChunkWriter", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(writer, nil) + } + mockObj := &gcs.MinObject{Name: "testObject", Size: 0} + mockBucket.On("FlushPendingWrites", mock.Anything, writer).Return(mockObj, nil) + bwh, err := NewBWHandler(&CreateBWHandlerRequest{ + Object: tc.obj, + ObjectName: "testObject", + Bucket: mockBucket, + BlockSize: blockSize, + MaxBlocksPerFile: 10, + GlobalMaxBlocksSem: testSuite.globalSemaphore, + ChunkRetryDeadlineSecs: chunkRetryDeadlineSecs, + ChunkTransferTimeoutSecs: chunkTransferTimeoutSecs, + TraceHandle: tracing.NewNoopTracer(), + }) + require.Nil(testSuite.T(), err) + err = bwh.Write(context.Background(), []byte("hi"), 0) + require.Nil(testSuite.T(), err) + + obj, err := bwh.Sync(context.Background()) + + require.Error(testSuite.T(), err) + assert.Contains(testSuite.T(), err.Error(), "could not upload entire data, expected size 2, got 0") + assert.Nil(testSuite.T(), obj) + }) + } +} + +func (testSuite *BufferedWriteTest) TestFlushWithMultiBlockWritesAndSignalUploadFailureInBetween() { + buffer, err := operations.GenerateRandomData(blockSize) + assert.NoError(testSuite.T(), err) + // Upload and sync 5 blocks. + testSuite.TestSync5InProgressBlocks() + // Set an error to simulate failure in uploader. + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + bwhImpl.uploadHandler.uploadError.Store(&errUploadFailure) + // Write 5 more blocks. + for i := range 5 { + err := testSuite.bwh.Write(context.Background(), buffer, int64(blockSize*(i+5))) + require.Error(testSuite.T(), err) + assert.Equal(testSuite.T(), errUploadFailure, err) + } + + obj, err := testSuite.bwh.Flush(context.Background()) + + require.Error(testSuite.T(), err) + assert.Equal(testSuite.T(), err, errUploadFailure) + assert.Nil(testSuite.T(), obj) +} + +func (testSuite *BufferedWriteTest) TestSync5InProgressBlocks() { + buffer, err := operations.GenerateRandomData(blockSize) + assert.NoError(testSuite.T(), err) + // Write 5 blocks. + for i := range 5 { + err = testSuite.bwh.Write(context.Background(), buffer, int64(blockSize*i)) + require.Nil(testSuite.T(), err) + } + + // Wait for 5 blocks to upload successfully. + o, err := testSuite.bwh.Sync(context.Background()) + + assert.NoError(testSuite.T(), err) + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + assert.Equal(testSuite.T(), 0, len(bwhImpl.uploadHandler.uploadCh)) + assert.Equal(testSuite.T(), 0, bwhImpl.uploadHandler.blockPool.TotalFreeBlocks()) + assert.Nil(testSuite.T(), o) +} + +func (testSuite *BufferedWriteTest) TestSyncPartialBlockTableDriven() { + testCases := []struct { + name string + bucketType gcs.BucketType + numBlocks float32 + }{ + { + name: "multi_regional_bucket_2.5_blocks", + bucketType: gcs.BucketType{}, + numBlocks: 2.5, + }, + { + name: "multi_regional_bucket_.5_blocks", + bucketType: gcs.BucketType{}, + numBlocks: .5, + }, + { + name: "zonal_bucket_2.5_blocks", + bucketType: gcs.BucketType{Zonal: true}, + numBlocks: 2.5, + }, + { + name: "zonal_bucket_.5_blocks", + bucketType: gcs.BucketType{Zonal: true}, + numBlocks: .5, + }, + { + name: "pirlo_bucket_rapid_writes_2.5_blocks", + bucketType: gcs.BucketType{Pirlo: gcs.PirloStateRapidWritesEnabled}, + numBlocks: 2.5, + }, + { + name: "pirlo_bucket_rapid_writes_.5_blocks", + bucketType: gcs.BucketType{Pirlo: gcs.PirloStateRapidWritesEnabled}, + numBlocks: .5, + }, + } + + for _, tc := range testCases { + testSuite.Run(tc.name, func() { + testSuite.setupTestWithBucketType(tc.bucketType) + buffer, err := operations.GenerateRandomData(int64(blockSize * tc.numBlocks)) + assert.NoError(testSuite.T(), err) + err = testSuite.bwh.Write(context.Background(), buffer, 0) + require.Nil(testSuite.T(), err) + + // Wait for blocks to upload successfully. + o, err := testSuite.bwh.Sync(context.Background()) + + require.NoError(testSuite.T(), err) + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + // Current block should also be uploaded. + assert.Nil(testSuite.T(), bwhImpl.current) + assert.Equal(testSuite.T(), 0, len(bwhImpl.uploadHandler.uploadCh)) + assert.Equal(testSuite.T(), 0, bwhImpl.uploadHandler.blockPool.TotalFreeBlocks()) + // Read the object from back door. + content, err := storageutil.ReadObject(context.Background(), bwhImpl.uploadHandler.bucket, bwhImpl.uploadHandler.objectName) + if tc.bucketType.RapidWritesEnabled() { + require.NotNil(testSuite.T(), o) + assert.EqualValues(testSuite.T(), int64(blockSize*tc.numBlocks), o.Size) + require.NoError(testSuite.T(), err) + assert.Equal(testSuite.T(), buffer, content) + } else { + require.Nil(testSuite.T(), o) + // Since the object is not finalized, the object will not be available + // on GCS for non-zonal buckets. + require.Error(testSuite.T(), err) + var notFoundErr *gcs.NotFoundError + assert.ErrorAs(testSuite.T(), err, ¬FoundErr) + } + }) + } +} + +func (testSuite *BufferedWriteTest) TestSyncBlocksWithError() { + buffer, err := operations.GenerateRandomData(blockSize) + assert.NoError(testSuite.T(), err) + // Write 5 blocks. + for i := range 5 { + err = testSuite.bwh.Write(context.Background(), buffer, int64(blockSize*i)) + require.Nil(testSuite.T(), err) + } + // Set an error to simulate failure in uploader. + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + bwhImpl.uploadHandler.uploadError.Store(&errUploadFailure) + + o, err := testSuite.bwh.Sync(context.Background()) + + assert.Error(testSuite.T(), err) + assert.Equal(testSuite.T(), errUploadFailure, err) + assert.Nil(testSuite.T(), o) +} + +func (testSuite *BufferedWriteTest) TestFlushWithNonZeroTruncatedLengthForEmptyObject() { + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + require.Nil(testSuite.T(), bwhImpl.current) + bwhImpl.truncatedSize = 10 + + _, err := testSuite.bwh.Flush(context.Background()) + + assert.NoError(testSuite.T(), err) + assert.Equal(testSuite.T(), int64(10), bwhImpl.totalSize) + assert.Equal(testSuite.T(), int64(-1), bwhImpl.truncatedSize) +} + +func (testSuite *BufferedWriteTest) TestFlushWithTruncatedLengthGreaterThanObjectSize() { + err := testSuite.bwh.Write(context.Background(), []byte("hi"), 0) + require.Nil(testSuite.T(), err) + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + bwhImpl.truncatedSize = 10 + + _, err = testSuite.bwh.Flush(context.Background()) + + assert.NoError(testSuite.T(), err) + assert.Equal(testSuite.T(), int64(10), bwhImpl.totalSize) + assert.Equal(testSuite.T(), int64(-1), bwhImpl.truncatedSize) +} + +func (testSuite *BufferedWriteTest) TestTruncateWithLesserSize() { + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + bwhImpl.totalSize = 10 + + err := testSuite.bwh.Truncate(2) + + assert.Error(testSuite.T(), err) + assert.Equal(testSuite.T(), ErrOutOfOrderWrite, err) +} + +func (testSuite *BufferedWriteTest) TestTruncateWithSizeGreaterThanCurrentObjectSize() { + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + bwhImpl.totalSize = 10 + + err := testSuite.bwh.Truncate(12) + + assert.NoError(testSuite.T(), err) + assert.Equal(testSuite.T(), int64(12), bwhImpl.truncatedSize) +} + +func (testSuite *BufferedWriteTest) TestWriteFileInfoWithTruncatedLengthLessThanTotalSize() { + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + bwhImpl.totalSize = 10 + bwhImpl.truncatedSize = 5 + + fileInfo := testSuite.bwh.WriteFileInfo() + + assert.Equal(testSuite.T(), bwhImpl.totalSize, fileInfo.TotalSize) +} + +func (testSuite *BufferedWriteTest) TestWriteFileInfoWithTruncatedLengthGreaterThanTotalSize() { + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + bwhImpl.totalSize = 10 + bwhImpl.truncatedSize = 20 + + fileInfo := testSuite.bwh.WriteFileInfo() + + assert.Equal(testSuite.T(), bwhImpl.truncatedSize, fileInfo.TotalSize) +} +func (testSuite *BufferedWriteTest) TestDestroyShouldClearFreeBlockChannel() { + // Try to write 4 blocks of data. + contents := strings.Repeat("A", blockSize*4) + err := testSuite.bwh.Write(context.Background(), []byte(contents), 0) + require.Nil(testSuite.T(), err) + + err = testSuite.bwh.Destroy() + + require.Nil(testSuite.T(), err) + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + assert.Equal(testSuite.T(), 0, bwhImpl.uploadHandler.blockPool.TotalFreeBlocks()) + assert.Equal(testSuite.T(), 0, len(bwhImpl.uploadHandler.uploadCh)) +} + +func (testSuite *BufferedWriteTest) TestUnlinkBeforeWrite() { + testSuite.bwh.Unlink() + + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + assert.Nil(testSuite.T(), bwhImpl.uploadHandler.cancelFunc) + assert.Equal(testSuite.T(), 0, len(bwhImpl.uploadHandler.uploadCh)) + assert.Equal(testSuite.T(), 0, bwhImpl.uploadHandler.blockPool.TotalFreeBlocks()) + // Check if semaphore is released correctly. Last block should not be released. + assert.True(testSuite.T(), testSuite.globalSemaphore.TryAcquire(9)) + assert.False(testSuite.T(), testSuite.globalSemaphore.TryAcquire(1)) +} + +func (testSuite *BufferedWriteTest) TestUnlinkAfterWrite() { + buffer, err := operations.GenerateRandomData(blockSize) + assert.NoError(testSuite.T(), err) + // Write 5 blocks. + for i := range 5 { + err = testSuite.bwh.Write(context.Background(), buffer, int64(blockSize*i)) + require.Nil(testSuite.T(), err) + } + cancelCalled := false + bwhImpl := testSuite.bwh.(*bufferedWriteHandlerImpl) + bwhImpl.uploadHandler.cancelFunc = func() { cancelCalled = true } + + testSuite.bwh.Unlink() + + assert.True(testSuite.T(), cancelCalled) + assert.Equal(testSuite.T(), 0, len(bwhImpl.uploadHandler.uploadCh)) + assert.Equal(testSuite.T(), 0, bwhImpl.uploadHandler.blockPool.TotalFreeBlocks()) + // Check if semaphore is released correctly. Last block should not be released. + assert.True(testSuite.T(), testSuite.globalSemaphore.TryAcquire(9)) + assert.False(testSuite.T(), testSuite.globalSemaphore.TryAcquire(1)) +} + +func (testSuite *BufferedWriteTest) TestReFlushAfterUploadFails() { + testSuite.TestFlushWithMultiBlockWritesAndSignalUploadFailureInBetween() + + // Re-flush. + obj, err := testSuite.bwh.Flush(context.Background()) + + require.Error(testSuite.T(), err) + assert.Nil(testSuite.T(), obj) + assert.ErrorContains(testSuite.T(), err, errUploadFailure.Error()) } diff --git a/internal/bufferedwrites/upload_handler.go b/internal/bufferedwrites/upload_handler.go index 828be02122..148bf8897a 100644 --- a/internal/bufferedwrites/upload_handler.go +++ b/internal/bufferedwrites/upload_handler.go @@ -12,17 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Note: All the write operations take inode lock in fs.go, hence we don't need +// any locks here as we will get calls to these methods serially. + package bufferedwrites import ( "context" + "errors" "fmt" "io" "sync" + "sync/atomic" - "github.com/googlecloudplatform/gcsfuse/v2/internal/block" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/block" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" ) // UploadHandler is responsible for synchronized uploads of the filled blocks @@ -34,96 +40,250 @@ type UploadHandler struct { // Wait group for waiting for the uploader goroutine to finish. wg sync.WaitGroup - // Channel on which uploaded block will be posted for reuse. - freeBlocksCh chan block.Block + // Used to release the free (uploaded) block back to the pool. + blockPool *block.GenBlockPool[block.Block] // writer to resumable upload the blocks to GCS. - writer io.WriteCloser + writer gcs.Writer + + // uploadError stores atomic pointer to the error seen by uploader. + uploadError atomic.Pointer[error] + // CancelFunc persisted to cancel the uploads in case of unlink operation. + cancelFunc context.CancelFunc + startUploader sync.Once // Parameters required for creating a new GCS chunk writer. - bucket gcs.Bucket - objectName string - blockSize int64 + bucket gcs.Bucket + objectName string + obj *gcs.Object + chunkRetryDeadline int64 + chunkTransferTimeout int64 + blockSize int64 + + traceHandle tracing.TraceHandle +} + +type CreateUploadHandlerRequest struct { + Object *gcs.Object + ObjectName string + Bucket gcs.Bucket + BlockPool *block.GenBlockPool[block.Block] + MaxBlocksPerFile int64 + BlockSize int64 + ChunkRetryDeadlineSecs int64 + ChunkTransferTimeoutSecs int64 + TraceHandle tracing.TraceHandle } // newUploadHandler creates the UploadHandler struct. -func newUploadHandler(objectName string, bucket gcs.Bucket, freeBlocksCh chan block.Block, blockSize int64) *UploadHandler { +func newUploadHandler(req *CreateUploadHandlerRequest) *UploadHandler { uh := &UploadHandler{ - uploadCh: make(chan block.Block), - wg: sync.WaitGroup{}, - freeBlocksCh: freeBlocksCh, - bucket: bucket, - objectName: objectName, - blockSize: blockSize, + uploadCh: make(chan block.Block, req.MaxBlocksPerFile), + wg: sync.WaitGroup{}, + blockPool: req.BlockPool, + bucket: req.Bucket, + objectName: req.ObjectName, + obj: req.Object, + blockSize: req.BlockSize, + chunkRetryDeadline: req.ChunkRetryDeadlineSecs, + chunkTransferTimeout: req.ChunkTransferTimeoutSecs, + traceHandle: req.TraceHandle, } return uh } // Upload adds a block to the upload queue. -func (uh *UploadHandler) Upload(block block.Block) error { +func (uh *UploadHandler) Upload(ctx context.Context, block block.Block) error { uh.wg.Add(1) - if uh.writer == nil { - // Lazily create the object writer. - err := uh.createObjectWriter() - if err != nil { - return fmt.Errorf("createObjectWriter: %w", err) - } - // Start the uploader goroutine. - go uh.uploader() + err := uh.ensureWriter(ctx) + if err != nil { + return fmt.Errorf("uh.ensureWriter() failed: %v", err) } - + // Start the uploader goroutine but only once. + uh.startUploader.Do(func() { + go uh.uploader(ctx) + }) uh.uploadCh <- block return nil } // createObjectWriter creates a GCS object writer. -func (uh *UploadHandler) createObjectWriter() (err error) { - var preCond int64 - req := &gcs.CreateObjectRequest{ - Name: uh.objectName, - GenerationPrecondition: &preCond, - Metadata: make(map[string]string), - } +func (uh *UploadHandler) createObjectWriter(ctx context.Context) (err error) { + req := gcs.NewCreateObjectRequest(uh.obj, uh.objectName, nil, uh.chunkRetryDeadline, uh.chunkTransferTimeout) // We need a new context here, since the first writeFile() call will be complete // (and context will be cancelled) by the time complete upload is done. - uh.writer, err = uh.bucket.CreateObjectChunkWriter(context.Background(), req, int(uh.blockSize), nil) + ctx, uh.cancelFunc = context.WithCancel(uh.traceHandle.PropagateTraceContext(context.Background(), ctx)) + if uh.bucket.BucketType().RapidWritesEnabled() && (uh.obj != nil && uh.obj.Finalized.IsZero()) { + chunkWriterReq := gcs.CreateObjectChunkWriterRequest{ + CreateObjectRequest: *req, + ChunkSize: int(uh.blockSize), + Offset: int64(uh.obj.Size), + } + uh.writer, err = uh.bucket.CreateAppendableObjectWriter(ctx, &chunkWriterReq) + } else { + uh.writer, err = uh.bucket.CreateObjectChunkWriter(ctx, req, int(uh.blockSize), nil) + } + return +} + +func (uh *UploadHandler) UploadError() (err error) { + if uploadError := uh.uploadError.Load(); uploadError != nil { + err = *uploadError + } return } // uploader is the single-threaded goroutine that uploads blocks. -func (uh *UploadHandler) uploader() { +func (uh *UploadHandler) uploader(ctx context.Context) { + _, finishSpan := uh.traceHandle.TraceUpload(context.Background(), tracing.StreamingUploader, "", nil, nil) + defer finishSpan() for currBlock := range uh.uploadCh { - _, err := io.Copy(uh.writer, currBlock.Reader()) - if err != nil { - logger.Errorf("upload failed: error in io.Copy: %v", err) - uh.wg.Done() - // TODO: handle failure scenario: finalize the upload and trigger edit flow. - } + uh.uploadBlock(ctx, currBlock) + + // Put back the uploaded block to the pool for re-use, + // irrespective of whether the upload was successful or not. + uh.blockPool.Release(currBlock) uh.wg.Done() + } +} - // Put back the uploaded block on the freeBlocksChannel for re-use. - uh.freeBlocksCh <- currBlock +// uploadBlock uploads the block content to GCS writer. +// It is called by the uploader goroutine. +// If the block is nil, it logs a warning and returns. +// If there is already an error in uploadError, it returns without doing anything. +// If there is an error during upload, it returns after storing the error in uploadError. +func (uh *UploadHandler) uploadBlock(ctx context.Context, b block.Block) { + var written int64 + var err error + _, finishSpan := uh.traceHandle.TraceUpload(ctx, tracing.StreamingUploadBlock, uh.objectName, &written, &err) + defer finishSpan() + + if b == nil { + logger.Warnf("uploadBlock: received nil block for object %s", uh.objectName) + return + } + + if uh.UploadError() != nil { + return + } + + // Reset the readSeek to 0 before uploading. + if off, err := b.Seek(0, io.SeekStart); err != nil || off != 0 { + err := fmt.Errorf("buffered write upload failed for object %s: error in block.Seek: %v with offset: %d", uh.objectName, err, off) + uh.uploadError.Store(&err) + logger.Errorf("uploadBlock: %v", err) + return + } + + written, err = io.Copy(uh.writer, b) + if errors.Is(err, context.Canceled) { + // Context canceled error indicates that the file was deleted from the + // same mount. In this case, we suppress the error to match local + // filesystem behavior. + err = nil + } + if err != nil { + err = gcs.GetGCSError(err) + uh.uploadError.Store(&err) + logger.Errorf("uploadBlock: failed for object %s: error in io.Copy: %v", uh.objectName, err) } } // Finalize finalizes the upload. -func (uh *UploadHandler) Finalize() error { +func (uh *UploadHandler) Finalize(ctx context.Context) (obj *gcs.MinObject, err error) { + ctx = uh.traceHandle.PropagateTraceContext(context.Background(), ctx) + bytes := int64(0) + _, finishSpan := uh.traceHandle.TraceUpload(ctx, tracing.StreamingUploadFinalize, uh.objectName, &bytes, &err) + defer finishSpan() uh.wg.Wait() close(uh.uploadCh) + // Writer may not have been created for empty file creation flow or for very + // small writes of size less than 1 block. + err = uh.ensureWriter(ctx) + if err != nil { + return nil, fmt.Errorf("uh.ensureWriter() failed: %v", err) + } + + obj, err = uh.bucket.FinalizeUpload(ctx, uh.writer) + if err != nil { + // FinalizeUpload already returns GCSerror so no need to convert again. + uh.uploadError.Store(&err) + logger.Errorf("FinalizeUpload failed for object %s: %v", uh.objectName, err) + return nil, err + } + if obj != nil { + bytes = int64(obj.Size) + } + return obj, nil +} + +func (uh *UploadHandler) ensureWriter(ctx context.Context) error { if uh.writer == nil { - // Writer may not have been created for empty file creation flow. - err := uh.createObjectWriter() - if err != nil { - return fmt.Errorf("createObjectWriter: %w", err) + if err := uh.createObjectWriter(ctx); err != nil { + return fmt.Errorf("createObjectWriter failed for object %s: %w", uh.objectName, err) } } + return nil +} + +// FlushPendingWrites uploads any data in the write buffer. +func (uh *UploadHandler) FlushPendingWrites(ctx context.Context) (o *gcs.MinObject, err error) { + bytes := int64(0) + _, finishSpan := uh.traceHandle.TraceUpload(ctx, tracing.StreamingUploadFlush, uh.objectName, &bytes, &err) + defer finishSpan() + uh.wg.Wait() - err := uh.writer.Close() + // Writer may not have been created for empty file creation flow or for very + // small writes of size less than 1 block. + err = uh.ensureWriter(ctx) if err != nil { - logger.Errorf("UploadHandler.Finalize(): %v", err) - return fmt.Errorf("writer.Close: %w", err) + return nil, fmt.Errorf("uh.ensureWriter() failed: %v", err) + } + + o, err = uh.bucket.FlushPendingWrites(ctx, uh.writer) + if err != nil { + // FlushUpload already returns GCS error so no need to convert again. + uh.uploadError.Store(&err) + logger.Errorf("FlushUpload failed for object %s: %v", uh.objectName, err) + return nil, err + } + if o != nil { + bytes = int64(o.Size) + } + return o, nil +} + +func (uh *UploadHandler) CancelUpload() { + if uh.cancelFunc != nil { + // cancel the context to cancel the ongoing GCS upload. + uh.cancelFunc() + } + // Wait for all in progress buffers to be added to the free channel. + uh.wg.Wait() +} + +func (uh *UploadHandler) AwaitBlocksUpload() { + uh.wg.Wait() +} + +func (uh *UploadHandler) Destroy() { + // Move all pending blocks to freeBlockCh and close the channel if not done. + for { + select { + case currBlock, ok := <-uh.uploadCh: + // Not ok means channel closed. Return. + if !ok { + return + } + uh.blockPool.Release(currBlock) + // Marking as wg.Done to ensure any waiters are unblocked. + uh.wg.Done() + default: + // This will get executed when there are no blocks pending in uploadCh and its not closed. + close(uh.uploadCh) + return + } } - return nil } diff --git a/internal/bufferedwrites/upload_handler_test.go b/internal/bufferedwrites/upload_handler_test.go index 15d101ad0d..f1f523faa4 100644 --- a/internal/bufferedwrites/upload_handler_test.go +++ b/internal/bufferedwrites/upload_handler_test.go @@ -15,14 +15,17 @@ package bufferedwrites import ( + "context" "errors" "fmt" + "strconv" "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/block" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage" - storagemock "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/mock" + "github.com/googlecloudplatform/gcsfuse/v3/internal/block" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + storagemock "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/mock" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -31,13 +34,18 @@ import ( ) const ( - blockSize = 1024 + blockSize = 1024 + maxBlocks int64 = 5 + objectName = "testObject" + objectSize uint64 = 1024 ) +var finalized = time.Date(2025, time.June, 18, 23, 30, 0, 0, time.UTC) + type UploadHandlerTest struct { uh *UploadHandler - blockPool *block.BlockPool - mockBucket *storage.TestifyMockBucket + blockPool *block.GenBlockPool[block.Block] + mockBucket *storagemock.TestifyMockBucket suite.Suite } @@ -46,103 +54,559 @@ func TestUploadHandlerTestSuite(t *testing.T) { } func (t *UploadHandlerTest) SetupTest() { - t.mockBucket = new(storage.TestifyMockBucket) + t.mockBucket = new(storagemock.TestifyMockBucket) var err error - t.blockPool, err = block.NewBlockPool(blockSize, 5, semaphore.NewWeighted(5)) + t.blockPool, err = block.NewBlockPool(blockSize, maxBlocks, 1, semaphore.NewWeighted(maxBlocks)) require.NoError(t.T(), err) - t.uh = newUploadHandler("testObject", t.mockBucket, t.blockPool.FreeBlocksChannel(), blockSize) + t.uh = newUploadHandler(&CreateUploadHandlerRequest{ + Object: nil, + ObjectName: "testObject", + Bucket: t.mockBucket, + BlockPool: t.blockPool, + MaxBlocksPerFile: maxBlocks, + BlockSize: blockSize, + ChunkRetryDeadlineSecs: chunkRetryDeadlineSecs, + ChunkTransferTimeoutSecs: chunkTransferTimeoutSecs, + TraceHandle: tracing.NewNoopTracer(), + }) } -func (t *UploadHandlerTest) TestMultipleBlockUpload() { - // Create some blocks. - var blocks []block.Block - for i := 0; i < 5; i++ { - b, err := t.blockPool.Get() - require.NoError(t.T(), err) - blocks = append(blocks, b) +func (t *UploadHandlerTest) SetupSubTest() { + t.SetupTest() +} + +func (t *UploadHandlerTest) createUploadHandlerWithObjectOfGivenSize(size uint64, finalized time.Time) { + t.uh = newUploadHandler(&CreateUploadHandlerRequest{ + Object: &gcs.Object{ + Name: objectName, + Size: size, + Finalized: finalized, + }, + ObjectName: "testObject", + Bucket: t.mockBucket, + BlockPool: t.blockPool, + MaxBlocksPerFile: maxBlocks, + BlockSize: blockSize, + ChunkRetryDeadlineSecs: chunkRetryDeadlineSecs, + ChunkTransferTimeoutSecs: chunkTransferTimeoutSecs, + TraceHandle: tracing.NewNoopTracer(), + }) +} + +func (t *UploadHandlerTest) TestCreateObjectWriter_CreateAppendableObjectWriterCalled() { + t.createUploadHandlerWithObjectOfGivenSize(objectSize, time.Time{}) + t.mockBucket.On("BucketType").Return(gcs.BucketType{Zonal: true}) + t.mockBucket.On("CreateAppendableObjectWriter", mock.Anything, mock.Anything).Return(&storagemock.Writer{}, nil) + + _ = t.uh.createObjectWriter(context.Background()) + + t.mockBucket.AssertCalled(t.T(), "CreateAppendableObjectWriter", mock.Anything, mock.Anything) +} + +func (t *UploadHandlerTest) TestCreateObjectWriter_Pirlo() { + testCases := []struct { + name string + pirloState gcs.PirloState + expectedMethod string + }{ + { + name: "RapidWritesEnabled", + pirloState: gcs.PirloStateRapidWritesEnabled, + expectedMethod: "CreateAppendableObjectWriter", + }, + { + name: "RapidWritesDisabled", + pirloState: gcs.PirloStateRapidWritesDisabled, + expectedMethod: "CreateObjectChunkWriter", + }, } + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.SetupSubTest() + t.createUploadHandlerWithObjectOfGivenSize(objectSize, time.Time{}) + t.mockBucket.On("BucketType").Return(gcs.BucketType{Pirlo: tc.pirloState}) + if tc.expectedMethod == "CreateAppendableObjectWriter" { + t.mockBucket.On("CreateAppendableObjectWriter", mock.Anything, mock.Anything).Return(&storagemock.Writer{}, nil) + } else { + t.mockBucket.On("CreateObjectChunkWriter", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&storagemock.Writer{}, nil) + } + + _ = t.uh.createObjectWriter(context.Background()) + + if tc.expectedMethod == "CreateAppendableObjectWriter" { + t.mockBucket.AssertCalled(t.T(), "CreateAppendableObjectWriter", mock.Anything, mock.Anything) + } else { + t.mockBucket.AssertCalled(t.T(), "CreateObjectChunkWriter", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + } + }) + } +} + +func (t *UploadHandlerTest) TestCreateObjectWriter_CreateObjectChunkWriterCalled() { + t.createUploadHandlerWithObjectOfGivenSize(0, finalized) + t.mockBucket.On("BucketType").Return(gcs.BucketType{}) + t.mockBucket.On("CreateObjectChunkWriter", mock.Anything, mock.Anything, mock.Anything).Return(&storagemock.Writer{}, nil) + + _ = t.uh.createObjectWriter(context.Background()) + + t.mockBucket.AssertCalled(t.T(), "CreateObjectChunkWriter", mock.Anything, mock.Anything) +} + +func (t *UploadHandlerTest) TestCreateObjectWriter_CreateObjectChunkWriterCalledForLocalFile() { + t.mockBucket.On("BucketType").Return(gcs.BucketType{}) + t.mockBucket.On("CreateObjectChunkWriter", mock.Anything, mock.Anything, mock.Anything).Return(&storagemock.Writer{}, nil) + + _ = t.uh.createObjectWriter(context.Background()) + + t.mockBucket.AssertCalled(t.T(), "CreateObjectChunkWriter", mock.Anything, mock.Anything) +} + +func (t *UploadHandlerTest) TestEnsureWriter_CreateAppendableWriterIsSuccessful() { + t.createUploadHandlerWithObjectOfGivenSize(objectSize, time.Time{}) + t.mockBucket.On("BucketType").Return(gcs.BucketType{Zonal: true}) + writer := &storagemock.Writer{} + t.mockBucket.On("CreateAppendableObjectWriter", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(writer, nil) + + err := t.uh.createObjectWriter(context.Background()) + + assert.Nil(t.T(), err) + assert.NotNil(t.T(), t.uh.writer) +} +func (t *UploadHandlerTest) TestEnsureWriter_CreateAppendableWriterReturnsError() { + t.createUploadHandlerWithObjectOfGivenSize(objectSize, time.Time{}) + t.mockBucket.On("BucketType").Return(gcs.BucketType{Zonal: true}) + expectedErr := fmt.Errorf("createAppendableObjectWriter failed") + t.mockBucket.On("CreateAppendableObjectWriter", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, expectedErr) + + err := t.uh.ensureWriter(context.Background()) + + assert.NotNil(t.T(), err) + assert.Nil(t.T(), t.uh.writer) +} + +func (t *UploadHandlerTest) TestEnsureWriter_CreateObjectChunkWriterIsSuccessful() { + t.createUploadHandlerWithObjectOfGivenSize(0, finalized) + t.mockBucket.On("BucketType").Return(gcs.BucketType{}) + writer := &storagemock.Writer{} + t.mockBucket.On("CreateObjectChunkWriter", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(writer, nil) + + err := t.uh.ensureWriter(context.Background()) + + assert.Nil(t.T(), err) + assert.NotNil(t.T(), t.uh.writer) +} +func (t *UploadHandlerTest) TestEnsureWriter_CreateObjectChunkWriterReturnsError() { + t.createUploadHandlerWithObjectOfGivenSize(0, finalized) + t.mockBucket.On("BucketType").Return(gcs.BucketType{}) + expectedErr := fmt.Errorf("createObjectChunkWriter failed") + t.mockBucket.On("CreateObjectChunkWriter", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, expectedErr) + + err := t.uh.ensureWriter(context.Background()) + + assert.NotNil(t.T(), err) + assert.Nil(t.T(), t.uh.writer) +} + +func (t *UploadHandlerTest) TestMultipleBlockUpload() { // CreateObjectChunkWriter -- should be called once. - writer := storagemock.NewMockWriter("mockObject", false, false) + writer := &storagemock.Writer{} + mockObj := &gcs.MinObject{} + t.mockBucket.On("BucketType").Return(gcs.BucketType{}) t.mockBucket.On("CreateObjectChunkWriter", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(writer, nil) + t.mockBucket.On("FinalizeUpload", mock.Anything, writer).Return(mockObj, nil) // Upload the blocks. + blocks := t.createBlocks(5) for _, b := range blocks { - err := t.uh.Upload(b) + err := t.uh.Upload(context.Background(), b) require.NoError(t.T(), err) } // Finalize. - err := t.uh.Finalize() + obj, err := t.uh.Finalize(context.Background()) require.NoError(t.T(), err) - // The blocks should be available on the free channel for reuse. + require.NotNil(t.T(), obj) + assert.Equal(t.T(), mockObj, obj) + // All the 5 blocks should be available on the free channel for reuse. + assert.Equal(t.T(), 5, t.uh.blockPool.TotalFreeBlocks()) for _, expect := range blocks { - got := <-t.uh.freeBlocksCh + got, err := t.uh.blockPool.Get() + require.NoError(t.T(), err) assert.Equal(t.T(), expect, got) } - // All goroutines for upload should have exited. - done := make(chan struct{}) - go func() { - t.uh.wg.Wait() - close(done) - }() - select { - case <-done: - case <-time.After(100 * time.Millisecond): - t.T().Error("Timeout waiting for WaitGroup") - } + assert.Equal(t.T(), 0, t.uh.blockPool.TotalFreeBlocks()) + assertAllBlocksProcessed(t.T(), t.uh) } -func (t *UploadHandlerTest) TestUpload_CreateObjectWriterFails() { +func (t *UploadHandlerTest) TestUploadWhenCreateObjectWriterFails() { // Create a block. b, err := t.blockPool.Get() require.NoError(t.T(), err) // CreateObjectChunkWriter -- should be called once. + t.mockBucket.On("BucketType").Return(gcs.BucketType{}) t.mockBucket.On("CreateObjectChunkWriter", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("taco")) // Upload the block. - err = t.uh.Upload(b) + err = t.uh.Upload(context.Background(), b) + require.Error(t.T(), err) assert.ErrorContains(t.T(), err, "createObjectWriter") assert.ErrorContains(t.T(), err, "taco") } func (t *UploadHandlerTest) TestFinalizeWithWriterAlreadyPresent() { - writer := storagemock.NewMockWriter("mockObject", false, false) + writer := &storagemock.Writer{} + mockObj := &gcs.MinObject{} + t.mockBucket.On("FinalizeUpload", mock.Anything, writer).Return(mockObj, nil) t.uh.writer = writer - err := t.uh.Finalize() + obj, err := t.uh.Finalize(context.Background()) - assert.NoError(t.T(), err) + require.NoError(t.T(), err) + require.NotNil(t.T(), obj) + assert.Equal(t.T(), mockObj, obj) } func (t *UploadHandlerTest) TestFinalizeWithNoWriter() { - writer := storagemock.NewMockWriter("mockObject", false, false) + writer := &storagemock.Writer{} + t.mockBucket.On("BucketType").Return(gcs.BucketType{}) t.mockBucket.On("CreateObjectChunkWriter", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(writer, nil) assert.Nil(t.T(), t.uh.writer) + mockObj := &gcs.MinObject{} + t.mockBucket.On("FinalizeUpload", mock.Anything, writer).Return(mockObj, nil) - err := t.uh.Finalize() + obj, err := t.uh.Finalize(context.Background()) - assert.NoError(t.T(), err) + require.NoError(t.T(), err) + require.NotNil(t.T(), obj) + assert.Equal(t.T(), mockObj, obj) } -func (t *UploadHandlerTest) TestFinalizeWithNoWriter_CreateObjectWriterFails() { +func (t *UploadHandlerTest) TestFinalizeWithNoWriterWhenCreateObjectWriterFails() { + t.mockBucket.On("BucketType").Return(gcs.BucketType{}) t.mockBucket.On("CreateObjectChunkWriter", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("taco")) assert.Nil(t.T(), t.uh.writer) - err := t.uh.Finalize() + obj, err := t.uh.Finalize(context.Background()) - assert.Error(t.T(), err) + require.Error(t.T(), err) assert.ErrorContains(t.T(), err, "taco") assert.ErrorContains(t.T(), err, "createObjectWriter") + assert.Nil(t.T(), obj) } -func (t *UploadHandlerTest) TestFinalize_WriterCloseFails() { - writer := storagemock.NewMockWriter("mockObject", false, true) +func (t *UploadHandlerTest) TestFinalizeWhenFinalizeUploadFails() { + writer := &storagemock.Writer{} t.mockBucket.On("CreateObjectChunkWriter", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(writer, nil) assert.Nil(t.T(), t.uh.writer) + mockObj := &gcs.MinObject{} + t.mockBucket.On("BucketType").Return(gcs.BucketType{}) + t.mockBucket.On("FinalizeUpload", mock.Anything, writer).Return(mockObj, fmt.Errorf("taco")) + + obj, err := t.uh.Finalize(context.Background()) + + require.Error(t.T(), err) + assert.Nil(t.T(), obj) + assert.ErrorContains(t.T(), err, "taco") + assertUploadFailureError(t.T(), t.uh) +} + +func (t *UploadHandlerTest) TestFlushWithWriterAlreadyPresent() { + writer := &storagemock.Writer{} + mockObject := &gcs.MinObject{Size: 100} + t.mockBucket.On("FlushPendingWrites", mock.Anything, writer).Return(mockObject, nil) + t.uh.writer = writer + + o, err := t.uh.FlushPendingWrites(context.Background()) + + require.NoError(t.T(), err) + assert.Equal(t.T(), mockObject, o) +} + +func (t *UploadHandlerTest) TestFlushWithNoWriter() { + writer := &storagemock.Writer{} + t.mockBucket.On("BucketType").Return(gcs.BucketType{}) + t.mockBucket.On("CreateObjectChunkWriter", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(writer, nil) + assert.Nil(t.T(), t.uh.writer) + mockObject := &gcs.MinObject{Size: 10} + t.mockBucket.On("FlushPendingWrites", mock.Anything, writer).Return(mockObject, nil) + + o, err := t.uh.FlushPendingWrites(context.Background()) + + require.NoError(t.T(), err) + assert.Equal(t.T(), mockObject, o) +} + +func (t *UploadHandlerTest) TestFlushWithNoWriterWhenCreateObjectWriterFails() { + t.mockBucket.On("BucketType").Return(gcs.BucketType{}) + t.mockBucket.On("CreateObjectChunkWriter", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("taco")) + assert.Nil(t.T(), t.uh.writer) + + o, err := t.uh.FlushPendingWrites(context.Background()) + + require.Error(t.T(), err) + assert.ErrorContains(t.T(), err, "taco") + assert.ErrorContains(t.T(), err, "createObjectWriter") + assert.Nil(t.T(), o) +} + +func (t *UploadHandlerTest) TestFlushWhenFlushPendingWritesFails() { + writer := &storagemock.Writer{} + t.mockBucket.On("BucketType").Return(gcs.BucketType{}) + t.mockBucket.On("CreateObjectChunkWriter", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(writer, nil) + assert.Nil(t.T(), t.uh.writer) + var minObj *gcs.MinObject = nil + t.mockBucket.On("FlushPendingWrites", mock.Anything, writer).Return(minObj, fmt.Errorf("taco")) + + o, err := t.uh.FlushPendingWrites(context.Background()) + + require.Error(t.T(), err) + assert.Nil(t.T(), nil, o) + assert.ErrorContains(t.T(), err, "taco") + assertUploadFailureError(t.T(), t.uh) +} - err := t.uh.Finalize() +func (t *UploadHandlerTest) TestUploadSingleBlockThrowsErrorInCopy() { + // Create a block with test data. + b, err := t.blockPool.Get() + require.NoError(t.T(), err) + _, err = b.Write([]byte("test data")) + require.NoError(t.T(), err) + // CreateObjectChunkWriter -- should be called once. + writer := &storagemock.Writer{} + t.mockBucket.On("BucketType").Return(gcs.BucketType{}) + t.mockBucket.On("CreateObjectChunkWriter", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(writer, nil) + // First write will be an error and Close will be successful. + writer.On("Write", mock.Anything).Return(0, fmt.Errorf("taco")).Once() + + // Upload the block. + err = t.uh.Upload(context.Background(), b) + + require.NoError(t.T(), err) + // Expect an error on upload due to error while copying content to GCS writer. + assertUploadFailureError(t.T(), t.uh) + assertAllBlocksProcessed(t.T(), t.uh) + assert.Equal(t.T(), 1, t.uh.blockPool.TotalFreeBlocks()) +} + +func (t *UploadHandlerTest) TestUploadMultipleBlocksThrowsErrorInCopy() { + // Create some blocks. + blocks := t.createBlocks(4) + for i := range 4 { + n, err := blocks[i].Write([]byte("testdata" + strconv.Itoa(i) + " ")) + require.Equal(t.T(), 10, n) + require.NoError(t.T(), err) + } + // CreateObjectChunkWriter -- should be called once. + writer := &storagemock.Writer{} + t.mockBucket.On("BucketType").Return(gcs.BucketType{}) + t.mockBucket.On("CreateObjectChunkWriter", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(writer, nil) + // Second write will be an error and rest of the operations will be successful. + writer. + On("Write", mock.Anything).Return(10, nil).Once(). + On("Write", mock.Anything).Return(0, fmt.Errorf("taco")) + + // Upload the blocks. + for _, b := range blocks { + err := t.uh.Upload(context.Background(), b) + require.NoError(t.T(), err) + } + + assertUploadFailureError(t.T(), t.uh) + assertAllBlocksProcessed(t.T(), t.uh) + assert.Equal(t.T(), 4, t.uh.blockPool.TotalFreeBlocks()) +} + +func assertUploadFailureError(t *testing.T, handler *UploadHandler) { + t.Helper() + for { + select { + case <-time.After(200 * time.Millisecond): + t.Error("Expected an error in uploader") + default: + if handler.UploadError() != nil { + return + } + } + } +} + +func assertAllBlocksProcessed(t *testing.T, handler *UploadHandler) { + t.Helper() + + // All blocks for upload should have been processed. + done := make(chan struct{}) + go func() { + handler.wg.Wait() + close(done) + }() + select { + case <-done: + case <-time.After(100 * time.Millisecond): + t.Error("Timeout waiting for WaitGroup") + } +} + +func TestUploadErrorReturnsError(t *testing.T) { + mockUploadError := fmt.Errorf("error") + uploadHandler := &UploadHandler{} + uploadHandler.uploadError.Store(&mockUploadError) + + actualUploadError := uploadHandler.UploadError() + + assert.Equal(t, mockUploadError, actualUploadError) +} + +func TestUploadErrorReturnsNil(t *testing.T) { + uploadHandler := &UploadHandler{} + + actualUploadError := uploadHandler.UploadError() + + assert.Nil(t, actualUploadError) +} + +func (t *UploadHandlerTest) TestMultipleBlockAwaitBlocksUpload() { + // CreateObjectChunkWriter -- should be called once. + writer := &storagemock.Writer{} + t.mockBucket.On("BucketType").Return(gcs.BucketType{}) + t.mockBucket.On("CreateObjectChunkWriter", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(writer, nil) + // Upload the blocks. + for _, b := range t.createBlocks(5) { + err := t.uh.Upload(context.Background(), b) + require.NoError(t.T(), err) + } + + // AwaitBlocksUpload. + t.uh.AwaitBlocksUpload() + + assert.Equal(t.T(), 5, t.uh.blockPool.TotalFreeBlocks()) + assert.Equal(t.T(), 0, len(t.uh.uploadCh)) + assertAllBlocksProcessed(t.T(), t.uh) +} + +func (t *UploadHandlerTest) TestUploadHandlerCancelUpload() { + cancelCalled := false + t.uh.cancelFunc = func() { cancelCalled = true } + + t.uh.CancelUpload() + + assert.True(t.T(), cancelCalled) +} + +func (t *UploadHandlerTest) TestCreateObjectChunkWriterIsCalledWithCorrectRequestParametersForEmptyGCSObject() { + t.uh.obj = &gcs.Object{ + Name: t.uh.objectName, + ContentType: "image/png", + Size: 0, + ContentEncoding: "gzip", + Generation: 10, + MetaGeneration: 20, + Acl: nil, + Finalized: finalized, + } + + // CreateObjectChunkWriter -- should be called once with correct request parameters. + writer := &storagemock.Writer{} + mockObj := &gcs.Object{} + t.mockBucket.On("BucketType").Return(gcs.BucketType{}) + t.mockBucket.On("CreateObjectChunkWriter", + mock.Anything, + mock.MatchedBy(func(req *gcs.CreateObjectRequest) bool { + return req.Name == t.uh.objectName && + *req.GenerationPrecondition == t.uh.obj.Generation && + *req.MetaGenerationPrecondition == t.uh.obj.MetaGeneration && + req.ContentEncoding == t.uh.obj.ContentEncoding && + req.ContentType == t.uh.obj.ContentType && + req.ChunkTransferTimeoutSecs == chunkTransferTimeoutSecs + }), + mock.Anything, + mock.Anything).Return(writer, nil) + t.mockBucket.On("FinalizeUpload", mock.Anything, writer).Return(mockObj, nil) + + // Create a block. + b, err := t.blockPool.Get() + require.NoError(t.T(), err) + // Upload the block. + err = t.uh.Upload(context.Background(), b) + require.NoError(t.T(), err) +} + +func (t *UploadHandlerTest) TestCreateObjectChunkWriterIsCalledWithCorrectRequestParametersForLocalInode() { + assert.Nil(t.T(), t.uh.obj) + + // CreateObjectChunkWriter -- should be called once with correct request parameters. + writer := &storagemock.Writer{} + mockObj := &gcs.Object{} + t.mockBucket.On("BucketType").Return(gcs.BucketType{}) + t.mockBucket.On("CreateObjectChunkWriter", + mock.Anything, + mock.MatchedBy(func(req *gcs.CreateObjectRequest) bool { + return req.Name == t.uh.objectName && + *req.GenerationPrecondition == 0 && + req.MetaGenerationPrecondition == nil && + req.ChunkTransferTimeoutSecs == chunkTransferTimeoutSecs + }), + mock.Anything, + mock.Anything).Return(writer, nil) + t.mockBucket.On("FinalizeUpload", mock.Anything, writer).Return(mockObj, nil) + + // Create a block. + b, err := t.blockPool.Get() + require.NoError(t.T(), err) + // Upload the block. + err = t.uh.Upload(context.Background(), b) + require.NoError(t.T(), err) +} + +func (t *UploadHandlerTest) TestDestroy() { + testCases := []struct { + name string + uploadChClosed bool + }{ + { + name: "UploadChNotClosed", + uploadChClosed: false, + }, + { + name: "UploadChClosed", + uploadChClosed: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + // Add blocks to uploadCh. + for _, b := range t.createBlocks(5) { + t.uh.uploadCh <- b + t.uh.wg.Add(1) + } + if tc.uploadChClosed { + close(t.uh.uploadCh) + } + + t.uh.Destroy() + + assertAllBlocksProcessed(t.T(), t.uh) + assert.Equal(t.T(), 5, t.uh.blockPool.TotalFreeBlocks()) + assert.Equal(t.T(), 0, len(t.uh.uploadCh)) + // Check if uploadCh is closed. + select { + case <-t.uh.uploadCh: + default: + assert.Fail(t.T(), "uploadCh not closed") + } + }) + } +} + +func (t *UploadHandlerTest) createBlocks(count int) []block.Block { + var blocks []block.Block + for range count { + b, err := t.blockPool.Get() + require.NoError(t.T(), err) + blocks = append(blocks, b) + } - assert.Error(t.T(), err) - assert.ErrorContains(t.T(), err, "writer.Close") + return blocks } diff --git a/internal/cache/data/byte_range_map.go b/internal/cache/data/byte_range_map.go new file mode 100644 index 0000000000..9636f37b94 --- /dev/null +++ b/internal/cache/data/byte_range_map.go @@ -0,0 +1,158 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package data + +import ( + "slices" + "sync" +) + +const DefaultChunkSize = 1024 * 1024 // 1MB + +// ByteRangeMap tracks which chunks have been downloaded in a sparse file. +// The chunk size should match the actual download chunk size for efficient tracking. +type ByteRangeMap struct { + mu sync.RWMutex + chunkSize uint64 + fileSize uint64 // Total size of the file + chunks map[uint64]bool // chunk ID -> downloaded + totalBytes uint64 // Total bytes downloaded +} + +// NewByteRangeMap creates a new empty ByteRangeMap with the specified chunk size and file size. +// The chunkSize should match the download chunk size (e.g., DownloadChunkSizeMb * 1MB). +func NewByteRangeMap(chunkSize, fileSize uint64) *ByteRangeMap { + if chunkSize == 0 { + chunkSize = DefaultChunkSize + } + return &ByteRangeMap{ + chunkSize: chunkSize, + fileSize: fileSize, + chunks: make(map[uint64]bool), + } +} + +// chunkID returns the chunk ID for a given byte offset +func (brm *ByteRangeMap) chunkID(offset uint64) uint64 { + return offset / brm.chunkSize +} + +// chunkSizeOf returns the size of a specific chunk, handling the last chunk which might be smaller. +func (brm *ByteRangeMap) chunkSizeOf(chunkID uint64) uint64 { + chunkStart := chunkID * brm.chunkSize + if chunkStart >= brm.fileSize { + return 0 + } + chunkEnd := chunkStart + brm.chunkSize + if chunkEnd > brm.fileSize { + return brm.fileSize - chunkStart + } + return brm.chunkSize +} + +// AddRange marks all chunks in the range [start, end) as downloaded. +// Returns the total number of new bytes added. +func (brm *ByteRangeMap) AddRange(start, end uint64) uint64 { + brm.mu.Lock() + defer brm.mu.Unlock() + + if start >= end { + return 0 + } + + startChunk := brm.chunkID(start) + endChunk := brm.chunkID(end - 1) // inclusive end + + bytesAdded := uint64(0) + for chunkID := startChunk; chunkID <= endChunk; chunkID++ { + if !brm.chunks[chunkID] { + brm.chunks[chunkID] = true + bytesAdded += brm.chunkSizeOf(chunkID) + } + } + + brm.totalBytes += bytesAdded + return bytesAdded +} + +// ContainsRange checks if all chunks covering [start, end) have been downloaded +func (brm *ByteRangeMap) ContainsRange(start, end uint64) bool { + brm.mu.RLock() + defer brm.mu.RUnlock() + + if start >= end { + return true + } + + startChunk := brm.chunkID(start) + endChunk := brm.chunkID(end - 1) + + for chunkID := startChunk; chunkID <= endChunk; chunkID++ { + if !brm.chunks[chunkID] { + return false + } + } + return true +} + +// GetMissingChunks returns the IDs of chunks that haven't been downloaded. +func (brm *ByteRangeMap) GetMissingChunks(start, end uint64) []uint64 { + brm.mu.RLock() + defer brm.mu.RUnlock() + + if start >= end { + return nil + } + + var missing []uint64 + startChunk := brm.chunkID(start) + endChunk := brm.chunkID(end - 1) + + for chunkID := startChunk; chunkID <= endChunk; chunkID++ { + if !brm.chunks[chunkID] { + missing = append(missing, chunkID) + } + } + + return missing +} + +// TotalBytes returns the total number of bytes downloaded (sum of chunk sizes) +func (brm *ByteRangeMap) TotalBytes() uint64 { + brm.mu.RLock() + defer brm.mu.RUnlock() + return brm.totalBytes +} + +// Clear removes all chunk records +func (brm *ByteRangeMap) Clear() { + brm.mu.Lock() + defer brm.mu.Unlock() + brm.chunks = make(map[uint64]bool) + brm.totalBytes = 0 +} + +// Chunks returns a sorted list of all downloaded chunk IDs. (for debugging/testing) +func (brm *ByteRangeMap) Chunks() []uint64 { + brm.mu.RLock() + defer brm.mu.RUnlock() + + chunks := make([]uint64, 0, len(brm.chunks)) + for id := range brm.chunks { + chunks = append(chunks, id) + } + slices.Sort(chunks) + return chunks +} diff --git a/internal/cache/data/byte_range_map_test.go b/internal/cache/data/byte_range_map_test.go new file mode 100644 index 0000000000..814d3140b6 --- /dev/null +++ b/internal/cache/data/byte_range_map_test.go @@ -0,0 +1,334 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package data + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +const MB = 1024 * 1024 + +func TestByteRangeMap_AddRange(t *testing.T) { + tests := []struct { + name string + initialRanges [][2]uint64 // [start, end] pairs + addStart uint64 + addEnd uint64 + expectedChunks []uint64 + expectedAdded uint64 // bytes added (chunk size multiples) + }{ + { + name: "add to empty map - single chunk", + initialRanges: [][2]uint64{}, + addStart: 0, + addEnd: MB, + expectedChunks: []uint64{0}, + expectedAdded: MB, + }, + { + name: "add to empty map - partial chunk becomes full chunk", + initialRanges: [][2]uint64{}, + addStart: 100, + addEnd: 200, + expectedChunks: []uint64{0}, + expectedAdded: MB, // tracks full chunk + }, + { + name: "add non-overlapping chunk", + initialRanges: [][2]uint64{{0, MB}}, + addStart: 2 * MB, + addEnd: 3 * MB, + expectedChunks: []uint64{0, 2}, + expectedAdded: MB, + }, + { + name: "add already downloaded chunk", + initialRanges: [][2]uint64{{0, MB}}, + addStart: 100, + addEnd: 200, + expectedChunks: []uint64{0}, + expectedAdded: 0, // chunk already tracked + }, + { + name: "add range spanning two chunks", + initialRanges: [][2]uint64{}, + addStart: 0, + addEnd: 2 * MB, + expectedChunks: []uint64{0, 1}, + expectedAdded: 2 * MB, + }, + { + name: "add range spanning partial chunks", + initialRanges: [][2]uint64{}, + addStart: MB / 2, + addEnd: MB + MB/2, + expectedChunks: []uint64{0, 1}, + expectedAdded: 2 * MB, + }, + { + name: "add range that fills gap", + initialRanges: [][2]uint64{{0, MB}, {2 * MB, 3 * MB}}, + addStart: MB, + addEnd: 2 * MB, + expectedChunks: []uint64{0, 1, 2}, + expectedAdded: MB, + }, + { + name: "add overlapping chunks - some new", + initialRanges: [][2]uint64{{0, MB}}, + addStart: 0, + addEnd: 2 * MB, + expectedChunks: []uint64{0, 1}, + expectedAdded: MB, // only chunk 1 is new + }, + { + name: "add invalid range (start >= end)", + initialRanges: [][2]uint64{{0, MB}}, + addStart: 2 * MB, + addEnd: 2 * MB, + expectedChunks: []uint64{0}, + expectedAdded: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + brm := NewByteRangeMap(DefaultChunkSize, 100*MB) + // Add initial ranges + for _, r := range tt.initialRanges { + brm.AddRange(r[0], r[1]) + } + + // Add the test range + added := brm.AddRange(tt.addStart, tt.addEnd) + + // Check results + assert.Equal(t, tt.expectedAdded, added, "bytes added mismatch") + assert.Equal(t, tt.expectedChunks, brm.Chunks(), "chunks mismatch") + }) + } +} + +func TestByteRangeMap_ContainsRange(t *testing.T) { + brm := NewByteRangeMap(DefaultChunkSize, 100*MB) + brm.AddRange(0, MB) // chunk 0 + brm.AddRange(2*MB, 3*MB) // chunk 2 + brm.AddRange(5*MB, 6*MB) // chunk 5 + + tests := []struct { + name string + start uint64 + end uint64 + expected bool + }{ + {"fully contained in first chunk", 100, 200, true}, + {"exact match first chunk", 0, MB, true}, + {"fully contained in middle chunk", 2*MB + 100, 2*MB + 200, true}, + {"spans downloaded and missing chunk", MB / 2, MB + MB/2, false}, + {"starts in downloaded, ends in missing", 100, MB + 100, false}, + {"completely outside ranges", 7 * MB, 8 * MB, false}, + {"empty range", MB + MB/2, MB + MB/2, true}, // empty ranges are considered contained + {"spans two downloaded chunks with gap", 0, 3 * MB, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := brm.ContainsRange(tt.start, tt.end) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestByteRangeMap_GetMissingChunks(t *testing.T) { + brm := NewByteRangeMap(DefaultChunkSize, 100*MB) + brm.AddRange(0, MB) // chunk 0 + brm.AddRange(2*MB, 3*MB) // chunk 2 + brm.AddRange(5*MB, 6*MB) // chunk 5 + + tests := []struct { + name string + start uint64 + end uint64 + expected []uint64 + }{ + { + name: "fully covered range", + start: 100, + end: 200, + expected: nil, + }, + { + name: "single missing chunk", + start: MB, + end: 2 * MB, + expected: []uint64{1}, + }, + { + name: "multiple missing chunks", + start: 0, + end: 6 * MB, + expected: []uint64{1, 3, 4}, + }, + { + name: "completely missing range", + start: 10 * MB, + end: 11 * MB, + expected: []uint64{10}, + }, + { + name: "partial chunk request - missing", + start: MB + 100, + end: MB + 200, + expected: []uint64{1}, + }, + { + name: "partial chunk request - present", + start: 100, + end: 200, + expected: nil, + }, + { + name: "empty range", + start: MB + MB/2, + end: MB + MB/2, + expected: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := brm.GetMissingChunks(tt.start, tt.end) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestByteRangeMap_TotalBytes(t *testing.T) { + brm := NewByteRangeMap(DefaultChunkSize, 100*MB) + + assert.Equal(t, uint64(0), brm.TotalBytes(), "empty map should have 0 bytes") + + brm.AddRange(0, MB) // 1 chunk + assert.Equal(t, uint64(MB), brm.TotalBytes()) + + brm.AddRange(2*MB, 4*MB) // 2 chunks + assert.Equal(t, uint64(3*MB), brm.TotalBytes()) + + brm.AddRange(MB, 2*MB) // 1 chunk, fills gap + assert.Equal(t, uint64(4*MB), brm.TotalBytes()) // 4 contiguous chunks +} + +func TestByteRangeMap_TotalBytes_PartialLastChunk(t *testing.T) { + // Test partial last chunk + // Chunk size 100 bytes, File size 110 bytes + brmSmall := NewByteRangeMap(100, 110) + brmSmall.AddRange(100, 110) + + // Should be 10 bytes, not 100 + assert.Equal(t, uint64(10), brmSmall.TotalBytes(), "partial last chunk size mismatch") +} + +func TestByteRangeMap_Clear(t *testing.T) { + brm := NewByteRangeMap(DefaultChunkSize, 100*MB) + brm.AddRange(0, MB) + brm.AddRange(2*MB, 3*MB) + + assert.Equal(t, uint64(2*MB), brm.TotalBytes()) + + brm.Clear() + + assert.Equal(t, uint64(0), brm.TotalBytes()) + assert.Empty(t, brm.Chunks()) +} + +func TestByteRangeMap_ConcurrentAccess(t *testing.T) { + brm := NewByteRangeMap(DefaultChunkSize, 100*MB) + + // This test just ensures no race conditions occur + // Run with -race flag to detect issues + done := make(chan bool) + + // Writer goroutine + go func() { + for i := uint64(0); i < 10; i++ { + brm.AddRange(i*MB, (i+1)*MB) + } + done <- true + }() + + // Reader goroutine + go func() { + for i := uint64(0); i < 10; i++ { + brm.ContainsRange(i*MB, (i+1)*MB) + brm.GetMissingChunks(i*MB, (i+2)*MB) + brm.TotalBytes() + } + done <- true + }() + + <-done + <-done +} + +func TestByteRangeMap_ChunkAlignment(t *testing.T) { + brm := NewByteRangeMap(DefaultChunkSize, 100*MB) + + // Test that partial byte ranges get tracked as full chunks + brm.AddRange(100, 200) + + // Should track chunk 0 + assert.True(t, brm.ContainsRange(0, MB)) + assert.True(t, brm.ContainsRange(100, 200)) + assert.True(t, brm.ContainsRange(0, 1000)) + assert.Equal(t, uint64(MB), brm.TotalBytes()) + + // Should not contain chunk 1 + assert.False(t, brm.ContainsRange(MB, MB+1)) + assert.False(t, brm.ContainsRange(100, MB+100)) +} + +func TestByteRangeMap_Chunks(t *testing.T) { + brm := NewByteRangeMap(DefaultChunkSize, 100*MB) + brm.AddRange(0, MB) // chunk 0 + brm.AddRange(2*MB, 3*MB) // chunk 2 + brm.AddRange(5*MB, 6*MB) // chunk 5 + + expected := []uint64{0, 2, 5} + assert.Equal(t, expected, brm.Chunks()) +} + +func TestByteRangeMap_chunkSizeOf(t *testing.T) { + chunkSize := uint64(100) + fileSize := uint64(150) + brm := NewByteRangeMap(chunkSize, fileSize) + + tests := []struct { + name string + chunkID uint64 + expected uint64 + }{ + {"chunk 0 (full)", 0, 100}, + {"chunk 1 (partial)", 1, 50}, + {"chunk 2 (out of bounds)", 2, 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, brm.chunkSizeOf(tt.chunkID)) + }) + } +} diff --git a/internal/cache/data/file_info.go b/internal/cache/data/file_info.go index 96366416ce..18f08a0c0e 100644 --- a/internal/cache/data/file_info.go +++ b/internal/cache/data/file_info.go @@ -16,9 +16,11 @@ package data import ( "errors" - "fmt" "os" + "strconv" "time" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/util/diskutil" ) const InvalidKeyAttributes = "key attributes not initialised" @@ -39,23 +41,61 @@ func GetFileInfoKeyName(objectName string, bucketCreationTime time.Time, bucketN if bucketName == "" || objectName == "" { return "", errors.New(InvalidKeyAttributes) } - unixTimeString := fmt.Sprintf("%d", bucketCreationTime.Unix()) - return bucketName + unixTimeString + objectName, nil + size := len(bucketName) + len(objectName) + 20 + keyBytes := make([]byte, 0, size) + keyBytes = append(keyBytes, bucketName...) + keyBytes = strconv.AppendInt(keyBytes, bucketCreationTime.Unix(), 10) + keyBytes = append(keyBytes, objectName...) + return string(keyBytes), nil } type FileInfo struct { Key FileInfoKey ObjectGeneration int64 - Offset uint64 + Offset uint64 // For non-sparse files: bytes downloaded so far. For sparse files: set to MaxUint64 as sentinel FileSize uint64 + SparseMode bool // Whether this file is using sparse file mode + DownloadedChunks *ByteRangeMap // For sparse files: tracks which chunks have been downloaded + // CacheDirVolumeBlockSize is used to round-up the FileSize to calculate the speculative + // disk utilization of this file in cache-directory. + // Speculative size = Round-up of FileSize to the next multiple of CacheDirVolumeBlockSize. + // 0 or 1 mean size in cache-dir is same as FileSize. + CacheDirVolumeBlockSize uint64 } -func (fi FileInfo) Size() uint64 { +// ContentSize returns the logical size of the given file, or in other words, the size +// of the corresponding GCS object. +func (fi FileInfo) ContentSize() uint64 { + // For sparse files, return actual downloaded bytes, not full file size + if fi.SparseMode && fi.DownloadedChunks != nil { + return fi.DownloadedChunks.TotalBytes() + } return fi.FileSize } +// Size returns the speculative physical size on disk, rounded up to the volume block size. +// If CacheDirVolumeBlockSize is 0 or 1, it returns the exact logical ContentSize. +// This satisfies the LRU ValueType interface for eviction accounting. +func (fi FileInfo) Size() uint64 { + return diskutil.GetSpeculativeFileSizeOnDisk(fi.ContentSize(), fi.CacheDirVolumeBlockSize) +} + type FileSpec struct { Path string FilePerm os.FileMode DirPerm os.FileMode } + +// NewFileInfo creates and returns a new FileInfo struct, ensuring that +// the CacheDirVolumeBlockSize field is explicitly provided. +func NewFileInfo(key FileInfoKey, objectGeneration int64, fileSize uint64, offset uint64, sparseMode bool, downloadedChunks *ByteRangeMap, cacheDirVolumeBlockSize uint64) FileInfo { + return FileInfo{ + Key: key, + ObjectGeneration: objectGeneration, + FileSize: fileSize, + Offset: offset, + SparseMode: sparseMode, + DownloadedChunks: downloadedChunks, + CacheDirVolumeBlockSize: cacheDirVolumeBlockSize, + } +} diff --git a/internal/cache/data/file_info_benchmark_test.go b/internal/cache/data/file_info_benchmark_test.go new file mode 100644 index 0000000000..a4e076cd1d --- /dev/null +++ b/internal/cache/data/file_info_benchmark_test.go @@ -0,0 +1,28 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package data + +import ( + "testing" + "time" +) + +func BenchmarkGetFileInfoKeyName_Optimized(b *testing.B) { + bucketCreationTime := time.Unix(TestTimeInEpoch, 0) + b.ResetTimer() + for b.Loop() { + _, _ = GetFileInfoKeyName(TestObjectName, bucketCreationTime, TestBucketName) + } +} diff --git a/internal/cache/data/file_info_test.go b/internal/cache/data/file_info_test.go index 72c957302d..d2a8842a60 100644 --- a/internal/cache/data/file_info_test.go +++ b/internal/cache/data/file_info_test.go @@ -19,17 +19,9 @@ import ( "testing" "time" - . "github.com/jacobsa/ogletest" + "github.com/stretchr/testify/assert" ) -func TestFileInfo(t *testing.T) { RunTests(t) } - -type fileInfoTestKey struct { -} - -type fileInfoTest struct { -} - const TestDataFileSize uint64 = 23 const TestTimeInEpoch int64 = 1654041600 const TestBucketName = "test_bucket" @@ -37,11 +29,6 @@ const TestObjectName = "test/a.txt" const TestGeneration = 12345678 const ExpectedFileInfoKey = "test_bucket1654041600test/a.txt" -func init() { - RegisterTestSuite(&fileInfoTestKey{}) - RegisterTestSuite(&fileInfoTest{}) -} - func getTestFileInfoKey() FileInfoKey { return FileInfoKey{ BucketName: TestBucketName, @@ -50,51 +37,55 @@ func getTestFileInfoKey() FileInfoKey { } } -func (t *fileInfoTestKey) TestKeyMethod() { +func TestKeyMethod(t *testing.T) { fik := getTestFileInfoKey() key, err := fik.Key() - AssertEq(nil, err) - ExpectEq(ExpectedFileInfoKey, key) + assert.NoError(t, err) + unixCreationTimeString := fmt.Sprintf("%d", fik.BucketCreationTime.Unix()) + assert.Equal(t, fik.BucketName+unixCreationTimeString+fik.ObjectName, key) } -func (t *fileInfoTestKey) TestKeyMethodWithEmptyBucketName() { +func TestKeyMethodWithEmptyBucketName(t *testing.T) { fik := getTestFileInfoKey() fik.BucketName = "" key, err := fik.Key() - AssertEq(InvalidKeyAttributes, err.Error()) - ExpectEq("", key) + assert.Equal(t, InvalidKeyAttributes, err.Error()) + assert.Equal(t, "", key) } -func (t *fileInfoTestKey) TestKeyMethodWithZeroBucketCreationTime() { +func TestKeyMethodWithEmptyObjectName(t *testing.T) { fik := getTestFileInfoKey() + fik.ObjectName = "" key, err := fik.Key() - ExpectEq(nil, err) - unixCreationTimeString := fmt.Sprintf("%d", fik.BucketCreationTime.Unix()) - ExpectEq(fik.BucketName+unixCreationTimeString+fik.ObjectName, key) + assert.Equal(t, InvalidKeyAttributes, err.Error()) + assert.Equal(t, "", key) } -func (t *fileInfoTestKey) TestKeyMethodWithEmptyObjectName() { - fik := getTestFileInfoKey() - fik.ObjectName = "" +func TestContentSizeMethod(t *testing.T) { + fileContentSize := uint64(23) + blockSize := uint64(4096) + fi := NewFileInfo(getTestFileInfoKey(), TestGeneration, fileContentSize, 0, false, nil, blockSize) - key, err := fik.Key() - AssertEq(InvalidKeyAttributes, err.Error()) + contentSize := fi.ContentSize() - ExpectEq("", key) + // ContentSize() always returns content-size. + assert.Equal(t, fileContentSize, contentSize) } -func (t *fileInfoTest) TestSizeMethod() { - fi := FileInfo{ - Key: getTestFileInfoKey(), - ObjectGeneration: TestGeneration, - FileSize: TestDataFileSize, - } +func TestSizeMethod(t *testing.T) { + fileContentSize := uint64(23) + blockSize := uint64(4096) + fi := NewFileInfo(getTestFileInfoKey(), TestGeneration, fileContentSize, 0, false, nil, blockSize) + + size := fi.Size() - ExpectEq(TestDataFileSize, fi.Size()) + // Size() returns size on disk, which is always multiples of block-size. So, + // if content-size < block-size, then Size() return block-size. + assert.Equal(t, blockSize, size) } diff --git a/internal/cache/file/cache_handle.go b/internal/cache/file/cache_handle.go index 35229ef9aa..deb1cbbd8b 100644 --- a/internal/cache/file/cache_handle.go +++ b/internal/cache/file/cache_handle.go @@ -16,16 +16,17 @@ package file import ( "context" - "errors" "fmt" "io" "os" - - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/data" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/file/downloader" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/lru" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/util" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "sync/atomic" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/data" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/file/downloader" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" ) type CacheHandle struct { @@ -45,32 +46,33 @@ type CacheHandle struct { // isSequential saves if the current read performed via cache handle is sequential or // random. - isSequential bool + isSequential atomic.Bool // prevOffset stores the offset of previous cache handle read call. This is used // to decide the type of read. - prevOffset int64 + prevOffset atomic.Int64 } func NewCacheHandle(localFileHandle *os.File, fileDownloadJob *downloader.Job, fileInfoCache *lru.Cache, cacheFileForRangeRead bool, initialOffset int64) *CacheHandle { - return &CacheHandle{ + fch := CacheHandle{ fileHandle: localFileHandle, fileDownloadJob: fileDownloadJob, fileInfoCache: fileInfoCache, cacheFileForRangeRead: cacheFileForRangeRead, - isSequential: initialOffset == 0, - prevOffset: initialOffset, } + fch.isSequential.Store(initialOffset == 0) + fch.prevOffset.Store(initialOffset) + return &fch } func (fch *CacheHandle) validateCacheHandle() error { if fch.fileHandle == nil { - return errors.New(util.InvalidFileHandleErrMsg) + return util.ErrInvalidFileHandle } if fch.fileInfoCache == nil { - return errors.New(util.InvalidFileInfoCacheErrMsg) + return util.ErrInvalidFileInfoCache } return nil @@ -82,28 +84,25 @@ func (fch *CacheHandle) shouldReadFromCache(jobStatus *downloader.JobStatus, req if jobStatus.Err != nil || jobStatus.Name == downloader.Invalid || jobStatus.Name == downloader.Failed { - err := fmt.Errorf("%s: jobStatus: %s jobError: %w", util.InvalidFileDownloadJobErrMsg, jobStatus.Name, jobStatus.Err) - return err + return fmt.Errorf("%w: jobStatus: %s jobError: %w", util.ErrInvalidFileDownloadJob, jobStatus.Name, jobStatus.Err) } else if jobStatus.Offset < requiredOffset { - err := fmt.Errorf("%s: jobOffset: %d is less than required offset: %d", util.FallbackToGCSErrMsg, jobStatus.Offset, requiredOffset) - return err + return fmt.Errorf("%w: jobOffset: %d is less than required offset: %d", util.ErrFallbackToGCS, jobStatus.Offset, requiredOffset) } return err } -// validateEntryInFileInfoCache checks if entry is present for a given object in -// file info cache with same generation and at least requiredOffset. -// It returns nil if entry is present, otherwise returns an appropriate error. +// getFileInfoData returns the file-info cache entry for the given object +// in the associated bucket. // Whether to change the order in cache while lookup is controlled via // changeCacheOrder. -func (fch *CacheHandle) validateEntryInFileInfoCache(bucket gcs.Bucket, object *gcs.MinObject, requiredOffset uint64, changeCacheOrder bool) error { +func (fch *CacheHandle) getFileInfoData(bucket gcs.Bucket, object *gcs.MinObject, changeCacheOrder bool) (*data.FileInfo, error) { fileInfoKey := data.FileInfoKey{ BucketName: bucket.Name(), ObjectName: object.Name, } fileInfoKeyName, err := fileInfoKey.Key() if err != nil { - return fmt.Errorf("error while creating key for bucket %s and object %s: %w", bucket.Name(), object.Name, err) + return nil, fmt.Errorf("error while creating key for bucket %s and object %s: %w", bucket.Name(), object.Name, err) } var fileInfo lru.ValueType @@ -113,21 +112,35 @@ func (fch *CacheHandle) validateEntryInFileInfoCache(bucket gcs.Bucket, object * fileInfo = fch.fileInfoCache.LookUpWithoutChangingOrder(fileInfoKeyName) } if fileInfo == nil { - err = fmt.Errorf("%v: no entry found in file info cache for key %v", util.InvalidFileInfoCacheErrMsg, fileInfoKeyName) - return err + return nil, fmt.Errorf("%w: no entry found in file info cache for key %v", util.ErrInvalidFileInfoCache, fileInfoKeyName) } // The generation check below is required because it may happen that file // being read is evicted from cache during or after reading the required offset // from local cached file to `dst` buffer. - fileInfoData := fileInfo.(data.FileInfo) + fileInfoData, ok := fileInfo.(data.FileInfo) + if !ok { + return nil, fmt.Errorf("getFileInfoData: failed to get fileInfoData from file-cache for %q: %w", object.Name, util.ErrInvalidFileHandle) + } + + return &fileInfoData, nil +} + +// validateEntryInFileInfoCache checks if entry is present for a given object in +// file info cache with same generation and at least requiredOffset. +// It returns nil if entry is present, otherwise returns an appropriate error. +// Whether to change the order in cache while lookup is controlled via +// changeCacheOrder. +func (fch *CacheHandle) validateEntryInFileInfoCache(bucket gcs.Bucket, object *gcs.MinObject, requiredOffset uint64, changeCacheOrder bool) error { + fileInfoData, err := fch.getFileInfoData(bucket, object, changeCacheOrder) + if err != nil { + return fmt.Errorf("validateEntryInFileInfoCache: failed to get fileInfoData for %q: %w", object.Name, err) + } if fileInfoData.ObjectGeneration != object.Generation { - err = fmt.Errorf("%v: generation of cached object: %v is different from required generation: %v", util.InvalidFileInfoCacheErrMsg, fileInfoData.ObjectGeneration, object.Generation) - return err + return fmt.Errorf("%w: generation of cached object: %v is different from required generation: %v", util.ErrInvalidFileInfoCache, fileInfoData.ObjectGeneration, object.Generation) } if fileInfoData.Offset < requiredOffset { - err = fmt.Errorf("%v offset of cached object: %v is less than required offset %v", util.InvalidFileInfoCacheErrMsg, fileInfoData.Offset, requiredOffset) - return err + return fmt.Errorf("%w offset of cached object: %v is less than required offset %v", util.ErrInvalidFileInfoCache, fileInfoData.Offset, requiredOffset) } return nil @@ -148,11 +161,28 @@ func (fch *CacheHandle) Read(ctx context.Context, bucket gcs.Bucket, object *gcs return 0, false, fmt.Errorf("wrong offset requested: %d, object size: %d", offset, object.Size) } + fileInfoData, errFileInfo := fch.getFileInfoData(bucket, object, false) + if errFileInfo != nil { + return 0, false, fmt.Errorf("%w Error in getCachedFileInfo: %v", util.ErrInvalidFileInfoCache, errFileInfo) + } + + // New check to ensure we bail out early in case the requested read-offset is beyond the cached size for an unfinalized object. + // If the end-offset is within the cached size, then the read will be served from the cache. + // If offset is within the cached size and end-offset is beyond the cached-size, then + // we will fallback to GCS in the later logic, but we should still create the cache download job even in that case. + // + // Note: Change the below check to `(offset + len(dst)) > int64(fileInfoData.FileSize))` if the below + // check causes a problem in any edge-case. + if bucket.BucketType().Zonal && object.IsUnfinalized() && offset >= int64(fileInfoData.FileSize) { + err = util.ErrFallbackToGCS + return + } + // Checking before updating the previous offset. isSequentialRead := fch.IsSequential(offset) waitForDownload := true if !isSequentialRead { - fch.isSequential = false + fch.isSequential.Store(false) waitForDownload = false } @@ -167,9 +197,20 @@ func (fch *CacheHandle) Read(ctx context.Context, bucket gcs.Bucket, object *gcs requiredOffset = objSize } - // If fileDownloadJob is not nil, it's better to get status of cache file - // from the job itself than to use file info cache. - if fch.fileDownloadJob != nil { + // Handle sparse file reads + if fileInfoData.SparseMode { + cacheHit, err = fch.fileDownloadJob.HandleSparseRead(ctx, offset, requiredOffset) + if err != nil { + // Log the error and fallback to GCS + logger.Infof("Sparse file read failed: %v. Falling back to GCS.", err) + return 0, false, util.ErrFallbackToGCS + } + if !cacheHit { + return 0, false, util.ErrFallbackToGCS + } + } else if fch.fileDownloadJob != nil { + // If fileDownloadJob is not nil, it's better to get status of cache file + // from the job itself than to use file info cache. jobStatus := fch.fileDownloadJob.GetStatus() // If cacheFileForRangeRead is false and readType is random, download will // not be initiated. @@ -183,9 +224,9 @@ func (fch *CacheHandle) Read(ctx context.Context, bucket gcs.Bucket, object *gcs cacheHit = true } - fch.prevOffset = offset + fch.prevOffset.Store(offset) - if fch.fileDownloadJob.IsParallelDownloadsEnabled() { + if fch.fileDownloadJob.IsParallelDownloadsEnabled() && !fch.fileDownloadJob.IsExperimentalParallelDownloadsDefaultOn() { waitForDownload = false } @@ -204,7 +245,7 @@ func (fch *CacheHandle) Read(ctx context.Context, bucket gcs.Bucket, object *gcs // If fileDownloadJob is nil then it means either the job is successfully // completed or failed. The offset must be equal to size of object for job // to be completed. - err = fch.validateEntryInFileInfoCache(bucket, object, object.Size, false) + err = fch.validateEntryInFileInfoCache(bucket, object, fileInfoData.FileSize, false) if err != nil { return 0, false, err } @@ -222,14 +263,12 @@ func (fch *CacheHandle) Read(ctx context.Context, bucket gcs.Bucket, object *gcs // Ensure that the number of bytes read into dst buffer is equal to what is // requested. It will also help catch cases where file in cache is truncated // externally to size offset + x where x < requestedNumBytes. - errMsg := fmt.Sprintf("%s, number of bytes read from file in cache: %v are not equal to requested: %v", util.ErrInReadingFileHandleMsg, n, requestedNumBytes) - return 0, false, errors.New(errMsg) + return 0, false, fmt.Errorf("%w, number of bytes read from file in cache: %v are not equal to requested: %v", util.ErrInReadingFileHandle, n, requestedNumBytes) } err = nil } if err != nil { - err = fmt.Errorf("%s: while reading from %d offset of the local file: %w", util.ErrInReadingFileHandleMsg, offset, err) - return 0, false, err + return 0, false, fmt.Errorf("%w: while reading from %d offset of the local file: %w", util.ErrInReadingFileHandle, offset, err) } // Look up of file being read in file info cache is required to update the LRU @@ -246,15 +285,15 @@ func (fch *CacheHandle) Read(ctx context.Context, bucket gcs.Bucket, object *gcs // IsSequential returns true if the sequential read is being performed, false for // random read. func (fch *CacheHandle) IsSequential(currentOffset int64) bool { - if !fch.isSequential { + if !fch.isSequential.Load() { return false } - if currentOffset < fch.prevOffset { + if currentOffset < fch.prevOffset.Load() { return false } - if currentOffset-fch.prevOffset > downloader.ReadChunkSize { + if currentOffset-fch.prevOffset.Load() > downloader.ReadChunkSize { return false } diff --git a/internal/cache/file/cache_handle_test.go b/internal/cache/file/cache_handle_test.go index 4ea51e0c2e..655fc08fe6 100644 --- a/internal/cache/file/cache_handle_test.go +++ b/internal/cache/file/cache_handle_test.go @@ -18,7 +18,6 @@ import ( "context" "crypto/rand" "errors" - "fmt" "io" "math" "os" @@ -28,17 +27,21 @@ import ( "strings" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/data" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/file/downloader" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/lru" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/util" - "github.com/googlecloudplatform/gcsfuse/v2/internal/locker" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" + "cloud.google.com/go/storage/control/apiv2/controlpb" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/data" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/file/downloader" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/locker" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" "golang.org/x/sync/semaphore" ) @@ -71,12 +74,7 @@ func (cht *cacheHandleTest) addTestFileInfoEntryInCache() { BucketName: storage.TestBucketName, ObjectName: TestObjectName, } - fileInfo := data.FileInfo{ - Key: fileInfoKey, - ObjectGeneration: cht.object.Generation, - FileSize: cht.object.Size, - Offset: 0, - } + fileInfo := data.NewFileInfo(fileInfoKey, cht.object.Generation, cht.object.Size, 0, false, nil, 1) fileInfoKeyName, err := fileInfoKey.Key() assert.Nil(cht.T(), err) @@ -109,9 +107,14 @@ func (cht *cacheHandleTest) SetupTest() { ctx := context.Background() // Create bucket in fake storage. - cht.fakeStorage = storage.NewFakeStorage() + var err error + mockClient := new(storage.MockStorageControlClient) + cht.fakeStorage = storage.NewFakeStorageWithMockClient(mockClient, cfg.HTTP2) storageHandle := cht.fakeStorage.CreateStorageHandle() - cht.bucket = storageHandle.BucketHandle(ctx, storage.TestBucketName, "") + mockClient.On("GetStorageLayout", mock.Anything, mock.Anything, mock.Anything). + Return(&controlpb.StorageLayout{}, nil) + cht.bucket, err = storageHandle.BucketHandle(ctx, storage.TestBucketName, "") + assert.Nil(cht.T(), err) // Create test object in the bucket. testObjectContent := make([]byte, TestObjectSize) @@ -148,12 +151,19 @@ func (cht *cacheHandleTest) SetupTest() { func() {}, fileCacheConfig, semaphore.NewWeighted(math.MaxInt64), + metrics.NewNoopMetrics(), + tracing.NewNoopTracer(), + 1, ) cht.cacheHandle = NewCacheHandle(readLocalFileHandle, fileDownloadJob, cht.cache, false, 0) } func (cht *cacheHandleTest) TearDownTest() { + if cht.cacheHandle != nil && cht.cacheHandle.fileDownloadJob != nil { + cht.cacheHandle.fileDownloadJob.Invalidate() + } + cht.fakeStorage.ShutDown() err := cht.cacheHandle.Close() @@ -167,7 +177,7 @@ func (cht *cacheHandleTest) Test_validateCacheHandle_WithNilFileHandle() { err := cht.cacheHandle.validateCacheHandle() - assert.Equal(cht.T(), util.InvalidFileHandleErrMsg, err.Error()) + assert.True(cht.T(), errors.Is(err, util.ErrInvalidFileHandle)) } func (cht *cacheHandleTest) Test_validateCacheHandle_WithNilFileDownloadJob() { @@ -183,7 +193,7 @@ func (cht *cacheHandleTest) Test_validateCacheHandle_WithNilFileInfoCache() { err := cht.cacheHandle.validateCacheHandle() - assert.Equal(cht.T(), util.InvalidFileInfoCacheErrMsg, err.Error()) + assert.True(cht.T(), errors.Is(err, util.ErrInvalidFileInfoCache)) } func (cht *cacheHandleTest) Test_validateCacheHandle_WithNonNilMemberAttributes() { @@ -209,7 +219,7 @@ func (cht *cacheHandleTest) Test_Close_WithNilFileHandle() { } func (cht *cacheHandleTest) Test_IsSequential_WhenReadTypeIsNotSequential() { - cht.cacheHandle.isSequential = false + cht.cacheHandle.isSequential.Store(false) currentOffset := int64(3) isSeq := cht.cacheHandle.IsSequential(currentOffset) @@ -218,8 +228,8 @@ func (cht *cacheHandleTest) Test_IsSequential_WhenReadTypeIsNotSequential() { } func (cht *cacheHandleTest) Test_IsSequential_WhenPrevOffsetGreaterThanCurrent() { - cht.cacheHandle.isSequential = true - cht.cacheHandle.prevOffset = 5 + cht.cacheHandle.isSequential.Store(true) + cht.cacheHandle.prevOffset.Store(5) currentOffset := int64(3) isSeq := cht.cacheHandle.IsSequential(currentOffset) @@ -228,8 +238,8 @@ func (cht *cacheHandleTest) Test_IsSequential_WhenPrevOffsetGreaterThanCurrent() } func (cht *cacheHandleTest) Test_IsSequential_WhenOffsetDiffIsMoreThanMaxAllowed() { - cht.cacheHandle.isSequential = true - cht.cacheHandle.prevOffset = 5 + cht.cacheHandle.isSequential.Store(true) + cht.cacheHandle.prevOffset.Store(5) currentOffset := int64(8 + downloader.ReadChunkSize) isSeq := cht.cacheHandle.IsSequential(currentOffset) @@ -238,8 +248,8 @@ func (cht *cacheHandleTest) Test_IsSequential_WhenOffsetDiffIsMoreThanMaxAllowed } func (cht *cacheHandleTest) Test_IsSequential_WhenOffsetDiffIsLessThanMaxAllowed() { - cht.cacheHandle.isSequential = true - cht.cacheHandle.prevOffset = 5 + cht.cacheHandle.isSequential.Store(true) + cht.cacheHandle.prevOffset.Store(5) currentOffset := int64(10) isSeq := cht.cacheHandle.IsSequential(currentOffset) @@ -248,8 +258,8 @@ func (cht *cacheHandleTest) Test_IsSequential_WhenOffsetDiffIsLessThanMaxAllowed } func (cht *cacheHandleTest) Test_IsSequential_WhenOffsetDiffIsEqualToMaxAllowed() { - cht.cacheHandle.isSequential = true - cht.cacheHandle.prevOffset = 5 + cht.cacheHandle.isSequential.Store(true) + cht.cacheHandle.prevOffset.Store(5) currentOffset := int64(5 + downloader.ReadChunkSize) isSeq := cht.cacheHandle.IsSequential(currentOffset) @@ -265,7 +275,7 @@ func (cht *cacheHandleTest) Test_shouldReadFromCache_WithJobStateIsNotStarted() err := cht.cacheHandle.shouldReadFromCache(&jobStatus, requiredOffset) assert.NotNil(cht.T(), err) - assert.True(cht.T(), strings.Contains(err.Error(), util.FallbackToGCSErrMsg)) + assert.True(cht.T(), errors.Is(err, util.ErrFallbackToGCS)) } func (cht *cacheHandleTest) Test_shouldReadFromCache_WithJobStateIsFailed() { @@ -276,7 +286,7 @@ func (cht *cacheHandleTest) Test_shouldReadFromCache_WithJobStateIsFailed() { err := cht.cacheHandle.shouldReadFromCache(&jobStatus, requiredOffset) assert.NotNil(cht.T(), err) - assert.True(cht.T(), strings.Contains(err.Error(), util.InvalidFileDownloadJobErrMsg)) + assert.True(cht.T(), errors.Is(err, util.ErrInvalidFileDownloadJob)) } func (cht *cacheHandleTest) Test_shouldReadFromCache_WithJobStateIsInvalid() { @@ -287,7 +297,7 @@ func (cht *cacheHandleTest) Test_shouldReadFromCache_WithJobStateIsInvalid() { err := cht.cacheHandle.shouldReadFromCache(&jobStatus, requiredOffset) assert.NotNil(cht.T(), err) - assert.True(cht.T(), strings.Contains(err.Error(), util.InvalidFileDownloadJobErrMsg)) + assert.True(cht.T(), errors.Is(err, util.ErrInvalidFileDownloadJob)) } func (cht *cacheHandleTest) Test_shouldReadFromCache_WithJobStateIsCompleted() { @@ -310,7 +320,7 @@ func (cht *cacheHandleTest) Test_shouldReadFromCache_WithJobDownloadedOffsetIsLe err := cht.cacheHandle.shouldReadFromCache(&jobStatus, requiredOffset) assert.NotNil(cht.T(), err) - assert.True(cht.T(), strings.Contains(err.Error(), util.FallbackToGCSErrMsg)) + assert.True(cht.T(), errors.Is(err, util.ErrFallbackToGCS)) } func (cht *cacheHandleTest) Test_shouldReadFromCache_WithJobDownloadedOffsetSameAsRequiredOffset() { @@ -345,7 +355,7 @@ func (cht *cacheHandleTest) Test_shouldReadFromCache_WithNonNilJobStatusErr() { err := cht.cacheHandle.shouldReadFromCache(&jobStatus, requiredOffset) assert.NotNil(cht.T(), err) - assert.True(cht.T(), strings.Contains(err.Error(), util.InvalidFileDownloadJobErrMsg)) + assert.True(cht.T(), errors.Is(err, util.ErrInvalidFileDownloadJob)) } func (cht *cacheHandleTest) Test_validateEntryInFileInfoCache_FileInfoPresent() { @@ -355,12 +365,7 @@ func (cht *cacheHandleTest) Test_validateEntryInFileInfoCache_FileInfoPresent() } fileInfoKeyName, err := fileInfoKey.Key() assert.Nil(cht.T(), err) - fileInfo := data.FileInfo{ - Key: fileInfoKey, - ObjectGeneration: cht.object.Generation, - FileSize: cht.object.Size, - Offset: cht.object.Size, - } + fileInfo := data.NewFileInfo(fileInfoKey, cht.object.Generation, cht.object.Size, cht.object.Size, false, nil, 1) _, err = cht.cache.Insert(fileInfoKeyName, fileInfo) assert.Nil(cht.T(), err) @@ -380,8 +385,7 @@ func (cht *cacheHandleTest) Test_validateEntryInFileInfoCache_FileInfoNotPresent _ = cht.cache.Erase(fileInfoKeyName) err = cht.cacheHandle.validateEntryInFileInfoCache(cht.bucket, cht.object, 0, false) - expectedErr := fmt.Errorf("%v: no entry found in file info cache for key %v", util.InvalidFileInfoCacheErrMsg, fileInfoKeyName) - assert.True(cht.T(), strings.Contains(err.Error(), expectedErr.Error())) + assert.True(cht.T(), errors.Is(err, util.ErrInvalidFileInfoCache)) } func (cht *cacheHandleTest) Test_validateEntryInFileInfoCache_FileInfoGenerationChanged() { @@ -391,19 +395,13 @@ func (cht *cacheHandleTest) Test_validateEntryInFileInfoCache_FileInfoGeneration } fileInfoKeyName, err := fileInfoKey.Key() assert.Nil(cht.T(), err) - fileInfo := data.FileInfo{ - Key: fileInfoKey, - ObjectGeneration: cht.object.Generation + 1, - FileSize: cht.object.Size, - Offset: cht.object.Size, - } + fileInfo := data.NewFileInfo(fileInfoKey, cht.object.Generation+1, cht.object.Size, cht.object.Size, false, nil, 1) _, err = cht.cache.Insert(fileInfoKeyName, fileInfo) assert.Nil(cht.T(), err) err = cht.cacheHandle.validateEntryInFileInfoCache(cht.bucket, cht.object, cht.object.Size-1, true) - expectedErr := fmt.Errorf("%v: generation of cached object: %v is different from required generation: ", util.InvalidFileInfoCacheErrMsg, fileInfo.ObjectGeneration) - assert.True(cht.T(), strings.Contains(err.Error(), expectedErr.Error())) + assert.True(cht.T(), errors.Is(err, util.ErrInvalidFileInfoCache)) } func (cht *cacheHandleTest) Test_validateEntryInFileInfoCache_FileInfoOffsetLessThanRequired() { @@ -413,20 +411,15 @@ func (cht *cacheHandleTest) Test_validateEntryInFileInfoCache_FileInfoOffsetLess } fileInfoKeyName, err := fileInfoKey.Key() assert.Nil(cht.T(), err) - fileInfo := data.FileInfo{ - Key: fileInfoKey, - ObjectGeneration: cht.object.Generation, - FileSize: cht.object.Size, - Offset: 10, // Insert offset less than required - } + fileInfo := data.NewFileInfo(fileInfoKey, cht.object.Generation, cht.object.Size, 10, // Insert offset less than required, + false, nil, 1) _, err = cht.cache.Insert(fileInfoKeyName, fileInfo) assert.Nil(cht.T(), err) err = cht.cacheHandle.validateEntryInFileInfoCache(cht.bucket, cht.object, 11, true) assert.NotNil(cht.T(), err) - expectedErr := fmt.Errorf("%v offset of cached object: %v is less than required offset %v", util.InvalidFileInfoCacheErrMsg, 10, 11) - assert.Equal(cht.T(), expectedErr.Error(), err.Error()) + assert.True(cht.T(), errors.Is(err, util.ErrInvalidFileInfoCache)) } func (cht *cacheHandleTest) Test_validateEntryInFileInfoCache_changeCacheOrderIsTrue() { @@ -440,12 +433,10 @@ func (cht *cacheHandleTest) Test_validateEntryInFileInfoCache_changeCacheOrderIs } fileInfoKeyName, err := fileInfoKey.Key() assert.Nil(cht.T(), err) - fileInfo := data.FileInfo{ - Key: fileInfoKey, - ObjectGeneration: 1, // Adding random generation. - FileSize: CacheMaxSize - cht.object.Size, // This makes cache size full. - Offset: 1, // Insert offset less than required - } + fileInfo := data.NewFileInfo(fileInfoKey, 1, // Adding random generation., + CacheMaxSize-cht.object.Size, // This makes cache size full., + 1, // Insert offset less than required, + false, nil, 1) evictedEntries, err := cht.cache.Insert(fileInfoKeyName, fileInfo) assert.Nil(cht.T(), err) assert.Equal(cht.T(), 0, len(evictedEntries)) @@ -462,12 +453,7 @@ func (cht *cacheHandleTest) Test_validateEntryInFileInfoCache_changeCacheOrderIs } fileInfoKeyName, err = fileInfoKey.Key() assert.Nil(cht.T(), err) - fileInfo = data.FileInfo{ - Key: fileInfoKey, - ObjectGeneration: 1, - FileSize: 1, - Offset: 1, - } + fileInfo = data.NewFileInfo(fileInfoKey, 1, 1, 1, false, nil, 1) evictedEntries, err = cht.cache.Insert(fileInfoKeyName, fileInfo) assert.Nil(cht.T(), err) assert.Equal(cht.T(), 1, len(evictedEntries)) @@ -485,12 +471,10 @@ func (cht *cacheHandleTest) Test_validateEntryInFileInfoCache_changeCacheOrderIs } fileInfoKeyName, err := fileInfoKey.Key() assert.Nil(cht.T(), err) - fileInfo := data.FileInfo{ - Key: fileInfoKey, - ObjectGeneration: 1, // Adding random generation. - FileSize: CacheMaxSize - cht.object.Size, // This makes cache size full. - Offset: 1, // Insert offset less than required - } + fileInfo := data.NewFileInfo(fileInfoKey, 1, // Adding random generation., + CacheMaxSize-cht.object.Size, // This makes cache size full., + 1, // Insert offset less than required, + false, nil, 1) evictedEntries, err := cht.cache.Insert(fileInfoKeyName, fileInfo) assert.Nil(cht.T(), err) assert.Equal(cht.T(), 0, len(evictedEntries)) @@ -506,12 +490,7 @@ func (cht *cacheHandleTest) Test_validateEntryInFileInfoCache_changeCacheOrderIs } fileInfoKeyName, err = fileInfoKey.Key() assert.Nil(cht.T(), err) - fileInfo = data.FileInfo{ - Key: fileInfoKey, - ObjectGeneration: 1, - FileSize: 1, - Offset: 1, - } + fileInfo = data.NewFileInfo(fileInfoKey, 1, 1, 1, false, nil, 1) evictedEntries, err = cht.cache.Insert(fileInfoKeyName, fileInfo) assert.Nil(cht.T(), err) assert.Equal(cht.T(), 1, len(evictedEntries)) @@ -540,7 +519,7 @@ func (cht *cacheHandleTest) Test_Read_WithNilFileHandle() { assert.NotNil(cht.T(), err) assert.Equal(cht.T(), 0, n) assert.False(cht.T(), cacheHit) - assert.Equal(cht.T(), util.InvalidFileHandleErrMsg, err.Error()) + assert.True(cht.T(), errors.Is(err, util.ErrInvalidFileHandle)) } func (cht *cacheHandleTest) Test_Read_WithNilFileDownloadJobAndCacheMiss() { @@ -556,7 +535,7 @@ func (cht *cacheHandleTest) Test_Read_WithNilFileDownloadJobAndCacheMiss() { assert.NotNil(cht.T(), err) assert.Equal(cht.T(), 0, n) assert.False(cht.T(), cacheHit) - assert.True(cht.T(), strings.Contains(err.Error(), util.InvalidFileInfoCacheErrMsg)) + assert.True(cht.T(), errors.Is(err, util.ErrInvalidFileInfoCache)) } func (cht *cacheHandleTest) Test_Read_WithNilFileDownloadJobAndCacheHit() { @@ -567,7 +546,7 @@ func (cht *cacheHandleTest) Test_Read_WithNilFileDownloadJobAndCacheHit() { assert.Equal(cht.T(), downloader.Downloading, jobStatus.Name) dst := make([]byte, cht.object.Size) offset := int64(0) - cht.cacheHandle.isSequential = false + cht.cacheHandle.isSequential.Store(false) cht.cacheHandle.fileDownloadJob = nil // Because the whole object is downloaded into the cache, file info cache @@ -583,7 +562,7 @@ func (cht *cacheHandleTest) Test_Read_WithNilFileDownloadJobAndCacheHit() { func (cht *cacheHandleTest) Test_RandomRead() { dst := make([]byte, ReadContentSize) offset := int64(cht.object.Size - ReadContentSize) - cht.cacheHandle.isSequential = false + cht.cacheHandle.isSequential.Store(false) cht.cacheHandle.cacheFileForRangeRead = true // Since, it's a random read hence will not wait to download till requested offset. @@ -595,13 +574,13 @@ func (cht *cacheHandleTest) Test_RandomRead() { assert.Equal(cht.T(), 0, n) assert.False(cht.T(), cacheHit) assert.NotNil(cht.T(), err) - assert.True(cht.T(), strings.Contains(err.Error(), util.FallbackToGCSErrMsg)) + assert.True(cht.T(), errors.Is(err, util.ErrFallbackToGCS)) } func (cht *cacheHandleTest) Test_RandomRead_CacheForRangeReadFalse() { dst := make([]byte, ReadContentSize) offset := int64(cht.object.Size - ReadContentSize) - cht.cacheHandle.isSequential = false + cht.cacheHandle.isSequential.Store(false) // Since, it's a random read hence will not wait to download till requested offset. n, cacheHit, err := cht.cacheHandle.Read(context.Background(), cht.bucket, cht.object, offset, dst) @@ -612,7 +591,7 @@ func (cht *cacheHandleTest) Test_RandomRead_CacheForRangeReadFalse() { assert.Equal(cht.T(), n, 0) assert.False(cht.T(), cacheHit) assert.NotNil(cht.T(), err) - assert.True(cht.T(), strings.Contains(err.Error(), util.FallbackToGCSErrMsg)) + assert.True(cht.T(), errors.Is(err, util.ErrFallbackToGCS)) } func (cht *cacheHandleTest) Test_RandomRead_CacheForRangeReadFalseButCacheHit() { @@ -623,7 +602,7 @@ func (cht *cacheHandleTest) Test_RandomRead_CacheForRangeReadFalseButCacheHit() assert.Equal(cht.T(), downloader.Downloading, jobStatus.Name) dst := make([]byte, ReadContentSize) offset := int64(1) - cht.cacheHandle.isSequential = false + cht.cacheHandle.isSequential.Store(false) // Since, it's a random read hence will not wait to download till requested offset. _, cacheHit, err := cht.cacheHandle.Read(context.Background(), cht.bucket, cht.object, offset, dst) @@ -639,8 +618,8 @@ func (cht *cacheHandleTest) Test_RandomRead_CacheForRangeReadFalseButCacheHit() func (cht *cacheHandleTest) Test_SequentialRead() { dst := make([]byte, ReadContentSize) offset := int64(cht.object.Size - ReadContentSize) - cht.cacheHandle.isSequential = true - cht.cacheHandle.prevOffset = offset - util.MiB + cht.cacheHandle.isSequential.Store(true) + cht.cacheHandle.prevOffset.Store(offset - util.MiB) cht.cacheHandle.cacheFileForRangeRead = false // Since, it's a sequential read, hence will wait to download till requested offset. @@ -664,19 +643,17 @@ func (cht *cacheHandleTest) Test_Read_ChangeCacheOrder() { } fileInfoKeyName, err := fileInfoKey.Key() assert.Nil(cht.T(), err) - fileInfo := data.FileInfo{ - Key: fileInfoKey, - ObjectGeneration: 1, // Adding random generation. - FileSize: CacheMaxSize - cht.object.Size, // This makes cache size full. - Offset: 1, // Insert offset less than required - } + fileInfo := data.NewFileInfo(fileInfoKey, 1, // Adding random generation., + CacheMaxSize-cht.object.Size, // This makes cache size full., + 1, // Insert offset less than required, + false, nil, 1) evictedEntries, err := cht.cache.Insert(fileInfoKeyName, fileInfo) assert.Nil(cht.T(), err) assert.Equal(cht.T(), 0, len(evictedEntries)) dst := make([]byte, ReadContentSize) offset := int64(cht.object.Size - ReadContentSize) - cht.cacheHandle.isSequential = true - cht.cacheHandle.prevOffset = offset - util.MiB + cht.cacheHandle.isSequential.Store(true) + cht.cacheHandle.prevOffset.Store(offset - util.MiB) cht.cacheHandle.cacheFileForRangeRead = false // Read should change the order in cache and bring cht.Object to most recently @@ -695,12 +672,7 @@ func (cht *cacheHandleTest) Test_Read_ChangeCacheOrder() { } fileInfoKeyName, err = fileInfoKey.Key() assert.Nil(cht.T(), err) - fileInfo = data.FileInfo{ - Key: fileInfoKey, - ObjectGeneration: 1, - FileSize: 1, - Offset: 1, - } + fileInfo = data.NewFileInfo(fileInfoKey, 1, 1, 1, false, nil, 1) evictedEntries, err = cht.cache.Insert(fileInfoKeyName, fileInfo) assert.Nil(cht.T(), err) assert.Equal(cht.T(), 1, len(evictedEntries)) @@ -710,7 +682,7 @@ func (cht *cacheHandleTest) Test_Read_ChangeCacheOrder() { func (cht *cacheHandleTest) Test_SequentialReadToRandom() { dst := make([]byte, ReadContentSize) firstReqOffset := int64(0) - cht.cacheHandle.isSequential = true + cht.cacheHandle.isSequential.Store(true) cht.cacheHandle.cacheFileForRangeRead = true // Since, it's a sequential read, hence will wait to download till requested offset. _, cacheHit, err := cht.cacheHandle.Read(context.Background(), cht.bucket, cht.object, firstReqOffset, dst) @@ -718,15 +690,15 @@ func (cht *cacheHandleTest) Test_SequentialReadToRandom() { jobStatus := cht.cacheHandle.fileDownloadJob.GetStatus() assert.GreaterOrEqual(cht.T(), jobStatus.Offset, firstReqOffset) assert.False(cht.T(), cacheHit) - assert.True(cht.T(), cht.cacheHandle.isSequential) + assert.True(cht.T(), cht.cacheHandle.isSequential.Load()) secondReqOffset := int64(cht.object.Size - ReadContentSize) // type will change to random. _, cacheHit, err = cht.cacheHandle.Read(context.Background(), cht.bucket, cht.object, secondReqOffset, dst) assert.NotNil(cht.T(), err) - assert.True(cht.T(), strings.Contains(err.Error(), util.FallbackToGCSErrMsg)) + assert.True(cht.T(), errors.Is(err, util.ErrFallbackToGCS)) assert.False(cht.T(), cacheHit) - assert.False(cht.T(), cht.cacheHandle.isSequential) + assert.False(cht.T(), cht.cacheHandle.isSequential.Load()) jobStatus = cht.cacheHandle.fileDownloadJob.GetStatus() assert.LessOrEqual(cht.T(), jobStatus.Offset, secondReqOffset) } @@ -736,8 +708,8 @@ func (cht *cacheHandleTest) Test_Read_WhenDstBufferIsMoreContentToBeRead() { extraBuffer := 2 dst := make([]byte, ReadContentSize+extraBuffer) offset := int64(cht.object.Size - ReadContentSize) - cht.cacheHandle.isSequential = true - cht.cacheHandle.prevOffset = offset - util.MiB + cht.cacheHandle.isSequential.Store(true) + cht.cacheHandle.prevOffset.Store(offset - util.MiB) cht.cacheHandle.cacheFileForRangeRead = true // Since, it's a sequential read, hence will wait to download till requested offset. @@ -752,7 +724,7 @@ func (cht *cacheHandleTest) Test_Read_WhenDstBufferIsMoreContentToBeRead() { func (cht *cacheHandleTest) Test_Read_FileInfoRemoved() { dst := make([]byte, ReadContentSize) - cht.cacheHandle.isSequential = true + cht.cacheHandle.isSequential.Store(true) cht.cacheHandle.cacheFileForRangeRead = true // First let the cache populated (we are doing this so that we can externally // modify file info cache for this unit test without hampering download job). @@ -773,14 +745,13 @@ func (cht *cacheHandleTest) Test_Read_FileInfoRemoved() { _, cacheHit, err = cht.cacheHandle.Read(context.Background(), cht.bucket, cht.object, 0, dst) assert.NotNil(cht.T(), err) - expectedErr := fmt.Errorf("%v: no entry found in file info cache for key %v", util.InvalidFileInfoCacheErrMsg, fileInfoKeyName) - assert.True(cht.T(), strings.Contains(err.Error(), expectedErr.Error())) + assert.True(cht.T(), errors.Is(err, util.ErrInvalidFileInfoCache)) assert.False(cht.T(), cacheHit) } func (cht *cacheHandleTest) Test_Read_FileInfoGenerationChanged() { dst := make([]byte, ReadContentSize) - cht.cacheHandle.isSequential = true + cht.cacheHandle.isSequential.Store(true) cht.cacheHandle.cacheFileForRangeRead = true // First let the cache populated (we are doing this so that we can externally // modify file info cache for this unit test without hampering download job). @@ -805,16 +776,15 @@ func (cht *cacheHandleTest) Test_Read_FileInfoGenerationChanged() { _, cacheHit, err = cht.cacheHandle.Read(context.Background(), cht.bucket, cht.object, 0, dst) assert.NotNil(cht.T(), err) - expectedErr := fmt.Errorf("%v: generation of cached object: %v is different from required generation: ", util.InvalidFileInfoCacheErrMsg, fileInfoData.ObjectGeneration) - assert.True(cht.T(), strings.Contains(err.Error(), expectedErr.Error())) + assert.True(cht.T(), errors.Is(err, util.ErrInvalidFileInfoCache)) assert.False(cht.T(), cacheHit) } func (cht *cacheHandleTest) Test_MultipleReads_CacheHitShouldBeFalseThenTrue() { dst := make([]byte, ReadContentSize) offset := int64(cht.object.Size - ReadContentSize) - cht.cacheHandle.isSequential = true - cht.cacheHandle.prevOffset = offset - util.MiB + cht.cacheHandle.isSequential.Store(true) + cht.cacheHandle.prevOffset.Store(offset - util.MiB) cht.cacheHandle.cacheFileForRangeRead = true // First read should be cache miss. n, cacheHit, err := cht.cacheHandle.Read(context.Background(), cht.bucket, cht.object, offset, dst) @@ -836,8 +806,8 @@ func (cht *cacheHandleTest) Test_MultipleReads_CacheHitShouldBeFalseThenTrue() { func (cht *cacheHandleTest) Test_SequentialRead_Parallel_Download_True() { dst := make([]byte, ReadContentSize) offset := int64(cht.object.Size - ReadContentSize) - cht.cacheHandle.isSequential = true - cht.cacheHandle.prevOffset = offset - util.MiB + cht.cacheHandle.isSequential.Store(true) + cht.cacheHandle.prevOffset.Store(offset - util.MiB) cht.cacheHandle.cacheFileForRangeRead = false fileCacheConfig := &cfg.FileCacheConfig{ EnableCrc: true, @@ -855,6 +825,9 @@ func (cht *cacheHandleTest) Test_SequentialRead_Parallel_Download_True() { func() {}, fileCacheConfig, semaphore.NewWeighted(math.MaxInt64), + metrics.NewNoopMetrics(), + tracing.NewNoopTracer(), + 1, ) cht.cacheHandle.fileDownloadJob = fileDownloadJob @@ -865,13 +838,13 @@ func (cht *cacheHandleTest) Test_SequentialRead_Parallel_Download_True() { assert.Equal(cht.T(), downloader.Downloading, jobStatus.Name) assert.Equal(cht.T(), 0, n) assert.False(cht.T(), cacheHit) - assert.ErrorContains(cht.T(), err, util.FallbackToGCSErrMsg) + assert.True(cht.T(), errors.Is(err, util.ErrFallbackToGCS)) } func (cht *cacheHandleTest) Test_RandomRead_Parallel_Download_True() { dst := make([]byte, ReadContentSize) offset := int64(cht.object.Size - ReadContentSize) - cht.cacheHandle.isSequential = false + cht.cacheHandle.isSequential.Store(false) cht.cacheHandle.cacheFileForRangeRead = true fileCacheConfig := &cfg.FileCacheConfig{ EnableCrc: true, @@ -889,6 +862,9 @@ func (cht *cacheHandleTest) Test_RandomRead_Parallel_Download_True() { func() {}, fileCacheConfig, semaphore.NewWeighted(math.MaxInt64), + metrics.NewNoopMetrics(), + tracing.NewNoopTracer(), + 1, ) cht.cacheHandle.fileDownloadJob = fileDownloadJob @@ -899,13 +875,13 @@ func (cht *cacheHandleTest) Test_RandomRead_Parallel_Download_True() { assert.Equal(cht.T(), downloader.Downloading, jobStatus.Name) assert.Equal(cht.T(), 0, n) assert.False(cht.T(), cacheHit) - assert.ErrorContains(cht.T(), err, util.FallbackToGCSErrMsg) + assert.True(cht.T(), errors.Is(err, util.ErrFallbackToGCS)) } func (cht *cacheHandleTest) Test_RandomRead_CacheForRangeReadFalse_And_ParallelDownloadsEnabled() { dst := make([]byte, ReadContentSize) offset := int64(cht.object.Size - ReadContentSize) - cht.cacheHandle.isSequential = false + cht.cacheHandle.isSequential.Store(false) cht.cacheHandle.cacheFileForRangeRead = false fileCacheConfig := &cfg.FileCacheConfig{ EnableCrc: true, @@ -923,6 +899,9 @@ func (cht *cacheHandleTest) Test_RandomRead_CacheForRangeReadFalse_And_ParallelD func() {}, fileCacheConfig, semaphore.NewWeighted(math.MaxInt64), + metrics.NewNoopMetrics(), + tracing.NewNoopTracer(), + 1, ) // Since, it's a random read, download job will not start. @@ -933,5 +912,5 @@ func (cht *cacheHandleTest) Test_RandomRead_CacheForRangeReadFalse_And_ParallelD assert.Less(cht.T(), jobStatus.Offset, offset) assert.Equal(cht.T(), n, 0) assert.False(cht.T(), cacheHit) - assert.ErrorContains(cht.T(), err, util.FallbackToGCSErrMsg) + assert.True(cht.T(), errors.Is(err, util.ErrFallbackToGCS)) } diff --git a/internal/cache/file/cache_handler.go b/internal/cache/file/cache_handler.go index 1c85e86640..cd6bb955c8 100644 --- a/internal/cache/file/cache_handler.go +++ b/internal/cache/file/cache_handler.go @@ -17,14 +17,16 @@ package file import ( "fmt" "os" - - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/data" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/file/downloader" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/lru" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/util" - "github.com/googlecloudplatform/gcsfuse/v2/internal/locker" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "path" + "regexp" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/data" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/file/downloader" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/locker" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" ) // CacheHandler is responsible for creating CacheHandle and invalidating file cache @@ -51,19 +53,54 @@ type CacheHandler struct { // mu guards the handling of insertion into and eviction from file cache. mu locker.Locker + + // excludeRegex is the compiled regex for excluding files from cache + excludeRegex *regexp.Regexp + + // includeRegex is the compiled regex for including files from cache + includeRegex *regexp.Regexp + + // isSparse indicates whether sparse file mode is enabled + isSparse bool + + // volumeBlockSize caches the block size of the local volume for speculative size accounting + volumeBlockSize uint64 } -func NewCacheHandler(fileInfoCache *lru.Cache, jobManager *downloader.JobManager, cacheDir string, filePerm os.FileMode, dirPerm os.FileMode) *CacheHandler { +func NewCacheHandler(fileInfoCache *lru.Cache, jobManager *downloader.JobManager, cacheDir string, filePerm os.FileMode, dirPerm os.FileMode, excludeRegex string, includeRegex string, isSparse bool, volumeBlockSize uint64) *CacheHandler { + var compiledExcludeRegex *regexp.Regexp + var compiledIncludeRegex *regexp.Regexp + + compiledExcludeRegex = compileRegex(excludeRegex) + compiledIncludeRegex = compileRegex(includeRegex) + return &CacheHandler{ - fileInfoCache: fileInfoCache, - jobManager: jobManager, - cacheDir: cacheDir, - filePerm: filePerm, - dirPerm: dirPerm, - mu: locker.New("FileCacheHandler", func() {}), + fileInfoCache: fileInfoCache, + jobManager: jobManager, + cacheDir: cacheDir, + filePerm: filePerm, + dirPerm: dirPerm, + mu: locker.New("FileCacheHandler", func() {}), + excludeRegex: compiledExcludeRegex, + includeRegex: compiledIncludeRegex, + isSparse: isSparse, + volumeBlockSize: volumeBlockSize, } } +func compileRegex(regexString string) *regexp.Regexp { + var compiledRegex *regexp.Regexp + + if regexString != "" { + var err error + compiledRegex, err = regexp.Compile(regexString) + if err != nil { + logger.Warnf("Failed to compile regex %q: %v", regexString, err) + } + } + return compiledRegex +} + func (chr *CacheHandler) createLocalFileReadHandle(objectName string, bucketName string) (*os.File, error) { fileSpec := data.FileSpec{ Path: util.GetDownloadPath(chr.cacheDir, util.GetObjectPath(bucketName, objectName)), @@ -128,7 +165,7 @@ func (chr *CacheHandler) addFileInfoEntryAndCreateDownloadJob(object *gcs.MinObj filePath := util.GetDownloadPath(chr.cacheDir, util.GetObjectPath(bucket.Name(), object.Name)) _, err := os.Stat(filePath) if err != nil && os.IsNotExist(err) { - return fmt.Errorf("addFileInfoEntryAndCreateDownloadJob: %s: %s", util.FileNotPresentInCacheErrMsg, filePath) + return fmt.Errorf("addFileInfoEntryAndCreateDownloadJob: %w: %s", util.ErrFileNotPresentInCache, filePath) } // Evict object in cache if the generation of object in cache is different @@ -140,7 +177,7 @@ func (chr *CacheHandler) addFileInfoEntryAndCreateDownloadJob(object *gcs.MinObj // If offset in file info cache is less than object size and there is no // reference to download job then it means the job has failed. existingJob := chr.jobManager.GetJob(object.Name, bucket.Name()) - shouldInvalidate := (existingJob == nil) && (fileInfoData.Offset < object.Size) + shouldInvalidate := (existingJob == nil) && (fileInfoData.Offset < fileInfoData.FileSize) if (!shouldInvalidate) && (existingJob != nil) { existingJobStatus := existingJob.GetStatus().Name shouldInvalidate = (existingJobStatus == downloader.Failed) || (existingJobStatus == downloader.Invalid) @@ -159,14 +196,17 @@ func (chr *CacheHandler) addFileInfoEntryAndCreateDownloadJob(object *gcs.MinObj } if addEntryToCache { - fileInfo = data.FileInfo{ - Key: fileInfoKey, - ObjectGeneration: object.Generation, - Offset: 0, - FileSize: object.Size, + newFileInfo := data.NewFileInfo(fileInfoKey, object.Generation, object.Size, 0, chr.isSparse, nil, chr.volumeBlockSize) + // For sparse files, set Offset to MaxUint64 as a sentinel to indicate + // sparse mode, so Offset < requiredOffset checks always fail + if chr.isSparse { + newFileInfo.Offset = ^uint64(0) // math.MaxUint64 + // Use download chunk size for ByteRangeMap tracking granularity + chunkSizeBytes := uint64(chr.jobManager.DownloadChunkSizeMb()) * 1024 * 1024 + newFileInfo.DownloadedChunks = data.NewByteRangeMap(chunkSizeBytes, object.Size) } - evictedValues, err := chr.fileInfoCache.Insert(fileInfoKeyName, fileInfo) + evictedValues, err := chr.fileInfoCache.Insert(fileInfoKeyName, newFileInfo) if err != nil { return fmt.Errorf("addFileInfoEntryAndCreateDownloadJob: while inserting into the cache: %w", err) } @@ -202,10 +242,16 @@ func (chr *CacheHandler) GetCacheHandle(object *gcs.MinObject, bucket gcs.Bucket chr.mu.Lock() defer chr.mu.Unlock() - // If cacheForRangeRead is set to False, initialOffset is non-zero (i.e. random read) - // and entry for file doesn't already exist in fileInfoCache then no need to - // create file in cache. - if !cacheForRangeRead && initialOffset != 0 { + // Check if file should be excluded from cache + if chr.shouldExcludeFromCache(bucket, object) { + return nil, util.ErrFileExcludedFromCacheByRegex + } + + // If cacheForRangeRead is set to False, initialOffset is non-zero (i.e. random read), + // not in sparse mode, and entry for file doesn't already exist in fileInfoCache + // then no need to create file in cache. Sparse files need cache handles even for + // random reads to track downloaded ranges. + if !cacheForRangeRead && initialOffset != 0 && !chr.isSparse { fileInfoKey := data.FileInfoKey{ BucketName: bucket.Name(), ObjectName: object.Name, @@ -217,7 +263,7 @@ func (chr *CacheHandler) GetCacheHandle(object *gcs.MinObject, bucket gcs.Bucket fileInfo := chr.fileInfoCache.LookUpWithoutChangingOrder(fileInfoKeyName) if fileInfo == nil { - return nil, fmt.Errorf("addFileInfoEntryAndCreateDownloadJob: %s", util.CacheHandleNotRequiredForRandomReadErrMsg) + return nil, fmt.Errorf("addFileInfoEntryAndCreateDownloadJob: %w", util.ErrCacheHandleNotRequiredForRandomRead) } } @@ -274,3 +320,26 @@ func (chr *CacheHandler) Destroy() (err error) { chr.jobManager.Destroy() return } + +// shouldExcludeFromCache checks if the object should be excluded from cache +// based on the configured regex pattern of include and/or exclude regex. +func (chr *CacheHandler) shouldExcludeFromCache(bucket gcs.Bucket, object *gcs.MinObject) bool { + // If no regex is configured, nothing is excluded. + if chr.includeRegex == nil && chr.excludeRegex == nil { + return false + } + + objectName := path.Join(bucket.Name(), bucket.GCSName(object)) + + // Exclude if it matches the exclude pattern. + // Exclude flag take precedence over Include regex (if matched). + if chr.excludeRegex != nil && chr.excludeRegex.MatchString(objectName) { + return true + } + // Exclude if an include pattern is present and it doesn't match. + if chr.includeRegex != nil && !chr.includeRegex.MatchString(objectName) { + return true + } + + return false +} diff --git a/internal/cache/file/cache_handler_test.go b/internal/cache/file/cache_handler_test.go index 2daa17883d..060429913b 100644 --- a/internal/cache/file/cache_handler_test.go +++ b/internal/cache/file/cache_handler_test.go @@ -17,6 +17,7 @@ package file import ( "context" "crypto/rand" + "errors" "os" "path" "strconv" @@ -24,17 +25,23 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/data" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/file/downloader" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/lru" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/util" - "github.com/googlecloudplatform/gcsfuse/v2/internal/locker" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util/diskutil" + + "cloud.google.com/go/storage/control/apiv2/controlpb" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/data" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/file/downloader" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/locker" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -58,32 +65,42 @@ func initializeCacheHandlerTestArgs(t *testing.T, fileCacheConfig *cfg.FileCache locker.EnableInvariantsCheck() // Create bucket in fake storage. - fakeStorage := storage.NewFakeStorage() + mockClient := new(storage.MockStorageControlClient) + fakeStorage := storage.NewFakeStorageWithMockClient(mockClient, cfg.HTTP2) t.Cleanup(func() { fakeStorage.ShutDown() }) storageHandle := fakeStorage.CreateStorageHandle() + mockClient.On("GetStorageLayout", mock.Anything, mock.Anything, mock.Anything). + Return(&controlpb.StorageLayout{}, nil) ctx := context.Background() - bucket := storageHandle.BucketHandle(ctx, storage.TestBucketName, "") + bucket, err := storageHandle.BucketHandle(ctx, storage.TestBucketName, "") + require.NoError(t, err) // Create test object in the bucket. testObjectContent := make([]byte, TestObjectSize) - _, err := rand.Read(testObjectContent) + _, err = rand.Read(testObjectContent) require.NoError(t, err) object := createObject(t, bucket, TestObjectName, testObjectContent) // fileInfoCache with testFileInfoEntry cache := lru.NewCache(HandlerCacheMaxSize) + // Calculate block size + cacheDirVolumeBlockSize := diskutil.GetVolumeBlockSize(cacheDir) + // Job manager jobManager := downloader.NewJobManager(cache, util.DefaultFilePerm, - util.DefaultDirPerm, cacheDir, DefaultSequentialReadSizeMb, fileCacheConfig) - + util.DefaultDirPerm, cacheDir, DefaultSequentialReadSizeMb, fileCacheConfig, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), cacheDirVolumeBlockSize) + t.Cleanup(func() { + jobManager.Destroy() + }) // Mocked cached handler object. - cacheHandler := NewCacheHandler(cache, jobManager, cacheDir, util.DefaultFilePerm, util.DefaultDirPerm) + isSparse := fileCacheConfig != nil && fileCacheConfig.ExperimentalEnableChunkCache + cacheHandler := NewCacheHandler(cache, jobManager, cacheDir, util.DefaultFilePerm, util.DefaultDirPerm, fileCacheConfig.ExcludeRegex, fileCacheConfig.IncludeRegex, isSparse, cacheDirVolumeBlockSize) // Follow consistency, local-cache file, entry in fileInfo cache and job should exist initially. - fileInfoKeyName := addTestFileInfoEntryInCache(t, cache, object, storage.TestBucketName) + fileInfoKeyName := addTestFileInfoEntryInCache(t, cache, object, storage.TestBucketName, cacheDirVolumeBlockSize) downloadPath := util.GetDownloadPath(cacheHandler.cacheDir, util.GetObjectPath(bucket.Name(), object.Name)) _, err = util.CreateFile(data.FileSpec{Path: downloadPath, FilePerm: util.DefaultFilePerm, DirPerm: util.DefaultDirPerm}, os.O_RDONLY) t.Cleanup(func() { @@ -121,19 +138,14 @@ func createObject(t *testing.T, bucket gcs.Bucket, objName string, objContent [] return minObject } -func addTestFileInfoEntryInCache(t *testing.T, cache *lru.Cache, object *gcs.MinObject, bucketName string) string { +func addTestFileInfoEntryInCache(t *testing.T, cache *lru.Cache, object *gcs.MinObject, bucketName string, cacheDirVolumeBlockSize uint64) string { t.Helper() // Add an entry into fileInfoKey := data.FileInfoKey{ BucketName: bucketName, ObjectName: object.Name, } - fileInfo := data.FileInfo{ - Key: fileInfoKey, - ObjectGeneration: object.Generation, - FileSize: object.Size, - Offset: 0, - } + fileInfo := data.NewFileInfo(fileInfoKey, object.Generation, object.Size, 0, false, nil, cacheDirVolumeBlockSize) fileInfoKeyName, err := fileInfoKey.Key() require.NoError(t, err) @@ -325,7 +337,7 @@ func Test_addFileInfoEntryAndCreateDownloadJob_IfLocalFileGetsDeleted(t *testing // Hence, this will return error containing util.FileNotPresentInCacheErrMsg. err = chTestArgs.cacheHandler.addFileInfoEntryAndCreateDownloadJob(chTestArgs.object, chTestArgs.bucket) - assert.ErrorContains(t, err, util.FileNotPresentInCacheErrMsg) + assert.True(t, errors.Is(err, util.ErrFileNotPresentInCache)) } func Test_addFileInfoEntryAndCreateDownloadJob_WhenJobHasCompleted(t *testing.T) { @@ -523,7 +535,7 @@ func Test_GetCacheHandle_IfLocalFileGetsDeleted(t *testing.T) { cacheHandle, err := chTestArgs.cacheHandler.GetCacheHandle(chTestArgs.object, chTestArgs.bucket, false, 0) - assert.ErrorContains(t, err, util.FileNotPresentInCacheErrMsg) + assert.True(t, errors.Is(err, util.ErrFileNotPresentInCache)) assert.Nil(t, cacheHandle) // Check file info and download job are not removed assert.True(t, isEntryInFileInfoCache(t, chTestArgs.cache, chTestArgs.object.Name, chTestArgs.bucket.Name())) @@ -532,6 +544,76 @@ func Test_GetCacheHandle_IfLocalFileGetsDeleted(t *testing.T) { assert.Equal(t, downloader.NotStarted, existingJob.GetStatus().Name) } +func Test_GetCacheHandle_ExcludeFromCache(t *testing.T) { + regex := ".*object_1" + cacheDir := path.Join(os.Getenv("HOME"), "CacheHandlerTest/dir") + chTestArgs := initializeCacheHandlerTestArgs(t, &cfg.FileCacheConfig{EnableCrc: true, ExcludeRegex: regex}, cacheDir) + + // Check cache handle is not created for excluded file + chTestArgs.object.Name = "object_1" + cacheHandle, err := chTestArgs.cacheHandler.GetCacheHandle(chTestArgs.object, chTestArgs.bucket, false, 0) + assert.True(t, errors.Is(err, util.ErrFileExcludedFromCacheByRegex)) + assert.Nil(t, cacheHandle) + + // Check cache handle is created for file not excluded. + chTestArgs.object.Name = "object_2" + cacheHandle, err = chTestArgs.cacheHandler.GetCacheHandle(chTestArgs.object, chTestArgs.bucket, false, 0) + assert.NoError(t, err) + assert.Nil(t, cacheHandle.validateCacheHandle()) + +} + +func Test_GetCacheHandle_IncludeInCache(t *testing.T) { + regex := ".*object_1" + cacheDir := path.Join(os.Getenv("HOME"), "CacheHandlerTest/dir") + chTestArgs := initializeCacheHandlerTestArgs(t, &cfg.FileCacheConfig{EnableCrc: true, IncludeRegex: regex}, cacheDir) + + // Check cache handle is created for included file. + chTestArgs.object.Name = "object_1" + cacheHandle, err := chTestArgs.cacheHandler.GetCacheHandle(chTestArgs.object, chTestArgs.bucket, false, 0) + assert.NoError(t, err) + assert.Nil(t, cacheHandle.validateCacheHandle()) + + // Check cache handle is not created for file not included. + chTestArgs.object.Name = "object_2" + cacheHandle, err = chTestArgs.cacheHandler.GetCacheHandle(chTestArgs.object, chTestArgs.bucket, false, 0) + assert.True(t, errors.Is(err, util.ErrFileExcludedFromCacheByRegex)) + assert.Nil(t, cacheHandle) +} + +func Test_GetCacheHandle_IncludeAndExclude(t *testing.T) { + includeRegex := ".*\\.txt" + excludeRegex := ".*_internal\\.txt" + cacheDir := path.Join(os.Getenv("HOME"), "CacheHandlerTest/dir") + chTestArgs := initializeCacheHandlerTestArgs(t, &cfg.FileCacheConfig{EnableCrc: true, IncludeRegex: includeRegex, ExcludeRegex: excludeRegex}, cacheDir) + + // Check cache handle is created for included file. + chTestArgs.object.Name = "some_file.txt" + cacheHandle, err := chTestArgs.cacheHandler.GetCacheHandle(chTestArgs.object, chTestArgs.bucket, false, 0) + assert.NoError(t, err) + assert.Nil(t, cacheHandle.validateCacheHandle()) + + // Check cache handle is not created for excluded file even if it matches include pattern. + chTestArgs.object.Name = "some_file_internal.txt" + cacheHandle, err = chTestArgs.cacheHandler.GetCacheHandle(chTestArgs.object, chTestArgs.bucket, false, 0) + assert.True(t, errors.Is(err, util.ErrFileExcludedFromCacheByRegex)) + assert.Nil(t, cacheHandle) +} + +func Test_GetCacheHandle_SameIncludeAndExcludeRegex(t *testing.T) { + regex := ".*\\.txt" + cacheDir := path.Join(os.Getenv("HOME"), "CacheHandlerTest/dir") + chTestArgs := initializeCacheHandlerTestArgs(t, &cfg.FileCacheConfig{EnableCrc: true, IncludeRegex: regex, ExcludeRegex: regex}, cacheDir) + + // Check cache handle is not created for a file that matches both include and + // exclude regex, as exclude takes precedence. + chTestArgs.object.Name = "some_file.txt" + cacheHandle, err := chTestArgs.cacheHandler.GetCacheHandle(chTestArgs.object, chTestArgs.bucket, false, 0) + + assert.True(t, errors.Is(err, util.ErrFileExcludedFromCacheByRegex)) + assert.Nil(t, cacheHandle) +} + func Test_GetCacheHandle_CacheForRangeRead(t *testing.T) { tbl := []struct { name string @@ -570,7 +652,7 @@ func Test_GetCacheHandle_CacheForRangeRead(t *testing.T) { assert.NoError(t, err1) assert.Nil(t, cacheHandle1.validateCacheHandle()) - assert.ErrorContains(t, err2, util.CacheHandleNotRequiredForRandomReadErrMsg) + assert.True(t, errors.Is(err2, util.ErrCacheHandleNotRequiredForRandomRead)) assert.Nil(t, cacheHandle2) assert.NoError(t, err3) assert.Nil(t, cacheHandle3.validateCacheHandle()) @@ -624,7 +706,7 @@ func Test_GetCacheHandle_ConcurrentSameFile(t *testing.T) { } // Start concurrent GetCacheHandle() - for i := 0; i < 5; i++ { + for range 5 { wg.Add(1) go getCacheHandleTestFun(t) } @@ -660,7 +742,7 @@ func Test_GetCacheHandle_ConcurrentDifferentFiles(t *testing.T) { } // Start concurrent GetCacheHandle() - for i := 0; i < 5; i++ { + for i := range 5 { wg.Add(1) go getCacheHandleTestFun(i) } @@ -832,7 +914,7 @@ func Test_InvalidateCache_ConcurrentSameFile(t *testing.T) { } // Start concurrent GetCacheHandle() - for i := 0; i < 5; i++ { + for range 5 { wg.Add(1) go invalidateCacheTestFun(t) } @@ -883,7 +965,7 @@ func Test_InvalidateCache_ConcurrentDifferentFiles(t *testing.T) { } // Start concurrent GetCacheHandle() - for i := 0; i < 5; i++ { + for i := range 5 { wg.Add(1) go invalidateCacheTestFun(i) } @@ -944,7 +1026,7 @@ func Test_InvalidateCache_GetCacheHandle_Concurrent(t *testing.T) { } // Start concurrent GetCacheHandle() - for i := 0; i < 5; i++ { + for i := range 5 { wg.Add(1) go invalidateCacheTestFun(i) wg.Add(1) @@ -1040,3 +1122,54 @@ func Test_Destroy(t *testing.T) { }) } } + +func Test_NewCacheHandler_WithSizeCalcFix(t *testing.T) { + cacheDir := t.TempDir() + cache := lru.NewCache(100) + cacheDirVolumeBlockSize := diskutil.GetVolumeBlockSize(cacheDir) + // Create with volumeBlockSize + handler := NewCacheHandler(cache, nil, cacheDir, util.DefaultFilePerm, util.DefaultDirPerm, "", "", false, cacheDirVolumeBlockSize) + require.NotNil(t, handler) + // Verify that inserting a 1-byte file via the handler's calculated block size would overflow a 100-byte cache. + fi := data.NewFileInfo( + data.FileInfoKey{ + ObjectName: "test.txt", + }, + 1, // dummy generation number + 1, // file-size + 0, // offset + false, // sparse-mode + nil, // ranges + cacheDirVolumeBlockSize) + + // Inserting should immediately fail because volumeBlockSize (e.g. 4096) > 100. + _, err := cache.Insert("test_key", fi) + + require.Error(t, err) + require.ErrorIs(t, err, lru.ErrInvalidEntrySize) +} + +func Test_NewCacheHandler_WithoutSizeCalcFix(t *testing.T) { + cacheDir := t.TempDir() + cache := lru.NewCache(100) + cacheDirVolumeBlockSize := uint64(1) + handler := NewCacheHandler(cache, nil, cacheDir, util.DefaultFilePerm, util.DefaultDirPerm, "", "", false, cacheDirVolumeBlockSize) + require.NotNil(t, handler) + // Verify that inserting a 5-byte file via the handler's block size of would work. + fi := data.NewFileInfo( + data.FileInfoKey{ + ObjectName: "test.txt", + }, + 1, // dummy generation number + 5, // file-size + 0, // offset + false, // sparse-mode + nil, // ranges + cacheDirVolumeBlockSize) + + // Inserting should work because Max(volumeBlockSize (1), item-size(5)) <= cache-size (100). + evicted, err := cache.Insert("test_key", fi) + + require.NoError(t, err) + require.Nil(t, evicted) +} diff --git a/internal/cache/file/downloader/downloader.go b/internal/cache/file/downloader/downloader.go index 3e4c98aa6f..7db4c976cf 100644 --- a/internal/cache/file/downloader/downloader.go +++ b/internal/cache/file/downloader/downloader.go @@ -18,12 +18,14 @@ import ( "math" "os" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/data" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/lru" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/util" - "github.com/googlecloudplatform/gcsfuse/v2/internal/locker" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/data" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/locker" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" "golang.org/x/sync/semaphore" ) @@ -57,13 +59,17 @@ type JobManager struct { // concatenation of bucket name, "/", and object name. e.g. object path for an // object named "a/b/foo.txt" in bucket named "test_bucket" would be // "test_bucket/a/b/foo.txt" - jobs map[string]*Job - mu locker.Locker - maxParallelismSem *semaphore.Weighted + jobs map[string]*Job + mu locker.Locker + maxParallelismSem *semaphore.Weighted + metricHandle metrics.MetricHandle + traceHandle tracing.TraceHandle + cacheDirVolumeBlockSize uint64 } func NewJobManager(fileInfoCache *lru.Cache, filePerm os.FileMode, dirPerm os.FileMode, - cacheDir string, sequentialReadSizeMb int32, c *cfg.FileCacheConfig) (jm *JobManager) { + cacheDir string, sequentialReadSizeMb int32, c *cfg.FileCacheConfig, + metricHandle metrics.MetricHandle, traceHandle tracing.TraceHandle, cacheDirVolumeBlockSize uint64) (jm *JobManager) { maxParallelDownloads := int64(math.MaxInt64) if c.MaxParallelDownloads > 0 { maxParallelDownloads = c.MaxParallelDownloads @@ -76,7 +82,10 @@ func NewJobManager(fileInfoCache *lru.Cache, filePerm os.FileMode, dirPerm os.Fi sequentialReadSizeMb: sequentialReadSizeMb, fileCacheConfig: c, // Shared between jobs - Limits the overall concurrency of downloads. - maxParallelismSem: semaphore.NewWeighted(maxParallelDownloads), + maxParallelismSem: semaphore.NewWeighted(maxParallelDownloads), + metricHandle: metricHandle, + traceHandle: traceHandle, + cacheDirVolumeBlockSize: cacheDirVolumeBlockSize, } jm.mu = locker.New("JobManager", func() {}) jm.jobs = make(map[string]*Job) @@ -114,7 +123,7 @@ func (jm *JobManager) CreateJobIfNotExists(object *gcs.MinObject, bucket gcs.Buc removeJobCallback := func() { jm.removeJob(object.Name, bucket.Name()) } - job = NewJob(object, bucket, jm.fileInfoCache, jm.sequentialReadSizeMb, fileSpec, removeJobCallback, jm.fileCacheConfig, jm.maxParallelismSem) + job = NewJob(object, bucket, jm.fileInfoCache, jm.sequentialReadSizeMb, fileSpec, removeJobCallback, jm.fileCacheConfig, jm.maxParallelismSem, jm.metricHandle, jm.traceHandle, jm.cacheDirVolumeBlockSize) jm.jobs[objectPath] = job return job } @@ -148,6 +157,11 @@ func (jm *JobManager) InvalidateAndRemoveJob(objectName string, bucketName strin } } +// DownloadChunkSizeMb returns the download chunk size in MB from the file cache config. +func (jm *JobManager) DownloadChunkSizeMb() int64 { + return jm.fileCacheConfig.DownloadChunkSizeMb +} + // Destroy invalidates and deletes all the jobs that job manager is managing. // // Acquires and releases Lock(jm.mu) diff --git a/internal/cache/file/downloader/downloader_test.go b/internal/cache/file/downloader/downloader_test.go index 8e16926aac..ad55b304e7 100644 --- a/internal/cache/file/downloader/downloader_test.go +++ b/internal/cache/file/downloader/downloader_test.go @@ -22,16 +22,20 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/data" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/lru" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/util" - "github.com/googlecloudplatform/gcsfuse/v2/internal/locker" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - testutil "github.com/googlecloudplatform/gcsfuse/v2/internal/util" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" + "cloud.google.com/go/storage/control/apiv2/controlpb" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/data" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/locker" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + testutil "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" . "github.com/jacobsa/ogletest" + "github.com/stretchr/testify/mock" ) var cacheDir = path.Join(os.Getenv("HOME"), "cache/dir") @@ -56,21 +60,32 @@ func (dt *downloaderTest) setupHelper() { operations.RemoveDir(cacheDir) // Create bucket in fake storage. - dt.fakeStorage = storage.NewFakeStorage() + var err error + mockClient := new(storage.MockStorageControlClient) + dt.fakeStorage = storage.NewFakeStorageWithMockClient(mockClient, cfg.HTTP2) storageHandle := dt.fakeStorage.CreateStorageHandle() + mockClient.On("GetStorageLayout", mock.Anything, mock.Anything, mock.Anything). + Return(&controlpb.StorageLayout{}, nil) ctx := context.Background() - dt.bucket = storageHandle.BucketHandle(ctx, storage.TestBucketName, "") + dt.bucket, err = storageHandle.BucketHandle(ctx, storage.TestBucketName, "") + ExpectEq(nil, err) dt.initJobTest(DefaultObjectName, []byte("taco"), DefaultSequentialReadSizeMb, CacheMaxSize, func() {}) - dt.jm = NewJobManager(dt.cache, util.DefaultFilePerm, util.DefaultDirPerm, cacheDir, DefaultSequentialReadSizeMb, dt.defaultFileCacheConfig) + dt.jm = NewJobManager(dt.cache, util.DefaultFilePerm, util.DefaultDirPerm, cacheDir, DefaultSequentialReadSizeMb, dt.defaultFileCacheConfig, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), 1) } func (dt *downloaderTest) SetUp(*TestInfo) { - dt.defaultFileCacheConfig = &cfg.FileCacheConfig{EnableCrc: true} + dt.defaultFileCacheConfig = &cfg.FileCacheConfig{EnableCrc: true, ExperimentalParallelDownloadsDefaultOn: true} dt.setupHelper() } func (dt *downloaderTest) TearDown() { + if dt.job != nil { + dt.job.Invalidate() + } + if dt.jm != nil { + dt.jm.Destroy() + } dt.fakeStorage.ShutDown() operations.RemoveDir(cacheDir) } @@ -198,7 +213,7 @@ func (dt *downloaderTest) Test_GetJob_Concurrent() { } // make concurrent requests - for i := 0; i < 5; i++ { + for i := range 5 { wg.Add(1) go getFunc(i) } @@ -206,7 +221,7 @@ func (dt *downloaderTest) Test_GetJob_Concurrent() { dt.verifyJob(dt.job, &dt.object, dt.bucket, dt.jm.sequentialReadSizeMb) // Verify all jobs - for i := 0; i < 5; i++ { + for i := range 5 { ExpectEq(dt.job, jobs[i]) } } @@ -253,7 +268,7 @@ func (dt *downloaderTest) Test_InvalidateAndRemoveJob_Concurrent() { wg := sync.WaitGroup{} // Make concurrent requests - for i := 0; i < 5; i++ { + for range 5 { wg.Add(1) invalidateFunc := func() { dt.jm.InvalidateAndRemoveJob(dt.object.Name, dt.bucket.Name()) @@ -316,7 +331,7 @@ func (dt *downloaderTest) Test_CreateJobIfNotExists_InvalidateAndRemoveJob_Concu wg.Done() } - for i := 0; i < 5; i++ { + for range 5 { wg.Add(2) go createNewJob() go invalidateJob() diff --git a/internal/cache/file/downloader/jm_parallel_downloads_test.go b/internal/cache/file/downloader/jm_parallel_downloads_test.go index 1b4d355c8b..4e48005384 100644 --- a/internal/cache/file/downloader/jm_parallel_downloads_test.go +++ b/internal/cache/file/downloader/jm_parallel_downloads_test.go @@ -22,14 +22,18 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/data" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/lru" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/util" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + "cloud.google.com/go/storage/control/apiv2/controlpb" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/data" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -49,8 +53,11 @@ func createObjectInBucket(t *testing.T, objPath string, objSize int64, bucket gc func configureFakeStorage(t *testing.T) storage.StorageHandle { t.Helper() - fakeStorage := storage.NewFakeStorage() + mockClient := new(storage.MockStorageControlClient) + fakeStorage := storage.NewFakeStorageWithMockClient(mockClient, cfg.HTTP2) t.Cleanup(func() { fakeStorage.ShutDown() }) + mockClient.On("GetStorageLayout", mock.Anything, mock.Anything, mock.Anything). + Return(&controlpb.StorageLayout{}, nil) return fakeStorage.CreateStorageHandle() } @@ -73,12 +80,7 @@ func createObjectInStoreAndInitCache(t *testing.T, cache *lru.Cache, bucket gcs. BucketName: storage.TestBucketName, ObjectName: objectName, } - fileInfo := data.FileInfo{ - Key: fileInfoKey, - ObjectGeneration: minObj.Generation, - FileSize: minObj.Size, - Offset: 0, - } + fileInfo := data.NewFileInfo(fileInfoKey, minObj.Generation, minObj.Size, 0, false, nil, 1) fileInfoKeyName, err := fileInfoKey.Key() if err != nil { t.Fatalf("Error occurred while retrieving fileInfoKey: %v", err) @@ -139,7 +141,8 @@ func TestParallelDownloads(t *testing.T) { cache, cacheDir := configureCache(t, 2*tc.objectSize) storageHandle := configureFakeStorage(t) ctx := context.Background() - bucket := storageHandle.BucketHandle(ctx, storage.TestBucketName, "") + bucket, err := storageHandle.BucketHandle(ctx, storage.TestBucketName, "") + assert.Nil(t, err) minObj, content := createObjectInStoreAndInitCache(t, cache, bucket, "path/in/gcs/foo.txt", tc.objectSize) fileCacheConfig := &cfg.FileCacheConfig{ EnableParallelDownloads: true, @@ -149,11 +152,12 @@ func TestParallelDownloads(t *testing.T) { WriteBufferSize: 4 * 1024 * 1024, EnableODirect: tc.enableODirect, } - jm := NewJobManager(cache, util.DefaultFilePerm, util.DefaultDirPerm, cacheDir, 2, fileCacheConfig) + jm := NewJobManager(cache, util.DefaultFilePerm, util.DefaultDirPerm, cacheDir, 2, fileCacheConfig, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), 1) + t.Cleanup(func() { jm.Destroy() }) job := jm.CreateJobIfNotExists(&minObj, bucket) subscriberC := job.subscribe(tc.subscribedOffset) - _, err := job.Download(context.Background(), 10, false) + _, err = job.Download(context.Background(), 10, false) timeout := time.After(1 * time.Second) for { @@ -180,7 +184,8 @@ func TestMultipleConcurrentDownloads(t *testing.T) { storageHandle := configureFakeStorage(t) cache, cacheDir := configureCache(t, 30*util.MiB) ctx := context.Background() - bucket := storageHandle.BucketHandle(ctx, storage.TestBucketName, "") + bucket, err := storageHandle.BucketHandle(ctx, storage.TestBucketName, "") + assert.Nil(t, err) minObj1, content1 := createObjectInStoreAndInitCache(t, cache, bucket, "path/in/gcs/foo.txt", 10*util.MiB) minObj2, content2 := createObjectInStoreAndInitCache(t, cache, bucket, "path/in/gcs/bar.txt", 5*util.MiB) fileCacheConfig := &cfg.FileCacheConfig{ @@ -191,7 +196,8 @@ func TestMultipleConcurrentDownloads(t *testing.T) { MaxParallelDownloads: 2, WriteBufferSize: 4 * 1024 * 1024, } - jm := NewJobManager(cache, util.DefaultFilePerm, util.DefaultDirPerm, cacheDir, 2, fileCacheConfig) + jm := NewJobManager(cache, util.DefaultFilePerm, util.DefaultDirPerm, cacheDir, 2, fileCacheConfig, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), 1) + t.Cleanup(func() { jm.Destroy() }) job1 := jm.CreateJobIfNotExists(&minObj1, bucket) job2 := jm.CreateJobIfNotExists(&minObj2, bucket) s1 := job1.subscribe(10 * util.MiB) diff --git a/internal/cache/file/downloader/job.go b/internal/cache/file/downloader/job.go index c0fb2b9aa0..b80616d838 100644 --- a/internal/cache/file/downloader/job.go +++ b/internal/cache/file/downloader/job.go @@ -22,18 +22,18 @@ import ( "io/fs" "os" "reflect" - "strings" "syscall" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/data" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/lru" - cacheutil "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/util" - "github.com/googlecloudplatform/gcsfuse/v2/internal/locker" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" - "github.com/googlecloudplatform/gcsfuse/v2/internal/monitor" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/data" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + cacheutil "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/locker" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "golang.org/x/net/context" "golang.org/x/sync/semaphore" ) @@ -57,12 +57,13 @@ type Job struct { // Constant data ///////////////////////// - object *gcs.MinObject - bucket gcs.Bucket - fileInfoCache *lru.Cache - sequentialReadSizeMb int32 - fileSpec data.FileSpec - fileCacheConfig *cfg.FileCacheConfig + object *gcs.MinObject + bucket gcs.Bucket + fileInfoCache *lru.Cache + sequentialReadSizeMb int32 + fileSpec data.FileSpec + fileCacheConfig *cfg.FileCacheConfig + cacheDirVolumeBlockSize uint64 ///////////////////////// // Mutable state @@ -97,6 +98,15 @@ type Job struct { // Channel which is used by goroutines to know which ranges need to be // downloaded when parallel download is enabled. rangeChan chan data.ObjectRange + + metricsHandle metrics.MetricHandle + + traceHandle tracing.TraceHandle + + // inflightChunks tracks chunks currently being downloaded to prevent redundant requests. + // Map key is chunkID. Value is a channel that closes when download completes. + // This is used for sparse files. + inflightChunks map[uint64]chan struct{} } // JobStatus represents the status of job. @@ -122,16 +132,26 @@ func NewJob( removeJobCallback func(), fileCacheConfig *cfg.FileCacheConfig, maxParallelismSem *semaphore.Weighted, + metricHandle metrics.MetricHandle, + traceHandle tracing.TraceHandle, + cacheDirVolumeBlockSize uint64, ) (job *Job) { + if traceHandle == nil { + traceHandle = tracing.NewNoopTracer() + } + job = &Job{ - object: object, - bucket: bucket, - fileInfoCache: fileInfoCache, - sequentialReadSizeMb: sequentialReadSizeMb, - fileSpec: fileSpec, - removeJobCallback: removeJobCallback, - fileCacheConfig: fileCacheConfig, - maxParallelismSem: maxParallelismSem, + object: object, + bucket: bucket, + fileInfoCache: fileInfoCache, + sequentialReadSizeMb: sequentialReadSizeMb, + fileSpec: fileSpec, + removeJobCallback: removeJobCallback, + fileCacheConfig: fileCacheConfig, + maxParallelismSem: maxParallelismSem, + metricsHandle: metricHandle, + traceHandle: traceHandle, + cacheDirVolumeBlockSize: cacheDirVolumeBlockSize, } job.mu = locker.New("Job-"+fileSpec.Path, job.checkInvariants) job.init() @@ -156,6 +176,7 @@ func (job *Job) init() { job.status = JobStatus{NotStarted, nil, 0} job.subscribers = list.List{} job.doneCh = make(chan struct{}) + job.inflightChunks = make(map[uint64]chan struct{}) } // cancel is helper function to cancel the in-progress job.downloadAsync goroutine. @@ -270,10 +291,7 @@ func (job *Job) updateStatusOffset(downloadedOffset int64) (err error) { return err } - updatedFileInfo := data.FileInfo{ - Key: fileInfoKey, ObjectGeneration: job.object.Generation, - FileSize: job.object.Size, Offset: uint64(downloadedOffset), - } + updatedFileInfo := data.NewFileInfo(fileInfoKey, job.object.Generation, job.object.Size, uint64(downloadedOffset), false, nil, job.cacheDirVolumeBlockSize) err = job.fileInfoCache.UpdateWithoutChangingOrder(fileInfoKeyName, updatedFileInfo) if err == nil { @@ -289,10 +307,11 @@ func (job *Job) updateStatusOffset(downloadedOffset int64) (err error) { } // downloadObjectToFile downloads the backing object from GCS into the given -// file and updates the file info cache. It uses gcs.Bucket's NewReader method +// file and updates the file info cache. It uses gcs.Bucket's NewReaderWithReadHandle method // to download the object. func (job *Job) downloadObjectToFile(cacheFile *os.File) (err error) { - var newReader io.ReadCloser + var newReader gcs.StorageReader + var readHandle []byte var start, end, sequentialReadSize, newReaderLimit int64 end = int64(job.object.Size) sequentialReadSize = int64(job.sequentialReadSizeMb) * cacheutil.MiB @@ -304,7 +323,7 @@ func (job *Job) downloadObjectToFile(cacheFile *os.File) (err error) { for start < end { if newReader == nil { newReaderLimit = min(start+sequentialReadSize, end) - newReader, err = job.bucket.NewReader( + newReader, err = job.bucket.NewReaderWithReadHandle( job.cancelCtx, &gcs.ReadObjectRequest{ Name: job.object.Name, @@ -314,12 +333,16 @@ func (job *Job) downloadObjectToFile(cacheFile *os.File) (err error) { Limit: uint64(newReaderLimit), }, ReadCompressed: job.object.HasContentEncodingGzip(), + ReadHandle: readHandle, }) if err != nil { err = fmt.Errorf("downloadObjectToFile: error in creating NewReader with start %d and limit %d: %w", start, newReaderLimit, err) return err } - monitor.CaptureGCSReadMetrics(job.cancelCtx, util.Sequential, newReaderLimit-start) + if newReader != nil { + readHandle = newReader.ReadHandle() + } + metrics.CaptureGCSReadMetrics(job.metricsHandle, metrics.ReadTypeNames[metrics.ReadTypeSequential], newReaderLimit-start) } maxRead := min(ReadChunkSize, newReaderLimit-start) @@ -359,10 +382,10 @@ func (job *Job) downloadObjectToFile(cacheFile *os.File) (err error) { // // Acquires and releases LOCK(job.mu) func (job *Job) cleanUpDownloadAsyncJob() { - // Close the job.doneCh, clear the cancelFunc & cancelCtx and call the + // Clear the cancelFunc & cancelCtx and call the // remove job callback function. + // Finally, close the job.doneCh. job.cancelFunc() - close(job.doneCh) job.mu.Lock() if job.removeJobCallback != nil { @@ -371,6 +394,7 @@ func (job *Job) cleanUpDownloadAsyncJob() { } job.cancelCtx, job.cancelFunc = nil, nil job.mu.Unlock() + close(job.doneCh) } // createCacheFile is a helper function which creates file in cache using @@ -431,7 +455,7 @@ func (job *Job) downloadObjectAsync() { // downloading. If the entry is deleted in between which is expected // to happen at the time of eviction, then the job should be // marked Invalid instead of Failed. - if strings.Contains(err.Error(), lru.EntryNotExistErrMsg) { + if errors.Is(err, lru.ErrEntryNotExist) { job.updateStatusAndNotifySubscribers(Invalid, err) return } @@ -441,7 +465,7 @@ func (job *Job) downloadObjectAsync() { // Truncate as the parallel downloads can create file with size little higher // than the actual object size because writing with O_DIRECT happens in size - // multiple of cacheutil.MinimumAlignSizeForWriting. + // multiple of cfg.MinimumAlignSizeForWriting. err = cacheFile.Truncate(int64(job.object.Size)) if err != nil { err = fmt.Errorf("downloadObjectAsync: error while truncating cache file: %w", err) @@ -468,7 +492,7 @@ func (job *Job) Download(ctx context.Context, offset int64, waitForDownload bool job.mu.Lock() if int64(job.object.Size) < offset { defer job.mu.Unlock() - err = fmt.Errorf(fmt.Sprintf("Download: the requested offset %d is greater than the size of object %d", offset, job.object.Size)) + err = fmt.Errorf("download: the requested offset %d is greater than the size of object %d", offset, job.object.Size) return job.status, err } @@ -478,7 +502,8 @@ func (job *Job) Download(ctx context.Context, offset int64, waitForDownload bool } else if job.status.Name == NotStarted { // Start the async download job.status.Name = Downloading - job.cancelCtx, job.cancelFunc = context.WithCancel(context.Background()) + spanCtx := job.traceHandle.PropagateTraceContext(context.Background(), ctx) + job.cancelCtx, job.cancelFunc = context.WithCancel(spanCtx) go job.downloadObjectAsync() } else if job.status.Name == Failed || job.status.Name == Invalid || job.status.Offset >= offset { defer job.mu.Unlock() @@ -517,7 +542,8 @@ func (job *Job) GetStatus() JobStatus { // Compares CRC32 of the downloaded file with the CRC32 from GCS object metadata. // In case of mismatch deletes the file and corresponding entry from file cache. func (job *Job) validateCRC() (err error) { - if !job.fileCacheConfig.EnableCrc { + // Todo (b/446440219): Enable crc check for ZB once it is fixed + if !job.fileCacheConfig.EnableCrc || job.bucket.BucketType().Zonal { return } @@ -574,3 +600,10 @@ func (job *Job) IsParallelDownloadsEnabled() bool { } return false } + +func (job *Job) IsExperimentalParallelDownloadsDefaultOn() bool { + if job.fileCacheConfig != nil && job.fileCacheConfig.ExperimentalParallelDownloadsDefaultOn { + return true + } + return false +} diff --git a/internal/cache/file/downloader/job_test.go b/internal/cache/file/downloader/job_test.go index 172a08e693..9b7828500d 100644 --- a/internal/cache/file/downloader/job_test.go +++ b/internal/cache/file/downloader/job_test.go @@ -28,16 +28,21 @@ import ( "sync/atomic" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/data" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/lru" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/util" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" - testutil "github.com/googlecloudplatform/gcsfuse/v2/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/data" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + testutil "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" . "github.com/jacobsa/ogletest" "golang.org/x/sync/semaphore" ) +// NOTE: Please add new tests in job_testify_test.go file. This file +// is deprecated and these tests will be moved to the job_testify_test.go. + //////////////////////////////////////////////////////////////////////// // Boilerplate //////////////////////////////////////////////////////////////////////// @@ -59,17 +64,12 @@ func (dt *downloaderTest) initJobTest(objectName string, objectContent []byte, s } dt.cache = lru.NewCache(lruCacheSize) - dt.job = NewJob(&dt.object, dt.bucket, dt.cache, sequentialReadSize, dt.fileSpec, removeCallback, dt.defaultFileCacheConfig, semaphore.NewWeighted(math.MaxInt64)) + dt.job = NewJob(&dt.object, dt.bucket, dt.cache, sequentialReadSize, dt.fileSpec, removeCallback, dt.defaultFileCacheConfig, semaphore.NewWeighted(math.MaxInt64), metrics.NewNoopMetrics(), tracing.NewNoopTracer(), 1) fileInfoKey := data.FileInfoKey{ BucketName: storage.TestBucketName, ObjectName: objectName, } - fileInfo := data.FileInfo{ - Key: fileInfoKey, - ObjectGeneration: dt.object.Generation, - FileSize: dt.object.Size, - Offset: 0, - } + fileInfo := data.NewFileInfo(fileInfoKey, dt.object.Generation, dt.object.Size, 0, false, nil, 1) fileInfoKeyName, err := fileInfoKey.Key() AssertEq(nil, err) _, err = dt.cache.Insert(fileInfoKeyName, fileInfo) @@ -77,7 +77,7 @@ func (dt *downloaderTest) initJobTest(objectName string, objectContent []byte, s } func (dt *downloaderTest) verifyInvalidError(err error) { - AssertTrue((nil == err) || (errors.Is(err, context.Canceled)) || (strings.Contains(err.Error(), lru.EntryNotExistErrMsg)), + AssertTrue((nil == err) || (errors.Is(err, context.Canceled)) || errors.Is(err, lru.ErrEntryNotExist), fmt.Sprintf("actual error:%v is not as expected", err)) } @@ -97,7 +97,7 @@ func (dt *downloaderTest) verifyFileInfoEntry(offset uint64) { AssertTrue(fileInfo != nil) AssertEq(dt.object.Generation, fileInfo.(data.FileInfo).ObjectGeneration) AssertLe(offset, fileInfo.(data.FileInfo).Offset) - AssertEq(dt.object.Size, fileInfo.(data.FileInfo).Size()) + AssertEq(dt.object.Size, fileInfo.(data.FileInfo).ContentSize()) } func (dt *downloaderTest) getFileInfo() lru.ValueType { @@ -236,12 +236,7 @@ func (dt *downloaderTest) Test_updateStatusOffset_UpdateEntry() { BucketName: storage.TestBucketName, ObjectName: DefaultObjectName, } - fileInfo := data.FileInfo{ - Key: fileInfoKey, - ObjectGeneration: dt.job.object.Generation, - FileSize: dt.job.object.Size, - Offset: 0, - } + fileInfo := data.NewFileInfo(fileInfoKey, dt.job.object.Generation, dt.job.object.Size, 0, false, nil, 1) fileInfoKeyName, err := fileInfoKey.Key() AssertEq(nil, err) _, err = dt.cache.Insert(fileInfoKeyName, fileInfo) @@ -283,7 +278,7 @@ func (dt *downloaderTest) Test_updateStatusOffset_InsertNew() { err = dt.job.updateStatusOffset(10) AssertNe(nil, err) - AssertTrue(strings.Contains(err.Error(), lru.EntryNotExistErrMsg)) + AssertTrue(errors.Is(err, lru.ErrEntryNotExist)) // Confirm job's status offset AssertEq(0, dt.job.status.Offset) } @@ -293,12 +288,7 @@ func (dt *downloaderTest) Test_updateStatusOffset_Fail() { BucketName: storage.TestBucketName, ObjectName: DefaultObjectName, } - fileInfo := data.FileInfo{ - Key: fileInfoKey, - ObjectGeneration: dt.job.object.Generation, - FileSize: dt.job.object.Size, - Offset: 0, - } + fileInfo := data.NewFileInfo(fileInfoKey, dt.job.object.Generation, dt.job.object.Size, 0, false, nil, 1) fileInfoKeyName, err := fileInfoKey.Key() AssertEq(nil, err) _, err = dt.cache.Insert(fileInfoKeyName, fileInfo) @@ -309,7 +299,7 @@ func (dt *downloaderTest) Test_updateStatusOffset_Fail() { err = dt.job.updateStatusOffset(15) AssertNe(nil, err) - AssertTrue(strings.Contains(err.Error(), lru.InvalidUpdateEntrySizeErrorMsg)) + AssertTrue(errors.Is(err, lru.ErrInvalidUpdateEntrySize)) // Confirm job's status offset AssertEq(0, dt.job.status.Offset) } @@ -577,7 +567,9 @@ func (dt *downloaderTest) Test_Download_WhenAsyncFails() { // Verify that jobStatus is failed AssertEq(Failed, jobStatus.Name) AssertGe(jobStatus.Offset, 0) - AssertTrue(strings.Contains(jobStatus.Err.Error(), lru.InvalidUpdateEntrySizeErrorMsg)) + AssertTrue(errors.Is(jobStatus.Err, lru.ErrInvalidUpdateEntrySize)) + // Wait for the async goroutine to complete its cleanup. + <-dt.job.doneCh // Verify callback is executed AssertTrue(callbackExecuted.Load()) } @@ -588,7 +580,7 @@ func (dt *downloaderTest) Test_Download_AlreadyFailed() { objectContent := testutil.GenerateRandomBytes(objectSize) dt.initJobTest(objectName, objectContent, DefaultSequentialReadSizeMb, uint64(objectSize), func() {}) dt.job.mu.Lock() - dt.job.status = JobStatus{Failed, fmt.Errorf(lru.InvalidUpdateEntrySizeErrorMsg), 8} + dt.job.status = JobStatus{Failed, lru.ErrInvalidUpdateEntrySize, 8} dt.job.mu.Unlock() // Requesting again from download job which is in failed state @@ -596,7 +588,7 @@ func (dt *downloaderTest) Test_Download_AlreadyFailed() { AssertEq(nil, err) AssertEq(Failed, jobStatus.Name) - AssertTrue(strings.Contains(jobStatus.Err.Error(), lru.InvalidUpdateEntrySizeErrorMsg)) + AssertTrue(errors.Is(jobStatus.Err, lru.ErrInvalidUpdateEntrySize)) } func (dt *downloaderTest) Test_Download_AlreadyInvalid() { @@ -630,7 +622,7 @@ func (dt *downloaderTest) Test_Download_InvalidOffset() { jobStatus, err := dt.job.Download(context.Background(), offset, true) AssertNe(nil, err) - AssertTrue(strings.Contains(err.Error(), fmt.Sprintf("Download: the requested offset %d is greater than the size of object %d", offset, dt.object.Size))) + AssertTrue(strings.Contains(err.Error(), fmt.Sprintf("download: the requested offset %d is greater than the size of object %d", offset, dt.object.Size))) expectedJobStatus := JobStatus{NotStarted, nil, 0} AssertTrue(reflect.DeepEqual(expectedJobStatus, jobStatus)) AssertFalse(callbackExecuted.Load()) @@ -680,7 +672,7 @@ func (dt *downloaderTest) Test_Download_Concurrent() { ctx := context.Background() wg := sync.WaitGroup{} offsets := []int64{0, 4 * util.MiB, 16 * util.MiB, 8 * util.MiB, int64(objectSize), int64(objectSize) + 1} - expectedErrs := []error{nil, nil, nil, nil, nil, fmt.Errorf(fmt.Sprintf("Download: the requested offset %d is greater than the size of object %d", int64(objectSize)+1, int64(objectSize)))} + expectedErrs := []error{nil, nil, nil, nil, nil, fmt.Errorf("download: the requested offset %d is greater than the size of object %d", int64(objectSize)+1, int64(objectSize))} downloadFunc := func(expectedOffset int64, expectedErr error) { defer wg.Done() var jobStatus JobStatus @@ -823,7 +815,7 @@ func (dt *downloaderTest) Test_Invalidate_Concurrent() { } // start concurrent Invalidate - for i := 0; i < 5; i++ { + for range 5 { wg.Add(1) go invalidateFunc() } @@ -866,7 +858,7 @@ func (dt *downloaderTest) Test_Invalidate_Download_Concurrent() { // Start concurrent invalidate and download offsets := [6]int64{0, util.MiB, 5 * util.MiB, 0, 2 * util.MiB, 10 * util.MiB} - for i := 0; i < len(offsets); i++ { + for i := range len(offsets) { wg.Add(2) waitForDownload := false if i%2 == 0 { @@ -949,6 +941,8 @@ func (dt *downloaderTest) Test_validateCRC_ForTamperedFileWhenEnableCRCIsFalse() dt.verifyFileInfoEntry(uint64(jobStatus.Offset)) } +// Disabled because it's flaky. +/* func (dt *downloaderTest) Test_validateCRC_WheContextIsCancelled() { objectName := "path/in/gcs/file2.txt" objectSize := 10 * util.MiB @@ -962,12 +956,18 @@ func (dt *downloaderTest) Test_validateCRC_WheContextIsCancelled() { AssertEq(nil, dt.job.status.Err) AssertGe(dt.job.status.Offset, offset) - dt.job.cancelFunc() + // Taking lock to ensure cancelFunc is valid before calling it. + dt.job.mu.Lock() + if dt.job.cancelFunc != nil { + dt.job.cancelFunc() + } + dt.job.mu.Unlock() dt.waitForCrcCheckToBeCompleted() AssertEq(Invalid, dt.job.status.Name) dt.verifyInvalidError(dt.job.status.Err) } +*/ func (dt *downloaderTest) Test_handleError_SetStatusAsInvalidWhenContextIsCancelled() { subscriberOffset := int64(1) @@ -1019,6 +1019,22 @@ func (dt *downloaderTest) Test_When_Parallel_Download_Is_Disabled() { AssertFalse(result) } +func (dt *downloaderTest) Test_When_Experimental_Default_Parallel_Download_Explicitly_Set_On() { + //Arrange - initJobTest is being called in setup of downloader.go + dt.job.fileCacheConfig.ExperimentalParallelDownloadsDefaultOn = true + + result := dt.job.IsExperimentalParallelDownloadsDefaultOn() + + AssertTrue(result) +} + +func (dt *downloaderTest) Test_When_Experimental_Default_Parallel_Download_On() { + + result := dt.job.IsExperimentalParallelDownloadsDefaultOn() + + AssertTrue(result) +} + func (dt *downloaderTest) Test_createCacheFile_WhenNonParallelDownloads() { //Arrange - initJobTest is being called in setup of downloader.go dt.job.fileCacheConfig.EnableParallelDownloads = false diff --git a/internal/cache/file/downloader/job_testify_test.go b/internal/cache/file/downloader/job_testify_test.go new file mode 100644 index 0000000000..6e65a1a719 --- /dev/null +++ b/internal/cache/file/downloader/job_testify_test.go @@ -0,0 +1,128 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package downloader + +import ( + "io" + "math" + "os" + "path" + "strings" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/util/diskutil" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "golang.org/x/net/context" + "golang.org/x/sync/semaphore" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/data" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + testutil "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type JobTestifyTest struct { + suite.Suite + ctx context.Context + defaultFileCacheConfig *cfg.FileCacheConfig + job *Job + object gcs.MinObject + cache *lru.Cache + fileSpec data.FileSpec + mockBucket *storage.TestifyMockBucket +} + +func TestJobTestifyTestSuite(testSuite *testing.T) { suite.Run(testSuite, new(JobTestifyTest)) } + +func (t *JobTestifyTest) initReadCacheTestifyTest(objectName string, objectContent []byte, sequentialReadSize int32, lruCacheSize uint64, removeCallback func()) { + // mock stat object call + minObject := gcs.MinObject{ + Name: objectName, + Size: uint64(len(objectContent)), + } + t.object = minObject + t.fileSpec = data.FileSpec{ + Path: path.Join(path.Join(os.Getenv("HOME"), "cache/dir"), t.object.Name), + FilePerm: util.DefaultFilePerm, + DirPerm: util.DefaultDirPerm, + } + t.cache = lru.NewCache(lruCacheSize) + cacheDirVolumeBlockSize := diskutil.GetVolumeBlockSize(t.fileSpec.Path) + t.job = NewJob(&t.object, t.mockBucket, t.cache, sequentialReadSize, t.fileSpec, removeCallback, t.defaultFileCacheConfig, semaphore.NewWeighted(math.MaxInt64), metrics.NewNoopMetrics(), tracing.NewNoopTracer(), cacheDirVolumeBlockSize) + fileInfoKey := data.FileInfoKey{ + BucketName: storage.TestBucketName, + ObjectName: objectName, + } + fileInfo := data.NewFileInfo(fileInfoKey, t.object.Generation, t.object.Size, 0, false, nil, 1) + fileInfoKeyName, err := fileInfoKey.Key() + assert.Equal(t.T(), nil, err) + _, err = t.cache.Insert(fileInfoKeyName, fileInfo) + assert.Equal(t.T(), nil, err) +} + +func (t *JobTestifyTest) SetupTest() { + t.ctx, _ = context.WithCancel(context.Background()) + t.mockBucket = new(storage.TestifyMockBucket) +} + +func (t *JobTestifyTest) Test_downloadObjectToFile_WithReadHandle() { + objectName := "path/in/gcs/foo.txt" + objectSize := 10 * util.MiB + objectContent := testutil.GenerateRandomBytes(objectSize) + t.initReadCacheTestifyTest(objectName, objectContent, 5, uint64(2*objectSize), func() {}) + t.job.cancelCtx, t.job.cancelFunc = context.WithCancel(context.Background()) + defer t.job.cancelFunc() + file, err := util.CreateFile(data.FileSpec{Path: t.job.fileSpec.Path, + FilePerm: os.FileMode(0600), DirPerm: os.FileMode(0700)}, os.O_TRUNC|os.O_RDWR) + defer func() { + _ = file.Close() + }() + // Add subscriber + subscribedOffset := int64(10 * util.MiB) + notificationC := t.job.subscribe(subscribedOffset) + assert.Equal(t.T(), nil, err) + rc := io.NopCloser(strings.NewReader(string(objectContent))) + rd := &fake.FakeReader{ReadCloser: rc, Handle: []byte("opaque-handle")} + t.mockBucket.On("Name").Return(storage.TestBucketName) + readObjectReq := gcs.ReadObjectRequest{Name: objectName, Generation: 0, Range: &gcs.ByteRange{Start: 0, Limit: 5 * util.MiB}, ReadCompressed: false, ReadHandle: nil} + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, &readObjectReq).Return(rd, nil) + readObjectReq2 := gcs.ReadObjectRequest{Name: objectName, Generation: 0, Range: &gcs.ByteRange{Start: 5 * util.MiB, Limit: 10 * util.MiB}, ReadCompressed: false, ReadHandle: []byte("opaque-handle")} + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, &readObjectReq2).Return(rd, nil) + + // Start download + err = t.job.downloadObjectToFile(file) + + t.mockBucket.AssertExpectations(t.T()) + assert.Nil(t.T(), err) + jobStatus, ok := <-notificationC + assert.Equal(t.T(), true, ok) + // Check the notification is sent after subscribed offset + assert.GreaterOrEqual(t.T(), jobStatus.Offset, subscribedOffset) + t.job.mu.Lock() + defer t.job.mu.Unlock() + // Verify file is downloaded + verifyCompleteFile(t.T(), t.fileSpec, objectContent) + // Verify fileInfoCache update + verifyFileInfoEntry(t.T(), t.mockBucket, t.object, t.cache, uint64(objectSize)) +} diff --git a/internal/cache/file/downloader/parallel_downloads_job.go b/internal/cache/file/downloader/parallel_downloads_job.go index 132e91aea4..89bdb20167 100644 --- a/internal/cache/file/downloader/parallel_downloads_job.go +++ b/internal/cache/file/downloader/parallel_downloads_job.go @@ -21,12 +21,11 @@ import ( "io" "os" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/data" - cacheutil "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/util" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" - "github.com/googlecloudplatform/gcsfuse/v2/internal/monitor" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/data" + cacheutil "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" "golang.org/x/sync/errgroup" ) @@ -34,8 +33,8 @@ import ( // GCS into given destination writer. // // This function doesn't take locks and can be executed parallely. -func (job *Job) downloadRange(ctx context.Context, dstWriter io.Writer, start, end int64) error { - newReader, err := job.bucket.NewReader( +func (job *Job) downloadRange(ctx context.Context, dstWriter io.Writer, start, end int64, readHandle []byte, rangeMap map[int64]int64) ([]byte, error) { + newReader, err := job.bucket.NewReaderWithReadHandle( ctx, &gcs.ReadObjectRequest{ Name: job.object.Name, @@ -45,10 +44,11 @@ func (job *Job) downloadRange(ctx context.Context, dstWriter io.Writer, start, e Limit: uint64(end), }, ReadCompressed: job.object.HasContentEncodingGzip(), + ReadHandle: readHandle, }) if err != nil { err = fmt.Errorf("downloadRange: error in creating NewReader with start %d and limit %d: %w", start, end, err) - return err + return nil, err } defer func() { // Reader is closed after the data has been read and the error from closure @@ -60,27 +60,72 @@ func (job *Job) downloadRange(ctx context.Context, dstWriter io.Writer, start, e } }() - monitor.CaptureGCSReadMetrics(ctx, util.Parallel, end-start) + metrics.CaptureGCSReadMetrics(job.metricsHandle, metrics.ReadTypeNames[metrics.ReadTypeParallel], end-start) // Use standard copy function if O_DIRECT is disabled and memory aligned // buffer otherwise. if !job.fileCacheConfig.EnableODirect { - _, err = io.CopyN(dstWriter, newReader, end-start) + if job.IsExperimentalParallelDownloadsDefaultOn() { + for start < end { + writeSize := min(end-start, ReadChunkSize) + _, err = io.CopyN(dstWriter, newReader, writeSize) + if err != nil { + err = fmt.Errorf("downloadRange: error at the time of copying content to cache file %w", err) + return newReader.ReadHandle(), err + } + + err = job.updateRangeMap(rangeMap, start, start+writeSize) + if err != nil { + // should return existing read handle with error + return newReader.ReadHandle(), err + } + + start = start + writeSize + } + } else { + _, err = io.CopyN(dstWriter, newReader, end-start) + } } else { - _, err = cacheutil.CopyUsingMemoryAlignedBuffer(ctx, newReader, dstWriter, end-start, - job.fileCacheConfig.WriteBufferSize) - // If context is canceled while reading/writing in CopyUsingMemoryAlignedBuffer - // then it returns error different from context cancelled (invalid argument), - // and we need to report that error as context cancelled. - if !errors.Is(err, context.Canceled) && errors.Is(ctx.Err(), context.Canceled) { - err = errors.Join(err, ctx.Err()) + if job.IsExperimentalParallelDownloadsDefaultOn() { + for start < end { + writeSize := min(end-start, ReadChunkSize) + _, err = cacheutil.CopyUsingMemoryAlignedBuffer(ctx, newReader, dstWriter, writeSize, + job.fileCacheConfig.WriteBufferSize) + // If context is canceled while reading/writing in CopyUsingMemoryAlignedBuffer + // then it returns error different from context cancelled (invalid argument), + // and we need to report that error as context cancelled. + if !errors.Is(err, context.Canceled) && errors.Is(ctx.Err(), context.Canceled) { + err = errors.Join(err, ctx.Err()) + return newReader.ReadHandle(), err + } + if err != nil { + err = fmt.Errorf("downloadRange: error at the time of copying content to cache file %w", err) + return newReader.ReadHandle(), err + } + + err = job.updateRangeMap(rangeMap, start, start+writeSize) + if err != nil { + // should return existing read handle with error + return newReader.ReadHandle(), err + } + start = start + writeSize + } + } else { + _, err = cacheutil.CopyUsingMemoryAlignedBuffer(ctx, newReader, dstWriter, end-start, + job.fileCacheConfig.WriteBufferSize) + // If context is canceled while reading/writing in CopyUsingMemoryAlignedBuffer + // then it returns error different from context cancelled (invalid argument), + // and we need to report that error as context cancelled. + if !errors.Is(err, context.Canceled) && errors.Is(ctx.Err(), context.Canceled) { + err = errors.Join(err, ctx.Err()) + } } } if err != nil { err = fmt.Errorf("downloadRange: error at the time of copying content to cache file %w", err) } - return err + return newReader.ReadHandle(), err } // RangeMap maintains the ranges downloaded by the different goroutines. This @@ -136,6 +181,8 @@ func (job *Job) downloadOffsets(ctx context.Context, goroutineIndex int64, cache if goroutineIndex > 0 { defer job.maxParallelismSem.Release(1) } + var readHandle []byte + var err error for { // Read the offset to be downloaded from the channel. @@ -145,15 +192,17 @@ func (job *Job) downloadOffsets(ctx context.Context, goroutineIndex int64, cache return nil } - offsetWriter := io.NewOffsetWriter(cacheFile, int64(objectRange.Start)) - err := job.downloadRange(ctx, offsetWriter, objectRange.Start, objectRange.End) + offsetWriter := io.NewOffsetWriter(cacheFile, objectRange.Start) + readHandle, err = job.downloadRange(ctx, offsetWriter, objectRange.Start, objectRange.End, readHandle, rangeMap) if err != nil { return err } - err = job.updateRangeMap(rangeMap, objectRange.Start, objectRange.End) - if err != nil { - return err + if !job.IsExperimentalParallelDownloadsDefaultOn() { + err = job.updateRangeMap(rangeMap, objectRange.Start, objectRange.End) + if err != nil { + return err + } } } } diff --git a/internal/cache/file/downloader/parallel_downloads_job_test.go b/internal/cache/file/downloader/parallel_downloads_job_test.go index d6b186c9a9..9ce5412123 100644 --- a/internal/cache/file/downloader/parallel_downloads_job_test.go +++ b/internal/cache/file/downloader/parallel_downloads_job_test.go @@ -27,10 +27,10 @@ import ( "sync/atomic" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/data" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/util" - testutil "github.com/googlecloudplatform/gcsfuse/v2/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/data" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + testutil "github.com/googlecloudplatform/gcsfuse/v3/internal/util" . "github.com/jacobsa/ogletest" ) @@ -47,11 +47,12 @@ func init() { RegisterTestSuite(¶llelDownloaderTest{}) } func (dt *parallelDownloaderTest) SetUp(*TestInfo) { dt.defaultFileCacheConfig = &cfg.FileCacheConfig{ - EnableParallelDownloads: true, - ParallelDownloadsPerFile: 3, - DownloadChunkSizeMb: 3, - EnableCrc: true, - WriteBufferSize: 4 * 1024 * 1024, + ExperimentalParallelDownloadsDefaultOn: true, + EnableParallelDownloads: true, + ParallelDownloadsPerFile: 3, + DownloadChunkSizeMb: 3, + EnableCrc: true, + WriteBufferSize: 4 * 1024 * 1024, } dt.setupHelper() } @@ -65,6 +66,7 @@ func (dt *parallelDownloaderTest) Test_downloadRange() { removeCallback := func() { callbackExecuted.Store(true) } dt.initJobTest(objectName, objectContent, DefaultSequentialReadSizeMb, uint64(2*objectSize), removeCallback) dt.job.cancelCtx, dt.job.cancelFunc = context.WithCancel(context.Background()) + defer dt.job.cancelFunc() file, err := util.CreateFile(data.FileSpec{Path: dt.job.fileSpec.Path, FilePerm: os.FileMode(0600), DirPerm: os.FileMode(0700)}, os.O_TRUNC|os.O_RDWR) AssertEq(nil, err) @@ -81,30 +83,35 @@ func (dt *parallelDownloaderTest) Test_downloadRange() { // Download end 1MiB of object start, end := int64(9*util.MiB), int64(10*util.MiB) offsetWriter := io.NewOffsetWriter(file, start) - err = dt.job.downloadRange(context.Background(), offsetWriter, start, end) + rangeMap := make(map[int64]int64) + + _, err = dt.job.downloadRange(context.Background(), offsetWriter, start, end, nil, rangeMap) AssertEq(nil, err) verifyContentAtOffset(file, start, end) // Download start 4MiB of object start, end = int64(0*util.MiB), int64(4*util.MiB) offsetWriter = io.NewOffsetWriter(file, start) - err = dt.job.downloadRange(context.Background(), offsetWriter, start, end) + _, err = dt.job.downloadRange(context.Background(), offsetWriter, start, end, nil, rangeMap) AssertEq(nil, err) verifyContentAtOffset(file, start, end) + AssertEq(int64(4*util.MiB), rangeMap[start]) // Download middle 1B of object start, end = int64(5*util.MiB), int64(5*util.MiB+1) offsetWriter = io.NewOffsetWriter(file, start) - err = dt.job.downloadRange(context.Background(), offsetWriter, start, end) + _, err = dt.job.downloadRange(context.Background(), offsetWriter, start, end, nil, rangeMap) AssertEq(nil, err) verifyContentAtOffset(file, start, end) + AssertEq(int64(5*util.MiB+1), rangeMap[start]) // Download 0B of object start, end = int64(5*util.MiB), int64(5*util.MiB) offsetWriter = io.NewOffsetWriter(file, start) - err = dt.job.downloadRange(context.Background(), offsetWriter, start, end) + _, err = dt.job.downloadRange(context.Background(), offsetWriter, start, end, nil, rangeMap) AssertEq(nil, err) verifyContentAtOffset(file, start, end) + AssertEq(int64(5*util.MiB+1), rangeMap[start]) } func (dt *parallelDownloaderTest) Test_parallelDownloadObjectToFile() { @@ -113,6 +120,7 @@ func (dt *parallelDownloaderTest) Test_parallelDownloadObjectToFile() { objectContent := testutil.GenerateRandomBytes(objectSize) dt.initJobTest(objectName, objectContent, DefaultSequentialReadSizeMb, uint64(2*objectSize), func() {}) dt.job.cancelCtx, dt.job.cancelFunc = context.WithCancel(context.Background()) + defer dt.job.cancelFunc() // Add subscriber subscribedOffset := int64(1 * util.MiB) notificationC := dt.job.subscribe(subscribedOffset) diff --git a/internal/cache/file/downloader/parallel_downloads_job_testify_test.go b/internal/cache/file/downloader/parallel_downloads_job_testify_test.go new file mode 100644 index 0000000000..c208cab47b --- /dev/null +++ b/internal/cache/file/downloader/parallel_downloads_job_testify_test.go @@ -0,0 +1,164 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package downloader + +import ( + "io" + "os" + "strings" + "sync" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/data" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + testutil "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + "golang.org/x/net/context" +) + +type ParallelDownloaderJobTestifyTest struct { + JobTestifyTest +} + +func TestParallelDownloaderJobTestifyTestSuite(testSuite *testing.T) { + suite.Run(testSuite, new(ParallelDownloaderJobTestifyTest)) +} + +func (t *ParallelDownloaderJobTestifyTest) SetupTest() { + t.defaultFileCacheConfig = &cfg.FileCacheConfig{ + EnableParallelDownloads: true, + ParallelDownloadsPerFile: 3, + DownloadChunkSizeMb: 3, + EnableCrc: true, + WriteBufferSize: 4 * 1024 * 1024, + } + t.ctx, _ = context.WithCancel(context.Background()) + t.mockBucket = new(storage.TestifyMockBucket) +} + +func (t *ParallelDownloaderJobTestifyTest) Test_ParallelDownloadObjectToFile_NewReaderWithReadHandle() { + objectName := "path/in/gcs/foo.txt" + objectSize := 10 * util.MiB + chunkSize := 3 * util.MiB + objectContent := testutil.GenerateRandomBytes(objectSize) + t.initReadCacheTestifyTest(objectName, objectContent, DefaultSequentialReadSizeMb, uint64(2*objectSize), func() {}) + t.job.cancelCtx, t.job.cancelFunc = context.WithCancel(context.Background()) + defer t.job.cancelFunc() + // Add subscriber + subscribedOffset := int64(1 * util.MiB) + notificationC := t.job.subscribe(subscribedOffset) + file, err := util.CreateFile(data.FileSpec{Path: t.job.fileSpec.Path, + FilePerm: os.FileMode(0600), DirPerm: os.FileMode(0700)}, os.O_TRUNC|os.O_RDWR) + assert.Equal(t.T(), nil, err) + defer func() { + _ = file.Close() + }() + // To download a file of 10mb using ParallelDownloadsPerFile = 3 and + // DownloadChunkSizeMb = 3mb there will be one call to NewReaderWithReadHandle + // with read handle. + handle := []byte("opaque-handle") + var ( + actualCallCount int64 + nilHandleCallCount int64 + mu sync.Mutex // To protect counter updates from concurrent mock calls + ) + // Reset counters + actualCallCount = 0 + nilHandleCallCount = 0 + + // Helper function to increment counters based on the actual request seen by the mock + incrementCounters := func(args mock.Arguments) { + req := args.Get(1).(*gcs.ReadObjectRequest) // Second arg to NewReaderWithReadHandle + mu.Lock() + actualCallCount++ + if req.ReadHandle == nil { + nilHandleCallCount++ + } + mu.Unlock() + } + + // Define ranges for clarity + rangeR1 := &gcs.ByteRange{Start: uint64(0 * chunkSize), Limit: uint64(1 * chunkSize)} + rangeR2 := &gcs.ByteRange{Start: uint64(1 * chunkSize), Limit: uint64(2 * chunkSize)} + rangeR3 := &gcs.ByteRange{Start: uint64(2 * chunkSize), Limit: uint64(3 * chunkSize)} + rangeR4 := &gcs.ByteRange{Start: uint64(3 * chunkSize), Limit: uint64(objectSize)} // Last chunk + + // Create FakeReaders for each chunk, all will return the same propagatedHandle + readerR1 := &fake.FakeReader{ReadCloser: io.NopCloser(strings.NewReader(string(objectContent[0*chunkSize : 1*chunkSize]))), Handle: handle} + readerR2 := &fake.FakeReader{ReadCloser: io.NopCloser(strings.NewReader(string(objectContent[1*chunkSize : 2*chunkSize]))), Handle: handle} + readerR3 := &fake.FakeReader{ReadCloser: io.NopCloser(strings.NewReader(string(objectContent[2*chunkSize : 3*chunkSize]))), Handle: handle} + readerR4 := &fake.FakeReader{ReadCloser: io.NopCloser(strings.NewReader(string(objectContent[3*chunkSize : objectSize]))), Handle: handle} + + t.mockBucket.On("Name").Return(storage.TestBucketName) + + // Chunk 1 (R1): Must be ReadHandle: nil + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(req *gcs.ReadObjectRequest) bool { + return req.Range.Start == rangeR1.Start && req.Range.Limit == rangeR1.Limit && + req.ReadHandle == nil + })).Run(incrementCounters).Return(readerR1, nil).Once() + + // Chunk 2 (R2): ReadHandle can be nil or propagated. Match primarily on range. + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(req *gcs.ReadObjectRequest) bool { + return req.Range.Start == rangeR2.Start && req.Range.Limit == rangeR2.Limit + })).Run(incrementCounters).Return(readerR2, nil).Once() + + // Chunk 3 (R3): ReadHandle can be nil or propagated. Match primarily on range. + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(req *gcs.ReadObjectRequest) bool { + return req.Range.Start == rangeR3.Start && req.Range.Limit == rangeR3.Limit + })).Run(incrementCounters).Return(readerR3, nil).Once() + + // Chunk 4 (R4): ReadHandle should not be nil + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(req *gcs.ReadObjectRequest) bool { + return req.Range.Start == rangeR4.Start && req.Range.Limit == rangeR4.Limit && req.ReadHandle != nil + })).Run(incrementCounters).Return(readerR4, nil).Once() + + // Start download + err = t.job.parallelDownloadObjectToFile(file) + + assert.Equal(t.T(), nil, err, "parallelDownloadObjectToFile should not return an error") + assert.Equal(t.T(), int64(4), actualCallCount, "Total calls to NewReaderWithReadHandle should be 4") + // Assert the number of calls with ReadHandle: nil falls within the expected range. + // For ParallelDownloadsPerFile = 3 and 4 chunks: + // - nilHandleCallCount must be at least 1 (for the very first chunk processed by any worker). + // - nilHandleCallCount can be at most ParallelDownloadsPerFile (3), as each of the 3 launched workers + // uses a nil handle only for its first operation. + // 1 <= nilHandleCallCount <= 3. + minExpectedNilCalls := int64(1) + maxExpectedNilCalls := int64(3) + numberOfChunks := int64(4) // Based on objectSize and chunkSize + + assert.True(t.T(), nilHandleCallCount >= minExpectedNilCalls && nilHandleCallCount <= maxExpectedNilCalls, + "Expected nilHandleCallCount to be between %d and %d (inclusive), but got %d. ParallelDownloadsPerFile=%d, Chunks=%d", + minExpectedNilCalls, maxExpectedNilCalls, nilHandleCallCount, t.job.fileCacheConfig.ParallelDownloadsPerFile, numberOfChunks) + + t.mockBucket.AssertExpectations(t.T()) + assert.Equal(t.T(), nil, err) + jobStatus, ok := <-notificationC + assert.Equal(t.T(), true, ok) + // Check the notification is sent after subscribed offset + assert.GreaterOrEqual(t.T(), jobStatus.Offset, subscribedOffset) + t.job.mu.Lock() + defer t.job.mu.Unlock() + // Verify file is downloaded + verifyCompleteFile(t.T(), t.fileSpec, objectContent) + // Verify fileInfoCache update + verifyFileInfoEntry(t.T(), t.mockBucket, t.object, t.cache, uint64(objectSize)) +} diff --git a/internal/cache/file/downloader/sparse_downloads_job.go b/internal/cache/file/downloader/sparse_downloads_job.go new file mode 100644 index 0000000000..c8fbd72490 --- /dev/null +++ b/internal/cache/file/downloader/sparse_downloads_job.go @@ -0,0 +1,270 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package downloader + +import ( + "context" + "fmt" + "io" + "os" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/data" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "golang.org/x/sync/errgroup" +) + +// HandleSparseRead manages the download and validation of sparse file ranges. +// It checks if the requested range is already downloaded, calculates chunk +// boundaries, downloads missing chunks, and validates the cache hit status. +// +// Returns: +// - cacheHit: true if the requested range is available in cache, false if fallback to GCS is needed +// - error: non-nil if there was an error during the operation +// +// Note: The FileInfo in the cache is automatically updated by downloadSparseRange. +func (job *Job) HandleSparseRead(ctx context.Context, startOffset, endOffset int64) (cacheHit bool, err error) { + // Get current file info from cache + fileInfo, err := job.getFileInfo() + if err != nil { + return false, fmt.Errorf("HandleSparseRead: error getting file info: %w", err) + } + + // Check if the requested range is already downloaded + if fileInfo.DownloadedChunks.ContainsRange(uint64(startOffset), uint64(endOffset)) { + // Range already downloaded, return cache hit + return true, nil + } + + // Calculate the missing ranges to download and filter out in-flight chunks + chunksToDownload, waitChans, err := job.getChunksToDownload(fileInfo, startOffset, endOffset) + if err != nil { + return false, fmt.Errorf("HandleSparseRead: error calculating ranges: %w", err) + } + + downloadErrGroup, downloadErrGroupCtx := errgroup.WithContext(ctx) + chunkSize := uint64(job.fileCacheConfig.DownloadChunkSizeMb) * 1024 * 1024 + + // Download missing chunks in parallel + for _, chunkID := range chunksToDownload { + downloadErrGroup.Go(func() error { + // Acquire semaphore to limit concurrency across all jobs + if err := job.maxParallelismSem.Acquire(downloadErrGroupCtx, 1); err != nil { + return err + } + defer job.maxParallelismSem.Release(1) + start := chunkID * chunkSize + end := start + chunkSize + return job.downloadSparseRange(downloadErrGroupCtx, start, end) + }) + } + + downloadErr := downloadErrGroup.Wait() + + // Cleanup inflight chunks for all download attempts. + job.mu.Lock() + for _, chunkID := range chunksToDownload { + if ch, ok := job.inflightChunks[chunkID]; ok { + close(ch) + delete(job.inflightChunks, chunkID) + } + } + job.mu.Unlock() + + if downloadErr != nil { + return false, fmt.Errorf("sparse download failed: %w", downloadErr) + } + + // Wait for other inflight chunks + for _, ch := range waitChans { + select { + case <-ch: + case <-ctx.Done(): + return false, ctx.Err() + } + } + + // Verify the download was successful + cacheHit, err = job.verifySparseRangeDownloaded(startOffset, endOffset) + if err != nil { + return false, fmt.Errorf("error verifying download: %w", err) + } + + if !cacheHit { + return false, fmt.Errorf("cache miss after download: range [%d, %d) not found in downloaded ranges", startOffset, endOffset) + } + + return true, nil +} + +// getChunksToDownload calculates the missing chunks that need to be downloaded +// and filters out chunks that are already in-flight. It returns a list of +// individual chunks to download and marks them as in-flight. +func (job *Job) getChunksToDownload(fileInfo data.FileInfo, startOffset, endOffset int64) ([]uint64, []chan struct{}, error) { + if startOffset < 0 || endOffset < 0 || startOffset >= endOffset { + return nil, nil, fmt.Errorf("invalid offset range: [%d, %d)", startOffset, endOffset) + } + + // Get missing chunks from ByteRangeMap + missingChunks := fileInfo.DownloadedChunks.GetMissingChunks(uint64(startOffset), uint64(endOffset)) + + var chunksToDownload []uint64 + var waitChans []chan struct{} + + job.mu.Lock() + defer job.mu.Unlock() + + for _, chunkID := range missingChunks { + if ch, ok := job.inflightChunks[chunkID]; ok { + // Chunk is inflight, wait for it + waitChans = append(waitChans, ch) + } else { + // Chunk is not inflight, mark it and add to download list + ch := make(chan struct{}) + job.inflightChunks[chunkID] = ch + + chunksToDownload = append(chunksToDownload, chunkID) + } + } + return chunksToDownload, waitChans, nil +} + +// getFileInfo retrieves the FileInfo from cache for this job's object. +func (job *Job) getFileInfo() (data.FileInfo, error) { + fileInfoKey := data.FileInfoKey{ + BucketName: job.bucket.Name(), + ObjectName: job.object.Name, + } + fileInfoKeyName, err := fileInfoKey.Key() + if err != nil { + return data.FileInfo{}, fmt.Errorf("error creating fileInfoKeyName: %w", err) + } + + fileInfoVal := job.fileInfoCache.LookUpWithoutChangingOrder(fileInfoKeyName) + if fileInfoVal == nil { + return data.FileInfo{}, fmt.Errorf("file info not found in cache") + } + + fileInfo, ok := fileInfoVal.(data.FileInfo) + if !ok { + return data.FileInfo{}, fmt.Errorf("getFileInfo: cached value has wrong type") + } + + return fileInfo, nil +} + +// verifySparseRangeDownloaded verifies that the requested range has been +// successfully downloaded by checking the updated FileInfo from the cache. +// +// Returns: +// - cacheHit: whether the range is present in the downloaded ranges +// - err: any error that occurred while fetching the FileInfo +func (job *Job) verifySparseRangeDownloaded(startOffset, endOffset int64) (cacheHit bool, err error) { + // Fetch updated file info from cache + fileInfo, err := job.getFileInfo() + if err != nil { + return false, fmt.Errorf("verifySparseRangeDownloaded: %w", err) + } + + // Check if the range is downloaded + cacheHit = fileInfo.DownloadedChunks != nil && fileInfo.DownloadedChunks.ContainsRange(uint64(startOffset), uint64(endOffset)) + + logger.Tracef("Sparse file cache hit check: startOffset=%d, endOffset=%d, DownloadedRanges=%v, cacheHit=%t", + startOffset, endOffset, fileInfo.DownloadedChunks != nil, cacheHit) + + return cacheHit, nil +} + +// downloadSparseRange downloads a specific byte range [start, end) from the GCS object +// for sparse file support. It writes the data to the cache file at the appropriate +// offset. +// +// Acquires and releases LOCK(job.mu) +func (job *Job) downloadSparseRange(ctx context.Context, start, end uint64) error { + if start >= end { + return fmt.Errorf("downloadSparseRange: invalid range [%d, %d)", start, end) + } + + if end > job.object.Size { + end = job.object.Size + } + + // Create GCS reader for the specific range + newReader, err := job.bucket.NewReaderWithReadHandle( + ctx, + &gcs.ReadObjectRequest{ + Name: job.object.Name, + Generation: job.object.Generation, + Range: &gcs.ByteRange{ + Start: start, + Limit: end, + }, + ReadCompressed: job.object.HasContentEncodingGzip(), + ReadHandle: nil, + }) + if err != nil { + return fmt.Errorf("downloadSparseRange: error creating reader for range [%d, %d): %w", start, end, err) + } + defer newReader.Close() + + metrics.CaptureGCSReadMetrics(job.metricsHandle, metrics.ReadTypeNames[metrics.ReadTypeRandom], int64(end-start)) + + // Open cache file for writing + cacheFile, err := os.OpenFile(job.fileSpec.Path, os.O_WRONLY, job.fileSpec.FilePerm) + if err != nil { + return fmt.Errorf("downloadSparseRange: error opening cache file: %w", err) + } + defer cacheFile.Close() + + // Download from GCS and write to cache file + offsetWriter := io.NewOffsetWriter(cacheFile, int64(start)) + bytesWritten, err := io.CopyN(offsetWriter, newReader, int64(end-start)) + if err != nil { + return fmt.Errorf("downloadSparseRange: error copying data: %w", err) + } + + // Update FileInfo with downloaded range + job.mu.Lock() + defer job.mu.Unlock() + + // Re-fetch FileInfo in case it changed + fileInfo, err := job.getFileInfo() + if err != nil { + return fmt.Errorf("downloadSparseRange: %w", err) + } + + // Add the downloaded range + bytesAdded := fileInfo.DownloadedChunks.AddRange(start, start+uint64(bytesWritten)) + + // Update LRU cache size accounting + fileInfoKey := data.FileInfoKey{ + BucketName: job.bucket.Name(), + ObjectName: job.object.Name, + } + fileInfoKeyName, err := fileInfoKey.Key() + if err != nil { + return fmt.Errorf("downloadSparseRange: error creating fileInfoKeyName: %w", err) + } + err = job.fileInfoCache.UpdateSize(fileInfoKeyName, bytesAdded) + if err != nil { + return fmt.Errorf("downloadSparseRange: error updating cache size: %w", err) + } + + logger.Tracef("Job:%p (%s:/%s) downloaded range [%d, %d), added %d bytes to sparse file", + job, job.bucket.Name(), job.object.Name, start, end, bytesAdded) + + return nil +} diff --git a/internal/cache/file/downloader/sparse_downloads_job_test.go b/internal/cache/file/downloader/sparse_downloads_job_test.go new file mode 100644 index 0000000000..91221b00f9 --- /dev/null +++ b/internal/cache/file/downloader/sparse_downloads_job_test.go @@ -0,0 +1,328 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// File that contains tests specific to sparse downloads job. + +package downloader + +import ( + "context" + "fmt" + "os" + "reflect" + "sync/atomic" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/data" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + testutil "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + . "github.com/jacobsa/ogletest" +) + +// TestSparseDownloader runs all tests for sparse file downloads +func TestSparseDownloader(t *testing.T) { RunTests(t) } + +type sparseDownloaderTest struct { + downloaderTest +} + +func init() { RegisterTestSuite(&sparseDownloaderTest{}) } + +func (dt *sparseDownloaderTest) SetUp(*TestInfo) { + dt.defaultFileCacheConfig = &cfg.FileCacheConfig{ + ExperimentalEnableChunkCache: true, + DownloadChunkSizeMb: 20, // 20MB chunks for sparse files + EnableCrc: true, + ExperimentalParallelDownloadsDefaultOn: true, + } + dt.setupHelper() +} + +func (dt *sparseDownloaderTest) Test_getChunksToDownload() { + tests := []struct { + name string + offset int64 + requiredOffset int64 + chunkSizeMb int64 + objectSize uint64 + expectedChunks []uint64 + expectError bool + }{ + { + name: "single chunk - aligned", + offset: 0, + requiredOffset: 10 * util.MiB, + chunkSizeMb: 20, + objectSize: 100 * util.MiB, + expectedChunks: []uint64{0}, + expectError: false, + }, + { + name: "single chunk - unaligned start", + offset: 5 * util.MiB, + requiredOffset: 10 * util.MiB, + chunkSizeMb: 20, + objectSize: 100 * util.MiB, + expectedChunks: []uint64{0}, + expectError: false, + }, + { + name: "single chunk - unaligned end", + offset: 15 * util.MiB, + requiredOffset: 25 * util.MiB, + chunkSizeMb: 20, + objectSize: 100 * util.MiB, + expectedChunks: []uint64{0, 1}, + expectError: false, + }, + { + name: "chunk end capped at object size", + offset: 90 * util.MiB, + requiredOffset: 95 * util.MiB, + chunkSizeMb: 20, + objectSize: 100 * util.MiB, + expectedChunks: []uint64{4}, + expectError: false, + }, + { + name: "invalid range - offset >= requiredOffset", + offset: 10, + requiredOffset: 10, + chunkSizeMb: 20, + objectSize: 100 * util.MiB, + expectError: true, + }, + { + name: "invalid range - negative offset", + offset: -1, + requiredOffset: 10, + chunkSizeMb: 20, + objectSize: 100 * util.MiB, + expectError: true, + }, + } + + for _, tt := range tests { + objectName := "test/sparse_chunk_boundaries.txt" + dt.initJobTest(objectName, nil, DefaultSequentialReadSizeMb, tt.objectSize, func() {}) + dt.job.fileCacheConfig.DownloadChunkSizeMb = tt.chunkSizeMb + // Manually insert FileInfo with DownloadedChunks + fileInfoKey := data.FileInfoKey{ + BucketName: dt.job.bucket.Name(), + ObjectName: dt.job.object.Name, + } + chunkSizeBytes := uint64(tt.chunkSizeMb) * util.MiB + fileInfo := data.NewFileInfo(fileInfoKey, dt.job.object.Generation, tt.objectSize, ^uint64(0), true, data.NewByteRangeMap(chunkSizeBytes, tt.objectSize), 1) + + chunks, _, err := dt.job.getChunksToDownload(fileInfo, tt.offset, tt.requiredOffset) + + if tt.expectError { + AssertNe(nil, err, fmt.Sprintf("Test case %q: expected error but got none", tt.name)) + } else { + AssertEq(nil, err, fmt.Sprintf("Test case %q: unexpected error: %v", tt.name, err)) + AssertEq(len(tt.expectedChunks), len(chunks), fmt.Sprintf("Test case %q: chunks count mismatch", tt.name)) + for i, chunkID := range chunks { + AssertEq(tt.expectedChunks[i], chunkID, fmt.Sprintf("Test case %q: chunk %d mismatch", tt.name, i)) + } + } + } +} + +func (dt *sparseDownloaderTest) Test_getChunksToDownload_WithInflight() { + objectName := "test/sparse_inflight.txt" + objectSize := 100 * util.MiB + dt.initJobTest(objectName, nil, DefaultSequentialReadSizeMb, uint64(objectSize), func() {}) + dt.job.fileCacheConfig.DownloadChunkSizeMb = 20 // 20MB chunks + // Scenario: Request [0, 60MB) -> Chunks 0, 1, 2. + // Chunk 1 (20MB-40MB) is already in-flight. + // Expected: Download Chunk 0 and Chunk 2. Wait for Chunk 1. + inflightCh := make(chan struct{}) + dt.job.mu.Lock() + dt.job.inflightChunks[1] = inflightCh // Chunk 1 is in-flight + dt.job.mu.Unlock() + // Manually insert FileInfo with DownloadedChunks + fileInfoKey := data.FileInfoKey{ + BucketName: dt.job.bucket.Name(), + ObjectName: dt.job.object.Name, + } + chunkSizeBytes := uint64(20) * 1024 * 1024 + fileInfo := data.NewFileInfo(fileInfoKey, dt.job.object.Generation, uint64(objectSize), ^uint64(0), true, data.NewByteRangeMap(chunkSizeBytes, uint64(objectSize)), 1) + + chunks, waitChans, err := dt.job.getChunksToDownload(fileInfo, 0, 60*util.MiB) + + AssertEq(nil, err) + // Should have 2 chunks to download: Chunk 0 and Chunk 2 + AssertEq(2, len(chunks)) + AssertEq(uint64(0), chunks[0]) // Chunk 0 + AssertEq(uint64(2), chunks[1]) // Chunk 2 + // Should have 1 wait channel + AssertEq(1, len(waitChans)) + AssertEq(inflightCh, waitChans[0]) + // Verify inflightChunks map was updated + dt.job.mu.Lock() + _, ok0 := dt.job.inflightChunks[0] + _, ok1 := dt.job.inflightChunks[1] + _, ok2 := dt.job.inflightChunks[2] + dt.job.mu.Unlock() + AssertTrue(ok0, "Chunk 0 should be marked inflight") + AssertTrue(ok1, "Chunk 1 should remain inflight") + AssertTrue(ok2, "Chunk 2 should be marked inflight") +} + +func (dt *sparseDownloaderTest) Test_DownloadRange() { + objectName := "test/sparse_download_range.txt" + objectSize := 50 * util.MiB + objectContent := testutil.GenerateRandomBytes(objectSize) + var callbackExecuted atomic.Bool + removeCallback := func() { callbackExecuted.Store(true) } + dt.initJobTest(objectName, objectContent, DefaultSequentialReadSizeMb, uint64(objectSize), removeCallback) + + // Set up sparse file mode + dt.job.fileCacheConfig.ExperimentalEnableChunkCache = true + dt.job.fileCacheConfig.DownloadChunkSizeMb = 20 + + // Create the cache file + file, err := util.CreateFile(data.FileSpec{ + Path: dt.job.fileSpec.Path, + FilePerm: os.FileMode(0600), + DirPerm: os.FileMode(0700), + }, os.O_TRUNC|os.O_RDWR) + AssertEq(nil, err) + defer file.Close() + + // Set up sparse file info in cache + fileInfoKey := data.FileInfoKey{ + BucketName: dt.bucket.Name(), + ObjectName: objectName, + } + fileInfoKeyName, err := fileInfoKey.Key() + AssertEq(nil, err) + + chunkSizeBytes := uint64(20) * 1024 * 1024 + fileInfo := data.NewFileInfo(fileInfoKey, dt.object.Generation, uint64(objectSize), ^uint64(0), true, data.NewByteRangeMap(chunkSizeBytes, uint64(objectSize)), 1) + _, err = dt.cache.Insert(fileInfoKeyName, fileInfo) + AssertEq(nil, err) + + // Download a range [10MB, 30MB) + start := uint64(10 * util.MiB) + end := uint64(30 * util.MiB) + err = dt.job.downloadSparseRange(context.Background(), start, end) + AssertEq(nil, err) + + // Verify the content was written correctly + _, err = file.Seek(int64(start), 0) + AssertEq(nil, err) + buf := make([]byte, end-start) + _, err = file.Read(buf) + AssertEq(nil, err) + AssertTrue(reflect.DeepEqual(objectContent[start:end], buf), "Downloaded content doesn't match") + + // Verify the downloaded range was tracked in ByteRangeMap + updatedFileInfoVal := dt.cache.LookUpWithoutChangingOrder(fileInfoKeyName) + AssertTrue(updatedFileInfoVal != nil, "FileInfo should exist in cache") + updatedFileInfo := updatedFileInfoVal.(data.FileInfo) + AssertTrue(updatedFileInfo.DownloadedChunks.ContainsRange(start, end), "Downloaded range not tracked in ByteRangeMap") +} + +func (dt *sparseDownloaderTest) Test_HandleSparseRead_AlreadyDownloaded() { + objectName := "test/sparse_already_downloaded.txt" + objectSize := 50 * util.MiB + objectContent := testutil.GenerateRandomBytes(objectSize) + dt.initJobTest(objectName, objectContent, DefaultSequentialReadSizeMb, uint64(objectSize), func() {}) + + dt.job.fileCacheConfig.ExperimentalEnableChunkCache = true + dt.job.fileCacheConfig.DownloadChunkSizeMb = 20 + + // Set up sparse file info with pre-downloaded range + fileInfoKey := data.FileInfoKey{ + BucketName: dt.bucket.Name(), + ObjectName: objectName, + } + fileInfoKeyName, err := fileInfoKey.Key() + AssertEq(nil, err) + + chunkSizeBytes := uint64(20) * 1024 * 1024 + downloadedRanges := data.NewByteRangeMap(chunkSizeBytes, uint64(objectSize)) + downloadedRanges.AddRange(0, 40*util.MiB) // Mark first 40MB as downloaded + + fileInfo := data.NewFileInfo(fileInfoKey, dt.object.Generation, uint64(objectSize), ^uint64(0), true, downloadedRanges, 1) + _, err = dt.cache.Insert(fileInfoKeyName, fileInfo) + AssertEq(nil, err) + + // Request a range that's already downloaded [5MB, 10MB) + offset := int64(5 * util.MiB) + requiredOffset := int64(10 * util.MiB) + cacheHit, err := dt.job.HandleSparseRead(context.Background(), offset, requiredOffset) + + AssertEq(nil, err) + AssertTrue(cacheHit, "Should be a cache hit for already downloaded range") +} + +func (dt *sparseDownloaderTest) Test_HandleSparseRead_NeedsDownload() { + objectName := "test/sparse_needs_download.txt" + objectSize := 100 * util.MiB + objectContent := testutil.GenerateRandomBytes(objectSize) + dt.initJobTest(objectName, objectContent, DefaultSequentialReadSizeMb, uint64(objectSize), func() {}) + + dt.job.fileCacheConfig.ExperimentalEnableChunkCache = true + dt.job.fileCacheConfig.DownloadChunkSizeMb = 20 + + // Create the cache file + file, err := util.CreateFile(data.FileSpec{ + Path: dt.job.fileSpec.Path, + FilePerm: os.FileMode(0600), + DirPerm: os.FileMode(0700), + }, os.O_TRUNC|os.O_RDWR) + AssertEq(nil, err) + defer file.Close() + + // Set up sparse file info with empty downloaded ranges + fileInfoKey := data.FileInfoKey{ + BucketName: dt.bucket.Name(), + ObjectName: objectName, + } + fileInfoKeyName, err := fileInfoKey.Key() + AssertEq(nil, err) + + chunkSizeBytes := uint64(20) * 1024 * 1024 + fileInfo := data.NewFileInfo(fileInfoKey, dt.object.Generation, uint64(objectSize), ^uint64(0), true, data.NewByteRangeMap(chunkSizeBytes, uint64(objectSize)), 1) + _, err = dt.cache.Insert(fileInfoKeyName, fileInfo) + AssertEq(nil, err) + + // Request a range that needs to be downloaded [15MB, 25MB) + offset := int64(15 * util.MiB) + requiredOffset := int64(25 * util.MiB) + cacheHit, err := dt.job.HandleSparseRead(context.Background(), offset, requiredOffset) + + AssertEq(nil, err) + AssertTrue(cacheHit, "Should be a cache hit after successful download") + + // Verify the chunk was downloaded [0, 40MB) due to alignment + // offset 15MB rounds down to 0, requiredOffset 25MB rounds up to 40MB + updatedFileInfoVal := dt.cache.LookUpWithoutChangingOrder(fileInfoKeyName) + AssertTrue(updatedFileInfoVal != nil, "FileInfo should exist in cache") + updatedFileInfo := updatedFileInfoVal.(data.FileInfo) + AssertTrue(updatedFileInfo.DownloadedChunks.ContainsRange(0, 40*util.MiB), + "Expected range [0, 40MB) to be downloaded") + + // Verify the content + _, err = file.Seek(int64(offset), 0) + AssertEq(nil, err) + buf := make([]byte, requiredOffset-offset) + _, err = file.Read(buf) + AssertEq(nil, err) + AssertTrue(reflect.DeepEqual(objectContent[offset:requiredOffset], buf), + "Downloaded content doesn't match") +} diff --git a/internal/cache/file/downloader/test_util.go b/internal/cache/file/downloader/test_util.go index 1fbd201ed4..10d5804eec 100644 --- a/internal/cache/file/downloader/test_util.go +++ b/internal/cache/file/downloader/test_util.go @@ -18,10 +18,13 @@ import ( "context" "fmt" "os" + "reflect" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/data" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/data" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "github.com/stretchr/testify/assert" ) @@ -51,3 +54,29 @@ func verifyFileTillOffset(t *testing.T, spec data.FileSpec, offset int64, conten } } + +func verifyCompleteFile(t *testing.T, spec data.FileSpec, content []byte) { + fileStat, err := os.Stat(spec.Path) + assert.Equal(t, nil, err) + assert.Equal(t, spec.FilePerm, fileStat.Mode()) + assert.LessOrEqual(t, int64(len(content)), fileStat.Size()) + // Verify the content of file downloaded only till the size of content passed. + fileContent, err := os.ReadFile(spec.Path) + assert.Equal(t, nil, err) + assert.True(t, reflect.DeepEqual(content, fileContent[:len(content)])) +} + +func verifyFileInfoEntry(t *testing.T, mockBucket *storage.TestifyMockBucket, object gcs.MinObject, cache *lru.Cache, offset uint64) { + fileInfo := getFileInfo(t, mockBucket, object, cache) + assert.True(t, fileInfo != nil) + assert.Equal(t, object.Generation, fileInfo.(data.FileInfo).ObjectGeneration) + assert.LessOrEqual(t, offset, fileInfo.(data.FileInfo).Offset) + assert.Equal(t, object.Size, fileInfo.(data.FileInfo).ContentSize()) +} + +func getFileInfo(t *testing.T, mockBucket *storage.TestifyMockBucket, object gcs.MinObject, cache *lru.Cache) lru.ValueType { + fileInfoKey := data.FileInfoKey{BucketName: mockBucket.Name(), ObjectName: object.Name} + fileInfoKeyName, err := fileInfoKey.Key() + assert.Equal(t, nil, err) + return cache.LookUp(fileInfoKeyName) +} diff --git a/internal/cache/file/shared_chunk_cache_manager.go b/internal/cache/file/shared_chunk_cache_manager.go new file mode 100644 index 0000000000..42182e2f36 --- /dev/null +++ b/internal/cache/file/shared_chunk_cache_manager.go @@ -0,0 +1,181 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package file + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "math/rand/v2" + "os" + "path/filepath" + "regexp" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" +) + +// SharedChunkCacheManager manages a file cache that can be safely shared across +// multiple gcsfuse mount instances using lock-free atomic operations mkdir and +// rename. It organizes cache files in a directory structure based on the hash of +// bucket name, object name, and generation, and uses fixed-size chunk files for +// caching object data. It also supports regex-based inclusion/exclusion of files +// from caching. +type SharedChunkCacheManager struct { + // cacheDir is the local path which contains the cache data + cacheDir string + + // chunkSize is the size of each chunk for chunk-based caching + chunkSize int64 + + // filePerm parameter specifies the permission of file in cache + filePerm os.FileMode + + // dirPerm parameter specifies the permission of cache directory + dirPerm os.FileMode + + // excludeRegex is the compiled regex for excluding files from cache + excludeRegex *regexp.Regexp + + // includeRegex is the compiled regex for including files from cache + includeRegex *regexp.Regexp + + // config contains file cache configuration + config *cfg.FileCacheConfig +} + +// NewSharedChunkCacheManager creates a new shared chunk cache handler. +func NewSharedChunkCacheManager( + cacheDir string, + filePerm os.FileMode, + dirPerm os.FileMode, + config *cfg.FileCacheConfig, +) (*SharedChunkCacheManager, error) { + // Determine chunk size + chunkSize := config.SharedCacheChunkSizeMb * 1024 * 1024 + + // Compile regex patterns + var err error + var excludeRegex, includeRegex *regexp.Regexp + if config.ExcludeRegex != "" { + excludeRegex, err = regexp.Compile(config.ExcludeRegex) + if err != nil { + logger.Warnf("Failed to compile exclude regex %q: %v", config.ExcludeRegex, err) + } + } + if config.IncludeRegex != "" { + includeRegex, err = regexp.Compile(config.IncludeRegex) + if err != nil { + logger.Warnf("Failed to compile include regex %q: %v", config.IncludeRegex, err) + } + } + + handler := &SharedChunkCacheManager{ + cacheDir: cacheDir, + chunkSize: chunkSize, + filePerm: filePerm, + dirPerm: dirPerm, + excludeRegex: excludeRegex, + includeRegex: includeRegex, + config: config, + } + + return handler, nil +} + +// ShouldExcludeFromCache checks if the file should be excluded from caching. +func (sccm *SharedChunkCacheManager) ShouldExcludeFromCache(bucket gcs.Bucket, object *gcs.MinObject) bool { + objectPath := filepath.Join(bucket.Name(), object.Name) + + // If include regex is set, only include matching files + if sccm.includeRegex != nil { + if !sccm.includeRegex.MatchString(objectPath) { + return true + } + } + + // Exclude files matching exclude regex + if sccm.excludeRegex != nil { + if sccm.excludeRegex.MatchString(objectPath) { + return true + } + } + + return false +} + +// GenerateTmpPath generates a unique temporary file path in the object directory. +// The temporary file name includes a random prefix to avoid conflicts. +func (sccm *SharedChunkCacheManager) GenerateTmpPath(bucketName, objectName string, generation int64, chunkIndex int64) string { + // Generate random 8-character hex prefix + randomPrefix := fmt.Sprintf("%016x", rand.Uint64()) + chunkPath := sccm.GetChunkPath(bucketName, objectName, generation, chunkIndex) + return chunkPath + "." + randomPrefix + ".tmp" +} + +// GetFilePerm returns the file permission used by this handler. +func (sccm *SharedChunkCacheManager) GetFilePerm() os.FileMode { + return sccm.filePerm +} + +// GetDirPerm returns the directory permission used by this handler. +func (sccm *SharedChunkCacheManager) GetDirPerm() os.FileMode { + return sccm.dirPerm +} + +// GetChunkIndex calculates which chunk contains the given offset. +func (sccm *SharedChunkCacheManager) GetChunkIndex(offset int64) int64 { + return offset / sccm.chunkSize +} + +// GetChunkSize returns the chunk size used by this handler. +func (sccm *SharedChunkCacheManager) GetChunkSize() int64 { + return sccm.chunkSize +} + +// computeObjectHash computes SHA256 hash of bucketName, objectName, and generation. +func computeObjectHash(bucketName, objectName string, generation int64) string { + h := sha256.New() + + // Use length prefixes to prevent different (bucket, objectName) tuples from resulting in + // the same pre-hash string. This is defensive, even though bucket names currently cannot + // contain '/'. + // Format: <bucketNameLength>:<bucketName><objectNameLength>:<objectName>:<generation> + fmt.Fprintf(h, "%d:%s", len(bucketName), bucketName) + fmt.Fprintf(h, "%d:%s", len(objectName), objectName) + fmt.Fprintf(h, ":%d", generation) + + return hex.EncodeToString(h.Sum(nil)) +} + +// GetObjectDir returns the directory path for an object with generation encoded. +// Format: /cache/<prefix1>/<prefix2>/<full-sha256-hash>/ +// where prefix1 and prefix2 are the first four hex digits of the SHA256 hash (2 chars each) +func (sccm *SharedChunkCacheManager) GetObjectDir(bucketName, objectName string, generation int64) string { + hash := computeObjectHash(bucketName, objectName, generation) + prefix1 := hash[0:2] // First 2 characters + prefix2 := hash[2:4] // Next 2 characters + return filepath.Join(sccm.cacheDir, prefix1, prefix2, hash) +} + +// GetChunkPath returns the path to a specific chunk file. +// Format: /cache/<prefix1>/<prefix2>/<hash>/<start-offset>_<end-offset>.bin +func (sccm *SharedChunkCacheManager) GetChunkPath(bucketName, objectName string, generation int64, chunkIndex int64) string { + objDir := sccm.GetObjectDir(bucketName, objectName, generation) + startOffset := chunkIndex * sccm.chunkSize + endOffset := startOffset + sccm.chunkSize + return filepath.Join(objDir, fmt.Sprintf("%d_%d.bin", startOffset, endOffset)) +} diff --git a/internal/cache/file/shared_chunk_cache_manager_test.go b/internal/cache/file/shared_chunk_cache_manager_test.go new file mode 100644 index 0000000000..ca7dfbb48d --- /dev/null +++ b/internal/cache/file/shared_chunk_cache_manager_test.go @@ -0,0 +1,431 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package file + +import ( + "os" + "path/filepath" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/jacobsa/timeutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewSharedChunkCacheManager(t *testing.T) { + // Arrange + tmpDir := t.TempDir() + + // Act + manager, err := NewSharedChunkCacheManager( + tmpDir, + 0644, + 0755, + &cfg.FileCacheConfig{}, + ) + + // Assert + assert.NoError(t, err) + assert.NotNil(t, manager) + assert.NotEmpty(t, manager.cacheDir) + assert.Equal(t, int64(manager.config.SharedCacheChunkSizeMb*1024*1024), manager.chunkSize) +} + +func TestSharedChunkCacheManager_ShouldExcludeFromCache(t *testing.T) { + tests := []struct { + name string + includeRegex string + excludeRegex string + bucketName string + objectName string + wantExcluded bool + }{ + { + name: "No regex - should not exclude", + includeRegex: "", + excludeRegex: "", + bucketName: "test-bucket", + objectName: "file.txt", + wantExcluded: false, + }, + { + name: "Include regex matches - should not exclude", + includeRegex: ".*\\.txt$", + excludeRegex: "", + bucketName: "test-bucket", + objectName: "file.txt", + wantExcluded: false, + }, + { + name: "Include regex does not match - should exclude", + includeRegex: ".*\\.txt$", + excludeRegex: "", + bucketName: "test-bucket", + objectName: "file.log", + wantExcluded: true, + }, + { + name: "Exclude regex matches - should exclude", + includeRegex: "", + excludeRegex: ".*\\.log$", + bucketName: "test-bucket", + objectName: "file.log", + wantExcluded: true, + }, + { + name: "Exclude regex does not match - should not exclude", + includeRegex: "", + excludeRegex: ".*\\.log$", + bucketName: "test-bucket", + objectName: "file.txt", + wantExcluded: false, + }, + { + name: "Both regexes - include matches, exclude does not - should not exclude", + includeRegex: ".*\\.txt$", + excludeRegex: ".*temp.*", + bucketName: "test-bucket", + objectName: "file.txt", + wantExcluded: false, + }, + { + name: "Both regexes - include matches, exclude also matches - should exclude", + includeRegex: ".*\\.txt$", + excludeRegex: ".*temp.*", + bucketName: "test-bucket", + objectName: "temp.txt", + wantExcluded: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Arrange + tmpDir := t.TempDir() + config := &cfg.FileCacheConfig{ + IncludeRegex: tt.includeRegex, + ExcludeRegex: tt.excludeRegex, + } + manager, err := NewSharedChunkCacheManager(tmpDir, 0644, 0755, config) + require.NoError(t, err) + bucket := fake.NewFakeBucket(timeutil.RealClock(), tt.bucketName, gcs.BucketType{}) + object := &gcs.MinObject{Name: tt.objectName} + + // Act + result := manager.ShouldExcludeFromCache(bucket, object) + + // Assert + assert.Equal(t, tt.wantExcluded, result) + }) + } +} + +func TestSharedChunkCacheManager_GetChunkIndex(t *testing.T) { + // Arrange + tmpDir := t.TempDir() + config := &cfg.FileCacheConfig{ + SharedCacheChunkSizeMb: 8, // 8 MB chunks + } + manager, err := NewSharedChunkCacheManager(tmpDir, 0644, 0755, config) + require.NoError(t, err) + + tests := []struct { + name string + offset int64 + wantIndex int64 + }{ + {"First byte", 0, 0}, + {"Last byte of chunk 0", 8*1024*1024 - 1, 0}, + {"First byte of chunk 1", 8 * 1024 * 1024, 1}, + {"Middle of chunk 1", 10 * 1024 * 1024, 1}, + {"Chunk 2", 16 * 1024 * 1024, 2}, + {"Chunk 10", 80 * 1024 * 1024, 10}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Act + result := manager.GetChunkIndex(tt.offset) + + // Assert + assert.Equal(t, tt.wantIndex, result) + }) + } +} + +func TestSharedChunkCacheManager_GetChunkSize(t *testing.T) { + tests := []struct { + name string + chunkSizeMb int64 + expectedChunkSize int64 + }{ + { + name: "Default chunk size", + chunkSizeMb: 0, // 0 means use default + expectedChunkSize: 0, + }, + { + name: "Custom 16MB chunk size", + chunkSizeMb: 16, + expectedChunkSize: 16 * 1024 * 1024, + }, + { + name: "Custom 32MB chunk size", + chunkSizeMb: 32, + expectedChunkSize: 32 * 1024 * 1024, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Arrange + tmpDir := t.TempDir() + config := &cfg.FileCacheConfig{ + SharedCacheChunkSizeMb: tt.chunkSizeMb, + } + manager, err := NewSharedChunkCacheManager(tmpDir, 0644, 0755, config) + require.NoError(t, err) + + // Act + result := manager.GetChunkSize() + + // Assert + assert.Equal(t, tt.expectedChunkSize, result) + }) + } +} + +func TestSharedChunkCacheManager_GetObjectDir(t *testing.T) { + // Arrange + tmpDir := t.TempDir() + manager, err := NewSharedChunkCacheManager(tmpDir, 0644, 0755, &cfg.FileCacheConfig{}) + require.NoError(t, err) + tests := []struct { + name string + bucketName string + objectName string + generation int64 + expectedCacheDir string + }{ + { + name: "Simple object", + bucketName: "my-bucket", + objectName: "file.txt", + generation: 12345, + expectedCacheDir: "41/7d/417d6e4989a22cfa815f9e622a859475121dacee0793e846b03e089b9d837e6a", + }, + { + name: "Object with path", + bucketName: "my-bucket", + objectName: "dir/subdir/file.txt", + generation: 67890, + expectedCacheDir: "73/8e/738ef21e631dc30612f56ccc87eba8f76bd714ccc22238a2549c7a3177f44bfa", + }, + { + name: "Zero generation", + bucketName: "bucket", + objectName: "obj", + generation: 0, + expectedCacheDir: "51/e5/51e59a4c0ab165b7b8c93715fcba438c133f8e8a14d94452dc7b2ce7315ae321", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Arrange + expectedCacheDir := filepath.Join(tmpDir, tt.expectedCacheDir) + + // Act + result := manager.GetObjectDir(tt.bucketName, tt.objectName, tt.generation) + + // Assert + assert.Equal(t, expectedCacheDir, result) + }) + } +} + +func TestSharedChunkCacheManager_GetChunkPath(t *testing.T) { + // Arrange + tmpDir := t.TempDir() + manager, err := NewSharedChunkCacheManager(tmpDir, 0644, 0755, &cfg.FileCacheConfig{ + SharedCacheChunkSizeMb: 8, // 8 MB chunks + }) + require.NoError(t, err) + + tests := []struct { + name string + bucketName string + objectName string + generation int64 + chunkIndex int64 + expectedPath string + }{ + { + name: "Chunk 0", + bucketName: "my-bucket", + objectName: "file.txt", + generation: 12345, + chunkIndex: 0, + expectedPath: "41/7d/417d6e4989a22cfa815f9e622a859475121dacee0793e846b03e089b9d837e6a/0_8388608.bin", + }, + { + name: "Chunk 5", + bucketName: "my-bucket", + objectName: "dir/file.txt", + generation: 67890, + chunkIndex: 5, + expectedPath: "8e/0b/8e0bf1cb92e4496f8107137549a19d9c429fdea785e54258e529751c8cc98093/41943040_50331648.bin", + }, + { + name: "Large chunk index", + bucketName: "bucket", + objectName: "object", + generation: 1, + chunkIndex: 999, + expectedPath: "1a/8b/1a8b371ee4267a3bbed920ce28dc0e8796a7bfe39b17e3b312f4469c6c25a7b1/8380219392_8388608000.bin", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Arrange + expectedPath := filepath.Join(tmpDir, tt.expectedPath) + + // Act + result := manager.GetChunkPath(tt.bucketName, tt.objectName, tt.generation, tt.chunkIndex) + + // Assert + assert.Equal(t, expectedPath, result) + }) + } +} + +func TestSharedChunkCacheManager_GenerateTmpPath(t *testing.T) { + // Arrange + tmpDir := t.TempDir() + manager, err := NewSharedChunkCacheManager(tmpDir, 0644, 0755, &cfg.FileCacheConfig{ + SharedCacheChunkSizeMb: 8, // 8 MB chunks + }) + require.NoError(t, err) + + tests := []struct { + name string + bucketName string + objectName string + generation int64 + chunkIndex int64 + expectedBasePath string // Path without random prefix + }{ + { + name: "Chunk 0", + bucketName: "my-bucket", + objectName: "file.txt", + generation: 12345, + chunkIndex: 0, + expectedBasePath: "41/7d/417d6e4989a22cfa815f9e622a859475121dacee0793e846b03e089b9d837e6a/0_8388608.bin", + }, + { + name: "Chunk 5", + bucketName: "my-bucket", + objectName: "dir/file.txt", + generation: 67890, + chunkIndex: 5, + expectedBasePath: "8e/0b/8e0bf1cb92e4496f8107137549a19d9c429fdea785e54258e529751c8cc98093/41943040_50331648.bin", + }, + { + name: "Large chunk index", + bucketName: "bucket", + objectName: "object", + generation: 1, + chunkIndex: 999, + expectedBasePath: "1a/8b/1a8b371ee4267a3bbed920ce28dc0e8796a7bfe39b17e3b312f4469c6c25a7b1/8380219392_8388608000.bin", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Arrange + expectedBasePath := filepath.Join(tmpDir, tt.expectedBasePath) + + // Act + result := manager.GenerateTmpPath(tt.bucketName, tt.objectName, tt.generation, tt.chunkIndex) + + // Assert - Verify pattern: <base>.<16-hex-chars>.tmp + assert.Contains(t, result, expectedBasePath, "Tmp path should contain base chunk path") + assert.True(t, filepath.Ext(result) == ".tmp", "Tmp path should end with .tmp extension") + // Verify it has the format: <base>.<random>.tmp + // The random part should be 16 hex characters (8 bytes encoded) + assert.Regexp(t, `\.bin\.[0-9a-f]{16}\.tmp$`, result, "Tmp path should have random hex prefix before .tmp") + }) + } +} + +func TestSharedChunkCacheManager_GenerateTmpPath_NoCollisions(t *testing.T) { + // Arrange + tmpDir := t.TempDir() + manager, err := NewSharedChunkCacheManager(tmpDir, 0644, 0755, &cfg.FileCacheConfig{ + SharedCacheChunkSizeMb: 8, + }) + require.NoError(t, err) + bucketName := "test-bucket" + objectName := "test-object.txt" + generation := int64(12345) + chunkIndex := int64(0) + numPaths := 100 + + // Act - Generate multiple tmp paths for the same chunk + generatedPaths := make(map[string]bool) + for i := 0; i < numPaths; i++ { + path := manager.GenerateTmpPath(bucketName, objectName, generation, chunkIndex) + generatedPaths[path] = true + } + + // Assert - All paths should be unique (no collisions) + assert.Equal(t, numPaths, len(generatedPaths), "All generated tmp paths should be unique - no collisions") + // Assert - Verify all paths follow the expected pattern + for path := range generatedPaths { + assert.Regexp(t, `\.bin\.[0-9a-f]{16}\.tmp$`, path, "Each tmp path should have random hex prefix before .tmp") + } +} + +func TestSharedChunkCacheManager_GetFilePerm(t *testing.T) { + // Arrange + tmpDir := t.TempDir() + filePerm := os.FileMode(0600) + manager, err := NewSharedChunkCacheManager(tmpDir, filePerm, 0755, &cfg.FileCacheConfig{}) + require.NoError(t, err) + + // Act + result := manager.GetFilePerm() + + // Assert + assert.Equal(t, filePerm, result) +} + +func TestSharedChunkCacheManager_GetDirPerm(t *testing.T) { + // Arrange + tmpDir := t.TempDir() + dirPerm := os.FileMode(0700) + manager, err := NewSharedChunkCacheManager(tmpDir, 0644, dirPerm, &cfg.FileCacheConfig{}) + require.NoError(t, err) + + // Act + result := manager.GetDirPerm() + + // Assert + assert.Equal(t, dirPerm, result) +} diff --git a/internal/cache/lru/lru.go b/internal/cache/lru/lru.go index 611512a34f..10b0e5eb80 100644 --- a/internal/cache/lru/lru.go +++ b/internal/cache/lru/lru.go @@ -21,15 +21,15 @@ import ( "reflect" "strings" - "github.com/googlecloudplatform/gcsfuse/v2/internal/locker" + "github.com/googlecloudplatform/gcsfuse/v3/internal/locker" ) -// Predefined error messages returned by the Cache. -const ( - InvalidEntrySizeErrorMsg = "size of the entry is more than the cache's maxSize" - InvalidEntryErrorMsg = "nil values are not supported" - InvalidUpdateEntrySizeErrorMsg = "size of entry to be updated is not same as existing size" - EntryNotExistErrMsg = "entry with given key does not exist" +// Predefined errors returned by the Cache. +var ( + ErrInvalidEntrySize = errors.New("size of the entry is more than the cache's maxSize") + ErrInvalidEntry = errors.New("nil values are not supported") + ErrInvalidUpdateEntrySize = errors.New("size of entry to be updated is not same as existing size") + ErrEntryNotExist = errors.New("entry with given key does not exist") ) // Cache is a LRU cache for any lru.ValueType indexed by string keys. @@ -149,17 +149,17 @@ func (c *Cache) Insert( key string, value ValueType) ([]ValueType, error) { if value == nil { - return nil, errors.New(InvalidEntryErrorMsg) + return nil, ErrInvalidEntry } + c.mu.Lock() + defer c.mu.Unlock() + valueSize := value.Size() if valueSize > c.maxSize { - return nil, errors.New(InvalidEntrySizeErrorMsg) + return nil, ErrInvalidEntrySize } - c.mu.Lock() - defer c.mu.Unlock() - e, ok := c.index[key] if ok { // Update an entry if already exist. @@ -183,11 +183,10 @@ func (c *Cache) Insert( return evictedValues, nil } -// Erase any entry for the supplied key, also returns the value of erased key. -func (c *Cache) Erase(key string) (value ValueType) { - c.mu.Lock() - defer c.mu.Unlock() - +// eraseInternal removes any entry for the supplied key from the cache without acquiring locks. +// It returns the value of the erased key, or nil if not present. +// LOCKS_REQUIRED(c.mu) +func (c *Cache) eraseInternal(key string) (value ValueType) { e, ok := c.index[key] if !ok { return @@ -202,6 +201,24 @@ func (c *Cache) Erase(key string) (value ValueType) { return deletedEntry } +// eraseKeys removes a list of keys from the cache. +func (c *Cache) eraseKeys(keys []string) { + c.mu.Lock() + defer c.mu.Unlock() + + for _, key := range keys { + c.eraseInternal(key) + } +} + +// Erase any entry for the supplied key, also returns the value of erased key. +func (c *Cache) Erase(key string) (value ValueType) { + c.mu.Lock() + defer c.mu.Unlock() + + return c.eraseInternal(key) +} + // LookUp a previously-inserted value for the given key. Return nil if no // value is present. func (c *Cache) LookUp(key string) (value ValueType) { @@ -248,7 +265,7 @@ func (c *Cache) UpdateWithoutChangingOrder( key string, value ValueType) error { if value == nil { - return errors.New(InvalidEntryErrorMsg) + return ErrInvalidEntry } c.mu.Lock() @@ -256,11 +273,11 @@ func (c *Cache) UpdateWithoutChangingOrder( e, ok := c.index[key] if !ok { - return errors.New(EntryNotExistErrMsg) + return ErrEntryNotExist } if value.Size() != e.Value.(entry).Value.Size() { - return errors.New(InvalidUpdateEntrySizeErrorMsg) + return ErrInvalidUpdateEntrySize } e.Value = entry{key, value} @@ -269,10 +286,38 @@ func (c *Cache) UpdateWithoutChangingOrder( return nil } +// UpdateSize updates the currentSize accounting when an entry's size has changed. +// This is needed for entries whose size grows incrementally (e.g., sparse files). +// Eviction is deferred until the next Insert() call. +// The entry's order in the LRU is not changed. +func (c *Cache) UpdateSize(key string, sizeDelta uint64) error { + c.mu.Lock() + defer c.mu.Unlock() + + _, ok := c.index[key] + if !ok { + return ErrEntryNotExist + } + + // Update currentSize accounting + // Note: This may temporarily violate currentSize <= maxSize invariant + // Eviction will happen on the next Insert() call + c.currentSize += sizeDelta + + return nil +} + func (c *Cache) EraseEntriesWithGivenPrefix(prefix string) { + c.mu.RLock() + var keysToDelete []string for key := range c.index { if strings.HasPrefix(key, prefix) { - c.Erase(key) + keysToDelete = append(keysToDelete, key) } } + c.mu.RUnlock() + + if len(keysToDelete) > 0 { + c.eraseKeys(keysToDelete) + } } diff --git a/internal/cache/lru/lru_benchmark_test.go b/internal/cache/lru/lru_benchmark_test.go new file mode 100644 index 0000000000..eb89a4a0b2 --- /dev/null +++ b/internal/cache/lru/lru_benchmark_test.go @@ -0,0 +1,139 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lru_test + +import ( + "fmt" + "math/rand" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" +) + +func BenchmarkInsert(b *testing.B) { + cache := lru.NewCache(10000000) // 10MB + data := testData{Value: 1, DataSize: 10} + + b.ResetTimer() + for i := range b.N { + key := fmt.Sprintf("key-%d", i) + _, _ = cache.Insert(key, data) + } +} + +func BenchmarkLookUp(b *testing.B) { + cache := lru.NewCache(10000000) // 10MB + data := testData{Value: 1, DataSize: 10} + + // Pre-populate + for i := range 10000 { + key := fmt.Sprintf("key-%d", i) + _, _ = cache.Insert(key, data) + } + + b.ResetTimer() + for i := range b.N { + key := fmt.Sprintf("key-%d", i%10000) + _ = cache.LookUp(key) + } +} + +func BenchmarkErase(b *testing.B) { + cache := lru.NewCache(10000000) // 10MB + data := testData{Value: 1, DataSize: 10} + + b.ResetTimer() + for i := range b.N { + b.StopTimer() + key := fmt.Sprintf("key-%d", i) + _, _ = cache.Insert(key, data) + b.StartTimer() + + _ = cache.Erase(key) + } +} + +func BenchmarkConcurrency(b *testing.B) { + cache := lru.NewCache(50000000) // 50MB + data := testData{Value: 1, DataSize: 10} + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + for pb.Next() { + op := r.Intn(100) + key := fmt.Sprintf("key-%d", r.Intn(10000)) + if op < 30 { + // 30% inserts + _, _ = cache.Insert(key, data) + } else if op < 90 { + // 60% lookups + _ = cache.LookUp(key) + } else { + // 10% erases + _ = cache.Erase(key) + } + } + }) +} + +func BenchmarkInsert1Million(b *testing.B) { + const numEntries = 1000000 + data := testData{Value: 1, DataSize: 10} + cacheMaxSize := uint64(numEntries * 20) // ensure enough size so no evictions + + for range b.N { + b.StopTimer() + cache := lru.NewCache(cacheMaxSize) + + // Insert 1 million entries + for j := range numEntries { + key := fmt.Sprintf("prefix/key-%d", j) + _, _ = cache.Insert(key, data) + } + + b.StartTimer() + + } +} + +func BenchmarkEraseEntriesWithGivenPrefix_1Million(b *testing.B) { + const numEntries = 1000000 + data := testData{Value: 1, DataSize: 10} + cacheMaxSize := uint64(numEntries * 20) // ensure enough size so no evictions + + for range b.N { + b.StopTimer() + cache := lru.NewCache(cacheMaxSize) + + // Insert 1 million entries + for j := range numEntries { + var key string + // Add a specific prefix to half the keys + if j%2 == 0 { + key = fmt.Sprintf("prefix/key-%d", j) + } else { + key = fmt.Sprintf("other/key-%d", j) + } + _, _ = cache.Insert(key, data) + } + + b.StartTimer() + + // Delete entries with "prefix/" + cache.EraseEntriesWithGivenPrefix("prefix/") + } +} diff --git a/internal/cache/lru/lru_test.go b/internal/cache/lru/lru_test.go index 80128f3613..45b6b41e08 100644 --- a/internal/cache/lru/lru_test.go +++ b/internal/cache/lru/lru_test.go @@ -16,13 +16,13 @@ package lru_test import ( "errors" + "fmt" "math/rand" - "strings" "sync" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/lru" - "github.com/googlecloudplatform/gcsfuse/v2/internal/locker" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/locker" . "github.com/jacobsa/ogletest" ) @@ -81,7 +81,7 @@ func (t *CacheTest) LookUpInEmptyCache() { } func (t *CacheTest) InsertNilValue() { - t.insertAndAssert("taco", nil, []int64{}, errors.New(lru.InvalidEntryErrorMsg)) + t.insertAndAssert("taco", nil, []int64{}, lru.ErrInvalidEntry) } func (t *CacheTest) LookUpUnknownKey() { @@ -155,7 +155,7 @@ func (t *CacheTest) TestWhenEntrySizeMoreThanCacheMaxSize() { t.insertAndAssert("burrito", testData{Value: 23, DataSize: 4}, []int64{}, nil) // Insert entry with size greater than maxSize of cache. - t.insertAndAssert("taco", testData{Value: 26, DataSize: MaxSize + 1}, []int64{}, errors.New(lru.InvalidEntrySizeErrorMsg)) + t.insertAndAssert("taco", testData{Value: 26, DataSize: MaxSize + 1}, []int64{}, lru.ErrInvalidEntrySize) ExpectEq(23, t.cache.LookUp("burrito").(testData).Value) } @@ -242,7 +242,7 @@ func (t *CacheTest) TestUpdateWhenKeyNotPresent() { err := t.cache.UpdateWithoutChangingOrder(key, data) ExpectNe(nil, err) - ExpectTrue(strings.Contains(err.Error(), lru.EntryNotExistErrMsg)) + ExpectTrue(errors.Is(err, lru.ErrEntryNotExist)) } func (t *CacheTest) TestUpdateWhenSizeIsDifferent() { @@ -254,7 +254,7 @@ func (t *CacheTest) TestUpdateWhenSizeIsDifferent() { err := t.cache.UpdateWithoutChangingOrder(key, newData) ExpectNe(nil, err) - ExpectTrue(strings.Contains(err.Error(), lru.InvalidUpdateEntrySizeErrorMsg)) + ExpectTrue(errors.Is(err, lru.ErrInvalidUpdateEntrySize)) } func (t *CacheTest) TestUpdateNotChangeOrder() { @@ -320,7 +320,7 @@ func (t *CacheTest) TestRaceCondition() { go func() { defer wg.Done() - for i := 0; i < OperationCount; i++ { + for i := range OperationCount { _, err := t.cache.Insert("key", testData{ Value: int64(i), DataSize: uint64(rand.Intn(MaxSize)), @@ -332,28 +332,28 @@ func (t *CacheTest) TestRaceCondition() { go func() { defer wg.Done() - for i := 0; i < OperationCount; i++ { + for range OperationCount { t.cache.Erase("key") } }() go func() { defer wg.Done() - for i := 0; i < OperationCount; i++ { + for range OperationCount { t.cache.LookUp("key") } }() go func() { defer wg.Done() - for i := 0; i < OperationCount; i++ { + for range OperationCount { t.cache.LookUpWithoutChangingOrder("key") } }() go func() { defer wg.Done() - for i := 0; i < OperationCount; i++ { + for i := range OperationCount { _ = t.cache.UpdateWithoutChangingOrder("key", testData{ Value: int64(i), DataSize: uint64(rand.Intn(MaxSize)), @@ -363,3 +363,29 @@ func (t *CacheTest) TestRaceCondition() { wg.Wait() } + +func (t *CacheTest) Test_EraseEntriesWithGivenPrefix_Concurrent() { + c := lru.NewCache(100000) + + // Pre-fill the cache + for i := range 1000 { + _, _ = c.Insert(fmt.Sprintf("dir1/file%d", i), testData{10, 10}) + } + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + for i := 1000; i < 2000; i++ { + _, _ = c.Insert(fmt.Sprintf("dir2/file%d", i), testData{10, 10}) + } + }() + + go func() { + defer wg.Done() + c.EraseEntriesWithGivenPrefix("dir1/") + }() + + wg.Wait() +} diff --git a/internal/cache/metadata/stat_cache.go b/internal/cache/metadata/stat_cache.go index 96c88da9ce..d0e056c284 100644 --- a/internal/cache/metadata/stat_cache.go +++ b/internal/cache/metadata/stat_cache.go @@ -18,9 +18,10 @@ import ( "math" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/lru" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" ) // A cache mapping from name to most recent known record for the object of that @@ -37,6 +38,8 @@ type StatCache interface { // The entry will expire after the supplied time. Insert(m *gcs.MinObject, expiration time.Time) + InsertImplicitDir(objectName string, expiration time.Time) + // Set up a negative entry for the given name, indicating that the name // doesn't exist. Overwrite any existing entry for the name, positive or // negative. @@ -111,33 +114,35 @@ type entry struct { m *gcs.MinObject f *gcs.Folder expiration time.Time - key string + // Set to true only for implicit directory entries. This flag will always remain false for negative entries and explicit objects. + implicitDir bool } -// Size returns the memory-size (resident set size) of the receiver entry. -// The size calculated by the unsafe.Sizeof calls, and -// NestedSizeOfGcsMinObject etc. does not account for -// hidden members in data structures like maps, slices, linked-lists etc. -// To account for those, we are adding a fixed constant of 515 bytes (deduced from -// benchmark runs) to heap-size per positive stat-cache entry -// to calculate a size closer to the actual memory utilization. -func (e entry) Size() (size uint64) { - // First, calculate size on heap (including folder size also in case of hns buckets, in case of non-hns buckets 0 will be added as e.f will be Nil ). - // Additional 2*util.UnsafeSizeOf(&e.key) is to account for the copies of string - // struct stored in the cache map and in the cache linked-list. - size = uint64(util.UnsafeSizeOf(&e) + len(e.key) + 2*util.UnsafeSizeOf(&e.key) + util.NestedSizeOfGcsMinObject(e.m)) +// Size returns the approximate memory-size (resident set size) of the receiver entry. +// It estimates the memory consumption on the heap and converts it to an estimated RSS: +// 1. util.UnsafeSizeOf(&e): The base size of the entry struct itself. +// 2. util.NestedSizeOfGcsMinObject(e.m): The deep size of the gcs.MinObject, if present. +// 3. util.NestedSizeOfGcsFolder(e.f): The deep size of the gcs.Folder, if present. +// 4. A fixed 515-byte constant is added for positive MinObject entries. Because +// unsafe.Sizeof and NestedSizeOfGcsMinObject do not account for hidden allocations +// in nested structures (like maps, slices, linked-lists etc.), this constant +// was deduced from benchmark runs to approximate actual memory utilization. +// 5. The final heap size is multiplied by util.HeapSizeToRssConversionFactor to +// estimate the Resident Set Size (RSS). +func (e entry) Size() uint64 { + size := uint64(util.UnsafeSizeOf(&e) + util.NestedSizeOfGcsMinObject(e.m)) if e.m != nil { size += 515 } if e.f != nil { - size += uint64(util.UnsafeSizeOf(&e.f)) + size += uint64(util.NestedSizeOfGcsFolder(e.f)) } // Convert heap-size to RSS (resident set size). size = uint64(math.Ceil(util.HeapSizeToRssConversionFactor * float64(size))) - return + return size } // Should the supplied object for a new positive entry replace the given @@ -187,7 +192,6 @@ func (sc *statCacheBucketView) Insert(m *gcs.MinObject, expiration time.Time) { e := entry{ m: m, expiration: expiration, - key: name, } if _, err := sc.sharedCache.Insert(name, e); err != nil { @@ -195,6 +199,42 @@ func (sc *statCacheBucketView) Insert(m *gcs.MinObject, expiration time.Time) { } } +func (sc *statCacheBucketView) InsertImplicitDir(objectName string, expiration time.Time) { + name := sc.key(objectName) + + // Is there already a better entry? + if existing := sc.sharedCache.LookUp(name); existing != nil { + e := existing.(entry) + // The ListObjects response handles directories in two ways: + // 1. 'MinObject' returns explicit directory objects containing full metadata. + // 2. 'CollapseRun' generates placeholders for these same directories; if no + // explicit object exists, it treats them as "implicit" (inferred). + // + // We attempt to create implicit directories for all entries in 'CollapseRun'. + // However, since 'ListObject' returns explicit directories in the 'MinObject' + // list as well, this could result in redundant implicit entries for + // every explicit directory already processed. + // + // To prevent this, we check if an entry with the same name already exists + // with non-nil metadata. If metadata is present, we skip the implicit + // creation to avoid overwriting a real, explicit object with an inferred + // placeholder (which would lack metadata and have 'Generation 0'). + if e.m != nil { + return + } + } + + // Insert an entry. + e := entry{ + implicitDir: true, + expiration: expiration, + } + + if _, err := sc.sharedCache.Insert(name, e); err != nil { + logger.Errorf("Failed to insert implicit dir stat cache entry for %q: %v", name, err) + } +} + func (sc *statCacheBucketView) AddNegativeEntry(objectName string, expiration time.Time) { name := sc.key(objectName) @@ -202,7 +242,6 @@ func (sc *statCacheBucketView) AddNegativeEntry(objectName string, expiration ti e := entry{ m: nil, expiration: expiration, - key: name, } if _, err := sc.sharedCache.Insert(name, e); err != nil { @@ -217,7 +256,6 @@ func (sc *statCacheBucketView) AddNegativeEntryForFolder(folderName string, expi e := entry{ f: nil, expiration: expiration, - key: name, } if _, err := sc.sharedCache.Insert(name, e); err != nil { @@ -236,6 +274,9 @@ func (sc *statCacheBucketView) LookUp( // Look up in the LRU cache. hit, entry := sc.sharedCacheLookup(objectName, now) if hit { + if entry.implicitDir { + return true, &gcs.MinObject{Name: objectName} + } return hit, entry.m } @@ -278,7 +319,6 @@ func (sc *statCacheBucketView) InsertFolder(f *gcs.Folder, expiration time.Time) e := entry{ f: f, expiration: expiration, - key: name, } if _, err := sc.sharedCache.Insert(name, e); err != nil { diff --git a/internal/cache/metadata/stat_cache_test.go b/internal/cache/metadata/stat_cache_test.go index d96fa94098..45300f4120 100644 --- a/internal/cache/metadata/stat_cache_test.go +++ b/internal/cache/metadata/stat_cache_test.go @@ -18,10 +18,10 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/lru" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/metadata" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) @@ -45,6 +45,12 @@ func (c *testHelperCache) Insert( c.wrapped.Insert(m, expiration) } +func (c *testHelperCache) InsertImplicitDir( + name string, + expiration time.Time) { + c.wrapped.InsertImplicitDir(name, expiration) +} + func (c *testHelperCache) AddNegativeEntry( name string, expiration time.Time) { @@ -517,15 +523,15 @@ func (t *StatCacheTest) Test_ShouldReturnHitTrueWhenOnlyObjectAlreadyHasEntry() } func (t *StatCacheTest) Test_ShouldEvictEntryOnFullCapacityIncludingFolderSize() { - localCache := lru.NewCache(uint64(3000)) + localCache := lru.NewCache(uint64(2800)) t.statCache = metadata.NewStatCacheBucketView(localCache, "local_bucket") objectEntry1 := &gcs.MinObject{Name: "1"} objectEntry2 := &gcs.MinObject{Name: "2"} folderEntry := &gcs.Folder{ Name: "3/", } - t.statCache.Insert(objectEntry1, expiration) // adds size of 1428 - t.statCache.Insert(objectEntry2, expiration) // adds size of 1428 + t.statCache.Insert(objectEntry1, expiration) // adds size of 1368 + t.statCache.Insert(objectEntry2, expiration) // adds size of 1368 hit1, entry1 := t.statCache.LookUp("1", someTime) hit2, entry2 := t.statCache.LookUp("2", someTime) @@ -535,7 +541,7 @@ func (t *StatCacheTest) Test_ShouldEvictEntryOnFullCapacityIncludingFolderSize() assert.True(t.T(), hit2) assert.Equal(t.T(), "2", entry2.Name) - t.statCache.InsertFolder(folderEntry, expiration) //adds size of 220 and exceeds capacity + t.statCache.InsertFolder(folderEntry, expiration) //adds size of 180 and exceeds capacity hit1, entry1 = t.statCache.LookUp("1", someTime) hit2, entry2 = t.statCache.LookUp("2", someTime) @@ -592,3 +598,57 @@ func (t *StatCacheTest) Test_ShouldEvictAllEntriesWithPrefixFolder() { assert.True(t.T(), hit6) assert.Equal(t.T(), "d", entry6.Name) } + +func (t *StatCacheTest) Test_InsertImplicitDir() { + const name = "dir/" + t.cache.InsertImplicitDir(name, expiration) + + m := t.cache.LookUpOrNil(name, someTime) + + assert.NotNil(t.T(), m) + assert.Equal(t.T(), name, m.Name) + assert.Equal(t.T(), int64(0), m.Generation) +} + +func (t *StatCacheTest) Test_ImplicitDirSizeEfficiency() { + // Standard entry size ~1640 bytes (according to Test_FillUpToCapacity comments). + // Implicit entry size should be much smaller (around 100-200 bytes). + // So we should be able to store many more implicit entries than explicit ones. + // capacity is 3 in SetupTest. Max size = 3 * (AvgPos + AvgNeg) ~= 5000 bytes. + // 1. Fill with implicit dirs + // Insert 20 implicit dirs. They should all fit if size is small. + for i := 0; i < 20; i++ { + name := string(rune('a'+i)) + "/" + t.cache.InsertImplicitDir(name, expiration) + } + + // Verify all are present + for i := 0; i < 20; i++ { + name := string(rune('a'+i)) + "/" + assert.True(t.T(), t.cache.Hit(name, someTime)) + } +} + +func (t *StatCacheTest) Test_InsertImplicitDir_DoesNotOverwriteExplicit() { + const name = "dir/" + m := &gcs.MinObject{Name: name, Generation: 1} + t.statCache.Insert(m, expiration) + + t.statCache.InsertImplicitDir(name, expiration) + + hit, result := t.statCache.LookUp(name, someTime) + assert.True(t.T(), hit) + assert.Equal(t.T(), m, result) +} + +func (t *StatCacheTest) Test_Insert_OverwritesImplicitDir() { + const name = "dir/" + t.statCache.InsertImplicitDir(name, expiration) + m := &gcs.MinObject{Name: name, Generation: 1} + + t.statCache.Insert(m, expiration) + + hit, result := t.statCache.LookUp(name, someTime) + assert.True(t.T(), hit) + assert.Equal(t.T(), m, result) +} diff --git a/internal/cache/metadata/type_cache.go b/internal/cache/metadata/type_cache.go index 35e34b3a82..ce9f4946e7 100644 --- a/internal/cache/metadata/type_cache.go +++ b/internal/cache/metadata/type_cache.go @@ -19,8 +19,8 @@ import ( "math" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/lru" - "github.com/googlecloudplatform/gcsfuse/v2/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" ) type Type int diff --git a/internal/cache/metadata/type_cache_test.go b/internal/cache/metadata/type_cache_test.go index c7268c836f..166ea19cd0 100644 --- a/internal/cache/metadata/type_cache_test.go +++ b/internal/cache/metadata/type_cache_test.go @@ -19,7 +19,7 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" . "github.com/jacobsa/ogletest" ) diff --git a/internal/cache/util/util.go b/internal/cache/util/util.go index f1784a8933..1a9d70fb72 100644 --- a/internal/cache/util/util.go +++ b/internal/cache/util/util.go @@ -23,32 +23,33 @@ import ( "os" "path" "path/filepath" - "strings" "unsafe" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/data" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/data" "github.com/jacobsa/fuse/fsutil" ) -const ( - InvalidFileHandleErrMsg = "invalid file handle" - InvalidFileDownloadJobErrMsg = "invalid download job" - InvalidFileInfoCacheErrMsg = "invalid file info cache" - ErrInSeekingFileHandleMsg = "error while seeking file handle" - ErrInReadingFileHandleMsg = "error while reading file handle" - FallbackToGCSErrMsg = "read via gcs" - FileNotPresentInCacheErrMsg = "file is not present in cache" - CacheHandleNotRequiredForRandomReadErrMsg = "cacheFileForRangeRead is false, read type random read and fileInfo entry is absent" +var ( + ErrInvalidFileHandle = errors.New("invalid file handle") + ErrInvalidFileDownloadJob = errors.New("invalid download job") + ErrInvalidFileInfoCache = errors.New("invalid file info cache") + ErrInReadingFileHandle = errors.New("error while reading file handle") + ErrFallbackToGCS = errors.New("read via gcs") + ErrFileNotPresentInCache = errors.New("file is not present in cache") + ErrCacheHandleNotRequiredForRandomRead = errors.New("cacheFileForRangeRead is false, read type random read and fileInfo entry is absent") + ErrFileExcludedFromCacheByRegex = errors.New("file excluded from cache by regex") + ErrShortRead = errors.New("short read") ) const ( - MiB = 1024 * 1024 - KiB = 1024 - DefaultFilePerm = os.FileMode(0600) - DefaultDirPerm = os.FileMode(0700) - FileCache = "gcsfuse-file-cache" - BufferSizeForCRC = 65536 - MinimumAlignSizeForWriting = 4096 + MiB = 1024 * 1024 + KiB = 1024 + DefaultFilePerm = os.FileMode(0600) + DefaultDirPerm = os.FileMode(0700) + FileCache = "gcsfuse-file-cache" + SharedChunkCache = "gcsfuse-shared-chunk-cache" + BufferSizeForCRC = 65536 ) // CreateFile creates file with given file spec i.e. permissions and returns @@ -99,11 +100,10 @@ func GetDownloadPath(cacheDir string, objectPath string) string { // If it's invalid then we should close that cacheHandle and create new cacheHandle // for next call onwards. func IsCacheHandleInvalid(readErr error) bool { - return strings.Contains(readErr.Error(), InvalidFileHandleErrMsg) || - strings.Contains(readErr.Error(), InvalidFileDownloadJobErrMsg) || - strings.Contains(readErr.Error(), InvalidFileInfoCacheErrMsg) || - strings.Contains(readErr.Error(), ErrInSeekingFileHandleMsg) || - strings.Contains(readErr.Error(), ErrInReadingFileHandleMsg) + return errors.Is(readErr, ErrInvalidFileHandle) || + errors.Is(readErr, ErrInvalidFileDownloadJob) || + errors.Is(readErr, ErrInvalidFileInfoCache) || + errors.Is(readErr, ErrInReadingFileHandle) } // CreateCacheDirectoryIfNotPresentAt Creates directory at given path with @@ -210,7 +210,7 @@ func GetMemoryAlignedBuffer(bufferSize int64, alignSize int64) (buffer []byte, e // Though we haven't seen any error while aligning buffer but still it is safer // to attempt few times in case alignment fails. - for try := 0; try < 3; try++ { + for range 3 { buffer, err = createAndAlignBuffer() if err == nil { return buffer, err @@ -221,13 +221,13 @@ func GetMemoryAlignedBuffer(bufferSize int64, alignSize int64) (buffer []byte, e // CopyUsingMemoryAlignedBuffer copies content from src reader to dst writer // by staging content into a memory aligned buffer of size bufferSize and -// aligned to multiple of MinimumAlignSizeForWriting. Note: The minimum write -// size is MinimumAlignSizeForWriting which means the total size of content -// written to dst writer is always in multiple of MinimumAlignSizeForWriting. -// If contentSize is lesser than MinimumAlignSizeForWriting then extra null data +// aligned to multiple of cfg.CacheUtilMinimumAlignSizeForWriting. Note: The minimum write +// size is cfg.CacheUtilMinimumAlignSizeForWriting which means the total size of content +// written to dst writer is always in multiple of cfg.CacheUtilMinimumAlignSizeForWriting. +// If contentSize is lesser than cfg.CacheUtilMinimumAlignSizeForWriting then extra null data // is written at the last. func CopyUsingMemoryAlignedBuffer(ctx context.Context, src io.Reader, dst io.Writer, contentSize, bufferSize int64) (n int64, err error) { - var alignSize int64 = MinimumAlignSizeForWriting + var alignSize int64 = cfg.CacheUtilMinimumAlignSizeForWriting if bufferSize < alignSize || ((bufferSize % alignSize) != 0) { return 0, fmt.Errorf("buffer size (%v) should be a multiple of %v", bufferSize, alignSize) } diff --git a/internal/cache/util/util_test.go b/internal/cache/util/util_test.go index 6864c09416..00761aec30 100644 --- a/internal/cache/util/util_test.go +++ b/internal/cache/util/util_test.go @@ -27,9 +27,9 @@ import ( "testing" "unsafe" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/data" - testutil "github.com/googlecloudplatform/gcsfuse/v2/internal/util" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/data" + testutil "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" . "github.com/jacobsa/ogletest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -206,7 +206,7 @@ func (ut *utilTest) Test_getObjectPath() { expectedOutPuts := [5]string{"", "a/b", "a/b/c/d", "a", "a"} results := [5]string{} - for i := 0; i < 5; i++ { + for i := range 5 { results[i] = GetDownloadPath(inputs[i][0], inputs[i][1]) } @@ -220,7 +220,7 @@ func (ut *utilTest) Test_getDownloadPath() { cacheDir + "/a/b/c/d", cacheDir + "/a", cacheDir + "/a"} results := [5]string{} - for i := 0; i < 5; i++ { + for i := range 5 { results[i] = GetDownloadPath(cacheDir, inputs[i]) } @@ -228,27 +228,26 @@ func (ut *utilTest) Test_getDownloadPath() { } func (ut *utilTest) Test_IsCacheHandleValid_True() { - errMessages := []string{ - InvalidFileHandleErrMsg + "test", - InvalidFileDownloadJobErrMsg + "test", - InvalidFileInfoCacheErrMsg + "test", - ErrInSeekingFileHandleMsg + "test", - ErrInReadingFileHandleMsg + "test", + errs := []error{ + fmt.Errorf("%w: %s", ErrInvalidFileHandle, "test"), + fmt.Errorf("%w: %s", ErrInvalidFileDownloadJob, "test"), + fmt.Errorf("%w: %s", ErrInvalidFileInfoCache, "test"), + fmt.Errorf("%w: %s", ErrInReadingFileHandle, "test"), } - for _, errMsg := range errMessages { - ExpectTrue(IsCacheHandleInvalid(errors.New(errMsg))) + for _, err := range errs { + ExpectTrue(IsCacheHandleInvalid(err)) } } func (ut *utilTest) Test_IsCacheHandleValid_False() { - errMessages := []string{ - FallbackToGCSErrMsg + "test", - "random error message", + errs := []error{ + fmt.Errorf("%w: %s", ErrFallbackToGCS, "test"), + fmt.Errorf("random error message"), } - for _, errMsg := range errMessages { - ExpectFalse(IsCacheHandleInvalid(errors.New(errMsg))) + for _, err := range errs { + ExpectFalse(IsCacheHandleInvalid(err)) } } diff --git a/internal/canned/canned.go b/internal/canned/canned.go index ef6a48e1ea..663ecffdfa 100644 --- a/internal/canned/canned.go +++ b/internal/canned/canned.go @@ -21,8 +21,8 @@ import ( "log" "strings" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/fake" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "golang.org/x/net/context" "github.com/jacobsa/timeutil" @@ -57,7 +57,7 @@ const ( // Create a fake bucket with canned contents as described in the comments for // FakeBucketName. func MakeFakeBucket(ctx context.Context) (b gcs.Bucket) { - b = fake.NewFakeBucket(timeutil.RealClock(), FakeBucketName, gcs.NonHierarchical) + b = fake.NewFakeBucket(timeutil.RealClock(), FakeBucketName, gcs.BucketType{}) // Set up contents. contents := map[string]string{ diff --git a/internal/clock/clock.go b/internal/clock/clock.go new file mode 100644 index 0000000000..521ef882ca --- /dev/null +++ b/internal/clock/clock.go @@ -0,0 +1,23 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clock + +import "time" + +// Interface to provide regular clock functionalities. +// Creating an interface so that a fake can be created for unit tests. +type Clock interface { + After(d time.Duration) <-chan time.Time +} diff --git a/internal/clock/fake_clock.go b/internal/clock/fake_clock.go new file mode 100644 index 0000000000..16338af941 --- /dev/null +++ b/internal/clock/fake_clock.go @@ -0,0 +1,33 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clock + +import "time" + +// Implements clock interface. It should be used during tests to mimic waiting. +type FakeClock struct { + WaitTime time.Duration +} + +// Notifies on the returned channel after the wait time specified during +// creation of FakeClock. +func (mc *FakeClock) After(time.Duration) <-chan time.Time { + ch := make(chan time.Time) + go func() { + time.Sleep(mc.WaitTime) + ch <- time.Now() + }() + return ch +} diff --git a/internal/clock/real_clock.go b/internal/clock/real_clock.go new file mode 100644 index 0000000000..68b4845736 --- /dev/null +++ b/internal/clock/real_clock.go @@ -0,0 +1,25 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clock + +import "time" + +// Implements Clock interface. +type RealClock struct{} + +// Notifies on the return channel after the specified time has passed. +func (RealClock) After(d time.Duration) <-chan time.Time { + return time.After(d) +} diff --git a/internal/clock/simulated_clock.go b/internal/clock/simulated_clock.go new file mode 100644 index 0000000000..d279396632 --- /dev/null +++ b/internal/clock/simulated_clock.go @@ -0,0 +1,115 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clock + +import ( + "sync" + "time" +) + +// afterRequest holds the information for a pending After call in SimulatedClock. +type afterRequest struct { + targetTime time.Time + ch chan time.Time +} + +// SimulatedClock is a clock that allows for manipulation of the time, +// which does not change unless AdvanceTime or SetTime is called. +// The zero value is a clock initialized to the zero time. +type SimulatedClock struct { + mu sync.RWMutex + t time.Time // GUARDED_BY(mu) + pending []*afterRequest // GUARDED_BY(mu) +} + +func NewSimulatedClock(startTime time.Time) *SimulatedClock { + return &SimulatedClock{ + t: startTime, + pending: nil, + } +} + +func (sc *SimulatedClock) Now() time.Time { + sc.mu.RLock() + defer sc.mu.RUnlock() + + return sc.t +} + +// SetTime sets the current time according to the clock. +// It also processes any pending After calls that should fire. +func (sc *SimulatedClock) SetTime(t time.Time) { + sc.mu.Lock() + defer sc.mu.Unlock() + + sc.t = t + sc.processPending() +} + +// AdvanceTime advances the current time according to the clock by the supplied duration. +// It also processes any pending After calls that should fire. +func (sc *SimulatedClock) AdvanceTime(d time.Duration) { + sc.mu.Lock() + defer sc.mu.Unlock() + + sc.t = sc.t.Add(d) + sc.processPending() +} + +// After returns a time channel and the expectedFired time is sent over the returned +// channel after the provided duration. +// If the duration is zero or negative, it sends the current simulated time immediately. +func (sc *SimulatedClock) After(d time.Duration) <-chan time.Time { + sc.mu.Lock() + defer sc.mu.Unlock() + + ch := make(chan time.Time, 1) + effectiveTargetTime := sc.t.Add(d) + + // If duration is non-positive, fire immediately with the current time. + // Or if the target time is not after the current time (e.g. d <= 0) + if !effectiveTargetTime.After(sc.t) { + ch <- sc.t // Send current time as per time.After behavior for d <= 0 + return ch + } + + // Otherwise, schedule it. + ar := &afterRequest{ + targetTime: effectiveTargetTime, + ch: ch, + } + sc.pending = append(sc.pending, ar) + + return ch +} + +// processPending checks all pending After requests and fires those +// whose target time has been reached or passed by the current simulated time. +// This method must be called with sc.mu held. +func (sc *SimulatedClock) processPending() { + var stillPending []*afterRequest + + for _, ar := range sc.pending { + // If current time sc.t is not before the targetTime (i.e., sc.t >= ar.targetTime) + if !sc.t.Before(ar.targetTime) { + ar.ch <- ar.targetTime // Send the time it was scheduled to fire + // Do not close the channel, to mimic time.After behavior + } else { + stillPending = append(stillPending, ar) + } + } + + sc.pending = stillPending +} diff --git a/internal/clock/simulated_clock_test.go b/internal/clock/simulated_clock_test.go new file mode 100644 index 0000000000..b3fe540349 --- /dev/null +++ b/internal/clock/simulated_clock_test.go @@ -0,0 +1,287 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clock + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + // A non-zero reference time for tests + shortTestTimeout = 10 * time.Millisecond // For non-blocking channel checks + fireTestTimeout = 50 * time.Millisecond // When expecting a channel to fire +) + +func TestSimulatedClock_Now(t *testing.T) { + testCases := []struct { + name string + initialTimeSetup func(sc *SimulatedClock) + expectedTime time.Time + }{ + { + name: "InitialState_IsZeroTime", + initialTimeSetup: func(sc *SimulatedClock) {}, + expectedTime: time.Date(2020, time.January, 1, 12, 0, 0, 0, time.UTC), + }, + { + name: "AfterSetTime_ReturnsSetTime", + initialTimeSetup: func(sc *SimulatedClock) { + sc.SetTime(time.Date(2020, time.January, 1, 12, 0, 0, 0, time.UTC)) + }, + expectedTime: time.Date(2020, time.January, 1, 12, 0, 0, 0, time.UTC), + }, + { + name: "AfterAdvanceTime_ReturnsAdvancedTime", + initialTimeSetup: func(sc *SimulatedClock) { + sc.SetTime(time.Date(2020, time.January, 1, 12, 0, 0, 0, time.UTC)) + sc.AdvanceTime(time.Hour) + }, + expectedTime: time.Date(2020, time.January, 1, 13, 0, 0, 0, time.UTC), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + clock := NewSimulatedClock(time.Date(2020, time.January, 1, 12, 0, 0, 0, time.UTC)) + tc.initialTimeSetup(clock) + + now := clock.Now() + + assert.Equal(t, tc.expectedTime, now) + }) + } +} + +func TestSimulatedClock_SetTime(t *testing.T) { + testCases := []struct { + name string + initialTimeSetup func(sc *SimulatedClock) // To set a pre-existing time if needed + timeToSet time.Time + expectedAfter time.Time + }{ + { + name: "SetFromZeroTime", + initialTimeSetup: func(sc *SimulatedClock) {}, + timeToSet: time.Date(2020, time.January, 1, 12, 0, 0, 0, time.UTC), + expectedAfter: time.Date(2020, time.January, 1, 12, 0, 0, 0, time.UTC), + }, + { + name: "OverwriteExistingTime", + initialTimeSetup: func(sc *SimulatedClock) { + sc.SetTime(time.Date(2020, time.January, 1, 11, 0, 0, 0, time.UTC)) // Start with a different time + }, + timeToSet: time.Date(2020, time.January, 1, 12, 0, 0, 0, time.UTC), + expectedAfter: time.Date(2020, time.January, 1, 12, 0, 0, 0, time.UTC), + }, + { + name: "SetToZeroTime", + initialTimeSetup: func(sc *SimulatedClock) { sc.SetTime(time.Date(2020, time.January, 1, 12, 0, 0, 0, time.UTC)) }, + timeToSet: time.Time{}, // Zero time + expectedAfter: time.Time{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + clock := NewSimulatedClock(time.Date(2020, time.January, 1, 12, 0, 0, 0, time.UTC)) + tc.initialTimeSetup(clock) + + clock.SetTime(tc.timeToSet) + + assert.Equal(t, tc.expectedAfter, clock.Now()) + }) + } +} + +func TestSimulatedClock_AdvanceTime(t *testing.T) { + testCases := []struct { + name string + initialTime time.Time + advanceBy time.Duration + expectedTime time.Time + }{ + { + name: "AdvancePositiveDuration", + initialTime: time.Date(2020, time.January, 1, 12, 0, 0, 0, time.UTC), + advanceBy: 5 * time.Minute, + expectedTime: time.Date(2020, time.January, 1, 12, 5, 0, 0, time.UTC), + }, + { + name: "AdvanceNegativeDuration", + initialTime: time.Date(2020, time.January, 1, 12, 0, 0, 0, time.UTC), + advanceBy: -2 * time.Hour, + expectedTime: time.Date(2020, time.January, 1, 10, 0, 0, 0, time.UTC), + }, + { + name: "AdvanceByZeroDuration", + initialTime: time.Date(2020, time.January, 1, 12, 0, 0, 0, time.UTC), + advanceBy: 0, + expectedTime: time.Date(2020, time.January, 1, 12, 0, 0, 0, time.UTC), + }, + { + name: "AdvanceFromZeroTime", + initialTime: time.Time{}, + advanceBy: time.Hour, + expectedTime: time.Date(1, time.January, 1, 1, 0, 0, 0, time.UTC), // zeroTime + time.Hour + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + clock := NewSimulatedClock(time.Date(2020, time.January, 1, 12, 0, 0, 0, time.UTC)) + clock.SetTime(tc.initialTime) // Set initial time for the clock + + clock.AdvanceTime(tc.advanceBy) + + assert.Equal(t, tc.expectedTime, clock.Now()) + }) + } +} + +func TestSimulatedClock_After_ShouldFireZeroOrNegativeDuration(t *testing.T) { + testCases := []struct { + name string + afterDuration time.Duration + action func(sc *SimulatedClock) // Action to manipulate the clock after After() is called + }{ + { + name: "ZeroDuration_FiresImmediately", + afterDuration: 0, + action: func(sc *SimulatedClock) { /* No action needed for immediate fire */ }, + }, + { + name: "NegativeDuration_FiresImmediately", + afterDuration: -5 * time.Second, + action: func(sc *SimulatedClock) { /* No action needed for immediate fire */ }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + clock := NewSimulatedClock(time.Date(2020, time.January, 1, 12, 0, 0, 0, time.UTC)) + clockTimeAtAfterCall := clock.Now() + + ch := clock.After(tc.afterDuration) + require.NotNil(t, ch, "Channel should not be nil") + + // Perform the action (if any) that might trigger the timer + tc.action(clock) + + // Fires at the same time for zero/negative duration. + expectedFireTimeOnChannel := clockTimeAtAfterCall + + select { + case actualTime := <-ch: + assert.Equal(t, expectedFireTimeOnChannel, actualTime) + + case <-time.After(fireTestTimeout): + t.Fatalf("Timeout waiting for time on channel. Expected after %v.", expectedFireTimeOnChannel) + } + }) + } +} + +func TestSimulatedClock_After_ShouldFirePositiveDuration(t *testing.T) { + testCases := []struct { + name string + afterDuration time.Duration + action func(sc *SimulatedClock) // Action to manipulate the clock after After() is called + }{ + { + name: "PositiveDuration_Fires_WhenTimeAdvancedPastDuration", + afterDuration: 10 * time.Second, + action: func(sc *SimulatedClock) { + sc.AdvanceTime(15 * time.Second) // Advance well past the duration + }, + }, + { + name: "PositiveDuration_Fires_WhenTimeSetPastDuration", + afterDuration: 10 * time.Second, + action: func(sc *SimulatedClock) { + sc.SetTime(time.Date(2020, time.January, 1, 12, 0, 15, 0, time.UTC)) // Set time well past the duration + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + clock := NewSimulatedClock(time.Date(2020, time.January, 1, 12, 0, 0, 0, time.UTC)) + clockTimeAtAfterCall := clock.Now() + + ch := clock.After(tc.afterDuration) + require.NotNil(t, ch, "Channel should not be nil") + + // Perform the action (if any) that might trigger the timer + tc.action(clock) + // Expected fire time, time + duration. + expectedFireTimeOnChannel := clockTimeAtAfterCall.Add(tc.afterDuration) + + select { + case actualTime := <-ch: + assert.Equal(t, expectedFireTimeOnChannel, actualTime) + + case <-time.After(fireTestTimeout): + t.Fatalf("Timeout waiting for time on channel. Expected after %v.", expectedFireTimeOnChannel) + } + }) + } +} + +func TestSimulatedClock_After_ShouldNotFire(t *testing.T) { + testCases := []struct { + name string + afterDuration time.Duration + action func(sc *SimulatedClock) // Action to manipulate the clock after After() is called + }{ + { + name: "PositiveDuration_DoesNotFire_WhenTimeAdvancedBeforeDuration", + afterDuration: 10 * time.Second, + action: func(sc *SimulatedClock) { + sc.AdvanceTime(5 * time.Second) // Advance, but not enough + }, + }, + { + name: "PositiveDuration_DoesNotFire_WhenTimeSetBeforeDuration", + afterDuration: 10 * time.Second, + action: func(sc *SimulatedClock) { + sc.SetTime(time.Date(2020, time.January, 1, 12, 0, 5, 0, time.UTC)) // Set time, but not enough + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + clock := NewSimulatedClock(time.Date(2020, time.January, 1, 12, 0, 0, 0, time.UTC)) + + ch := clock.After(tc.afterDuration) + require.NotNil(t, ch, "Channel should not be nil") + + // Perform the action (if any) that might trigger the timer + tc.action(clock) + + select { + case receivedTime := <-ch: + t.Fatalf("Expected no time on channel, but received %v.", receivedTime) + + case <-time.After(shortTestTimeout): + // Success, nothing received + } + }) + } +} diff --git a/internal/contentcache/contentcache.go b/internal/contentcache/contentcache.go index fef7554d13..4e84e893b8 100644 --- a/internal/contentcache/contentcache.go +++ b/internal/contentcache/contentcache.go @@ -26,8 +26,8 @@ import ( "regexp" "sync" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" "github.com/jacobsa/timeutil" ) diff --git a/internal/contentcache/contentcache_test.go b/internal/contentcache/contentcache_test.go index 3c28c9fe72..e3e249d618 100644 --- a/internal/contentcache/contentcache_test.go +++ b/internal/contentcache/contentcache_test.go @@ -21,7 +21,7 @@ import ( "sync" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/internal/contentcache" + "github.com/googlecloudplatform/gcsfuse/v3/internal/contentcache" "github.com/jacobsa/fuse/fsutil" . "github.com/jacobsa/ogletest" "github.com/jacobsa/timeutil" diff --git a/internal/fs/all_buckets_test.go b/internal/fs/all_buckets_test.go index 5132a42efd..804b498e5a 100644 --- a/internal/fs/all_buckets_test.go +++ b/internal/fs/all_buckets_test.go @@ -19,8 +19,8 @@ import ( "os" "path" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/fake" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/ogletest" "github.com/jacobsa/timeutil" @@ -41,9 +41,9 @@ func init() { func (t *AllBucketsTest) SetUpTestSuite() { mtimeClock = timeutil.RealClock() buckets = map[string]gcs.Bucket{ - "bucket-0": fake.NewFakeBucket(mtimeClock, "bucket-0", gcs.NonHierarchical), - "bucket-1": fake.NewFakeBucket(mtimeClock, "bucket-1", gcs.NonHierarchical), - "bucket-2": fake.NewFakeBucket(mtimeClock, "bucket-2", gcs.NonHierarchical), + "bucket-0": fake.NewFakeBucket(mtimeClock, "bucket-0", gcs.BucketType{}), + "bucket-1": fake.NewFakeBucket(mtimeClock, "bucket-1", gcs.BucketType{}), + "bucket-2": fake.NewFakeBucket(mtimeClock, "bucket-2", gcs.BucketType{}), } // buckets: {"some_bucket", "bucket-1", "bucket-2"} t.fsTest.SetUpTestSuite() diff --git a/internal/fs/caching_test.go b/internal/fs/caching_test.go index 8370a3d2d2..7c804709a9 100644 --- a/internal/fs/caching_test.go +++ b/internal/fs/caching_test.go @@ -20,14 +20,16 @@ import ( "path" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/lru" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/metadata" - "github.com/googlecloudplatform/gcsfuse/v2/internal/fs/inode" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/caching" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/fake" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/inode" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/caching" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" "github.com/jacobsa/fuse/fusetesting" . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/ogletest" @@ -39,6 +41,7 @@ import ( //////////////////////////////////////////////////////////////////////// const ttl = 10 * time.Minute +const negativeCacheTTL = 1 * time.Minute var ( uncachedBucket gcs.Bucket @@ -55,14 +58,18 @@ type cachingTestCommon struct { func (t *cachingTestCommon) SetUpTestSuite() { // Wrap the bucket in a stat caching layer for the purposes of the file // system. - uncachedBucket = fake.NewFakeBucket(timeutil.RealClock(), "some_bucket", gcs.NonHierarchical) + uncachedBucket = fake.NewFakeBucket(timeutil.RealClock(), "some_bucket", gcs.BucketType{}) lruCache := newLruCache(uint64(1000 * cfg.AverageSizeOfPositiveStatCacheEntry)) statCache := metadata.NewStatCacheBucketView(lruCache, "") bucket = caching.NewFastStatBucket( ttl, statCache, &cacheClock, - uncachedBucket) + uncachedBucket, + negativeCacheTTL, + IsTypeCacheDeprecated, + isImplicitDir, + ) // Enable directory type caching. t.serverCfg.DirTypeCacheTTL = ttl @@ -462,17 +469,23 @@ func (t *MultiBucketMountCachingTest) SetUpTestSuite() { // Create uncached buckets and wrap them in stat caching layer // for the purposes of the file system. for _, bucketName := range []string{bucket1Name, bucket2Name} { - uncachedBuckets[bucketName] = fake.NewFakeBucket(timeutil.RealClock(), bucketName, gcs.NonHierarchical) + uncachedBuckets[bucketName] = fake.NewFakeBucket(timeutil.RealClock(), bucketName, gcs.BucketType{}) statCache := metadata.NewStatCacheBucketView(sharedCache, bucketName) buckets[bucketName] = caching.NewFastStatBucket( ttl, statCache, &cacheClock, - uncachedBuckets[bucketName]) + uncachedBuckets[bucketName], + negativeCacheTTL, + IsTypeCacheDeprecated, + isImplicitDir, + ) } // Enable directory type caching. t.serverCfg.DirTypeCacheTTL = ttl + t.serverCfg.MetricHandle = metrics.NewNoopMetrics() + t.serverCfg.TraceHandle = tracing.NewNoopTracer() // Call through. t.fsTest.SetUpTestSuite() @@ -725,3 +738,107 @@ func (t *MultiBucketMountCachingTest) TypeOfNameChanges_RemoteModifier() { AssertEq(nil, err) ExpectFalse(fi.IsDir()) } + +// --------------------- Test for delete directory ----------------- +// Create directory +// stat directory +// Delete directory +// Create directory using storageutil +// Stat the directory - It should return not found error from cache although dir present on GCS +// Stat the directory after TTL expiry and it should appear +// ------------------------------------------------------------------ +func (t *MultiBucketMountCachingTest) DirectoryRemovedLocallyAddedRemotely() { + const name = "foo" + var fi os.FileInfo + var err error + bucket1MntDir := getMultiMountBucketDir(bucket1Name) + bucket1 := uncachedBuckets[bucket1Name] + + // Create a directory via the file system. + err = os.Mkdir(path.Join(bucket1MntDir, name), 0700) + AssertEq(nil, err) + + // Stat the directory + fi, err = os.Stat(path.Join(bucket1MntDir, name)) + AssertEq(nil, err) + ExpectTrue(fi.IsDir()) + + // Remove the directory locally. + err = os.RemoveAll(path.Join(bucket1MntDir, name)) + AssertEq(nil, err) + + // Create a directory with the same name via GCS. + _, err = storageutil.CreateObject( + ctx, + bucket1, + name+"/", + []byte("")) + + AssertEq(nil, err) + + // Because we are caching, the directory should still appear to not exist. + _, err = os.Stat(path.Join(bucket1MntDir, name)) + ExpectTrue(os.IsNotExist(err), "err: %v", err) + + // After the TTL elapses, we should see it reappear. + cacheClock.AdvanceTime(ttl + time.Millisecond) + + fi, err = os.Stat(path.Join(bucket1MntDir, name)) + AssertEq(nil, err) + ExpectTrue(fi.IsDir()) +} + +// --------------------- Test for delete object ------------------- +// Create directory +// Create object in directory +// Stat the object +// Delete object +// Create object using storageutil +// Stat the object. It should return not found error although object present. +// Stat the object after TTL expiry and it should appear +// ------------------------------------------------------------------ +func (t *MultiBucketMountCachingTest) ObjectRemovedLocallyAddedRemotely() { + const dirName = "foo" + const objName = "bar" + var fi os.FileInfo + var err error + bucket1MntDir := getMultiMountBucketDir(bucket1Name) + bucket1 := uncachedBuckets[bucket1Name] + + // Create a directory via the file system. + err = os.Mkdir(path.Join(bucket1MntDir, dirName), 0700) + AssertEq(nil, err) + + // Create an object in the directory via the file system. + err = os.WriteFile(path.Join(bucket1MntDir, dirName, objName), []byte("taco"), 0400) + AssertEq(nil, err) + + // Stat the object + fi, err = os.Stat(path.Join(bucket1MntDir, dirName, objName)) + AssertEq(nil, err) + ExpectFalse(fi.IsDir()) + + // Remove the object locally. + err = os.Remove(path.Join(bucket1MntDir, dirName, objName)) + AssertEq(nil, err) + + // Create an object with the same name via GCS. + _, err = storageutil.CreateObject( + ctx, + bucket1, + path.Join(dirName, objName), + []byte("burrito")) + + AssertEq(nil, err) + + // Because we are caching, the object should still appear to not exist. + _, err = os.Stat(path.Join(bucket1MntDir, dirName, objName)) + ExpectTrue(os.IsNotExist(err), "err: %v", err) + + // After the TTL elapses, we should see it reappear. + cacheClock.AdvanceTime(ttl + time.Millisecond) + + fi, err = os.Stat(path.Join(bucket1MntDir, dirName, objName)) + AssertEq(nil, err) + ExpectFalse(fi.IsDir()) +} diff --git a/internal/fs/flat_bucket_test.go b/internal/fs/flat_bucket_test.go new file mode 100644 index 0000000000..e92dcbf11d --- /dev/null +++ b/internal/fs/flat_bucket_test.go @@ -0,0 +1,65 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type FlatBucketTests struct { + suite.Suite + fsTest + RenameDirTests + RenameFileTests +} + +func TestFlatBucketTests(t *testing.T) { suite.Run(t, new(FlatBucketTests)) } + +func (t *FlatBucketTests) SetT(testingT *testing.T) { + t.Suite.SetT(testingT) + t.RenameDirTests.SetT(testingT) + t.RenameFileTests.SetT(testingT) +} + +func (t *FlatBucketTests) SetupSuite() { + t.serverCfg.RenameDirLimit = 20 + t.serverCfg.ImplicitDirectories = true + t.fsTest.SetUpTestSuite() +} + +func (t *FlatBucketTests) TearDownSuite() { + t.fsTest.TearDownTestSuite() +} + +func (t *FlatBucketTests) SetupTest() { + err := t.createObjects( + map[string]string{ + "foo/test2/": "", + "foo/test/": "", + "foo/file1.txt": file1Content, + "foo/file2.txt": file2Content, + "foo/test/file3.txt": "xyz", + "foo/implicit_dir/file3.txt": "xxw", + "bar/file1.txt": "-1234556789", + }) + require.NoError(t.T(), err) +} + +func (t *FlatBucketTests) TearDownTest() { + t.fsTest.TearDown() +} diff --git a/internal/fs/foreign_modifications_test.go b/internal/fs/foreign_modifications_test.go index fbb25bb44c..30d89a04bc 100644 --- a/internal/fs/foreign_modifications_test.go +++ b/internal/fs/foreign_modifications_test.go @@ -29,10 +29,10 @@ import ( "syscall" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" - "github.com/googlecloudplatform/gcsfuse/v2/internal/fs/inode" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/inode" "github.com/jacobsa/fuse/fusetesting" . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/ogletest" @@ -601,7 +601,7 @@ func (t *ForeignModsTest) ReadBeyondEndOfFile() { AssertEq(contents[contentLen-1], buf[0]) if err == nil { - n, err = f.Read(buf) + n, _ = f.Read(buf) AssertEq(0, n) } } diff --git a/internal/fs/fs.go b/internal/fs/fs.go index bbeb7ab3ef..08ebc66990 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -20,32 +20,45 @@ import ( "fmt" "io" iofs "io/fs" + "maps" "math" "os" "path" "reflect" + "slices" "strings" "syscall" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/file" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/file/downloader" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/lru" - cacheutil "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/util" - "github.com/googlecloudplatform/gcsfuse/v2/internal/contentcache" - "github.com/googlecloudplatform/gcsfuse/v2/internal/fs/handle" - "github.com/googlecloudplatform/gcsfuse/v2/internal/fs/inode" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" - "github.com/googlecloudplatform/gcsfuse/v2/internal/locker" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util/diskutil" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/gcsfuse_errors" + "github.com/googlecloudplatform/gcsfuse/v3/internal/kernelparams" + "github.com/googlecloudplatform/gcsfuse/v3/internal/workerpool" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + + "golang.org/x/sync/semaphore" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/file" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/file/downloader" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + cacheutil "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/contentcache" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/handle" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/inode" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/locker" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" "github.com/jacobsa/fuse" "github.com/jacobsa/fuse/fuseops" "github.com/jacobsa/fuse/fuseutil" "github.com/jacobsa/timeutil" - "golang.org/x/sync/semaphore" + "github.com/spf13/viper" ) type ServerConfig struct { @@ -125,6 +138,20 @@ type ServerConfig struct { // NewConfig has all the config specified by the user using config-file or CLI flags. NewConfig *cfg.Config + + // ViperConfig tracks which flags were explicitly set by the user (vs defaults) + // This is used to determine if optimization rules should be applied. + ViperConfig *viper.Viper + + MetricHandle metrics.MetricHandle + + TraceHandle tracing.TraceHandle + + // Notifier allows the file system to send invalidation messages to the FUSE + // kernel module. This enables proactive cache invalidation (e.g., for dentries) + // when underlying content changes, improving consistency while still leveraging + // kernel caching. + Notifier *fuse.Notifier } // Create a fuse file system server according to the supplied configuration. @@ -153,9 +180,10 @@ func NewFileSystem(ctx context.Context, serverCfg *ServerConfig) (fuseutil.FileS // Create file cache handler if cache is enabled by user. Cache is considered // enabled only if cache-dir is not empty and file-cache:max-size-mb is non 0. var fileCacheHandler *file.CacheHandler + var sharedChunkCacheManager *file.SharedChunkCacheManager if cfg.IsFileCacheEnabled(serverCfg.NewConfig) { var err error - fileCacheHandler, err = createFileCacheHandler(serverCfg) + fileCacheHandler, sharedChunkCacheManager, err = createFileCacheHandler(serverCfg) if err != nil { return nil, err } @@ -185,11 +213,36 @@ func NewFileSystem(ctx context.Context, serverCfg *ServerConfig) (fuseutil.FileS implicitDirInodes: make(map[inode.Name]inode.DirInode), folderInodes: make(map[inode.Name]inode.DirInode), localFileInodes: make(map[inode.Name]inode.Inode), - handles: make(map[fuseops.HandleID]interface{}), + handles: make(map[fuseops.HandleID]any), newConfig: serverCfg.NewConfig, fileCacheHandler: fileCacheHandler, + sharedChunkCacheManager: sharedChunkCacheManager, cacheFileForRangeRead: serverCfg.NewConfig.FileCache.CacheFileForRangeRead, - globalMaxBlocksSem: semaphore.NewWeighted(serverCfg.NewConfig.Write.GlobalMaxBlocks), + metricHandle: serverCfg.MetricHandle, + traceHandle: serverCfg.TraceHandle, + enableAtomicRenameObject: serverCfg.NewConfig.EnableAtomicRenameObject, + isTracingEnabled: cfg.IsTracingEnabled(serverCfg.NewConfig), + globalMaxWriteBlocksSem: semaphore.NewWeighted(serverCfg.NewConfig.Write.GlobalMaxBlocks), + globalMaxReadBlocksSem: semaphore.NewWeighted(serverCfg.NewConfig.Read.GlobalMaxBlocks), + globalMetadataPrefetchSem: semaphore.NewWeighted(serverCfg.NewConfig.MetadataCache.MetadataPrefetchMaxWorkers), + } + + // Initialize MRD cache if enabled + if serverCfg.NewConfig.FileSystem.InactiveMrdCacheSize > 0 { + fs.mrdCache = lru.NewCache(uint64(serverCfg.NewConfig.FileSystem.InactiveMrdCacheSize)) + logger.Infof("MRD cache (LRU-based) enabled with maxSize=%d (pools closed-file MRDs)", serverCfg.NewConfig.FileSystem.InactiveMrdCacheSize) + } + + if serverCfg.Notifier != nil { + fs.notifier = serverCfg.Notifier + } + + if serverCfg.NewConfig.Read.EnableBufferedRead { + var err error + fs.bufferedReadWorkerPool, err = workerpool.NewStaticWorkerPoolForCurrentCPU(serverCfg.NewConfig.Read.GlobalMaxBlocks) + if err != nil { + return nil, fmt.Errorf("failed to create worker pool for buffered read: %w", err) + } } // Set up root bucket @@ -199,11 +252,46 @@ func NewFileSystem(ctx context.Context, serverCfg *ServerConfig) (fuseutil.FileS root = makeRootForAllBuckets(fs) } else { logger.Info("Set up root directory for bucket " + serverCfg.BucketName) - syncerBucket, err := fs.bucketManager.SetUpBucket(ctx, serverCfg.BucketName, false) + syncerBucket, err := fs.bucketManager.SetUpBucket(ctx, serverCfg.BucketName, false, fs.metricHandle) if err != nil { return nil, fmt.Errorf("SetUpBucket: %w", err) } - root = makeRootForBucket(ctx, fs, syncerBucket) + + // Optimize flags for non-dynamic mounts based on the bucket type. + // WHY HERE: The bucket type is required for these optimizations, and this is the + // first point it becomes available. + // WHY NOT EARLIER: Although ideal to do this during cobra init in root.go, + // the bucket type isn't known then. A major refactor (involving creation and caching of + // bucketHandle to avoid duplicated network calls) would be needed to change this. + // IMPACT: Flags are used after this. Optimization/rationalization functions are called twice + // for non-dynamic mounts, but they are idempotent, so it's safe. + bucketType := syncerBucket.BucketType() + if serverCfg.ViperConfig != nil { + bucketTypeEnum := cfg.GetBucketType(bucketType.Hierarchical, bucketType.Zonal, bucketType.Pirlo != gcs.PirloStateNone) + optimizedFlags := serverCfg.NewConfig.ApplyOptimizations(serverCfg.ViperConfig, &cfg.OptimizationInput{ + BucketType: bucketTypeEnum, + }) + if len(optimizedFlags) > 0 { + logger.Info("GCSFuse Config", "Applied optimizations for bucket-type: ", bucketTypeEnum, "Full Config", optimizedFlags) + optimizedFlagNames := slices.Collect(maps.Keys(optimizedFlags)) + if err := cfg.Rationalize(serverCfg.ViperConfig, serverCfg.NewConfig, optimizedFlagNames); err != nil { + logger.Warnf("GCSFuse Config: error in rationalize after applying bucket-type optimizations: %v", err) + } + } + } else { + logger.Warnf("Cannot apply bucket-type optimizations as ViperConfig is nil") + } + // Write post mount kernel settings for Zonal Buckets when kernel reader is enabled in GKE environments for + // non dynamic mounts before user space mounting in GCSFuse. Mounting in GKE is already done at this point but + // writing kernel settings early ensures the asynchronous application of these settings happens as early as possible in GKE. + if serverCfg.NewConfig.FileSystem.KernelParamsFile != "" && bucketType.Zonal && serverCfg.NewConfig.FileSystem.EnableKernelReader { + kernelParams := kernelparams.NewKernelParamsManager() + kernelParams.SetReadAheadKb(int(serverCfg.NewConfig.FileSystem.MaxReadAheadKb)) + kernelParams.SetCongestionWindowThreshold(int(serverCfg.NewConfig.FileSystem.CongestionThreshold)) + kernelParams.SetMaxBackgroundRequests(int(serverCfg.NewConfig.FileSystem.MaxBackground)) + kernelParams.ApplyGKE(string(serverCfg.NewConfig.FileSystem.KernelParamsFile)) + } + root = makeRootForBucket(fs, syncerBucket) } root.Lock() root.IncrementLookupCount() @@ -217,43 +305,115 @@ func NewFileSystem(ctx context.Context, serverCfg *ServerConfig) (fuseutil.FileS return fs, nil } -func createFileCacheHandler(serverCfg *ServerConfig) (fileCacheHandler *file.CacheHandler, err error) { +// createFileCacheHandler either returns a regular file cache handler with an in-memory LRU cache, or +// a shared chunk cache manager that allows multiple gcsfuse instances to share the same cache directory +// on disk, based on the configuration. +func createFileCacheHandler(serverCfg *ServerConfig) (fileCacheHandler *file.CacheHandler, sharedChunkCacheManager *file.SharedChunkCacheManager, err error) { + baseCacheDir := string(serverCfg.NewConfig.CacheDir) + filePerm := cacheutil.DefaultFilePerm + dirPerm := cacheutil.DefaultDirPerm + + // Shared cache with external LRU cache eviction. + if serverCfg.NewConfig.FileCache.EnableExperimentalSharedChunkCache { + sharedChunkCacheManager, err := createSharedChunkCacheManager(baseCacheDir, filePerm, dirPerm, serverCfg) + return nil, sharedChunkCacheManager, err + } + + // Regular gcsfuse file-cache with memory based LRU cache. + fileCacheHandler, err = createSingleMountFileCacheHandler(baseCacheDir, filePerm, dirPerm, serverCfg) + return fileCacheHandler, nil, err +} + +// createSharedChunkCacheManager creates a shared chunk cache manager for multi-instance caching. +func createSharedChunkCacheManager(baseCacheDir string, filePerm, dirPerm os.FileMode, serverCfg *ServerConfig) (*file.SharedChunkCacheManager, error) { + // Use separate directory for shared chunk cache to avoid conflicts with regular file cache + cacheDir := path.Join(baseCacheDir, cacheutil.SharedChunkCache) + + if err := cacheutil.CreateCacheDirectoryIfNotPresentAt(cacheDir, dirPerm); err != nil { + return nil, fmt.Errorf("createSharedChunkCacheManager: while creating shared chunk cache directory: %w", err) + } + + sharedCacheManager, err := file.NewSharedChunkCacheManager( + cacheDir, + filePerm, + dirPerm, + &serverCfg.NewConfig.FileCache, + ) + if err != nil { + return nil, fmt.Errorf("createSharedChunkCacheManager: while creating shared chunk cache manager: %w", err) + } + + logger.Infof("File Cache: Shared chunk cache created successfully at %s", cacheDir) + return sharedCacheManager, nil +} + +func cacheDirVolumeBlockSize(serverCfg *ServerConfig, cacheDir string) uint64 { + if serverCfg.NewConfig.FileCache.ExperimentalDisableSizeCalculationFix { + return 1 + } + if serverCfg.NewConfig.FileCache.ExperimentalEnableChunkCache { + logger.Info("file-cache disk-utilization fix is not supported with sparse-mode, so is disabled.") + return 1 + } + return diskutil.GetVolumeBlockSize(cacheDir) +} + +// createSingleMountFileCacheHandler creates a file cache handler with an in-memory LRU cache specific to a single gcsfuse instance. +func createSingleMountFileCacheHandler(baseCacheDir string, filePerm, dirPerm os.FileMode, serverCfg *ServerConfig) (*file.CacheHandler, error) { + // Use separate directory for regular file cache + cacheDir := path.Join(baseCacheDir, cacheutil.FileCache) + + if err := cacheutil.CreateCacheDirectoryIfNotPresentAt(cacheDir, dirPerm); err != nil { + return nil, fmt.Errorf("createSingleMountFileCacheHandler: while creating file cache directory: %w", err) + } + + // Calculate cache size var sizeInBytes uint64 // -1 means unlimited size for cache, the underlying LRU cache doesn't handle // -1 explicitly, hence we pass MaxUint64 as capacity in that case. if serverCfg.NewConfig.FileCache.MaxSizeMb == -1 { sizeInBytes = math.MaxUint64 + logger.Infof("File Cache: Regular cache size: UNLIMITED") } else { sizeInBytes = uint64(serverCfg.NewConfig.FileCache.MaxSizeMb) * cacheutil.MiB + logger.Infof("File Cache: Regular cache size: %d MB (%d bytes)", serverCfg.NewConfig.FileCache.MaxSizeMb, sizeInBytes) } - fileInfoCache := lru.NewCache(sizeInBytes) - cacheDir := string(serverCfg.NewConfig.CacheDir) - // Adding a new directory inside cacheDir to keep file-cache separate from - // metadata cache if and when we support storing metadata cache on disk in - // the future. - cacheDir = path.Join(cacheDir, cacheutil.FileCache) - - filePerm := cacheutil.DefaultFilePerm - dirPerm := cacheutil.DefaultDirPerm - - cacheDirErr := cacheutil.CreateCacheDirectoryIfNotPresentAt(cacheDir, dirPerm) - if cacheDirErr != nil { - return nil, fmt.Errorf("createFileCacheHandler: while creating file cache directory: %w", cacheDirErr) - } + fileInfoCache := lru.NewCache(sizeInBytes) + cacheDirVolumeBlockSize := cacheDirVolumeBlockSize(serverCfg, cacheDir) + jobManager := downloader.NewJobManager( + fileInfoCache, + filePerm, + dirPerm, + cacheDir, + serverCfg.SequentialReadSizeMb, + &serverCfg.NewConfig.FileCache, + serverCfg.MetricHandle, + serverCfg.TraceHandle, + cacheDirVolumeBlockSize, + ) + fileCacheHandler := file.NewCacheHandler( + fileInfoCache, + jobManager, + cacheDir, + filePerm, + dirPerm, + serverCfg.NewConfig.FileCache.ExcludeRegex, + serverCfg.NewConfig.FileCache.IncludeRegex, + serverCfg.NewConfig.FileCache.ExperimentalEnableChunkCache, + cacheDirVolumeBlockSize, + ) - jobManager := downloader.NewJobManager(fileInfoCache, filePerm, dirPerm, cacheDir, serverCfg.SequentialReadSizeMb, &serverCfg.NewConfig.FileCache) - fileCacheHandler = file.NewCacheHandler(fileInfoCache, jobManager, cacheDir, filePerm, dirPerm) - return + return fileCacheHandler, nil } func makeRootForBucket( - ctx context.Context, fs *fileSystem, syncerBucket gcsx.SyncerBucket) inode.DirInode { return inode.NewDirInode( fuseops.RootInodeID, inode.NewRootName(""), + nil, // For root buckets, there is no parent and hence no parent context. fuseops.InodeAttributes{ Uid: fs.uid, Gid: fs.gid, @@ -265,14 +425,13 @@ func makeRootForBucket( Mtime: fs.mtimeClock.Now(), }, fs.implicitDirs, - fs.newConfig.List.EnableEmptyManagedFolders, fs.enableNonexistentTypeCache, fs.dirTypeCacheTTL, &syncerBucket, fs.mtimeClock, fs.cacheClock, - fs.newConfig.MetadataCache.TypeCacheMaxSizeMb, - fs.newConfig.EnableHns, + fs.globalMetadataPrefetchSem, + fs.newConfig, ) } @@ -291,6 +450,8 @@ func makeRootForAllBuckets(fs *fileSystem) inode.DirInode { Mtime: fs.mtimeClock.Now(), }, fs.bucketManager, + fs.metricHandle, + fs.newConfig.EnableTypeCacheDeprecation, ) } @@ -458,7 +619,7 @@ type fileSystem struct { // INVARIANT: All values are of type *dirHandle or *handle.FileHandle // // GUARDED_BY(mu) - handles map[fuseops.HandleID]interface{} + handles map[fuseops.HandleID]any // The next handle ID to hand out. We assume that this will never overflow. // @@ -474,17 +635,74 @@ type fileSystem struct { // file cache is enabled at the time of mounting. fileCacheHandler *file.CacheHandler + // sharedChunkCacheManager manages the shared chunk cache enable with + // enable-experimental-shared-cache. + // Non-nil only when file cache is enabled with enable-experimental-shared-cache flag. + sharedChunkCacheManager *file.SharedChunkCacheManager + // cacheFileForRangeRead when true downloads file into cache even for // random file access. cacheFileForRangeRead bool - globalMaxBlocksSem *semaphore.Weighted + metricHandle metrics.MetricHandle + + traceHandle tracing.TraceHandle + + enableAtomicRenameObject bool + + isTracingEnabled bool + + // Limits the max number of blocks that can be created across file system when + // streaming writes are enabled. + globalMaxWriteBlocksSem *semaphore.Weighted + + // notifier allows sending invalidation messages to the FUSE kernel module. + // It is used to invalidate the kernel's dentry cache, + // providing feedback to the kernel about dynamic content changes. + notifier *fuse.Notifier + + // bufferedReadWorkerPool is used for asynchronous prefetching of data for buffered reads. + // It executes download tasks associated with prefetch blocks. + bufferedReadWorkerPool workerpool.WorkerPool + + // globalMaxReadBlocksSem is a semaphore that limits the total number of blocks + // that can be allocated for buffered read across all file-handles in the file system. + // This helps control the overall memory usage for buffered reads. + globalMaxReadBlocksSem *semaphore.Weighted + + // Limits the max number of metadata prefetch background workers across file system when + // metadata prefetching is enabled. + globalMetadataPrefetchSem *semaphore.Weighted + + // mrdCache manages the cache of inactive MultiRangeDownloaders. + mrdCache *lru.Cache } //////////////////////////////////////////////////////////////////////// // Helpers //////////////////////////////////////////////////////////////////////// +// LOCKS_REQUIRED(fs.mu) +func (fs *fileSystem) findParentDirInode(childName inode.Name) inode.DirInode { + parentName, err := childName.ParentName() + if err != nil { + return nil + } + + if parentInode, ok := fs.implicitDirInodes[parentName]; ok { + return parentInode + } + if parentInode, ok := fs.folderInodes[parentName]; ok { + return parentInode + } + if parentInode, ok := fs.generationBackedInodes[parentName]; ok { + if dirInode, ok := parentInode.(inode.DirInode); ok { + return dirInode + } + } + return nil +} + func (fs *fileSystem) checkInvariantsForLocalFileInodes() { // INVARIANT: For each k/v, v.Name() == k for k, v := range fs.localFileInodes { @@ -704,10 +922,11 @@ func (fs *fileSystem) checkInvariants() { } } -func (fs *fileSystem) createExplicitDirInode(inodeID fuseops.InodeID, ic inode.Core) inode.Inode { +func (fs *fileSystem) createExplicitDirInode(inodeID fuseops.InodeID, ic inode.Core, parInodeCtx context.Context) inode.Inode { in := inode.NewExplicitDirInode( inodeID, ic.FullName, + parInodeCtx, ic.MinObject, fuseops.InodeAttributes{ Uid: fs.uid, @@ -720,14 +939,13 @@ func (fs *fileSystem) createExplicitDirInode(inodeID fuseops.InodeID, ic inode.C Mtime: fs.mtimeClock.Now(), }, fs.implicitDirs, - fs.newConfig.List.EnableEmptyManagedFolders, fs.enableNonexistentTypeCache, fs.dirTypeCacheTTL, ic.Bucket, fs.mtimeClock, fs.cacheClock, - fs.newConfig.MetadataCache.TypeCacheMaxSizeMb, - fs.newConfig.EnableHns) + fs.globalMetadataPrefetchSem, + fs.newConfig) return in } @@ -736,7 +954,7 @@ func (fs *fileSystem) createExplicitDirInode(inodeID fuseops.InodeID, ic inode.C // of that function. // // LOCKS_REQUIRED(fs.mu) -func (fs *fileSystem) mintInode(ic inode.Core) (in inode.Inode) { +func (fs *fileSystem) mintInode(ic inode.Core, parInodeCtx context.Context) (in inode.Inode, err error) { // Choose an ID. id := fs.nextInodeID fs.nextInodeID++ @@ -745,13 +963,14 @@ func (fs *fileSystem) mintInode(ic inode.Core) (in inode.Inode) { switch { // Explicit directories or folders in hierarchical bucket. case (ic.MinObject != nil && ic.FullName.IsDir()), ic.Folder != nil: - in = fs.createExplicitDirInode(id, ic) + in = fs.createExplicitDirInode(id, ic, parInodeCtx) // Implicit directories case ic.FullName.IsDir(): in = inode.NewDirInode( id, ic.FullName, + parInodeCtx, fuseops.InodeAttributes{ Uid: fs.uid, Gid: fs.gid, @@ -763,26 +982,30 @@ func (fs *fileSystem) mintInode(ic inode.Core) (in inode.Inode) { Mtime: fs.mtimeClock.Now(), }, fs.implicitDirs, - fs.newConfig.List.EnableEmptyManagedFolders, fs.enableNonexistentTypeCache, fs.dirTypeCacheTTL, ic.Bucket, fs.mtimeClock, fs.cacheClock, - fs.newConfig.MetadataCache.TypeCacheMaxSizeMb, - fs.newConfig.EnableHns, + fs.globalMetadataPrefetchSem, + fs.newConfig, ) case inode.IsSymlink(ic.MinObject): - in = inode.NewSymlinkInode( + in, err = inode.NewSymlinkInode( + parInodeCtx, id, ic.FullName, + ic.Bucket, ic.MinObject, fuseops.InodeAttributes{ Uid: fs.uid, Gid: fs.gid, Mode: fs.fileMode | os.ModeSymlink, }) + if err != nil { + return nil, err + } default: in = inode.NewFileInode( @@ -799,14 +1022,17 @@ func (fs *fileSystem) mintInode(ic inode.Core) (in inode.Inode) { fs.contentCache, fs.mtimeClock, ic.Local, - &fs.newConfig.Write, - fs.globalMaxBlocksSem) + fs.newConfig, + fs.globalMaxWriteBlocksSem, + fs.mrdCache, + fs.traceHandle, + fs.metricHandle) } // Place it in our map of IDs to inodes. fs.inodes[in.ID()] = in - return + return in, nil } // Return the dir Inode. @@ -814,21 +1040,24 @@ func (fs *fileSystem) mintInode(ic inode.Core) (in inode.Inode) { // LOCKS_EXCLUDED(fs.mu) // UNLOCK_FUNCTION(fs.mu) // LOCK_FUNCTION(in) -func (fs *fileSystem) createDirInode(ic inode.Core, inodes map[inode.Name]inode.DirInode) inode.Inode { +func (fs *fileSystem) createDirInode(ic inode.Core, inodes map[inode.Name]inode.DirInode, parInodeCtx context.Context) (inode.Inode, error) { if !ic.FullName.IsDir() { panic(fmt.Sprintf("Unexpected name for a directory: %q", ic.FullName)) } var maxTriesToCreateInode = 3 - for n := 0; n < maxTriesToCreateInode; n++ { + for range maxTriesToCreateInode { in, ok := (inodes)[ic.FullName] // Create a new inode when a folder is created first time, or when a folder is deleted and then recreated with the same name. if !ok || in.IsUnlinked() { - in := fs.mintInode(ic) + in, err := fs.mintInode(ic, parInodeCtx) + if err != nil { + return nil, err + } (inodes)[in.Name()] = in.(inode.DirInode) in.Lock() - return in + return in, nil } fs.mu.Unlock() @@ -840,10 +1069,10 @@ func (fs *fileSystem) createDirInode(ic inode.Core, inodes map[inode.Name]inode. continue } - return in + return in, nil } - return nil + return nil, fmt.Errorf("createDirInode: failed to create inode after %d tries", maxTriesToCreateInode) } // Attempt to find an inode for a backing object or an implicit directory. @@ -857,9 +1086,8 @@ func (fs *fileSystem) createDirInode(ic inode.Core, inodes map[inode.Name]inode. // LOCKS_EXCLUDED(fs.mu) // UNLOCK_FUNCTION(fs.mu) // LOCK_FUNCTION(in) -func (fs *fileSystem) lookUpOrCreateInodeIfNotStale(ic inode.Core) (in inode.Inode) { +func (fs *fileSystem) lookUpOrCreateInodeIfNotStale(parInodeCtx context.Context, ic inode.Core) (in inode.Inode, err error) { - // Sanity check. if err := ic.SanityCheck(); err != nil { panic(err.Error()) } @@ -878,17 +1106,18 @@ func (fs *fileSystem) lookUpOrCreateInodeIfNotStale(ic inode.Core) (in inode.Ino // Handle Folders in hierarchical bucket. if ic.Folder != nil { - return fs.createDirInode(ic, fs.folderInodes) + return fs.createDirInode(ic, fs.folderInodes, parInodeCtx) } // Handle implicit directories. if ic.MinObject == nil { - return fs.createDirInode(ic, fs.implicitDirInodes) + return fs.createDirInode(ic, fs.implicitDirInodes, parInodeCtx) } oGen := inode.Generation{ Object: ic.MinObject.Generation, Metadata: ic.MinObject.MetaGeneration, + Size: ic.MinObject.Size, } // Retry loop for the stale index entry case below. On entry, we hold fs.mu @@ -899,11 +1128,14 @@ func (fs *fileSystem) lookUpOrCreateInodeIfNotStale(ic inode.Core) (in inode.Ino // If we have no existing record, mint an inode and return it. if !ok { - in = fs.mintInode(ic) + in, err = fs.mintInode(ic, parInodeCtx) + if err != nil { + return nil, err + } fs.generationBackedInodes[in.Name()] = in.(inode.GenerationBackedInode) in.Lock() - return + return in, nil } // Otherwise we need to read the inode's source generation below, which @@ -929,6 +1161,15 @@ func (fs *fileSystem) lookUpOrCreateInodeIfNotStale(ic inode.Core) (in inode.Ino return } + // The existing inode has the same generation but a different size. + // Update the size and return the existing inode. + if cmp == 2 { + logger.Warnf("The size of object has changed remotely at the same generation. Updating the existing inode to reflect the size change.\n") + existingInode.UpdateSize(oGen.Size) + in = existingInode + return + } + // The existing inode is newer than the backing object. The caller // should call again with a newer backing object. if cmp == -1 { @@ -946,7 +1187,10 @@ func (fs *fileSystem) lookUpOrCreateInodeIfNotStale(ic inode.Core) (in inode.Ino // lock in accordance with our lock ordering rules. existingInode.Unlock() - in = fs.mintInode(ic) + in, err = fs.mintInode(ic, parInodeCtx) + if err != nil { + return nil, err + } fs.generationBackedInodes[in.Name()] = in.(inode.GenerationBackedInode) continue @@ -967,7 +1211,10 @@ func (fs *fileSystem) lookUpOrCreateChildInode( parent inode.DirInode, childName string) (child inode.Inode, err error) { // First check if the requested child is a localFileInode. - child = fs.lookUpLocalFileInode(parent, childName) + child, err = fs.lookUpLocalFileInode(parent, childName) + if err != nil { + return nil, err + } if child != nil { return } @@ -992,7 +1239,7 @@ func (fs *fileSystem) lookUpOrCreateChildInode( // Run a retry loop around lookUpOrCreateInodeIfNotStale. const maxTries = 3 - for n := 0; n < maxTries; n++ { + for range maxTries { // Create a record. var core *inode.Core core, err = getLookupResult() @@ -1007,7 +1254,10 @@ func (fs *fileSystem) lookUpOrCreateChildInode( } // Attempt to create the inode. Return if successful. - child = fs.lookUpOrCreateInodeIfNotStale(*core) + child, err = fs.lookUpOrCreateInodeIfNotStale(parent.Context(), *core) + if err != nil { + return nil, err + } if child != nil { return } @@ -1023,7 +1273,17 @@ func (fs *fileSystem) lookUpOrCreateChildInode( // LOCKS_EXCLUDED(parent) // UNLOCK_FUNCTION(fs.mu) // LOCK_FUNCTION(child) -func (fs *fileSystem) lookUpLocalFileInode(parent inode.DirInode, childName string) (child inode.Inode) { +func (fs *fileSystem) lookUpLocalFileInode(parent inode.DirInode, childName string) (child inode.Inode, err error) { + // If the path specified is "a/\n", the child would come as \n which is not a valid childname. + // In such cases, simply return a file-not-found. + if childName == inode.ConflictingFileNameSuffix { + return nil, syscall.ENOENT + } + // Trim the suffix assigned to fix conflicting names. + childName = strings.TrimSuffix(childName, inode.ConflictingFileNameSuffix) + fileName := inode.NewFileName(parent.Name(), childName) + + fs.mu.Lock() defer func() { if child != nil { child.IncrementLookupCount() @@ -1031,13 +1291,8 @@ func (fs *fileSystem) lookUpLocalFileInode(parent inode.DirInode, childName stri fs.mu.Unlock() }() - // Trim the suffix assigned to fix conflicting names. - childName = strings.TrimSuffix(childName, inode.ConflictingFileNameSuffix) - fileName := inode.NewFileName(parent.Name(), childName) - - fs.mu.Lock() var maxTriesToLookupInode = 3 - for n := 0; n < maxTriesToLookupInode; n++ { + for range maxTriesToLookupInode { child = fs.localFileInodes[fileName] if child == nil { @@ -1104,53 +1359,154 @@ func (fs *fileSystem) lookUpOrCreateChildDirInode( return child, nil } -// Synchronize the supplied file inode to GCS, updating the index as +// promoteToGenerationBacked updates the file system maps for the given file inode +// after it has been synced to GCS. +// The inode is removed from the localFileInodes map and added to the +// generationBackedInodes map. +// +// LOCKS_EXCLUDED(fs.mu) +// LOCKS_REQUIRED(f) +func (fs *fileSystem) promoteToGenerationBacked(f *inode.FileInode) { + fs.mu.Lock() + delete(fs.localFileInodes, f.Name()) + if _, ok := fs.generationBackedInodes[f.Name()]; !ok { + fs.generationBackedInodes[f.Name()] = f + } + fs.mu.Unlock() + + // We need not update fileIndex: + // + // We've held the inode lock the whole time, so there's no way that this + // inode could have been booted from the index. Therefore, if it's not in the + // index at the moment, it must not have been in there when we started. That + // is, it must have been clobbered remotely. + // + // In other words, either this inode is still in the index or it has been + // clobbered and *should* be anonymous. +} + +// Flushes the supplied file inode to GCS, updating the index as +// appropriate. +// +// LOCKS_EXCLUDED(fs.mu) +// LOCKS_REQUIRED(f) +func (fs *fileSystem) flushFile( + ctx context.Context, + f *inode.FileInode) error { + // FlushFile mirrors the behavior of native filesystems by not returning an error + // when file to be synced has been unlinked from the same mount. + if f.IsUnlinked() { + return nil + } + + fs.mu.Lock() + parInode := fs.findParentDirInode(f.Name()) + fs.mu.Unlock() + if parInode != nil { + // Increment active writers so no new prefetch gets triggered until write operation completes. + parInode.IncrementActiveWriters() + defer parInode.DecrementActiveWriters() + // Cancel current directory's prefetch so stale data is not updated in cache. + parInode.CancelCurrDirPrefetcher() + } + + // Flush the inode. + err := f.Flush(ctx) + if err != nil { + err = fmt.Errorf("FileInode.Sync: %w", err) + // If the inode was local file inode, treat it as unlinked. + fs.mu.Lock() + delete(fs.localFileInodes, f.Name()) + fs.mu.Unlock() + return err + } + + // Promote the inode to generationBackedInodes in fs maps. + fs.promoteToGenerationBacked(f) + return nil +} + +// Synchronizes the supplied file inode to GCS, updating the index as // appropriate. // // LOCKS_EXCLUDED(fs.mu) // LOCKS_REQUIRED(f) func (fs *fileSystem) syncFile( ctx context.Context, - f *inode.FileInode) (err error) { - // SyncFile can be triggered for unlinked files if the fileHandle is open by - // same or another user. Silently ignore the syncFile call. - // This is in sync with non-local file behaviour. - if f.IsLocal() && f.IsUnlinked() { - return + f *inode.FileInode) error { + // SyncFile mirrors the behavior of native filesystems by not returning an error + // when file to be synced has been unlinked from the same mount. + if f.IsUnlinked() { + return nil + } + + fs.mu.Lock() + parInode := fs.findParentDirInode(f.Name()) + fs.mu.Unlock() + if parInode != nil { + // Increment active writers so no new prefetch gets triggered until write operation completes. + parInode.IncrementActiveWriters() + defer parInode.DecrementActiveWriters() + // Cancel current directory's prefetch so stale data is not updated in cache. + parInode.CancelCurrDirPrefetcher() } // Sync the inode. - err = f.Sync(ctx) + gcsSynced, err := f.Sync(ctx) if err != nil { err = fmt.Errorf("FileInode.Sync: %w", err) // If the inode was local file inode, treat it as unlinked. fs.mu.Lock() delete(fs.localFileInodes, f.Name()) fs.mu.Unlock() - return + return err } - // Once the inode is synced to GCS, it is no longer an localFileInode. - // Delete the entry from localFileInodes map and add it to generationBackedInodes. - fs.mu.Lock() - delete(fs.localFileInodes, f.Name()) - _, ok := fs.generationBackedInodes[f.Name()] - if !ok { - fs.generationBackedInodes[f.Name()] = f + // If gcsSynced is true, it means the inode was fully synced to GCS In this + // case, we need to promote the inode to generationBackedInodes in fs maps. + if gcsSynced { + fs.promoteToGenerationBacked(f) } - fs.mu.Unlock() + return nil +} - // We need not update fileIndex: - // - // We've held the inode lock the whole time, so there's no way that this - // inode could have been booted from the index. Therefore if it's not in the - // index at the moment, it must not have been in there when we started. That - // is, it must have been clobbered remotely, which we treat as unlinking. - // - // In other words, either this inode is still in the index or it has been - // unlinked and *should* be anonymous. +// Initializes Buffered Write Handler if Eligible and synchronizes the file inode to GCS if initialization succeeds. +// Otherwise creates an empty temp writer if temp file nil. +// +// LOCKS_EXCLUDED(fs.mu) +// LOCKS_REQUIRED(f.mu) +func (fs *fileSystem) createBufferedWriteHandlerAndSyncOrTempWriter(ctx context.Context, f *inode.FileInode, openMode util.OpenMode) error { + err := fs.initBufferedWriteHandlerAndSyncFileIfEligible(ctx, f, openMode) + if err != nil { + return err + } + err = f.CreateEmptyTempFile(ctx) + if err != nil { + return err + } + return nil +} - return +// Initializes Buffered Write Handler if Eligible and synchronizes the file inode to GCS if initialization succeeds. +// +// LOCKS_EXCLUDED(fs.mu) +// LOCKS_REQUIRED(f.mu) +func (fs *fileSystem) initBufferedWriteHandlerAndSyncFileIfEligible(ctx context.Context, f *inode.FileInode, openMode util.OpenMode) error { + initialized, err := f.InitBufferedWriteHandlerIfEligible(ctx, openMode) + if err != nil { + return err + } + if initialized { + // Calling syncFile is safe here as we have file inode lock and BWH is initialized. + // Thus sync method of BWH will be invoked. + // 1. In case of zonal bucket it creates unfinalized new generation object. + // 2. In case of non zonal bucket it's no-op as we don't have pending buffers to upload. + err = fs.syncFile(ctx, f) + if err != nil { + return err + } + } + return nil } // Decrement the supplied inode's lookup count, destroying it if the inode says @@ -1248,7 +1604,7 @@ func (fs *fileSystem) getAttributes( expiration time.Time, err error) { // Call through. - attr, err = in.Attributes(ctx) + attr, err = in.Attributes(ctx, true) if err != nil { return } @@ -1319,79 +1675,251 @@ func (fs *fileSystem) symlinkInodeOrDie( // invalidateChildFileCacheIfExist invalidates the file in read cache. This is used to // invalidate the file in read cache after deletion of original file. -// -// LOCKS_REQUIRED(fs.mu) func (fs *fileSystem) invalidateChildFileCacheIfExist(parentInode inode.DirInode, objectGCSName string) (err error) { - if fs.fileCacheHandler != nil { - if bucketOwnedDirInode, ok := parentInode.(inode.BucketOwnedDirInode); ok { - bucketName := bucketOwnedDirInode.Bucket().Name() - // Invalidate the file cache entry if it exists. - err := fs.fileCacheHandler.InvalidateCache(objectGCSName, bucketName) + if bucketOwnedDirInode, ok := parentInode.(inode.BucketOwnedDirInode); ok { + bucketName := bucketOwnedDirInode.Bucket().Name() + // Invalidate the file cache entry if it exists. + if fs.fileCacheHandler != nil { + err = fs.fileCacheHandler.InvalidateCache(objectGCSName, bucketName) if err != nil { return fmt.Errorf("invalidateChildFileCacheIfExist: while invalidating the file cache: %w", err) } - } else { - // The parentInode is not owned by any bucket, which means it's the base - // directory that holds all the buckets' root directories. So, this op - // is to delete a bucket, which is not supported. - return fmt.Errorf("invalidateChildFileCacheIfExist: not an BucketOwnedDirInode: %w", syscall.ENOTSUP) } + } else { + // The parentInode is not owned by any bucket, which means it's the base + // directory that holds all the buckets' root directories. So, this op + // is to delete a bucket, which is not supported. + return fmt.Errorf("invalidateChildFileCacheIfExist: not an BucketOwnedDirInode: %w", syscall.ENOTSUP) } return nil } -//////////////////////////////////////////////////////////////////////// -// fuse.FileSystem methods -//////////////////////////////////////////////////////////////////////// +// coreToDirentPlus creates a fuseutil.DirentPlus entry from an inode core. +// LOCKS_EXCLUDED(fs.mu) +func (fs *fileSystem) coreToDirentPlus(ctx context.Context, fullName inode.Name, core inode.Core, parInodeCtx context.Context) (entryPlus *fuseutil.DirentPlus, err error) { + // Look up or create the inode for the core. + child, err := fs.lookUpOrCreateInodeIfNotStale(parInodeCtx, core) + if err != nil { + return nil, fmt.Errorf("coreToDirentPlus: lookUpOrCreateInodeIfNotStale: %w", err) + } + if child == nil { + return nil, fmt.Errorf("coreToDirentPlus: stale record for %s", path.Base(fullName.LocalName())) + } + defer child.Unlock() -func (fs *fileSystem) Destroy() { - fs.bucketManager.ShutDown() - if fs.fileCacheHandler != nil { - _ = fs.fileCacheHandler.Destroy() + // Extract the child's attributes. + attributes, err := child.Attributes(ctx, false) + if err != nil { + // The inode is valid, but we couldn't get attributes. + return nil, fmt.Errorf("coreToDirentPlus: unable to fetch attributes for %s: %w", path.Base(fullName.LocalName()), err) } -} -func (fs *fileSystem) StatFS( - ctx context.Context, - op *fuseops.StatFSOp) (err error) { - // Simulate a large amount of free space so that the Finder doesn't refuse to - // copy in files. (See issue #125.) Use 2^17 as the block size because that - // is the largest that OS X will pass on. - op.BlockSize = 1 << 17 - op.Blocks = 1 << 33 - op.BlocksFree = op.Blocks - op.BlocksAvailable = op.Blocks + expiration := time.Now().Add(fs.inodeAttributeCacheTTL) + entryPlus = &fuseutil.DirentPlus{ + Dirent: fuseutil.Dirent{ + Name: path.Base(fullName.LocalName()), + Type: fuseutil.DT_Unknown, + Inode: child.ID(), + }, + Entry: fuseops.ChildInodeEntry{ + Child: child.ID(), + Attributes: attributes, + AttributesExpiration: expiration, + }, + } + if fs.newConfig.FileSystem.ExperimentalEnableDentryCache { + entryPlus.Entry.EntryExpiration = expiration + } - // Similarly with inodes. - op.Inodes = 1 << 50 - op.InodesFree = op.Inodes + // Set the directory entry type based on the core type. + switch core.Type() { + case metadata.SymlinkType: + entryPlus.Dirent.Type = fuseutil.DT_Link + case metadata.RegularFileType: + entryPlus.Dirent.Type = fuseutil.DT_File + case metadata.ImplicitDirType, metadata.ExplicitDirType: + entryPlus.Dirent.Type = fuseutil.DT_Directory + } - // Prefer large transfers. This is the largest value that OS X will - // faithfully pass on, according to fuseops/ops.go. - op.IoSize = 1 << 20 + return entryPlus, nil +} +// LocalFileEntries lists the local files (file that is not yet present on GCS) present in the directory. +// For each entry, only the Dirent field is populated; the ChildInodeEntry field is not set. +// LOCKS_REQUIRED(fs.mu) +func (fs *fileSystem) localFileEntriesPlus(parent inode.Name) (localEntriesPlus map[string]fuseutil.DirentPlus) { + localEntriesPlus = make(map[string]fuseutil.DirentPlus) + + for localInodeName, in := range fs.localFileInodes { + // It is possible that the local file inode has been unlinked, but + // still present in localFileInodes map because of open file handle. + // So, if the inode has been unlinked, skip the entry. + file, ok := in.(*inode.FileInode) + if ok && file.IsUnlinked() { + continue + } + if localInodeName.IsDirectChildOf(parent) { + entry := fuseutil.DirentPlus{ + Dirent: fuseutil.Dirent{ + Name: path.Base(localInodeName.LocalName()), + Type: fuseutil.DT_File, + Inode: in.ID(), + }, + } + localEntriesPlus[entry.Dirent.Name] = entry + } + } return } +// lookupAndFetchAttributesForLocalFileEntriesPlus performs a lookup for each local file entry, +// fetches its attributes, and updates the corresponding DirentPlus.Entry field. // LOCKS_EXCLUDED(fs.mu) -func (fs *fileSystem) LookUpInode( - ctx context.Context, - op *fuseops.LookUpInodeOp) (err error) { - if fs.newConfig.FileSystem.IgnoreInterrupts { - // When ignore interrupts config is set, we are creating a new context not - // cancellable by parent context. - var cancel context.CancelFunc - ctx, cancel = util.IsolateContextFromParentContext(ctx) - defer cancel() - } - // Find the parent directory in question. - fs.mu.Lock() - parent := fs.dirInodeOrDie(op.Parent) - fs.mu.Unlock() - - // Find or create the child inode. - child, err := fs.lookUpOrCreateChildInode(ctx, parent, op.Name) +func (fs *fileSystem) lookupAndFetchAttributesForLocalFileEntriesPlus(parentName inode.DirInode, localFileEntriesPlus map[string]fuseutil.DirentPlus) (err error) { + for localEntryName, localEntryPlus := range localFileEntriesPlus { + // Lookup the child inode under the parent directory. + child, err := fs.lookUpLocalFileInode(parentName, localEntryName) + if err != nil { + return fmt.Errorf("lookupAndFetchAttributesForLocalFileEntriesPlus: while looking up local file %q: %w", localEntryName, err) + } + if child == nil { + // This indicates a potential race condition where the file was removed after being listed. + return fmt.Errorf("lookupAndFetchAttributesForLocalFileEntriesPlus: local file %q disappeared", localEntryName) + } + // Fetch attributes from the child inode. + attrs, err := child.Attributes(context.Background(), false) + if err != nil { + child.Unlock() + return fmt.Errorf("lookupAndFetchAttributesForLocalFileEntriesPlus: unable to fetch attributes for %s: %w", localEntryName, err) + } + // Unlock the inode after retrieving its attributes. + child.Unlock() + + expiration := time.Now().Add(fs.inodeAttributeCacheTTL) + childInodeEntry := fuseops.ChildInodeEntry{ + Child: child.ID(), + Attributes: attrs, + AttributesExpiration: expiration, + } + if fs.newConfig.FileSystem.ExperimentalEnableDentryCache { + childInodeEntry.EntryExpiration = expiration + } + localEntryPlus.Entry = childInodeEntry + localFileEntriesPlus[localEntryName] = localEntryPlus + } + return +} + +// invalidateCachedEntry sends a notification to the kernel to invalidate a stale +// directory entry, ensuring consistency when file content changes dynamically. +// It identifies the parent of the given child inode and sends a notification +// to the kernel to remove the entry corresponding to the child's +// name within that parent directory. +// +// LOCKS_EXCLUDED(fs.mu) +func (fs *fileSystem) invalidateCachedEntry(childID fuseops.InodeID) error { + fs.mu.Lock() + childInode, ok := fs.inodes[childID] + fs.mu.Unlock() + if !ok { + return fmt.Errorf("invalidateCachedEntry: inode with ID %d not found", childID) + } + + childName := childInode.Name() + parentPath := path.Dir(childName.LocalName()) + // If the parent path resolves to the current directory ".", it means the parent + // is the root of the file system. + if parentPath == "." { + return fs.notifier.InvalidateEntry(fuseops.RootInodeID, path.Base(childInode.Name().LocalName())) + } + + parentName, err := childName.ParentName() + if err != nil { + return fmt.Errorf("invalidateCachedEntry: cannot find Parent name: %w", err) + } + childBase := path.Base(childName.LocalName()) + + fs.mu.Lock() + defer fs.mu.Unlock() + + var parentInodeID fuseops.InodeID + // Check in all maps: implicit dirs → folders → generation-backed + if parentInode, ok := fs.implicitDirInodes[parentName]; ok { + parentInodeID = parentInode.ID() + } else if parentInode, ok := fs.folderInodes[parentName]; ok { + parentInodeID = parentInode.ID() + } else if parentInode, ok := fs.generationBackedInodes[parentName]; ok { + parentInodeID = parentInode.ID() + } else { + return fmt.Errorf("invalidateCachedEntry: failed to invalidate the entry, parent inode not found for child ID %d (parent: %s)", childID, parentName.String()) + } + + return fs.notifier.InvalidateEntry(parentInodeID, childBase) +} + +//////////////////////////////////////////////////////////////////////// +// fuse.FileSystem methods +//////////////////////////////////////////////////////////////////////// + +func (fs *fileSystem) Destroy() { + fs.bucketManager.ShutDown() + if fs.fileCacheHandler != nil { + _ = fs.fileCacheHandler.Destroy() + } + if fs.bufferedReadWorkerPool != nil { + fs.bufferedReadWorkerPool.Stop() + } +} + +func (fs *fileSystem) StatFS( + ctx context.Context, + op *fuseops.StatFSOp) (err error) { + // Simulate a large amount of free space so that the Finder doesn't refuse to + // copy in files. (See issue #125.) Use 2^17 as the block size because that + // is the largest that OS X will pass on. + op.BlockSize = 1 << 17 + op.Blocks = 1 << 33 + op.BlocksFree = op.Blocks + op.BlocksAvailable = op.Blocks + + // Similarly with inodes. + op.Inodes = 1 << 50 + op.InodesFree = op.Inodes + + // Prefer large transfers. This is the largest value that OS X will + // faithfully pass on, according to fuseops/ops.go. + op.IoSize = 1 << 20 + + return +} + +// getInterruptlessContext returns a new context that is not cancellable by the +// parent context if the ignore-interrupts flag is set. Otherwise, it returns +// the original context. +func (fs *fileSystem) getInterruptlessContext(ctx context.Context) context.Context { + if fs.newConfig.FileSystem.IgnoreInterrupts { + // When ignore interrupts config is set, we are creating a new context not + // cancellable by parent context. + newCtx := context.Background() + return fs.traceHandle.PropagateTraceContext(newCtx, ctx) + } + + return ctx +} + +// LOCKS_EXCLUDED(fs.mu) +func (fs *fileSystem) LookUpInode( + ctx context.Context, + op *fuseops.LookUpInodeOp) (err error) { + ctx = fs.getInterruptlessContext(ctx) + // Find the parent directory in question. + fs.mu.Lock() + parent := fs.dirInodeOrDie(op.Parent) + fs.mu.Unlock() + + // Find or create the child inode. + child, err := fs.lookUpOrCreateChildInode(ctx, parent, op.Name) if err != nil { return err } @@ -1402,6 +1930,9 @@ func (fs *fileSystem) LookUpInode( e := &op.Entry e.Child = child.ID() e.Attributes, e.AttributesExpiration, err = fs.getAttributes(ctx, child) + if fs.newConfig.FileSystem.ExperimentalEnableDentryCache { + e.EntryExpiration = e.AttributesExpiration + } if err != nil { return err @@ -1414,13 +1945,7 @@ func (fs *fileSystem) LookUpInode( func (fs *fileSystem) GetInodeAttributes( ctx context.Context, op *fuseops.GetInodeAttributesOp) (err error) { - if fs.newConfig.FileSystem.IgnoreInterrupts { - // When ignore interrupts config is set, we are creating a new context not - // cancellable by parent context. - var cancel context.CancelFunc - ctx, cancel = util.IsolateContextFromParentContext(ctx) - defer cancel() - } + ctx = fs.getInterruptlessContext(ctx) // Find the inode. fs.mu.Lock() in := fs.inodeOrDie(op.Inode) @@ -1442,13 +1967,7 @@ func (fs *fileSystem) GetInodeAttributes( func (fs *fileSystem) SetInodeAttributes( ctx context.Context, op *fuseops.SetInodeAttributesOp) (err error) { - if fs.newConfig.FileSystem.IgnoreInterrupts { - // When ignore interrupts config is set, we are creating a new context not - // cancellable by parent context. - var cancel context.CancelFunc - ctx, cancel = util.IsolateContextFromParentContext(ctx) - defer cancel() - } + ctx = fs.getInterruptlessContext(ctx) // Find the inode. fs.mu.Lock() in := fs.inodeOrDie(op.Inode) @@ -1469,7 +1988,17 @@ func (fs *fileSystem) SetInodeAttributes( // Truncate files. if isFile && op.Size != nil { - err = file.Truncate(ctx, int64(*op.Size)) + // Initialize BWH if eligible and Sync file inode. + err = fs.initBufferedWriteHandlerAndSyncFileIfEligible(ctx, file, util.NewOpenMode(util.WriteOnly, 0)) + if err != nil { + return + } + gcsSynced, err := file.Truncate(ctx, int64(*op.Size)) + // Sync the inode if finalize during truncate is successful + // even if the truncate operation later resulted error. + if gcsSynced { + fs.promoteToGenerationBacked(file) + } if err != nil { err = fmt.Errorf("truncate: %w", err) return err @@ -1508,13 +2037,7 @@ func (fs *fileSystem) ForgetInode( func (fs *fileSystem) MkDir( ctx context.Context, op *fuseops.MkDirOp) (err error) { - if fs.newConfig.FileSystem.IgnoreInterrupts { - // When ignore interrupts config is set, we are creating a new context not - // cancellable by parent context. - var cancel context.CancelFunc - ctx, cancel = util.IsolateContextFromParentContext(ctx) - defer cancel() - } + ctx = fs.getInterruptlessContext(ctx) // Find the parent. fs.mu.Lock() parent := fs.dirInodeOrDie(op.Parent) @@ -1542,7 +2065,10 @@ func (fs *fileSystem) MkDir( // Attempt to create a child inode using the object we created. If we fail to // do so, it means someone beat us to the punch with a newer generation // (unlikely, so we're probably okay with failing here). - child := fs.lookUpOrCreateInodeIfNotStale(*result) + child, err := fs.lookUpOrCreateInodeIfNotStale(parent.Context(), *result) + if err != nil { + return err + } if child == nil { err = fmt.Errorf("newly-created record is already stale") return err @@ -1567,19 +2093,13 @@ func (fs *fileSystem) MkDir( func (fs *fileSystem) MkNode( ctx context.Context, op *fuseops.MkNodeOp) (err error) { - if fs.newConfig.FileSystem.IgnoreInterrupts { - // When ignore interrupts config is set, we are creating a new context not - // cancellable by parent context. - var cancel context.CancelFunc - ctx, cancel = util.IsolateContextFromParentContext(ctx) - defer cancel() - } + ctx = fs.getInterruptlessContext(ctx) if (op.Mode & (iofs.ModeNamedPipe | iofs.ModeSocket)) != 0 { return syscall.ENOTSUP } // Create the child. - child, err := fs.createFile(ctx, op.Parent, op.Name, op.Mode) + child, err := fs.createFile(ctx, op.Parent, op.Name) if err != nil { return err } @@ -1607,8 +2127,7 @@ func (fs *fileSystem) MkNode( func (fs *fileSystem) createFile( ctx context.Context, parentID fuseops.InodeID, - name string, - mode os.FileMode) (child inode.Inode, err error) { + name string) (child inode.Inode, err error) { // Find the parent. fs.mu.Lock() parent := fs.dirInodeOrDie(parentID) @@ -1636,7 +2155,10 @@ func (fs *fileSystem) createFile( // Attempt to create a child inode using the object we created. If we fail to // do so, it means someone beat us to the punch with a newer generation // (unlikely, so we're probably okay with failing here). - child = fs.lookUpOrCreateInodeIfNotStale(*result) + child, err = fs.lookUpOrCreateInodeIfNotStale(parent.Context(), *result) + if err != nil { + return nil, err + } if child == nil { err = fmt.Errorf("newly-created record is already stale") return @@ -1649,15 +2171,16 @@ func (fs *fileSystem) createFile( // LOCKS_EXCLUDED(fs.mu) // UNLOCK_FUNCTION(fs.mu) // LOCK_FUNCTION(child) -func (fs *fileSystem) createLocalFile( - parentID fuseops.InodeID, - name string) (child inode.Inode, err error) { +func (fs *fileSystem) createLocalFile(ctx context.Context, parentID fuseops.InodeID, name string, openMode util.OpenMode) (child inode.Inode, err error) { // Find the parent. fs.mu.Lock() parent := fs.dirInodeOrDie(parentID) defer func() { if err != nil { + if child == nil { + return + } // fs.mu lock is already taken delete(fs.localFileInodes, child.Name()) } @@ -1683,24 +2206,26 @@ func (fs *fileSystem) createLocalFile( if err != nil { return } - child = fs.mintInode(core) - fs.localFileInodes[child.Name()] = child - // For buffered writes, we don't create temp files on disk. - if !fs.newConfig.Write.ExperimentalEnableStreamingWrites { - // Empty file is created to be able to set attributes on the file. - fileInode := child.(*inode.FileInode) - if err := fileInode.CreateEmptyTempFile(); err != nil { - return nil, err - } + child, err = fs.mintInode(core, parent.Context()) + if err != nil { + return nil, err } + fs.localFileInodes[child.Name()] = child + fileInode := child.(*inode.FileInode) + // Use deferred locking on filesystem so that it is locked before the defer call that unlocks the mutex and it doesn't fail. + // We need to release the filesystem lock before acquiring the inode lock. fs.mu.Unlock() + defer fs.mu.Lock() + fileInode.Lock() + err = fs.createBufferedWriteHandlerAndSyncOrTempWriter(ctx, fileInode, openMode) + fileInode.Unlock() + if err != nil { + return + } parent.Lock() defer parent.Unlock() parent.InsertFileIntoTypeCache(name) - // Even though there is no action here that requires locking, adding locking - // so that the defer call that unlocks the mutex doesn't fail. - fs.mu.Lock() return child, nil } @@ -1708,19 +2233,14 @@ func (fs *fileSystem) createLocalFile( func (fs *fileSystem) CreateFile( ctx context.Context, op *fuseops.CreateFileOp) (err error) { - if fs.newConfig.FileSystem.IgnoreInterrupts { - // When ignore interrupts config is set, we are creating a new context not - // cancellable by parent context. - var cancel context.CancelFunc - ctx, cancel = util.IsolateContextFromParentContext(ctx) - defer cancel() - } + ctx = fs.getInterruptlessContext(ctx) // Create the child. var child inode.Inode + openMode := util.FileOpenMode(op.OpenFlags) if fs.newConfig.Write.CreateEmptyFile { - child, err = fs.createFile(ctx, op.Parent, op.Name, op.Mode) + child, err = fs.createFile(ctx, op.Parent, op.Name) } else { - child, err = fs.createLocalFile(op.Parent, op.Name) + child, err = fs.createLocalFile(ctx, op.Parent, op.Name, openMode) } if err != nil { @@ -1732,11 +2252,24 @@ func (fs *fileSystem) CreateFile( // Allocate a handle. fs.mu.Lock() - handleID := fs.nextHandleID + op.Handle = fs.nextHandleID fs.nextHandleID++ - fs.handles[handleID] = handle.NewFileHandle(child.(*inode.FileInode), fs.fileCacheHandler, fs.cacheFileForRangeRead) - op.Handle = handleID + // CreateFile() invoked to create new files, can be safely considered as filehandle + // opened in append mode. + fs.handles[op.Handle] = handle.NewFileHandle( + child.(*inode.FileInode), + fs.fileCacheHandler, + fs.sharedChunkCacheManager, + fs.cacheFileForRangeRead, + fs.metricHandle, + fs.traceHandle, + openMode, + fs.newConfig, + fs.bufferedReadWorkerPool, + fs.globalMaxReadBlocksSem, + op.Handle, + ) fs.mu.Unlock() @@ -1757,13 +2290,7 @@ func (fs *fileSystem) CreateFile( func (fs *fileSystem) CreateSymlink( ctx context.Context, op *fuseops.CreateSymlinkOp) (err error) { - if fs.newConfig.FileSystem.IgnoreInterrupts { - // When ignore interrupts config is set, we are creating a new context not - // cancellable by parent context. - var cancel context.CancelFunc - ctx, cancel = util.IsolateContextFromParentContext(ctx) - defer cancel() - } + ctx = fs.getInterruptlessContext(ctx) // Find the parent. fs.mu.Lock() parent := fs.dirInodeOrDie(op.Parent) @@ -1790,7 +2317,10 @@ func (fs *fileSystem) CreateSymlink( // Attempt to create a child inode using the object we created. If we fail to // do so, it means someone beat us to the punch with a newer generation // (unlikely, so we're probably okay with failing here). - child := fs.lookUpOrCreateInodeIfNotStale(*result) + child, err := fs.lookUpOrCreateInodeIfNotStale(parent.Context(), *result) + if err != nil { + return err + } if child == nil { err = fmt.Errorf("newly-created record is already stale") return err @@ -1827,13 +2357,7 @@ func (fs *fileSystem) RmDir( ctx context.Context, op *fuseops.RmDirOp) (err error) { - if fs.newConfig.FileSystem.IgnoreInterrupts { - // When ignore interrupts config is set, we are creating a new context not - // cancellable by parent context. - var cancel context.CancelFunc - ctx, cancel = util.IsolateContextFromParentContext(ctx) - defer cancel() - } + ctx = fs.getInterruptlessContext(ctx) // Find the parent. fs.mu.Lock() parent := fs.dirInodeOrDie(op.Parent) @@ -1888,12 +2412,24 @@ func (fs *fileSystem) RmDir( var tok string for { var entries []fuseutil.Dirent - entries, tok, err = childDir.ReadEntries(ctx, tok) + var unsupportedPaths []string + entries, unsupportedPaths, tok, err = childDir.ReadEntries(ctx, tok) if err != nil { err = fmt.Errorf("ReadEntries: %w", err) return err } + // TODO: Remove this check once we gain confidence that it is not causing any issues. + if fs.newConfig.EnableUnsupportedPathSupport { + // If there are unsupported objects, delete them recursively. + if len(unsupportedPaths) > 0 { + err = childDir.DeleteObjects(ctx, unsupportedPaths) + if err != nil { + return fmt.Errorf("RmDir: failed to delete unsupported objects: %w", err) + } + } + } + if fs.kernelListCacheTTL > 0 { // Clear kernel list cache after removing a directory. This ensures remote // GCS files are included in future directory listings for unlinking. @@ -1935,69 +2471,119 @@ func (fs *fileSystem) RmDir( func (fs *fileSystem) Rename( ctx context.Context, op *fuseops.RenameOp) (err error) { - if fs.newConfig.FileSystem.IgnoreInterrupts { - // When ignore interrupts config is set, we are creating a new context not - // cancellable by parent context. - var cancel context.CancelFunc - ctx, cancel = util.IsolateContextFromParentContext(ctx) - defer cancel() - } + ctx = fs.getInterruptlessContext(ctx) // Find the old and new parents. fs.mu.Lock() oldParent := fs.dirInodeOrDie(op.OldParent) newParent := fs.dirInodeOrDie(op.NewParent) fs.mu.Unlock() - if oldInode, ok := oldParent.(inode.BucketOwnedInode); !ok { + if oldParentInode, ok := oldParent.(inode.BucketOwnedInode); !ok { // The old parent is not owned by any bucket, which means it's the base // directory that holds all the buckets' root directories. So, this op // is to rename a bucket, which is not supported. return fmt.Errorf("rename a bucket: %w", syscall.ENOTSUP) } else { // The target path must exist in the same bucket. - oldBucket := oldInode.Bucket().Name() - if newInode, ok := newParent.(inode.BucketOwnedInode); !ok || oldBucket != newInode.Bucket().Name() { + oldBucket := oldParentInode.Bucket().Name() + if newParentInode, ok := newParent.(inode.BucketOwnedInode); !ok || oldBucket != newParentInode.Bucket().Name() { return fmt.Errorf("move out of bucket %q: %w", oldBucket, syscall.ENOTSUP) } } - // If object to be renamed is a local file inode (un-synced), rename operation is not supported. - localChild := fs.lookUpLocalFileInode(oldParent, op.OldName) - if localChild != nil { - fs.unlockAndDecrementLookupCount(localChild, 1) - return fmt.Errorf("cannot rename open file %q: %w", op.OldName, syscall.ENOTSUP) - } - - // Else find the object in the old location (on GCS). - oldParent.Lock() - child, err := oldParent.LookUpChild(ctx, op.OldName) - oldParent.Unlock() - + child, err := fs.lookUpOrCreateChildInode(ctx, oldParent, op.OldName) if err != nil { - err = fmt.Errorf("LookUpChild: %w", err) return err } - if child == nil { - err = fuse.ENOENT - return err + return fuse.ENOENT + } + child.DecrementLookupCount(1) + child.Unlock() + + childBktOwned, ok := child.(inode.BucketOwnedInode) + if !ok { // Won't happen in ideal case. + return fmt.Errorf("child inode (id %v) is not owned by any bucket", child.ID()) } - if child.FullName.IsDir() { + if child.Name().IsDir() { // If 'enable-hns' flag is false, the bucket type is set to 'NonHierarchical' even for HNS buckets because the control client is nil. // Therefore, an additional 'enable hns' check is not required here. - if child.Bucket.BucketType() == gcs.Hierarchical { + if childBktOwned.Bucket().BucketType().Hierarchical { return fs.renameHierarchicalDir(ctx, oldParent, op.OldName, newParent, op.NewName) } return fs.renameNonHierarchicalDir(ctx, oldParent, op.OldName, newParent, op.NewName) } - return fs.renameFile(ctx, oldParent, op.OldName, child.MinObject, newParent, op.NewName) + + return fs.renameFile(ctx, op, childBktOwned, oldParent, newParent) +} + +// LOCKS_EXCLUDED(oldParent) +// LOCKS_EXCLUDED(newParent) +func (fs *fileSystem) renameFile(ctx context.Context, op *fuseops.RenameOp, child inode.BucketOwnedInode, oldParent, newParent inode.DirInode) error { + var updatedMinObject *gcs.MinObject + var err error + + switch c := child.(type) { + case *inode.FileInode: + updatedMinObject, err = fs.flushPendingWrites(ctx, c) + if err != nil { + return fmt.Errorf("flushPendingWrites: %w", err) + } + case *inode.SymlinkInode: + updatedMinObject = c.Source() + default: + return fmt.Errorf("child inode (id %v) is not a file or symlink inode", child.ID()) + } + if fs.enableAtomicRenameObject || child.Bucket().BucketType().Zonal { + return fs.atomicRename(ctx, oldParent, op.OldName, updatedMinObject, newParent, op.NewName) + } + return fs.nonAtomicRename(ctx, oldParent, op.OldName, updatedMinObject, newParent, op.NewName) +} + +// LOCKS_EXCLUDED(fileInode) +func (fs *fileSystem) flushPendingWrites(ctx context.Context, fileInode *inode.FileInode) (minObject *gcs.MinObject, err error) { + // We will return modified minObject if flush is done, otherwise the original + // minObject is returned. Original minObject is the one passed in the request. + fileInode.Lock() + defer fileInode.Unlock() + minObject = fileInode.Source() + // Try to flush if there are any pending writes. + err = fs.flushFile(ctx, fileInode) + minObject = fileInode.Source() + return +} + +// LOCKS_EXCLUDED(oldParent) +// LOCKS_EXCLUDED(newParent) +func (fs *fileSystem) atomicRename(ctx context.Context, oldParent inode.DirInode, oldName string, oldObject *gcs.MinObject, newParent inode.DirInode, newName string) error { + oldParent.Lock() + defer oldParent.Unlock() + + if newParent != oldParent { + newParent.Lock() + defer newParent.Unlock() + } + + newFileName := inode.NewFileName(newParent.Name(), newName) + + if _, err := oldParent.RenameFile(ctx, oldObject, newFileName.GcsObjectName()); err != nil { + return fmt.Errorf("renameFile: while renaming file: %w", err) + } + + if err := fs.invalidateChildFileCacheIfExist(oldParent, oldName); err != nil { + return fmt.Errorf("atomicRename: while invalidating cache for renamed file: %w", err) + } + + // Insert new file in type cache. + newParent.InsertFileIntoTypeCache(newName) + + return nil } -// LOCKS_EXCLUDED(fs.mu) // LOCKS_EXCLUDED(oldParent) // LOCKS_EXCLUDED(newParent) -func (fs *fileSystem) renameFile( +func (fs *fileSystem) nonAtomicRename( ctx context.Context, oldParent inode.DirInode, oldName string, @@ -2017,24 +2603,32 @@ func (fs *fileSystem) renameFile( // Delete behind. Make sure to delete exactly the generation we cloned, in // case the referent of the name has changed in the meantime. oldParent.Lock() - err = oldParent.DeleteChildFile( + defer oldParent.Unlock() + + deleteErr := oldParent.DeleteChildFile( ctx, oldName, oldObject.Generation, &oldObject.MetaGeneration) - if err := fs.invalidateChildFileCacheIfExist(oldParent, oldObject.Name); err != nil { - return fmt.Errorf("renameFile: while invalidating cache for delete file: %w", err) + // In case the delete is successful or a precondition error is encountered, + // then file cache becomes unusable, thus, should be invalidated. + // File cache must not be invalidated in case of any other errors encountered + // while deletion. + if deleteErr == nil { + if invErr := fs.invalidateChildFileCacheIfExist(oldParent, oldName); invErr != nil { + logger.Warnf("File cache eviction failed after successful delete on GCS: %v", invErr) + } + return nil } - oldParent.Unlock() - - if err != nil { - err = fmt.Errorf("DeleteChildFile: %w", err) - return err + var precondErr *gcs.PreconditionError + if errors.As(deleteErr, &precondErr) { + if invErr := fs.invalidateChildFileCacheIfExist(oldParent, oldName); invErr != nil { + logger.Warnf("File cache eviction failed after precondition error during delete: %v", invErr) + } } - - return nil + return fmt.Errorf("DeleteChildFile: %w", deleteErr) } func (fs *fileSystem) releaseInodes(inodes *[]inode.DirInode) { @@ -2063,18 +2657,6 @@ func (fs *fileSystem) ensureNoLocalFilesInDirectory(dir inode.BucketOwnedDirInod return nil } -func (fs *fileSystem) checkDirNotEmpty(dir inode.BucketOwnedDirInode, name string) error { - unexpected, err := dir.ReadDescendants(context.Background(), 1) - if err != nil { - return fmt.Errorf("read descendants of the new directory %q: %w", name, err) - } - - if len(unexpected) > 0 { - return fuse.ENOTEMPTY - } - return nil -} - // Rename an old folder to a new folder in a hierarchical bucket. If the new folder already // exists and is non-empty, return ENOTEMPTY. If old folder have open files then return // ENOTSUP. @@ -2105,18 +2687,22 @@ func (fs *fileSystem) renameHierarchicalDir(ctx context.Context, oldParent inode // If the call for getBucketDirInode fails it means directory does not exist. newDirInode, err := fs.getBucketDirInode(ctx, newParent, newName) if err == nil { - // If the directory exists, then check if it is empty or not. - if err = fs.checkDirNotEmpty(newDirInode, newName); err != nil { - return err - } + pendingInodes = append(pendingInodes, newDirInode) - // This refers to an empty destination directory. - // The RenameFolder API does not allow renaming to an existing empty directory. - // To make this work, we delete the empty directory first from gcsfuse and then perform rename. + // The RenameFolder API does not allow renaming to an empty existing directory. + // To make this work, we attempt to delete the destination directory first. + // If it is non-empty, this deletion will fail with a PreconditionError, + // in which case we immediately return ENOTEMPTY. newParent.Lock() - _ = newParent.DeleteChildDir(ctx, newName, false, newDirInode) + deleteErr := newParent.DeleteChildDir(ctx, newName, false, newDirInode) newParent.Unlock() - pendingInodes = append(pendingInodes, newDirInode) + if deleteErr != nil { + var precondErr *gcs.PreconditionError + if errors.As(deleteErr, &precondErr) { + return fuse.ENOTEMPTY + } + return fmt.Errorf("DeleteChildDir: %w", deleteErr) + } } // Note:The renameDirLimit is not utilized in the folder rename operation because there is no user-defined limit on new renames. @@ -2129,7 +2715,7 @@ func (fs *fileSystem) renameHierarchicalDir(ctx context.Context, oldParent inode } // Rename old directory to the new directory, keeping both parent directories locked. - _, err = oldParent.RenameFolder(ctx, oldDirName.GcsObjectName(), newDirName.GcsObjectName()) + _, err = oldParent.RenameFolder(ctx, oldDirName.GcsObjectName(), newDirName.GcsObjectName(), oldDirInode) if err != nil { return fmt.Errorf("failed to rename folder: %w", err) } @@ -2137,6 +2723,18 @@ func (fs *fileSystem) renameHierarchicalDir(ctx context.Context, oldParent inode return } +func (fs *fileSystem) checkDirNotEmpty(dir inode.BucketOwnedDirInode, name string) error { + unexpected, err := dir.ReadDescendants(context.Background(), 1) + if err != nil { + return fmt.Errorf("read descendants of the new directory %q: %w", name, err) + } + + if len(unexpected) > 0 { + return fuse.ENOTEMPTY + } + return nil +} + // Rename an old directory to a new directory in a non-hierarchical bucket. If the new directory already // exists and is non-empty, return ENOTEMPTY. // @@ -2207,11 +2805,26 @@ func (fs *fileSystem) renameNonHierarchicalDir( } o := descendant.MinObject - if _, err := newDir.CloneToChildFile(ctx, nameDiff, o); err != nil { - return fmt.Errorf("copy file %q: %w", o.Name, err) - } - if err := oldDir.DeleteChildFile(ctx, nameDiff, o.Generation, &o.MetaGeneration); err != nil { - return fmt.Errorf("delete file %q: %w", o.Name, err) + // Use copy-delete if atomic rename is disabled, or if the object is a directory or of unknown type. + // Otherwise, for files with atomic rename enabled, use move. + isDirOrUnknown := descendant.Type() == metadata.ExplicitDirType || descendant.Type() == metadata.UnknownType + if !fs.enableAtomicRenameObject || isDirOrUnknown { + if _, err = newDir.CloneToChildFile(ctx, nameDiff, o); err != nil { + return fmt.Errorf("copy file %q: %w", o.Name, err) + } + if err = oldDir.DeleteChildFile(ctx, nameDiff, o.Generation, &o.MetaGeneration); err != nil { + return fmt.Errorf("delete file %q: %w", o.Name, err) + } + } else { + // For regular files, perform an in-place rename by constructing the new GCS object name. + // Standard path.Join is avoided here because object names in GCS are distinct from + // directory prefixes; the "/" character is *always* treated as a separate directory + // element, not part of the object's base name. This manual approach correctly + // handles those GCS naming edge cases (like objects with unsupported characters). + newObject := newDir.Name().GcsObjectName() + nameDiff + if _, err = oldDir.RenameFile(ctx, o, newObject); err != nil { + return fmt.Errorf("renameFile: while renaming file: %w", err) + } } if err = fs.invalidateChildFileCacheIfExist(oldDir, o.Name); err != nil { @@ -2239,49 +2852,62 @@ func (fs *fileSystem) renameNonHierarchicalDir( func (fs *fileSystem) Unlink( ctx context.Context, op *fuseops.UnlinkOp) (err error) { - if fs.newConfig.FileSystem.IgnoreInterrupts { - // When ignore interrupts config is set, we are creating a new context not - // cancellable by parent context. - var cancel context.CancelFunc - ctx, cancel = util.IsolateContextFromParentContext(ctx) - defer cancel() - } - // Find the parent. + ctx = fs.getInterruptlessContext(ctx) + fs.mu.Lock() + + // Find the parent and file name. parent := fs.dirInodeOrDie(op.Parent) + fileName := inode.NewFileName(parent.Name(), op.Name) + + // Get the inode for the given file. + // Files must have an associated inode, which can be found in either: + // - localFileInodes: For files created locally. + // - generationBackedInodes: For files backed by an object. + // We are not checking implicitDirInodes or folderInodes because + // the unlink operation is only applicable to files. + in, isLocalFile := fs.localFileInodes[fileName] + if !isLocalFile { + in = fs.generationBackedInodes[fileName] + } + fs.mu.Unlock() - // if inode is a local file, mark it unlinked. - fileName := inode.NewFileName(parent.Name(), op.Name) - fs.mu.Lock() - fileInode, ok := fs.localFileInodes[fileName] - if ok { - file := fs.fileInodeOrDie(fileInode.ID()) - fs.mu.Unlock() - file.Lock() - defer file.Unlock() - file.Unlink() + if in != nil { + // Perform the unlink operation on the inode. + in.Lock() + in.Unlink() + in.Unlock() + } + + // If the inode represents a local file, we don't need to delete + // the backing object on GCS, so return early. + if isLocalFile { return } - fs.mu.Unlock() - // else delete the backing object present on GCS. + // Delete the backing object present on GCS. parent.Lock() defer parent.Unlock() - // Delete the backing object. err = parent.DeleteChildFile( ctx, op.Name, 0, // Latest generation nil) // No meta-generation precondition - if err != nil { - err = fmt.Errorf("DeleteChildFile: %w", err) - return err + var preconditionErr *gcs.PreconditionError + // Do not invalidate the file cache in case of error while deleting file + // which is not precondition error. + if err != nil && !errors.As(err, &preconditionErr) { + return fmt.Errorf("DeleteChildFile: %w", err) } + // In case of successful delete or precondition error, we should invalidate + // the file cache as it is no longer usable. if err := fs.invalidateChildFileCacheIfExist(parent, fileName.GcsObjectName()); err != nil { + // In case of file cache invalidation error, we should return the error, + // but still consider the unlink operation successful. return fmt.Errorf("unlink: while invalidating cache for delete file: %w", err) } @@ -2323,13 +2949,7 @@ func (fs *fileSystem) OpenDir( func (fs *fileSystem) ReadDir( ctx context.Context, op *fuseops.ReadDirOp) (err error) { - if fs.newConfig.FileSystem.IgnoreInterrupts { - // When ignore interrupts config is set, we are creating a new context not - // cancellable by parent context. - var cancel context.CancelFunc - ctx, cancel = util.IsolateContextFromParentContext(ctx) - defer cancel() - } + ctx = fs.getInterruptlessContext(ctx) // Find the handle. fs.mu.Lock() dh := fs.handles[op.Handle].(*handle.DirHandle) @@ -2349,6 +2969,51 @@ func (fs *fileSystem) ReadDir( return } +// LOCKS_EXCLUDED(fs.mu) +func (fs *fileSystem) ReadDirPlus(ctx context.Context, op *fuseops.ReadDirPlusOp) (err error) { + ctx = fs.getInterruptlessContext(ctx) + // Find the handle. + fs.mu.Lock() + dh := fs.handles[op.Handle].(*handle.DirHandle) + in := fs.dirInodeOrDie(op.Inode) + // Fetch local file entries beforehand for passing it to directory handle as + // we need fs lock to fetch local file entries. + localFileEntriesPlus := fs.localFileEntriesPlus(in.Name()) + // Unlock fs lock and fetch attributes for local file entries as it requires inode lock. + fs.mu.Unlock() + + err = fs.lookupAndFetchAttributesForLocalFileEntriesPlus(in, localFileEntriesPlus) + if err != nil { + return err + } + + dh.Mu.Lock() + defer dh.Mu.Unlock() + // Serve the request. + var cores map[inode.Name]*inode.Core + cores, err = dh.FetchEntryCores(ctx, op) + if err != nil { + return fmt.Errorf("FetchDirCores: %w", err) + } + // dh.mu lock is not required during iteration over cores, but was acquired earlier + // for code readability and to use a common defer unlock pattern. Holding the + // lock here has no performance overhead. + var entriesPlus []fuseutil.DirentPlus + for fullName, core := range cores { + entry, err := fs.coreToDirentPlus(ctx, fullName, *core, in.Context()) + if err != nil { + return err + } + entriesPlus = append(entriesPlus, *entry) + } + + if err := dh.ReadDirPlus(op, entriesPlus, localFileEntriesPlus); err != nil { + return err + } + + return nil +} + // LOCKS_EXCLUDED(fs.mu) func (fs *fileSystem) ReleaseDirHandle( ctx context.Context, @@ -2369,18 +3034,49 @@ func (fs *fileSystem) ReleaseDirHandle( func (fs *fileSystem) OpenFile( ctx context.Context, op *fuseops.OpenFileOp) (err error) { + // For file handles opened in O_DIRECT mode, we must set UseDirectIO. + // This tells the kernel to bypass its page cache (for file reads and + // writes) and **size checks**, forwarding read requests directly to + // GCSFuse. This is critical for tailing reads on remotely updated files. + // This should be enabled for all file handles being opened, if direct + // IO is configured at file system level. + if fs.newConfig.FileSystem.ExperimentalODirect || op.OpenFlags.IsDirect() { + op.UseDirectIO = true + } + fs.mu.Lock() - defer fs.mu.Unlock() // Find the inode. in := fs.fileInodeOrDie(op.Inode) + // Follow lock ordering rules to get inode lock. + // Inode lock is required to register fileHandle with the inode. + fs.mu.Unlock() + in.Lock() + defer in.Unlock() + + // Get the fs lock again. + fs.mu.Lock() + defer fs.mu.Unlock() // Allocate a handle. - handleID := fs.nextHandleID + op.Handle = fs.nextHandleID fs.nextHandleID++ - fs.handles[handleID] = handle.NewFileHandle(in, fs.fileCacheHandler, fs.cacheFileForRangeRead) - op.Handle = handleID + // Figure out the mode in which the file is being opened. + openMode := util.FileOpenMode(op.OpenFlags) + fs.handles[op.Handle] = handle.NewFileHandle( + in, + fs.fileCacheHandler, + fs.sharedChunkCacheManager, + fs.cacheFileForRangeRead, + fs.metricHandle, + fs.traceHandle, + openMode, + fs.newConfig, + fs.bufferedReadWorkerPool, + fs.globalMaxReadBlocksSem, + op.Handle, + ) // When we observe object generations that we didn't create, we assign them // new inode IDs. So for a given inode, all modifications go through the @@ -2395,27 +3091,69 @@ func (fs *fileSystem) OpenFile( func (fs *fileSystem) ReadFile( ctx context.Context, op *fuseops.ReadFileOp) (err error) { - if fs.newConfig.FileSystem.IgnoreInterrupts { - // When ignore interrupts config is set, we are creating a new context not - // cancellable by parent context. - var cancel context.CancelFunc - ctx, cancel = util.IsolateContextFromParentContext(ctx) - defer cancel() - } - // Save readOp in context for access in logs. - ctx = context.WithValue(ctx, gcsx.ReadOp, op) + ctx = fs.getInterruptlessContext(ctx) // Find the handle and lock it. fs.mu.Lock() fh := fs.handles[op.Handle].(*handle.FileHandle) fs.mu.Unlock() - fh.Lock() - defer fh.Unlock() + fh.Inode().Lock() + if fh.Inode().IsUsingBWH() { + // Flush/Sync Pending streaming writes and issue read within same inode lock. + // With rapid buckets, we can read from unfinalized objects as well. + // Hence, there is no need to finalize the object from here for rapid buckets. + // Hence, if FinalizeFileOnClose is set, then we will call syncFile otherwise + // we can call flushFile (as it will not finalize when FinalizeFileOnClose is false) itself. + if fh.Inode().Bucket().BucketType().RapidWritesEnabled() && fs.newConfig.Write.FinalizeFileOnClose { + err = fs.syncFile(ctx, fh.Inode()) + } else { + err = fs.flushFile(ctx, fh.Inode()) + } + if err != nil { + fh.Inode().Unlock() + return err + } + } // Serve the read. - op.BytesRead, err = fh.Read(ctx, op.Dst, op.Offset, fs.sequentialReadSizeMb) + if fs.newConfig.FileSystem.EnableKernelReader { + var resp gcsx.ReadResponse + req := &gcsx.ReadRequest{ + Buffer: op.Dst, + Offset: op.Offset, + } + resp, err = fh.ReadWithMrdKernelReader(ctx, req) + op.BytesRead = resp.Size + op.Data = resp.Data + op.Callback = resp.Callback + } else if fs.newConfig.EnableNewReader { + var resp gcsx.ReadResponse + req := &gcsx.ReadRequest{ + Buffer: op.Dst, + Offset: op.Offset, + } + resp, err = fh.ReadWithReadManager(ctx, req, fs.sequentialReadSizeMb) + op.BytesRead = resp.Size + op.Data = resp.Data + op.Callback = resp.Callback + } else { + op.Dst, op.BytesRead, err = fh.Read(ctx, op.Dst, op.Offset, fs.sequentialReadSizeMb) + } + + // A FileClobberedError indicates the underlying GCS object has changed, + // making the kernel's dentry for this file stale. We use the notifier to + // invalidate this entry, providing feedback to the kernel about the dynamic + // content change and ensuring subsequent lookups fetch the correct metadata. + if fs.newConfig.FileSystem.ExperimentalEnableDentryCache { + var clobberedErr *gcsfuse_errors.FileClobberedError + if err != nil && errors.As(err, &clobberedErr) { + if invalidateErr := fs.invalidateCachedEntry(op.Inode); invalidateErr != nil { + err = fmt.Errorf("%w; additionally failed to invalidate entry: %w", err, invalidateErr) + } + } + } // As required by fuse, we don't treat EOF as an error. if err == io.EOF { err = nil @@ -2446,26 +3184,42 @@ func (fs *fileSystem) ReadSymlink( func (fs *fileSystem) WriteFile( ctx context.Context, op *fuseops.WriteFileOp) (err error) { - if fs.newConfig.FileSystem.IgnoreInterrupts { - // When ignore interrupts config is set, we are creating a new context not - // cancellable by parent context. - var cancel context.CancelFunc - ctx, cancel = util.IsolateContextFromParentContext(ctx) - defer cancel() - } - // Find the inode. + ctx = fs.getInterruptlessContext(ctx) + + // Find the inode( and file handle in case of appends). fs.mu.Lock() + fh := fs.handles[op.Handle].(*handle.FileHandle) in := fs.fileInodeOrDie(op.Inode) fs.mu.Unlock() + var gcsSynced bool in.Lock() defer in.Unlock() - - // Serve the request. - if err := in.Write(ctx, op.Data, op.Offset); err != nil { + if err = fs.initBufferedWriteHandlerAndSyncFileIfEligible(ctx, in, fh.OpenMode()); err != nil { + // A FileClobberedError on write indicates the file was modified in GCS, + // making the kernel's dentry stale. By invalidating the cache + // entry, we ensure the filesystem corrects the inconsistency caused by this + // dynamic content change. + if fs.newConfig.FileSystem.ExperimentalEnableDentryCache { + var clobberedErr *gcsfuse_errors.FileClobberedError + if errors.As(err, &clobberedErr) { + if invalidateErr := fs.invalidateCachedEntry(op.Inode); invalidateErr != nil { + err = fmt.Errorf("%w; additionally failed to invalidate entry: %w", err, invalidateErr) + } + } + } return err } - + // Serve the request. + gcsSynced, err = in.Write(ctx, op.Data, op.Offset, fh.OpenMode()) + if err != nil { + return + } + // Sync the inode if finalize during write is successful + // even if the write operation later resulted in error. + if gcsSynced { + fs.promoteToGenerationBacked(in) + } return } @@ -2473,13 +3227,7 @@ func (fs *fileSystem) WriteFile( func (fs *fileSystem) SyncFile( ctx context.Context, op *fuseops.SyncFileOp) (err error) { - if fs.newConfig.FileSystem.IgnoreInterrupts { - // When ignore interrupts config is set, we are creating a new context not - // cancellable by parent context. - var cancel context.CancelFunc - ctx, cancel = util.IsolateContextFromParentContext(ctx) - defer cancel() - } + ctx = fs.getInterruptlessContext(ctx) // Find the inode. fs.mu.Lock() in := fs.inodeOrDie(op.Inode) @@ -2506,13 +3254,7 @@ func (fs *fileSystem) SyncFile( func (fs *fileSystem) FlushFile( ctx context.Context, op *fuseops.FlushFileOp) (err error) { - if fs.newConfig.FileSystem.IgnoreInterrupts { - // When ignore interrupts config is set, we are creating a new context not - // cancellable by parent context. - var cancel context.CancelFunc - ctx, cancel = util.IsolateContextFromParentContext(ctx) - defer cancel() - } + ctx = fs.getInterruptlessContext(ctx) // Find the inode. fs.mu.Lock() in := fs.fileInodeOrDie(op.Inode) @@ -2522,7 +3264,7 @@ func (fs *fileSystem) FlushFile( defer in.Unlock() // Sync it. - if err := fs.syncFile(ctx, in); err != nil { + if err := fs.flushFile(ctx, in); err != nil { return err } @@ -2534,13 +3276,17 @@ func (fs *fileSystem) ReleaseFileHandle( ctx context.Context, op *fuseops.ReleaseFileHandleOp) (err error) { fs.mu.Lock() - defer fs.mu.Unlock() - - // Destroy the handle. - fs.handles[op.Handle].(*handle.FileHandle).Destroy() - // Update the map. + fileHandle := fs.handles[op.Handle].(*handle.FileHandle) + // Update the map. We are okay updating the map before destroy is called + // since destroy is doing only internal cleanup. delete(fs.handles, op.Handle) + fs.mu.Unlock() + + // Destroy the handle. + fileHandle.Lock() + defer fileHandle.Unlock() + fileHandle.Destroy() return } @@ -2556,3 +3302,9 @@ func (fs *fileSystem) ListXattr( op *fuseops.ListXattrOp) error { return syscall.ENOSYS } + +func (fs *fileSystem) SyncFS( + ctx context.Context, + op *fuseops.SyncFSOp) error { + return syscall.ENOSYS +} diff --git a/internal/fs/fs_internal_test.go b/internal/fs/fs_internal_test.go new file mode 100644 index 0000000000..74ec63dc56 --- /dev/null +++ b/internal/fs/fs_internal_test.go @@ -0,0 +1,69 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util/diskutil" + "github.com/stretchr/testify/assert" +) + +func TestCacheDirVolumeBlockSize(t *testing.T) { + cacheDir := t.TempDir() + actualBlockSize := diskutil.GetVolumeBlockSize(cacheDir) + + for _, tc := range []struct { + name string + disableSizeCalculationFix bool + enableExperimentalChunkCache bool + expectedBlockSize uint64 + }{ + { + name: "SizeCalcFixEnabled_NotSparse", + disableSizeCalculationFix: false, + enableExperimentalChunkCache: false, + expectedBlockSize: actualBlockSize, + }, + { + name: "SizeCalcFixDisabled", + disableSizeCalculationFix: true, + enableExperimentalChunkCache: false, + expectedBlockSize: 1, + }, + { + name: "SparseModeEnabled", + disableSizeCalculationFix: false, + enableExperimentalChunkCache: true, + expectedBlockSize: 1, + }, + } { + t.Run(tc.name, func(t *testing.T) { + serverCfg := &ServerConfig{ + NewConfig: &cfg.Config{ + FileCache: cfg.FileCacheConfig{ + ExperimentalDisableSizeCalculationFix: tc.disableSizeCalculationFix, + ExperimentalEnableChunkCache: tc.enableExperimentalChunkCache, + }, + }, + } + + blockSize := cacheDirVolumeBlockSize(serverCfg, cacheDir) + + assert.Equal(t, tc.expectedBlockSize, blockSize) + }) + } +} diff --git a/internal/fs/fs_test.go b/internal/fs/fs_test.go index e9f24d3bb7..1515741e3d 100644 --- a/internal/fs/fs_test.go +++ b/internal/fs/fs_test.go @@ -29,15 +29,17 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/fs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" - "github.com/googlecloudplatform/gcsfuse/v2/internal/locker" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" - "github.com/googlecloudplatform/gcsfuse/v2/internal/perms" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/fake" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/locker" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/perms" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" "github.com/jacobsa/fuse" "github.com/jacobsa/fuse/fusetesting" . "github.com/jacobsa/ogletest" @@ -128,20 +130,22 @@ func (t *fsTest) SetUpTestSuite() { mtimeClock = timeutil.RealClock() cacheClock.SetTime(time.Date(2015, 4, 5, 2, 15, 0, 0, time.Local)) t.serverCfg.CacheClock = &cacheClock - if bucketType == gcs.Nil { - bucketType = gcs.NonHierarchical - } + + mountCfg := t.mountCfg + mountCfg.OpContext = ctx if buckets != nil { // mount all buckets bucket = nil t.serverCfg.BucketName = "" + mountCfg.FSName = "gcsfuse" } else { // mount a single bucket if bucket == nil { bucket = fake.NewFakeBucket(mtimeClock, "some_bucket", bucketType) } t.serverCfg.BucketName = bucket.Name() + mountCfg.FSName = bucket.Name() buckets = map[string]gcs.Bucket{bucket.Name(): bucket} } @@ -149,10 +153,13 @@ func (t *fsTest) SetUpTestSuite() { // This bucket manager is allowed to open these buckets buckets: buckets, // Configs for the syncer when setting up buckets - appendThreshold: 0, - tmpObjectPrefix: ".gcsfuse_tmp/", + appendThreshold: 0, + chunkTransferTimeoutSecs: 10, + tmpObjectPrefix: ".gcsfuse_tmp/", + } + if t.serverCfg.RenameDirLimit == 0 { + t.serverCfg.RenameDirLimit = RenameDirLimit } - t.serverCfg.RenameDirLimit = RenameDirLimit t.serverCfg.SequentialReadSizeMb = SequentialReadSizeMb if t.serverCfg.NewConfig == nil { @@ -160,12 +167,15 @@ func (t *fsTest) SetUpTestSuite() { FileCache: defaultFileCacheConfig(), MetadataCache: cfg.MetadataCacheConfig{ // Setting default values. - StatCacheMaxSizeMb: 32, + StatCacheMaxSizeMb: 33, TtlSecs: 60, TypeCacheMaxSizeMb: 4, }, + EnableNewReader: true, } } + t.serverCfg.MetricHandle = metrics.NewNoopMetrics() + t.serverCfg.TraceHandle = tracing.NewNoopTracer() // Set up ownership. t.serverCfg.Uid, t.serverCfg.Gid, err = perms.MyUserAndGroup() @@ -183,18 +193,15 @@ func (t *fsTest) SetUpTestSuite() { server, err := fs.NewServer(ctx, &t.serverCfg) AssertEq(nil, err) - // Mount the file system. - mountCfg := t.mountCfg - mountCfg.OpContext = ctx - + // Initialize Fuse Loggers. if mountCfg.ErrorLogger == nil { - mountCfg.ErrorLogger = logger.NewLegacyLogger(logger.LevelError, "fuse_errors: ") + mountCfg.ErrorLogger = logger.NewLegacyLogger(logger.LevelError, "fuse_errors: ", mountCfg.FSName) } if *fDebug { - mountCfg.DebugLogger = logger.NewLegacyLogger(logger.LevelDebug, "fuse: ") + mountCfg.DebugLogger = logger.NewLegacyLogger(logger.LevelDebug, "fuse: ", mountCfg.FSName) } - + // Mount the file system. mfs, err = fuse.Mount(mntDir, server, &mountCfg) AssertEq(nil, err) } @@ -226,7 +233,7 @@ func (t *fsTest) TearDownTestSuite() { // Unlink the mount point. if err = os.Remove(mntDir); err != nil { - err = fmt.Errorf("Unlinking mount point: %w", err) + logger.Errorf("Unlinking mount point: %v", err) return } @@ -284,9 +291,8 @@ func (t *fsTest) createEmptyObjects(names []string) error { } func (t *fsTest) createFolders(folders []string) error { - for i := 0; i < len(folders); i++ { - _, err = bucket.CreateFolder(ctx, folders[i]) - if err != nil { + for i := range folders { + if _, err := bucket.CreateFolder(ctx, folders[i]); err != nil { return err } } @@ -359,20 +365,24 @@ func currentGid() uint32 { } type fakeBucketManager struct { - buckets map[string]gcs.Bucket - appendThreshold int64 - tmpObjectPrefix string + buckets map[string]gcs.Bucket + appendThreshold int64 + chunkRetryDeadlineSecs int64 + chunkTransferTimeoutSecs int64 + tmpObjectPrefix string } func (bm *fakeBucketManager) ShutDown() {} func (bm *fakeBucketManager) SetUpBucket( ctx context.Context, - name string, isMultibucketMount bool) (sb gcsx.SyncerBucket, err error) { + name string, isMultibucketMount bool, _ metrics.MetricHandle) (sb gcsx.SyncerBucket, err error) { bucket, ok := bm.buckets[name] if ok { sb = gcsx.NewSyncerBucket( bm.appendThreshold, + bm.chunkRetryDeadlineSecs, + bm.chunkTransferTimeoutSecs, bm.tmpObjectPrefix, gcsx.NewContentTypeBucket(bucket), ) diff --git a/internal/fs/gcs_metrics_test.go b/internal/fs/gcs_metrics_test.go new file mode 100644 index 0000000000..dc3a5e25df --- /dev/null +++ b/internal/fs/gcs_metrics_test.go @@ -0,0 +1,465 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs_test + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/wrappers" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/monitor" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/jacobsa/fuse/fuseops" + "github.com/jacobsa/fuse/fuseutil" + "github.com/jacobsa/timeutil" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/metric" + "google.golang.org/api/googleapi" +) + +type fakeBucketManagerWithMetrics struct { + buckets map[string]gcs.Bucket +} + +func (bm *fakeBucketManagerWithMetrics) SetUpBucket( + ctx context.Context, + name string, + isMultibucketMount bool, + mh metrics.MetricHandle) (sb gcsx.SyncerBucket, err error) { + bucket, ok := bm.buckets[name] + if !ok { + err = fmt.Errorf("Bucket %q does not exist", name) + return sb, err + } + + // Wrap bucket with monitor.NewMonitoringBucket to enable GCS metrics. + sb = gcsx.NewSyncerBucket( + /*appendThreshold=*/ 0, + /*chunkRetryDeadlineSecs=*/ 120, + /*chunkTransferTimeoutSecs=*/ 10, + ".gcsfuse_tmp/", + gcsx.NewContentTypeBucket(monitor.NewMonitoringBucket(bucket, mh)), + ) + return sb, err +} + +func (bm *fakeBucketManagerWithMetrics) ShutDown() {} + +func createTestFileSystemWithMonitoredBucket(ctx context.Context, t *testing.T, params *serverConfigParams) (gcs.Bucket, fuseutil.FileSystem, metrics.MetricHandle, *metric.ManualReader) { + t.Helper() + origProvider := otel.GetMeterProvider() + t.Cleanup(func() { otel.SetMeterProvider(origProvider) }) + reader := metric.NewManualReader() + provider := metric.NewMeterProvider(metric.WithReader(reader)) + otel.SetMeterProvider(provider) + + mh, err := metrics.NewOTelMetrics(ctx, 1, 100) + require.NoError(t, err, "metrics.NewOTelMetrics") + bucketName := "test-bucket" + bucket := fake.NewFakeBucket(timeutil.RealClock(), bucketName, gcs.BucketType{Hierarchical: false}) + serverCfg := &fs.ServerConfig{ + NewConfig: &cfg.Config{ + Write: cfg.WriteConfig{ + GlobalMaxBlocks: 1, + }, + Read: cfg.ReadConfig{ + EnableBufferedRead: params.enableBufferedRead, + GlobalMaxBlocks: 1, + BlockSizeMb: 1, + MaxBlocksPerHandle: 10, + }, + EnableNewReader: true, // Not much use testing the case where it's false + }, + MetricHandle: mh, + TraceHandle: tracing.NewNoopTracer(), + CacheClock: &timeutil.SimulatedClock{}, + BucketName: bucketName, + BucketManager: &fakeBucketManagerWithMetrics{ + buckets: map[string]gcs.Bucket{ + bucketName: bucket, + }, + }, + SequentialReadSizeMb: 200, + } + + if params.enableFileCache || params.enableSparseFileCache { + cacheDir := t.TempDir() + t.Cleanup(func() { + os.RemoveAll(cacheDir) + }) + serverCfg.NewConfig.CacheDir = cfg.ResolvedPath(cacheDir) + serverCfg.NewConfig.FileCache = cfg.FileCacheConfig{ + MaxSizeMb: 100, + CacheFileForRangeRead: true, + ExperimentalEnableChunkCache: params.enableSparseFileCache, + DownloadChunkSizeMb: 1, // 1MB chunks for testing + EnableParallelDownloads: params.enableParallelDownloads, + ExperimentalParallelDownloadsDefaultOn: params.enableParallelDownloadsBlocking, + ParallelDownloadsPerFile: 16, + } + } + + server, err := fs.NewFileSystem(ctx, serverCfg) + require.NoError(t, err, "NewFileSystem") + return bucket, server, mh, reader +} + +// TestGCSMetrics_RequestCount_StatObject validates the "gcs/request_count" metric for StatObject calls. +// +// Expected Behavior: +// - LookUpInode invokes StatObject 3 times in this test scenario: +// 1. Lookup Directory: Check if the object is a directory. +// 2. Lookup File: Check if the object itself exists. +// 3. Attribute Refresh: Fetch fresh attributes to ensure validity for the new inode. +// - GetInodeAttributes invokes StatObject 1 time to refresh attributes. +// - Therefore, we verify that "gcs/request_count" with "gcs_method=StatObject" is recorded as 4. +func TestGCSMetrics_RequestCount_StatObject(t *testing.T) { + ctx := context.Background() + bucket, server, _, reader := createTestFileSystemWithMonitoredBucket(ctx, t, defaultServerConfigParams()) + fileName := "test.txt" + createWithContents(ctx, t, bucket, fileName, "test") + lookupOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + + err := server.LookUpInode(ctx, lookupOp) + require.NoError(t, err) + waitForMetricsProcessing() + + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/request_count", + attribute.NewSet(attribute.String("gcs_method", "StatObject")), + 3) + + // Trigger another StatObject via GetInodeAttributes to verify stat count increments. + err = server.GetInodeAttributes(ctx, &fuseops.GetInodeAttributesOp{Inode: lookupOp.Entry.Child}) + require.NoError(t, err) + waitForMetricsProcessing() + + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/request_count", + attribute.NewSet(attribute.String("gcs_method", "StatObject")), + 4) // Previously 3, now incremented by 1 +} + +// TestGCSMetrics_RequestCount_CreateObject validates the "gcs/request_count" metric for CreateObject calls. +// +// Expected Behavior: +// - CreateFile alone creates a file handle and potentially a temporary object in the GCS bucket. +// - The actual upload (CreateObject GCS call) happens when the file is synced or closed. +// - We verify that "gcs/request_count" with "gcs_method=CreateObject" is incremented by 1 after the SyncFile operation. +func TestGCSMetrics_RequestCount_CreateObject(t *testing.T) { + ctx := context.Background() + _, server, mh, reader := createTestFileSystemWithMonitoredBucket(ctx, t, defaultServerConfigParams()) + server = wrappers.WithMonitoring(server, mh) + fileName := "new_file.txt" + + // CreateFile -> CreateObject + createOp := &fuseops.CreateFileOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + Mode: 0644, + } + err := server.CreateFile(ctx, createOp) + require.NoError(t, err) + // Sync or Close to trigger upload to GCS + syncOp := &fuseops.SyncFileOp{ + Inode: createOp.Entry.Child, + Handle: createOp.Handle, + } + err = server.SyncFile(ctx, syncOp) + require.NoError(t, err) + waitForMetricsProcessing() + + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/request_count", + attribute.NewSet(attribute.String("gcs_method", "CreateObject")), + 1) +} + +// TestGCSMetrics_RequestLatencies validates the "gcs/request_latencies" histogram metric. +// +// Expected Behavior: +// - Similar to TestGCSMetrics_RequestCount_StatObject, this operation triggers 3 StatObject calls. +// - We verify that the "gcs/request_latencies" histogram with "gcs_method=StatObject" has recorded 3 events. +// - This test ensures that latency tracking is active for GCS requests. +func TestGCSMetrics_RequestLatencies(t *testing.T) { + ctx := context.Background() + bucket, server, mh, reader := createTestFileSystemWithMonitoredBucket(ctx, t, defaultServerConfigParams()) + server = wrappers.WithMonitoring(server, mh) + fileName := "test.txt" + createWithContents(ctx, t, bucket, fileName, "test") + lookupOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + + err := server.LookUpInode(ctx, lookupOp) + require.NoError(t, err) + waitForMetricsProcessing() + + metrics.VerifyHistogramMetric(t, ctx, reader, "gcs/request_latencies", + attribute.NewSet(attribute.String("gcs_method", "StatObject")), + 3) +} + +// TestGCSMetrics_DownloadBytesCount_Explicit validates the "gcs/download_bytes_count" metric. +// +// Expected Behavior: +// - With buffered reading enabled, the file content is downloaded from GCS. +// - The length of the downloaded content (len("1234567890") = 10 bytes) should be recorded. +// - We verify that "gcs/download_bytes_count" with "read_type=Buffered" equals the file size. +// - This confirms that payload bytes from GCS response bodies are correctly counted. +func TestGCSMetrics_DownloadBytesCount_Explicit(t *testing.T) { + // Explicitly test gcs/download_bytes_count using monitored bucket. + ctx := context.Background() + params := defaultServerConfigParams() + params.enableBufferedRead = true + bucket, server, mh, reader := createTestFileSystemWithMonitoredBucket(ctx, t, params) + server = wrappers.WithMonitoring(server, mh) + fileName := "test.txt" + content := "1234567890" + createWithContents(ctx, t, bucket, fileName, content) + lookupOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookupOp) + require.NoError(t, err) + openOp := &fuseops.OpenFileOp{ + Inode: lookupOp.Entry.Child, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err) + + readOp := &fuseops.ReadFileOp{ + Inode: lookupOp.Entry.Child, + Handle: openOp.Handle, + Offset: 0, + Dst: make([]byte, len(content)), + } + + err = server.ReadFile(ctx, readOp) + require.NoError(t, err) + waitForMetricsProcessing() + + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/download_bytes_count", + attribute.NewSet(attribute.String("read_type", string(metrics.ReadTypeBufferedAttr))), + int64(len(content))) + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/reader_count", + attribute.NewSet(attribute.String("io_method", "opened")), + 1) + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/reader_count", + attribute.NewSet(attribute.String("io_method", "closed")), + 1) +} + +// TestGCSMetrics_With_FileCache validates GCS metrics behavior when file cache is enabled. +// +// Expected Behavior: +// 1. First Read (Cache Miss): +// - The file content is not in cache, so it must be downloaded from GCS. +// - File cache always uses "Sequential" read type for downloading. +// - "gcs/download_bytes_count" with "read_type=Sequential" should equal the file size. +// +// 2. Second Read (Cache Hit): +// - The file content is served from the local file cache. +// - No further GCS downloads should occur. +// - The "gcs/download_bytes_count" metric should remain unchanged. +func TestGCSMetrics_WithFileCache(t *testing.T) { + // TestGCSMetrics_WithFileCache verifies metrics when reading a file with file cache enabled. + ctx := context.Background() + params := defaultServerConfigParams() + params.enableFileCache = true + bucket, server, mh, reader := createTestFileSystemWithMonitoredBucket(ctx, t, params) + server = wrappers.WithMonitoring(server, mh) + fileName := "file_cache_miss.txt" + content := "file_cache_content" + createWithContents(ctx, t, bucket, fileName, content) + lookupOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + + err := server.LookUpInode(ctx, lookupOp) + require.NoError(t, err) + openOp := &fuseops.OpenFileOp{ + Inode: lookupOp.Entry.Child, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err) + readOp := &fuseops.ReadFileOp{ + Inode: lookupOp.Entry.Child, + Handle: openOp.Handle, + Offset: 0, + Dst: make([]byte, len(content)), + } + err = server.ReadFile(ctx, readOp) + require.NoError(t, err) + waitForMetricsProcessing() + + // Expect download bytes from GCS + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/download_bytes_count", + attribute.NewSet(attribute.String("read_type", "Sequential")), // File cache uses sequential read + int64(len(content))) + // gcs/read_count - Sequential + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/read_count", + attribute.NewSet(attribute.String("read_type", "Sequential")), + 1) + // gcs/reader_count - opened + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/reader_count", + attribute.NewSet(attribute.String("io_method", "opened")), + 1) + // gcs/reader_count - closed + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/reader_count", + attribute.NewSet(attribute.String("io_method", "closed")), + 1) + // gcs/read_bytes_count - 0 attributes + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/read_bytes_count", + attribute.NewSet(), + int64(len(content))) + + // Second Read - Should hit cache + readOp2 := &fuseops.ReadFileOp{ + Inode: lookupOp.Entry.Child, + Handle: openOp.Handle, + Offset: 0, + Dst: make([]byte, len(content)), + } + err = server.ReadFile(ctx, readOp2) + require.NoError(t, err) + waitForMetricsProcessing() + + // Count should still be the same (no new GCS downloads) + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/download_bytes_count", + attribute.NewSet(attribute.String("read_type", "Sequential")), + int64(len(content))) + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/read_count", + attribute.NewSet(attribute.String("read_type", "Sequential")), + 1) + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/reader_count", + attribute.NewSet(attribute.String("io_method", "opened")), + 1) + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/reader_count", + attribute.NewSet(attribute.String("io_method", "closed")), + 1) + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/read_bytes_count", + attribute.NewSet(), + int64(len(content))) +} + +// TestGCSMetrics_ParallelDownloads validates GCS metrics when parallel downloads are enabled. +// +// Expected Behavior: +// - With parallel downloads enabled, large files are downloaded in chunks in parallel. +// - We verify that "gcs/download_bytes_count" matches the file size. +// - We verify that "gcs/read_count" reflects the number of chunks downloaded (since each chunk triggers a GCS read). +func TestGCSMetrics_ParallelDownloads(t *testing.T) { + ctx := context.Background() + params := defaultServerConfigParams() + params.enableFileCache = true + params.enableParallelDownloads = true + // Enable blocking for parallel downloads to prevent fallback to GCS. + // Without this, the read operation might not wait for the async download to complete, + // triggering a redundant sequential GCS read and doubling the read metrics. + params.enableParallelDownloadsBlocking = true + bucket, server, mh, reader := createTestFileSystemWithMonitoredBucket(ctx, t, params) + server = wrappers.WithMonitoring(server, mh) + // Create a file larger than the chunk size (1MB) to trigger parallel downloads. + // 5MB file, 1MB chunks -> 5 chunks. + fileSize := 5 * 1024 * 1024 + fileName := "parallel_download.txt" + content := string(make([]byte, fileSize)) + createWithContents(ctx, t, bucket, fileName, content) + lookupOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookupOp) + require.NoError(t, err) + + openOp := &fuseops.OpenFileOp{ + Inode: lookupOp.Entry.Child, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err) + readOp := &fuseops.ReadFileOp{ + Inode: lookupOp.Entry.Child, + Handle: openOp.Handle, + Offset: 0, + Dst: make([]byte, fileSize), + } + err = server.ReadFile(ctx, readOp) + require.NoError(t, err) + waitForMetricsProcessing() + + // Verify download bytes count + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/download_bytes_count", + attribute.NewSet(attribute.String("read_type", "Parallel")), + int64(fileSize)) + // Verify read count. + // With parallel downloads and 1MB chunks, a 5MB file should trigger 5 GCS reads (one per chunk). + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/read_count", + attribute.NewSet(attribute.String("read_type", "Parallel")), + 5) + // Verify request count for NewReader (which corresponds to GetObject requests). + // Parallel downloads trigger multiple NewReader calls (one per chunk). + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/request_count", + attribute.NewSet(attribute.String("gcs_method", "NewReader")), + 5) + // Verify read bytes count + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/read_bytes_count", + attribute.NewSet(), + int64(fileSize)) +} + +// TestGCSMetrics_RetryCount validates the "gcs/retry_count" metric. +func TestGCSMetrics_RetryCount(t *testing.T) { + ctx := context.Background() + _, _, mh, reader := createTestFileSystemWithMonitoredBucket(ctx, t, defaultServerConfigParams()) + + // Simulate a retryable error (e.g. 429) + var err error = &googleapi.Error{Code: 429} + shouldRetry := storageutil.ShouldRetryWithMonitoring(ctx, err, mh) + require.True(t, shouldRetry) + + waitForMetricsProcessing() + + // Verify gcs/retry_count with retry_error_category="OTHER_ERRORS" + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/retry_count", + attribute.NewSet(attribute.String("retry_error_category", "OTHER_ERRORS")), + 1) + + // Simulate a DeadlineExceeded error (Stalled Read) + err = context.DeadlineExceeded + shouldRetry = storageutil.ShouldRetryWithMonitoring(ctx, err, mh) + require.True(t, shouldRetry) + + waitForMetricsProcessing() + + // Verify gcs/retry_count with retry_error_category="STALLED_READ_REQUEST" + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/retry_count", + attribute.NewSet(attribute.String("retry_error_category", "STALLED_READ_REQUEST")), + 1) +} diff --git a/internal/fs/gcsfuse_errors/gcsfuse_errors.go b/internal/fs/gcsfuse_errors/gcsfuse_errors.go new file mode 100644 index 0000000000..f2f5564a53 --- /dev/null +++ b/internal/fs/gcsfuse_errors/gcsfuse_errors.go @@ -0,0 +1,34 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsfuse_errors + +import ( + "fmt" +) + +// FileClobberedError represents a file clobbering scenario where a file was +// modified or deleted while it was being accessed. +type FileClobberedError struct { + Err error + ObjectName string +} + +func (fce *FileClobberedError) Error() string { + return fmt.Sprintf("The file %q was modified or deleted by another process, possibly due to concurrent modification: %v", fce.ObjectName, fce.Err) +} + +func (fce *FileClobberedError) Unwrap() error { + return fce.Err +} diff --git a/internal/fs/gcsfuse_errors/gcsfuse_errors_test.go b/internal/fs/gcsfuse_errors/gcsfuse_errors_test.go new file mode 100644 index 0000000000..8050d1e206 --- /dev/null +++ b/internal/fs/gcsfuse_errors/gcsfuse_errors_test.go @@ -0,0 +1,61 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsfuse_errors + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFileClobberedError(t *testing.T) { + testCases := []struct { + name string + objectName string + err error + wantErrMsg string + }{ + { + name: "with_underlying_error", + objectName: "a/b/c/foo.txt", + err: fmt.Errorf("some error"), + wantErrMsg: "The file \"a/b/c/foo.txt\" was modified or deleted by another process, possibly due to concurrent modification: some error", + }, + { + name: "without_underlying_error", + objectName: "bar.txt", + err: nil, + wantErrMsg: "The file \"bar.txt\" was modified or deleted by another process, possibly due to concurrent modification: <nil>", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + clobberedErr := &FileClobberedError{ + Err: tc.err, + ObjectName: tc.objectName, + } + + gotErrMsg := clobberedErr.Error() + + assert.Equal(t, tc.wantErrMsg, gotErrMsg) + if tc.err != nil { + assert.True(t, errors.Is(clobberedErr, tc.err)) + } + }) + } +} diff --git a/internal/fs/grpc_metrics_test.go b/internal/fs/grpc_metrics_test.go new file mode 100644 index 0000000000..09fba207e4 --- /dev/null +++ b/internal/fs/grpc_metrics_test.go @@ -0,0 +1,497 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs_test + +import ( + "context" + "fmt" + "net" + "net/http" + "os" + "strings" + "testing" + "time" + + "cloud.google.com/go/storage/control/apiv2/controlpb" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/wrappers" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/jacobsa/fuse/fuseops" + "github.com/jacobsa/fuse/fuseutil" + "github.com/jacobsa/timeutil" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/metric" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/reflect/protoregistry" + "google.golang.org/protobuf/types/dynamicpb" +) + +// StorageServer is a dummy interface for gRPC registration. +type StorageServer any + +// reflectFakeServer implements the gRPC Storage service using reflection and dynamicpb +// to avoid importing the conflicting storagepb package. A functional server is +// needed to trigger client-side gRPC metrics and satisfy GCSFuse's stateful +// expectations (e.g., GetObject must succeed before ReadObject is called). +type reflectFakeServer struct{} + +func (s *reflectFakeServer) GetObject(ctx context.Context, req any) (any, error) { + dm := req.(*dynamicpb.Message) + objectName := dm.Get(dm.Descriptor().Fields().ByName("object")).String() + bucketName := dm.Get(dm.Descriptor().Fields().ByName("bucket")).String() + fmt.Printf("Fake GCS: GetObject %s in bucket %s\n", objectName, bucketName) + + if strings.HasSuffix(objectName, "/") { + return nil, status.Error(codes.NotFound, "not found") + } + + // Dynamically create an Object message using the global registry. + msgType, _ := protoregistry.GlobalTypes.FindMessageByName("google.storage.v2.Object") + obj := dynamicpb.NewMessage(msgType.Descriptor()) + obj.Set(obj.Descriptor().Fields().ByName("name"), protoreflect.ValueOf(objectName)) + obj.Set(obj.Descriptor().Fields().ByName("bucket"), protoreflect.ValueOf(bucketName)) + obj.Set(obj.Descriptor().Fields().ByName("size"), protoreflect.ValueOf(int64(12))) + + return obj, nil +} + +func (s *reflectFakeServer) ListObjects(ctx context.Context, req any) (any, error) { + msgType, _ := protoregistry.GlobalTypes.FindMessageByName("google.storage.v2.ListObjectsResponse") + return dynamicpb.NewMessage(msgType.Descriptor()), nil +} + +func (s *reflectFakeServer) ReadObject(req any, stream grpc.ServerStream) error { + dm := req.(*dynamicpb.Message) + objectName := dm.Get(dm.Descriptor().Fields().ByName("object")).String() + fmt.Printf("Fake GCS: ReadObject %s\n", objectName) + + // Send one response. + respType, _ := protoregistry.GlobalTypes.FindMessageByName("google.storage.v2.ReadObjectResponse") + resp := dynamicpb.NewMessage(respType.Descriptor()) + + // Set data. + dataField := resp.Descriptor().Fields().ByName("checksummed_data") + dataMsg := dynamicpb.NewMessage(dataField.Message()) + dataMsg.Set(dataMsg.Descriptor().Fields().ByName("content"), protoreflect.ValueOf([]byte("test content"))) + resp.Set(dataField, protoreflect.ValueOfMessage(dataMsg)) + + // Set metadata. + metaField := resp.Descriptor().Fields().ByName("metadata") + metaMsg := dynamicpb.NewMessage(metaField.Message()) + metaMsg.Set(metaMsg.Descriptor().Fields().ByName("name"), protoreflect.ValueOf(objectName)) + metaMsg.Set(metaMsg.Descriptor().Fields().ByName("size"), protoreflect.ValueOf(int64(12))) + resp.Set(metaField, protoreflect.ValueOfMessage(metaMsg)) + + return stream.SendMsg(resp) +} + +func (s *reflectFakeServer) StartResumableWrite(ctx context.Context, req any) (any, error) { + respType, _ := protoregistry.GlobalTypes.FindMessageByName("google.storage.v2.StartResumableWriteResponse") + resp := dynamicpb.NewMessage(respType.Descriptor()) + resp.Set(resp.Descriptor().Fields().ByName("upload_id"), protoreflect.ValueOf("upload-id")) + return resp, nil +} + +func (s *reflectFakeServer) WriteObject(stream grpc.ServerStream) error { + msgType, _ := protoregistry.GlobalTypes.FindMessageByName("google.storage.v2.WriteObjectRequest") + req := dynamicpb.NewMessage(msgType.Descriptor()) + if err := stream.RecvMsg(req); err != nil { + return err + } + + specField := req.Descriptor().Fields().ByName("write_object_spec") + if req.Has(specField) { + spec := req.Get(specField).Message() + resField := spec.Descriptor().Fields().ByName("resource") + if spec.Has(resField) { + res := spec.Get(resField).Message() + nameField := res.Descriptor().Fields().ByName("name") + fmt.Printf("Fake GCS: WriteObject %s\n", res.Get(nameField).String()) + } + } + + respType, _ := protoregistry.GlobalTypes.FindMessageByName("google.storage.v2.WriteObjectResponse") + resp := dynamicpb.NewMessage(respType.Descriptor()) + return stream.SendMsg(resp) +} + +func registerFakeStorageServer(s *grpc.Server, srv *reflectFakeServer) { + // Dummy interface type pointer. + var storageServerPtr *StorageServer + + desc := &grpc.ServiceDesc{ + ServiceName: "google.storage.v2.Storage", + HandlerType: storageServerPtr, + Methods: []grpc.MethodDesc{ + { + MethodName: "GetObject", + Handler: func(srv any, ctx context.Context, dec func(any) error, interceptor grpc.UnaryServerInterceptor) (any, error) { + msgType, _ := protoregistry.GlobalTypes.FindMessageByName("google.storage.v2.GetObjectRequest") + in := dynamicpb.NewMessage(msgType.Descriptor()) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(*reflectFakeServer).GetObject(ctx, in) + } + return interceptor(ctx, in, &grpc.UnaryServerInfo{Server: srv, FullMethod: "/google.storage.v2.Storage/GetObject"}, func(ctx context.Context, req any) (any, error) { + return srv.(*reflectFakeServer).GetObject(ctx, req) + }) + }, + }, + { + MethodName: "ListObjects", + Handler: func(srv any, ctx context.Context, dec func(any) error, interceptor grpc.UnaryServerInterceptor) (any, error) { + msgType, _ := protoregistry.GlobalTypes.FindMessageByName("google.storage.v2.ListObjectsRequest") + in := dynamicpb.NewMessage(msgType.Descriptor()) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(*reflectFakeServer).ListObjects(ctx, in) + } + return interceptor(ctx, in, &grpc.UnaryServerInfo{Server: srv, FullMethod: "/google.storage.v2.Storage/ListObjects"}, func(ctx context.Context, req any) (any, error) { + return srv.(*reflectFakeServer).ListObjects(ctx, req) + }) + }, + }, + { + MethodName: "StartResumableWrite", + Handler: func(srv any, ctx context.Context, dec func(any) error, interceptor grpc.UnaryServerInterceptor) (any, error) { + msgType, _ := protoregistry.GlobalTypes.FindMessageByName("google.storage.v2.StartResumableWriteRequest") + in := dynamicpb.NewMessage(msgType.Descriptor()) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(*reflectFakeServer).StartResumableWrite(ctx, in) + } + return interceptor(ctx, in, &grpc.UnaryServerInfo{Server: srv, FullMethod: "/google.storage.v2.Storage/StartResumableWrite"}, func(ctx context.Context, req any) (any, error) { + return srv.(*reflectFakeServer).StartResumableWrite(ctx, req) + }) + }, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "ReadObject", + Handler: func(srv any, stream grpc.ServerStream) error { + msgType, _ := protoregistry.GlobalTypes.FindMessageByName("google.storage.v2.ReadObjectRequest") + m := dynamicpb.NewMessage(msgType.Descriptor()) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(*reflectFakeServer).ReadObject(m, stream) + }, + ServerStreams: true, + }, + { + StreamName: "WriteObject", + Handler: func(srv any, stream grpc.ServerStream) error { + return srv.(*reflectFakeServer).WriteObject(stream) + }, + ClientStreams: true, + }, + { + StreamName: "BidiWriteObject", + Handler: func(srv any, stream grpc.ServerStream) error { + return status.Error(codes.Unimplemented, "unimplemented") + }, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "google/storage/v2/storage.proto", + } + s.RegisterService(desc, srv) +} + +type fakeStorageControlServer struct { + controlpb.UnimplementedStorageControlServer +} + +func (s *fakeStorageControlServer) GetStorageLayout(ctx context.Context, req *controlpb.GetStorageLayoutRequest) (*controlpb.StorageLayout, error) { + return &controlpb.StorageLayout{ + Name: req.Name, + HierarchicalNamespace: &controlpb.StorageLayout_HierarchicalNamespace{ + Enabled: false, + }, + }, nil +} + +func createTestFileSystemWithGrpcMetrics(ctx context.Context, t *testing.T, params *serverConfigParams) (storage.StorageHandle, fuseutil.FileSystem, metrics.MetricHandle, *metric.ManualReader) { + t.Helper() + + // Bypass the protobuf registration conflict panic. + _ = os.Setenv("GOLANG_PROTOBUF_REGISTRATION_CONFLICT", "ignore") + _ = os.Setenv("GOOGLE_CLOUD_PROJECT", "test-project") + + // Start fake gRPC server + lis, err := net.Listen("tcp", "localhost:0") + require.NoError(t, err) + grpcServer := grpc.NewServer() + registerFakeStorageServer(grpcServer, &reflectFakeServer{}) + controlpb.RegisterStorageControlServer(grpcServer, &fakeStorageControlServer{}) + go func() { + _ = grpcServer.Serve(lis) + }() + t.Cleanup(func() { + grpcServer.Stop() + }) + + // Start fake metadata server to provide project ID for gRPC metrics + metadataLis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + mux := http.NewServeMux() + mux.HandleFunc("/computeMetadata/v1/project/project-id", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Metadata-Flavor", "Google") + _, _ = w.Write([]byte("test-project")) + }) + metadataServer := &http.Server{Handler: mux} + go func() { + _ = metadataServer.Serve(metadataLis) + }() + t.Cleanup(func() { + _ = metadataServer.Close() + }) + _ = os.Setenv("GCE_METADATA_HOST", metadataLis.Addr().String()) + t.Cleanup(func() { _ = os.Unsetenv("GCE_METADATA_HOST") }) + + origProvider := otel.GetMeterProvider() + t.Cleanup(func() { otel.SetMeterProvider(origProvider) }) + reader := metric.NewManualReader() + provider := metric.NewMeterProvider(metric.WithReader(reader)) + otel.SetMeterProvider(provider) + + mh, err := metrics.NewOTelMetrics(ctx, 1, 100) + require.NoError(t, err, "metrics.NewOTelMetrics") + + clientConfig := storageutil.StorageClientConfig{ + ClientProtocol: cfg.GRPC, + CustomEndpoint: lis.Addr().String(), + EnableGrpcMetrics: true, + IsGKE: true, + AnonymousAccess: true, + MetricHandle: mh, + WriteConfig: &cfg.WriteConfig{}, + } + + sh, err := storage.NewStorageHandle(ctx, clientConfig, "") + require.NoError(t, err) + + // Poke the storage client to trigger internal DirectPath checks with a short timeout. + // This prevents the subsequent real operations from hanging for 60s. + shortCtx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) + _, _ = sh.BucketHandle(shortCtx, "test-bucket", "") + cancel() + + bucketName := "test-bucket" + bucketConfig := gcsx.BucketConfig{ + BillingProject: "", + StatCacheMaxSizeMB: 32, + StatCacheTTL: time.Minute, + NegativeStatCacheTTL: time.Minute, + TmpObjectPrefix: ".gcsfuse_tmp/", + } + + bm := gcsx.NewBucketManager(bucketConfig, sh) + + serverCfg := &fs.ServerConfig{ + NewConfig: &cfg.Config{ + Write: cfg.WriteConfig{ + GlobalMaxBlocks: 1, + }, + Read: cfg.ReadConfig{ + EnableBufferedRead: params.enableBufferedRead, + GlobalMaxBlocks: 1, + BlockSizeMb: 1, + MaxBlocksPerHandle: 10, + }, + EnableNewReader: true, + Metrics: cfg.MetricsConfig{ + ExperimentalEnableGrpcMetrics: true, + }, + }, + MetricHandle: mh, + TraceHandle: tracing.NewNoopTracer(), + CacheClock: &timeutil.SimulatedClock{}, + BucketName: bucketName, + BucketManager: bm, + } + + if params.enableFileCache || params.enableSparseFileCache { + cacheDir := t.TempDir() + serverCfg.NewConfig.CacheDir = cfg.ResolvedPath(cacheDir) + serverCfg.NewConfig.FileCache = cfg.FileCacheConfig{ + MaxSizeMb: 100, + CacheFileForRangeRead: true, + ExperimentalEnableChunkCache: params.enableSparseFileCache, + DownloadChunkSizeMb: 1, + EnableParallelDownloads: params.enableParallelDownloads, + ExperimentalParallelDownloadsDefaultOn: params.enableParallelDownloadsBlocking, + ParallelDownloadsPerFile: 16, + } + } + if serverCfg.NewConfig.MetadataCache.TtlSecs == 0 { + serverCfg.NewConfig.MetadataCache.TtlSecs = 60 + } + + server, err := fs.NewFileSystem(ctx, serverCfg) + require.NoError(t, err, "NewFileSystem") + return sh, server, mh, reader +} + +func TestGrpcMetrics_LookUpInode(t *testing.T) { + ctx := context.Background() + params := defaultServerConfigParams() + + _, server, mh, reader := createTestFileSystemWithGrpcMetrics(ctx, t, params) + server = wrappers.WithMonitoring(server, mh) + + fileName := "test.txt" + + lookupOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + + // HACK: Every gRPC-enabled mount in GCSFuse performs a synchronous + // "DirectPath connectivity check" during the first few operations on a bucket. + // In non-GCP environments (like local development), this library check + // hangs for exactly 60 seconds before failing and falling back to standard gRPC. + // + // We perform a dummy LookUpInode with a 100ms timeout here to "poke" this check + // and allow it to fail early. This prevents the initial test operation from + // appearing to hang indefinitely in an IDE, although subsequent real calls + // will still wait for the library's internal 60s timeout to expire naturally. + // + // This approach is used to keep the production codebase (storage_handle.go) + // completely pristine while still allowing the tests to eventually pass. + shortCtx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) + _ = server.LookUpInode(shortCtx, lookupOp) + cancel() + + // Now run the real call. It should be fast because DirectPath check (even if hanging) + // won't block the actual gRPC calls once the client is initialized. + err := server.LookUpInode(ctx, lookupOp) + require.NoError(t, err) + + waitForMetricsProcessing() + time.Sleep(501 * time.Millisecond) + + // Verify that grpc.client.attempt.started was emitted. + metrics.VerifyCounterMetric(t, ctx, reader, "grpc.client.attempt.started", + attribute.NewSet(attribute.String("grpc.method", "google.storage.v2.Storage/GetObject")), + 1, metrics.AtLeast(), metrics.Subset()) + metrics.VerifyHistogramMetric(t, ctx, reader, "grpc.client.call.duration", + attribute.NewSet(attribute.String("grpc.method", "google.storage.v2.Storage/GetObject")), + 1, metrics.AtLeast(), metrics.Subset()) +} + +func TestGrpcMetrics_ReadFile(t *testing.T) { + ctx := context.Background() + params := defaultServerConfigParams() + + _, server, mh, reader := createTestFileSystemWithGrpcMetrics(ctx, t, params) + server = wrappers.WithMonitoring(server, mh) + + fileName := "test.txt" + + lookupOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + // Bypass the DirectPath hang (see explanation in TestGrpcMetrics_LookUpInode). + shortCtx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) + _ = server.LookUpInode(shortCtx, lookupOp) + cancel() + err := server.LookUpInode(ctx, lookupOp) + require.NoError(t, err) + + openOp := &fuseops.OpenFileOp{ + Inode: lookupOp.Entry.Child, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err) + + readOp := &fuseops.ReadFileOp{ + Inode: lookupOp.Entry.Child, + Handle: openOp.Handle, + Offset: 0, + Dst: make([]byte, 12), + } + err = server.ReadFile(ctx, readOp) + require.NoError(t, err) + waitForMetricsProcessing() + time.Sleep(501 * time.Millisecond) + + // Verify ReadObject metric. + metrics.VerifyCounterMetric(t, ctx, reader, "grpc.client.attempt.started", + attribute.NewSet(attribute.String("grpc.method", "google.storage.v2.Storage/ReadObject")), + 1, metrics.AtLeast(), metrics.Subset()) + metrics.VerifyHistogramMetric(t, ctx, reader, "grpc.client.call.duration", + attribute.NewSet(attribute.String("grpc.method", "google.storage.v2.Storage/ReadObject")), + 1, metrics.AtLeast(), metrics.Subset()) +} + +func TestGrpcMetrics_CreateFile(t *testing.T) { + ctx := context.Background() + params := defaultServerConfigParams() + + _, server, mh, reader := createTestFileSystemWithGrpcMetrics(ctx, t, params) + server = wrappers.WithMonitoring(server, mh) + + fileName := "new_file.txt" + + createOp := &fuseops.CreateFileOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + Mode: 0644, + } + // Bypass the DirectPath hang (see explanation in TestGrpcMetrics_LookUpInode). + shortCtx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) + _ = server.CreateFile(shortCtx, createOp) + cancel() + err := server.CreateFile(ctx, createOp) + require.NoError(t, err) + + syncOp := &fuseops.SyncFileOp{ + Inode: createOp.Entry.Child, + Handle: createOp.Handle, + } + // SyncFile might fail if BidiWriteObject is unimplemented, but we check metrics. + _ = server.SyncFile(ctx, syncOp) + + waitForMetricsProcessing() + time.Sleep(501 * time.Millisecond) + + // Verify BidiWriteObject metric. + metrics.VerifyCounterMetric(t, ctx, reader, "grpc.client.attempt.started", + attribute.NewSet(attribute.String("grpc.method", "google.storage.v2.Storage/BidiWriteObject")), + 1, metrics.AtLeast(), metrics.Subset()) + metrics.VerifyHistogramMetric(t, ctx, reader, "grpc.client.call.duration", + attribute.NewSet(attribute.String("grpc.method", "google.storage.v2.Storage/BidiWriteObject")), + 1, metrics.AtLeast(), metrics.Subset()) +} diff --git a/internal/fs/handle/dir_handle.go b/internal/fs/handle/dir_handle.go index 86b05117b6..9df8164751 100644 --- a/internal/fs/handle/dir_handle.go +++ b/internal/fs/handle/dir_handle.go @@ -15,17 +15,43 @@ package handle import ( + "cmp" "fmt" - "sort" + "maps" + "slices" - "github.com/googlecloudplatform/gcsfuse/v2/internal/fs/inode" - "github.com/googlecloudplatform/gcsfuse/v2/internal/locker" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/inode" + "github.com/googlecloudplatform/gcsfuse/v3/internal/locker" "github.com/jacobsa/fuse" "github.com/jacobsa/fuse/fuseops" "github.com/jacobsa/fuse/fuseutil" "golang.org/x/net/context" ) +// DirEntry is a generic interface for directory entries that expose +// name, type, and offset. It is implemented by fuseutil.Dirent and fuseutil.DirentPlus. +type DirEntry interface { + EntryName() string + SetName(name string) + EntryType() fuseutil.DirentType + SetOffset(offset fuseops.DirOffset) +} + +type dirent fuseutil.Dirent +type direntPlus fuseutil.DirentPlus + +// For Dirent +func (d *dirent) EntryName() string { return d.Name } +func (d *dirent) SetName(name string) { d.Name = name } +func (d *dirent) EntryType() fuseutil.DirentType { return d.Type } +func (d *dirent) SetOffset(offset fuseops.DirOffset) { d.Offset = offset } + +// For DirentPlus +func (dp *direntPlus) EntryName() string { return dp.Dirent.Name } +func (dp *direntPlus) SetName(name string) { dp.Dirent.Name = name } +func (dp *direntPlus) EntryType() fuseutil.DirentType { return dp.Dirent.Type } +func (dp *direntPlus) SetOffset(offset fuseops.DirOffset) { dp.Dirent.Offset = offset } + // DirHandle is the state required for reading from directories. type DirHandle struct { ///////////////////////// @@ -48,12 +74,26 @@ type DirHandle struct { // GUARDED_BY(Mu) entries []fuseutil.Dirent + // All entries in the directory along with their attributes. Populated the first time we need one. + // + // INVARIANT: For each i, entriesPlus[i+1].Offset == entriesPlus[i].Offset + 1 + // + // GUARDED_BY(Mu) + entriesPlus []fuseutil.DirentPlus + // Has entries yet been populated? // // INVARIANT: If !entriesValid, then len(entries) == 0 // // GUARDED_BY(Mu) entriesValid bool + + // Has entriesPlus yet been populated? + // + // INVARIANT: If !entriesPlusValid, then len(entriesPlus) == 0 + // + // GUARDED_BY(Mu) + entriesPlusValid bool } // NewDirHandle creates a directory handle that obtains listings from the supplied inode. @@ -76,13 +116,6 @@ func NewDirHandle( // Helpers //////////////////////////////////////////////////////////////////////// -// Directory entries, sorted by name. -type sortedDirents []fuseutil.Dirent - -func (p sortedDirents) Len() int { return len(p) } -func (p sortedDirents) Less(i, j int) bool { return p[i].Name < p[j].Name } -func (p sortedDirents) Swap(i, j int) { p[i], p[j] = p[j], p[i] } - func (dh *DirHandle) checkInvariants() { // INVARIANT: For each i, entries[i+1].Offset == entries[i].Offset + 1 for i := 0; i < len(dh.entries)-1; i++ { @@ -99,6 +132,28 @@ func (dh *DirHandle) checkInvariants() { if !dh.entriesValid && len(dh.entries) != 0 { panic("Unexpected non-empty entries slice") } + + // INVARIANT: For each i, entriesPlus[i+1].Dirent.Offset == entriesPlus[i].Dirent.Offset + 1 + for i := 0; i < len(dh.entriesPlus)-1; i++ { + if !(dh.entriesPlus[i+1].Dirent.Offset == dh.entriesPlus[i].Dirent.Offset+1) { + panic( + fmt.Sprintf( + "Unexpected offset sequence: %v, %v", + dh.entriesPlus[i].Dirent.Offset, + dh.entriesPlus[i+1].Dirent.Offset)) + } + } + + // INVARIANT: If !entriesPlusValid, then len(entriesPlus) == 0 + if !dh.entriesPlusValid && len(dh.entriesPlus) != 0 { + panic("Unexpected non-empty entries slice") + } +} + +// compareEntriesByName provides a comparison function for sorting directory entries +// by name. +func compareEntriesByName[T DirEntry](a, b T) int { + return cmp.Compare(a.EntryName(), b.EntryName()) } // Resolve name conflicts between file objects and directory objects (e.g. the @@ -106,37 +161,37 @@ func (dh *DirHandle) checkInvariants() { // GCS object names, to conflicting file names. // // Input must be sorted by name. -func fixConflictingNames(entries []fuseutil.Dirent, localEntries map[string]fuseutil.Dirent) (output []fuseutil.Dirent, err error) { +func fixConflictingNames[T DirEntry](entries []T, localEntries map[string]T) (output []T, err error) { // Sanity check. - if !sort.IsSorted(sortedDirents(entries)) { + if !slices.IsSortedFunc(entries, compareEntriesByName) { err = fmt.Errorf("expected sorted input") return } // Examine each adjacent pair of names. for i := range entries { - e := &entries[i] + e := entries[i] // Find the previous entry. if i == 0 { - output = append(output, *e) + output = append(output, e) continue } - prev := &output[len(output)-1] + prev := output[len(output)-1] // Does the pair have matching names? - if e.Name != prev.Name { - output = append(output, *e) + if e.EntryName() != prev.EntryName() { + output = append(output, e) continue } // We expect exactly one to be a directory. - eIsDir := e.Type == fuseutil.DT_Directory - prevIsDir := prev.Type == fuseutil.DT_Directory + eIsDir := e.EntryType() == fuseutil.DT_Directory + prevIsDir := prev.EntryType() == fuseutil.DT_Directory if eIsDir == prevIsDir { - if _, ok := localEntries[e.Name]; ok && !eIsDir { + if _, ok := localEntries[e.EntryName()]; ok && !eIsDir { // We have found same entry in GCS and local file entries, i.e, the // entry is uploaded to GCS but not yet deleted from local entries. // Do not return the duplicate entry as part of list response. @@ -144,26 +199,69 @@ func fixConflictingNames(entries []fuseutil.Dirent, localEntries map[string]fuse } else { err = fmt.Errorf( "weird dirent type pair for name %q: %v, %v", - e.Name, - e.Type, - prev.Type) + e.EntryName(), + e.EntryType(), + prev.EntryType()) return } } // Repair whichever is not the directory. if eIsDir { - prev.Name += inode.ConflictingFileNameSuffix + prev.SetName(prev.EntryName() + inode.ConflictingFileNameSuffix) } else { - e.Name += inode.ConflictingFileNameSuffix + e.SetName(e.EntryName() + inode.ConflictingFileNameSuffix) } - output = append(output, *e) + output = append(output, e) } return } +// sortAndResolveEntries is a generic helper function that takes a list of +// directory entries, sorts them, resolves name conflicts, and sets their +// offsets. +// +// This generic function supports both fuseutil.Dirent and fuseutil.DirentPlus +// by wrapping/ unwrapping them into DirEntry interface-compatible types. +func sortAndResolveEntries[Entry any, WrappedEntry DirEntry](entries []Entry, localEntries map[string]Entry, wrap func(Entry) WrappedEntry, unwrap func(WrappedEntry) Entry) ([]Entry, error) { + // Wrap and append local file entry (not synced to GCS). + wrappedLocalEntries := make(map[string]WrappedEntry) + for name, localEntry := range localEntries { + wrappedLocalEntries[name] = wrap(localEntry) + entries = append(entries, localEntry) + } + wrappedEntries := make([]WrappedEntry, 0, len(entries)) + for _, entry := range entries { + wrappedEntries = append(wrappedEntries, wrap(entry)) + } + + // Ensure that the entries are sorted, for use in fixConflictingNames + // below. + slices.SortFunc(wrappedEntries, compareEntriesByName) + + // Fix name conflicts. + // When a local file is synced to GCS but not removed from the local file map, + // the entries list will have two duplicate entries. + // To handle this scenario, we are removing the duplicate entry before + // returning the response to kernel. + fixedEntries, err := fixConflictingNames(wrappedEntries, wrappedLocalEntries) + if err != nil { + err = fmt.Errorf("fixConflictingNames: %w", err) + return nil, err + } + + // Fix up offset fields and unwrap. + finalEntries := make([]Entry, 0, len(fixedEntries)) + for i, fe := range fixedEntries { + fe.SetOffset(fuseops.DirOffset(i) + 1) + finalEntries = append(finalEntries, unwrap(fe)) + } + + return finalEntries, nil +} + // Read all entries for the directory, fix up conflicting names, and fill in // offset fields. // @@ -179,7 +277,7 @@ func readAllEntries( // Read a batch. var batch []fuseutil.Dirent - batch, tok, err = in.ReadEntries(ctx, tok) + batch, _, tok, err = in.ReadEntries(ctx, tok) if err != nil { err = fmt.Errorf("ReadEntries: %w", err) return @@ -194,29 +292,10 @@ func readAllEntries( } } - // Append local file entries (not synced to GCS). - for _, localEntry := range localEntries { - entries = append(entries, localEntry) - } - - // Ensure that the entries are sorted, for use in fixConflictingNames - // below. - sort.Sort(sortedDirents(entries)) - - // Fix name conflicts. - // When a local file is synced to GCS but not removed from the local file map, - // the entries list will have two duplicate entries. - // To handle this scenario, we are removing the duplicate entry before - // returning the response to kernel. - entries, err = fixConflictingNames(entries, localEntries) + // Sort, resolve conflicts, and set offsets. + entries, err = sortAndResolveEntries(entries, localEntries, func(e fuseutil.Dirent) *dirent { d := dirent(e); return &d }, func(w *dirent) fuseutil.Dirent { return fuseutil.Dirent(*w) }) if err != nil { - err = fmt.Errorf("fixConflictingNames: %w", err) - return - } - - // Fix up offset fields. - for i := 0; i < len(entries); i++ { - entries[i].Offset = fuseops.DirOffset(i) + 1 + return nil, err } // Return a bogus inode ID for each entry, but not the root inode ID. @@ -239,6 +318,33 @@ func readAllEntries( return } +// readAllEntryCores retrieves all directory entry cores for the given inode, +// handling pagination and accumulating the results. +// LOCKS_REQUIRED(in) +func readAllEntryCores(ctx context.Context, in inode.DirInode) (cores map[inode.Name]*inode.Core, err error) { + // Read entries from GCS. + // Read one batch at a time. + var tok string + cores = make(map[inode.Name]*inode.Core) + for { + // Read a batch from GCS + var batch map[inode.Name]*inode.Core + batch, _, tok, err = in.ReadEntryCores(ctx, tok) + if err != nil { + return + } + // Accumulate. + maps.Copy(cores, batch) + + // Are we done? + if tok == "" { + break + } + } + + return +} + // LOCKS_REQUIRED(dh.Mu) // LOCKS_EXCLUDED(dh.in) func (dh *DirHandle) ensureEntries(ctx context.Context, localFileEntries map[string]fuseutil.Dirent) (err error) { @@ -311,3 +417,69 @@ func (dh *DirHandle) ReadDir( return } + +// FetchEntryCores retrieves the core inode data for all entries within the directory from GCS. +// +// Special case: If the request offset is zero, it assumes the directory is being read from the +// beginning and resets the cached list of entries. +// +// LOCKS_REQUIRED(dh.Mu) +// LOCKS_EXCLUDED(dh.in) +func (dh *DirHandle) FetchEntryCores(ctx context.Context, op *fuseops.ReadDirPlusOp) (cores map[inode.Name]*inode.Core, err error) { + // If the request is for offset zero, we assume that either this is the first + // call or rewinddir has been called. Reset state. + if op.Offset == 0 { + dh.entriesPlus = nil + dh.entriesPlusValid = false + } + + // Do we need to read entries from GCS? + if !dh.entriesPlusValid { + dh.in.Lock() + cores, err = readAllEntryCores(ctx, dh.in) + if err != nil { + dh.in.Unlock() + return + } + dh.in.Unlock() + } + + return +} + +// ReadDirPlus populates the FUSE response buffer using a pre-processed list +// of directory entries. +// LOCKS_REQUIRED(dh.Mu) +func (dh *DirHandle) ReadDirPlus(op *fuseops.ReadDirPlusOp, entries []fuseutil.DirentPlus, localEntries map[string]fuseutil.DirentPlus) (err error) { + // If entriesPlus has not been populated yet, populate it. + if !dh.entriesPlusValid { + // Sort, resolve conflicts, and set offsets. + entries, err = sortAndResolveEntries(entries, localEntries, func(e fuseutil.DirentPlus) *direntPlus { dp := direntPlus(e); return &dp }, func(w *direntPlus) fuseutil.DirentPlus { return fuseutil.DirentPlus(*w) }) + if err != nil { + return + } + // Update state. + dh.entriesPlus = entries + dh.entriesPlusValid = true + } + + // Is the offset past the end of what we have buffered? If so, this must be + // an invalid seekdir according to posix. + index := int(op.Offset) + if index > len(dh.entriesPlus) { + err = fuse.EINVAL + return + } + + //We copy out entries until we run out of entries or space. + for i := index; i < len(dh.entriesPlus); i++ { + n := fuseutil.WriteDirentPlus(op.Dst[op.BytesRead:], dh.entriesPlus[i]) + if n == 0 { + break + } + + op.BytesRead += n + } + + return +} diff --git a/internal/fs/handle/dir_handle_test.go b/internal/fs/handle/dir_handle_test.go index fc17b0f7ab..07ec05c2bb 100644 --- a/internal/fs/handle/dir_handle_test.go +++ b/internal/fs/handle/dir_handle_test.go @@ -16,24 +16,27 @@ package handle import ( "context" - "math" + "path" "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/contentcache" - "github.com/googlecloudplatform/gcsfuse/v2/internal/fs/inode" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/fake" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + cfg2 "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata" + "golang.org/x/sync/semaphore" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/inode" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" "github.com/jacobsa/fuse/fuseops" "github.com/jacobsa/fuse/fuseutil" . "github.com/jacobsa/ogletest" "github.com/jacobsa/timeutil" - "golang.org/x/sync/semaphore" ) +const testDirentName = "sameName" + func TestDirHandle(t *testing.T) { RunTests(t) } //////////////////////////////////////////////////////////////////////// @@ -56,7 +59,10 @@ func init() { RegisterTestSuite(&DirHandleTest{}) } func (t *DirHandleTest) SetUp(ti *TestInfo) { t.ctx = ti.Ctx t.bucket = gcsx.NewSyncerBucket( - 1, ".gcsfuse_tmp/", fake.NewFakeBucket(&t.clock, "some_bucket", gcs.NonHierarchical)) + /*appendThreshold=*/ 1, + /*chunkRetryDeadlineSecs=*/ 120, + /*chunkTransferTimeoutSecs=*/ 10, + ".gcsfuse_tmp/", fake.NewFakeBucket(&t.clock, "some_bucket", gcs.BucketType{})) t.clock.SetTime(time.Date(2022, 8, 15, 22, 56, 0, 0, time.Local)) t.resetDirHandle() } @@ -67,23 +73,30 @@ func (t *DirHandleTest) TearDown() {} // Helpers // ////////////////////////////////////////////////////////////////////// func (t *DirHandleTest) resetDirHandle() { + cfg := &cfg2.Config{ + List: cfg2.ListConfig{EnableEmptyManagedFolders: true}, + MetadataCache: cfg2.MetadataCacheConfig{TypeCacheMaxSizeMb: 0}, + EnableHns: false, + EnableUnsupportedPathSupport: true, + EnableTypeCacheDeprecation: isTypeCacheDeprecationEnabled, + } dirInode := inode.NewDirInode( 17, inode.NewDirName(inode.NewRootName(""), "testDir"), + nil, fuseops.InodeAttributes{ Uid: 123, Gid: 456, Mode: 0712, }, false, // implicitDirs, - true, // enableManagedFoldersListing false, // enableNonExistentTypeCache 0, // typeCacheTTL &t.bucket, &t.clock, &t.clock, - 0, - false) + semaphore.NewWeighted(10), + cfg) t.dh = NewDirHandle( dirInode, @@ -91,31 +104,49 @@ func (t *DirHandleTest) resetDirHandle() { ) } -func (t *DirHandleTest) createLocalFileInode(name string, id fuseops.InodeID) (in inode.Inode) { - in = inode.NewFileInode( - id, - inode.NewFileName(t.dh.in.Name(), name), - nil, - fuseops.InodeAttributes{ - Uid: 123, - Gid: 456, - Mode: 0712, - }, - &t.bucket, - false, // localFileCache - contentcache.New("", &t.clock), - &t.clock, - true, // localFile - &cfg.WriteConfig{}, - semaphore.NewWeighted(math.MaxInt64)) - return -} - func (t *DirHandleTest) validateEntry(entry fuseutil.Dirent, name string, filetype fuseutil.DirentType) { AssertEq(name, entry.Name) AssertEq(filetype, entry.Type) } +func (t *DirHandleTest) createTestDirentPlus(dtype fuseutil.DirentType, childInodeID fuseops.InodeID, size uint64) fuseutil.DirentPlus { + attrs := fuseops.InodeAttributes{ + Size: size, + Mode: 0777, + Nlink: 1, + Uid: 123, + Gid: 456, + } + if dtype != fuseutil.DT_Directory { + attrs.Mode = 0666 + } + + return fuseutil.DirentPlus{ + Dirent: fuseutil.Dirent{ + Name: testDirentName, + Type: dtype, + }, + Entry: fuseops.ChildInodeEntry{ + Child: childInodeID, + Attributes: attrs, + }, + } +} + +func (t *DirHandleTest) validateEntryPlus(entry fuseutil.DirentPlus, expectedName string, expectedType fuseutil.DirentType, expectedChildInodeID fuseops.InodeID) { + AssertEq(expectedName, entry.Dirent.Name) + AssertEq(expectedType, entry.Dirent.Type) + AssertEq(expectedChildInodeID, entry.Entry.Child) +} + +func (t *DirHandleTest) validateFileType(core *inode.Core, expectedName string, expectedMinObjectName string) { + AssertNe(nil, core) + AssertNe(nil, core.MinObject) + AssertEq(expectedName, path.Base(core.FullName.LocalName())) + AssertEq(expectedMinObjectName, core.MinObject.Name) + AssertEq(metadata.RegularFileType, core.Type()) +} + //////////////////////////////////////////////////////////////////////// // Tests //////////////////////////////////////////////////////////////////////// @@ -278,3 +309,128 @@ func (t *DirHandleTest) EnsureEntriesWithOneLocalFile() { AssertEq(1, len(t.dh.entries)) t.validateEntry(t.dh.entries[0], localFileName1, fuseutil.DT_File) } + +func (t *DirHandleTest) ReadAllEntryCoresWithNoEntry() { + cores, err := readAllEntryCores(t.ctx, t.dh.in) + + AssertEq(nil, err) + AssertEq(0, len(cores)) +} + +func (t *DirHandleTest) ReadAllEntryCoresReturnsAllEntryCores() { + // Setup GCS objects + _, err := storageutil.CreateObject(t.ctx, t.bucket, "testDir/gcsObject1", nil) + AssertEq(nil, err) + _, err = storageutil.CreateObject(t.ctx, t.bucket, "testDir/gcsObject2", nil) + AssertEq(nil, err) + + // read all entry cores + cores, err := readAllEntryCores(t.ctx, t.dh.in) + + // validations + AssertEq(nil, err) + AssertEq(2, len(cores)) + entry1, ok := cores[inode.NewFileName(t.dh.in.Name(), "gcsObject1")] + AssertTrue(ok, "Core for gcsObject1 not found") + t.validateFileType(entry1, "gcsObject1", "testDir/gcsObject1") + entry2, ok := cores[inode.NewFileName(t.dh.in.Name(), "gcsObject2")] + AssertTrue(ok, "Core for gcsObject2 not found") + t.validateFileType(entry2, "gcsObject2", "testDir/gcsObject2") +} + +func (t *DirHandleTest) FetchEntryCoresFetchesCores() { + _, err := storageutil.CreateObject(t.ctx, t.bucket, "testDir/testFile", nil) + AssertEq(nil, err) + op := &fuseops.ReadDirPlusOp{ + ReadDirOp: fuseops.ReadDirOp{Offset: 0}, + } + t.dh.entriesPlusValid = false + + cores, err := t.dh.FetchEntryCores(t.ctx, op) + + AssertEq(nil, err) + AssertEq(1, len(cores)) + entry, ok := cores[inode.NewFileName(t.dh.in.Name(), "testFile")] + AssertTrue(ok, "Core for gcsFile1 not found") + t.validateFileType(entry, "testFile", "testDir/testFile") +} + +func (t *DirHandleTest) FetchEntryCoresNonZeroOffsetNoFetchIfCacheValid() { + t.dh.entriesPlus = []fuseutil.DirentPlus{{}} + t.dh.entriesPlusValid = true + op := &fuseops.ReadDirPlusOp{ + ReadDirOp: fuseops.ReadDirOp{Offset: 1}, + } + + cores, err := t.dh.FetchEntryCores(t.ctx, op) + + AssertEq(nil, err) + AssertEq(nil, cores) + AssertTrue(t.dh.entriesPlusValid) +} + +func (t *DirHandleTest) FetchEntryCoresNonZeroOffsetFetchesIfCacheInvalid() { + t.dh.entriesPlusValid = false + _, err := storageutil.CreateObject(t.ctx, t.bucket, "testDir/fetchThis", nil) + AssertEq(nil, err) + op := &fuseops.ReadDirPlusOp{ + ReadDirOp: fuseops.ReadDirOp{Offset: 1}, + } + + cores, err := t.dh.FetchEntryCores(t.ctx, op) + + AssertEq(nil, err) + AssertEq(1, len(cores)) + entry, ok := cores[inode.NewFileName(t.dh.in.Name(), "fetchThis")] + AssertTrue(ok, "Core for fetchThis not found") + t.validateFileType(entry, "fetchThis", "testDir/fetchThis") +} + +func (t *DirHandleTest) ReadDirPlusResponseForNoFile() { + op := &fuseops.ReadDirPlusOp{ + ReadDirOp: fuseops.ReadDirOp{Dst: make([]byte, 1024)}, + } + var gcsEntries []fuseutil.DirentPlus + localFileEntries := make(map[string]fuseutil.DirentPlus) + + err := t.dh.ReadDirPlus(op, gcsEntries, localFileEntries) + + AssertEq(nil, err) + AssertEq(0, op.BytesRead) + AssertTrue(t.dh.entriesPlusValid) + AssertEq(0, len(t.dh.entriesPlus)) +} + +func (t *DirHandleTest) ReadDirPlusSameNameLocalAndGCSFile() { + op := &fuseops.ReadDirPlusOp{ + ReadDirOp: fuseops.ReadDirOp{Dst: make([]byte, 1024)}, + } + gcsFile := t.createTestDirentPlus(fuseutil.DT_File, 1001, 10) + localFile := t.createTestDirentPlus(fuseutil.DT_File, 1002, 0) + gcsEntriesPlus := []fuseutil.DirentPlus{gcsFile} + localFileEntriesPlus := map[string]fuseutil.DirentPlus{testDirentName: localFile} + + err := t.dh.ReadDirPlus(op, gcsEntriesPlus, localFileEntriesPlus) + + AssertEq(nil, err) + AssertEq(1, len(t.dh.entriesPlus)) + t.validateEntryPlus(t.dh.entriesPlus[0], testDirentName, fuseutil.DT_File, 1001) +} + +func (t *DirHandleTest) ReadDirPlusSameNameLocalFileAndGCSDirectory() { + op := &fuseops.ReadDirPlusOp{ + ReadDirOp: fuseops.ReadDirOp{Dst: make([]byte, 1024)}, + } + gcsDir := t.createTestDirentPlus(fuseutil.DT_Directory, 1001, 0) + gcsEntriesPlus := []fuseutil.DirentPlus{gcsDir} + localFile := t.createTestDirentPlus(fuseutil.DT_File, 2001, 20) + localFileEntriesPlus := map[string]fuseutil.DirentPlus{testDirentName: localFile} + + err := t.dh.ReadDirPlus(op, gcsEntriesPlus, localFileEntriesPlus) + + AssertEq(nil, err) + AssertEq(2, len(t.dh.entriesPlus)) + t.validateEntryPlus(t.dh.entriesPlus[0], testDirentName, fuseutil.DT_Directory, 1001) + t.validateEntryPlus(t.dh.entriesPlus[1], testDirentName+inode.ConflictingFileNameSuffix, fuseutil.DT_File, 2001) + AssertEq(t.dh.entriesPlus[1].Dirent.Offset, t.dh.entriesPlus[0].Dirent.Offset+1) +} diff --git a/internal/fs/handle/file.go b/internal/fs/handle/file.go index d10abe1c3e..fe0287a01c 100644 --- a/internal/fs/handle/file.go +++ b/internal/fs/handle/file.go @@ -15,14 +15,25 @@ package handle import ( + "errors" "fmt" "io" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/file" - "github.com/googlecloudplatform/gcsfuse/v2/internal/fs/inode" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/file" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/inode" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx/read_manager" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/workerpool" + "github.com/googlecloudplatform/gcsfuse/v3/internal/workloadinsight" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/jacobsa/fuse/fuseops" "github.com/jacobsa/syncutil" "golang.org/x/net/context" + "golang.org/x/sync/semaphore" ) type FileHandle struct { @@ -38,22 +49,79 @@ type FileHandle struct { // GUARDED_BY(mu) reader gcsx.RandomReader + // A readManager configured to some (potentially previous) generation of + // the object backing the inode, or nil. + // + // INVARIANT: If readManager != nil, readManager.CheckInvariants() doesn't panic. + // + // GUARDED_BY(mu) + readManager gcsx.ReadManager + + // MrdKernelReader is a reader that uses an MRD instance to read data from a GCS + // object. This reader is kernel-optimized & reads whatever is requested as is. + mrdKernelReader *gcsx.MrdKernelReader // fileCacheHandler is used to get file cache handle and read happens using that. // This will be nil if the file cache is disabled. - fileCacheHandler *file.CacheHandler + // Exactly one of these should be set: + fileCacheHandler *file.CacheHandler + SharedChunkCacheManager *file.SharedChunkCacheManager // cacheFileForRangeRead is also valid for cache workflow, if true, object content // will be downloaded for random reads as well too. cacheFileForRangeRead bool + metricHandle metrics.MetricHandle + traceHandle tracing.TraceHandle + + // openMode is used to store the mode in which the file is opened. + openMode util.OpenMode + + // Mount configuration. + config *cfg.Config + + // bufferedReadWorkerPool is used to execute download tasks for buffered reads. + bufferedReadWorkerPool workerpool.WorkerPool + + // globalMaxReadBlocksSem is a semaphore that limits the total number of blocks + // that can be allocated for buffered read across all files in the file system. + globalMaxReadBlocksSem *semaphore.Weighted + + // HandleID is an opaque 64-bit number used to create this File Handle, used for logging. + handleID fuseops.HandleID } -func NewFileHandle(inode *inode.FileInode, fileCacheHandler *file.CacheHandler, cacheFileForRangeRead bool) (fh *FileHandle) { +// LOCKS_REQUIRED(fh.inode.mu) +func NewFileHandle( + inode *inode.FileInode, + fileCacheHandler *file.CacheHandler, + sharedChunkCacheManager *file.SharedChunkCacheManager, + cacheFileForRangeRead bool, + metricHandle metrics.MetricHandle, + traceHandle tracing.TraceHandle, + openMode util.OpenMode, + c *cfg.Config, + bufferedReadWorkerPool workerpool.WorkerPool, + globalMaxReadBlocksSem *semaphore.Weighted, + handleID fuseops.HandleID, +) (fh *FileHandle) { fh = &FileHandle{ - inode: inode, - fileCacheHandler: fileCacheHandler, - cacheFileForRangeRead: cacheFileForRangeRead, + inode: inode, + fileCacheHandler: fileCacheHandler, + SharedChunkCacheManager: sharedChunkCacheManager, + cacheFileForRangeRead: cacheFileForRangeRead, + metricHandle: metricHandle, + traceHandle: traceHandle, + openMode: openMode, + config: c, + bufferedReadWorkerPool: bufferedReadWorkerPool, + globalMaxReadBlocksSem: globalMaxReadBlocksSem, + handleID: handleID, } + if c.FileSystem.EnableKernelReader { + fh.mrdKernelReader = gcsx.NewMrdKernelReader(inode.GetMRDInstance(), metricHandle) + } + + fh.inode.RegisterFileHandle(fh.openMode.AccessMode() == util.ReadOnly) fh.mu = syncutil.NewInvariantMutex(fh.checkInvariants) return @@ -61,10 +129,24 @@ func NewFileHandle(inode *inode.FileInode, fileCacheHandler *file.CacheHandler, // Destroy any resources associated with the handle, which must not be used // again. +// LOCKS_REQUIRED(fh.mu) +// LOCK_FUNCTION(fh.inode.mu) +// UNLOCK_FUNCTION(fh.inode.mu) func (fh *FileHandle) Destroy() { + // Deregister the fileHandle with the inode. + fh.inode.Lock() + fh.inode.DeRegisterFileHandle(fh.openMode.AccessMode() == util.ReadOnly) + fh.inode.Unlock() if fh.reader != nil { fh.reader.Destroy() } + if fh.readManager != nil { + fh.readManager.Destroy() + } + if fh.mrdKernelReader != nil { + fh.mrdKernelReader.Destroy() + fh.mrdKernelReader = nil + } } // Inode returns the inode backing this handle. @@ -80,47 +162,208 @@ func (fh *FileHandle) Unlock() { fh.mu.Unlock() } +// lockHandleAndRelockInode is a helper function which locks fh.mu maintaing the locking +// order, i.e. it first unlocks inode lock, then locks fh.mu (RLock or RWlock) and then +// relocks inode lock. +// LOCKS_REQUIRED(fh.inode.mu) +func (fh *FileHandle) lockHandleAndRelockInode(rLock bool) { + if rLock { + fh.inode.Unlock() + fh.mu.RLock() + fh.inode.Lock() + } else { + fh.inode.Unlock() + fh.mu.Lock() + fh.inode.Lock() + } +} + +// unlockHandleAndInode is a helper function which unlocks fh.inode.mu & fh.mu in order. +// LOCKS_REQUIRED(fh.mu) +// LOCKS_REQUIRED(fh.inode.mu) +func (fh *FileHandle) unlockHandleAndInode(rLock bool) { + if rLock { + fh.inode.Unlock() + fh.mu.RUnlock() + } else { + fh.inode.Unlock() + fh.mu.Unlock() + } +} + +// ReadWithReadManager reads data at the given offset using the read manager if available, +// falling back to inode.Read otherwise. It may be more efficient than directly calling inode.Read. +// +// LOCKS_REQUIRED(fh.inode.mu) +// UNLOCK_FUNCTION(fh.inode.mu) +func (fh *FileHandle) ReadWithReadManager(ctx context.Context, req *gcsx.ReadRequest, sequentialReadSizeMb int32) (gcsx.ReadResponse, error) { + // If content cache enabled, CacheEnsureContent forces the file handler to fall through to the inode + // and fh.inode.SourceGenerationIsAuthoritative() will return false + if err := fh.inode.CacheEnsureContent(ctx); err != nil { + fh.inode.Unlock() + return gcsx.ReadResponse{}, fmt.Errorf("failed to ensure inode content: %w", err) + } + + if !fh.inode.SourceGenerationIsAuthoritative() { + // Read from inode if source generation is not authoratative + defer fh.inode.Unlock() + n, err := fh.inode.Read(ctx, req.Buffer, req.Offset) + return gcsx.ReadResponse{Size: n}, err + } + + fh.lockHandleAndRelockInode(true) + defer fh.mu.RUnlock() + + // If the inode is dirty, there's nothing we can do. Throw away our readManager if + // we have one & create a new readManager + if fh.isValidReadManager() { + fh.inode.Unlock() + } else { + minObj := fh.inode.Source() + bucket := fh.inode.Bucket() + mrdWrapper := fh.inode.MRDWrapper + + // Acquire a RWLock on file handle as we will update readManager + fh.unlockHandleAndInode(true) + fh.mu.Lock() + + fh.destroyReadManager() + // Create a new read manager for the current inode state. + fh.readManager = read_manager.NewReadManager(minObj, bucket, &read_manager.ReadManagerConfig{ + SequentialReadSizeMB: sequentialReadSizeMb, + FileCacheHandler: fh.fileCacheHandler, + SharedChunkCacheManager: fh.SharedChunkCacheManager, + CacheFileForRangeRead: fh.cacheFileForRangeRead, + MetricHandle: fh.metricHandle, + TraceHandle: fh.traceHandle, + MrdWrapper: mrdWrapper, + Config: fh.config, + GlobalMaxBlocksSem: fh.globalMaxReadBlocksSem, + WorkerPool: fh.bufferedReadWorkerPool, + HandleID: fh.handleID, + InitialOffset: req.Offset, + }) + + // Override the read-manager with visual-read-manager (a wrapper over read_manager with visualizer) if configured. + if fh.config.WorkloadInsight.Visualize { + if renderer, err := workloadinsight.NewRenderer(); err == nil { + fh.readManager = read_manager.NewVisualReadManager(fh.readManager, renderer, fh.config.WorkloadInsight) + } else { + logger.Warnf("Failed to construct workload insight visualizer: %v", err) + } + } + + // Release RWLock and take RLock on file handle again. Inode lock is not needed now. + fh.mu.Unlock() + fh.mu.RLock() + } + + // For unfinalized objects, we may need to read past the known size. + // Update the request to skip size checks if required. + req.SkipSizeChecks = fh.shouldSkipSizeChecks(req) + + // Use the readManager to read data. + var readResponse gcsx.ReadResponse + readResponse, err := fh.readManager.ReadAt(ctx, req) + switch { + case errors.Is(err, io.EOF): + if err != io.EOF { + logger.Warnf("Unexpected EOF error encountered while reading, err: %v type: %T ", err, err) + } + return gcsx.ReadResponse{}, io.EOF + + case err != nil: + return gcsx.ReadResponse{}, fmt.Errorf("fh.readManager.ReadAt: %w", err) + } + + return readResponse, nil +} + +// ReadWithMrdKernelReader reads data at the given offset using the mrd kernel reader. +// +// LOCKS_REQUIRED(fh.inode.mu) +// UNLOCK_FUNCTION(fh.inode.mu) +func (fh *FileHandle) ReadWithMrdKernelReader(ctx context.Context, req *gcsx.ReadRequest) (gcsx.ReadResponse, error) { + if !fh.inode.SourceGenerationIsAuthoritative() { + // Read from inode if source generation is not authoritative. + defer fh.inode.Unlock() + n, err := fh.inode.Read(ctx, req.Buffer, req.Offset) + return gcsx.ReadResponse{Size: n}, err + } + fh.inode.Unlock() + + fh.mu.RLock() + defer fh.mu.RUnlock() + + if fh.mrdKernelReader == nil { + return gcsx.ReadResponse{}, errors.New("mrdKernelReader is not initialized") + } + + return fh.mrdKernelReader.ReadAt(ctx, req) +} + // Equivalent to locking fh.Inode() and calling fh.Inode().Read, but may be // more efficient. // -// LOCKS_REQUIRED(fh) -// LOCKS_EXCLUDED(fh.inode) -func (fh *FileHandle) Read(ctx context.Context, dst []byte, offset int64, sequentialReadSizeMb int32) (n int, err error) { - // Lock the inode and attempt to ensure that we have a reader for its current - // state, or clear fh.reader if it's not possible to create one (probably - // because the inode is dirty). - fh.inode.Lock() - err = fh.tryEnsureReader(ctx, sequentialReadSizeMb) +// LOCKS_REQUIRED(fh.inode.mu) +// UNLOCK_FUNCTION(fh.inode.mu) +func (fh *FileHandle) Read(ctx context.Context, dst []byte, offset int64, sequentialReadSizeMb int32) (output []byte, n int, err error) { + // If content cache enabled, CacheEnsureContent forces the file handler to fall through to the inode + // and fh.inode.SourceGenerationIsAuthoritative() will return false + err = fh.inode.CacheEnsureContent(ctx) if err != nil { fh.inode.Unlock() - err = fmt.Errorf("tryEnsureReader: %w", err) - return + return nil, 0, fmt.Errorf("failed to ensure inode content: %w", err) } - // If we have an appropriate reader, unlock the inode and use that. This - // allows reads to proceed concurrently with other operations; in particular, - // multiple reads can run concurrently. It's safe because the user can't tell - // if a concurrent write started during or after a read. - if fh.reader != nil { + // If the inode is dirty, there's nothing we can do. Throw away our reader if + // we have one. + if !fh.inode.SourceGenerationIsAuthoritative() { + defer fh.inode.Unlock() + n, err = fh.inode.Read(ctx, dst, offset) + return dst, n, err + } + + fh.lockHandleAndRelockInode(true) + defer fh.mu.RUnlock() + + if fh.isValidReader() { fh.inode.Unlock() + } else { + minObj := fh.inode.Source() + bucket := fh.inode.Bucket() + mrdWrapper := fh.inode.MRDWrapper + + // Acquire a RWLock on file handle as we will update reader + fh.unlockHandleAndInode(true) + fh.mu.Lock() - n, _, err = fh.reader.ReadAt(ctx, dst, offset) - switch { - case err == io.EOF: - return + fh.destroyReader() + fh.reader = gcsx.NewRandomReader(minObj, bucket, sequentialReadSizeMb, fh.fileCacheHandler, fh.cacheFileForRangeRead, fh.metricHandle, fh.traceHandle, mrdWrapper, fh.config, fh.handleID) - case err != nil: - err = fmt.Errorf("fh.reader.ReadAt: %w", err) - return + // Release RWLock and take RLock on file handle again + fh.mu.Unlock() + fh.mu.RLock() + } + + // Use the reader to read data. + var objectData gcsx.ObjectData + objectData, err = fh.reader.ReadAt(ctx, dst, offset) + switch { + case errors.Is(err, io.EOF): + if err != io.EOF { + logger.Warnf("Unexpected EOF error encountered while reading, err: %v type: %T ", err, err) + err = io.EOF } + return + case err != nil: + err = fmt.Errorf("fh.reader.ReadAt: %w", err) return } - // Otherwise we must fall through to the inode. - defer fh.inode.Unlock() - n, err = fh.inode.Read(ctx, dst, offset) - + output = objectData.DataBuf + n = objectData.Size return } @@ -134,44 +377,89 @@ func (fh *FileHandle) checkInvariants() { if fh.reader != nil { fh.reader.CheckInvariants() } + + // INVARIANT: If readManager != nil, readManager.CheckInvariants() doesn't panic. + if fh.readManager != nil { + fh.readManager.CheckInvariants() + } } -// If possible, ensure that fh.reader is set to an appropriate random reader -// for the current state of the inode. Otherwise set it to nil. -// -// LOCKS_REQUIRED(fh) -// LOCKS_REQUIRED(fh.inode) -func (fh *FileHandle) tryEnsureReader(ctx context.Context, sequentialReadSizeMb int32) (err error) { - // If content cache enabled, CacheEnsureContent forces the file handler to fall through to the inode - // and fh.inode.SourceGenerationIsAuthoritative() will return false - err = fh.inode.CacheEnsureContent(ctx) - if err != nil { +// destroyReadManager is a helper function to safely destroy the readManager & set it to nil. +// LOCKS_REQUIRED(fh.mu) +// LOCKS_REQUIRED(fh.inode.mu) +func (fh *FileHandle) destroyReadManager() { + if fh.readManager == nil { return } - // If the inode is dirty, there's nothing we can do. Throw away our reader if - // we have one. - if !fh.inode.SourceGenerationIsAuthoritative() { - if fh.reader != nil { - fh.reader.Destroy() - fh.reader = nil - } + fh.readManager.Destroy() + fh.readManager = nil +} +// isValidReadManager is a helper function which validates & returns whether the +// current readManager is valid or not. +// LOCKS_REQUIRED(fh.mu.RLock) +// LOCKS_REQUIRED(fh.inode.mu) +func (fh *FileHandle) isValidReadManager() bool { + // If we already have a readManager, and it's at the appropriate generation, we + // can use it otherwise we must throw it away. + if fh.readManager != nil && fh.readManager.Object().Generation == fh.inode.SourceGeneration().Object { + // Update reader object size to source object size. + fh.readManager.Object().Size = fh.inode.SourceGeneration().Size + return true + } + return false +} + +// destroyReader is a helper function to safely destroy the reader and set it to nil. +// LOCKS_REQUIRED(fh.mu) +// LOCKS_REQUIRED(fh.inode.mu) +func (fh *FileHandle) destroyReader() { + if fh.reader == nil { return } + fh.reader.Destroy() + fh.reader = nil +} +// isValidReader is a helper function which validates & returns whether the +// current reader is valid or not. +// LOCKS_REQUIRED(fh.mu.RLock) +// LOCKS_REQUIRED(fh.inode.mu) +func (fh *FileHandle) isValidReader() bool { // If we already have a reader, and it's at the appropriate generation, we - // can use it. Otherwise we must throw it away. - if fh.reader != nil { - if fh.reader.Object().Generation == fh.inode.SourceGeneration().Object { - return - } - fh.reader.Destroy() - fh.reader = nil + // can use it otherwise we must throw it away. + if fh.reader != nil && fh.reader.Object().Generation == fh.inode.SourceGeneration().Object { + // Update reader object size to source object size. + fh.reader.Object().Size = fh.inode.SourceGeneration().Size + return true } + return false +} - // Attempt to create an appropriate reader. - rr := gcsx.NewRandomReader(fh.inode.Source(), fh.inode.Bucket(), sequentialReadSizeMb, fh.fileCacheHandler, fh.cacheFileForRangeRead) +func (fh *FileHandle) OpenMode() util.OpenMode { + return fh.openMode +} - fh.reader = rr - return +// shouldSkipSizeChecks determines if the read request should skip object size checks. +// This is true for direct I/O reads on unfinalized objects that extend beyond +// the known object size. This allows reading from an object that is being +// concurrently written to. +func (fh *FileHandle) shouldSkipSizeChecks(req *gcsx.ReadRequest) bool { + if !fh.inode.Bucket().BucketType().RapidWritesEnabled() { + return false + } + if !fh.readManager.Object().IsUnfinalized() { + return false + } + if !fh.OpenMode().IsDirect() { + return false + } + if req.Offset < 0 { + return false + } + if req.Offset+int64(len(req.Buffer)) <= int64(fh.readManager.Object().Size) { + return false + } + + return true } diff --git a/internal/fs/handle/file_test.go b/internal/fs/handle/file_test.go new file mode 100644 index 0000000000..105f1b7d71 --- /dev/null +++ b/internal/fs/handle/file_test.go @@ -0,0 +1,1376 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handle + +import ( + "bytes" + "context" + crypto_rand "crypto/rand" + "errors" + "fmt" + "io" + "math/rand" + "os" + "sync" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/contentcache" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/inode" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx/read_manager" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/workerpool" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/jacobsa/fuse/fuseops" + "github.com/jacobsa/timeutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "golang.org/x/sync/semaphore" +) + +const testDirName = "parentRoot" +const isTypeCacheDeprecationEnabled = false + +var readMode = util.NewOpenMode(util.ReadOnly, 0) +var writeMode = util.NewOpenMode(util.WriteOnly, 0) + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +type fileTest struct { + suite.Suite + ctx context.Context + clock timeutil.SimulatedClock + bucket gcsx.SyncerBucket +} + +func TestFileTestSuite(t *testing.T) { + suite.Run(t, new(fileTest)) +} + +func (t *fileTest) SetupTest() { + t.ctx = context.TODO() + t.clock.SetTime(time.Date(2015, 4, 5, 2, 15, 0, 0, time.Local)) + t.bucket = gcsx.NewSyncerBucket( + /*appendThreshold=*/ 1, + /*chunkRetryDeadlineSecs=*/ 120, + /*chunkTransferTimeoutSecs=*/ 10, + ".gcsfuse_tmp/", fake.NewFakeBucket(&t.clock, "some_bucket", gcs.BucketType{}), + ) +} + +func (t *fileTest) TearDownTest() { +} + +//////////////////////////////////////////////////////////////////////// +// Helpers +//////////////////////////////////////////////////////////////////////// + +// createDirInode helps create the parent directory inode for the file inode +// which will be used for testing methods defined on the fileHandle. +func createDirInode( + bucket *gcsx.SyncerBucket, + clock *timeutil.SimulatedClock) inode.DirInode { + config := &cfg.Config{ + List: cfg.ListConfig{EnableEmptyManagedFolders: false}, + MetadataCache: cfg.MetadataCacheConfig{TypeCacheMaxSizeMb: 4}, + EnableHns: false, + EnableUnsupportedPathSupport: true, + EnableTypeCacheDeprecation: isTypeCacheDeprecationEnabled, + } + + parInodeCtx := context.Background() + return inode.NewDirInode( + 1, + inode.NewDirName(inode.NewRootName(""), testDirName), + parInodeCtx, + fuseops.InodeAttributes{ + Uid: 0, + Gid: 0, + Mode: 0712, + }, + false, + true, + 0, + bucket, + clock, + clock, + semaphore.NewWeighted(10), + config, + ) +} + +// createFileInode is a helper to create a FileInode for testing. +func createFileInode( + t *testing.T, + bucket *gcsx.SyncerBucket, + clock *timeutil.SimulatedClock, + config *cfg.Config, + parent inode.DirInode, + objectName string, + content []byte, + localFileCache bool) *inode.FileInode { + + fullObjectName := parent.Name().GcsObjectName() + objectName + obj := &gcs.MinObject{ + Name: fullObjectName, + Size: uint64(len(content)), + Generation: 1, + MetaGeneration: 1, + Updated: clock.Now(), + } + + // Create object in the fake bucket to simulate existing GCS object + _, err := bucket.CreateObject(context.Background(), &gcs.CreateObjectRequest{ + Name: fullObjectName, + Contents: io.NopCloser(bytes.NewReader(content)), + }) + if err != nil { + t.Fatalf("Failed to create object in fake bucket: %v", err) + } + + return inode.NewFileInode( + fuseops.InodeID(2), + inode.NewFileName(parent.Name(), objectName), + obj, + fuseops.InodeAttributes{}, + bucket, + localFileCache, + contentcache.New("", clock), + clock, + false, + config, + semaphore.NewWeighted(100), + nil, + tracing.NewNoopTracer(), + metrics.NewNoopMetrics(), + ) +} + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func (t *fileTest) Test_IsValidReadManager_NilReadManager() { + parent := createDirInode(&t.bucket, &t.clock) + config := &cfg.Config{} + const objectName = "test_obj" + const objectContent = "some data" + in := createFileInode(t.T(), &t.bucket, &t.clock, config, parent, objectName, []byte(objectContent), false) + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, config, nil, nil, 0) + fh.inode.Lock() + defer fh.inode.Unlock() + fh.readManager = nil + + result := fh.isValidReadManager() + + assert.False(t.T(), result) +} + +func (t *fileTest) Test_IsValidReadManager_GenerationValidation() { + parent := createDirInode(&t.bucket, &t.clock) + config := &cfg.Config{} + const objectName = "test_obj" + const objectContent = "some data" + in := createFileInode(t.T(), &t.bucket, &t.clock, config, parent, objectName, []byte(objectContent), false) + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, config, nil, nil, 0) + fh.inode.Lock() + defer fh.inode.Unlock() + + testCases := []struct { + name string + readerGeneration int64 + expectedIsValid bool + }{ + { + name: "Generation mismatch", + readerGeneration: 2, // Inode has generation 1 + expectedIsValid: false, + }, + { + name: "Generation match", + readerGeneration: 1, // Inode has generation 1 + expectedIsValid: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + minObj := in.Source() + minObj.Generation = tc.readerGeneration + fh.readManager = read_manager.NewReadManager(minObj, &t.bucket, &read_manager.ReadManagerConfig{Config: config}) + + result := fh.isValidReadManager() + + assert.Equal(t.T(), tc.expectedIsValid, result) + }) + } +} + +func (t *fileTest) Test_IsValidReader_NilReader() { + parent := createDirInode(&t.bucket, &t.clock) + config := &cfg.Config{} + const objectName = "test_obj" + const objectContent = "some data" + in := createFileInode(t.T(), &t.bucket, &t.clock, config, parent, objectName, []byte(objectContent), false) + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, config, nil, nil, 0) + fh.inode.Lock() + defer fh.inode.Unlock() + fh.reader = nil + + result := fh.isValidReader() + + assert.False(t.T(), result) +} + +func (t *fileTest) Test_IsValidReader_GenerationValidation() { + parent := createDirInode(&t.bucket, &t.clock) + config := &cfg.Config{} + const objectName = "test_obj" + const objectContent = "some data" + in := createFileInode(t.T(), &t.bucket, &t.clock, config, parent, objectName, []byte(objectContent), false) + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, config, nil, nil, 0) + fh.inode.Lock() + defer fh.inode.Unlock() + + testCases := []struct { + name string + readerGeneration int64 + expectedIsValid bool + }{ + { + name: "Generation mismatch", + readerGeneration: 2, // Inode has generation 1 + expectedIsValid: false, + }, + { + name: "Generation match", + readerGeneration: 1, // Inode has generation 1 + expectedIsValid: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + minObj := in.Source() + minObj.Generation = tc.readerGeneration + fh.reader = gcsx.NewRandomReader(minObj, &t.bucket, 200, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), in.MRDWrapper, config, 0) + + result := fh.isValidReader() + + assert.Equal(t.T(), tc.expectedIsValid, result) + }) + } +} + +// Test_Read_Success validates successful read behavior using the random reader. +func (t *fileTest) Test_Read_Success() { + expectedData := []byte("hello from reader") + parent := createDirInode(&t.bucket, &t.clock) + in := createFileInode(t.T(), &t.bucket, &t.clock, &cfg.Config{}, parent, "test_obj_reader", expectedData, false) + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, &cfg.Config{}, nil, nil, 0) + buf := make([]byte, len(expectedData)) + fh.inode.Lock() + + output, n, err := fh.Read(t.ctx, buf, 0, 200) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), len(expectedData), n) + assert.Equal(t.T(), expectedData, output) +} + +// Test_ReadWithReadManager_Success validates successful read behavior using the readManager. +func (t *fileTest) Test_ReadWithReadManager_Success() { + expectedData := []byte("hello from readManager") + parent := createDirInode(&t.bucket, &t.clock) + in := createFileInode(t.T(), &t.bucket, &t.clock, &cfg.Config{}, parent, "test_obj_readManager", expectedData, false) + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, &cfg.Config{}, nil, nil, 0) + buf := make([]byte, len(expectedData)) + fh.inode.Lock() + + resp, err := fh.ReadWithReadManager(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: 0, + }, 200) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), len(expectedData), resp.Size) + assert.Equal(t.T(), expectedData, buf[:resp.Size]) +} + +// Test_ReadWithReadManager_Concurrent validates concurrent read behavior using the readManager. +func (t *fileTest) Test_ReadWithReadManager_Concurrent() { + // Setup + const ( + fileSize = 1 * 1024 * 1024 // 1 MiB + numReaders = 20 + maxReadSize = 16 * 1024 // 16 KiB + ) + // Create large content for the file. + objectContent := make([]byte, fileSize) + _, err := crypto_rand.Read(objectContent) + assert.NoError(t.T(), err) + + parent := createDirInode(&t.bucket, &t.clock) + in := createFileInode(t.T(), &t.bucket, &t.clock, &cfg.Config{}, parent, "concurrent_read_obj", objectContent, false) + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, &cfg.Config{}, nil, nil, 0) + + var wg sync.WaitGroup + wg.Add(numReaders) + + // Run concurrent reads + for range numReaders { + go func() { + defer wg.Done() + + // Each goroutine reads a random chunk. + readSize := rand.Intn(maxReadSize-1) + 1 // Ensure readSize > 0 + offset := rand.Intn(fileSize - readSize) + dst := make([]byte, readSize) + + fh.inode.Lock() // Lock required by ReadWithReadManager + // The method is responsible for unlocking. + resp, err := fh.ReadWithReadManager(t.ctx, &gcsx.ReadRequest{ + Buffer: dst, + Offset: int64(offset), + }, 200) + + // Assertions + assert.NoError(t.T(), err) + assert.Equal(t.T(), readSize, resp.Size) + assert.Equal(t.T(), objectContent[offset:offset+readSize], dst[:resp.Size]) + }() + } + + // Wait for all goroutines to finish, with a timeout to detect deadlocks. + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + select { + case <-done: + // Test completed successfully. + case <-time.After(10 * time.Second): + t.T().Fatal("Test timed out, potential deadlock in ReadWithReadManager") + } +} + +// Test_Read_Concurrent validates concurrent read behavior using the random reader +func (t *fileTest) Test_Read_Concurrent() { + // Setup + const ( + fileSize = 1 * 1024 * 1024 // 1 MiB + numReaders = 20 + maxReadSize = 16 * 1024 // 16 KiB + ) + // Create large content for the file. + objectContent := make([]byte, fileSize) + _, err := crypto_rand.Read(objectContent) + assert.NoError(t.T(), err) + + parent := createDirInode(&t.bucket, &t.clock) + in := createFileInode(t.T(), &t.bucket, &t.clock, &cfg.Config{}, parent, "concurrent_read_obj", objectContent, false) + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, &cfg.Config{}, nil, nil, 0) + + var wg sync.WaitGroup + wg.Add(numReaders) + + // Run concurrent reads + for range numReaders { + go func() { + defer wg.Done() + + // Each goroutine reads a random chunk. + readSize := rand.Intn(maxReadSize-1) + 1 // Ensure readSize > 0 + offset := rand.Intn(fileSize - readSize) + dst := make([]byte, readSize) + + fh.inode.Lock() // Lock required by ReadWithReadManager + // The method is responsible for unlocking. + _, n, err := fh.Read(t.ctx, dst, int64(offset), 200) + + // Assertions + assert.NoError(t.T(), err) + assert.Equal(t.T(), readSize, n) + assert.Equal(t.T(), objectContent[offset:offset+readSize], dst[:n]) + }() + } + + // Wait for all goroutines to finish, with a timeout to detect deadlocks. + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + select { + case <-done: + // Test completed successfully. + case <-time.After(10 * time.Second): + t.T().Fatal("Test timed out, potential deadlock in ReadWithReadManager") + } +} + +// Test_ReadWithReadManager_ErrorScenarios verifies error handling in ReadWithReadManager. +func (t *fileTest) Test_ReadWithReadManager_ErrorScenarios() { + type testCase struct { + name string + returnErr error + } + + object := gcs.MinObject{Name: "test_obj", Generation: 1} + mockErr := fmt.Errorf("mock error") + dst := make([]byte, 100) + + testCases := []testCase{ + {name: "EOF via readManager", returnErr: io.EOF}, + {name: "mock error via readManager", returnErr: mockErr}, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.SetupTest() + parent := createDirInode(&t.bucket, &t.clock) + testInode := createFileInode(t.T(), &t.bucket, &t.clock, &cfg.Config{}, parent, object.Name, []byte("data"), false) + fh := NewFileHandle(testInode, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, &cfg.Config{}, nil, nil, 0) + fh.inode.Lock() + mockRM := new(read_manager.MockReadManager) + mockRM.On("ReadAt", t.ctx, mock.AnythingOfType("*gcsx.ReadRequest")).Return(gcsx.ReadResponse{}, tc.returnErr) + mockRM.On("Object").Return(&object) + fh.readManager = mockRM + + resp, err := fh.ReadWithReadManager(t.ctx, &gcsx.ReadRequest{ + Buffer: dst, + Offset: 0, + }, 200) + + assert.Zero(t.T(), resp.Size, "expected 0 bytes read") + assert.True(t.T(), errors.Is(err, tc.returnErr), "expected error to match") + mockRM.AssertExpectations(t.T()) + }) + } +} + +// Test_Read_ErrorScenarios verifies error handling in Read (random reader). +func (t *fileTest) Test_Read_ErrorScenarios() { + type testCase struct { + name string + returnErr error + } + + object := gcs.MinObject{Name: "test_obj", Generation: 1} + mockErr := fmt.Errorf("mock error") + dst := make([]byte, 100) + + testCases := []testCase{ + {name: "EOF via random reader", returnErr: io.EOF}, + {name: "mock error via random reader", returnErr: mockErr}, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.SetupTest() + parent := createDirInode(&t.bucket, &t.clock) + testInode := createFileInode(t.T(), &t.bucket, &t.clock, &cfg.Config{}, parent, object.Name, []byte("data"), false) + fh := NewFileHandle(testInode, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, &cfg.Config{}, nil, nil, 0) + fh.inode.Lock() + mockReader := new(gcsx.MockRandomReader) + mockReader.On("ReadAt", t.ctx, dst, int64(0)).Return(gcsx.ObjectData{}, tc.returnErr) + mockReader.On("Object").Return(&object) + fh.reader = mockReader + + output, n, err := fh.Read(t.ctx, dst, 0, 200) + + assert.Zero(t.T(), n, "expected 0 bytes read") + assert.Nil(t.T(), output, "expected output to be nil") + assert.True(t.T(), errors.Is(err, tc.returnErr), "expected error to match") + mockReader.AssertExpectations(t.T()) + }) + } +} + +// Test_ReadWithReadManager_FallbackToInode verifies that ReadWithReadManager +// falls back to inode object data when readManager is not valid. +func (t *fileTest) Test_ReadWithReadManager_FallbackToInode() { + dst := make([]byte, 100) + objectData := []byte("fallback data") + object := gcs.MinObject{Name: "test_obj"} + parent := createDirInode(&t.bucket, &t.clock) + in := createFileInode(t.T(), &t.bucket, &t.clock, &cfg.Config{}, parent, object.Name, objectData, true) + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, &cfg.Config{}, nil, nil, 0) + fh.inode.Lock() + mockRM := new(read_manager.MockReadManager) + fh.readManager = mockRM + + resp, err := fh.ReadWithReadManager(t.ctx, &gcsx.ReadRequest{ + Buffer: dst, + Offset: 0, + }, 200) + + assert.Equal(t.T(), io.EOF, err) + assert.Equal(t.T(), len(objectData), resp.Size) + assert.Equal(t.T(), objectData, dst[:resp.Size]) + mockRM.AssertExpectations(t.T()) +} + +// Test_Read_FallbackToInode verifies that Read falls back to inode object data +// when reader is not valid. +func (t *fileTest) Test_Read_FallbackToInode() { + dst := make([]byte, 100) + objectData := []byte("fallback data") + object := gcs.MinObject{Name: "test_obj"} + parent := createDirInode(&t.bucket, &t.clock) + in := createFileInode(t.T(), &t.bucket, &t.clock, &cfg.Config{}, parent, object.Name, objectData, true) + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, &cfg.Config{}, nil, nil, 0) + fh.inode.Lock() + mockR := new(gcsx.MockRandomReader) + fh.reader = mockR + + output, n, err := fh.Read(t.ctx, dst, 0, 200) + + assert.Equal(t.T(), io.EOF, err) + assert.Equal(t.T(), len(objectData), n) + assert.Equal(t.T(), objectData, output[:n]) + mockR.AssertExpectations(t.T()) +} + +func (t *fileTest) Test_ReadWithReadManager_ReadManagerInvalidatedByGenerationChange() { + content1 := []byte("content1") + content2 := []byte("content2-larger") + dst := make([]byte, len(content2)) + objectName := "test_obj_rm_gen_change" + + parent := createDirInode(&t.bucket, &t.clock) + in := createFileInode(t.T(), &t.bucket, &t.clock, &cfg.Config{}, parent, objectName, content1, false) + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, &cfg.Config{}, nil, nil, 0) + + // First read, to create a readManager. + fh.inode.Lock() + _, err := fh.ReadWithReadManager(t.ctx, &gcsx.ReadRequest{ + Buffer: make([]byte, len(content1)), + Offset: 0, + }, 200) + assert.NoError(t.T(), err) + assert.NotNil(t.T(), fh.readManager) + oldReadManager := fh.readManager + + // Now, update the object in GCS, which changes its generation. + in.Lock() + gcsSynced, err := in.Write(t.ctx, content2, 0, writeMode) + assert.NoError(t.T(), err) + assert.False(t.T(), gcsSynced) + gcsSynced, err = in.Sync(t.ctx) + assert.NoError(t.T(), err) + assert.True(t.T(), gcsSynced) + in.Unlock() + + // The existing readManager is now for an old generation. + // The next ReadWithReadManager call should detect this, destroy the old one, + // create a new one, and read the new content. + fh.inode.Lock() + resp, err := fh.ReadWithReadManager(t.ctx, &gcsx.ReadRequest{ + Buffer: dst, + Offset: 0, + }, 200) + + assert.NoError(t.T(), err) + assert.NotNil(t.T(), fh.readManager) + assert.NotEqual(t.T(), oldReadManager, fh.readManager) + assert.Equal(t.T(), len(content2), resp.Size) + assert.Equal(t.T(), content2, dst) +} + +func (t *fileTest) Test_Read_ReaderInvalidatedByGenerationChange() { + content1 := []byte("content1") + content2 := []byte("content2-larger") + dst := make([]byte, len(content2)) + objectName := "test_obj_rm_gen_change" + + parent := createDirInode(&t.bucket, &t.clock) + in := createFileInode(t.T(), &t.bucket, &t.clock, &cfg.Config{}, parent, objectName, content1, false) + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, &cfg.Config{}, nil, nil, 0) + + // First read, to create a reader. + fh.inode.Lock() + _, _, err := fh.Read(t.ctx, make([]byte, len(content1)), 0, 200) + assert.NoError(t.T(), err) + assert.NotNil(t.T(), fh.reader) + oldReader := fh.reader + + // Now, update the object in GCS, which changes its generation. + in.Lock() + gcsSynced, err := in.Write(t.ctx, content2, 0, writeMode) + assert.NoError(t.T(), err) + assert.False(t.T(), gcsSynced) + gcsSynced, err = in.Sync(t.ctx) + assert.NoError(t.T(), err) + assert.True(t.T(), gcsSynced) + in.Unlock() + + // The existing reader is now for an old generation. + // The next Read call should detect this, destroy the old one, + // create a new one, and read the new content. + fh.inode.Lock() + output, n, err := fh.Read(t.ctx, dst, 0, 200) + + assert.NoError(t.T(), err) + assert.NotNil(t.T(), fh.reader) + assert.NotEqual(t.T(), oldReader, fh.reader) + assert.Equal(t.T(), len(content2), n) + assert.Equal(t.T(), content2, output) +} + +func (t *fileTest) Test_ReadWithMrdKernelReader_Success() { + // 1. Setup + expectedData := []byte("hello from mrd reader") + objectName := "test_obj_mrd_reader" + // Create a mock bucket that behaves like a zonal bucket. + mockBucket := new(storage.TestifyMockBucket) + mockBucket.On("BucketType").Return(gcs.BucketType{Zonal: true}) + // Mock CreateObject which is called by createFileInode. + mockBucket.On("CreateObject", mock.Anything, mock.Anything).Return(&gcs.Object{}, nil).Once() + // Create File Inode. + mockSyncerBucket := gcsx.NewSyncerBucket( + /*appendThreshold=*/ 1, + /*chunkRetryDeadlineSecs=*/ 120, + /*chunkTransferTimeoutSecs=*/ 10, + ".gcsfuse_tmp/", mockBucket, + ) + parent := createDirInode(&mockSyncerBucket, &t.clock) + in := createFileInode(t.T(), &mockSyncerBucket, &t.clock, &cfg.Config{}, parent, objectName, expectedData, false) + // Create File Handle. + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, &cfg.Config{FileSystem: cfg.FileSystemConfig{EnableKernelReader: true}}, nil, nil, 0) + require.NotNil(t.T(), fh.mrdKernelReader) + // Mock the downloader that mrdKernelReader will use. + fakeMRD := fake.NewFakeMultiRangeDownloader(in.Source(), expectedData) + mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + // Create read request and take inode lock. + buf := make([]byte, len(expectedData)) + req := &gcsx.ReadRequest{ + Buffer: buf, + Offset: 0, + } + fh.inode.Lock() // Required by the function signature. + + // 2. Call ReadWithMrdKernelReader. + resp, err := fh.ReadWithMrdKernelReader(t.ctx, req) + + // 3. Assertions + assert.NoError(t.T(), err) + assert.Equal(t.T(), len(expectedData), resp.Size) + assert.Equal(t.T(), expectedData, buf[:resp.Size]) + mockBucket.AssertExpectations(t.T()) +} + +func (t *fileTest) Test_ReadWithMrdKernelReader_NotAuthoritative() { + // 1. Setup + zonalBucket := gcsx.NewSyncerBucket( + /*appendThreshold=*/ 1, + /*chunkRetryDeadlineSecs=*/ 120, + /*chunkTransferTimeoutSecs=*/ 10, + ".gcsfuse_tmp/", fake.NewFakeBucket(&t.clock, "zonal_bucket", gcs.BucketType{Zonal: true}), + ) + originalData := []byte("some data") // 9 bytes + parent := createDirInode(&zonalBucket, &t.clock) + in := createFileInode(t.T(), &zonalBucket, &t.clock, &cfg.Config{FileSystem: cfg.FileSystemConfig{EnableKernelReader: true}}, parent, "test_obj", originalData, false) + // Make inode dirty. + in.Lock() + _, err := in.Write(t.ctx, []byte("dirty"), 0, writeMode) // 5 bytes + in.Unlock() + require.NoError(t.T(), err) + // After write, content should be "dirtydata". + expectedReadData := "dirtydata" + // Create file handle. + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, &cfg.Config{FileSystem: cfg.FileSystemConfig{EnableKernelReader: true}}, nil, nil, 0) + require.NotNil(t.T(), fh.mrdKernelReader) + // Create read request and take inode lock. + buf := make([]byte, len(expectedReadData)) + req := &gcsx.ReadRequest{ + Buffer: buf, + Offset: 0, + } + fh.inode.Lock() + + // 2. Call ReadWithMrdKernelReader + resp, err := fh.ReadWithMrdKernelReader(t.ctx, req) + + // 3. Assertions + // It should read from inode, which contains "dirty data". + assert.NoError(t.T(), err) + assert.Equal(t.T(), len(expectedReadData), resp.Size) + assert.Equal(t.T(), expectedReadData, string(buf[:resp.Size])) +} + +func (t *fileTest) Test_ReadWithMrdKernelReader_NilReader() { + // 1. Setup with a non-zonal bucket. + nonZonalBucket := gcsx.NewSyncerBucket( + /*appendThreshold=*/ 1, + /*chunkRetryDeadlineSecs=*/ 120, + /*chunkTransferTimeoutSecs=*/ 10, + ".gcsfuse_tmp/", fake.NewFakeBucket(&t.clock, "non_zonal_bucket", gcs.BucketType{Zonal: true}), + ) + parent := createDirInode(&nonZonalBucket, &t.clock) + in := createFileInode(t.T(), &nonZonalBucket, &t.clock, &cfg.Config{}, parent, "test_obj", []byte("data"), false) + // Create file handle. + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, &cfg.Config{FileSystem: cfg.FileSystemConfig{EnableKernelReader: false}}, nil, nil, 0) + require.Nil(t.T(), fh.mrdKernelReader) + // Create read request and take inode lock. + req := &gcsx.ReadRequest{ + Buffer: make([]byte, 4), + Offset: 0, + } + fh.inode.Lock() + + // 2. Call ReadWithMrdKernelReader. + _, err := fh.ReadWithMrdKernelReader(t.ctx, req) + + // 3. Assertions + assert.Error(t.T(), err) + assert.Equal(t.T(), "mrdKernelReader is not initialized", err.Error()) +} + +func (t *fileTest) Test_ReadWithMrdKernelReader_ReadAtError() { + // 1. Setup + expectedData := []byte("hello from mrd reader") + objectName := "test_obj_mrd_reader_error" + // Mock required functions from mock bucket. + mockBucket := new(storage.TestifyMockBucket) + mockBucket.On("BucketType").Return(gcs.BucketType{Zonal: true}) + mockBucket.On("CreateObject", mock.Anything, mock.Anything).Return(&gcs.Object{}, nil).Once() + // Create file inode. + mockSyncerBucket := gcsx.NewSyncerBucket( + /*appendThreshold=*/ 1, + /*chunkRetryDeadlineSecs=*/ 120, + /*chunkTransferTimeoutSecs=*/ 10, + ".gcsfuse_tmp/", mockBucket, + ) + parent := createDirInode(&mockSyncerBucket, &t.clock) + in := createFileInode(t.T(), &mockSyncerBucket, &t.clock, &cfg.Config{FileSystem: cfg.FileSystemConfig{EnableKernelReader: true}}, parent, objectName, expectedData, false) + // Create file handle. + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, &cfg.Config{FileSystem: cfg.FileSystemConfig{EnableKernelReader: true}}, nil, nil, 0) + require.NotNil(t.T(), fh.mrdKernelReader) + // Mock the downloader to return an error. + expectedErr := errors.New("mrd read error") + fakeMRD := fake.NewFakeMultiRangeDownloaderWithSleepAndDefaultError(in.Source(), expectedData, 0, expectedErr) + mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + // Create read request & take inode lock. + buf := make([]byte, len(expectedData)) + req := &gcsx.ReadRequest{ + Buffer: buf, + Offset: 0, + } + fh.inode.Lock() + + // 2. Call ReadWithMrdKernelReader. + resp, err := fh.ReadWithMrdKernelReader(t.ctx, req) + + // 3. Assertions + assert.Error(t.T(), err) + assert.Contains(t.T(), err.Error(), expectedErr.Error()) + assert.Zero(t.T(), resp.Size) + mockBucket.AssertExpectations(t.T()) +} + +func (t *fileTest) TestOpenMode() { + testCases := []struct { + name string + openMode util.OpenMode + }{ + { + name: "OpenModeRead", + openMode: util.NewOpenMode(util.ReadOnly, 0), + }, + { + name: "OpenModeWrite", + openMode: util.NewOpenMode(util.WriteOnly, 0), + }, + { + name: "OpenModeAppend", + openMode: util.NewOpenMode(util.WriteOnly, util.O_APPEND), + }, + { + name: "OpenModeReadWithODirect", + openMode: util.NewOpenMode(util.ReadOnly, util.O_DIRECT), + }, + { + name: "OpenModeWriteWithODirect", + openMode: util.NewOpenMode(util.WriteOnly, util.O_DIRECT), + }, + { + name: "OpenModeAppendWithODirect", + openMode: util.NewOpenMode(util.WriteOnly, util.O_APPEND|util.O_DIRECT), + }, + } + for _, tc := range testCases { + parent := createDirInode(&t.bucket, &t.clock) + config := &cfg.Config{Write: cfg.WriteConfig{EnableStreamingWrites: false}} + in := createFileInode(t.T(), &t.bucket, &t.clock, config, parent, "test_obj", nil, false) + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), tc.openMode, &cfg.Config{}, nil, nil, 0) + + openMode := fh.OpenMode() + + assert.Equal(t.T(), tc.openMode, openMode) + } +} + +func (t *fileTest) TestFileHandle_Destroy_WithReaderAndReadManager() { + parent := createDirInode(&t.bucket, &t.clock) + config := &cfg.Config{} + fileInode := createFileInode(t.T(), &t.bucket, &t.clock, config, parent, "destroy_test_obj", nil, false) + // Create mocks + mockReader := new(gcsx.MockRandomReader) + mockReadManager := new(read_manager.MockReadManager) + // Expect Destroy to be called on both + mockReader.On("Destroy").Once() + mockReadManager.On("Destroy").Once() + // Construct file handle with mocks + fh := NewFileHandle(fileInode, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, config, nil, nil, 0) + fh.reader = mockReader + fh.readManager = mockReadManager + + fh.Destroy() + + // Assert expectations + mockReader.AssertExpectations(t.T()) + mockReadManager.AssertExpectations(t.T()) +} + +func (t *fileTest) TestFileHandle_Destroy_WithNilReaderAndReadManager() { + parent := createDirInode(&t.bucket, &t.clock) + config := &cfg.Config{} + fileInode := createFileInode(t.T(), &t.bucket, &t.clock, config, parent, "destroy_test_nil_obj", nil, false) + // Construct file handle with nils + fh := NewFileHandle(fileInode, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, config, nil, nil, 0) + fh.reader = nil + fh.readManager = nil + + // Should not panic + assert.NotPanics(t.T(), func() { + fh.Destroy() + }) +} + +func (t *fileTest) TestFileHandle_CheckInvariants_WithNonNilReaderAndManager() { + parent := createDirInode(&t.bucket, &t.clock) + config := &cfg.Config{} + fileInode := createFileInode(t.T(), &t.bucket, &t.clock, config, parent, "destroy_test_obj", nil, false) + // Create mocks + mockReader := new(gcsx.MockRandomReader) + mockRM := new(read_manager.MockReadManager) + // Expectations + mockReader.On("CheckInvariants").Once() + mockRM.On("CheckInvariants").Once() + fh := NewFileHandle(fileInode, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, config, nil, nil, 0) + fh.reader = mockReader + fh.readManager = mockRM + + assert.NotPanics(t.T(), func() { + fh.checkInvariants() + }) + + mockReader.AssertExpectations(t.T()) + mockRM.AssertExpectations(t.T()) +} + +func (t *fileTest) TestFileHandle_CheckInvariants_WithNilReaderAndManager() { + parent := createDirInode(&t.bucket, &t.clock) + config := &cfg.Config{} + in := createFileInode(t.T(), &t.bucket, &t.clock, config, parent, "test_check_invariants_nil", nil, false) + + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, config, nil, nil, 0) + + // Should not panic even if both are nil + assert.NotPanics(t.T(), func() { + fh.checkInvariants() + }) +} + +func (t *fileTest) Test_LockHandleAndRelockInode_Lock_NoDeadlockWithContention() { + parent := createDirInode(&t.bucket, &t.clock) + config := &cfg.Config{} + in := createFileInode(t.T(), &t.bucket, &t.clock, config, parent, "test_obj_deadlock", []byte("content"), false) + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, config, nil, nil, 0) + var wg sync.WaitGroup + const numContenders = 10 + wg.Add(2 * numContenders) + done := make(chan struct{}) + + // Simulate the flow that uses lockHandleAndRelockInode + for range numContenders { + go func() { + defer wg.Done() + fh.inode.Lock() + fh.lockHandleAndRelockInode(false) // This should not deadlock + fh.inode.Unlock() + fh.mu.Unlock() + }() + } + + // Simulate conflicting lock acquisition order + for range numContenders { + go func() { + defer wg.Done() + fh.mu.Lock() + defer fh.mu.Unlock() + // Now try to get the inode lock + fh.inode.Lock() + defer fh.inode.Unlock() + }() + } + + go func() { + wg.Wait() + close(done) + }() + + select { + case <-done: + // Success, no deadlock + case <-time.After(2 * time.Second): + t.T().Fatal("Potential deadlock detected with Lock") + } +} + +func (t *fileTest) Test_LockHandleAndRelockInode_RLock_NoDeadlockWithContention() { + parent := createDirInode(&t.bucket, &t.clock) + config := &cfg.Config{} + in := createFileInode(t.T(), &t.bucket, &t.clock, config, parent, "test_obj_deadlock", []byte("content"), false) + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, config, nil, nil, 0) + var wg sync.WaitGroup + const numContenders = 10 + wg.Add(2 * numContenders) + done := make(chan struct{}) + + // Simulate the flow that uses lockHandleAndRelockInode + for range numContenders { + go func() { + defer wg.Done() + fh.inode.Lock() + fh.lockHandleAndRelockInode(true) // This should not deadlock + fh.inode.Unlock() + fh.mu.RUnlock() + }() + } + + // Simulate conflicting lock acquisition order + for range numContenders { + go func() { + defer wg.Done() + fh.mu.Lock() + defer fh.mu.Unlock() + // Now try to get the inode lock + fh.inode.Lock() + defer fh.inode.Unlock() + }() + } + + go func() { + wg.Wait() + close(done) + }() + + select { + case <-done: + // Success, no deadlock + case <-time.After(2 * time.Second): + t.T().Fatal("Potential deadlock detected with RLock") + } +} + +func (t *fileTest) Test_LockHandleAndRelockInode_Mixed_NoDeadlockWithContention() { + parent := createDirInode(&t.bucket, &t.clock) + config := &cfg.Config{} + in := createFileInode(t.T(), &t.bucket, &t.clock, config, parent, "test_obj_deadlock", []byte("content"), false) + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, config, nil, nil, 0) + var wg sync.WaitGroup + const numRContenders = 10 + const numWContenders = 10 + wg.Add(numRContenders + numWContenders) + done := make(chan struct{}) + + // Simulate the flow that uses lockHandleAndRelockInode(false) + for range numWContenders { + go func() { + defer wg.Done() + fh.inode.Lock() + fh.lockHandleAndRelockInode(false) // This should not deadlock + fh.mu.Unlock() + fh.inode.Unlock() + }() + } + + // Simulate the flow that uses lockHandleAndRelockInode(true) + for range numRContenders { + go func() { + defer wg.Done() + fh.inode.Lock() + fh.lockHandleAndRelockInode(true) // This should not deadlock + fh.mu.RUnlock() + fh.inode.Unlock() + }() + } + + go func() { + wg.Wait() + close(done) + }() + + select { + case <-done: + // Success, no deadlock + case <-time.After(2 * time.Second): + t.T().Fatal("Potential deadlock detected with mixed Lock/RLock") + } +} + +func (t *fileTest) Test_UnlockHandleAndInode() { + parent := createDirInode(&t.bucket, &t.clock) + config := &cfg.Config{} + in := createFileInode(t.T(), &t.bucket, &t.clock, config, parent, "test_obj_deadlock", []byte("content"), false) + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, config, nil, nil, 0) + + var wg sync.WaitGroup + const numContenders = 10 + wg.Add(3 * numContenders) + done := make(chan struct{}) + + for range numContenders { + go func() { + defer wg.Done() + fh.mu.Lock() + fh.inode.Lock() + fh.unlockHandleAndInode(false) + }() + } + + for range numContenders { + go func() { + defer wg.Done() + fh.mu.RLock() + fh.inode.Lock() + fh.unlockHandleAndInode(true) + }() + } + + for range numContenders { + go func() { + defer wg.Done() + fh.mu.Lock() + defer fh.mu.Unlock() + fh.inode.Lock() + defer fh.inode.Unlock() + }() + } + + go func() { + wg.Wait() + close(done) + }() + + select { + case <-done: + // Success: locks were re-acquired without blocking. + case <-time.After(2 * time.Second): + t.T().Fatal("Potential deadlock detected: locks were not released for write lock.") + } +} + +func (t *fileTest) Test_ReadWithReadManager_FullReadSuccessWithBufferedRead() { + const ( + fileSize = 1 * 1024 * 1024 // 1 MiB + ) + expectedData := util.GenerateRandomBytes(fileSize) + // Setup for Buffered Read test case + config := &cfg.Config{ + Read: cfg.ReadConfig{ + EnableBufferedRead: true, + MaxBlocksPerHandle: 10, + BlockSizeMb: 1, + StartBlocksPerHandle: 2, + }, + } + workerPool, err := workerpool.NewStaticWorkerPoolForCurrentCPU(20) + require.NoError(t.T(), err) + defer workerPool.Stop() + globalSemaphore := semaphore.NewWeighted(20) // Sufficient blocks for the test + parent := createDirInode(&t.bucket, &t.clock) + in := createFileInode(t.T(), &t.bucket, &t.clock, config, parent, "read_obj", expectedData, false) + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, config, workerPool, globalSemaphore, 0) + fh.inode.Lock() + buf := make([]byte, fileSize) + + // ReadWithReadManager will unlock the inode. + resp, err := fh.ReadWithReadManager(context.Background(), &gcsx.ReadRequest{ + Buffer: buf, + Offset: 0, + }, 200) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), fileSize, resp.Size) + assert.Equal(t.T(), expectedData, util.ConvertReadResponseToBytes(resp.Data, resp.Size)) +} + +func (t *fileTest) Test_ShouldSkipSizeChecks() { + const objectSize = 100 + unfinalizedObject := &gcs.MinObject{Name: "unfinalized", Size: objectSize} + finalizedObject := &gcs.MinObject{Name: "finalized", Size: objectSize, Finalized: time.Now()} + directIOReadMode := util.NewOpenMode(util.ReadOnly, util.O_DIRECT) + readOnlyMode := util.NewOpenMode(util.ReadOnly, 0) + + testCases := []struct { + name string + object *gcs.MinObject + openMode util.OpenMode + offset int64 + bufferSize int + expectedForRapid bool + useNilReadManager bool + }{ + { + name: "All conditions met: unfinalized, direct I/O, positive offset, extends beyond size", + object: unfinalizedObject, + openMode: directIOReadMode, + offset: 50, + bufferSize: 60, // 50 + 60 > 100 + expectedForRapid: true, + }, + { + name: "Finalized object: should not skip", + object: finalizedObject, + openMode: directIOReadMode, + offset: 50, + bufferSize: 60, + expectedForRapid: false, + }, + { + name: "Not direct I/O: should not skip", + object: unfinalizedObject, + openMode: readOnlyMode, + offset: 50, + bufferSize: 60, + expectedForRapid: false, + }, + { + name: "Negative offset: should not skip", + object: unfinalizedObject, + openMode: directIOReadMode, + offset: -10, + bufferSize: 20, + expectedForRapid: false, + }, + { + name: "Read within size: should not skip", + object: unfinalizedObject, + openMode: directIOReadMode, + offset: 50, + bufferSize: 50, // 50 + 50 <= 100 + expectedForRapid: false, + }, + { + name: "Read exactly at size boundary: should not skip", + object: unfinalizedObject, + openMode: directIOReadMode, + offset: 100, + bufferSize: 0, + expectedForRapid: false, + }, + { + name: "Read starts at size and extends: should skip", + object: unfinalizedObject, + openMode: directIOReadMode, + offset: 100, + bufferSize: 10, // 100 + 10 > 100 + expectedForRapid: true, + }, + { + name: "Read starts before size and extends: should skip", + object: unfinalizedObject, + openMode: directIOReadMode, + offset: 101, + bufferSize: 10, // 101 + 10 > 100 + expectedForRapid: true, + }, + } + bucketTypes := []struct { + name string + bucketType gcs.BucketType + isRapid bool + }{ + {name: "Zonal", bucketType: gcs.BucketType{Zonal: true}, isRapid: true}, + {name: "PirloRapidWriteEnabled", bucketType: gcs.BucketType{Pirlo: gcs.PirloStateRapidWritesEnabled}, isRapid: true}, + {name: "PirloRapidWriteDisabled", bucketType: gcs.BucketType{Pirlo: gcs.PirloStateRapidWritesDisabled}, isRapid: false}, + {name: "Standard", bucketType: gcs.BucketType{}, isRapid: false}, + } + + for _, bt := range bucketTypes { + for _, tc := range testCases { + t.Run(bt.name+"_"+tc.name, func() { + // Setup test bucket. + t.bucket = gcsx.NewSyncerBucket( + /*appendThreshold=*/ 1, + /*chunkRetryDeadlineSecs=*/ 120, + /*chunkTransferTimeoutSecs=*/ 10, + ".gcsfuse_tmp/", fake.NewFakeBucket(&t.clock, "some_bucket", bt.bucketType), + ) + parent := createDirInode(&t.bucket, &t.clock) + config := &cfg.Config{} + in := createFileInode(t.T(), &t.bucket, &t.clock, config, parent, tc.object.Name, nil, false) + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), tc.openMode, config, nil, nil, 0) + if tc.useNilReadManager { + fh.readManager = nil + req := &gcsx.ReadRequest{Offset: tc.offset, Buffer: make([]byte, tc.bufferSize)} + assert.Panics(t.T(), func() { fh.shouldSkipSizeChecks(req) }) + return + } + rmConfig := &read_manager.ReadManagerConfig{Config: config} + fh.readManager = read_manager.NewReadManager(tc.object, &t.bucket, rmConfig) + req := &gcsx.ReadRequest{Offset: tc.offset, Buffer: make([]byte, tc.bufferSize)} + + skip := fh.shouldSkipSizeChecks(req) + + expected := tc.expectedForRapid && bt.isRapid + assert.Equal(t.T(), expected, skip) + }) + } + } +} +func (t *fileTest) Test_ReadWithReadManager_ConcurrentReadsWithBufferedReader() { + const ( + fileSize = 9 * 1024 * 1024 // 9 MiB + numGoroutines = 3 + ) + // Create expected data for the file. + expectedData := util.GenerateRandomBytes(fileSize) + // Setup configuration for buffered read. + config := &cfg.Config{ + Read: cfg.ReadConfig{ + EnableBufferedRead: true, + MaxBlocksPerHandle: 10, + StartBlocksPerHandle: 2, + BlockSizeMb: 1, + RandomSeekThreshold: 3, + }, + } + workerPool, err := workerpool.NewStaticWorkerPoolForCurrentCPU(20) + require.NoError(t.T(), err) + defer workerPool.Stop() + globalSemaphore := semaphore.NewWeighted(20) + // Create mock inode and file handle. + parent := createDirInode(&t.bucket, &t.clock) + in := createFileInode(t.T(), &t.bucket, &t.clock, config, parent, "read_obj", expectedData, false) + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, config, workerPool, globalSemaphore, 0) + // Use a WaitGroup to synchronize goroutines. + var wg sync.WaitGroup + wg.Add(numGoroutines) + readSize := fileSize / numGoroutines + results := make([][]byte, numGoroutines) + for i := range numGoroutines { + go func(index int) { + defer wg.Done() + offset := int64(index * readSize) + readBuf := make([]byte, readSize) + fh.inode.Lock() + + // Each goroutine use same file handle. + resp, err := fh.ReadWithReadManager(context.Background(), &gcsx.ReadRequest{ + Buffer: readBuf, + Offset: offset, + }, int32(readSize)) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), readSize, resp.Size) + results[index] = util.ConvertReadResponseToBytes(resp.Data, resp.Size) + }(i) + } + // Wait for all goroutines to finish. + wg.Wait() + // Combine the results from all goroutines. + combinedResult := make([]byte, 0, fileSize) + for _, res := range results { + combinedResult = append(combinedResult, res...) + } + // Final assertion: compare the combined result with the original expected data. + assert.Equal(t.T(), expectedData, combinedResult, "Combined result should match expected data.") + // Clean up the original file handle. + fh.Destroy() +} + +// Test_ReadWithReadManager_WorkloadInsightVisual validates that when +// Workload Insight visualization is enabled, the output file is created +// after performing reads with ReadWithReadManager. +func (t *fileTest) Test_ReadWithReadManager_WorkloadInsightVisual() { + config := &cfg.Config{ + WorkloadInsight: cfg.WorkloadInsightConfig{ + Visualize: true, + OutputFile: "test.txt", + }, + } + const ( + fileSize = 9 * 1024 * 1024 // 9 MiB + MiB = 1024 * 1024 + ) + content := util.GenerateRandomBytes(fileSize) + // Create a new file handle with the updated config. + parent := createDirInode(&t.bucket, &t.clock) + in := createFileInode(t.T(), &t.bucket, &t.clock, config, parent, "test_obj_visual", content, false) + in.Lock() + fh := NewFileHandle(in, nil, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), readMode, config, nil, nil, 0) + in.Unlock() + + // Perform multiple reads and destroy the file-handle. + fh.inode.Lock() + dst := make([]byte, MiB) + // First read. + resp1, err := fh.ReadWithReadManager(t.ctx, &gcsx.ReadRequest{ + Buffer: dst, + Offset: MiB, + }, 200) + require.NoError(t.T(), err) + require.Equal(t.T(), MiB, resp1.Size) + require.Equal(t.T(), content[MiB:2*MiB], dst[:resp1.Size]) + clear(dst) + // Second read. + fh.inode.Lock() + resp2, err := fh.ReadWithReadManager(t.ctx, &gcsx.ReadRequest{ + Buffer: dst, + Offset: 0, + }, 200) + require.NoError(t.T(), err) + assert.Equal(t.T(), MiB, resp2.Size) + assert.Equal(t.T(), content[0:MiB], dst[:resp2.Size]) + clear(dst) + // Third read. + fh.inode.Lock() + resp3, err := fh.ReadWithReadManager(t.ctx, &gcsx.ReadRequest{Buffer: dst, Offset: 2 * MiB}, 200) + require.NoError(t.T(), err) + assert.Equal(t.T(), MiB, resp3.Size) + assert.Equal(t.T(), content[2*MiB:3*MiB], dst[:resp3.Size]) + fh.Destroy() + + // Validate the output file creation for workload insight. + assert.FileExists(t.T(), "test.txt") + require.NoError(t.T(), os.Remove("test.txt")) // Clean up the file after test. +} diff --git a/internal/fs/hns_bucket_test.go b/internal/fs/hns_bucket_test.go index 767e23b32c..46377a10d2 100644 --- a/internal/fs/hns_bucket_test.go +++ b/internal/fs/hns_bucket_test.go @@ -15,15 +15,21 @@ package fs_test import ( - "fmt" "os" - "os/exec" "path" "strings" "testing" - - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/caching" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/jacobsa/timeutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -32,6 +38,8 @@ import ( type HNSBucketTests struct { suite.Suite fsTest + RenameFileTests + RenameDirTests } type dirEntry struct { @@ -39,6 +47,11 @@ type dirEntry struct { isDir bool } +const file1Content = "abcdef" +const file2Content = "file2" +const IsTypeCacheDeprecated = true +const isImplicitDir = true + var expectedFooDirEntries = []dirEntry{ {name: "test", isDir: true}, {name: "test2", isDir: true}, @@ -49,12 +62,21 @@ var expectedFooDirEntries = []dirEntry{ func TestHNSBucketTests(t *testing.T) { suite.Run(t, new(HNSBucketTests)) } +func (t *HNSBucketTests) SetT(testingT *testing.T) { + t.Suite.SetT(testingT) + t.RenameDirTests.SetT(testingT) + t.RenameFileTests.SetT(testingT) +} + func (t *HNSBucketTests) SetupSuite() { t.serverCfg.ImplicitDirectories = false t.serverCfg.NewConfig = &cfg.Config{ - EnableHns: true, + EnableHns: true, + EnableAtomicRenameObject: true, } - bucketType = gcs.Hierarchical + t.serverCfg.MetricHandle = metrics.NewNoopMetrics() + t.serverCfg.TraceHandle = tracing.NewNoopTracer() + bucketType = gcs.BucketType{Hierarchical: true} t.fsTest.SetUpTestSuite() } @@ -63,13 +85,13 @@ func (t *HNSBucketTests) TearDownSuite() { } func (t *HNSBucketTests) SetupTest() { - err = t.createFolders([]string{"foo/", "bar/", "foo/test2/", "foo/test/"}) + err := t.createFolders([]string{"foo/", "bar/", "foo/test2/", "foo/test/"}) require.NoError(t.T(), err) err = t.createObjects( map[string]string{ - "foo/file1.txt": "abcdef", - "foo/file2.txt": "xyz", + "foo/file1.txt": file1Content, + "foo/file2.txt": file2Content, "foo/test/file3.txt": "xyz", "foo/implicit_dir/file3.txt": "xxw", "bar/file1.txt": "-1234556789", @@ -101,7 +123,7 @@ func (t *HNSBucketTests) TestReadDir() { func (t *HNSBucketTests) TestDeleteFolder() { dirPath := path.Join(mntDir, "foo") - err = os.RemoveAll(dirPath) + err := os.RemoveAll(dirPath) assert.NoError(t.T(), err) _, err = os.Stat(dirPath) @@ -112,7 +134,7 @@ func (t *HNSBucketTests) TestDeleteFolder() { func (t *HNSBucketTests) TestDeleteImplicitDir() { dirPath := path.Join(mntDir, "foo", "implicit_dir") - err = os.RemoveAll(dirPath) + err := os.RemoveAll(dirPath) assert.NoError(t.T(), err) _, err = os.Stat(dirPath) @@ -120,260 +142,139 @@ func (t *HNSBucketTests) TestDeleteImplicitDir() { assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) } -func (t *HNSBucketTests) TestRenameFolderWithSrcDirectoryDoesNotExist() { - oldDirPath := path.Join(mntDir, "foo_not_exist") - newDirPath := path.Join(mntDir, "foo_rename") - - err = os.Rename(oldDirPath, newDirPath) - - assert.Error(t.T(), err) - assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) - _, err = os.Stat(newDirPath) - assert.Error(t.T(), err) - assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) -} - -func (t *HNSBucketTests) TestRenameFolderWithDstDirectoryNotEmpty() { - oldDirPath := path.Join(mntDir, "foo") - _, err = os.Stat(oldDirPath) - assert.NoError(t.T(), err) - // In the setup phase, we created file1.txt within the bar directory. - newDirPath := path.Join(mntDir, "bar") - _, err = os.Stat(newDirPath) - assert.NoError(t.T(), err) +// ////////////////////////////////////////////////////////////////////// +// HNS bucket with caching support tests +// ////////////////////////////////////////////////////////////////////// +const ( + cachedHnsBucketName string = "cachedHnsBucket" +) - err = os.Rename(oldDirPath, newDirPath) +var ( + uncachedHNSBucket gcs.Bucket +) - assert.Error(t.T(), err) - assert.True(t.T(), strings.Contains(err.Error(), "file exists")) +type HNSCachedBucketMountTest struct { + suite.Suite + fsTest } -func (t *HNSBucketTests) TestRenameFolderWithEmptySourceDirectory() { - oldDirPath := path.Join(mntDir, "foo", "test2") - _, err = os.Stat(oldDirPath) - assert.NoError(t.T(), err) - newDirPath := path.Join(mntDir, "foo_rename") - _, err = os.Stat(newDirPath) - assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) - - err = os.Rename(oldDirPath, newDirPath) - - assert.NoError(t.T(), err) - _, err = os.Stat(oldDirPath) - assert.Error(t.T(), err) - assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) - _, err = os.Stat(newDirPath) - assert.NoError(t.T(), err) - dirEntries, err := os.ReadDir(newDirPath) - assert.NoError(t.T(), err) - assert.Equal(t.T(), 0, len(dirEntries)) +func TestHNSCachedBucketTests(t *testing.T) { suite.Run(t, new(HNSCachedBucketMountTest)) } + +func (t *HNSCachedBucketMountTest) SetupSuite() { + bucketType = gcs.BucketType{Hierarchical: true} + uncachedHNSBucket = fake.NewFakeBucket(timeutil.RealClock(), cachedHnsBucketName, bucketType) + lruCache := newLruCache(uint64(1000 * cfg.AverageSizeOfPositiveStatCacheEntry)) + statCache := metadata.NewStatCacheBucketView(lruCache, "") + bucket = caching.NewFastStatBucket( + ttl, + statCache, + &cacheClock, + uncachedHNSBucket, + negativeCacheTTL, + IsTypeCacheDeprecated, + isImplicitDir) + + // Enable directory type caching. + t.serverCfg.DirTypeCacheTTL = ttl + t.serverCfg.ImplicitDirectories = false + t.serverCfg.NewConfig = &cfg.Config{ + EnableHns: true, + FileCache: defaultFileCacheConfig(), + MetadataCache: cfg.MetadataCacheConfig{ + // Setting default values. + StatCacheMaxSizeMb: 33, + TtlSecs: 60, + TypeCacheMaxSizeMb: 4, + }, + } + // Call through. + t.fsTest.SetUpTestSuite() } -func (t *HNSBucketTests) TestRenameFolderWithSourceDirectoryHaveLocalFiles() { - oldDirPath := path.Join(mntDir, "foo", "test") - _, err = os.Stat(oldDirPath) - assert.NoError(t.T(), err) - file, err := os.OpenFile(path.Join(oldDirPath, "file4.txt"), os.O_RDWR|os.O_CREATE, filePerms) - assert.NoError(t.T(), err) - defer file.Close() - newDirPath := path.Join(mntDir, "bar", "foo_rename") - - err = os.Rename(oldDirPath, newDirPath) - - assert.Error(t.T(), err) - // In the logs, we encountered the following error: - // "Rename: operation not supported, can't rename directory 'test' with open files: operation not supported." - // This was translated to an "operation not supported" error at the kernel level. - assert.True(t.T(), strings.Contains(err.Error(), "operation not supported")) +func (t *HNSCachedBucketMountTest) TearDownSuite() { + t.fsTest.TearDownTestSuite() } -func (t *HNSBucketTests) TestRenameFolderWithSameParent() { - oldDirPath := path.Join(mntDir, "foo") - _, err = os.Stat(oldDirPath) +func (t *HNSCachedBucketMountTest) SetupTest() { + err := t.createFolders([]string{"hns/", "hns/cache/"}) require.NoError(t.T(), err) - newDirPath := path.Join(mntDir, "foo_rename") - _, err = os.Stat(newDirPath) - require.Error(t.T(), err) - assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) - - err = os.Rename(oldDirPath, newDirPath) +} - assert.NoError(t.T(), err) - _, err = os.Stat(oldDirPath) - assert.Error(t.T(), err) - assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) - _, err = os.Stat(newDirPath) - assert.NoError(t.T(), err) - dirEntries, err := os.ReadDir(newDirPath) - assert.NoError(t.T(), err) - assert.Equal(t.T(), 5, len(dirEntries)) - actualDirEntries := []dirEntry{} - for _, d := range dirEntries { - actualDirEntries = append(actualDirEntries, dirEntry{ - name: d.Name(), - isDir: d.IsDir(), - }) - } - assert.ElementsMatch(t.T(), actualDirEntries, expectedFooDirEntries) +func (t *HNSCachedBucketMountTest) TearDownTest() { + t.fsTest.TearDown() } -func (t *HNSBucketTests) TestRenameFolderWithExistingEmptyDestDirectory() { - oldDirPath := path.Join(mntDir, "foo", "test") - _, err = os.Stat(oldDirPath) +// ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +// --------------------- Test for delete object ------------------- +// Create directory +// Create object in directory +// Stat the object +// Delete object +// Create object using other client +// Stat the object immediately, It should return not found error although object present +// Stat the object after TTL expiry and it should appear +func (t *HNSCachedBucketMountTest) TestLocalFileIsInaccessibleAfterDeleteObjectButPresentRemotely() { + dirPath := path.Join(mntDir, "hns", "cache") + filePath := path.Join(dirPath, "file1.txt") + // Create local file inside it. + ff, err := os.Create(filePath) + require.NoError(t.T(), ff.Close()) require.NoError(t.T(), err) - newDirPath := path.Join(mntDir, "foo", "test2") - _, err = os.Stat(newDirPath) + _, err = os.Stat(filePath) require.NoError(t.T(), err) - - // Go's Rename function does not support renaming a directory into an existing empty directory. - // To achieve this, we call a Python rename function as a workaround. - cmd := exec.Command("python3", "-c", fmt.Sprintf("import os; os.rename('%s', '%s')", oldDirPath, newDirPath)) - _, err = cmd.CombinedOutput() - - assert.NoError(t.T(), err) - _, err = os.Stat(oldDirPath) - assert.Error(t.T(), err) - assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) - _, err = os.Stat(newDirPath) - assert.NoError(t.T(), err) - dirEntries, err := os.ReadDir(newDirPath) - assert.NoError(t.T(), err) - assert.Equal(t.T(), 1, len(dirEntries)) - assert.Equal(t.T(), "file3.txt", dirEntries[0].Name()) - assert.False(t.T(), dirEntries[0].IsDir()) -} - -func (t *HNSBucketTests) TestRenameFolderWithDifferentParents() { - oldDirPath := path.Join(mntDir, "foo") - _, err = os.Stat(oldDirPath) + // Delete object + err = os.Remove(filePath) assert.NoError(t.T(), err) - newDirPath := path.Join(mntDir, "bar", "foo_rename") - - err = os.Rename(oldDirPath, newDirPath) - - assert.NoError(t.T(), err) - _, err = os.Stat(oldDirPath) - assert.Error(t.T(), err) - assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) - _, err = os.Stat(newDirPath) - assert.NoError(t.T(), err) - dirEntries, err := os.ReadDir(newDirPath) + // Create an object with the same name via GCS. + _, err = storageutil.CreateObject( + ctx, + uncachedHNSBucket, + path.Join("hns", "cache", "file1.txt"), + []byte("burrito")) assert.NoError(t.T(), err) - assert.Equal(t.T(), 5, len(dirEntries)) - actualDirEntries := []dirEntry{} - for _, d := range dirEntries { - actualDirEntries = append(actualDirEntries, dirEntry{ - name: d.Name(), - isDir: d.IsDir(), - }) - } - assert.ElementsMatch(t.T(), actualDirEntries, expectedFooDirEntries) -} - -func (t *HNSBucketTests) TestRenameFolderWithOpenGCSFile() { - oldDirPath := path.Join(mntDir, "bar") - _, err = os.Stat(oldDirPath) - assert.NoError(t.T(), err) - newDirPath := path.Join(mntDir, "bar_rename") - filePath := path.Join(oldDirPath, "file1.txt") - f, err := os.Open(filePath) - require.NoError(t.T(), err) - - err = os.Rename(oldDirPath, newDirPath) - - require.NoError(t.T(), err) - _, err = f.WriteString("test") - assert.Error(t.T(), err) - assert.True(t.T(), strings.Contains(err.Error(), "bad file descriptor")) - assert.NoError(t.T(), f.Close()) - _, err = os.Stat(oldDirPath) + // Stat the object --> It should return not found error although object present + _, err = os.Stat(filePath) assert.Error(t.T(), err) assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) - _, err = os.Stat(newDirPath) - assert.NoError(t.T(), err) - dirEntries, err := os.ReadDir(newDirPath) - assert.NoError(t.T(), err) - assert.Equal(t.T(), 1, len(dirEntries)) - assert.Equal(t.T(), "file1.txt", dirEntries[0].Name()) - assert.False(t.T(), dirEntries[0].IsDir()) -} - -// Create directory foo. -// Stat the directory foo. -// Rename directory foo --> foo_rename -// Stat the old directory. -// Stat the new directory. -// Read new directory and validate. -// Create old directory again with same name - foo -// Stat the directory - foo -// Read directory again and validate it is empty. -func (t *HNSBucketTests) TestCreateDirectoryWithSameNameAfterRename() { - oldDirPath := path.Join(mntDir, "foo") - _, err = os.Stat(oldDirPath) - require.NoError(t.T(), err) - newDirPath := path.Join(mntDir, "foo_rename") - // Rename directory foo --> foo_rename - err = os.Rename(oldDirPath, newDirPath) - require.NoError(t.T(), err) - // Stat old directory. - _, err = os.Stat(oldDirPath) - require.Error(t.T(), err) - require.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) - // Stat new directory. - _, err = os.Stat(newDirPath) - require.NoError(t.T(), err) - // Read new directory and validate. - dirEntries, err := os.ReadDir(newDirPath) - require.NoError(t.T(), err) - require.Equal(t.T(), 5, len(dirEntries)) - actualDirEntries := []dirEntry{} - for _, d := range dirEntries { - actualDirEntries = append(actualDirEntries, dirEntry{ - name: d.Name(), - isDir: d.IsDir(), - }) - } - require.ElementsMatch(t.T(), actualDirEntries, expectedFooDirEntries) - - // Create old directory again. - err = os.Mkdir(oldDirPath, dirPerms) - - assert.NoError(t.T(), err) - _, err = os.Stat(oldDirPath) - assert.NoError(t.T(), err) - dirEntries, err = os.ReadDir(oldDirPath) + // After the TTL elapses, we should see it reappear. + cacheClock.AdvanceTime(ttl + time.Millisecond) + _, err = os.Stat(filePath) assert.NoError(t.T(), err) - assert.Equal(t.T(), 0, len(dirEntries)) } -// Create directory - foo/test2 -// Create local file in directory - foo/test2/test.txt -// Stat the local file - foo/test2/test.txt -// Delete directory - rm -r foo/test2 -// Create directory again - foo/test2 -// Create local file with the same name in directory - foo/test2/test.txt -// Stat the local file - foo/test2/test.txt -func (t *HNSBucketTests) TestCreateLocalFileInSamePathAfterDeletingParentDirectory() { - dirPath := path.Join(mntDir, "foo", "test2") - filePath := path.Join(dirPath, "test.txt") - // Create local file in side it. - f1, err := os.Create(filePath) - defer require.NoError(t.T(), f1.Close()) +// --------------------- Test for delete directory ----------------- +// Create directory +// stat directory +// Delete directory +// Create directory using other client +// Stat the directory immeditely, It should return not found error from cache although dir present +// Stat the directory after TTL expiry and it should appear +// ------------------------------------------------------------------ +func (t *HNSCachedBucketMountTest) TestLocalDirectoryIsInaccessibleAfterDeleteDirectoryButPresentRemotely() { + dirPath := path.Join(mntDir, "hns", "cache", "test") + // Create directory - foo/test2 + err := os.Mkdir(dirPath, dirPerms) require.NoError(t.T(), err) - _, err = os.Stat(filePath) + // stat directory - foo/test2 + _, err = os.Stat(dirPath) require.NoError(t.T(), err) - // Delete directory rm -r foo/test2 + // Delete directory - rm -r foo/test2 err = os.RemoveAll(dirPath) assert.NoError(t.T(), err) - // Create directory again foo/test2 - err = os.Mkdir(dirPath, dirPerms) + // Create a directory with the same name via GCS. + _, err = storageutil.CreateObject( + ctx, + uncachedHNSBucket, + path.Join("hns", "cache", "test")+"/", + []byte("")) assert.NoError(t.T(), err) + // Stat the directory - foo/test2/test.txt --> It should return not found error from cache although dir present + _, err = os.Stat(dirPath) + assert.Error(t.T(), err) + assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) - // Create local file again. - f2, err := os.Create(filePath) - defer require.NoError(t.T(), f2.Close()) - - assert.NoError(t.T(), err) - _, err = os.Stat(filePath) + // After the TTL elapses, we should see it reappear. + cacheClock.AdvanceTime(ttl + time.Millisecond) + _, err = os.Stat(dirPath) assert.NoError(t.T(), err) } diff --git a/internal/fs/implicit_dirs_test.go b/internal/fs/implicit_dirs_test.go index e5000c53be..49a7a2dcd3 100644 --- a/internal/fs/implicit_dirs_test.go +++ b/internal/fs/implicit_dirs_test.go @@ -24,7 +24,9 @@ import ( "syscall" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" "github.com/jacobsa/fuse/fusetesting" . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/ogletest" @@ -45,6 +47,8 @@ func init() { func (t *ImplicitDirsTest) SetUpTestSuite() { t.serverCfg.ImplicitDirectories = true + t.serverCfg.MetricHandle = metrics.NewNoopMetrics() + t.serverCfg.TraceHandle = tracing.NewNoopTracer() t.fsTest.SetUpTestSuite() } @@ -569,3 +573,32 @@ func (t *ImplicitDirsTest) AtimeCtimeAndMtime() { ExpectThat(ctime, timeutil.TimeNear(mountTime, delta)) ExpectThat(mtime, timeutil.TimeNear(mountTime, delta)) } + +func (t *ImplicitDirsTest) RenameSymlinkToImplicitDir() { + // Create an implicit directory "foo" by creating a file within it. + err := t.createWithContents("foo/bar", "taco") + AssertEq(nil, err) + // Create a symlink. + oldPath := path.Join(mntDir, "symlink") + target := "foo/bar" + err = os.Symlink(target, oldPath) + AssertEq(nil, err) + newPath := path.Join(mntDir, "symlink_new") + + // Attempt to rename the symlink to a new path. + err = os.Rename(oldPath, newPath) + + AssertEq(nil, err) + // The old path should no longer exist. + _, err = os.Lstat(oldPath) + AssertNe(nil, err) + AssertTrue(os.IsNotExist(err), "err: %v", err) + // The new path should now be a symlink. + fi, err := os.Lstat(newPath) + AssertEq(nil, err) + AssertEq(os.ModeSymlink, fi.Mode()&os.ModeSymlink) + // The new symlink should point to the correct target. + targetRead, err := os.Readlink(newPath) + AssertEq(nil, err) + AssertEq(target, targetRead) +} diff --git a/internal/fs/implicit_dirs_with_cache_test.go b/internal/fs/implicit_dirs_with_cache_test.go index db900392c2..60c363ebdb 100644 --- a/internal/fs/implicit_dirs_with_cache_test.go +++ b/internal/fs/implicit_dirs_with_cache_test.go @@ -24,6 +24,8 @@ import ( "path" "time" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/ogletest" ) @@ -43,6 +45,8 @@ func init() { func (t *ImplicitDirsWithCacheTest) SetUpTestSuite() { t.serverCfg.ImplicitDirectories = true t.serverCfg.DirTypeCacheTTL = time.Minute * 3 + t.serverCfg.MetricHandle = metrics.NewNoopMetrics() + t.serverCfg.TraceHandle = tracing.NewNoopTracer() t.fsTest.SetUpTestSuite() } diff --git a/internal/fs/inode/base_dir.go b/internal/fs/inode/base_dir.go index 8c7b52ab43..5a3169b07b 100644 --- a/internal/fs/inode/base_dir.go +++ b/internal/fs/inode/base_dir.go @@ -18,12 +18,13 @@ import ( "syscall" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/locker" + "github.com/googlecloudplatform/gcsfuse/v3/internal/locker" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "github.com/jacobsa/fuse" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" "github.com/jacobsa/fuse/fuseops" "github.com/jacobsa/fuse/fuseutil" "golang.org/x/net/context" @@ -60,6 +61,10 @@ type baseDirInode struct { // GUARDED_BY(mu) buckets map[string]gcsx.SyncerBucket + + metricHandle metrics.MetricHandle + + isEnableTypeCacheDeprecation bool } // NewBaseDirInode returns a baseDirInode that acts as the directory of @@ -68,13 +73,17 @@ func NewBaseDirInode( id fuseops.InodeID, name Name, attrs fuseops.InodeAttributes, - bm gcsx.BucketManager) (d DirInode) { + bm gcsx.BucketManager, + metricHandle metrics.MetricHandle, + isEnableTypeCacheDeprecation bool) (d DirInode) { typed := &baseDirInode{ - id: id, - name: NewRootName(""), - attrs: attrs, - bucketManager: bm, - buckets: make(map[string]gcsx.SyncerBucket), + id: id, + name: NewRootName(""), + attrs: attrs, + bucketManager: bm, + buckets: make(map[string]gcsx.SyncerBucket), + metricHandle: metricHandle, + isEnableTypeCacheDeprecation: isEnableTypeCacheDeprecation, } typed.lc.Init(id) typed.mu = locker.NewRW("BaseDirInode"+name.GcsObjectName(), func() {}) @@ -142,7 +151,7 @@ func (d *baseDirInode) Destroy() (err error) { // LOCKS_REQUIRED(d) func (d *baseDirInode) Attributes( - ctx context.Context) (attrs fuseops.InodeAttributes, err error) { + ctx context.Context, clobberedCheck bool) (attrs fuseops.InodeAttributes, err error) { // Set up basic attributes. attrs = d.attrs attrs.Nlink = 1 @@ -150,12 +159,16 @@ func (d *baseDirInode) Attributes( return } +func (d *baseDirInode) UpdateSize(size uint64) { + // No-op for directories. +} + // LOCKS_REQUIRED(d) func (d *baseDirInode) LookUpChild(ctx context.Context, name string) (*Core, error) { var err error bucket, ok := d.buckets[name] if !ok { - bucket, err = d.bucketManager.SetUpBucket(ctx, name, true) + bucket, err = d.bucketManager.SetUpBucket(ctx, name, true, d.metricHandle) if err != nil { return nil, err } @@ -177,13 +190,23 @@ func (d *baseDirInode) ReadDescendants(ctx context.Context, limit int) (map[Name // LOCKS_REQUIRED(d) func (d *baseDirInode) ReadEntries( ctx context.Context, - tok string) (entries []fuseutil.Dirent, newTok string, err error) { + tok string) (entries []fuseutil.Dirent, unsupportedPaths []string, newTok string, err error) { // The subdirectories of the base directory should be all the accessible // buckets. Although the user is allowed to visit each individual // subdirectory, listing all the subdirectories (i.e. the buckets) can be // very expensive and currently not supported. - return nil, "", syscall.ENOTSUP + return nil, nil, "", syscall.ENOTSUP +} + +// LOCKS_REQUIRED(d) +func (d *baseDirInode) ReadEntryCores(ctx context.Context, tok string) (cores map[Name]*Core, unsupportedPaths []string, newTok string, err error) { + + // The subdirectories of the base directory should be all the accessible + // buckets. Although the user is allowed to visit each individual + // subdirectory, listing all the subdirectories (i.e. the buckets) can be + // very expensive and currently not supported. + return nil, nil, "", syscall.ENOTSUP } //////////////////////////////////////////////////////////////////////// @@ -237,6 +260,10 @@ func (d *baseDirInode) DeleteChildDir( return } +func (d *baseDirInode) DeleteObjects(ctx context.Context, objectNames []string) error { + return fuse.ENOSYS +} + func (d *baseDirInode) LocalFileEntries(localFileInodes map[Name]Inode) (localEntries map[string]fuseutil.Dirent) { // Base directory can not contain local files. return nil @@ -251,7 +278,12 @@ func (d *baseDirInode) ShouldInvalidateKernelListCache(ttl time.Duration) bool { // List operation is not supported for baseDirInode. func (d *baseDirInode) InvalidateKernelListCache() {} -func (d *baseDirInode) RenameFolder(ctx context.Context, folderName string, destinationFolderId string) (op *gcs.Folder, err error) { +func (d *baseDirInode) RenameFile(ctx context.Context, fileToRename *gcs.MinObject, destinationFileName string) (*gcs.Object, error) { + err := fuse.ENOSYS + return nil, err +} + +func (d *baseDirInode) RenameFolder(ctx context.Context, folderName string, destinationFolderId string, folderInode DirInode) (op *gcs.Folder, err error) { err = fuse.ENOSYS return } @@ -263,3 +295,22 @@ func (d *baseDirInode) IsUnlinked() bool { func (d *baseDirInode) Unlink() { } + +func (d *baseDirInode) IsTypeCacheDeprecated() bool { + return d.isEnableTypeCacheDeprecation +} + +func (d *baseDirInode) CancelCurrDirPrefetcher() {} + +func (d *baseDirInode) CancelSubdirectoryPrefetches() {} + +func (d *baseDirInode) Context() context.Context { + // TODO: Consider implementing Context() if it simplifies the code in the future. + // Currently, baseDirInode is the root for dynamic mounts where listing (and thus prefetching) + // is not allowed, so a nil context suffices. + return nil +} + +func (d *baseDirInode) IncrementActiveWriters() {} + +func (d *baseDirInode) DecrementActiveWriters() {} diff --git a/internal/fs/inode/base_dir_test.go b/internal/fs/inode/base_dir_test.go index 369dcc816b..396f1e5f29 100644 --- a/internal/fs/inode/base_dir_test.go +++ b/internal/fs/inode/base_dir_test.go @@ -17,20 +17,27 @@ package inode import ( "fmt" "os" + "syscall" "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/metadata" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/fake" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" "golang.org/x/net/context" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" "github.com/jacobsa/fuse/fuseops" . "github.com/jacobsa/ogletest" "github.com/jacobsa/timeutil" ) +const ( + chunkRetryDeadlineSecs = 120 + chunkTransferTimeoutSecs = 10 +) + func TestBaseDir(t *testing.T) { RunTests(t) } //////////////////////////////////////////////////////////////////////// @@ -58,14 +65,18 @@ func (t *BaseDirTest) SetUp(ti *TestInfo) { buckets: make(map[string]gcsx.SyncerBucket), } t.bm.buckets["bucketA"] = gcsx.NewSyncerBucket( - 1, // Append threshold + /*appendThreshold=*/ 1, + chunkRetryDeadlineSecs, + chunkTransferTimeoutSecs, ".gcsfuse_tmp/", - fake.NewFakeBucket(&t.clock, "bucketA", gcs.NonHierarchical), + fake.NewFakeBucket(&t.clock, "bucketA", gcs.BucketType{}), ) t.bm.buckets["bucketB"] = gcsx.NewSyncerBucket( - 1, // Append threshold + /*appendThreshold=*/ 1, + chunkRetryDeadlineSecs, + chunkTransferTimeoutSecs, ".gcsfuse_tmp/", - fake.NewFakeBucket(&t.clock, "bucketB", gcs.NonHierarchical), + fake.NewFakeBucket(&t.clock, "bucketB", gcs.BucketType{}), ) // Create the inode. No implicit dirs by default. @@ -87,7 +98,7 @@ type fakeBucketManager struct { func (bm *fakeBucketManager) SetUpBucket( ctx context.Context, - name string, isMultibucketMount bool) (sb gcsx.SyncerBucket, err error) { + name string, isMultibucketMount bool, _ metrics.MetricHandle) (sb gcsx.SyncerBucket, err error) { bm.setupTimes++ var ok bool @@ -118,7 +129,9 @@ func (t *BaseDirTest) resetInode() { Gid: gid, Mode: dirMode, }, - t.bm) + t.bm, + metrics.NewNoopMetrics(), + isTypeCacheDeprecationEnabled) t.in.Lock() } @@ -146,8 +159,18 @@ func (t *BaseDirTest) LookupCount() { ExpectTrue(t.in.DecrementLookupCount(1)) } -func (t *BaseDirTest) Attributes() { - attrs, err := t.in.Attributes(t.ctx) +func (t *BaseDirTest) Attributes_ClobberedCheckTrue() { + attrs, err := t.in.Attributes(t.ctx, true) + + AssertEq(nil, err) + ExpectEq(uid, attrs.Uid) + ExpectEq(gid, attrs.Gid) + ExpectEq(dirMode|os.ModeDir, attrs.Mode) +} + +func (t *BaseDirTest) Attributes_ClobberedCheckFalse() { + attrs, err := t.in.Attributes(t.ctx, false) + AssertEq(nil, err) ExpectEq(uid, attrs.Uid) ExpectEq(gid, attrs.Gid) @@ -212,3 +235,45 @@ func (t *BaseDirTest) Test_ShouldInvalidateKernelListCache_TtlExpired() { AssertEq(true, t.in.ShouldInvalidateKernelListCache(ttl)) } + +func (t *BaseDirTest) TestReadEntryCores() { + cores, unsupportedPaths, newTok, err := t.in.ReadEntryCores(t.ctx, "") + + // Should return ENOTSUP because listing is unsupported. + ExpectEq(nil, cores) + ExpectEq(nil, unsupportedPaths) + ExpectEq("", newTok) + ExpectEq(syscall.ENOTSUP, err) +} + +func (t *BaseDirTest) Test_IsTypeCacheDeprecated_false() { + dInode := NewBaseDirInode( + dirInodeID, + NewRootName(""), + fuseops.InodeAttributes{ + Uid: uid, + Gid: gid, + Mode: dirMode, + }, + t.bm, + metrics.NewNoopMetrics(), + false) + + AssertFalse(dInode.IsTypeCacheDeprecated()) +} + +func (t *BaseDirTest) Test_IsTypeCacheDeprecated_true() { + dInode := NewBaseDirInode( + dirInodeID, + NewRootName(""), + fuseops.InodeAttributes{ + Uid: uid, + Gid: gid, + Mode: dirMode, + }, + t.bm, + metrics.NewNoopMetrics(), + true) + + AssertTrue(dInode.IsTypeCacheDeprecated()) +} diff --git a/internal/fs/inode/core.go b/internal/fs/inode/core.go index 96df808561..bbcb617e62 100644 --- a/internal/fs/inode/core.go +++ b/internal/fs/inode/core.go @@ -17,9 +17,9 @@ package inode import ( "fmt" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/metadata" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" ) // Core contains critical information about an inode before its creation. diff --git a/internal/fs/inode/core_test.go b/internal/fs/inode/core_test.go index 5e484a4ad8..bc135a5f3c 100644 --- a/internal/fs/inode/core_test.go +++ b/internal/fs/inode/core_test.go @@ -18,12 +18,12 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/metadata" - "github.com/googlecloudplatform/gcsfuse/v2/internal/fs/inode" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/fake" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/inode" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" . "github.com/jacobsa/ogletest" "github.com/jacobsa/timeutil" "golang.org/x/net/context" @@ -49,7 +49,10 @@ func init() { RegisterTestSuite(&CoreTest{}) } func (t *CoreTest) SetUp(ti *TestInfo) { t.ctx = ti.Ctx t.bucket = gcsx.NewSyncerBucket( - 1, ".gcsfuse_tmp/", fake.NewFakeBucket(&t.clock, "some_bucket", gcs.NonHierarchical)) + /*appendThreshold=*/ 1, + /*chunkRetryDeadlineSecs=*/ 120, + /*chunkTransferTimeoutSecs=*/ 10, + ".gcsfuse_tmp/", fake.NewFakeBucket(&t.clock, "some_bucket", gcs.BucketType{})) t.clock.SetTime(time.Date(2012, 8, 15, 22, 56, 0, 0, time.Local)) } diff --git a/internal/fs/inode/dir.go b/internal/fs/inode/dir.go index cdffed15a4..d16763561b 100644 --- a/internal/fs/inode/dir.go +++ b/internal/fs/inode/dir.go @@ -15,23 +15,27 @@ package inode import ( + "context" "errors" "fmt" "path" "strings" + "sync/atomic" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/metadata" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" - "github.com/googlecloudplatform/gcsfuse/v2/internal/locker" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/locker" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/caching" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" "github.com/jacobsa/fuse/fuseops" "github.com/jacobsa/fuse/fuseutil" "github.com/jacobsa/timeutil" - "golang.org/x/net/context" "golang.org/x/sync/errgroup" + "golang.org/x/sync/semaphore" ) // ListObjects call supports fetching upto 5000 results when projection is noAcl @@ -65,8 +69,11 @@ type DirInode interface { // true. LookUpChild(ctx context.Context, name string) (*Core, error) - // Rename the directiory/folder. - RenameFolder(ctx context.Context, folderName string, destinationFolderId string) (*gcs.Folder, error) + // Rename the file. + RenameFile(ctx context.Context, fileToRename *gcs.MinObject, destinationFileName string) (*gcs.Object, error) + + // Rename the directory/folder. It accepts folderInode to trigger recursive prefetch cancellation. + RenameFolder(ctx context.Context, folderName string, destinationFolderId string, folderInode DirInode) (*gcs.Folder, error) // Read the children objects of this dir, recursively. The result count // is capped at the given limit. Internal caches are not refreshed from this @@ -86,7 +93,18 @@ type DirInode interface { // undefined. ReadEntries( ctx context.Context, - tok string) (entries []fuseutil.Dirent, newTok string, err error) + tok string) (entries []fuseutil.Dirent, unsupportedPaths []string, newTok string, err error) + + // ReadEntryCores reads a batch of directory entries and returns them as a + // map of `inode.Core` objects along with a continuation token that can be + // used to pick up the read operation where it left off. + // Supply the empty token on the first call. + // + // At the end of the directory, the returned continuation token will be + // empty. Otherwise it will be non-empty. There is no guarantee about the + // number of entries returned; it may be zero even with a non-empty + // continuation token. + ReadEntryCores(ctx context.Context, tok string) (cores map[Name]*Core, unsupportedPaths []string, newTok string, err error) // Create an empty child file with the supplied (relative) name, failing with // *gcs.PreconditionError if a backing object already exists in GCS. @@ -139,6 +157,9 @@ type DirInode interface { isImplicitDir bool, dirInode DirInode) (err error) + // DeleteObjects recursively deletes the given objects and prefixes. + DeleteObjects(ctx context.Context, objectNames []string) error + // LocalFileEntries lists the local files present in the directory. // Local means that the file is not yet present on GCS. LocalFileEntries(localFileInodes map[Name]Inode) (localEntries map[string]fuseutil.Dirent) @@ -158,6 +179,17 @@ type DirInode interface { // served from GCSFuse. InvalidateKernelListCache() + // CancelCurrDirPrefetcher stops only the *current* prefetch run for this dir. + CancelCurrDirPrefetcher() + + // CancelSubdirectoryPrefetches permanently stops the context for this inode AND all its descendants. + // This is used for Directory Rename/Delete operations. This context is not refreshed + // with the assumption that Rename/Delete directory operations will lead to new inodes with fresh contexts. + CancelSubdirectoryPrefetches() + + // Context provides the lifecycle context of the inode. + Context() context.Context + // RLock readonly lock. RLock() @@ -167,6 +199,17 @@ type DirInode interface { IsUnlinked() bool Unlink() + + IsTypeCacheDeprecated() bool + + // IncrementActiveWriters increments the active writer count for this directory. + // This should be called before any write operation (file create, delete, rename, etc.) + // within this directory begins. + IncrementActiveWriters() + + // DecrementActiveWriters decrements the active writer count for this directory. + // This should be called after any write operation within this directory completes. + DecrementActiveWriters() } // An inode that represents a directory from a GCS bucket. @@ -183,6 +226,13 @@ type dirInode struct { bucket *gcsx.SyncerBucket mtimeClock timeutil.Clock cacheClock timeutil.Clock + prefetcher *MetadataPrefetcher + + // ctx is the lifecycle context for this inode. + // It is derived from the parent directory's context. + // Parent directory's context is cancelled on directory rename/deletion. + ctx context.Context + cancel context.CancelFunc ///////////////////////// // Constant data @@ -222,9 +272,21 @@ type dirInode struct { prevDirListingTimeStamp time.Time isHNSEnabled bool + isStandardSymlinkRepresentationEnabled bool + + isUnsupportedPathSupportEnabled bool + + isEnableTypeCacheDeprecation bool + // Represents if folder has been unlinked in hierarchical bucket. This is not getting used in // non-hierarchical bucket. unlinked bool + + metadataCacheTtlSecs int64 + + // activeWriters tracks the number of ongoing write operations in this directory. + // It is used to prevent metadata prefetching while writes are in progress. + activeWriters atomic.Int32 } var _ DirInode = &dirInode{} @@ -251,35 +313,64 @@ var _ DirInode = &dirInode{} func NewDirInode( id fuseops.InodeID, name Name, + parentInodeCtx context.Context, attrs fuseops.InodeAttributes, implicitDirs bool, - includeFoldersAsPrefixes bool, enableNonexistentTypeCache bool, typeCacheTTL time.Duration, bucket *gcsx.SyncerBucket, mtimeClock timeutil.Clock, cacheClock timeutil.Clock, - typeCacheMaxSizeMB int64, - isHNSEnabled bool, + prefetchSem *semaphore.Weighted, + cfg *cfg.Config, ) (d DirInode) { if !name.IsDir() { panic(fmt.Sprintf("Unexpected name: %s", name)) } + // Establish the recursive context chain so if parent is cancelled, child inodes are also cancelled. + // If parent inode ctx is nil, set a new context. This will happen for bucket root inodes. + if parentInodeCtx == nil { + parentInodeCtx = context.Background() + } + ctx, cancel := context.WithCancel(parentInodeCtx) + typed := &dirInode{ - bucket: bucket, - mtimeClock: mtimeClock, - cacheClock: cacheClock, - id: id, - implicitDirs: implicitDirs, - includeFoldersAsPrefixes: includeFoldersAsPrefixes, - enableNonexistentTypeCache: enableNonexistentTypeCache, - name: name, - attrs: attrs, - cache: metadata.NewTypeCache(typeCacheMaxSizeMB, typeCacheTTL), - isHNSEnabled: isHNSEnabled, - unlinked: false, + bucket: bucket, + mtimeClock: mtimeClock, + cacheClock: cacheClock, + id: id, + implicitDirs: implicitDirs, + includeFoldersAsPrefixes: cfg.List.EnableEmptyManagedFolders, + enableNonexistentTypeCache: enableNonexistentTypeCache, + name: name, + attrs: attrs, + isHNSEnabled: cfg.EnableHns, + isStandardSymlinkRepresentationEnabled: cfg.EnableStandardSymlinks, + isUnsupportedPathSupportEnabled: cfg.EnableUnsupportedPathSupport, + isEnableTypeCacheDeprecation: cfg.EnableTypeCacheDeprecation, + unlinked: false, + ctx: ctx, + cancel: cancel, + metadataCacheTtlSecs: cfg.MetadataCache.TtlSecs, + } + + // Init Prefetcher only if it is enabled, stat cache ttl != 0 and stat cache size != 0. + // Prefetcher is bound to the inode's lifecycle context `ctx`. + // readObjectsUnlocked is used by the prefetcher so the background worker performs GCS I/O without the lock, + // acquiring d.mu only to update the cache. + // We pass a callback to check if there are active writers, to prevent prefetching during writes. + if cfg.MetadataCache.EnableMetadataPrefetch && cfg.MetadataCache.TtlSecs != 0 && cfg.MetadataCache.StatCacheMaxSizeMb != 0 { + typed.prefetcher = NewMetadataPrefetcher(ctx, cfg, prefetchSem, cacheClock, typed.readObjectsUnlocked, func() bool { + return typed.activeWriters.Load() == 0 + }) + } + + var cache metadata.TypeCache + if !cfg.EnableTypeCacheDeprecation { + cache = metadata.NewTypeCache(cfg.MetadataCache.TypeCacheMaxSizeMb, typeCacheTTL) + typed.cache = cache } typed.lc.Init(id) @@ -303,19 +394,19 @@ func (d *dirInode) checkInvariants() { } func (d *dirInode) lookUpChildFile(ctx context.Context, name string) (*Core, error) { - return findExplicitInode(ctx, d.Bucket(), NewFileName(d.Name(), name)) + return findExplicitInode(ctx, d.Bucket(), NewFileName(d.Name(), name), false) } func (d *dirInode) lookUpChildDir(ctx context.Context, name string) (*Core, error) { childName := NewDirName(d.Name(), name) if d.isBucketHierarchical() { - return findExplicitFolder(ctx, d.Bucket(), childName) + return findExplicitFolder(ctx, d.Bucket(), childName, false) } if d.implicitDirs { return findDirInode(ctx, d.Bucket(), childName) } - return findExplicitInode(ctx, d.Bucket(), childName) + return findExplicitInode(ctx, d.Bucket(), childName, false) } // Look up the file for a (file, dir) pair with conflicting names, overriding @@ -349,10 +440,11 @@ func (d *dirInode) lookUpConflicting(ctx context.Context, name string) (*Core, e // findExplicitInode finds the file or dir inode core backed by an explicit // object in GCS with the given name. Return nil if such object does not exist. -func findExplicitInode(ctx context.Context, bucket *gcsx.SyncerBucket, name Name) (*Core, error) { +func findExplicitInode(ctx context.Context, bucket *gcsx.SyncerBucket, name Name, fetchOnlyFromCache bool) (*Core, error) { // Call the bucket. req := &gcs.StatObjectRequest{ - Name: name.GcsObjectName(), + Name: name.GcsObjectName(), + FetchOnlyFromCache: fetchOnlyFromCache, } m, _, err := bucket.StatObject(ctx, req) @@ -368,6 +460,13 @@ func findExplicitInode(ctx context.Context, bucket *gcsx.SyncerBucket, name Name return nil, fmt.Errorf("StatObject: %w", err) } + // Treat this as an implicit directory. Since implicit objects lack metadata + // (only the name is preserved), we set MinObject to nil. This ensures the + // inode is correctly assigned the ImplicitDir Core Type. + if fetchOnlyFromCache && m.Generation == 0 { + m = nil + } + return &Core{ Bucket: bucket, FullName: name, @@ -375,8 +474,13 @@ func findExplicitInode(ctx context.Context, bucket *gcsx.SyncerBucket, name Name }, nil } -func findExplicitFolder(ctx context.Context, bucket *gcsx.SyncerBucket, name Name) (*Core, error) { - folder, err := bucket.GetFolder(ctx, name.GcsObjectName()) +func findExplicitFolder(ctx context.Context, bucket *gcsx.SyncerBucket, name Name, fetchOnlyFromCache bool) (*Core, error) { + // Call the bucket. + req := &gcs.GetFolderRequest{ + Name: name.GcsObjectName(), + FetchOnlyFromCache: fetchOnlyFromCache, + } + folder, err := bucket.GetFolder(ctx, req) // Suppress "not found" errors. var gcsErr *gcs.NotFoundError @@ -430,13 +534,14 @@ func findDirInode(ctx context.Context, bucket *gcsx.SyncerBucket, name Name) (*C func (d *dirInode) createNewObject( ctx context.Context, name Name, - metadata map[string]string) (o *gcs.Object, err error) { - // Create an empty backing object for the child, failing if it already + metadata map[string]string, + content string) (o *gcs.Object, err error) { + // Create a backing object for the child, failing if it already // exists. var precond int64 createReq := &gcs.CreateObjectRequest{ Name: name.GcsObjectName(), - Contents: strings.NewReader(""), + Contents: strings.NewReader(content), GenerationPrecondition: &precond, Metadata: metadata, } @@ -503,13 +608,48 @@ func (d *dirInode) DecrementLookupCount(n uint64) (destroy bool) { // LOCKS_REQUIRED(d) func (d *dirInode) Destroy() (err error) { - // Nothing interesting to do. + // When destroying the inode, we cancel its subdirectory prefetches. + // This cleans up any curr dir + child dir prefetchers. + d.CancelSubdirectoryPrefetches() + return +} + +// LOCKS_REQUIRED(d.mu.RLock) +func (d *dirInode) Context() context.Context { + return d.ctx +} + +// CancelSubdirectoryPrefetches permanently stops the context for this inode AND all its descendants. +// This is used for Directory Rename/Delete operations where the tree is being invalidated. +// LOCKS_REQUIRED(d.mu.WLock) +func (d *dirInode) CancelSubdirectoryPrefetches() { + if d.cancel != nil { + d.cancel() + d.cancel = nil + } +} + +// CancelCurrDirPrefetcher stops only the *current* prefetch run for this dir. +// This allows the prefetcher to be restarted later (unless the inode context itself is cancelled). +// LOCKS_REQUIRED(d.mu.WLock) +func (d *dirInode) CancelCurrDirPrefetcher() { + if d.prefetcher != nil { + d.prefetcher.Cancel() + } return } +// UpdateSize is a no-op for implicit directories. These directories are not +// backed by a real GCS object but are inferred from the presence of other +// objects (e.g., the directory "foo/" is implied by the object "foo/bar.txt"). +// Since there is no backing object, there is no size attribute to update. +func (d *dirInode) UpdateSize(size uint64) { + // No-op for directories. +} + // LOCKS_REQUIRED(d) func (d *dirInode) Attributes( - ctx context.Context) (attrs fuseops.InodeAttributes, err error) { + ctx context.Context, clobberedCheck bool) (attrs fuseops.InodeAttributes, err error) { // Set up basic attributes. attrs = d.attrs attrs.Nlink = 1 @@ -529,23 +669,87 @@ func (d *dirInode) Bucket() *gcsx.SyncerBucket { // See also the notes on DirInode.LookUpChild. const ConflictingFileNameSuffix = "\n" -// LOCKS_REQUIRED(d) +// LOCKS_REQUIRED(d.mu.RLock) func (d *dirInode) LookUpChild(ctx context.Context, name string) (*Core, error) { // Is this a conflict marker name? if strings.HasSuffix(name, ConflictingFileNameSuffix) { return d.lookUpConflicting(ctx, name) } + cachedType := metadata.UnknownType + + // 1. Optimization: If Type Cache is deprecated, attempt a lookup via the Stat Cache first. + // We skip this if the metadata cache TTL is 0, as the cache layer is inactive. + if d.IsTypeCacheDeprecated() && d.metadataCacheTtlSecs != 0 { + var cacheMissErr *caching.CacheMissError + + // 1. Try Directory FIRST (since it's the preferred return type) + var dirResult *Core + var err error + if d.Bucket().BucketType().Hierarchical { + dirResult, err = findExplicitFolder(ctx, d.Bucket(), NewDirName(d.Name(), name), true) + } else { + dirResult, err = findExplicitInode(ctx, d.Bucket(), NewDirName(d.Name(), name), true) + } + + // If we found a directory, we're done. Return it now. + if dirResult != nil { + return dirResult, nil + } + // If we hit a real error (not a cache miss), exit early. + if err != nil && !errors.As(err, &cacheMissErr) { + return nil, err + } + + // 2. Try File ONLY if directory wasn't found + fileResult, err := findExplicitInode(ctx, d.Bucket(), NewFileName(d.Name(), name), true) + if err != nil && !errors.As(err, &cacheMissErr) { + return nil, err + } + + if fileResult != nil { + return fileResult, nil + } + } + + // 2. Legacy: If Type Cache is NOT deprecated, fetch the type hint. + if !d.IsTypeCacheDeprecated() { + cachedType = d.cache.Get(d.cacheClock.Now(), name) + } + + // 3. Main Lookup Logic (Unified) + result, err := d.fetchCoreEntity(ctx, name, cachedType) + if err != nil { + return nil, err + } + + // 4. Legacy: Update Type Cache if needed. + if !d.IsTypeCacheDeprecated() { + if result != nil { + d.cache.Insert(d.cacheClock.Now(), name, result.Type()) + } else if d.enableNonexistentTypeCache && cachedType == metadata.UnknownType { + d.cache.Insert(d.cacheClock.Now(), name, metadata.NonexistentType) + } + } + + return result, nil +} + +// fetchCoreEntity contains all the existing logic for looking up children +// without worrying about the isTypeCacheDeprecated flag. +func (d *dirInode) fetchCoreEntity(ctx context.Context, name string, cachedType metadata.Type) (*Core, error) { group, ctx := errgroup.WithContext(ctx) var fileResult *Core var dirResult *Core + var err error + lookUpFile := func() (err error) { - fileResult, err = findExplicitInode(ctx, d.Bucket(), NewFileName(d.Name(), name)) + fileResult, err = findExplicitInode(ctx, d.Bucket(), NewFileName(d.Name(), name), false) return } lookUpExplicitDir := func() (err error) { - dirResult, err = findExplicitInode(ctx, d.Bucket(), NewDirName(d.Name(), name)) + dirResult, err = findExplicitInode(ctx, d.Bucket(), NewDirName(d.Name(), name), false) return } lookUpImplicitOrExplicitDir := func() (err error) { @@ -553,18 +757,17 @@ func (d *dirInode) LookUpChild(ctx context.Context, name string) (*Core, error) return } lookUpHNSDir := func() (err error) { - dirResult, err = findExplicitFolder(ctx, d.Bucket(), NewDirName(d.Name(), name)) + dirResult, err = findExplicitFolder(ctx, d.Bucket(), NewDirName(d.Name(), name), false) return } - cachedType := d.cache.Get(d.cacheClock.Now(), name) switch cachedType { case metadata.ImplicitDirType: - dirResult = &Core{ + return &Core{ Bucket: d.Bucket(), FullName: NewDirName(d.Name(), name), MinObject: nil, - } + }, nil case metadata.ExplicitDirType: if d.isBucketHierarchical() { group.Go(lookUpHNSDir) @@ -573,9 +776,16 @@ func (d *dirInode) LookUpChild(ctx context.Context, name string) (*Core, error) } case metadata.RegularFileType, metadata.SymlinkType: group.Go(lookUpFile) + case metadata.NonexistentType: return nil, nil case metadata.UnknownType: + // Entry not present in cache. + // Trigger prefetcher + if d.prefetcher != nil { + d.prefetcher.Run(NewFileName(d.Name(), name).GcsObjectName()) + } + group.Go(lookUpFile) if d.isBucketHierarchical() { group.Go(lookUpHNSDir) @@ -586,27 +796,16 @@ func (d *dirInode) LookUpChild(ctx context.Context, name string) (*Core, error) group.Go(lookUpExplicitDir) } } - } - if err := group.Wait(); err != nil { + if err = group.Wait(); err != nil { return nil, err } - var result *Core if dirResult != nil { - result = dirResult - } else if fileResult != nil { - result = fileResult - } - - if result != nil { - d.cache.Insert(d.cacheClock.Now(), name, result.Type()) - } else if d.enableNonexistentTypeCache && cachedType == metadata.UnknownType { - d.cache.Insert(d.cacheClock.Now(), name, metadata.NonexistentType) + return dirResult, nil } - - return result, nil + return fileResult, nil } func (d *dirInode) IsUnlinked() bool { @@ -617,6 +816,14 @@ func (d *dirInode) Unlink() { d.unlinked = true } +func (d *dirInode) IncrementActiveWriters() { + d.activeWriters.Add(1) +} + +func (d *dirInode) DecrementActiveWriters() { + d.activeWriters.Add(-1) +} + // LOCKS_REQUIRED(d) func (d *dirInode) ReadDescendants(ctx context.Context, limit int) (map[Name]*Core, error) { var tok string @@ -656,24 +863,28 @@ func (d *dirInode) ReadDescendants(ctx context.Context, limit int) (map[Name]*Co } -// LOCKS_REQUIRED(d) -func (d *dirInode) readObjects( - ctx context.Context, - tok string) (cores map[Name]*Core, newTok string, err error) { +// Helper function to handle the common listing logic and GCS request preparation. +func (d *dirInode) listObjectsAndBuildCores(ctx context.Context, tok string, maxListCallResults int, listStartOffset string) (cores map[Name]*Core, unsupportedPaths []string, newTok string, err error) { + + includeTrailingDelimeter := true // Important for flat bucket to list explicit directory. if d.isBucketHierarchical() { d.includeFoldersAsPrefixes = true + // Listing explicit directories are not required for HNS bucket as already listed as + // folder resources and setting it to true causes perf issue (b/465295359) + includeTrailingDelimeter = false } // Ask the bucket to list some objects. req := &gcs.ListObjectsRequest{ Delimiter: "/", - IncludeTrailingDelimiter: true, + IncludeTrailingDelimiter: includeTrailingDelimeter, Prefix: d.Name().GcsObjectName(), ContinuationToken: tok, - MaxResults: MaxResultsForListObjectsCall, + MaxResults: maxListCallResults, // Setting Projection param to noAcl since fetching owner and acls are not // required. ProjectionVal: gcs.NoAcl, IncludeFoldersAsPrefixes: d.includeFoldersAsPrefixes, + StartOffset: listStartOffset, } listing, err := d.bucket.ListObjects(ctx, req) @@ -683,14 +894,16 @@ func (d *dirInode) readObjects( } cores = make(map[Name]*Core) - defer func() { - now := d.cacheClock.Now() - for fullName, c := range cores { - d.cache.Insert(now, path.Base(fullName.LocalName()), c.Type()) + for _, o := range listing.MinObjects { + if storageutil.IsUnsupportedPath(o.Name) { + unsupportedPaths = append(unsupportedPaths, o.Name) + // Skip unsupported objects in the listing, as the kernel cannot process these file system elements. + // TODO: Remove this check once we gain confidence that it is not causing any issues. + if d.isUnsupportedPathSupportEnabled { + continue + } } - }() - for _, o := range listing.MinObjects { // Skip empty results or the directory object backing this inode. if o.Name == d.Name().GcsObjectName() || o.Name == "" { continue @@ -733,11 +946,15 @@ func (d *dirInode) readObjects( } // Add implicit directories into the result. - unsupportedPrefixes := []string{} for _, p := range listing.CollapsedRuns { pathBase := path.Base(p) - if storageutil.IsUnsupportedObjectName(p) { - unsupportedPrefixes = append(unsupportedPrefixes, p) + if storageutil.IsUnsupportedPath(p) { + unsupportedPaths = append(unsupportedPaths, p) + // Skip unsupported objects in the listing, as the kernel cannot process these file system elements. + // TODO: Remove this check once we gain confidence that it is not causing any issues. + if d.isUnsupportedPathSupportEnabled { + continue + } } dirName := NewDirName(d.Name(), pathBase) if d.isBucketHierarchical() { @@ -762,19 +979,63 @@ func (d *dirInode) readObjects( cores[dirName] = implicitDir } } - if len(unsupportedPrefixes) > 0 { - logger.Errorf("Encountered unsupported prefixes during listing: %v", unsupportedPrefixes) + if len(unsupportedPaths) > 0 { + logger.Warnf("Encountered unsupported prefixes during listing: %v", unsupportedPaths) + } + return +} + +// LOCKS_REQUIRED(d) +func (d *dirInode) insertToCache(cores map[Name]*Core) { + if d.IsTypeCacheDeprecated() { + return + } + now := d.cacheClock.Now() + for fullName, c := range cores { + d.cache.Insert(now, path.Base(fullName.LocalName()), c.Type()) + } +} + +// LOCKS_REQUIRED(d) +func (d *dirInode) readObjects( + ctx context.Context, + tok string) (cores map[Name]*Core, unsupportedPaths []string, newTok string, err error) { + + cores, unsupportedPaths, newTok, err = d.listObjectsAndBuildCores(ctx, tok, MaxResultsForListObjectsCall, "") + if err == nil { + d.insertToCache(cores) } return } +// LOCK_EXCLUDED(d) +// readObjectsUnlocked performs GCS I/O without the lock, acquiring d.mu only to update the cache. +func (d *dirInode) readObjectsUnlocked(ctx context.Context, tok string, startOffset string, maxListCallResults int) (cores map[Name]*Core, unsupportedPaths []string, newTok string, err error) { + cores, unsupportedPaths, newTok, err = d.listObjectsAndBuildCores(ctx, tok, maxListCallResults, startOffset) + if err != nil { + return + } + + d.mu.Lock() + defer d.mu.Unlock() + + // Critical check after acquiring lock: If the context was cancelled during the list call (e.g. by a Rename), + // we must not update the cache with this stale data. + if ctx != nil && ctx.Err() != nil { + return nil, nil, "", ctx.Err() + } + + d.insertToCache(cores) + return +} + +// LOCKS_REQUIRED(d) func (d *dirInode) ReadEntries( ctx context.Context, - tok string) (entries []fuseutil.Dirent, newTok string, err error) { + tok string) (entries []fuseutil.Dirent, unsupportedPaths []string, newTok string, err error) { var cores map[Name]*Core - cores, newTok, err = d.readObjects(ctx, tok) + cores, unsupportedPaths, newTok, err = d.ReadEntryCores(ctx, tok) if err != nil { - err = fmt.Errorf("read objects: %w", err) return } @@ -794,24 +1055,38 @@ func (d *dirInode) ReadEntries( entries = append(entries, entry) } + return +} + +// LOCKS_REQUIRED(d) +func (d *dirInode) ReadEntryCores(ctx context.Context, tok string) (cores map[Name]*Core, unsupportedPaths []string, newTok string, err error) { + cores, unsupportedPaths, newTok, err = d.readObjects(ctx, tok) + if err != nil { + err = fmt.Errorf("read objects: %w", err) + return + } + d.prevDirListingTimeStamp = d.cacheClock.Now() return } // LOCKS_REQUIRED(d) func (d *dirInode) CreateChildFile(ctx context.Context, name string) (*Core, error) { + // No need to cancel prefetch here as creation of new file can not lead to stale data in metadata cache. childMetadata := map[string]string{ FileMtimeMetadataKey: d.mtimeClock.Now().UTC().Format(time.RFC3339Nano), } fullName := NewFileName(d.Name(), name) - o, err := d.createNewObject(ctx, fullName, childMetadata) + o, err := d.createNewObject(ctx, fullName, childMetadata, "") if err != nil { return nil, err } m := storageutil.ConvertObjToMinObject(o) - d.cache.Insert(d.cacheClock.Now(), name, metadata.RegularFileType) + if !d.IsTypeCacheDeprecated() { + d.cache.Insert(d.cacheClock.Now(), name, metadata.RegularFileType) + } return &Core{ Bucket: d.Bucket(), FullName: fullName, @@ -830,18 +1105,30 @@ func (d *dirInode) CreateLocalChildFileCore(name string) (Core, error) { // LOCKS_REQUIRED(d) func (d *dirInode) InsertFileIntoTypeCache(name string) { - d.cache.Insert(d.cacheClock.Now(), name, metadata.RegularFileType) + if !d.IsTypeCacheDeprecated() { + d.cache.Insert(d.cacheClock.Now(), name, metadata.RegularFileType) + } } // LOCKS_REQUIRED(d) func (d *dirInode) EraseFromTypeCache(name string) { - d.cache.Erase(name) + if !d.IsTypeCacheDeprecated() { + d.cache.Erase(name) + } } // LOCKS_REQUIRED(d) func (d *dirInode) CloneToChildFile(ctx context.Context, name string, src *gcs.MinObject) (*Core, error) { - // Erase any existing type information for this name. - d.cache.Erase(name) + // Increment active writers on the directory so no new prefetch gets triggered until the write operation completes. + d.IncrementActiveWriters() + defer d.DecrementActiveWriters() + // Cancel prefetch of the current directory only. + d.CancelCurrDirPrefetcher() + + if !d.IsTypeCacheDeprecated() { + // Erase any existing type information for this name. + d.cache.Erase(name) + } fullName := NewFileName(d.Name(), name) // Clone over anything that might already exist for the name. @@ -863,24 +1150,41 @@ func (d *dirInode) CloneToChildFile(ctx context.Context, name string, src *gcs.M FullName: fullName, MinObject: m, } - d.cache.Insert(d.cacheClock.Now(), name, c.Type()) + if !d.IsTypeCacheDeprecated() { + d.cache.Insert(d.cacheClock.Now(), name, c.Type()) + } return c, nil } // LOCKS_REQUIRED(d) func (d *dirInode) CreateChildSymlink(ctx context.Context, name string, target string) (*Core, error) { + // No need to cancel prefetch here as creation of new symlink can not lead to stale data in metadata cache. + fullName := NewFileName(d.Name(), name) - childMetadata := map[string]string{ - SymlinkMetadataKey: target, + var childMetadata map[string]string + var content string + + if d.isStandardSymlinkRepresentationEnabled { + childMetadata = map[string]string{ + StandardSymlinkMetadataKey: "true", + SymlinkMetadataKey: target, + } + content = target + } else { + childMetadata = map[string]string{ + SymlinkMetadataKey: target, + } } - o, err := d.createNewObject(ctx, fullName, childMetadata) + o, err := d.createNewObject(ctx, fullName, childMetadata, content) if err != nil { return nil, err } m := storageutil.ConvertObjToMinObject(o) - d.cache.Insert(d.cacheClock.Now(), name, metadata.SymlinkType) + if !d.IsTypeCacheDeprecated() { + d.cache.Insert(d.cacheClock.Now(), name, metadata.SymlinkType) + } return &Core{ Bucket: d.Bucket(), @@ -891,6 +1195,7 @@ func (d *dirInode) CreateChildSymlink(ctx context.Context, name string, target s // LOCKS_REQUIRED(d) func (d *dirInode) CreateChildDir(ctx context.Context, name string) (*Core, error) { + // No need to cancel prefetch here as creation of new directory can not lead to stale data in metadata cache. // Generate the full name for the new directory. fullName := NewDirName(d.Name(), name) var m *gcs.MinObject @@ -907,7 +1212,7 @@ func (d *dirInode) CreateChildDir(ctx context.Context, name string) (*Core, erro } else { var o *gcs.Object // For non-hierarchical buckets, create a new object. - o, err = d.createNewObject(ctx, fullName, nil) + o, err = d.createNewObject(ctx, fullName, nil, "") if err != nil { return nil, err } @@ -916,7 +1221,9 @@ func (d *dirInode) CreateChildDir(ctx context.Context, name string) (*Core, erro } // Insert the new directory into the type cache. - d.cache.Insert(d.cacheClock.Now(), name, metadata.ExplicitDirType) + if !d.IsTypeCacheDeprecated() { + d.cache.Insert(d.cacheClock.Now(), name, metadata.ExplicitDirType) + } return &Core{ Bucket: d.Bucket(), @@ -932,7 +1239,12 @@ func (d *dirInode) DeleteChildFile( name string, generation int64, metaGeneration *int64) (err error) { - d.cache.Erase(name) + // Increment active writers on the directory so no new prefetch gets triggered until the write operation completes. + d.IncrementActiveWriters() + defer d.DecrementActiveWriters() + // Cancel prefetch of the current directory only. + d.CancelCurrDirPrefetcher() + childName := NewFileName(d.Name(), name) err = d.bucket.DeleteObject( @@ -943,13 +1255,21 @@ func (d *dirInode) DeleteChildFile( MetaGenerationPrecondition: metaGeneration, }) - if err != nil { - err = fmt.Errorf("DeleteObject: %w", err) + if err == nil { + if !d.IsTypeCacheDeprecated() { + d.cache.Erase(name) + } return } - d.cache.Erase(name) - - return + var notFoundError *gcs.NotFoundError + // DeleteObject returns notFoundError when the type has been modified remotely. + // So, evict from type-cache in such cases. + if errors.As(err, ¬FoundError) { + if !d.IsTypeCacheDeprecated() { + d.cache.Erase(name) + } + } + return fmt.Errorf("DeleteObject: %w", err) } // LOCKS_REQUIRED(d) @@ -958,47 +1278,166 @@ func (d *dirInode) DeleteChildDir( name string, isImplicitDir bool, dirInode DirInode) error { - d.cache.Erase(name) + // Increment active writers on the directory so no new prefetch gets triggered until the write operation completes. + d.IncrementActiveWriters() + defer d.DecrementActiveWriters() + // Cancel prefetch of the current directory only. + d.CancelCurrDirPrefetcher() + + if dirInode != nil { + // Recursively cancel prefetches for the deleted directory and its children. + dirInode.CancelSubdirectoryPrefetches() + } - // If the directory is an implicit directory, then no backing object - // exists in the gcs bucket, so returning from here. - // Hierarchical buckets don't have implicit dirs so this will be always false in hierarchical bucket case. - if isImplicitDir { - return nil + if !d.IsTypeCacheDeprecated() { + d.cache.Erase(name) } childName := NewDirName(d.Name(), name) + req := &gcs.DeleteObjectRequest{ + Name: childName.GcsObjectName(), + Generation: 0, // Delete the latest version. + } + + // Hierarchical Namespace (HNS) Buckets + if d.isBucketHierarchical() { + // Ignoring delete object error here, as in case of hns there is no way of knowing + // if underlying placeholder object exists or not in Hierarchical bucket. + // The DeleteFolder operation handles removing empty folders. + _ = d.bucket.DeleteObject(ctx, req) + + if err := d.bucket.DeleteFolder(ctx, req.Name); err != nil { + return fmt.Errorf("DeleteFolder: %w", err) + } + + if dirInode != nil { + dirInode.Unlink() + } + + return nil + } + + if isImplicitDir { + if !d.IsTypeCacheDeprecated() { + // If the directory is an implicit directory, then no backing object + // exists in the gcs bucket, so returning from here. + // Hierarchical buckets don't have implicit dirs so this will be always false in hierarchical bucket case. + return nil + } + // Implicit directories do not have a backing object in GCS. + // Set this flag to skip the GCS network call and only invalidate the local cache. + req.OnlyDeleteFromCache = true + } // Delete the backing object. Unfortunately we have no way to precondition // this on the directory being empty. - err := d.bucket.DeleteObject( - ctx, - &gcs.DeleteObjectRequest{ - Name: childName.GcsObjectName(), - Generation: 0, // Delete the latest version of object named after dir. - }) + if err := d.bucket.DeleteObject(ctx, req); err != nil { + return fmt.Errorf("DeleteObject: %w", err) + } - if !d.isBucketHierarchical() { - if err != nil { - return fmt.Errorf("DeleteObject: %w", err) + return nil +} + +// LOCKS_REQUIRED(d) +func (d *dirInode) DeleteObjects(ctx context.Context, objectNames []string) error { + for _, objectName := range objectNames { + if strings.HasSuffix(objectName, "/") { + // Initiate deletion for a prefix (directory). This logic handles pagination internally. + if err := d.deletePrefixRecursively(ctx, objectName); err != nil { + return fmt.Errorf("recursively deleting prefix %q: %w", objectName, err) + } + } else { + // Handle single file-like object deletion. + if err := d.deleteObject(ctx, objectName); err != nil { + return fmt.Errorf("deleting unsupported object %q: %w", objectName, err) + } + } + } + return nil +} + +// Helper to delete a single object, handling 'Not Found' errors gracefully. +// This is important for idempotency. For example, in a recursive delete, we +// list objects and then delete them. If an object is deleted by another process +// between our List and Delete calls, we'd get a 'Not Found' error. By ignoring +// it, we ensure the delete operation succeeds if the object is already gone. +func (d *dirInode) deleteObject(ctx context.Context, objectName string) error { + // For HNS buckets, the directory entry might be backed by: + // 1. **Only a folder:** Deleting the 'object' will fail (not found), and the DeleteFolder call is needed. + // 2. **A 0-byte placeholder object + an empty folder:** We first attempt to delete the 0-byte object. The subsequent DeleteFolder handles the folder removal. + + // 1. Attempt to delete the underlying GCS object (This handles files and 0-byte directory placeholders). + err := d.bucket.DeleteObject(ctx, &gcs.DeleteObjectRequest{Name: objectName}) + + // If it's a non-HNS bucket, return any error other than Not Found. + if !d.isBucketHierarchical() && err != nil { + var notFoundErr *gcs.NotFoundError + if !errors.As(err, ¬FoundErr) { + return err } - d.cache.Erase(name) - return nil } // Ignoring delete object error here, as in case of hns there is no way of knowing // if underlying placeholder object exists or not in Hierarchical bucket. // The DeleteFolder operation handles removing empty folders. - if err = d.bucket.DeleteFolder(ctx, childName.GcsObjectName()); err != nil { - return fmt.Errorf("DeleteFolder: %w", err) + if d.isBucketHierarchical() && strings.HasSuffix(objectName, "/") { + if err := d.bucket.DeleteFolder(ctx, objectName); err != nil { + var notFoundErr *gcs.NotFoundError + if !errors.As(err, ¬FoundErr) { + return err + } + } + return nil } - if d.isBucketHierarchical() { - dirInode.Unlink() + return nil +} + +// Core recursive function to list, delete, and handle pagination for a prefix. +func (d *dirInode) deletePrefixRecursively(ctx context.Context, prefix string) error { + var tok string + for { + objects, err := d.bucket.ListObjects(ctx, &gcs.ListObjectsRequest{ + Prefix: prefix, + MaxResults: MaxResultsForListObjectsCall, + Delimiter: "/", // Use Delimiter to separate nested folders (CollapsedRuns) + ContinuationToken: tok, + IncludeFoldersAsPrefixes: d.includeFoldersAsPrefixes, + }) + if err != nil { + return fmt.Errorf("listing objects under prefix %q: %w", prefix, err) + } + + // 1. Delete all file-like objects and recurse into subdirectories in parallel. + g, gCtx := errgroup.WithContext(ctx) + for _, obj := range objects.MinObjects { + // obj.Name is guaranteed to start with 'prefix'. + if !strings.HasSuffix(obj.Name, "/") { + // It's a file, delete it. + g.Go(func() error { + return d.deleteObject(gCtx, obj.Name) + }) + } + } + + for _, nestedPrefix := range objects.CollapsedRuns { + g.Go(func() error { + return d.deletePrefixRecursively(gCtx, nestedPrefix) + }) + } + + if err = g.Wait(); err != nil { + return err // Propagate the first error encountered. + } + + // If there are no more pages, we are done with this prefix's contents. + tok = objects.ContinuationToken + if tok == "" { + break + } } - d.cache.Erase(name) - return nil + return d.deleteObject(ctx, prefix) } // LOCKS_REQUIRED(fs) @@ -1038,7 +1477,45 @@ func (d *dirInode) ShouldInvalidateKernelListCache(ttl time.Duration) bool { return cachedDuration >= ttl } -func (d *dirInode) RenameFolder(ctx context.Context, folderName string, destinationFolderName string) (*gcs.Folder, error) { +// LOCKS_REQUIRED(d) +// LOCKS_REQUIRED(parent of destinationFileName) +func (d *dirInode) RenameFile(ctx context.Context, fileToRename *gcs.MinObject, destinationFileName string) (*gcs.Object, error) { + // Increment active writers on the src dir so no new prefetch gets triggered until the write operation completes. + // Note that prefetch on dest directory can still continue because it will not have stale data (only missing renamed file). + d.IncrementActiveWriters() + defer d.DecrementActiveWriters() + // Cancel prefetch of the current directory only. + d.CancelCurrDirPrefetcher() + + req := &gcs.MoveObjectRequest{ + SrcName: fileToRename.Name, + DstName: destinationFileName, + SrcGeneration: fileToRename.Generation, + SrcMetaGenerationPrecondition: &fileToRename.MetaGeneration, + } + + o, err := d.bucket.MoveObject(ctx, req) + + // Invalidate the cache entry for the old object name. + if !d.IsTypeCacheDeprecated() { + d.cache.Erase(fileToRename.Name) + } + + return o, err +} + +func (d *dirInode) RenameFolder(ctx context.Context, folderName string, destinationFolderName string, dstFolderInode DirInode) (*gcs.Folder, error) { + // Increment active writers on the directory so no new prefetch gets triggered until the write operation completes. + d.IncrementActiveWriters() + defer d.DecrementActiveWriters() + // Cancel prefetch of the current directory. + d.CancelCurrDirPrefetcher() + + if dstFolderInode != nil { + // Recursively cancel prefetches for the renamed folder and its children. + dstFolderInode.CancelSubdirectoryPrefetches() + } + folder, err := d.bucket.RenameFolder(ctx, folderName, destinationFolderName) if err != nil { return nil, err @@ -1046,7 +1523,9 @@ func (d *dirInode) RenameFolder(ctx context.Context, folderName string, destinat // TODO: Cache updates won't be necessary once type cache usage is removed from HNS. // Remove old entry from type cache. - d.cache.Erase(folderName) + if !d.IsTypeCacheDeprecated() { + d.cache.Erase(folderName) + } return folder, nil } @@ -1057,8 +1536,12 @@ func (d *dirInode) InvalidateKernelListCache() { } func (d *dirInode) isBucketHierarchical() bool { - if d.isHNSEnabled && d.bucket.BucketType() == gcs.Hierarchical { + if d.isHNSEnabled && d.bucket.BucketType().Hierarchical { return true } return false } + +func (d *dirInode) IsTypeCacheDeprecated() bool { + return d.isEnableTypeCacheDeprecation +} diff --git a/internal/fs/inode/dir_prefetcher.go b/internal/fs/inode/dir_prefetcher.go new file mode 100644 index 0000000000..72bfe0ace1 --- /dev/null +++ b/internal/fs/inode/dir_prefetcher.go @@ -0,0 +1,196 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inode + +import ( + "context" + "path" + "sync/atomic" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/jacobsa/timeutil" + "golang.org/x/sync/semaphore" +) + +// Constants for the metadata prefetch state. +const ( + prefetchReady uint32 = iota + prefetchInProgress +) + +type MetadataPrefetcher struct { + // Variables for metadata prefetching. + metadataCacheTTL time.Duration + state atomic.Uint32 // 0=Ready, 1=InProgress + + // inodeCtx is the lifecycle context of the owning inode. + // If this is cancelled (e.g. due to rename/delete folder), all prefetch runs including the subdirectories stop + // because they are all inherited from the same inodeCtx. + inodeCtx context.Context + + // runCancelFunc cancels the *current* prefetch run, without killing the inodeCtx. + runCancelFunc context.CancelFunc + + maxPrefetchCount int64 + cacheClock timeutil.Clock + lastPrefetchTime atomic.Pointer[time.Time] + // isLargeDir indicates if the directory size exceeds maxPrefetchCount. + // If true, we start prefetching from the looked-up object's offset. + isLargeDir atomic.Bool + + // sem limits the number of concurrent prefetch goroutines. + sem *semaphore.Weighted + + // listCallFunc allows the prefetcher to perform GCS List call and hydrate metadata cache. + listCallFunc func(ctx context.Context, tok string, startOffset string, limit int) (map[Name]*Core, []string, string, error) + + // shouldRun is a callback that returns true if prefetching is allowed. + // It checks if there are any active writers in the directory. + shouldRun func() bool +} + +func NewMetadataPrefetcher( + inodeCtx context.Context, // Passed from DirInode + cfg *cfg.Config, + prefetchSem *semaphore.Weighted, // Shared semaphore across all MetadataPrefetchers. + cacheClock timeutil.Clock, + listFunc func(context.Context, string, string, int) (map[Name]*Core, []string, string, error), + shouldRun func() bool, +) *MetadataPrefetcher { + return &MetadataPrefetcher{ + inodeCtx: inodeCtx, + metadataCacheTTL: time.Duration(cfg.MetadataCache.TtlSecs) * time.Second, + maxPrefetchCount: cfg.MetadataCache.MetadataPrefetchEntriesLimit, + cacheClock: cacheClock, + sem: prefetchSem, + listCallFunc: listFunc, + shouldRun: shouldRun, + // state is 0 (prefetchReady) by default. + } +} + +// Run attempts to prefetch metadata for the directory if a prefetch is due. +// It uses an atomic state to prevent concurrent execution. +// This function is already protected by directory mutex so no new mutex is required here for the setup part. +func (p *MetadataPrefetcher) Run(fullObjectName string) { + // Do not trigger prefetching if: + // 1. The inode context is nil or already cancelled (dir inode is dead/renamed). + // 2. If there are active writers in the directory, do not trigger prefetch. + if p.inodeCtx == nil || p.inodeCtx.Err() != nil || !p.shouldRun() { + return + } + + // Do not trigger prefetch if the last prefetch result is still within the TTL. + lastPrefetchTime := p.lastPrefetchTime.Load() + now := p.cacheClock.Now() + if lastPrefetchTime != nil && now.Sub(*lastPrefetchTime) < p.metadataCacheTTL { + return + } + + // Ensure only one prefetch runs at a time for this directory. + if !p.state.CompareAndSwap(prefetchReady, prefetchInProgress) { + return + } + + // Create a new context for this specific run, derived from the Inode's lifecycle context. + // This ensures that if the Inode is cancelled (Recursive), this run stops. + // We also keep runCancel so we can stop prefetch for just this run (for operations only effecting the curr dir). + ctx, cancel := context.WithCancel(p.inodeCtx) + p.runCancelFunc = cancel + + // Run in background to avoid blocking the main Lookup call. + go func() { + // Reset to Ready state when the worker finishes. + defer p.state.Store(prefetchReady) + // Ensure context resources are released when worker finishes. Calling cancel is idempotent, so it is safe even + // if p.Cancel() is called externally. + defer cancel() + + // Try to acquire a semaphore. If the semaphore is full, we skip this prefetch + // to avoid queuing stale background work. + if !p.sem.TryAcquire(1) { + return + } + defer p.sem.Release(1) + + dirName := path.Dir(fullObjectName) + var continuationToken string + var totalPrefetched int64 = 0 + + // If the directory was previously identified as 'large', we optimize by + // starting the listing from the current looked-up object to ensure its + // immediate siblings (lexicographically greater) are cached. + startOffset := "" + if p.isLargeDir.Load() { + startOffset = fullObjectName + } + + for totalPrefetched < p.maxPrefetchCount { + select { + case <-ctx.Done(): + logger.Debugf("Metadata prefetch for directory %s aborted: context cancelled.", dirName) + return + default: + } + + // Calculate how many results to ask for in this batch. + remaining := p.maxPrefetchCount - totalPrefetched + batchSize := min(remaining, MaxResultsForListObjectsCall) + + // Perform network I/O without holding the inode lock. + cores, _, newTok, err := p.listCallFunc(ctx, continuationToken, startOffset, int(batchSize)) + if err != nil { + if ctx == nil || ctx.Err() != nil { + logger.Debugf("Metadata prefetch for directory %s aborted during list call: %v", dirName, ctx.Err()) + } else { + logger.Warnf("Prefetch failed for %s: %v", dirName, err) + } + return + } + + totalPrefetched += int64(len(cores)) + + // If we hit the prefetch limit but there is still more data in GCS, + // mark this as a large directory for future targeted prefetches. + if totalPrefetched >= p.maxPrefetchCount { + if newTok != "" { + p.isLargeDir.Store(true) + } + break + } + + // End of directory reached. + if newTok == "" { + break + } + continuationToken = newTok + } + // Update lastPrefetchTime on successful completion. + now := p.cacheClock.Now() + p.lastPrefetchTime.Store(&now) + }() +} + +// Cancel stops the *current* prefetch run. It does NOT cancel the inodeCtx. +// Use this for operations to stop the prefetcher for this directory. +// This function is already protected by directory mutex so no new mutex is required here. +func (p *MetadataPrefetcher) Cancel() { + if p.runCancelFunc != nil { + p.runCancelFunc() + p.runCancelFunc = nil + } +} diff --git a/internal/fs/inode/dir_prefetcher_test.go b/internal/fs/inode/dir_prefetcher_test.go new file mode 100644 index 0000000000..708f166aa4 --- /dev/null +++ b/internal/fs/inode/dir_prefetcher_test.go @@ -0,0 +1,513 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inode + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/jacobsa/fuse/fuseops" + "github.com/jacobsa/timeutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "golang.org/x/sync/semaphore" +) + +type DirPrefetchTest struct { + ctx context.Context + bucket gcsx.SyncerBucket + fake gcs.Bucket + clock timeutil.SimulatedClock + in *dirInode + config *cfg.Config + suite.Suite +} + +func (t *DirPrefetchTest) setup(enablePrefetch bool, ttl time.Duration) (d *dirInode) { + t.T().Helper() + t.ctx = context.Background() + t.clock.SetTime(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + t.fake = fake.NewFakeBucket(&t.clock, "some_bucket", gcs.BucketType{}) + t.bucket = gcsx.NewSyncerBucket( + /*appendThreshold=*/ 1, + /*chunkRetryDeadlineSecs=*/ 120, + /*chunkTransferTimeoutSecs=*/ 10, + ".gcsfuse_tmp/", t.fake, + ) + + t.config = &cfg.Config{ + MetadataCache: cfg.MetadataCacheConfig{ + EnableMetadataPrefetch: enablePrefetch, + TypeCacheMaxSizeMb: 400, + StatCacheMaxSizeMb: 400, + TtlSecs: 60, + MetadataPrefetchEntriesLimit: 5000, + }, + } + + in := NewDirInode( + dirInodeID, + NewDirName(NewRootName(""), "dir/"), + nil, + fuseops.InodeAttributes{Mode: dirMode}, + true, // implicitDirs + false, + ttl, + &t.bucket, + &t.clock, + &t.clock, + semaphore.NewWeighted(10), + t.config, + ) + return in.(*dirInode) +} + +func (t *DirPrefetchTest) SetupTest() { + t.in = t.setup(true, time.Minute) + // Setup GCS state: a directory with files. + files := []string{"dir/file0001", "dir/file0002", "dir/implicitDir0003/a", "dir/file0004"} + require.NoError(t.T(), storageutil.CreateEmptyObjects(t.ctx, t.bucket, files)) +} + +func (t *DirPrefetchTest) TearDownTest() { + err := t.in.Destroy() + require.NoError(t.T(), err) +} + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func TestDirPrefetch(t *testing.T) { + suite.Run(t, new(DirPrefetchTest)) +} + +// Tests that LookUpChild triggers the background prefetch and populates siblings. +func (t *DirPrefetchTest) TestPrefetch_TriggersOnUnknownType() { + // Trigger LookUpChild for "implicitDir0003" which is not in cache. + t.in.mu.Lock() + _, err := t.in.LookUpChild(t.ctx, "implicitDir0003") + t.in.mu.Unlock() + require.NoError(t.T(), err) + + // Wait for the background worker to populate the cache for siblings. + assert.Eventually(t.T(), func() bool { + t.in.mu.RLock() + defer t.in.mu.RUnlock() + return t.in.cache.Get(t.clock.Now(), "file0001") == metadata.RegularFileType && + t.in.cache.Get(t.clock.Now(), "file0002") == metadata.RegularFileType && + t.in.cache.Get(t.clock.Now(), "implicitDir0003") == metadata.ImplicitDirType && + t.in.cache.Get(t.clock.Now(), "file0004") == metadata.RegularFileType + }, 2*time.Second, 10*time.Millisecond, "Prefetch should populate all siblings in cache") +} + +// Tests that if the directory is marked as large, prefetch starts from the looked-up object. +func (t *DirPrefetchTest) TestPrefetch_LargeDirUsesOffset() { + // Set the inode to LargeDir mode. + t.in.prefetcher.isLargeDir.Store(true) + // Trigger LookUpChild for "file0002" which is not in cache. + t.in.mu.Lock() + _, err := t.in.LookUpChild(t.ctx, "file0002") + t.in.mu.Unlock() + require.NoError(t.T(), err) + + // Wait for the background worker to populate the cache for siblings. + assert.Eventually(t.T(), func() bool { + return t.in.prefetcher.state.Load() == prefetchReady + }, 2*time.Second, 10*time.Millisecond) + t.in.mu.RLock() + defer t.in.mu.RUnlock() + assert.Equal(t.T(), metadata.RegularFileType, t.in.cache.Get(t.clock.Now(), "file0002")) + assert.Equal(t.T(), metadata.RegularFileType, t.in.cache.Get(t.clock.Now(), "file0004")) + assert.Equal(t.T(), metadata.ImplicitDirType, t.in.cache.Get(t.clock.Now(), "implicitDir0003")) + assert.Equal(t.T(), metadata.UnknownType, t.in.cache.Get(t.clock.Now(), "file0001"), "Objects before StartOffset should not be prefetched") +} + +// Tests that if prefetch is disabled in config, LookUpChild doesn't trigger it. +func (t *DirPrefetchTest) TestPrefetch_Disabled() { + t.in = t.setup(false, time.Minute) // Prefetch OFF + + t.in.mu.Lock() + _, err := t.in.LookUpChild(t.ctx, "file0001") + require.NoError(t.T(), err) + t.in.mu.Unlock() + + // Give it a moment to ensure no background task runs. + time.Sleep(100 * time.Millisecond) + t.in.mu.RLock() + defer t.in.mu.RUnlock() + assert.Equal(t.T(), metadata.UnknownType, t.in.cache.Get(t.clock.Now(), "file0002"), "Sibling should not be cached when prefetch is disabled") +} + +// Tests that only one prefetch can run at a time using the atomic state. +func (t *DirPrefetchTest) TestPrefetch_ConcurrentSafety() { + // 1. Manually set state to InProgress. + t.in.prefetcher.state.Store(prefetchInProgress) + + // 2. Call runOnDemandPrefetch. It should return immediately because of the state. + t.in.prefetcher.Run(NewFileName(t.in.Name(), "file0001").GcsObjectName()) + + // 3. Cache should remain empty because the function exited early. + t.in.mu.RLock() + defer t.in.mu.RUnlock() + assert.Equal(t.T(), metadata.UnknownType, t.in.cache.Get(t.clock.Now(), "file0001")) +} + +// Tests that the prefetcher respects the ctx and stops when Inode is destroyed. +func (t *DirPrefetchTest) TestPrefetch_CancellationOnDestroy() { + // Trigger a prefetch. + t.in.mu.Lock() + _, _ = t.in.LookUpChild(t.ctx, "file0001") + t.in.mu.Unlock() + + // Destroy the inode, which calls cancel(). + err := t.in.Destroy() + require.NoError(t.T(), err) + + // The state should eventually return to Ready and context should be cancelled. + assert.Equal(t.T(), t.in.prefetcher.inodeCtx.Err(), context.Canceled) + assert.Eventually(t.T(), func() bool { + return t.in.prefetcher.state.Load() == prefetchReady + }, 1*time.Second, 5*time.Millisecond) +} + +func (t *DirPrefetchTest) TestPrefetch_RespectsMaxPrefetchCount() { + t.in.prefetcher.maxPrefetchCount = 2 // Set to a small value. + + t.in.mu.Lock() + _, err := t.in.LookUpChild(t.ctx, "file0001") + t.in.mu.Unlock() + require.NoError(t.T(), err) + + assert.Eventually(t.T(), func() bool { + return t.in.prefetcher.state.Load() == prefetchReady + }, 2*time.Second, 10*time.Millisecond) + + t.in.mu.RLock() + defer t.in.mu.RUnlock() + assert.Equal(t.T(), metadata.RegularFileType, t.in.cache.Get(t.clock.Now(), "file0001")) + assert.Equal(t.T(), metadata.RegularFileType, t.in.cache.Get(t.clock.Now(), "file0002")) + assert.Equal(t.T(), metadata.UnknownType, t.in.cache.Get(t.clock.Now(), "implicitDir003"), "Sibling implicitDir003 should NOT be cached (limit reached)") + assert.Equal(t.T(), metadata.UnknownType, t.in.cache.Get(t.clock.Now(), "file0004"), "Sibling file0002 should NOT be cached (limit reached)") + assert.True(t.T(), t.in.prefetcher.isLargeDir.Load()) +} + +func (t *DirPrefetchTest) TestPrefetch_HandlesMultiplePages() { + // 1. Create 6000 objects with padded names to ensure consistent string sorting. + // Names will be: dir/file0000, dir/file0001, ... dir/file5999 + files := make([]string, 0, 6000) + for i := 0; i < 6000; i++ { + files = append(files, fmt.Sprintf("dir/file%04d", i)) + } + require.NoError(t.T(), storageutil.CreateEmptyObjects(t.ctx, t.bucket, files)) + // 2. Set maxPrefetchCount to 5500. + // This forces the prefetcher to perform: + // Page 1: 5000 results (MaxResultsForListObjectsCall) + // Page 2: 500 results (Remainder) + t.in.prefetcher.maxPrefetchCount = 5500 + + // 3. Trigger LookUpChild. We use a name that starts at the beginning + // of the sequence to ensure we fetch from file0000 onwards. + t.in.mu.Lock() + _, err := t.in.LookUpChild(t.ctx, "file0000") + t.in.mu.Unlock() + require.NoError(t.T(), err) + + // 4. Wait for the background worker to finish. + assert.Eventually(t.T(), func() bool { + return t.in.prefetcher.state.Load() == prefetchReady + }, 2*time.Second, 20*time.Millisecond) + // 5. Verify the large dir flag was set because listing wasn't finished. + assert.True(t.T(), t.in.prefetcher.isLargeDir.Load(), "Inode should be marked as large directory") + // 5. Verify the cache boundaries. + t.in.mu.RLock() + defer t.in.mu.RUnlock() + // Check the first item. + assert.Equal(t.T(), metadata.RegularFileType, t.in.cache.Get(t.clock.Now(), "file0000")) + // Check an item right at the first page boundary (5000). + assert.Equal(t.T(), metadata.RegularFileType, t.in.cache.Get(t.clock.Now(), "file4999")) + // Check the last item that SHOULD be cached (5499 is the 5500th item). + assert.Equal(t.T(), metadata.RegularFileType, t.in.cache.Get(t.clock.Now(), "file5499")) + // Check the first item that SHOULD NOT be cached (5500). + assert.Equal(t.T(), metadata.UnknownType, t.in.cache.Get(t.clock.Now(), "file5500"), + "Should have stopped prefetching after 5500 items") +} + +// mockListFunc returns a function that blocks until the provided channel is closed. +// This allows us to simulate long-running GCS calls to test concurrency limits. +func blockingListFunc(blockChan chan struct{}) func(context.Context, string, string, int) (map[Name]*Core, []string, string, error) { + return func(ctx context.Context, tok string, start string, limit int) (map[Name]*Core, []string, string, error) { + <-blockChan + return make(map[Name]*Core), nil, "", nil + } +} + +func (t *DirPrefetchTest) TestMetadataPrefetcher_ConcurrencyLimit() { + // Setup: Semaphore with a limit of 2 workers. + limit := int64(2) + sem := semaphore.NewWeighted(limit) + blockChan := make(chan struct{}) + ctx := context.Background() + p1 := NewMetadataPrefetcher(ctx, t.config, sem, &t.clock, blockingListFunc(blockChan), func() bool { return true }) + p2 := NewMetadataPrefetcher(ctx, t.config, sem, &t.clock, blockingListFunc(blockChan), func() bool { return true }) + p3 := NewMetadataPrefetcher(ctx, t.config, sem, &t.clock, blockingListFunc(blockChan), func() bool { return true }) + // 1. Run two prefetches to fill up the limit. + p1.Run("dir1/obj1") + p2.Run("dir2/obj2") + // 2. Wait until the semaphore is fully occupied. + time.Sleep(10 * time.Millisecond) + assert.False(t.T(), sem.TryAcquire(1), "Expected semaphore to be full") + + // 3. Trigger a third prefetch. + // Because TryAcquire(1) is used in Run(), it should skip immediately if full. + p3.Run("dir3/obj3") + + // 4. Release the blocked workers. + close(blockChan) + // 5. Use Eventually to wait until all permits are released. + t.Eventually(func() bool { + // Attempt to acquire the full weight. If successful, workers have finished. + if sem.TryAcquire(limit) { + sem.Release(limit) + return true + } + return false + }, 50*time.Millisecond, 5*time.Millisecond, "Expected all workers to release permits") +} + +func (t *DirPrefetchTest) TestMetadataPrefetcher_RespectsMaxParallelPrefetchesConfig() { + // User configures max 1 parallel prefetch. + sem := semaphore.NewWeighted(1) + blockChan := make(chan struct{}) + // Track how many times listFunc was actually entered. + var callCount int + var mu sync.Mutex + listFunc := func(ctx context.Context, tok string, start string, limit int) (map[Name]*Core, []string, string, error) { + mu.Lock() + callCount++ + mu.Unlock() + <-blockChan + return nil, nil, "", nil + } + ctx := context.Background() + p := NewMetadataPrefetcher(ctx, t.config, sem, &t.clock, listFunc, func() bool { return true }) + + // Trigger multiple runs on the same prefetcher (simulating different objects in same dir) + // and different prefetchers. + p.Run("a/1") + p.Run("a/2") // Will be skipped by atomic state check anyway + p2 := NewMetadataPrefetcher(ctx, t.config, sem, &t.clock, listFunc, func() bool { return true }) + p2.Run("b/1") // Should be skipped by semaphore check + + time.Sleep(10 * time.Millisecond) + mu.Lock() + assert.Equal(t.T(), 1, callCount, "Expected only 1 concurrent call based on semaphore") + mu.Unlock() + close(blockChan) +} + +func mockListFuncWithCtr() (*atomic.Int32, func(ctx context.Context, tok string, startOffset string, limit int) (map[Name]*Core, []string, string, error)) { + var listCalls atomic.Int32 + mockListFunc := func(ctx context.Context, tok string, startOffset string, limit int) (map[Name]*Core, []string, string, error) { + listCalls.Add(1) + return make(map[Name]*Core), nil, "", nil + } + return &listCalls, mockListFunc +} + +func (t *DirPrefetchTest) TestMetadataPrefetcher_TTLGuard() { + listCallCtr, mockListFunc := mockListFuncWithCtr() + sem := semaphore.NewWeighted(1) + ctx := context.Background() + p := NewMetadataPrefetcher(ctx, t.config, sem, &t.clock, mockListFunc, func() bool { return true }) + + // 1. Initial Run: Should trigger a prefetch. + p.Run("dir/obj1") + assert.Eventually(t.T(), func() bool { + return listCallCtr.Load() == 1 + }, 200*time.Millisecond, 10*time.Millisecond) + + // 2. Immediate Run: Should NOT trigger another prefetch because the + // TTL has not passed since the last run. + p.Run("dir/obj2") + // Wait to ensure no new prefetch is triggered. + time.Sleep(20 * time.Millisecond) + assert.Equal(t.T(), int32(1), listCallCtr.Load()) + + // 3. Run after TTL expiry: Should trigger a new prefetch. + t.clock.AdvanceTime(60 * time.Second) + p.Run("dir/obj3") + + assert.Eventually(t.T(), func() bool { + return listCallCtr.Load() == 2 + }, 200*time.Millisecond, 10*time.Millisecond) +} + +// TestPrefetch_RaceCondition_WriteCancelsPrefetch verifies that if a write operation +// (simulated by calling Cancel()) occurs while a prefetch list call is in progress, +// the prefetch is aborted and NO cache update occurs. +func (t *DirPrefetchTest) TestPrefetch_RaceCondition_WriteCancelsPrefetch() { + // 1. Setup: Create a mock list function that blocks until we allow it to proceed. + startChan := make(chan struct{}) // Signal that the list function has started + blockChan := make(chan struct{}) // Channel to block the list function + updatePerformed := false // Flag to track if the "critical section" was reached + // Mock ListFunc matching the signature expected by MetadataPrefetcher + mockListFunc := func(ctx context.Context, _ string, _ string, _ int) (map[Name]*Core, []string, string, error) { + close(startChan) // Notify test that we are running + <-blockChan // Wait here (simulating network latency) + // This simulates the check done before stat/type cache update. + // If the context is cancelled, we should NOT see an update. + if ctx.Err() != nil { + return nil, nil, "", ctx.Err() + } + updatePerformed = true + return nil, nil, "", nil + } + // Initialize Prefetcher with the mock + // We use context.Background() as the inodeCtx for this unit test. + p := NewMetadataPrefetcher(context.Background(), t.config, semaphore.NewWeighted(1), &t.clock, mockListFunc, func() bool { return true }) + + // 2. Act: Trigger the prefetch + p.Run("dir/obj") + + // Wait for the prefetch goroutine to start and reach the blocking point + <-startChan + // 3. Simulate a Write Operation: This calls Cancel() which should cancel the current run context. + p.Cancel() + // 4. Unblock the list function + close(blockChan) + // 5. Assert + // Wait for the prefetcher to return to 'Ready' state (worker finished) + assert.Eventually(t.T(), func() bool { + return p.state.Load() == prefetchReady + }, 1*time.Second, 10*time.Millisecond) + // CRITICAL: Verify that the update was NOT performed + assert.False(t.T(), updatePerformed, "Cache update should not happen if prefetch is cancelled") +} + +// TestPrefetch_RecursiveCancellation verifies that cancelling the Inode context +// (simulating a Directory Rename/Delete) stops future prefetch runs. +func (t *DirPrefetchTest) TestPrefetch_RecursiveCancellation() { + // Setup cancellable inode context. + inodeCtx, cancelInode := context.WithCancel(context.Background()) + _, mockListFunc := mockListFuncWithCtr() + p := NewMetadataPrefetcher(inodeCtx, t.config, semaphore.NewWeighted(1), &t.clock, mockListFunc, func() bool { return true }) + + // 1. Cancel the inode context (Simulate Rename/Delete Folder) + cancelInode() + + // 2. Try to run prefetch + p.Run("dir/obj") + // 3. Assert: Prefetch should NOT start because inodeCtx is cancelled + // The state should remain 0 (Ready) and not switch to 1 (InProgress) + // We wait briefly to ensure no async goroutine starts + time.Sleep(50 * time.Millisecond) + assert.Equal(t.T(), prefetchReady, p.state.Load()) +} + +// TestPrefetch_NilInodeContext verifies that a nil inode context does not cause a panic +// and that the prefetch operation is correctly skipped. +func (t *DirPrefetchTest) TestPrefetch_NilInodeContext() { + // Setup with a nil inode context. + listCallCtr, mockListFunc := mockListFuncWithCtr() + var nilCtx context.Context = nil + p := NewMetadataPrefetcher(nilCtx, t.config, semaphore.NewWeighted(1), &t.clock, mockListFunc, func() bool { return true }) + + // 1. Try to run prefetch. This should not panic. + p.Run("dir/obj") + + // 2. Assert: Prefetch should NOT start because inodeCtx is nil. + time.Sleep(50 * time.Millisecond) + assert.Equal(t.T(), prefetchReady, p.state.Load(), "State should remain ready") + assert.Equal(t.T(), int32(0), listCallCtr.Load(), "List function should not be called") +} + +// TestPrefetch_SkipIfActiveWriters verifies that if shouldRun returns false (simulating active writers), +// the prefetch operation is skipped. +func (t *DirPrefetchTest) TestPrefetch_SkipIfActiveWriters() { + listCallCtr, mockListFunc := mockListFuncWithCtr() + sem := semaphore.NewWeighted(1) + ctx := context.Background() + // Create prefetcher with shouldRun returning false. + p := NewMetadataPrefetcher(ctx, t.config, sem, &t.clock, mockListFunc, func() bool { return false }) + + // 1. Try to run prefetch. + p.Run("dir/obj1") + + // 2. Assert: Prefetch should NOT start because shouldRun is false. + time.Sleep(50 * time.Millisecond) + assert.Equal(t.T(), prefetchReady, p.state.Load(), "State should remain ready") + assert.Equal(t.T(), int32(0), listCallCtr.Load(), "List function should not be called") +} + +// TestPrefetch_RaceWithDeleteChildFile verifies that a slow prefetch triggered by LookUpChild +// does not overwrite fresh metadata created by a subsequent DeleteChildFile operation. +func (t *DirPrefetchTest) TestPrefetch_RaceWithDeleteChildFile() { + mockListFunc := func(ctx context.Context, tok string, start string, limit int) (map[Name]*Core, []string, string, error) { + // Simulate "Stale" data being returned for "file0001". + // This simulates the prefetcher finding an old version of the object. + res := make(map[Name]*Core) + res[Name{objectName: "file0001"}] = &Core{} + time.Sleep(700 * time.Millisecond) + return res, nil, "", nil + } + // Initialize the inode's prefetcher with the slow list function. + sem := semaphore.NewWeighted(1) + t.in.prefetcher = NewMetadataPrefetcher(t.ctx, t.config, sem, &t.clock, mockListFunc, func() bool { return true }) + + // 2. Act: Trigger a prefetch and validate that the result got cached. + // LookUpChild will trigger p.Run() which starts the background worker. + _, err := t.in.LookUpChild(t.ctx, "file0001") + require.NoError(t.T(), err) + t.in.mu.RLock() + cachedType := t.in.cache.Get(t.clock.Now(), "file0001") + assert.Equal(t.T(), metadata.RegularFileType, cachedType, + "The fresh RegularFile entry should not be overwritten by the stale prefetch result.") + t.in.mu.RUnlock() + + // 3. Perform a Write Operation: eg: DeleteFile. + // This operation is expected to finish "sooner" than the slow prefetch. + // It calls CancelCurrDirPrefetcher() internally. + var metaGen int64 = 1 + err = t.in.DeleteChildFile(t.ctx, "file0001", 1, &metaGen) + require.NoError(t.T(), err) + // Verify DeleteChildFile correctly updated the cache with "Fresh" data. + t.in.mu.RLock() + assert.Equal(t.T(), metadata.UnknownType, t.in.cache.Get(t.clock.Now(), "file0001")) + t.in.mu.RUnlock() + + // Wait for the prefetcher state to return to 'Ready', meaning the worker is done. + assert.Eventually(t.T(), func() bool { + return t.in.prefetcher.state.Load() == prefetchReady + }, 1*time.Second, 10*time.Millisecond) + + // 5. Final Assertion: The "stale" data from list must NOT be written to the cache. + t.in.mu.RLock() + cachedType = t.in.cache.Get(t.clock.Now(), "file0001") + assert.Equal(t.T(), metadata.UnknownType, cachedType, + "The fresh RegularFile entry should not be overwritten by the stale prefetch result.") + t.in.mu.RUnlock() +} diff --git a/internal/fs/inode/dir_test.go b/internal/fs/inode/dir_test.go index 714c364503..59c21a26ae 100644 --- a/internal/fs/inode/dir_test.go +++ b/internal/fs/inode/dir_test.go @@ -16,6 +16,7 @@ package inode import ( "errors" + "maps" "math" "os" "path" @@ -23,27 +24,30 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/util" - "golang.org/x/sync/semaphore" - - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/metadata" - "github.com/googlecloudplatform/gcsfuse/v2/internal/contentcache" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/fake" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata" + "github.com/googlecloudplatform/gcsfuse/v3/internal/contentcache" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/caching" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + storagemock "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/mock" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" "golang.org/x/net/context" + "golang.org/x/sync/semaphore" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" "github.com/jacobsa/fuse/fuseops" "github.com/jacobsa/fuse/fuseutil" - . "github.com/jacobsa/oglematchers" - . "github.com/jacobsa/ogletest" "github.com/jacobsa/timeutil" ) -func TestDir(t *testing.T) { RunTests(t) } - //////////////////////////////////////////////////////////////////////// // Boilerplate //////////////////////////////////////////////////////////////////////// @@ -52,6 +56,8 @@ const dirInodeID = 17 const dirInodeName = "foo/bar/" const dirMode os.FileMode = 0712 | os.ModeDir const typeCacheTTL = time.Second +const testSymlinkTarget = "blah" +const isTypeCacheDeprecationEnabled = false type DirTest struct { ctx context.Context @@ -60,26 +66,28 @@ type DirTest struct { in DirInode tc metadata.TypeCache + suite.Suite } -var _ SetUpInterface = &DirTest{} -var _ TearDownInterface = &DirTest{} - -func init() { RegisterTestSuite(&DirTest{}) } +func TestDirTest(t *testing.T) { + suite.Run(t, &DirTest{}) +} -func (t *DirTest) SetUp(ti *TestInfo) { - t.ctx = ti.Ctx +func (t *DirTest) SetupTest() { + t.ctx = context.Background() t.clock.SetTime(time.Date(2015, 4, 5, 2, 15, 0, 0, time.Local)) - bucket := fake.NewFakeBucket(&t.clock, "some_bucket", gcs.NonHierarchical) + bucket := fake.NewFakeBucket(&t.clock, "some_bucket", gcs.BucketType{}) t.bucket = gcsx.NewSyncerBucket( - 1, // Append threshold + /*appendThreshold=*/ 1, + chunkRetryDeadlineSecs, + chunkTransferTimeoutSecs, ".gcsfuse_tmp/", bucket) // Create the inode. No implicit dirs by default. - t.resetInode(false, false, true) + t.resetInode(false, false) } -func (t *DirTest) TearDown() { +func (t *DirTest) TearDownTestSuite() { t.in.Unlock() } @@ -96,8 +104,8 @@ func (p DirentSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } // NOTE: A limitation in the fake bucket's API prevents the direct creation of managed folders. // This poses a challenge for writing unit tests for includeFoldersAsPrefixes. -func (t *DirTest) resetInode(implicitDirs, enableNonexistentTypeCache, enableManagedFoldersListing bool) { - t.resetInodeWithTypeCacheConfigs(implicitDirs, enableNonexistentTypeCache, enableManagedFoldersListing, 4, typeCacheTTL) +func (t *DirTest) resetInode(implicitDirs, enableNonexistentTypeCache bool) { + t.resetInodeWithTypeCacheConfigs(implicitDirs, enableNonexistentTypeCache, true, 4, typeCacheTTL) } func (t *DirTest) resetInodeWithTypeCacheConfigs(implicitDirs, enableNonexistentTypeCache, enableManagedFoldersListing bool, typeCacheMaxSizeMB int64, typeCacheTTL time.Duration) { @@ -105,51 +113,80 @@ func (t *DirTest) resetInodeWithTypeCacheConfigs(implicitDirs, enableNonexistent t.in.Unlock() } + config := &cfg.Config{ + List: cfg.ListConfig{EnableEmptyManagedFolders: enableManagedFoldersListing}, + MetadataCache: cfg.MetadataCacheConfig{TypeCacheMaxSizeMb: typeCacheMaxSizeMB}, + EnableHns: false, + EnableUnsupportedPathSupport: true, + EnableTypeCacheDeprecation: isTypeCacheDeprecationEnabled, + } + + parInodeCtx := context.Background() t.in = NewDirInode( dirInodeID, NewDirName(NewRootName(""), dirInodeName), + parInodeCtx, fuseops.InodeAttributes{ Uid: uid, Gid: gid, Mode: dirMode, }, implicitDirs, - enableManagedFoldersListing, enableNonexistentTypeCache, typeCacheTTL, &t.bucket, &t.clock, &t.clock, - typeCacheMaxSizeMB, - false, + semaphore.NewWeighted(10), + config, ) d := t.in.(*dirInode) - AssertNe(nil, d) + require.NotNil(t.T(), d) t.tc = d.cache - AssertNe(nil, t.tc) + if !d.IsTypeCacheDeprecated() { + require.NotNil(t.T(), t.tc) + } else { + require.Nil(t.T(), t.tc) + } t.in.Lock() } func (t *DirTest) createDirInode(dirInodeName string) DirInode { + return t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, false) +} + +func (t *DirTest) createDirInodeWithTypeCacheDeprecationFlag(dirInodeName string, isTypeCacheDeprecated bool) DirInode { + config := &cfg.Config{ + List: cfg.ListConfig{EnableEmptyManagedFolders: false}, + MetadataCache: cfg.MetadataCacheConfig{ + TypeCacheMaxSizeMb: 4, + TtlSecs: 60, + }, + EnableHns: false, + EnableUnsupportedPathSupport: true, + EnableTypeCacheDeprecation: isTypeCacheDeprecated, + } + + parInodeCtx := context.Background() return NewDirInode( 5, NewDirName(NewRootName(""), dirInodeName), + parInodeCtx, fuseops.InodeAttributes{ Uid: uid, Gid: gid, Mode: dirMode, }, false, - false, true, typeCacheTTL, &t.bucket, &t.clock, &t.clock, - 4, - false, + semaphore.NewWeighted(10), + config, ) } @@ -162,7 +199,7 @@ func (t *DirTest) readAllEntries() (entries []fuseutil.Dirent, err error) { tok := "" for { var tmp []fuseutil.Dirent - tmp, tok, err = t.in.ReadEntries(t.ctx, tok) + tmp, _, tok, err = t.in.ReadEntries(t.ctx, tok) entries = append(entries, tmp...) if err != nil { return @@ -177,9 +214,31 @@ func (t *DirTest) readAllEntries() (entries []fuseutil.Dirent, err error) { return } +// Read all of the entry cores +func (t *DirTest) readAllEntryCores() (cores map[Name]*Core, unsupportedPaths []string, err error) { + cores = make(map[Name]*Core) + tok := "" + for { + var fetchedCores map[Name]*Core + var fetchedUnsupportedPaths []string + fetchedCores, fetchedUnsupportedPaths, tok, err = t.in.ReadEntryCores(t.ctx, tok) + if err != nil { + return nil, nil, err + } + maps.Copy(cores, fetchedCores) + unsupportedPaths = append(unsupportedPaths, fetchedUnsupportedPaths...) + + if tok == "" { + break + } + } + + return +} + func (t *DirTest) setSymlinkTarget( - objName string, - target string) (err error) { + objName string) (err error) { + target := testSymlinkTarget _, err = t.bucket.UpdateObject( t.ctx, &gcs.UpdateObjectRequest{ @@ -207,8 +266,11 @@ func (t *DirTest) createLocalFileInode(parent Name, name string, id fuseops.Inod contentcache.New("", &t.clock), &t.clock, true, //localFile - &cfg.WriteConfig{}, - semaphore.NewWeighted(math.MaxInt64)) + &cfg.Config{}, + semaphore.NewWeighted(math.MaxInt64), + nil, + tracing.NewNoopTracer(), + metrics.NewNoopMetrics()) // mrdCache return } @@ -216,48 +278,77 @@ func (t *DirTest) getLocalDirentKey(in Inode) string { return path.Base(in.Name().LocalName()) } +func (t *DirTest) validateCore(cores map[Name]*Core, entryName string, isDir bool, expectedType metadata.Type, expectedFullName string) { + var name Name + if isDir { + name = NewDirName(t.in.Name(), entryName) + } else { + name = NewFileName(t.in.Name(), entryName) + } + + core, ok := cores[name] + require.True(t.T(), ok, "entry for "+entryName+" not found") + assert.Equal(t.T(), expectedFullName, core.FullName.GcsObjectName()) + assert.Equal(t.T(), expectedType, core.Type()) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), expectedType, t.getTypeFromCache(entryName)) + } +} + //////////////////////////////////////////////////////////////////////// // Tests //////////////////////////////////////////////////////////////////////// -func (t *DirTest) ID() { - ExpectEq(dirInodeID, t.in.ID()) +func (t *DirTest) TestID() { + assert.EqualValues(t.T(), dirInodeID, t.in.ID()) } -func (t *DirTest) Name() { - ExpectEq(dirInodeName, t.in.Name().GcsObjectName()) +func (t *DirTest) TestName() { + assert.Equal(t.T(), dirInodeName, t.in.Name().GcsObjectName()) } -func (t *DirTest) LookupCount() { +func (t *DirTest) TestLookupCount() { // Increment thrice. The count should now be three. t.in.IncrementLookupCount() t.in.IncrementLookupCount() t.in.IncrementLookupCount() // Decrementing twice shouldn't cause destruction. But one more should. - AssertFalse(t.in.DecrementLookupCount(2)) - ExpectTrue(t.in.DecrementLookupCount(1)) + require.False(t.T(), t.in.DecrementLookupCount(2)) + assert.True(t.T(), t.in.DecrementLookupCount(1)) } -func (t *DirTest) Attributes() { - attrs, err := t.in.Attributes(t.ctx) - AssertEq(nil, err) - ExpectEq(uid, attrs.Uid) - ExpectEq(gid, attrs.Gid) - ExpectEq(dirMode|os.ModeDir, attrs.Mode) +func (t *DirTest) TestAttributes_WithClobberedCheckTrue() { + attrs, err := t.in.Attributes(t.ctx, true) + + require.NoError(t.T(), err) + assert.EqualValues(t.T(), uid, attrs.Uid) + assert.EqualValues(t.T(), gid, attrs.Gid) + assert.Equal(t.T(), dirMode|os.ModeDir, attrs.Mode) +} + +func (t *DirTest) TestAttributes_WithClobberedCheckFalse() { + attrs, err := t.in.Attributes(t.ctx, false) + + require.NoError(t.T(), err) + assert.EqualValues(t.T(), uid, attrs.Uid) + assert.EqualValues(t.T(), gid, attrs.Gid) + assert.Equal(t.T(), dirMode|os.ModeDir, attrs.Mode) } -func (t *DirTest) LookUpChild_NonExistent() { +func (t *DirTest) TestLookUpChild_NonExistent() { const name = "qux" result, err := t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertEq(nil, result) - ExpectEq(metadata.UnknownType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.Nil(t.T(), result) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.UnknownType, t.getTypeFromCache(name)) + } } -func (t *DirTest) LookUpChild_FileOnly() { +func (t *DirTest) TestLookUpChild_FileOnly() { const name = "qux" objName := path.Join(dirInodeName, name) @@ -265,28 +356,32 @@ func (t *DirTest) LookUpChild_FileOnly() { // Create a backing object. createObj, err := storageutil.CreateObject(t.ctx, t.bucket, objName, []byte("taco")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Look up with the proper name. result, err := t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.RegularFileType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.RegularFileType, t.getTypeFromCache(name)) + } - ExpectEq(objName, result.FullName.GcsObjectName()) - ExpectEq(objName, result.MinObject.Name) - ExpectEq(createObj.Generation, result.MinObject.Generation) - ExpectEq(createObj.Size, result.MinObject.Size) + assert.Equal(t.T(), objName, result.FullName.GcsObjectName()) + assert.Equal(t.T(), objName, result.MinObject.Name) + assert.Equal(t.T(), createObj.Generation, result.MinObject.Generation) + assert.Equal(t.T(), createObj.Size, result.MinObject.Size) // A conflict marker name shouldn't work. result, err = t.in.LookUpChild(t.ctx, name+ConflictingFileNameSuffix) - AssertEq(nil, err) - ExpectEq(nil, result) - ExpectEq(metadata.UnknownType, t.getTypeFromCache(name+ConflictingFileNameSuffix)) + require.NoError(t.T(), err) + assert.Nil(t.T(), result) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.UnknownType, t.getTypeFromCache(name+ConflictingFileNameSuffix)) + } } -func (t *DirTest) LookUpChild_DirOnly() { +func (t *DirTest) TestLookUpChild_DirOnly() { const name = "qux" objName := path.Join(dirInodeName, name) + "/" @@ -294,81 +389,93 @@ func (t *DirTest) LookUpChild_DirOnly() { // Create a backing object. createObj, err := storageutil.CreateObject(t.ctx, t.bucket, objName, []byte("")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Look up with the proper name. result, err := t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.ExplicitDirType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.getTypeFromCache(name)) + } - ExpectEq(objName, result.FullName.GcsObjectName()) - ExpectEq(objName, result.MinObject.Name) - ExpectEq(createObj.Generation, result.MinObject.Generation) - ExpectEq(createObj.Size, result.MinObject.Size) + assert.Equal(t.T(), objName, result.FullName.GcsObjectName()) + assert.Equal(t.T(), objName, result.MinObject.Name) + assert.Equal(t.T(), createObj.Generation, result.MinObject.Generation) + assert.Equal(t.T(), createObj.Size, result.MinObject.Size) // A conflict marker name shouldn't work. result, err = t.in.LookUpChild(t.ctx, name+ConflictingFileNameSuffix) - AssertEq(nil, err) - ExpectEq(nil, result) - ExpectEq(metadata.UnknownType, t.getTypeFromCache(name+ConflictingFileNameSuffix)) + require.NoError(t.T(), err) + assert.Nil(t.T(), result) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.UnknownType, t.getTypeFromCache(name+ConflictingFileNameSuffix)) + } } -func (t *DirTest) LookUpChild_ImplicitDirOnly_Disabled() { +func (t *DirTest) TestLookUpChild_ImplicitDirOnly_Disabled() { const name = "qux" var err error // Create an object that implicitly defines the directory. otherObjName := path.Join(dirInodeName, name) + "/asdf" _, err = storageutil.CreateObject(t.ctx, t.bucket, otherObjName, []byte("")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Looking up the name shouldn't work. result, err := t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - ExpectEq(nil, result) - ExpectEq(metadata.UnknownType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + assert.Nil(t.T(), result) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.UnknownType, t.getTypeFromCache(name)) + } // Ditto with a conflict marker. result, err = t.in.LookUpChild(t.ctx, name+ConflictingFileNameSuffix) - AssertEq(nil, err) - ExpectEq(nil, result) - ExpectEq(metadata.UnknownType, t.getTypeFromCache(name+ConflictingFileNameSuffix)) + require.NoError(t.T(), err) + assert.Nil(t.T(), result) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.UnknownType, t.getTypeFromCache(name+ConflictingFileNameSuffix)) + } } -func (t *DirTest) LookUpChild_ImplicitDirOnly_Enabled() { +func (t *DirTest) TestLookUpChild_ImplicitDirOnly_Enabled() { const name = "qux" objName := path.Join(dirInodeName, name) + "/" var err error // Enable implicit dirs. - t.resetInode(true, false, true) + t.resetInode(true, false) // Create an object that implicitly defines the directory. otherObjName := path.Join(objName, "asdf") _, err = storageutil.CreateObject(t.ctx, t.bucket, otherObjName, []byte("")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Looking up the name should work. result, err := t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - ExpectEq(nil, result.MinObject) - ExpectEq(metadata.ImplicitDirType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + assert.Nil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ImplicitDirType, t.getTypeFromCache(name)) + } - ExpectEq(objName, result.FullName.GcsObjectName()) - ExpectEq(metadata.ImplicitDirType, result.Type()) + assert.Equal(t.T(), objName, result.FullName.GcsObjectName()) + assert.Equal(t.T(), metadata.ImplicitDirType, result.Type()) // A conflict marker should not work. result, err = t.in.LookUpChild(t.ctx, name+ConflictingFileNameSuffix) - AssertEq(nil, err) - ExpectEq(nil, result) - ExpectEq(metadata.UnknownType, t.getTypeFromCache(name+ConflictingFileNameSuffix)) + require.NoError(t.T(), err) + assert.Nil(t.T(), result) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.UnknownType, t.getTypeFromCache(name+ConflictingFileNameSuffix)) + } } -func (t *DirTest) LookUpChild_FileAndDir() { +func (t *DirTest) TestLookUpChild_FileAndDir() { const name = "qux" fileObjName := path.Join(dirInodeName, name) dirObjName := path.Join(dirInodeName, name) + "/" @@ -377,37 +484,41 @@ func (t *DirTest) LookUpChild_FileAndDir() { // Create backing objects. fileObj, err := storageutil.CreateObject(t.ctx, t.bucket, fileObjName, []byte("taco")) - AssertEq(nil, err) + require.NoError(t.T(), err) dirObj, err := storageutil.CreateObject(t.ctx, t.bucket, dirObjName, []byte("")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Look up with the proper name. result, err := t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.ExplicitDirType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.getTypeFromCache(name)) + } - ExpectEq(dirObjName, result.FullName.GcsObjectName()) - ExpectEq(dirObjName, result.MinObject.Name) - ExpectEq(dirObj.Generation, result.MinObject.Generation) - ExpectEq(dirObj.Size, result.MinObject.Size) + assert.Equal(t.T(), dirObjName, result.FullName.GcsObjectName()) + assert.Equal(t.T(), dirObjName, result.MinObject.Name) + assert.Equal(t.T(), dirObj.Generation, result.MinObject.Generation) + assert.Equal(t.T(), dirObj.Size, result.MinObject.Size) // Look up with the conflict marker name. result, err = t.in.LookUpChild(t.ctx, name+ConflictingFileNameSuffix) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - AssertEq(metadata.UnknownType, t.getTypeFromCache(name+ConflictingFileNameSuffix)) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + require.Equal(t.T(), metadata.UnknownType, t.getTypeFromCache(name+ConflictingFileNameSuffix)) + } - ExpectEq(fileObjName, result.FullName.GcsObjectName()) - ExpectEq(fileObjName, result.MinObject.Name) - ExpectEq(fileObj.Generation, result.MinObject.Generation) - ExpectEq(fileObj.Size, result.MinObject.Size) + assert.Equal(t.T(), fileObjName, result.FullName.GcsObjectName()) + assert.Equal(t.T(), fileObjName, result.MinObject.Name) + assert.Equal(t.T(), fileObj.Generation, result.MinObject.Generation) + assert.Equal(t.T(), fileObj.Size, result.MinObject.Size) } -func (t *DirTest) LookUpChild_SymlinkAndDir() { +func (t *DirTest) TestLookUpChild_SymlinkAndDir() { const name = "qux" linkObjName := path.Join(dirInodeName, name) dirObjName := path.Join(dirInodeName, name) + "/" @@ -416,19 +527,19 @@ func (t *DirTest) LookUpChild_SymlinkAndDir() { // Create backing objects. linkObj, err := storageutil.CreateObject(t.ctx, t.bucket, linkObjName, []byte("taco")) - AssertEq(nil, err) + require.NoError(t.T(), err) - err = t.setSymlinkTarget(linkObjName, "blah") - AssertEq(nil, err) + err = t.setSymlinkTarget(linkObjName) + require.NoError(t.T(), err) dirObj, err := storageutil.CreateObject(t.ctx, t.bucket, dirObjName, []byte("")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Look up with the proper name. result, err := t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) // The following check should have been for metadata.SymlinkType, // because of the t.setSymlinkTarget call above, but it is not. @@ -436,27 +547,31 @@ func (t *DirTest) LookUpChild_SymlinkAndDir() { // created as a regular directory object on GCS, // and is read back the same to gcsfuse and is this stored in type-cache // also as a directory. - ExpectEq(metadata.ExplicitDirType, t.getTypeFromCache(name)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.getTypeFromCache(name)) + } - ExpectEq(dirObjName, result.FullName.GcsObjectName()) - ExpectEq(dirObjName, result.MinObject.Name) - ExpectEq(dirObj.Generation, result.MinObject.Generation) - ExpectEq(dirObj.Size, result.MinObject.Size) + assert.Equal(t.T(), dirObjName, result.FullName.GcsObjectName()) + assert.Equal(t.T(), dirObjName, result.MinObject.Name) + assert.Equal(t.T(), dirObj.Generation, result.MinObject.Generation) + assert.Equal(t.T(), dirObj.Size, result.MinObject.Size) // Look up with the conflict marker name. result, err = t.in.LookUpChild(t.ctx, name+ConflictingFileNameSuffix) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.UnknownType, t.getTypeFromCache(name+ConflictingFileNameSuffix)) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.UnknownType, t.getTypeFromCache(name+ConflictingFileNameSuffix)) + } - ExpectEq(linkObjName, result.FullName.GcsObjectName()) - ExpectEq(linkObjName, result.MinObject.Name) - ExpectEq(linkObj.Generation, result.MinObject.Generation) - ExpectEq(linkObj.Size, result.MinObject.Size) + assert.Equal(t.T(), linkObjName, result.FullName.GcsObjectName()) + assert.Equal(t.T(), linkObjName, result.MinObject.Name) + assert.Equal(t.T(), linkObj.Generation, result.MinObject.Generation) + assert.Equal(t.T(), linkObj.Size, result.MinObject.Size) } -func (t *DirTest) LookUpChild_FileAndDirAndImplicitDir_Disabled() { +func (t *DirTest) TestLookUpChild_FileAndDirAndImplicitDir_Disabled() { const name = "qux" fileObjName := path.Join(dirInodeName, name) dirObjName := path.Join(dirInodeName, name) + "/" @@ -465,42 +580,44 @@ func (t *DirTest) LookUpChild_FileAndDirAndImplicitDir_Disabled() { // Create backing objects. fileObj, err := storageutil.CreateObject(t.ctx, t.bucket, fileObjName, []byte("taco")) - AssertEq(nil, err) + require.NoError(t.T(), err) dirObj, err := storageutil.CreateObject(t.ctx, t.bucket, dirObjName, []byte("")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Create an object that implicitly defines the directory. otherObjName := path.Join(dirInodeName, name) + "/asdf" _, err = storageutil.CreateObject(t.ctx, t.bucket, otherObjName, []byte("")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Look up with the proper name. result, err := t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.ExplicitDirType, t.getTypeFromCache(name)) - ExpectEq(metadata.UnknownType, t.getTypeFromCache(path.Join(dirInodeName, name))) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.getTypeFromCache(name)) + assert.Equal(t.T(), metadata.UnknownType, t.getTypeFromCache(path.Join(dirInodeName, name))) + } - ExpectEq(dirObjName, result.FullName.GcsObjectName()) - ExpectEq(dirObjName, result.MinObject.Name) - ExpectEq(dirObj.Generation, result.MinObject.Generation) - ExpectEq(dirObj.Size, result.MinObject.Size) + assert.Equal(t.T(), dirObjName, result.FullName.GcsObjectName()) + assert.Equal(t.T(), dirObjName, result.MinObject.Name) + assert.Equal(t.T(), dirObj.Generation, result.MinObject.Generation) + assert.Equal(t.T(), dirObj.Size, result.MinObject.Size) // Look up with the conflict marker name. result, err = t.in.LookUpChild(t.ctx, name+ConflictingFileNameSuffix) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) - ExpectEq(fileObjName, result.FullName.GcsObjectName()) - ExpectEq(fileObjName, result.MinObject.Name) - ExpectEq(fileObj.Generation, result.MinObject.Generation) - ExpectEq(fileObj.Size, result.MinObject.Size) + assert.Equal(t.T(), fileObjName, result.FullName.GcsObjectName()) + assert.Equal(t.T(), fileObjName, result.MinObject.Name) + assert.Equal(t.T(), fileObj.Generation, result.MinObject.Generation) + assert.Equal(t.T(), fileObj.Size, result.MinObject.Size) } -func (t *DirTest) LookUpChild_FileAndDirAndImplicitDir_Enabled() { +func (t *DirTest) TestLookUpChild_FileAndDirAndImplicitDir_Enabled() { const name = "qux" fileObjName := path.Join(dirInodeName, name) dirObjName := path.Join(dirInodeName, name) + "/" @@ -508,47 +625,54 @@ func (t *DirTest) LookUpChild_FileAndDirAndImplicitDir_Enabled() { var err error // Enable implicit dirs. - t.resetInode(true, false, true) + t.resetInode(true, false) // Create backing objects. fileObj, err := storageutil.CreateObject(t.ctx, t.bucket, fileObjName, []byte("taco")) - AssertEq(nil, err) + require.NoError(t.T(), err) dirObj, err := storageutil.CreateObject(t.ctx, t.bucket, dirObjName, []byte("")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Create an object that implicitly defines the directory. otherObjName := path.Join(dirInodeName, name) + "/asdf" _, err = storageutil.CreateObject(t.ctx, t.bucket, otherObjName, []byte("")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Look up with the proper name. result, err := t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.ExplicitDirType, t.getTypeFromCache(name)) - ExpectEq(metadata.UnknownType, t.getTypeFromCache(path.Join(dirInodeName, name))) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.getTypeFromCache(name)) + assert.Equal(t.T(), metadata.UnknownType, t.getTypeFromCache(path.Join(dirInodeName, name))) + } - ExpectEq(dirObjName, result.FullName.GcsObjectName()) - ExpectEq(dirObjName, result.MinObject.Name) - ExpectEq(dirObj.Generation, result.MinObject.Generation) - ExpectEq(dirObj.Size, result.MinObject.Size) + assert.Equal(t.T(), dirObjName, result.FullName.GcsObjectName()) + assert.Equal(t.T(), dirObjName, result.MinObject.Name) + assert.Equal(t.T(), dirObj.Generation, result.MinObject.Generation) + assert.Equal(t.T(), dirObj.Size, result.MinObject.Size) // Look up with the conflict marker name. result, err = t.in.LookUpChild(t.ctx, name+ConflictingFileNameSuffix) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.UnknownType, t.getTypeFromCache(name+ConflictingFileNameSuffix)) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.UnknownType, t.getTypeFromCache(name+ConflictingFileNameSuffix)) + } - ExpectEq(fileObjName, result.FullName.GcsObjectName()) - ExpectEq(fileObjName, result.MinObject.Name) - ExpectEq(fileObj.Generation, result.MinObject.Generation) - ExpectEq(fileObj.Size, result.MinObject.Size) + assert.Equal(t.T(), fileObjName, result.FullName.GcsObjectName()) + assert.Equal(t.T(), fileObjName, result.MinObject.Name) + assert.Equal(t.T(), fileObj.Generation, result.MinObject.Generation) + assert.Equal(t.T(), fileObj.Size, result.MinObject.Size) } -func (t *DirTest) LookUpChild_TypeCaching() { +func (t *DirTest) TestLookUpChild_TypeCaching() { + if t.in.IsTypeCacheDeprecated() { + return + } const name = "qux" fileObjName := path.Join(dirInodeName, name) dirObjName := path.Join(dirInodeName, name) + "/" @@ -557,46 +681,55 @@ func (t *DirTest) LookUpChild_TypeCaching() { // Create a backing object for a file. _, err = storageutil.CreateObject(t.ctx, t.bucket, fileObjName, []byte("taco")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Look up; we should get the file. result, err := t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.RegularFileType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.RegularFileType, t.getTypeFromCache(name)) + } - ExpectEq(fileObjName, result.MinObject.Name) + assert.Equal(t.T(), fileObjName, result.MinObject.Name) // Create a backing object for a directory. _, err = storageutil.CreateObject(t.ctx, t.bucket, dirObjName, []byte("taco")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Look up again. Even though the directory should shadow the file, because // we've cached only seeing the file that's what we should get back. result, err = t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.RegularFileType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.RegularFileType, t.getTypeFromCache(name)) + } - ExpectEq(fileObjName, result.MinObject.Name) + assert.Equal(t.T(), fileObjName, result.MinObject.Name) // But after the TTL expires, the behavior should flip. t.clock.AdvanceTime(typeCacheTTL + time.Millisecond) result, err = t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.ExplicitDirType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.getTypeFromCache(name)) + } - ExpectEq(dirObjName, result.MinObject.Name) + assert.Equal(t.T(), dirObjName, result.MinObject.Name) } -func (t *DirTest) LookUpChild_NonExistentTypeCache_ImplicitDirsDisabled() { +func (t *DirTest) TestLookUpChild_NonExistentTypeCache_ImplicitDirsDisabled() { + if t.in.IsTypeCacheDeprecated() { + return + } // Enable enableNonexistentTypeCache for type cache - t.resetInode(false, true, true) + t.resetInode(false, true) const name = "qux" objName := path.Join(dirInodeName, name) + "/" @@ -604,19 +737,21 @@ func (t *DirTest) LookUpChild_NonExistentTypeCache_ImplicitDirsDisabled() { // Look up nonexistent object, return nil result, err := t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertEq(nil, result) + require.NoError(t.T(), err) + require.Nil(t.T(), result) // Create a backing object. createObj, err := storageutil.CreateObject(t.ctx, t.bucket, objName, []byte("")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Look up again, should still return nil due to cache result, err = t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertEq(nil, result) - ExpectEq(metadata.NonexistentType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.Nil(t.T(), result) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.NonexistentType, t.getTypeFromCache(name)) + } // But after the TTL expires, the behavior should flip. t.clock.AdvanceTime(typeCacheTTL + time.Millisecond) @@ -624,19 +759,24 @@ func (t *DirTest) LookUpChild_NonExistentTypeCache_ImplicitDirsDisabled() { // Look up again, should return correct object result, err = t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.ExplicitDirType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.getTypeFromCache(name)) + } - ExpectEq(objName, result.FullName.GcsObjectName()) - ExpectEq(objName, result.MinObject.Name) - ExpectEq(createObj.Generation, result.MinObject.Generation) - ExpectEq(createObj.Size, result.MinObject.Size) + assert.Equal(t.T(), objName, result.FullName.GcsObjectName()) + assert.Equal(t.T(), objName, result.MinObject.Name) + assert.Equal(t.T(), createObj.Generation, result.MinObject.Generation) + assert.Equal(t.T(), createObj.Size, result.MinObject.Size) } -func (t *DirTest) LookUpChild_NonExistentTypeCache_ImplicitDirsEnabled() { +func (t *DirTest) TestLookUpChild_NonExistentTypeCache_ImplicitDirsEnabled() { + if t.in.IsTypeCacheDeprecated() { + return + } // Enable implicitDirs and enableNonexistentTypeCache for type cache - t.resetInode(true, true, true) + t.resetInode(true, true) const name = "qux" objName := path.Join(dirInodeName, name) + "/" @@ -644,21 +784,25 @@ func (t *DirTest) LookUpChild_NonExistentTypeCache_ImplicitDirsEnabled() { // Look up nonexistent object, return nil result, err := t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertEq(nil, result) - ExpectEq(metadata.NonexistentType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.Nil(t.T(), result) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.NonexistentType, t.getTypeFromCache(name)) + } // Create an object that implicitly defines the directory. otherObjName := path.Join(objName, "asdf") _, err = storageutil.CreateObject(t.ctx, t.bucket, otherObjName, []byte("")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Look up again, should still return nil due to cache result, err = t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertEq(nil, result) - ExpectEq(metadata.NonexistentType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.Nil(t.T(), result) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.NonexistentType, t.getTypeFromCache(name)) + } // But after the TTL expires, the behavior should flip. t.clock.AdvanceTime(typeCacheTTL + time.Millisecond) @@ -666,20 +810,22 @@ func (t *DirTest) LookUpChild_NonExistentTypeCache_ImplicitDirsEnabled() { // Look up again, should return correct object result, err = t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - ExpectEq(nil, result.MinObject) + require.NoError(t.T(), err) + assert.Nil(t.T(), result.MinObject) - ExpectEq(objName, result.FullName.GcsObjectName()) - ExpectEq(metadata.ImplicitDirType, result.Type()) + assert.Equal(t.T(), objName, result.FullName.GcsObjectName()) + assert.Equal(t.T(), metadata.ImplicitDirType, result.Type()) // A conflict marker should not work. result, err = t.in.LookUpChild(t.ctx, name+ConflictingFileNameSuffix) - AssertEq(nil, err) - ExpectEq(nil, result) - ExpectEq(metadata.UnknownType, t.getTypeFromCache(name+ConflictingFileNameSuffix)) + require.NoError(t.T(), err) + assert.Nil(t.T(), result) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.UnknownType, t.getTypeFromCache(name+ConflictingFileNameSuffix)) + } } -func (t *DirTest) LookUpChild_TypeCacheEnabled() { +func (t *DirTest) TestLookUpChild_TypeCacheEnabled() { inputs := []struct { typeCacheMaxSizeMB int64 typeCacheTTL time.Duration @@ -700,29 +846,33 @@ func (t *DirTest) LookUpChild_TypeCacheEnabled() { // Create a backing object. o, err := storageutil.CreateObject(t.ctx, t.bucket, objName, []byte("taco")) - AssertEq(nil, err) - AssertNe(nil, o) + require.NoError(t.T(), err) + require.NotNil(t.T(), o) // Look up nonexistent object, return nil result, err := t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result) - ExpectEq(metadata.RegularFileType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.NotNil(t.T(), result) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.RegularFileType, t.getTypeFromCache(name)) + } } } -func (t *DirTest) LookUpChild_TypeCacheDisabled() { +func (t *DirTest) TestLookUpChild_TypeCacheDisabled() { inputs := []struct { typeCacheMaxSizeMB int64 typeCacheTTL time.Duration - }{{ - typeCacheMaxSizeMB: 0, - typeCacheTTL: time.Second, - }, { - typeCacheMaxSizeMB: 4, - typeCacheTTL: 0, - }} + }{ + { + typeCacheMaxSizeMB: 0, + typeCacheTTL: time.Second, + }, { + typeCacheMaxSizeMB: 4, + typeCacheTTL: 0, + }, + } for _, input := range inputs { t.resetInodeWithTypeCacheConfigs(true, true, true, input.typeCacheMaxSizeMB, input.typeCacheTTL) @@ -733,27 +883,29 @@ func (t *DirTest) LookUpChild_TypeCacheDisabled() { // Create a backing object. o, err := storageutil.CreateObject(t.ctx, t.bucket, objName, []byte("taco")) - AssertEq(nil, err) - AssertNe(nil, o) + require.NoError(t.T(), err) + require.NotNil(t.T(), o) // Look up nonexistent object, return nil result, err := t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result) - ExpectEq(metadata.UnknownType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.NotNil(t.T(), result) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.UnknownType, t.getTypeFromCache(name)) + } } } -func (t *DirTest) ReadDescendants_Empty() { +func (t *DirTest) TestReadDescendants_Empty() { descendants, err := t.in.ReadDescendants(t.ctx, 10) - AssertEq(nil, err) - ExpectEq(0, len(descendants)) + require.NoError(t.T(), err) + assert.Equal(t.T(), 0, len(descendants)) } -func (t *DirTest) ReadDescendants_NonEmpty() { +func (t *DirTest) TestReadDescendants_NonEmpty() { var err error // Set up contents. @@ -767,30 +919,30 @@ func (t *DirTest) ReadDescendants_NonEmpty() { } err = storageutil.CreateEmptyObjects(t.ctx, t.bucket, objs) - AssertEq(nil, err) + require.NoError(t.T(), err) descendants, err := t.in.ReadDescendants(t.ctx, 10) - AssertEq(nil, err) - ExpectEq(6, len(descendants)) + require.NoError(t.T(), err) + assert.Equal(t.T(), 6, len(descendants)) descendants, err = t.in.ReadDescendants(t.ctx, 2) - AssertEq(nil, err) - ExpectEq(2, len(descendants)) + require.NoError(t.T(), err) + assert.Equal(t.T(), 2, len(descendants)) } -func (t *DirTest) ReadEntries_Empty() { +func (t *DirTest) TestReadEntries_Empty() { d := t.in.(*dirInode) - AssertNe(nil, d) - AssertTrue(d.prevDirListingTimeStamp.IsZero()) + require.NotNil(t.T(), d) + require.True(t.T(), d.prevDirListingTimeStamp.IsZero()) entries, err := t.readAllEntries() - AssertEq(nil, err) - ExpectThat(entries, ElementsAre()) + require.NoError(t.T(), err) + assert.ElementsMatch(t.T(), []fuseutil.Dirent{}, entries) // Make sure prevDirListingTimeStamp is initialized. - AssertFalse(d.prevDirListingTimeStamp.IsZero()) + require.False(t.T(), d.prevDirListingTimeStamp.IsZero()) } -func (t *DirTest) ReadEntries_NonEmpty_ImplicitDirsDisabled() { +func (t *DirTest) TestReadEntries_NonEmpty_ImplicitDirsDisabled() { var err error var entry fuseutil.Dirent @@ -805,53 +957,61 @@ func (t *DirTest) ReadEntries_NonEmpty_ImplicitDirsDisabled() { } err = storageutil.CreateEmptyObjects(t.ctx, t.bucket, objs) - AssertEq(nil, err) + require.NoError(t.T(), err) // Set up the symlink target. - err = t.setSymlinkTarget(dirInodeName+"symlink", "blah") - AssertEq(nil, err) + err = t.setSymlinkTarget(dirInodeName + "symlink") + require.NoError(t.T(), err) // Nil prevDirListingTimeStamp d := t.in.(*dirInode) - AssertNe(nil, d) - AssertTrue(d.prevDirListingTimeStamp.IsZero()) + require.NotNil(t.T(), d) + require.True(t.T(), d.prevDirListingTimeStamp.IsZero()) // Read entries. entries, err := t.readAllEntries() - AssertEq(nil, err) - AssertEq(4, len(entries)) + require.NoError(t.T(), err) + require.Equal(t.T(), 4, len(entries)) entry = entries[0] - ExpectEq("backed_dir_empty", entry.Name) - ExpectEq(fuseutil.DT_Directory, entry.Type) - ExpectEq(metadata.ExplicitDirType, t.getTypeFromCache("backed_dir_empty")) + assert.Equal(t.T(), "backed_dir_empty", entry.Name) + assert.Equal(t.T(), fuseutil.DT_Directory, entry.Type) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.getTypeFromCache("backed_dir_empty")) + } entry = entries[1] - ExpectEq("backed_dir_nonempty", entry.Name) - ExpectEq(fuseutil.DT_Directory, entry.Type) - ExpectEq(metadata.ExplicitDirType, t.getTypeFromCache("backed_dir_nonempty")) + assert.Equal(t.T(), "backed_dir_nonempty", entry.Name) + assert.Equal(t.T(), fuseutil.DT_Directory, entry.Type) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.getTypeFromCache("backed_dir_nonempty")) + } entry = entries[2] - ExpectEq("file", entry.Name) - ExpectEq(fuseutil.DT_File, entry.Type) - ExpectEq(metadata.RegularFileType, t.getTypeFromCache("file")) + assert.Equal(t.T(), "file", entry.Name) + assert.Equal(t.T(), fuseutil.DT_File, entry.Type) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.RegularFileType, t.getTypeFromCache("file")) + } entry = entries[3] - ExpectEq("symlink", entry.Name) - ExpectEq(fuseutil.DT_Link, entry.Type) - ExpectEq(metadata.SymlinkType, t.getTypeFromCache("symlink")) + assert.Equal(t.T(), "symlink", entry.Name) + assert.Equal(t.T(), fuseutil.DT_Link, entry.Type) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.SymlinkType, t.getTypeFromCache("symlink")) + } // Make sure prevDirListingTimeStamp is initialized. - AssertFalse(d.prevDirListingTimeStamp.IsZero()) + require.False(t.T(), d.prevDirListingTimeStamp.IsZero()) } -func (t *DirTest) ReadEntries_NonEmpty_ImplicitDirsEnabled() { +func (t *DirTest) TestReadEntries_NonEmpty_ImplicitDirsEnabled() { var err error var entry fuseutil.Dirent // Enable implicit dirs. - t.resetInode(true, false, true) + t.resetInode(true, false) // Set up contents. objs := []string{ @@ -864,53 +1024,66 @@ func (t *DirTest) ReadEntries_NonEmpty_ImplicitDirsEnabled() { } err = storageutil.CreateEmptyObjects(t.ctx, t.bucket, objs) - AssertEq(nil, err) + require.NoError(t.T(), err) // Set up the symlink target. - err = t.setSymlinkTarget(dirInodeName+"symlink", "blah") - AssertEq(nil, err) + err = t.setSymlinkTarget(dirInodeName + "symlink") + require.NoError(t.T(), err) // Nil prevDirListingTimeStamp d := t.in.(*dirInode) - AssertNe(nil, d) - AssertTrue(d.prevDirListingTimeStamp.IsZero()) + require.NotNil(t.T(), d) + require.True(t.T(), d.prevDirListingTimeStamp.IsZero()) // Read entries. entries, err := t.readAllEntries() - AssertEq(nil, err) - AssertEq(5, len(entries)) + require.NoError(t.T(), err) + require.Equal(t.T(), 5, len(entries)) entry = entries[0] - ExpectEq("backed_dir_empty", entry.Name) - ExpectEq(fuseutil.DT_Directory, entry.Type) - ExpectEq(metadata.ExplicitDirType, t.getTypeFromCache("backed_dir_empty")) + assert.Equal(t.T(), "backed_dir_empty", entry.Name) + assert.Equal(t.T(), fuseutil.DT_Directory, entry.Type) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.getTypeFromCache("backed_dir_empty")) + } entry = entries[1] - ExpectEq("backed_dir_nonempty", entry.Name) - ExpectEq(fuseutil.DT_Directory, entry.Type) - ExpectEq(metadata.ExplicitDirType, t.getTypeFromCache("backed_dir_nonempty")) + assert.Equal(t.T(), "backed_dir_nonempty", entry.Name) + assert.Equal(t.T(), fuseutil.DT_Directory, entry.Type) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.getTypeFromCache("backed_dir_nonempty")) + } entry = entries[2] - ExpectEq("file", entry.Name) - ExpectEq(fuseutil.DT_File, entry.Type) - ExpectEq(metadata.RegularFileType, t.getTypeFromCache("file")) + assert.Equal(t.T(), "file", entry.Name) + assert.Equal(t.T(), fuseutil.DT_File, entry.Type) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.RegularFileType, t.getTypeFromCache("file")) + } entry = entries[3] - ExpectEq("implicit_dir", entry.Name) - ExpectEq(fuseutil.DT_Directory, entry.Type) - ExpectEq(metadata.ImplicitDirType, t.getTypeFromCache("implicit_dir")) + assert.Equal(t.T(), "implicit_dir", entry.Name) + assert.Equal(t.T(), fuseutil.DT_Directory, entry.Type) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ImplicitDirType, t.getTypeFromCache("implicit_dir")) + } entry = entries[4] - ExpectEq("symlink", entry.Name) - ExpectEq(fuseutil.DT_Link, entry.Type) - ExpectEq(metadata.SymlinkType, t.getTypeFromCache("symlink")) + assert.Equal(t.T(), "symlink", entry.Name) + assert.Equal(t.T(), fuseutil.DT_Link, entry.Type) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.SymlinkType, t.getTypeFromCache("symlink")) + } // Make sure prevDirListingTimeStamp is initialized. - AssertFalse(d.prevDirListingTimeStamp.IsZero()) + require.False(t.T(), d.prevDirListingTimeStamp.IsZero()) } -func (t *DirTest) ReadEntries_TypeCaching() { +func (t *DirTest) TestReadEntries_TypeCaching() { + if t.in.IsTypeCacheDeprecated() { + return + } const name = "qux" fileObjName := path.Join(dirInodeName, name) dirObjName := path.Join(dirInodeName, name) + "/" @@ -919,70 +1092,195 @@ func (t *DirTest) ReadEntries_TypeCaching() { // Create a backing object for a file. _, err = storageutil.CreateObject(t.ctx, t.bucket, fileObjName, []byte("taco")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Nil prevDirListingTimeStamp d := t.in.(*dirInode) - AssertNe(nil, d) - AssertTrue(d.prevDirListingTimeStamp.IsZero()) + require.NotNil(t.T(), d) + require.True(t.T(), d.prevDirListingTimeStamp.IsZero()) // Read the directory, priming the type cache. _, err = t.readAllEntries() - AssertEq(nil, err) - ExpectEq(metadata.RegularFileType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.RegularFileType, t.getTypeFromCache(name)) + } // Create a backing object for a directory. _, err = storageutil.CreateObject(t.ctx, t.bucket, dirObjName, []byte("taco")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Look up the name. Even though the directory should shadow the file, // because we've cached only seeing the file that's what we should get back. result, err := t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.RegularFileType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.RegularFileType, t.getTypeFromCache(name)) + } - ExpectEq(fileObjName, result.MinObject.Name) + assert.Equal(t.T(), fileObjName, result.MinObject.Name) // But after the TTL expires, the behavior should flip. t.clock.AdvanceTime(typeCacheTTL + time.Millisecond) result, err = t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.ExplicitDirType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.getTypeFromCache(name)) + } + + assert.Equal(t.T(), dirObjName, result.MinObject.Name) + + // Make sure prevDirListingTimeStamp is initialized. + require.False(t.T(), d.prevDirListingTimeStamp.IsZero()) +} + +func (t *DirTest) TestReadEntryCores_Empty() { + d := t.in.(*dirInode) + require.NotNil(t.T(), d) + require.True(t.T(), d.prevDirListingTimeStamp.IsZero()) + + cores, unsupportedPaths, err := t.readAllEntryCores() + + require.NoError(t.T(), err) + assert.Equal(t.T(), 0, len(cores)) + assert.Equal(t.T(), 0, len(unsupportedPaths)) + // Make sure prevDirListingTimeStamp is initialized. + require.False(t.T(), d.prevDirListingTimeStamp.IsZero()) +} + +func (t *DirTest) TestReadEntryCores_NonEmpty_ImplicitDirsDisabled() { + var err error + var cores map[Name]*Core + + // Set up contents. + backedDirEmptyName := path.Join(dirInodeName, "backed_dir_empty") + "/" + backedDirNonEmptyName := path.Join(dirInodeName, "backed_dir_nonempty") + "/" + backedDirNonEmptyFileName := path.Join(backedDirNonEmptyName, "blah") + testFileName := path.Join(dirInodeName, "file") + implicitDirObjName := path.Join(dirInodeName, "implicit_dir") + "/blah" + symlinkName := path.Join(dirInodeName, "symlink") + + objs := []string{ + backedDirEmptyName, + backedDirNonEmptyName, + backedDirNonEmptyFileName, + testFileName, + implicitDirObjName, + symlinkName, + } + + err = storageutil.CreateEmptyObjects(t.ctx, t.bucket, objs) + require.NoError(t.T(), err) + + // Set up the symlink target. + err = t.setSymlinkTarget(dirInodeName + "symlink") + require.NoError(t.T(), err) + + // Nil prevDirListingTimeStamp + d := t.in.(*dirInode) + require.NotNil(t.T(), d) + require.True(t.T(), d.prevDirListingTimeStamp.IsZero()) + + // Read cores. + cores, _, _, err = t.in.ReadEntryCores(t.ctx, "") + + require.NoError(t.T(), err) + require.Equal(t.T(), 4, len(cores)) + t.validateCore(cores, "backed_dir_empty", true, metadata.ExplicitDirType, backedDirEmptyName) + t.validateCore(cores, "backed_dir_nonempty", true, metadata.ExplicitDirType, backedDirNonEmptyName) + t.validateCore(cores, "file", false, metadata.RegularFileType, testFileName) + t.validateCore(cores, "symlink", false, metadata.SymlinkType, symlinkName) + // Make sure prevDirListingTimeStamp is initialized. + require.False(t.T(), d.prevDirListingTimeStamp.IsZero()) +} + +func (t *DirTest) TestReadEntryCores_NonEmpty_ImplicitDirsEnabled() { + var err error + var cores map[Name]*Core + var unsupportedPaths []string + + // Enable implicit dirs. + t.resetInode(true, false) + + // Set up contents. + backedDirEmptyName := path.Join(dirInodeName, "backed_dir_empty") + "/" + backedDirNonEmptyName := path.Join(dirInodeName, "backed_dir_nonempty") + "/" + backedDirNonEmptyFileName := path.Join(backedDirNonEmptyName, "blah") + testFileName := path.Join(dirInodeName, "file") + implicitDirObjName := path.Join(dirInodeName, "implicit_dir") + "/blah" + symlinkName := path.Join(dirInodeName, "symlink") + unsupportedPathName1 := dirInodeName + "//" + "a.txt" + unsupportedPathName2 := dirInodeName + "../" + "b.txt" - ExpectEq(dirObjName, result.MinObject.Name) + objs := []string{ + backedDirEmptyName, + backedDirNonEmptyName, + backedDirNonEmptyFileName, + testFileName, + implicitDirObjName, + symlinkName, + unsupportedPathName1, + unsupportedPathName2, + } + + err = storageutil.CreateEmptyObjects(t.ctx, t.bucket, objs) + require.NoError(t.T(), err) + // Set up the symlink target. + err = t.setSymlinkTarget(dirInodeName + "symlink") + require.NoError(t.T(), err) + + // Nil prevDirListingTimeStamp + d := t.in.(*dirInode) + require.NotNil(t.T(), d) + require.True(t.T(), d.prevDirListingTimeStamp.IsZero()) + + // Read cores. + cores, unsupportedPaths, err = t.readAllEntryCores() + + require.NoError(t.T(), err) + require.Equal(t.T(), 5, len(cores)) + require.Equal(t.T(), 2, len(unsupportedPaths)) + t.validateCore(cores, "backed_dir_empty", true, metadata.ExplicitDirType, backedDirEmptyName) + t.validateCore(cores, "backed_dir_nonempty", true, metadata.ExplicitDirType, backedDirNonEmptyName) + t.validateCore(cores, "file", false, metadata.RegularFileType, testFileName) + t.validateCore(cores, "implicit_dir", true, metadata.ImplicitDirType, path.Join(dirInodeName, "implicit_dir")+"/") + t.validateCore(cores, "symlink", false, metadata.SymlinkType, symlinkName) + assert.ElementsMatch(t.T(), []string{dirInodeName + "../", dirInodeName + "/"}, unsupportedPaths) // Make sure prevDirListingTimeStamp is initialized. - AssertFalse(d.prevDirListingTimeStamp.IsZero()) + require.False(t.T(), d.prevDirListingTimeStamp.IsZero()) } -func (t *DirTest) CreateChildFile_DoesntExist() { +func (t *DirTest) TestCreateChildFile_DoesntExist() { const name = "qux" objName := path.Join(dirInodeName, name) // Call the inode. result, err := t.in.CreateChildFile(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result) - AssertNe(nil, result.MinObject) - - ExpectEq(t.bucket.Name(), result.Bucket.Name()) - ExpectEq(result.FullName.GcsObjectName(), result.MinObject.Name) - ExpectEq(objName, result.MinObject.Name) - ExpectFalse(IsSymlink(result.MinObject)) - ExpectEq(metadata.RegularFileType, t.getTypeFromCache(name)) - - ExpectEq(1, len(result.MinObject.Metadata)) - ExpectEq( + require.NoError(t.T(), err) + require.NotNil(t.T(), result) + require.NotNil(t.T(), result.MinObject) + + assert.Equal(t.T(), t.bucket.Name(), result.Bucket.Name()) + assert.Equal(t.T(), result.FullName.GcsObjectName(), result.MinObject.Name) + assert.Equal(t.T(), objName, result.MinObject.Name) + assert.False(t.T(), IsSymlink(result.MinObject)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.RegularFileType, t.getTypeFromCache(name)) + } + + assert.Equal(t.T(), 1, len(result.MinObject.Metadata)) + assert.Equal(t.T(), t.clock.Now().UTC().Format(time.RFC3339Nano), result.MinObject.Metadata["gcsfuse_mtime"]) } -func (t *DirTest) CreateChildFile_Exists() { +func (t *DirTest) TestCreateChildFile_Exists() { const name = "qux" objName := path.Join(dirInodeName, name) @@ -990,17 +1288,22 @@ func (t *DirTest) CreateChildFile_Exists() { // Create an existing backing object. _, err = storageutil.CreateObject(t.ctx, t.bucket, objName, []byte("taco")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Call the inode. _, err = t.in.CreateChildFile(t.ctx, name) - ExpectThat(err, Error(HasSubstr("Precondition"))) - ExpectThat(err, Error(HasSubstr("exists"))) - ExpectEq(metadata.UnknownType, t.getTypeFromCache(name)) - ExpectEq(metadata.UnknownType, t.getTypeFromCache(name)) + assert.ErrorContains(t.T(), err, "Precondition") + assert.ErrorContains(t.T(), err, "exists") + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.UnknownType, t.getTypeFromCache(name)) + assert.Equal(t.T(), metadata.UnknownType, t.getTypeFromCache(name)) + } } -func (t *DirTest) CreateChildFile_TypeCaching() { +func (t *DirTest) TestCreateChildFile_TypeCaching() { + if t.in.IsTypeCacheDeprecated() { + return + } const name = "qux" fileObjName := path.Join(dirInodeName, name) dirObjName := path.Join(dirInodeName, name) + "/" @@ -1009,35 +1312,39 @@ func (t *DirTest) CreateChildFile_TypeCaching() { // Create the name. _, err = t.in.CreateChildFile(t.ctx, name) - AssertEq(nil, err) + require.NoError(t.T(), err) // Create a backing object for a directory. _, err = storageutil.CreateObject(t.ctx, t.bucket, dirObjName, []byte("taco")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Look up the name. Even though the directory should shadow the file, // because we've cached only seeing the file that's what we should get back. result, err := t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.RegularFileType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.RegularFileType, t.getTypeFromCache(name)) + } - ExpectEq(fileObjName, result.MinObject.Name) + assert.Equal(t.T(), fileObjName, result.MinObject.Name) // But after the TTL expires, the behavior should flip. t.clock.AdvanceTime(typeCacheTTL + time.Millisecond) result, err = t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.ExplicitDirType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.getTypeFromCache(name)) + } - ExpectEq(dirObjName, result.MinObject.Name) + assert.Equal(t.T(), dirObjName, result.MinObject.Name) } -func (t *DirTest) CloneToChildFile_SourceDoesntExist() { +func (t *DirTest) TestCloneToChildFile_SourceDoesntExist() { const srcName = "blah/baz" dstName := path.Join(dirInodeName, "qux") @@ -1045,84 +1352,97 @@ func (t *DirTest) CloneToChildFile_SourceDoesntExist() { // Create and then delete the source. src, err := storageutil.CreateObject(t.ctx, t.bucket, srcName, []byte("")) - AssertEq(nil, err) + require.NoError(t.T(), err) err = t.bucket.DeleteObject( t.ctx, &gcs.DeleteObjectRequest{Name: srcName}) - AssertEq(nil, err) + require.NoError(t.T(), err) // Call the inode. srcMinObject := storageutil.ConvertObjToMinObject(src) _, err = t.in.CloneToChildFile(t.ctx, path.Base(dstName), srcMinObject) var notFoundErr *gcs.NotFoundError - ExpectTrue(errors.As(err, ¬FoundErr)) - ExpectEq(metadata.UnknownType, t.getTypeFromCache(dstName)) + assert.True(t.T(), errors.As(err, ¬FoundErr)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.UnknownType, t.getTypeFromCache(dstName)) + } } -func (t *DirTest) CloneToChildFile_DestinationDoesntExist() { +func (t *DirTest) TestCloneToChildFile_DestinationDoesntExist() { const srcName = "blah/baz" dstName := path.Join(dirInodeName, "qux") // Create the source. src, err := storageutil.CreateObject(t.ctx, t.bucket, srcName, []byte("taco")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Call the inode. srcMinObject := storageutil.ConvertObjToMinObject(src) result, err := t.in.CloneToChildFile(t.ctx, path.Base(dstName), srcMinObject) - AssertEq(nil, err) - AssertNe(nil, result) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.RegularFileType, t.getTypeFromCache("qux")) + require.NoError(t.T(), err) + require.NotNil(t.T(), result) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.RegularFileType, t.getTypeFromCache("qux")) + } - ExpectEq(t.bucket.Name(), result.Bucket.Name()) - ExpectEq(result.FullName.GcsObjectName(), result.MinObject.Name) - ExpectEq(dstName, result.MinObject.Name) - ExpectFalse(IsSymlink(result.MinObject)) + assert.Equal(t.T(), t.bucket.Name(), result.Bucket.Name()) + assert.Equal(t.T(), result.FullName.GcsObjectName(), result.MinObject.Name) + assert.Equal(t.T(), dstName, result.MinObject.Name) + assert.False(t.T(), IsSymlink(result.MinObject)) // Check resulting contents. contents, err := storageutil.ReadObject(t.ctx, t.bucket, dstName) - AssertEq(nil, err) - ExpectEq("taco", string(contents)) - ExpectEq(metadata.RegularFileType, t.getTypeFromCache("qux")) + require.NoError(t.T(), err) + assert.Equal(t.T(), "taco", string(contents)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.RegularFileType, t.getTypeFromCache("qux")) + } } -func (t *DirTest) CloneToChildFile_DestinationExists() { +func (t *DirTest) TestCloneToChildFile_DestinationExists() { const srcName = "blah/baz" dstName := path.Join(dirInodeName, "qux") // Create the source. src, err := storageutil.CreateObject(t.ctx, t.bucket, srcName, []byte("taco")) - AssertEq(nil, err) + require.NoError(t.T(), err) // And a destination object that will be overwritten. _, err = storageutil.CreateObject(t.ctx, t.bucket, dstName, []byte("")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Call the inode. srcMinObject := storageutil.ConvertObjToMinObject(src) result, err := t.in.CloneToChildFile(t.ctx, path.Base(dstName), srcMinObject) - AssertEq(nil, err) - AssertNe(nil, result) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.RegularFileType, t.getTypeFromCache("qux")) + require.NoError(t.T(), err) + require.NotNil(t.T(), result) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.RegularFileType, t.getTypeFromCache("qux")) + } - ExpectEq(t.bucket.Name(), result.Bucket.Name()) - ExpectEq(result.FullName.GcsObjectName(), result.MinObject.Name) - ExpectEq(dstName, result.MinObject.Name) - ExpectFalse(IsSymlink(result.MinObject)) - ExpectEq(len("taco"), result.MinObject.Size) + assert.Equal(t.T(), t.bucket.Name(), result.Bucket.Name()) + assert.Equal(t.T(), result.FullName.GcsObjectName(), result.MinObject.Name) + assert.Equal(t.T(), dstName, result.MinObject.Name) + assert.False(t.T(), IsSymlink(result.MinObject)) + assert.EqualValues(t.T(), len("taco"), result.MinObject.Size) // Check resulting contents. contents, err := storageutil.ReadObject(t.ctx, t.bucket, dstName) - AssertEq(nil, err) - ExpectEq("taco", string(contents)) - ExpectEq(metadata.RegularFileType, t.getTypeFromCache("qux")) + require.NoError(t.T(), err) + assert.Equal(t.T(), "taco", string(contents)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.RegularFileType, t.getTypeFromCache("qux")) + } } -func (t *DirTest) CloneToChildFile_TypeCaching() { +func (t *DirTest) TestCloneToChildFile_TypeCaching() { + if t.in.IsTypeCacheDeprecated() { + return + } const srcName = "blah/baz" dstName := path.Join(dirInodeName, "qux") @@ -1130,59 +1450,65 @@ func (t *DirTest) CloneToChildFile_TypeCaching() { // Create the source. src, err := storageutil.CreateObject(t.ctx, t.bucket, srcName, []byte("")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Clone to the destination. srcMinObject := storageutil.ConvertObjToMinObject(src) _, err = t.in.CloneToChildFile(t.ctx, path.Base(dstName), srcMinObject) - AssertEq(nil, err) + require.NoError(t.T(), err) // Create a backing object for a directory. dirObjName := dstName + "/" _, err = storageutil.CreateObject(t.ctx, t.bucket, dirObjName, []byte("")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Look up the name. Even though the directory should shadow the file, // because we've cached only seeing the file that's what we should get back. result, err := t.in.LookUpChild(t.ctx, path.Base(dstName)) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.RegularFileType, t.getTypeFromCache("qux")) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.RegularFileType, t.getTypeFromCache("qux")) + } - ExpectEq(dstName, result.MinObject.Name) + assert.Equal(t.T(), dstName, result.MinObject.Name) // But after the TTL expires, the behavior should flip. t.clock.AdvanceTime(typeCacheTTL + time.Millisecond) result, err = t.in.LookUpChild(t.ctx, path.Base(dstName)) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.ExplicitDirType, t.getTypeFromCache("qux")) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.getTypeFromCache("qux")) + } - ExpectEq(dirObjName, result.MinObject.Name) + assert.Equal(t.T(), dirObjName, result.MinObject.Name) } -func (t *DirTest) CreateChildSymlink_DoesntExist() { +func (t *DirTest) TestCreateChildSymlink_DoesntExist() { const name = "qux" const target = "taco" objName := path.Join(dirInodeName, name) // Call the inode. result, err := t.in.CreateChildSymlink(t.ctx, name, target) - AssertEq(nil, err) - AssertNe(nil, result) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.SymlinkType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.NotNil(t.T(), result) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.SymlinkType, t.getTypeFromCache(name)) + } - ExpectEq(t.bucket.Name(), result.Bucket.Name()) - ExpectEq(result.FullName.GcsObjectName(), result.MinObject.Name) - ExpectEq(objName, result.MinObject.Name) - ExpectEq(target, result.MinObject.Metadata[SymlinkMetadataKey]) + assert.Equal(t.T(), t.bucket.Name(), result.Bucket.Name()) + assert.Equal(t.T(), result.FullName.GcsObjectName(), result.MinObject.Name) + assert.Equal(t.T(), objName, result.MinObject.Name) + assert.Equal(t.T(), target, result.MinObject.Metadata[SymlinkMetadataKey]) } -func (t *DirTest) CreateChildSymlink_Exists() { +func (t *DirTest) TestCreateChildSymlink_Exists() { const name = "qux" const target = "taco" objName := path.Join(dirInodeName, name) @@ -1191,16 +1517,21 @@ func (t *DirTest) CreateChildSymlink_Exists() { // Create an existing backing object. _, err = storageutil.CreateObject(t.ctx, t.bucket, objName, []byte("")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Call the inode. _, err = t.in.CreateChildSymlink(t.ctx, name, target) - ExpectThat(err, Error(HasSubstr("Precondition"))) - ExpectThat(err, Error(HasSubstr("exists"))) - ExpectEq(metadata.UnknownType, t.getTypeFromCache(name)) + assert.ErrorContains(t.T(), err, "Precondition") + assert.ErrorContains(t.T(), err, "exists") + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.UnknownType, t.getTypeFromCache(name)) + } } -func (t *DirTest) CreateChildSymlink_TypeCaching() { +func (t *DirTest) TestCreateChildSymlink_TypeCaching() { + if t.in.IsTypeCacheDeprecated() { + return + } const name = "qux" linkObjName := path.Join(dirInodeName, name) dirObjName := path.Join(dirInodeName, name) + "/" @@ -1209,53 +1540,157 @@ func (t *DirTest) CreateChildSymlink_TypeCaching() { // Create the name. _, err = t.in.CreateChildSymlink(t.ctx, name, "") - AssertEq(nil, err) + require.NoError(t.T(), err) // Create a backing object for a directory. _, err = storageutil.CreateObject(t.ctx, t.bucket, dirObjName, []byte("taco")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Look up the name. Even though the directory should shadow the symlink, // because we've cached only seeing the symlink that's what we should get // back. result, err := t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.SymlinkType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.SymlinkType, t.getTypeFromCache(name)) + } - ExpectEq(linkObjName, result.MinObject.Name) + assert.Equal(t.T(), linkObjName, result.MinObject.Name) // But after the TTL expires, the behavior should flip. t.clock.AdvanceTime(typeCacheTTL + time.Millisecond) result, err = t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.ExplicitDirType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.getTypeFromCache(name)) + } + + assert.Equal(t.T(), dirObjName, result.MinObject.Name) +} + +func (t *DirTest) TestCreateChildSymlink_StandardSymlinkEnabled() { + // Re-create inode with standard symlinks enabled. + t.in.Unlock() + config := &cfg.Config{ + List: cfg.ListConfig{EnableEmptyManagedFolders: true}, + MetadataCache: cfg.MetadataCacheConfig{TypeCacheMaxSizeMb: 4}, + EnableHns: false, + EnableUnsupportedPathSupport: true, + EnableTypeCacheDeprecation: isTypeCacheDeprecationEnabled, + EnableStandardSymlinks: true, + } + parInodeCtx := context.Background() + t.in = NewDirInode( + dirInodeID, + NewDirName(NewRootName(""), dirInodeName), + parInodeCtx, + fuseops.InodeAttributes{ + Uid: uid, + Gid: gid, + Mode: dirMode, + }, + false, // implicitDirs + false, // enableNonexistentTypeCache + typeCacheTTL, + &t.bucket, + &t.clock, + &t.clock, + semaphore.NewWeighted(10), + config, + ) + d := t.in.(*dirInode) + t.tc = d.cache + t.in.Lock() + const name = "qux" + const target = "taco" + objName := path.Join(dirInodeName, name) + + // Call the inode. + result, err := t.in.CreateChildSymlink(t.ctx, name, target) + + require.NoError(t.T(), err) + require.NotNil(t.T(), result) + require.NotNil(t.T(), result.MinObject) + assert.Equal(t.T(), metadata.SymlinkType, t.getTypeFromCache(name)) + assert.Equal(t.T(), t.bucket.Name(), result.Bucket.Name()) + assert.Equal(t.T(), result.FullName.GcsObjectName(), result.MinObject.Name) + assert.Equal(t.T(), objName, result.MinObject.Name) + // Check metadata for standard symlink + assert.Equal(t.T(), "true", result.MinObject.Metadata[StandardSymlinkMetadataKey]) + assert.Equal(t.T(), target, result.MinObject.Metadata[SymlinkMetadataKey]) + // Check content for standard symlink (should be target) + content, err := storageutil.ReadObject(t.ctx, t.bucket, objName) + require.NoError(t.T(), err) + assert.Equal(t.T(), target, string(content)) +} + +func (t *DirTest) TestDeleteChildFile_Succeeds_TypeCacheEvicted() { + const name = "qux" + objName := path.Join(dirInodeName, name) + var err error + // Create a backing object. + o, err := storageutil.CreateObject(t.ctx, t.bucket, objName, []byte("taco")) + require.NoError(t.T(), err) + // Prime the type cache. + t.in.InsertFileIntoTypeCache(name) + assert.Equal(t.T(), metadata.RegularFileType, t.getTypeFromCache(name)) + + // Call the inode. + err = t.in.DeleteChildFile(t.ctx, name, o.Generation, &o.MetaGeneration) + + require.NoError(t.T(), err) + // Check the bucket. + _, err = storageutil.ReadObject(t.ctx, t.bucket, objName) + var notFoundErr *gcs.NotFoundError + assert.ErrorAs(t.T(), err, ¬FoundErr) + // Check that the type cache has been updated. + assert.Equal(t.T(), metadata.UnknownType, t.getTypeFromCache(name)) +} + +func (t *DirTest) TestDeleteChildFile_ReturnsError_TypeCacheRetained() { + const name = "qux" + objName := path.Join(dirInodeName, name) + var err error + // Create a backing object. + o, err := storageutil.CreateObject(t.ctx, t.bucket, objName, []byte("taco")) + require.NoError(t.T(), err) + // Prime the type cache. + t.in.InsertFileIntoTypeCache(name) + assert.Equal(t.T(), metadata.RegularFileType, t.getTypeFromCache(name)) - ExpectEq(dirObjName, result.MinObject.Name) + // Call the inode with a meta-generation that will cause a precondition error. + wrongMetaGeneration := o.MetaGeneration + 1 + err = t.in.DeleteChildFile(t.ctx, name, o.Generation, &wrongMetaGeneration) + + assert.ErrorContains(t.T(), err, "DeleteObject: gcs.PreconditionError") + assert.Equal(t.T(), metadata.RegularFileType, t.getTypeFromCache(name)) } -func (t *DirTest) CreateChildDir_DoesntExist() { +func (t *DirTest) TestCreateChildDir_DoesntExist() { const name = "qux" objName := path.Join(dirInodeName, name) + "/" // Call the inode. result, err := t.in.CreateChildDir(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.ExplicitDirType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.NotNil(t.T(), result) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.getTypeFromCache(name)) + } - ExpectEq(t.bucket.Name(), result.Bucket.Name()) - ExpectEq(result.FullName.GcsObjectName(), result.MinObject.Name) - ExpectEq(objName, result.MinObject.Name) - ExpectFalse(IsSymlink(result.MinObject)) + assert.Equal(t.T(), t.bucket.Name(), result.Bucket.Name()) + assert.Equal(t.T(), result.FullName.GcsObjectName(), result.MinObject.Name) + assert.Equal(t.T(), objName, result.MinObject.Name) + assert.False(t.T(), IsSymlink(result.MinObject)) } -func (t *DirTest) CreateChildDir_Exists() { +func (t *DirTest) TestCreateChildDir_Exists() { const name = "qux" objName := path.Join(dirInodeName, name) + "/" @@ -1263,24 +1698,28 @@ func (t *DirTest) CreateChildDir_Exists() { // Create an existing backing object. _, err = storageutil.CreateObject(t.ctx, t.bucket, objName, []byte("taco")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Call the inode. _, err = t.in.CreateChildDir(t.ctx, name) - ExpectThat(err, Error(HasSubstr("Precondition"))) - ExpectThat(err, Error(HasSubstr("exists"))) - ExpectEq(metadata.UnknownType, t.getTypeFromCache(name)) + assert.ErrorContains(t.T(), err, "Precondition") + assert.ErrorContains(t.T(), err, "exists") + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.UnknownType, t.getTypeFromCache(name)) + } } -func (t *DirTest) DeleteChildFile_DoesntExist() { +func (t *DirTest) TestDeleteChildFile_DoesntExist() { const name = "qux" err := t.in.DeleteChildFile(t.ctx, name, 0, nil) - ExpectEq(nil, err) - ExpectEq(metadata.UnknownType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.UnknownType, t.getTypeFromCache(name)) + } } -func (t *DirTest) DeleteChildFile_WrongGeneration() { +func (t *DirTest) TestDeleteChildFile_WrongGeneration() { const name = "qux" objName := path.Join(dirInodeName, name) @@ -1288,19 +1727,19 @@ func (t *DirTest) DeleteChildFile_WrongGeneration() { // Create a backing object. o, err := storageutil.CreateObject(t.ctx, t.bucket, objName, []byte("taco")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Call the inode with the wrong generation. No error should be returned. err = t.in.DeleteChildFile(t.ctx, name, o.Generation+1, &o.MetaGeneration) - AssertEq(nil, err) + require.NoError(t.T(), err) // The original generation should still be there. contents, err := storageutil.ReadObject(t.ctx, t.bucket, objName) - AssertEq(nil, err) - ExpectEq("taco", string(contents)) + require.NoError(t.T(), err) + assert.Equal(t.T(), "taco", string(contents)) } -func (t *DirTest) DeleteChildFile_WrongMetaGeneration() { +func (t *DirTest) TestDeleteChildFile_WrongMetaGeneration() { const name = "qux" objName := path.Join(dirInodeName, name) @@ -1308,23 +1747,23 @@ func (t *DirTest) DeleteChildFile_WrongMetaGeneration() { // Create a backing object. o, err := storageutil.CreateObject(t.ctx, t.bucket, objName, []byte("taco")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Call the inode with the wrong meta-generation. No error should be // returned. precond := o.MetaGeneration + 1 err = t.in.DeleteChildFile(t.ctx, name, o.Generation, &precond) - ExpectThat(err, Error(HasSubstr("Precondition"))) - ExpectThat(err, Error(HasSubstr("meta-generation"))) + assert.ErrorContains(t.T(), err, "Precondition") + assert.ErrorContains(t.T(), err, "meta-generation") // The original generation should still be there. contents, err := storageutil.ReadObject(t.ctx, t.bucket, objName) - AssertEq(nil, err) - ExpectEq("taco", string(contents)) + require.NoError(t.T(), err) + assert.Equal(t.T(), "taco", string(contents)) } -func (t *DirTest) DeleteChildFile_LatestGeneration() { +func (t *DirTest) TestDeleteChildFile_LatestGeneration() { const name = "qux" objName := path.Join(dirInodeName, name) @@ -1332,19 +1771,19 @@ func (t *DirTest) DeleteChildFile_LatestGeneration() { // Create a backing object. _, err = storageutil.CreateObject(t.ctx, t.bucket, objName, []byte("taco")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Call the inode. err = t.in.DeleteChildFile(t.ctx, name, 0, nil) - AssertEq(nil, err) + require.NoError(t.T(), err) // Check the bucket. _, err = storageutil.ReadObject(t.ctx, t.bucket, objName) var notFoundErr *gcs.NotFoundError - ExpectTrue(errors.As(err, ¬FoundErr)) + assert.True(t.T(), errors.As(err, ¬FoundErr)) } -func (t *DirTest) DeleteChildFile_ParticularGenerationAndMetaGeneration() { +func (t *DirTest) TestDeleteChildFile_ParticularGenerationAndMetaGeneration() { const name = "qux" objName := path.Join(dirInodeName, name) @@ -1352,19 +1791,22 @@ func (t *DirTest) DeleteChildFile_ParticularGenerationAndMetaGeneration() { // Create a backing object. o, err := storageutil.CreateObject(t.ctx, t.bucket, objName, []byte("taco")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Call the inode. err = t.in.DeleteChildFile(t.ctx, name, o.Generation, &o.MetaGeneration) - AssertEq(nil, err) + require.NoError(t.T(), err) // Check the bucket. _, err = storageutil.ReadObject(t.ctx, t.bucket, objName) var notFoundErr *gcs.NotFoundError - ExpectTrue(errors.As(err, ¬FoundErr)) + assert.True(t.T(), errors.As(err, ¬FoundErr)) } -func (t *DirTest) DeleteChildFile_TypeCaching() { +func (t *DirTest) TestDeleteChildFile_TypeCaching() { + if t.in.IsTypeCacheDeprecated() { + return + } const name = "qux" fileObjName := path.Join(dirInodeName, name) dirObjName := path.Join(dirInodeName, name) + "/" @@ -1373,41 +1815,43 @@ func (t *DirTest) DeleteChildFile_TypeCaching() { // Create the name, priming the type cache. _, err = t.in.CreateChildFile(t.ctx, name) - AssertEq(nil, err) + require.NoError(t.T(), err) // Create a backing object for a directory. It should be shadowed by the // file. _, err = storageutil.CreateObject(t.ctx, t.bucket, dirObjName, []byte("taco")) - AssertEq(nil, err) + require.NoError(t.T(), err) result, err := t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - AssertEq(fileObjName, result.MinObject.Name) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + require.Equal(t.T(), fileObjName, result.MinObject.Name) // But after deleting the file via the inode, the directory should be // revealed. err = t.in.DeleteChildFile(t.ctx, name, 0, nil) - AssertEq(nil, err) + require.NoError(t.T(), err) result, err = t.in.LookUpChild(t.ctx, name) - AssertEq(nil, err) - AssertNe(nil, result.MinObject) - ExpectEq(metadata.ExplicitDirType, t.getTypeFromCache(name)) + require.NoError(t.T(), err) + require.NotNil(t.T(), result.MinObject) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.getTypeFromCache(name)) + } - ExpectEq(dirObjName, result.MinObject.Name) + assert.Equal(t.T(), dirObjName, result.MinObject.Name) } -func (t *DirTest) DeleteChildDir_DoesntExist() { +func (t *DirTest) TestDeleteChildDir_DoesntExist() { const name = "qux" err := t.in.DeleteChildDir(t.ctx, name, false, nil) - ExpectEq(nil, err) + require.NoError(t.T(), err) } -func (t *DirTest) DeleteChildDir_Exists() { +func (t *DirTest) TestDeleteChildDir_Exists() { const name = "qux" objName := path.Join(dirInodeName, name) + "/" @@ -1415,72 +1859,117 @@ func (t *DirTest) DeleteChildDir_Exists() { // Create a backing object. _, err = storageutil.CreateObject(t.ctx, t.bucket, objName, []byte("taco")) - AssertEq(nil, err) + require.NoError(t.T(), err) dirIn := t.createDirInode(objName) // Call the inode. err = t.in.DeleteChildDir(t.ctx, name, false, dirIn) - AssertEq(nil, err) + require.NoError(t.T(), err) // Check the bucket. _, err = storageutil.ReadObject(t.ctx, t.bucket, objName) var notFoundErr *gcs.NotFoundError - ExpectTrue(errors.As(err, ¬FoundErr)) - ExpectFalse(dirIn.IsUnlinked()) + assert.True(t.T(), errors.As(err, ¬FoundErr)) + assert.False(t.T(), dirIn.IsUnlinked()) } -func (t *DirTest) DeleteChildDir_ImplicitDirTrue() { +func (t *DirTest) TestDeleteChildDir_ImplicitDirTrue() { const name = "qux" objName := path.Join(dirInodeName, name) + "/" dirIn := t.createDirInode(objName) err := t.in.DeleteChildDir(t.ctx, name, true, dirIn) - ExpectEq(nil, err) - ExpectFalse(dirIn.IsUnlinked()) + require.NoError(t.T(), err) + assert.False(t.T(), dirIn.IsUnlinked()) } -func (t *DirTest) LocalChildFileCore() { +func (t *DirTest) TestLocalChildFileCore() { core, err := t.in.CreateLocalChildFileCore("qux") - AssertEq(nil, err) - AssertEq(t.bucket.Name(), core.Bucket.Name()) - AssertEq("foo/bar/qux", core.FullName.objectName) - AssertTrue(core.Local) - AssertEq(nil, core.MinObject) + require.NoError(t.T(), err) + assert.Equal(t.T(), t.bucket.Name(), core.Bucket.Name()) + assert.Equal(t.T(), "foo/bar/qux", core.FullName.objectName) + assert.True(t.T(), core.Local) + assert.Nil(t.T(), core.MinObject) result, err := t.in.LookUpChild(t.ctx, "qux") - AssertEq(nil, err) - AssertEq(nil, result) - ExpectEq(metadata.UnknownType, t.getTypeFromCache("qux")) + require.NoError(t.T(), err) + assert.Nil(t.T(), result) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.UnknownType, t.getTypeFromCache("qux")) + } } -func (t *DirTest) InsertIntoTypeCache() { +func (t *DirTest) TestInsertIntoTypeCache() { t.in.InsertFileIntoTypeCache("abc") - d := t.in.(*dirInode) - tp := t.tc.Get(d.cacheClock.Now(), "abc") - AssertEq(2, tp) + if !t.in.IsTypeCacheDeprecated() { + d := t.in.(*dirInode) + tp := t.tc.Get(d.cacheClock.Now(), "abc") + assert.EqualValues(t.T(), 2, tp) + } } -func (t *DirTest) EraseFromTypeCache() { +func (t *DirTest) TestEraseFromTypeCache() { + if t.in.IsTypeCacheDeprecated() { + return + } t.in.InsertFileIntoTypeCache("abc") t.in.EraseFromTypeCache("abc") d := t.in.(*dirInode) tp := d.cache.Get(d.cacheClock.Now(), "abc") - AssertEq(0, tp) + require.EqualValues(t.T(), 0, tp) } -func (t *DirTest) LocalFileEntriesEmpty() { +func (t *DirTest) TestDeleteObjects() { + // Arrange + parentDirGcsName := t.in.Name().GcsObjectName() // e.g., "foo/bar/" + d := t.in.(*dirInode) + // Define supported objects to create. + objectsToCreate := map[string]string{ + parentDirGcsName + "dir_to_delete/": "", // Explicit dir + parentDirGcsName + "dir_to_delete/file1.txt": "content1", + parentDirGcsName + "dir_to_delete/nested_dir/": "", + parentDirGcsName + "dir_to_delete/nested_dir/nested_file.txt": "content_nested", + parentDirGcsName + "file_to_delete.txt": "content_file", + } + for objName, content := range objectsToCreate { + _, err := storageutil.CreateObject(t.ctx, t.bucket, objName, []byte(content)) + require.NoError(t.T(), err) + } + // Verify initial state: all created objects exist. + for objName := range objectsToCreate { + _, err := storageutil.ReadObject(t.ctx, t.bucket, objName) + require.NoError(t.T(), err) + } + // Act: Call DeleteObjects with the list of supported objects. + objectsToDelete := []string{ + parentDirGcsName + "dir_to_delete/", + parentDirGcsName + "file_to_delete.txt", + } + + err := d.DeleteObjects(t.ctx, objectsToDelete) + + require.NoError(t.T(), err) + // Assert: All specified objects and their contents should be deleted. + for _, objName := range objectsToDelete { + _, err = storageutil.ReadObject(t.ctx, t.bucket, objName) + var notFoundErr *gcs.NotFoundError + assert.True(t.T(), errors.As(err, ¬FoundErr), "Object %s should be deleted. Error: %v", objName, err) + } +} + +func (t *DirTest) TestLocalFileEntriesEmpty() { localFileInodes := map[Name]Inode{} entries := t.in.LocalFileEntries(localFileInodes) - AssertEq(0, len(entries)) + require.Equal(t.T(), 0, len(entries)) } -func (t *DirTest) LocalFileEntriesWith2LocalChildFiles() { +func (t *DirTest) TestLocalFileEntriesWith2LocalChildFiles() { in1 := t.createLocalFileInode(t.in.Name(), "1_localChildInode", 1) in2 := t.createLocalFileInode(t.in.Name(), "2_localChildInode", 2) in3 := t.createLocalFileInode(Name{bucketName: "abc", objectName: "def/"}, "3_localNonChildInode", 3) @@ -1492,12 +1981,12 @@ func (t *DirTest) LocalFileEntriesWith2LocalChildFiles() { entries := t.in.LocalFileEntries(localFileInodes) - AssertEq(2, len(entries)) - AssertEq(entries[t.getLocalDirentKey(in1)].Name, "1_localChildInode") - AssertEq(entries[t.getLocalDirentKey(in2)].Name, "2_localChildInode") + require.Equal(t.T(), 2, len(entries)) + require.Equal(t.T(), entries[t.getLocalDirentKey(in1)].Name, "1_localChildInode") + require.Equal(t.T(), entries[t.getLocalDirentKey(in2)].Name, "2_localChildInode") } -func (t *DirTest) LocalFileEntriesWithNoLocalChildFiles() { +func (t *DirTest) TestLocalFileEntriesWithNoLocalChildFiles() { in1 := t.createLocalFileInode(Name{bucketName: "abc", objectName: "def/"}, "1_localNonChildInode", 4) in2 := t.createLocalFileInode(Name{bucketName: "abc", objectName: "def/"}, "2_localNonChildInode", 5) localFileInodes := map[Name]Inode{ @@ -1507,10 +1996,10 @@ func (t *DirTest) LocalFileEntriesWithNoLocalChildFiles() { entries := t.in.LocalFileEntries(localFileInodes) - AssertEq(0, len(entries)) + require.Equal(t.T(), 0, len(entries)) } -func (t *DirTest) LocalFileEntriesWithUnlinkedLocalChildFiles() { +func (t *DirTest) TestLocalFileEntriesWithUnlinkedLocalChildFiles() { // Create 2 local child inodes and 1 non child inode. in1 := t.createLocalFileInode(t.in.Name(), "1_localChildInode", 1) in2 := t.createLocalFileInode(t.in.Name(), "2_localChildInode", 2) @@ -1528,8 +2017,8 @@ func (t *DirTest) LocalFileEntriesWithUnlinkedLocalChildFiles() { entries := t.in.LocalFileEntries(localFileInodes) // Validate entries contains only linked child files. - AssertEq(1, len(entries)) - AssertEq(entries[t.getLocalDirentKey(in1)].Name, "1_localChildInode") + require.Equal(t.T(), 1, len(entries)) + require.Equal(t.T(), entries[t.getLocalDirentKey(in1)].Name, "1_localChildInode") } func (t *DirTest) Test_ShouldInvalidateKernelListCache_ListingNotHappenedYet() { @@ -1539,7 +2028,7 @@ func (t *DirTest) Test_ShouldInvalidateKernelListCache_ListingNotHappenedYet() { // Irrespective of the ttl value, this should always return true. shouldInvalidate := t.in.ShouldInvalidateKernelListCache(util.MaxTimeDuration) - AssertEq(true, shouldInvalidate) + require.Equal(t.T(), true, shouldInvalidate) } func (t *DirTest) Test_ShouldInvalidateKernelListCache_WithinTtl() { @@ -1550,7 +2039,7 @@ func (t *DirTest) Test_ShouldInvalidateKernelListCache_WithinTtl() { shouldInvalidate := t.in.ShouldInvalidateKernelListCache(ttl) - AssertEq(false, shouldInvalidate) + require.Equal(t.T(), false, shouldInvalidate) } func (t *DirTest) Test_ShouldInvalidateKernelListCache_ExpiredTtl() { @@ -1561,7 +2050,7 @@ func (t *DirTest) Test_ShouldInvalidateKernelListCache_ExpiredTtl() { shouldInvalidate := t.in.ShouldInvalidateKernelListCache(ttl) - AssertEq(true, shouldInvalidate) + require.Equal(t.T(), true, shouldInvalidate) } func (t *DirTest) Test_ShouldInvalidateKernelListCache_ZeroTtl() { @@ -1571,15 +2060,345 @@ func (t *DirTest) Test_ShouldInvalidateKernelListCache_ZeroTtl() { shouldInvalidate := t.in.ShouldInvalidateKernelListCache(ttl) - AssertEq(true, shouldInvalidate) + require.Equal(t.T(), true, shouldInvalidate) } func (t *DirTest) Test_InvalidateKernelListCache() { d := t.in.(*dirInode) d.prevDirListingTimeStamp = d.cacheClock.Now() - AssertFalse(d.prevDirListingTimeStamp.IsZero()) + assert.False(t.T(), d.prevDirListingTimeStamp.IsZero()) t.in.InvalidateKernelListCache() - AssertTrue(d.prevDirListingTimeStamp.IsZero()) + assert.True(t.T(), d.prevDirListingTimeStamp.IsZero()) +} + +func (t *DirTest) Test_ReadObjectsUnlocked() { + testCases := []struct { + name string + enableImplicitDirs bool + expectedCoresCount int + expectedUnsupCount int + startOffset string + }{ + { + name: "ImplicitDirsDisabled", + enableImplicitDirs: false, + expectedCoresCount: 4, // backed_dir_empty, backed_dir_nonempty, file2, symlink + expectedUnsupCount: 0, + startOffset: "", + }, + { + name: "ImplicitDirsEnabled", + enableImplicitDirs: true, + expectedCoresCount: 5, // Above + implicit_dir + expectedUnsupCount: 1, // dirInodeName + "//" + startOffset: "", + }, + { + name: "ImplicitDirsDisabledWithStartOffset", + enableImplicitDirs: false, + expectedCoresCount: 3, // backed_dir_nonempty, file2, symlink + expectedUnsupCount: 0, + startOffset: path.Join(dirInodeName, "backed_dir_nonempty") + "/", + }, + } + + // 1. Setup - Create a superset of all object types once for all cases. + objs := []string{ + path.Join(dirInodeName, "backed_dir_empty") + "/", + path.Join(dirInodeName, "backed_dir_nonempty") + "/", + path.Join(dirInodeName, "backed_dir_nonempty", "file1"), + path.Join(dirInodeName, "file2"), + path.Join(dirInodeName, "implicit_dir", "file3"), + path.Join(dirInodeName, "symlink"), + dirInodeName + "//" + "invalid", + } + err := storageutil.CreateEmptyObjects(t.ctx, t.bucket, objs) + require.NoError(t.T(), err) + err = t.setSymlinkTarget(path.Join(dirInodeName, "symlink")) + require.NoError(t.T(), err) + + for _, tc := range testCases { + t.T().Run(tc.name, func(st *testing.T) { + t.resetInode(tc.enableImplicitDirs, false) + d := t.in.(*dirInode) + + // Execute with lock management + t.in.Unlock() + cores, unsupported, _, err := d.readObjectsUnlocked(t.ctx, "", tc.startOffset, MaxResultsForListObjectsCall) + t.in.Lock() + + require.NoError(st, err) + assert.Equal(st, tc.expectedCoresCount, len(cores)) + assert.Equal(st, tc.expectedUnsupCount, len(unsupported)) + if tc.startOffset == "" { + t.validateCore(cores, "backed_dir_empty", true, metadata.ExplicitDirType, path.Join(dirInodeName, "backed_dir_empty")+"/") + } + t.validateCore(cores, "backed_dir_nonempty", true, metadata.ExplicitDirType, path.Join(dirInodeName, "backed_dir_nonempty")+"/") + t.validateCore(cores, "file2", false, metadata.RegularFileType, path.Join(dirInodeName, "file2")) + t.validateCore(cores, "symlink", false, metadata.SymlinkType, path.Join(dirInodeName, "symlink")) + if tc.enableImplicitDirs { + t.validateCore(cores, "implicit_dir", true, metadata.ImplicitDirType, path.Join(dirInodeName, "implicit_dir")+"/") + } + }) + } +} + +func (t *DirTest) Test_readObjectsUnlocked_Empty() { + // readObjectsUnlocked needs inode in unlocked state. + t.in.Unlock() + defer t.in.Lock() + d := t.in.(*dirInode) + assert.NotNil(t.T(), d) + + cores, unsupportedPaths, newTok, err := d.readObjectsUnlocked(t.ctx, "", "", MaxResultsForListObjectsCall) + + require.NoError(t.T(), err) + assert.Equal(t.T(), 0, len(cores)) + assert.Equal(t.T(), 0, len(unsupportedPaths)) + assert.Equal(t.T(), "", newTok) +} + +func (t *DirTest) TestLookUpChild_TypeCacheDeprecated_File() { + t.in.Unlock() + t.in = t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, true) + t.in.Lock() + const name = "file" + objName := path.Join(dirInodeName, name) + _, err := storageutil.CreateObject(t.ctx, t.bucket, objName, []byte("content")) + require.NoError(t.T(), err) + + entry, err := t.in.LookUpChild(t.ctx, name) + + require.NoError(t.T(), err) + require.NotNil(t.T(), entry) + assert.Equal(t.T(), objName, entry.FullName.GcsObjectName()) + assert.Equal(t.T(), metadata.RegularFileType, entry.Type()) +} + +func (t *DirTest) TestLookUpChild_TypeCacheDeprecated_ExplicitDir() { + t.in.Unlock() + t.in = t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, true) + t.in.Lock() + const name = "dir" + objName := path.Join(dirInodeName, name) + "/" + _, err := storageutil.CreateObject(t.ctx, t.bucket, objName, []byte("")) + require.NoError(t.T(), err) + + entry, err := t.in.LookUpChild(t.ctx, name) + + require.NoError(t.T(), err) + require.NotNil(t.T(), entry) + assert.Equal(t.T(), objName, entry.FullName.GcsObjectName()) + assert.Equal(t.T(), metadata.ExplicitDirType, entry.Type()) +} + +func (t *DirTest) TestLookUpChild_TypeCacheDeprecated_ImplicitDir() { + t.in.Unlock() + t.in = t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, true) + // Enable implicit dirs + t.in.(*dirInode).implicitDirs = true + t.in.Lock() + + const name = "implicit_dir" + // Create object that implies directory + objName := path.Join(dirInodeName, name, "file") + _, err := storageutil.CreateObject(t.ctx, t.bucket, objName, []byte("content")) + require.NoError(t.T(), err) + + entry, err := t.in.LookUpChild(t.ctx, name) + + require.NoError(t.T(), err) + require.NotNil(t.T(), entry) + assert.Equal(t.T(), path.Join(dirInodeName, name)+"/", entry.FullName.GcsObjectName()) + assert.Equal(t.T(), metadata.ImplicitDirType, entry.Type()) +} + +func (t *DirTest) Test_IsTypeCacheDeprecated_false() { + dInode := t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, false) + + assert.False(t.T(), dInode.IsTypeCacheDeprecated()) +} + +func (t *DirTest) Test_IsTypeCacheDeprecated_true() { + dInode := t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, true) + + assert.True(t.T(), dInode.IsTypeCacheDeprecated()) +} + +func (t *DirTest) TestLookUpChild_TypeCacheDeprecated_CacheMiss() { + mockBucket := new(storagemock.TestifyMockBucket) + mockBucket.On("BucketType").Return(gcs.BucketType{}) + syncerBucket := gcsx.NewSyncerBucket( + /*appendThreshold=*/ 1, + chunkRetryDeadlineSecs, + chunkTransferTimeoutSecs, + ".gcsfuse_tmp/", mockBucket, + ) + oldBucket := t.bucket + t.bucket = syncerBucket + defer func() { t.bucket = oldBucket }() + in := t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, true) + const name = "file" + objName := path.Join(dirInodeName, name) + dirObjName := objName + "/" + cacheMissErr := &caching.CacheMissError{} + // Expect cache lookup for file -> CacheMiss + mockBucket.On("StatObject", mock.Anything, mock.MatchedBy(func(req *gcs.StatObjectRequest) bool { + return req.Name == objName && req.FetchOnlyFromCache == true + })).Return(nil, nil, cacheMissErr).Once() + // Expect cache lookup for dir -> CacheMiss + mockBucket.On("StatObject", mock.Anything, mock.MatchedBy(func(req *gcs.StatObjectRequest) bool { + return req.Name == dirObjName && req.FetchOnlyFromCache == true + })).Return(nil, nil, cacheMissErr).Once() + // Expect actual lookup for file -> Success + minObject := &gcs.MinObject{ + Name: objName, + Generation: 1, + MetaGeneration: 1, + Size: 100, + } + mockBucket.On("StatObject", mock.Anything, mock.MatchedBy(func(req *gcs.StatObjectRequest) bool { + return req.Name == objName && req.FetchOnlyFromCache == false + })).Return(minObject, &gcs.ExtendedObjectAttributes{}, nil).Once() + // Expect actual lookup for dir -> NotFound + mockBucket.On("StatObject", mock.Anything, mock.MatchedBy(func(req *gcs.StatObjectRequest) bool { + return req.Name == dirObjName && req.FetchOnlyFromCache == false + })).Return(nil, nil, &gcs.NotFoundError{}).Once() + + in.Lock() + entry, err := in.LookUpChild(t.ctx, name) + in.Unlock() + + require.NoError(t.T(), err) + require.NotNil(t.T(), entry) + assert.Equal(t.T(), objName, entry.FullName.GcsObjectName()) + mockBucket.AssertExpectations(t.T()) +} + +func (t *DirTest) TestLookUpChild_TypeCacheDeprecated_CacheHit() { + mockBucket := new(storagemock.TestifyMockBucket) + mockBucket.On("BucketType").Return(gcs.BucketType{}) + syncerBucket := gcsx.NewSyncerBucket( + /*appendThreshold=*/ 1, + chunkRetryDeadlineSecs, + chunkTransferTimeoutSecs, + ".gcsfuse_tmp/", mockBucket, + ) + oldBucket := t.bucket + t.bucket = syncerBucket + defer func() { t.bucket = oldBucket }() + in := t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, true) + const name = "file" + objName := path.Join(dirInodeName, name) + dirObjName := objName + "/" + // Expect cache lookup for file -> Success + minObject := &gcs.MinObject{ + Name: objName, + Generation: 1, + MetaGeneration: 1, + Size: 100, + } + mockBucket.On("StatObject", mock.Anything, mock.MatchedBy(func(req *gcs.StatObjectRequest) bool { + return req.Name == objName && req.FetchOnlyFromCache == true + })).Return(minObject, &gcs.ExtendedObjectAttributes{}, nil).Once() + // Expect cache lookup for dir -> NotFound (nil, nil) + mockBucket.On("StatObject", mock.Anything, mock.MatchedBy(func(req *gcs.StatObjectRequest) bool { + return req.Name == dirObjName && req.FetchOnlyFromCache == true + })).Return(nil, nil, &gcs.NotFoundError{}).Once() + + in.Lock() + entry, err := in.LookUpChild(t.ctx, name) + in.Unlock() + + require.NoError(t.T(), err) + require.NotNil(t.T(), entry) + assert.Equal(t.T(), objName, entry.FullName.GcsObjectName()) + mockBucket.AssertExpectations(t.T()) +} + +// Test that Destroy() cancels the inode's lifecycle context. +func (t *DirTest) TestDestroy_CancelsContext() { + // 1. Setup + // Ensure the inode starts with a valid, non-cancelled context + assert.Nil(t.T(), t.in.Context().Err()) + + err := t.in.Destroy() + assert.Nil(nil, err) + + // The context should now be cancelled + assert.Equal(t.T(), context.Canceled, t.in.Context().Err()) +} + +// Test that a new DirInode has a derived context. +func (t *DirTest) TestNewDirInode_HasContext() { + assert.NotNil(t.T(), t.in.Context()) +} + +func (t *DirTest) TestMetadataPrefetcher_InitializationGuards() { + testCases := []struct { + name string + enablePrefetch bool + statCacheMaxSizeMb int64 + ttlSecs int64 + expectActive bool + }{ + { + name: "DisabledInConfig", + enablePrefetch: false, + statCacheMaxSizeMb: 4, + ttlSecs: 60, + expectActive: false, + }, + { + name: "ZeroCacheSize", + enablePrefetch: true, + statCacheMaxSizeMb: 0, + ttlSecs: 60, + expectActive: false, + }, + { + name: "ZeroTTL", + enablePrefetch: true, + statCacheMaxSizeMb: 4, + ttlSecs: 0, + expectActive: false, + }, + { + name: "ValidConfig", + enablePrefetch: true, + statCacheMaxSizeMb: 4, + ttlSecs: 60, + expectActive: true, + }, + } + + for _, tc := range testCases { + t.T().Run(tc.name, func(st *testing.T) { + config := &cfg.Config{ + MetadataCache: cfg.MetadataCacheConfig{ + EnableMetadataPrefetch: tc.enablePrefetch, + StatCacheMaxSizeMb: tc.statCacheMaxSizeMb, + TtlSecs: tc.ttlSecs, + MetadataPrefetchEntriesLimit: 500, + }, + } + + // Create a new inode with the specific config + inode := NewDirInode( + dirInodeID, + NewDirName(NewRootName(""), dirInodeName), + context.Background(), + fuseops.InodeAttributes{Mode: dirMode}, + false, false, time.Second, + &t.bucket, &t.clock, &t.clock, + semaphore.NewWeighted(10), + config, + ) + + d := inode.(*dirInode) + assert.Equal(st, tc.expectActive, d.prefetcher != nil) + }) + } } diff --git a/internal/fs/inode/explicit_dir.go b/internal/fs/inode/explicit_dir.go index 482836a869..0a5cb807f1 100644 --- a/internal/fs/inode/explicit_dir.go +++ b/internal/fs/inode/explicit_dir.go @@ -15,12 +15,15 @@ package inode import ( + "context" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "github.com/jacobsa/fuse/fuseops" "github.com/jacobsa/timeutil" + "golang.org/x/sync/semaphore" ) // An inode representing a directory backed by an object in GCS with a specific @@ -35,30 +38,30 @@ type ExplicitDirInode interface { func NewExplicitDirInode( id fuseops.InodeID, name Name, + parentInodeCtx context.Context, m *gcs.MinObject, attrs fuseops.InodeAttributes, implicitDirs bool, - includeFoldersAsPrefixes bool, enableNonexistentTypeCache bool, typeCacheTTL time.Duration, bucket *gcsx.SyncerBucket, mtimeClock timeutil.Clock, cacheClock timeutil.Clock, - typeCacheMaxSizeMB int64, - enableHNS bool) (d ExplicitDirInode) { + prefetchSem *semaphore.Weighted, + cfg *cfg.Config) (d ExplicitDirInode) { wrapped := NewDirInode( id, name, + parentInodeCtx, attrs, implicitDirs, - includeFoldersAsPrefixes, enableNonexistentTypeCache, typeCacheTTL, bucket, mtimeClock, cacheClock, - typeCacheMaxSizeMB, - enableHNS) + prefetchSem, + cfg) dirInode := &explicitDirInode{ dirInode: wrapped.(*dirInode), @@ -68,6 +71,7 @@ func NewExplicitDirInode( dirInode.generation = Generation{ Object: m.Generation, Metadata: m.MetaGeneration, + Size: m.Size, } } @@ -84,3 +88,7 @@ func (d *explicitDirInode) SourceGeneration() (gen Generation) { gen = d.generation return } + +func (d *explicitDirInode) UpdateSize(size uint64) { + // No-op for directories. +} diff --git a/internal/fs/inode/file.go b/internal/fs/inode/file.go index d1385cbfd5..67b7a08a4a 100644 --- a/internal/fs/inode/file.go +++ b/internal/fs/inode/file.go @@ -22,22 +22,35 @@ import ( "strings" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/bufferedwrites" - "github.com/googlecloudplatform/gcsfuse/v2/internal/contentcache" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + "context" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/block" + "github.com/googlecloudplatform/gcsfuse/v3/internal/bufferedwrites" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/contentcache" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/gcsfuse_errors" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" "github.com/jacobsa/fuse/fuseops" "github.com/jacobsa/syncutil" "github.com/jacobsa/timeutil" - "golang.org/x/net/context" "golang.org/x/sync/semaphore" ) // A GCS object metadata key for file mtimes. mtimes are UTC, and are stored in // the format defined by time.RFC3339Nano. -const FileMtimeMetadataKey = gcsx.MtimeMetadataKey +const ( + FileMtimeMetadataKey = gcs.MtimeMetadataKey + // TODO(b/447991081): Update streaming writes semantic message once semantics for ZB are updated on semantics doc. + StreamingWritesSemantics = "Streaming writes is supported for sequential writes to new/empty files. " + + "For more details, see: https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#writes" +) type FileInode struct { ///////////////////////// @@ -92,9 +105,33 @@ type FileInode struct { // Represents if local file has been unlinked. unlinked bool - bwh *bufferedwrites.BufferedWriteHandler - writeConfig *cfg.WriteConfig - globalMaxBlocksSem *semaphore.Weighted + // Wrapper object for multi range downloader. Needed as we will create the MRD in + // random reader and we can't pass fileInode object to random reader as it + // creates a cyclic dependency. + // Todo: Investigate if cyclic dependency can be removed by removing some unused + // code. + MRDWrapper *gcsx.MultiRangeDownloaderWrapper + + bwh bufferedwrites.BufferedWriteHandler + config *cfg.Config + + // Once write is started on the file i.e, bwh is initialized, any fileHandles + // opened in write mode before or after this and not yet closed are considered + // as writing to the file even though they are not writing. + // In case of successful flush, we will set bwh to nil. But in case of error, + // we will keep returning that error to all the fileHandles open during that time + // and set bwh to nil after all fileHandlers are closed. + // writeHandleCount tracks the count of open fileHandles in write mode. + writeHandleCount int32 + + // Limits the max number of blocks that can be created across file system when + // streaming writes are enabled. + globalMaxWriteBlocksSem *semaphore.Weighted + + // mrdInstance manages the MultiRangeDownloader instances for this inode. + mrdInstance *gcsx.MrdInstance + metricHandle metrics.MetricHandle + traceHandle tracing.TraceHandle } var _ Inode = &FileInode{} @@ -117,26 +154,40 @@ func NewFileInode( contentCache *contentcache.ContentCache, mtimeClock timeutil.Clock, localFile bool, - writeConfig *cfg.WriteConfig, - globalMaxBlocksSem *semaphore.Weighted) (f *FileInode) { + cfg *cfg.Config, + globalMaxBlocksSem *semaphore.Weighted, + mrdCache *lru.Cache, + traceHandle tracing.TraceHandle, + metricHandle metrics.MetricHandle) (f *FileInode) { // Set up the basic struct. var minObj gcs.MinObject if m != nil { minObj = *m } f = &FileInode{ - bucket: bucket, - mtimeClock: mtimeClock, - id: id, - name: name, - attrs: attrs, - localFileCache: localFileCache, - contentCache: contentCache, - src: minObj, - local: localFile, - unlinked: false, - writeConfig: writeConfig, - globalMaxBlocksSem: globalMaxBlocksSem, + bucket: bucket, + mtimeClock: mtimeClock, + id: id, + name: name, + attrs: attrs, + localFileCache: localFileCache, + contentCache: contentCache, + src: minObj, + local: localFile, + unlinked: false, + config: cfg, + globalMaxWriteBlocksSem: globalMaxBlocksSem, + traceHandle: traceHandle, + metricHandle: metricHandle, + } + + if f.bucket.BucketType().IsRapid() { + var err error + f.mrdInstance = gcsx.NewMrdInstance(&minObj, bucket, mrdCache, id, cfg) + f.MRDWrapper, err = gcsx.NewMultiRangeDownloaderWrapper(bucket, &minObj, cfg, mrdCache) + if err != nil { + logger.Errorf("NewFileInode: Error in creating MRDWrapper %v", err) + } } f.lc.Init(id) @@ -178,6 +229,22 @@ func (f *FileInode) checkInvariants() { } } +// clobbered checks if the FileInode's corresponding GCS object has been +// unexpectedly modified or deleted. +// +// It compares the GCS object's metadata (generation, size, meta-generation) +// against the inode's expected f.SourceGeneration(). +// +// Return Values (object *gcs.Object, isClobbered bool, err error): +// +// | Condition | oGen.Compare | GCS Error | Returns (*gcs.Object, bool, error) | +// |----------------------------------------------|--------------|-----------|---------------------------------------------| +// | Generations Match | 0 | nil | (latestObject, false, nil) | +// | GCS Object Size Greater at same generation | 2 | nil | (latestObject, true, nil) | +// | GCS Object Older/Divergent | -1 or 1 | nil | (nil, true, nil) | +// | Object Not Found | N/A | NotFound | (nil, true(false if local), nil) | +// | Other GCS Stat Error | N/A | Other | (nil, false, <GCS Error>) | +// // LOCKS_REQUIRED(f.mu) func (f *FileInode) clobbered(ctx context.Context, forceFetchFromGcs bool, includeExtendedObjectAttributes bool) (o *gcs.Object, b bool, err error) { // Stat the object in GCS. ForceFetchFromGcs ensures object is fetched from @@ -213,21 +280,43 @@ func (f *FileInode) clobbered(ctx context.Context, forceFetchFromGcs bool, inclu } // We are clobbered iff the generation doesn't match our source generation. - oGen := Generation{o.Generation, o.MetaGeneration} - b = f.SourceGeneration().Compare(oGen) != 0 - - return + oGen := Generation{o.Generation, o.MetaGeneration, o.Size} + cmp := oGen.Compare(f.SourceGeneration()) + switch cmp { + case 0: + // Generations and size match: Not clobbered. Return the fetched object. + return o, false, nil + case 2: + // The latest GCS object has greater size at the same generation. Return + // the fetched object. We also return isClobbered true to indicate the + // remote size change. + return o, true, nil + default: // -1 (GCS is older) or 1 (GCS has different gen/metagen) + // GCS object is older, or generation/metageneration mismatch: Clobbered. + // Return nil for the object as it's not the version we might want to use. + return nil, true, nil + } } // Open a reader for the generation of object we care about. func (f *FileInode) openReader(ctx context.Context) (io.ReadCloser, error) { - rc, err := f.bucket.NewReader( + rc, err := f.bucket.NewReaderWithReadHandle( ctx, &gcs.ReadObjectRequest{ Name: f.src.Name, Generation: f.src.Generation, ReadCompressed: f.src.HasContentEncodingGzip(), }) + // If the object with requested generation doesn't exist in GCS, it indicates + // a file clobbering scenario. This likely occurred because the file was + // modified/deleted leading to different generation number. + var notFoundError *gcs.NotFoundError + if errors.As(err, ¬FoundError) { + err = &gcsfuse_errors.FileClobberedError{ + Err: fmt.Errorf("NewReader: %w", err), + ObjectName: f.src.Name, + } + } if err != nil { err = fmt.Errorf("NewReader: %w", err) } @@ -318,6 +407,17 @@ func (f *FileInode) IsUnlinked() bool { func (f *FileInode) Unlink() { f.unlinked = true + + if f.bwh != nil { + f.bwh.Unlink() + } +} + +// Returns true if the fileInode is using Buffered Write Handler. +// +// LOCKS_REQUIRED(f.mu) +func (f *FileInode) IsUsingBWH() bool { + return f.bwh != nil } // Source returns a record for the GCS object from which this inode is branched. The @@ -330,22 +430,42 @@ func (f *FileInode) Source() *gcs.MinObject { return &o } +// Returns MrdInstace for this inode. +func (f *FileInode) GetMRDInstance() *gcsx.MrdInstance { + return f.mrdInstance +} + // If true, it is safe to serve reads directly from the object given by // f.Source(), rather than calling f.ReadAt. Doing so may be more efficient, // because f.ReadAt may cause the entire object to be faulted in and requires -// the inode to be locked during the read. +// the inode to be locked during the read. SourceGenerationAuthoritative requires +// SyncPendingBufferedWrites method has been called on f within same inode lock for +// streaming writes with zonal bucket. +// TODO(b/406160290): Check if this can be improved. // // LOCKS_REQUIRED(f.mu) func (f *FileInode) SourceGenerationIsAuthoritative() bool { - return f.content == nil + // Source generation is authoritative if: + // 1. No pending writes exists on the inode (both content and bwh are nil). + // 2. The bucket is rapid and there are no pending writes in the temporary file. + return (f.content == nil && f.bwh == nil) || (f.bucket.BucketType().RapidWritesEnabled() && f.content == nil) } // Equivalent to the generation returned by f.Source(). // -// LOCKS_REQUIRED(f) +// LOCKS_REQUIRED(f.mu) func (f *FileInode) SourceGeneration() (g Generation) { + g.Size = f.src.Size g.Object = f.src.Generation g.Metadata = f.src.MetaGeneration + // If bwh is not nil, it's size takes precedence as that is being actively + // written to GCS. + // Since temporary file does not write to GCS on the go, it's size is not + // used as source object's size. + if f.bwh != nil { + writeFileInfo := f.bwh.WriteFileInfo() + g.Size = uint64(writeFileInfo.TotalSize) + } return } @@ -360,6 +480,46 @@ func (f *FileInode) DecrementLookupCount(n uint64) (destroy bool) { return } +// LOCKS_REQUIRED(f.mu) +func (f *FileInode) RegisterFileHandle(readOnly bool) { + if !readOnly { + f.writeHandleCount++ + } +} + +// LOCKS_REQUIRED(f.mu) +func (f *FileInode) DeRegisterFileHandle(readOnly bool) { + if readOnly { + return + } + + if f.writeHandleCount <= 0 { + logger.Errorf("Mismatch in number of write file handles for inode :%d", f.id) + } + + f.writeHandleCount-- + + // All write fileHandles associated with bwh are closed. So safe to set bwh to nil. + if f.writeHandleCount == 0 && f.bwh != nil { + err := f.bwh.Destroy() + if err != nil { + logger.Warnf("Error while destroying the bufferedWritesHandler: %v", err) + } + f.bwh = nil + } +} + +// LOCKS_REQUIRED(f.mu) +// UpdateSize updates the size of the backing GCS object. It also calls +// updateMRD to ensure that the multi-range downloader (which is used +// for random reads) is aware of the new size. This prevents the downloader +// from operating on stale object information. +func (f *FileInode) UpdateSize(size uint64) { + f.src.Size = size + f.attrs.Size = size + f.updateMRD() +} + // LOCKS_REQUIRED(f.mu) func (f *FileInode) Destroy() (err error) { f.destroyed = true @@ -369,17 +529,19 @@ func (f *FileInode) Destroy() (err error) { } else if f.content != nil { f.content.Destroy() } + if f.mrdInstance != nil { + f.mrdInstance.Destroy() + } return } // LOCKS_REQUIRED(f.mu) func (f *FileInode) Attributes( - ctx context.Context) (attrs fuseops.InodeAttributes, err error) { + ctx context.Context, clobberedCheck bool) (attrs fuseops.InodeAttributes, err error) { attrs = f.attrs - // Obtain default information from the source object. attrs.Mtime = f.src.Updated - attrs.Size = uint64(f.src.Size) + attrs.Size = f.src.Size // If the source object has an mtime metadata key, use that instead of its // update time. @@ -414,22 +576,48 @@ func (f *FileInode) Attributes( } } + if f.bwh != nil { + writeFileInfo := f.bwh.WriteFileInfo() + attrs.Mtime = writeFileInfo.Mtime + attrs.Size = uint64(writeFileInfo.TotalSize) + } + // We require only that atime and ctime be "reasonable". attrs.Atime = attrs.Mtime attrs.Ctime = attrs.Mtime - // If the object has been clobbered, we reflect that as the inode being - // unlinked. - _, clobbered, err := f.clobbered(ctx, false, false) - if err != nil { - err = fmt.Errorf("clobbered: %w", err) - return + if clobberedCheck { + // If the object has been clobbered, we reflect that as the inode being + // unlinked. + var clobbered bool + var o *gcs.Object + o, clobbered, err = f.clobbered(ctx, false, false) + if err != nil { + err = fmt.Errorf("clobbered: %w", err) + return + } + if clobbered { + // If clobbered check is true but the minObject returned is not nil, it means the clobber + // was due to update in object size remotely (appends case). In this scenario, we will update + // the inode attributes to reflect latest size. + if o != nil { + f.UpdateSize(o.Size) + attrs = f.attrs + attrs.Nlink = 1 + return + + } + // If the minObj is nil, it means that file has been clobbered genuinely due to generation + // or metageneration changes. + attrs.Nlink = 0 + return + } } attrs.Nlink = 1 // For local files, also checking if file is unlinked locally. - if clobbered || (f.IsLocal() && f.IsUnlinked()) { + if f.IsLocal() && f.IsUnlinked() { attrs.Nlink = 0 } @@ -450,8 +638,8 @@ func (f *FileInode) Read( ctx context.Context, dst []byte, offset int64) (n int, err error) { - if f.IsLocal() && f.writeConfig.ExperimentalEnableStreamingWrites { - err = fmt.Errorf("cannot read a local file when upload in progress") + if f.bwh != nil { + err = fmt.Errorf("unexpected read call for %q when streaming write is in progress for it", f.Name().LocalName()) return } @@ -476,17 +664,56 @@ func (f *FileInode) Read( return } +func getMetricOpenMode(openMode util.OpenMode) metrics.OpenMode { + isAppend := openMode.IsAppend() + + switch openMode.AccessMode() { + case util.ReadWrite: + if isAppend { + return metrics.OpenModeReadWriteAppendAttr + } + return metrics.OpenModeReadWriteAttr + case util.WriteOnly: + if isAppend { + return metrics.OpenModeWriteOnlyAppendAttr + } + return metrics.OpenModeWriteOnlyAttr + default: + return metrics.OpenModeOtherAttr + } +} + +func recordStreamingWriteFallbackMetric(mh metrics.MetricHandle, openMode util.OpenMode, reason metrics.WriteFallbackReason) { + if mh == nil { + return + } + metricOpenMode := getMetricOpenMode(openMode) + mh.FsStreamingWriteFallbackCount(1, metricOpenMode, reason) +} + // Serve a write for this file with semantics matching fuseops.WriteFileOp. +// It returns true if the file is successfully synced during the write operation. // // LOCKS_REQUIRED(f.mu) func (f *FileInode) Write( ctx context.Context, data []byte, - offset int64) (err error) { - if f.local && f.writeConfig.ExperimentalEnableStreamingWrites { - return f.writeToBuffer(data, offset) + offset int64, + openMode util.OpenMode) (bool, error) { + if f.bwh != nil { + return f.writeUsingBufferedWrites(ctx, data, offset, openMode) } + return false, f.writeUsingTempFile(ctx, data, offset) +} + +// Helper function to serve write for file using temp file. +// +// LOCKS_REQUIRED(f.mu) +func (f *FileInode) writeUsingTempFile(ctx context.Context, data []byte, offset int64) (err error) { + bytes := int64(len(data)) + ctx, finishSpan := f.traceHandle.TraceUpload(ctx, tracing.WriteFileStaged, f.src.Name, &bytes, &err) + defer finishSpan() // Make sure f.content != nil. err = f.ensureContent(ctx) if err != nil { @@ -501,12 +728,108 @@ func (f *FileInode) Write( return } +// Helper function to serve write for file using buffered writes handler. +// +// LOCKS_REQUIRED(f.mu) +func (f *FileInode) writeUsingBufferedWrites(ctx context.Context, data []byte, offset int64, openMode util.OpenMode) (_ bool, err error) { + bytes := int64(len(data)) + ctx, finishSpan := f.traceHandle.TraceUpload(ctx, tracing.WriteFileStreaming, f.src.Name, &bytes, &err) + defer finishSpan() + err = f.bwh.Write(ctx, data, offset) + var preconditionErr *gcs.PreconditionError + if errors.As(err, &preconditionErr) { + return false, &gcsfuse_errors.FileClobberedError{ + Err: fmt.Errorf("f.bwh.Write(): %w", err), + ObjectName: f.src.Name, + } + } + // Fall back to temp file for Out-Of-Order Writes. + if errors.Is(err, bufferedwrites.ErrOutOfOrderWrite) { + logger.Infof("Out of order write detected. File %s will now use legacy staged writes. "+StreamingWritesSemantics, f.name.String()) + // Finalize the object. + err = f.flushUsingBufferedWriteHandler(ctx) + if err != nil { + return false, fmt.Errorf("could not finalize what has been written so far: %w", err) + } + recordStreamingWriteFallbackMetric(f.metricHandle, openMode, metrics.WriteFallbackReasonOutOfOrderAttr) + return true, f.writeUsingTempFile(ctx, data, offset) + } + if err != nil { + return false, fmt.Errorf("write to buffered write handler failed: %w", err) + } + return false, nil +} + +// Helper function to flush buffered writes handler and update inode state with +// new object. +// +// LOCKS_REQUIRED(f.mu) +func (f *FileInode) flushUsingBufferedWriteHandler(ctx context.Context) error { + obj, err := f.bwh.Flush(ctx) + var preconditionErr *gcs.PreconditionError + if errors.As(err, &preconditionErr) { + return &gcsfuse_errors.FileClobberedError{ + Err: fmt.Errorf("f.bwh.Flush(): %w", err), + ObjectName: f.src.Name, + } + } + if err != nil { + return fmt.Errorf("f.bwh.Flush(): %w", err) + } + // If we finalized the object, we need to update our state. + f.updateInodeStateAfterFlush(obj) + return nil +} + +// SyncPendingBufferedWrites flushes any pending writes on the bwh to GCS. +// It is a no-op when bwh is nil. +// +// LOCKS_REQUIRED(f.mu) +func (f *FileInode) SyncPendingBufferedWrites(ctx context.Context) (gcsSynced bool, err error) { + if f.bwh == nil { + return + } + minObj, err := f.bwh.Sync(ctx) + var preconditionErr *gcs.PreconditionError + if errors.As(err, &preconditionErr) { + err = &gcsfuse_errors.FileClobberedError{ + Err: fmt.Errorf("f.bwh.Sync(ctx): %w", err), + ObjectName: f.src.Name, + } + return + } + if err != nil { + err = fmt.Errorf("f.bwh.Sync(ctx): %w", err) + return + } + // We return gcsSynced as true when we get minObject from Sync() for Zonal Buckets. + // For Non-Zonal Buckets minObj is always nil. + gcsSynced = minObj != nil + // If we flushed out object, we need to update our state. + f.updateInodeStateAfterSync(minObj) + return +} + // Set the mtime for this file. May involve a round trip to GCS. // // LOCKS_REQUIRED(f.mu) func (f *FileInode) SetMtime( ctx context.Context, mtime time.Time) (err error) { + if f.IsUnlinked() { + // No need to update mtime on GCS for unlinked file. + return + } + + // When bufferedWritesHandler instance is not nil, set time on bwh. + // It will not be nil in 2 cases when bufferedWrites are enabled: + // 1. local files + // 2. After first write on empty GCS files. + if f.bwh != nil { + f.bwh.SetMtime(mtime) + return + } + // If we have a local temp file, stat it. var sr gcsx.StatResult if f.content != nil { @@ -552,6 +875,7 @@ func (f *FileInode) SetMtime( minObj = *minObjPtr } f.src = minObj + f.updateMRD() return } @@ -575,21 +899,7 @@ func (f *FileInode) SetMtime( return } -// Sync writes out contents to GCS. If this fails due to the generation having been -// clobbered, treat it as a non-error (simulating the inode having been -// unlinked). -// -// After this method succeeds, SourceGeneration will return the new generation -// by which this inode should be known (which may be the same as before). If it -// fails, the generation will not change. -// -// LOCKS_REQUIRED(f.mu) -func (f *FileInode) Sync(ctx context.Context) (err error) { - // If we have not been dirtied, there is nothing to do. - if f.content == nil { - return - } - +func (f *FileInode) fetchLatestGcsObject(ctx context.Context) (*gcs.Object, error) { // When listObjects call is made, we fetch data with projection set as noAcl // which means acls and owner properties are not returned. So the f.src object // here will not have acl information even though there are acls present on @@ -599,68 +909,239 @@ func (f *FileInode) Sync(ctx context.Context) (err error) { // default sets the projection to full, which fetches all the object // properties. latestGcsObj, isClobbered, err := f.clobbered(ctx, true, true) + if err != nil { + return nil, err + } + if isClobbered { + var err error + if latestGcsObj != nil { + err = fmt.Errorf("file was clobbered due to increase in size at same generation(remote appends)") + } else { + err = fmt.Errorf("file was clobbered due to generation/metageneration mismatch") + } + return nil, &gcsfuse_errors.FileClobberedError{ + Err: err, + ObjectName: f.src.Name, + } + } + return latestGcsObj, nil +} - // Clobbered is treated as being unlinked. There's no reason to return an - // error in that case. We simply return without syncing the object. - if err != nil || isClobbered { +// Sync writes out contents to GCS. If this fails due to the generation +// having been clobbered, failure is propagated back to the calling +// function as an error. +// +// For buffered writes, this method only waits for any partial buffers to be +// uploaded to GCS. It does not guarantee that the entire contents of the file +// have been persisted. +// +// For non-buffered writes, this method writes the entire contents to GCS. +// If this method succeeds, SourceGeneration will return the new generation by +// which this inode should be known (which may be the same as before). If it +// fails, the generation will not change. +// +// LOCKS_REQUIRED(f.mu) +func (f *FileInode) Sync(ctx context.Context) (gcsSynced bool, err error) { + // If we have not been dirtied, there is nothing to do. + if f.content == nil && f.bwh == nil { return } + if f.bwh != nil { + gcsSynced, err = f.SyncPendingBufferedWrites(ctx) + if err != nil { + err = fmt.Errorf("could not sync what has been written so far: %w", err) + } + return + } + err = f.syncUsingContent(ctx) + if err != nil { + return false, err + } + return true, nil +} + +// get the temp content size when tracing is enabled to set the BYTES_UPLOADED attribute +func (f *FileInode) getTempContentSizeForSpan() int64 { + if !cfg.IsTracingEnabled(f.config) { + return -1 + } + + st, err := f.content.Stat() + if err != nil { + logger.Errorf("failed getting content size for staged sync file span: %v", err) + return -1 + } + + return st.Size +} + +// syncUsingContent syncs the inode content to GCS. It fetches the latest GCS +// object, syncs the content and updates the inode state. +// +// LOCKS_REQUIRED(f.mu) +func (f *FileInode) syncUsingContent(ctx context.Context) (err error) { + bytes := f.getTempContentSizeForSpan() + ctx, finishSpan := f.traceHandle.TraceUpload(ctx, tracing.SyncFileStaged, f.src.Name, &bytes, &err) + defer finishSpan() + var latestGcsObj *gcs.Object + if !f.local { + latestGcsObj, err = f.fetchLatestGcsObject(ctx) + if err != nil { + return err + } + } + // Write out the contents if they are dirty. // Object properties are also synced as part of content sync. Hence, passing // the latest object fetched from gcs which has all the properties populated. newObj, err := f.bucket.SyncObject(ctx, f.Name().GcsObjectName(), latestGcsObj, f.content) - // Special case: a precondition error means we were clobbered, which we treat - // as being unlinked. There's no reason to return an error in that case. var preconditionErr *gcs.PreconditionError if errors.As(err, &preconditionErr) { - err = nil - return + return &gcsfuse_errors.FileClobberedError{ + Err: fmt.Errorf("SyncObject: %w", err), + ObjectName: f.src.Name, + } } // Propagate other errors. if err != nil { - err = fmt.Errorf("SyncObject: %w", err) - return + return fmt.Errorf("SyncObject: %w", err) + } + + if newObj != nil && f.content != nil { + st, statErr := f.content.Stat() + if statErr != nil { + return fmt.Errorf("SyncObject: stat temp file for size validation: %w", statErr) + } + + if newObj.Size != uint64(st.Size) { + return fmt.Errorf("SyncObject: could not upload entire data, expected size %d, got %d", st.Size, newObj.Size) + } } + minObj := storageutil.ConvertObjToMinObject(newObj) // If we wrote out a new object, we need to update our state. - if newObj != nil && !f.localFileCache { - var minObj gcs.MinObject - minObjPtr := storageutil.ConvertObjToMinObject(newObj) - if minObjPtr != nil { - minObj = *minObjPtr + f.updateInodeStateAfterFlush(minObj) + return nil +} + +// Flush writes out contents to GCS. If this fails due to the generation +// having been clobbered, failure is propagated back to the calling +// function as an error. +// +// After this method succeeds, SourceGeneration will return the new generation +// by which this inode should be known (which may be the same as before). If it +// fails, the generation will not change. +// +// LOCKS_REQUIRED(f.mu) +func (f *FileInode) Flush(ctx context.Context) (err error) { + // If we have not been dirtied, there is nothing to do. + if f.content == nil && f.bwh == nil { + return + } + + // Flush using the appropriate method based on whether we're using a + // buffered write handler. + if f.bwh != nil { + return f.flushUsingBufferedWriteHandler(ctx) + } + return f.syncUsingContent(ctx) +} + +func (f *FileInode) updateInodeStateAfterFlush(minObj *gcs.MinObject) { + if minObj != nil && !f.localFileCache { + // Set BWH to nil as as object has been finalized. + if f.bwh != nil { + f.bwh = nil } - f.src = minObj + f.updateInodeStateAfterSync(minObj) + } +} + +func (f *FileInode) updateInodeStateAfterSync(minObj *gcs.MinObject) { + if minObj != nil && !f.localFileCache { + f.src = *minObj + // Update MRDWrapper + f.updateMRD() // Convert localFile to nonLocalFile after it is synced to GCS. if f.IsLocal() { f.local = false } - f.content.Destroy() - f.content = nil + if f.content != nil { + f.content.Destroy() + f.content = nil + } } +} - return +// Updates the min object stored in MRDWrapper & MRDInstance corresponding to the inode. +// Should be called when minObject associated with inode is updated. +func (f *FileInode) updateMRD() { + // updateMRD will be a noop for regional bucket. + if !f.bucket.BucketType().Zonal { + return + } + minObj := f.Source() + if err := f.mrdInstance.SetMinObject(minObj); err != nil { + logger.Errorf("FileInode::updateMRD Error in setting minObject for MrdInstance %v", err) + } + if err := f.MRDWrapper.SetMinObject(minObj); err != nil { + logger.Errorf("FileInode::updateMRD Error in setting minObject for MRDWrapper %v", err) + } } -// Truncate the file to the specified size. +// truncateUsingBufferedWriteHandler attempts to truncate the file using the buffered +// write handler. If the requested size is smaller than the file's current size, +// it finalizes the existing writes, falls back to using a temporary file for +// the truncation, and returns true to indicate the file has been synced. // // LOCKS_REQUIRED(f.mu) -func (f *FileInode) Truncate( - ctx context.Context, - size int64) (err error) { - // Make sure f.content != nil. - err = f.ensureContent(ctx) +func (f *FileInode) truncateUsingBufferedWriteHandler(ctx context.Context, size int64) (bool, error) { + err := f.bwh.Truncate(size) + // If truncate size is less than the total file size resulting in OutOfOrder write, finalize and fall back to temp file. + if errors.Is(err, bufferedwrites.ErrOutOfOrderWrite) { + logger.Infof("Out of order write detected. File %s will now use legacy staged writes. "+StreamingWritesSemantics, f.name.String()) + // Finalize the object. + err = f.flushUsingBufferedWriteHandler(ctx) + if err != nil { + return false, fmt.Errorf("could not finalize what has been written so far: %w", err) + } + recordStreamingWriteFallbackMetric(f.metricHandle, util.NewOpenMode(util.WriteOnly, 0), metrics.WriteFallbackReasonOutOfOrderAttr) + return true, f.truncateUsingTempFile(ctx, size) + } if err != nil { - err = fmt.Errorf("ensureContent: %w", err) - return + return false, fmt.Errorf("got unexpected error from bwh.Truncate(): %w", err) } + return false, nil +} - // Call through. - err = f.content.Truncate(size) +// truncateUsingTempFile truncates the file by first ensuring a temporary file +// is available and then truncating that temporary file to the specified size. +// +// LOCKS_REQUIRED(f.mu) +func (f *FileInode) truncateUsingTempFile(ctx context.Context, size int64) error { + // Make sure f.content != nil. + err := f.ensureContent(ctx) + if err != nil { + return fmt.Errorf("ensureContent: %w", err) + } + // Truncate temp file. + return f.content.Truncate(size) +} - return +// Truncate the file to the specified size. +// It returns true if the file has been successfully synced to GCS. +// +// LOCKS_REQUIRED(f.mu) +func (f *FileInode) Truncate( + ctx context.Context, + size int64) (bool, error) { + if f.IsUsingBWH() { + return f.truncateUsingBufferedWriteHandler(ctx, size) + } + return false, f.truncateUsingTempFile(ctx, size) } // Ensures cache content on read if content cache enabled @@ -672,7 +1153,17 @@ func (f *FileInode) CacheEnsureContent(ctx context.Context) (err error) { return } -func (f *FileInode) CreateEmptyTempFile() (err error) { +// CreateEmptyTempFile creates an empty file with no contents when +// streaming writes are not in enabled. +// +// LOCKS_REQUIRED(f.mu) +func (f *FileInode) CreateEmptyTempFile(ctx context.Context) (err error) { + // Skip creating empty temp file when streaming writes are enabled + // or temp file is already created. + if f.bwh != nil || f.content != nil { + return + } + // Creating a file with no contents. The contents will be updated with // writeFile operations. f.content, err = f.contentCache.NewTempFile(io.NopCloser(strings.NewReader(""))) @@ -681,17 +1172,83 @@ func (f *FileInode) CreateEmptyTempFile() (err error) { return } -// writeToBuffer writes the given content to the in-memory buffer. -func (f *FileInode) writeToBuffer(data []byte, offset int64) (err error) { - // Initialize bufferedWriteHandler if not done already. - if f.bwh == nil { - f.bwh, err = bufferedwrites.NewBWHandler(f.name.GcsObjectName(), f.bucket, f.writeConfig.BlockSizeMb, f.writeConfig.MaxBlocksPerFile, f.globalMaxBlocksSem) +// Initializes Buffered Write Handler if the file inode is eligible and returns +// initialized as true when the new instance of buffered writer handler is created. +func (f *FileInode) InitBufferedWriteHandlerIfEligible(ctx context.Context, openMode util.OpenMode) (bool, error) { + // bwh already initialized, do nothing. + if f.bwh != nil { + return false, nil + } + + tempFileInUse := f.content != nil + if !f.config.Write.EnableStreamingWrites || tempFileInUse { + // bwh should not be initialized under these conditions. + return false, nil + } + + var latestGcsObj *gcs.Object + var err error + if !f.local { + if f.bucket.BucketType().RapidWritesEnabled() && openMode.IsAppend() { + // In case of rapid appends, we will rely on kernel's latest view of the object + // instead of reaching out to the server for latest metadata. This is done to avoid + // forceful overwrites of local and latest object metadata with possibly stale server + // response. Since appends happen at the same generation, StatObject() call is redundant. + latestGcsObj = storageutil.ConvertMinObjectToObject(&f.src) + } else { + // For regional buckets or overwrites for rapid buckets, call StatObject() to fetch extended + // attributes missing from the cached MinObject, which is required by the CreateObject request + // to create the new object generation. + latestGcsObj, err = f.fetchLatestGcsObject(ctx) + } if err != nil { - return fmt.Errorf("failed to create bufferedWriteHandler: %w", err) + return false, err } } - err = f.bwh.Write(data, offset) + if !f.areBufferedWritesSupported(openMode, latestGcsObj) { + return false, nil + } - return + if f.bwh == nil { + f.bwh, err = bufferedwrites.NewBWHandler(&bufferedwrites.CreateBWHandlerRequest{ + Object: latestGcsObj, + ObjectName: f.name.GcsObjectName(), + Bucket: f.bucket, + BlockSize: f.config.Write.BlockSizeMb * util.MiB, + MaxBlocksPerFile: f.config.Write.MaxBlocksPerFile, + GlobalMaxBlocksSem: f.globalMaxWriteBlocksSem, + ChunkRetryDeadlineSecs: f.config.GcsRetries.ChunkRetryDeadlineSecs, + ChunkTransferTimeoutSecs: f.config.GcsRetries.ChunkTransferTimeoutSecs, + TraceHandle: f.traceHandle, + }) + if errors.Is(err, block.CantAllocateAnyBlockError) { + logger.Warnf("File %s will use legacy staged writes because concurrent streaming write "+ + "limit (set by --write-global-max-blocks) has been reached. To allow more concurrent files "+ + "to use streaming writes, consider increasing this limit if sufficient memory is available. "+ + "For more details on memory usage, see: https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#writes", f.name.String()) + recordStreamingWriteFallbackMetric(f.metricHandle, openMode, metrics.WriteFallbackReasonConcurrencyLimitBreachedAttr) + return false, nil + } + if err != nil { + recordStreamingWriteFallbackMetric(f.metricHandle, openMode, metrics.WriteFallbackReasonOtherAttr) + return false, fmt.Errorf("failed to create bufferedWriteHandler: %w", err) + } + f.bwh.SetMtime(f.mtimeClock.Now()) + return true, nil + } + return false, nil +} + +func (f *FileInode) areBufferedWritesSupported(openMode util.OpenMode, obj *gcs.Object) bool { + // For new files and existing files of size 0, buffered writes are always supported. + if f.local || obj.Size == 0 { + return true + } + if f.config.Write.EnableRapidAppends && openMode.IsAppend() && f.bucket.BucketType().RapidWritesEnabled() && obj.Finalized.IsZero() { + return true + } + logger.Infof("Existing file %s of size %d bytes (non-zero) will use legacy staged writes. "+StreamingWritesSemantics, f.name.String(), obj.Size) + recordStreamingWriteFallbackMetric(f.metricHandle, openMode, metrics.WriteFallbackReasonExistingFileAttr) + return false } diff --git a/internal/fs/inode/file_mock_bucket_test.go b/internal/fs/inode/file_mock_bucket_test.go new file mode 100644 index 0000000000..68e3ac0e7c --- /dev/null +++ b/internal/fs/inode/file_mock_bucket_test.go @@ -0,0 +1,422 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inode + +import ( + "context" + "errors" + "math" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/contentcache" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/gcsfuse_errors" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + storagemock "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/mock" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/jacobsa/fuse/fuseops" + "github.com/jacobsa/syncutil" + "github.com/jacobsa/timeutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "golang.org/x/sync/semaphore" +) + +type FileMockBucketTest struct { + suite.Suite + ctx context.Context + bucket *storagemock.TestifyMockBucket + clock timeutil.SimulatedClock + backingObj *gcs.MinObject + in *FileInode +} + +func TestFileMockBucketTestSuite(t *testing.T) { + suite.Run(t, new(FileMockBucketTest)) +} + +func (t *FileMockBucketTest) SetupTest() { + // Enabling invariant check for all tests. + syncutil.EnableInvariantChecking() + t.ctx = context.Background() + t.clock.SetTime(time.Date(2012, 8, 15, 22, 56, 0, 0, time.Local)) + t.bucket = new(storagemock.TestifyMockBucket) + t.bucket.On("BucketType").Return(gcs.BucketType{Hierarchical: false, Zonal: false}) + + // Create the inode. + t.createLockedInode(fileName, localFile) +} + +func (t *FileMockBucketTest) TearDownTest() { + t.in.Unlock() +} + +func (t *FileMockBucketTest) createLockedInode(fileName string, fileType string) { + if fileType != emptyGCSFile && fileType != localFile { + t.T().Errorf("fileType should be either local or empty") + } + + name := NewFileName( + NewRootName(""), + fileName, + ) + syncerBucket := gcsx.NewSyncerBucket( + /*appendThreshold=*/ 1, + chunkRetryDeadlineSecs, + chunkTransferTimeoutSecs, + ".gcsfuse_tmp/", + t.bucket) + + isLocal := false + if fileType == localFile { + t.backingObj = nil + isLocal = true + } + + if fileType == emptyGCSFile { + object, err := storageutil.CreateObject( + t.ctx, + t.bucket, + fileName, + []byte{}) + t.backingObj = storageutil.ConvertObjToMinObject(object) + + assert.Nil(t.T(), err) + } + + t.in = NewFileInode( + fileInodeID, + name, + t.backingObj, + fuseops.InodeAttributes{ + Uid: uid, + Gid: gid, + Mode: fileMode, + }, + &syncerBucket, + false, // localFileCache + contentcache.New("", &t.clock), + &t.clock, + isLocal, + &cfg.Config{}, + semaphore.NewWeighted(math.MaxInt64), + nil, + tracing.NewNoopTracer(), + metrics.NewNoopMetrics()) + + // Create empty file for local inode created above. + err := t.in.CreateEmptyTempFile(t.ctx) + assert.Nil(t.T(), err) + + t.in.Lock() +} + +// createGCSBackedFileInode is a helper function to create and lock a FileInode for attribute tests. +// It initializes the inode with the provided backing object of non-zero size. +func (t *FileMockBucketTest) createGCSBackedFileInode(backingObj *gcs.MinObject) *FileInode { + t.T().Helper() + syncerBucket := gcsx.NewSyncerBucket( + /*appendThreshold=*/ 1, + chunkRetryDeadlineSecs, + chunkTransferTimeoutSecs, + ".gcsfuse_tmp/", + t.bucket) + + f := NewFileInode( + fileInodeID, + NewFileName(NewRootName(""), fileName), + backingObj, + fuseops.InodeAttributes{ + Uid: uid, + Gid: gid, + Mode: fileMode, + }, + &syncerBucket, + false, // localFileCache + contentcache.New("", &t.clock), + &t.clock, + false, // localFile + &cfg.Config{}, + semaphore.NewWeighted(math.MaxInt64), + nil, + tracing.NewNoopTracer(), + metrics.NewNoopMetrics()) + f.Lock() + return f +} + +func (t *FileMockBucketTest) TestFlushLocalFileDoesNotForceFetchObjectFromGCS() { + assert.True(t.T(), t.in.IsLocal()) + // Expect only CreateObject call on bucket. + t.bucket.On("CreateObject", t.ctx, mock.AnythingOfType("*gcs.CreateObjectRequest")). + Return(&gcs.Object{Name: fileName}, nil) + + err := t.in.Flush(t.ctx) + + require.NoError(t.T(), err) + t.bucket.AssertExpectations(t.T()) +} + +func (t *FileMockBucketTest) TestFlushLocalFile_SizeMismatch_ReturnsError() { + assert.True(t.T(), t.in.IsLocal()) + // Write some data so that the local temp file size becomes 4 + _, err := t.in.Write(t.ctx, []byte("data"), 0, util.NewOpenMode(util.WriteOnly, 0)) + require.NoError(t.T(), err) + // Mock CreateObject to return an object with a mismatched size + t.bucket.On("CreateObject", t.ctx, mock.AnythingOfType("*gcs.CreateObjectRequest")). + Return(&gcs.Object{Name: fileName, Size: 2}, nil) + + err = t.in.Flush(t.ctx) + + require.Error(t.T(), err) + assert.Contains(t.T(), err.Error(), "could not upload entire data, expected size 4, got 2") +} + +func (t *FileMockBucketTest) TestFlushSyncedFileForceFetchObjectFromGCS() { + // Expect a CreateObject call because createLockedInode creates a synced file + // inode (backed by GCS object). + t.bucket.On("CreateObject", t.ctx, mock.AnythingOfType("*gcs.CreateObjectRequest")). + Return(&gcs.Object{Name: fileName}, nil) + t.createLockedInode(fileName, emptyGCSFile) + assert.False(t.T(), t.in.IsLocal()) + // Expect both StatObject and CreateObject call on bucket. + t.bucket.On("StatObject", t.ctx, mock.AnythingOfType("*gcs.StatObjectRequest")). + Return(&gcs.MinObject{Name: fileName}, &gcs.ExtendedObjectAttributes{}, nil) + t.bucket.On("CreateObject", t.ctx, mock.AnythingOfType("*gcs.CreateObjectRequest")). + Return(&gcs.Object{Name: fileName}, nil) + + err := t.in.Flush(t.ctx) + + require.NoError(t.T(), err) + t.bucket.AssertExpectations(t.T()) +} + +func (t *FileMockBucketTest) TestSync_RemoteAppendClobber() { + // Expect CreateObject from createLockedInode + t.bucket.On("CreateObject", t.ctx, mock.AnythingOfType("*gcs.CreateObjectRequest")). + Return(&gcs.Object{Name: fileName, Generation: 1, MetaGeneration: 1}, nil) + t.createLockedInode(fileName, emptyGCSFile) + assert.False(t.T(), t.in.IsLocal()) + // Dirty the file. + _, err := t.in.Write(t.ctx, []byte("data"), 0, util.NewOpenMode(util.WriteOnly, 0)) + require.NoError(t.T(), err) + // Mock StatObject to return same generation but larger size (remote append). + // Current size is 0 (emptyGCSFile). + // We simulate remote append by returning size 100. + t.bucket.On("StatObject", t.ctx, mock.AnythingOfType("*gcs.StatObjectRequest")). + Return(&gcs.MinObject{ + Name: fileName, + Generation: t.in.SourceGeneration().Object, + MetaGeneration: t.in.SourceGeneration().Metadata, + Size: 100, // Larger size + }, &gcs.ExtendedObjectAttributes{}, nil) + + _, err = t.in.Sync(t.ctx) + + require.Error(t.T(), err) + var clobberedErr *gcsfuse_errors.FileClobberedError + require.True(t.T(), errors.As(err, &clobberedErr)) + assert.Contains(t.T(), clobberedErr.Err.Error(), "file was clobbered due to increase in size at same generation(remote appends)") +} + +func (t *FileMockBucketTest) TestSync_GenerationMismatchClobber() { + // Expect CreateObject from createLockedInode + t.bucket.On("CreateObject", t.ctx, mock.AnythingOfType("*gcs.CreateObjectRequest")). + Return(&gcs.Object{Name: fileName, Generation: 1, MetaGeneration: 1}, nil) + t.createLockedInode(fileName, emptyGCSFile) + assert.False(t.T(), t.in.IsLocal()) + // Dirty the file. + _, err := t.in.Write(t.ctx, []byte("data"), 0, util.NewOpenMode(util.WriteOnly, 0)) + require.NoError(t.T(), err) + // Mock StatObject to return different generation. + t.bucket.On("StatObject", t.ctx, mock.AnythingOfType("*gcs.StatObjectRequest")). + Return(&gcs.MinObject{ + Name: fileName, + Generation: t.in.SourceGeneration().Object + 1, // Different generation + MetaGeneration: t.in.SourceGeneration().Metadata, + Size: 0, + }, &gcs.ExtendedObjectAttributes{}, nil) + + _, err = t.in.Sync(t.ctx) + + require.Error(t.T(), err) + var clobberedErr *gcsfuse_errors.FileClobberedError + require.True(t.T(), errors.As(err, &clobberedErr)) + assert.Contains(t.T(), clobberedErr.Err.Error(), "file was clobbered due to generation/metageneration mismatch") +} + +func (t *FileMockBucketTest) TestAttributes_SizeIncreasedSameGeneration() { + initialSize := uint64(len("taco")) + initialGeneration := int64(123) + initialMetaGeneration := int64(456) + // Setup the minObject. + backingObj := &gcs.MinObject{ + Name: fileName, + Size: initialSize, + Generation: initialGeneration, + MetaGeneration: initialMetaGeneration, + } + f := t.createGCSBackedFileInode(backingObj) + defer f.Unlock() + statReq := &gcs.StatObjectRequest{ + Name: fileName, + ForceFetchFromGcs: false, + ReturnExtendedObjectAttributes: false, + } + // 1. First call to Attributes: Mock StatObject to return the original object. + t.bucket.On("StatObject", t.ctx, statReq). + Return(backingObj, &gcs.ExtendedObjectAttributes{}, nil).Once() + attrs1, err1 := f.Attributes(t.ctx, true) + require.NoError(t.T(), err1) + // Check that attributes match the initial object. + assert.Equal(t.T(), initialSize, attrs1.Size) + // 2. Second call to Attributes: Mock StatObject to return an updated object. + newSize := initialSize + 10 + updatedMinObject := &gcs.MinObject{ + Name: fileName, + Generation: initialGeneration, + MetaGeneration: initialMetaGeneration, + Size: newSize, + } + t.bucket.On("StatObject", t.ctx, statReq). + Return(updatedMinObject, &gcs.ExtendedObjectAttributes{}, nil).Once() + + attrs2, err2 := f.Attributes(t.ctx, true) + + require.NoError(t.T(), err2) + // Check that attributes are updated. + assert.Equal(t.T(), newSize, attrs2.Size) + // Check that internal state is updated. + assert.Equal(t.T(), newSize, f.Source().Size) + assert.Equal(t.T(), newSize, f.attrs.Size) + // Assert that all mock expectations were met. + t.bucket.AssertExpectations(t.T()) +} + +func (t *FileMockBucketTest) TestAttributes_NoChangeInAttributes() { + initialSize := uint64(4) + initialGeneration := int64(123) + initialMetaGeneration := int64(456) + initialTime := t.clock.Now() + // Setup the minObject + backingObj := &gcs.MinObject{ + Name: fileName, + Size: initialSize, + Generation: initialGeneration, + MetaGeneration: initialMetaGeneration, + Updated: initialTime, + } + f := t.createGCSBackedFileInode(backingObj) + defer f.Unlock() + t.clock.AdvanceTime(time.Minute) + // This object has a newer timestamp but the same size and generation. + updatedMinObject := &gcs.MinObject{ + Name: fileName, + Generation: initialGeneration, + MetaGeneration: initialMetaGeneration, + Size: initialSize, // Size is not greater + Updated: t.clock.Now(), + } + statReq := &gcs.StatObjectRequest{Name: fileName, ForceFetchFromGcs: false, ReturnExtendedObjectAttributes: false} + t.bucket.On("StatObject", t.ctx, statReq).Return(updatedMinObject, &gcs.ExtendedObjectAttributes{}, nil).Once() + + attrs, err := f.Attributes(t.ctx, true) + + require.NoError(t.T(), err) + t.bucket.AssertExpectations(t.T()) + // Attributes should NOT be updated because the size hasn't increased. + assert.Equal(t.T(), initialSize, attrs.Size) + assert.Equal(t.T(), initialTime, attrs.Mtime) + assert.Equal(t.T(), initialSize, f.Source().Size) + assert.Equal(t.T(), initialTime, f.Source().Updated) +} + +func (t *FileMockBucketTest) TestInitBufferedWriteHandlerIfEligible_ZonalBucket_DoesNotFetchMetadataFromGCS_ForAppends() { + // Setup Mock Bucket for Zonal + t.bucket = new(storagemock.TestifyMockBucket) + t.bucket.On("BucketType").Return(gcs.BucketType{Zonal: true}) + // Setup expectations for inode creation (emptyGCSFile triggers CreateObject) + t.bucket.On("CreateObject", t.ctx, mock.AnythingOfType("*gcs.CreateObjectRequest")). + Return(&gcs.Object{Name: fileName, Size: 0, Generation: 1, MetaGeneration: 1}, nil) + // Create Inode (Non-local) + t.createLockedInode(fileName, emptyGCSFile) + t.in.content = nil // Force content to nil to allow BWH init + t.in.config.Write = *getWriteConfigWithEnabledRapidAppends() + + // Call Method + // We do NOT expect StatObject to be called. + initialized, err := t.in.InitBufferedWriteHandlerIfEligible(t.ctx, util.NewOpenMode(util.ReadWrite, util.O_APPEND)) + + // Assertions + require.NoError(t.T(), err) + assert.True(t.T(), initialized) + t.bucket.AssertExpectations(t.T()) + t.bucket.AssertNotCalled(t.T(), "StatObject", mock.Anything, mock.Anything) +} + +func (t *FileMockBucketTest) TestInitBufferedWriteHandlerIfEligible_ZonalBucket_FetchesLatestMetadataFromGCS_ForOverwrites() { + // Setup Mock Bucket for Zonal + t.bucket = new(storagemock.TestifyMockBucket) + t.bucket.On("BucketType").Return(gcs.BucketType{Zonal: true}) + // Setup expectations for inode creation + t.bucket.On("CreateObject", t.ctx, mock.AnythingOfType("*gcs.CreateObjectRequest")). + Return(&gcs.Object{Name: fileName, Size: 0, Generation: 1, MetaGeneration: 1}, nil) + // Create Inode (Non-local) + t.createLockedInode(fileName, emptyGCSFile) + t.in.content = nil // Force content to nil to allow BWH init + t.in.config.Write = *getWriteConfigWithEnabledRapidAppends() + // We expect to make a StatObject() call to GCS to fetch the latest minObject. + t.bucket.On("StatObject", t.ctx, mock.AnythingOfType("*gcs.StatObjectRequest")). + Return(&gcs.MinObject{Name: fileName, Size: 0, Generation: 1, MetaGeneration: 1}, &gcs.ExtendedObjectAttributes{}, nil) + + // Call Method + initialized, err := t.in.InitBufferedWriteHandlerIfEligible(t.ctx, util.NewOpenMode(util.WriteOnly, 0)) + + // Assertions + require.NoError(t.T(), err) + assert.True(t.T(), initialized) + t.bucket.AssertExpectations(t.T()) +} + +func (t *FileMockBucketTest) TestInitBufferedWriteHandlerIfEligible_RegionalBucket_FetchesLatestMetadataFromGCS_Always() { + // Setup Mock Bucket for Non-Zonal + t.bucket = new(storagemock.TestifyMockBucket) + t.bucket.On("BucketType").Return(gcs.BucketType{Zonal: false}) + // Setup expectations for inode creation + t.bucket.On("CreateObject", t.ctx, mock.AnythingOfType("*gcs.CreateObjectRequest")). + Return(&gcs.Object{Name: fileName, Size: 0, Generation: 1, MetaGeneration: 1}, nil) + // Create Inode (Non-local) + t.createLockedInode(fileName, emptyGCSFile) + t.in.content = nil // Force content to nil to allow BWH init + t.in.config.Write = *getWriteConfigWithEnabledRapidAppends() + // We expect to make a StatObject() call to GCS to fetch the latest minObject. + t.bucket.On("StatObject", t.ctx, mock.AnythingOfType("*gcs.StatObjectRequest")). + Return(&gcs.MinObject{Name: fileName, Size: 0, Generation: 1, MetaGeneration: 1}, &gcs.ExtendedObjectAttributes{}, nil) + + // Call Method + initialized, err := t.in.InitBufferedWriteHandlerIfEligible(t.ctx, util.NewOpenMode(util.WriteOnly, 0)) + + // Assertions + require.NoError(t.T(), err) + assert.True(t.T(), initialized) + t.bucket.AssertExpectations(t.T()) +} diff --git a/internal/fs/inode/file_streaming_writes_test.go b/internal/fs/inode/file_streaming_writes_test.go new file mode 100644 index 0000000000..824490b436 --- /dev/null +++ b/internal/fs/inode/file_streaming_writes_test.go @@ -0,0 +1,863 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inode + +import ( + "context" + "errors" + "math" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/bufferedwrites" + "github.com/googlecloudplatform/gcsfuse/v3/internal/contentcache" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/gcsfuse_errors" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/jacobsa/fuse/fuseops" + "github.com/jacobsa/syncutil" + "github.com/jacobsa/timeutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "golang.org/x/sync/semaphore" +) + +const localFile = "local" +const emptyGCSFile = "emptyGCS" + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +type FileStreamingWritesCommon struct { + suite.Suite + ctx context.Context + bucket gcs.Bucket + clock timeutil.SimulatedClock + backingObj *gcs.MinObject + in *FileInode +} +type FileStreamingWritesTest struct { + FileStreamingWritesCommon +} + +type FileStreamingWritesZonalBucketTest struct { + FileStreamingWritesCommon +} + +// ////////////////////////////////////////////////////////////////////// +// Helper +// ////////////////////////////////////////////////////////////////////// + +func (t *FileStreamingWritesCommon) createBufferedWriteHandler() { + // Initialize BWH for local inode created above. + initialized, err := t.in.InitBufferedWriteHandlerIfEligible(t.ctx, WriteMode) + require.NoError(t.T(), err) + assert.True(t.T(), initialized) + assert.NotNil(t.T(), t.in.bwh) +} + +func (t *FileStreamingWritesCommon) setupTest() { + // Enabling invariant check for all tests. + syncutil.EnableInvariantChecking() + t.ctx = context.Background() + // Create the inode. + t.createInode(localFile) +} + +func (t *FileStreamingWritesZonalBucketTest) SetupTest() { + t.clock.SetTime(time.Date(2012, 8, 15, 22, 56, 0, 0, time.Local)) + t.bucket = fake.NewFakeBucket(&t.clock, "some_bucket", gcs.BucketType{Zonal: true}) + t.setupTest() +} + +func (t *FileStreamingWritesTest) SetupTest() { + t.clock.SetTime(time.Date(2012, 8, 15, 22, 56, 0, 0, time.Local)) + t.bucket = fake.NewFakeBucket(&t.clock, "some_bucket", gcs.BucketType{Zonal: false}) + t.setupTest() +} + +func (t *FileStreamingWritesCommon) TearDownTest() { + t.in.Unlock() +} + +func (t *FileStreamingWritesTest) SetupSubTest() { + t.SetupTest() +} + +func (t *FileStreamingWritesCommon) createInode(fileType string) { + if fileType != emptyGCSFile && fileType != localFile { + t.T().Errorf("fileType should be either local or empty") + } + + name := NewFileName( + NewRootName(""), + fileName, + ) + syncerBucket := gcsx.NewSyncerBucket( + /*appendThreshold=*/ 1, + chunkRetryDeadlineSecs, + chunkTransferTimeoutSecs, + ".gcsfuse_tmp/", + t.bucket) + + isLocal := false + if fileType == localFile { + t.backingObj = nil + isLocal = true + } + + if fileType == emptyGCSFile { + object, err := storageutil.CreateObject( + t.ctx, + t.bucket, + fileName, + []byte{}) + t.backingObj = storageutil.ConvertObjToMinObject(object) + + assert.Nil(t.T(), err) + } + + t.in = NewFileInode( + fileInodeID, + name, + t.backingObj, + fuseops.InodeAttributes{ + Uid: uid, + Gid: gid, + Mode: fileMode, + }, + &syncerBucket, + false, // localFileCache + contentcache.New("", &t.clock), + &t.clock, + isLocal, + &cfg.Config{}, + semaphore.NewWeighted(math.MaxInt64), + nil, + tracing.NewNoopTracer(), + metrics.NewNoopMetrics()) + + // Set buffered write config for created inode. + t.in.config = &cfg.Config{Write: cfg.WriteConfig{ + MaxBlocksPerFile: 5, + BlockSizeMb: 1, + EnableStreamingWrites: true, + GlobalMaxBlocks: 10, + }} + + t.in.Lock() +} + +//////////////////////////////////////////////////////////////////////// +// Common Tests +//////////////////////////////////////////////////////////////////////// + +func (t *FileStreamingWritesCommon) TestIsUsingBWH() { + assert.False(t.T(), t.in.IsUsingBWH()) + t.createBufferedWriteHandler() + assert.True(t.T(), t.in.IsUsingBWH()) +} + +func (t *FileStreamingWritesCommon) TestflushUsingBufferedWriteHandlerOnZeroSizeRecreatesBwhOnInitAgain() { + t.createBufferedWriteHandler() + err := t.in.flushUsingBufferedWriteHandler(context.Background()) + require.NoError(t.T(), err) + assert.Nil(t.T(), t.in.bwh) + + initialized, err := t.in.InitBufferedWriteHandlerIfEligible(t.ctx, WriteMode) + + require.NoError(t.T(), err) + assert.True(t.T(), initialized) + assert.NotNil(t.T(), t.in.bwh) +} + +func (t *FileStreamingWritesCommon) TestflushUsingBufferedWriteHandlerOnNonZeroSizeDoesNotRecreatesBwhOnInitAgain() { + t.createBufferedWriteHandler() + gcsSynced, err := t.in.Write(t.ctx, []byte("foobar"), 0, WriteMode) + assert.NoError(t.T(), err) + assert.False(t.T(), gcsSynced) + err = t.in.flushUsingBufferedWriteHandler(context.Background()) + require.NoError(t.T(), err) + assert.Nil(t.T(), t.in.bwh) + + initialized, err := t.in.InitBufferedWriteHandlerIfEligible(t.ctx, WriteMode) + + require.NoError(t.T(), err) + assert.False(t.T(), initialized) + assert.Nil(t.T(), t.in.bwh) +} + +func (t *FileStreamingWritesCommon) TestTruncateNegative() { + // Truncate neagtive. + gcsSynced, err := t.in.Truncate(t.ctx, -1) + + require.Error(t.T(), err) + assert.False(t.T(), gcsSynced) +} + +//////////////////////////////////////////////////////////////////////// +// Tests (Zonal Bucket) +//////////////////////////////////////////////////////////////////////// + +func TestFileStreamingWritesWithZonalBucketTestSuite(t *testing.T) { + suite.Run(t, new(FileStreamingWritesZonalBucketTest)) +} + +func (t *FileStreamingWritesZonalBucketTest) TestSourceGenerationIsAuthoritativeReturnsTrueForZonalBuckets() { + assert.True(t.T(), t.in.SourceGenerationIsAuthoritative()) +} + +func (t *FileStreamingWritesZonalBucketTest) TestSourceGenerationIsAuthoritativeReturnsTrueAfterWriteForZonalBuckets() { + t.createBufferedWriteHandler() + gcsSynced, err := t.in.Write(t.ctx, []byte("taco"), 0, WriteMode) + assert.NoError(t.T(), err) + assert.False(t.T(), gcsSynced) + assert.True(t.T(), t.in.SourceGenerationIsAuthoritative()) +} + +func (t *FileStreamingWritesZonalBucketTest) TestSyncPendingBufferedWritesForZonalBucketsPromotesInodeToNonLocal() { + t.createBufferedWriteHandler() + gcsSynced, err := t.in.Write(t.ctx, []byte("pizza"), 0, WriteMode) + assert.NoError(t.T(), err) + require.False(t.T(), gcsSynced) + + gcsSynced, err = t.in.SyncPendingBufferedWrites(context.Background()) + + require.NoError(t.T(), err) + assert.True(t.T(), gcsSynced) + assert.False(t.T(), t.in.IsLocal()) + content, err := storageutil.ReadObject(t.ctx, t.bucket, t.in.Name().GcsObjectName()) + require.NoError(t.T(), err) + assert.Equal(t.T(), "pizza", string(content)) +} + +func (t *FileStreamingWritesZonalBucketTest) TestSyncPendingBufferedWritesForZonalBucketsUpdatesSrcSize() { + t.createBufferedWriteHandler() + gcsSynced, err := t.in.Write(t.ctx, []byte("foobar"), 0, WriteMode) + assert.NoError(t.T(), err) + assert.False(t.T(), gcsSynced) + assert.Equal(t.T(), uint64(0), t.in.src.Size) + + gcsSynced, err = t.in.SyncPendingBufferedWrites(context.Background()) + + require.NoError(t.T(), err) + assert.True(t.T(), gcsSynced) + assert.Equal(t.T(), uint64(6), t.in.src.Size) +} + +// ////////////////////////////////////////////////////////////////////// +// Tests (Non Zonal Bucket) +// ////////////////////////////////////////////////////////////////////// + +func TestFileStreamingWritesTestSuite(t *testing.T) { + suite.Run(t, new(FileStreamingWritesTest)) +} + +func (t *FileStreamingWritesTest) TestSourceGenerationIsAuthoritativeReturnsTrueForNonZonalBuckets() { + assert.True(t.T(), t.in.SourceGenerationIsAuthoritative()) +} + +func (t *FileStreamingWritesTest) TestSourceGenerationIsAuthoritativeReturnsFalseAfterWriteForNonZonalBuckets() { + t.createBufferedWriteHandler() + gcsSynced, err := t.in.Write(t.ctx, []byte("taco"), 0, WriteMode) + assert.NoError(t.T(), err) + assert.False(t.T(), gcsSynced) + + assert.False(t.T(), t.in.SourceGenerationIsAuthoritative()) +} + +func (t *FileStreamingWritesTest) TestSyncPendingBufferedWritesForNonZonalBucketsDoesNotPromoteInodeToNonLocal() { + t.createBufferedWriteHandler() + gcsSynced, err := t.in.Write(t.ctx, []byte("taco"), 0, WriteMode) + assert.NoError(t.T(), err) + assert.False(t.T(), gcsSynced) + + gcsSynced, err = t.in.SyncPendingBufferedWrites(context.Background()) + + require.NoError(t.T(), err) + assert.False(t.T(), gcsSynced) + assert.True(t.T(), t.in.IsLocal()) + operations.ValidateObjectNotFoundErr(t.ctx, t.T(), t.bucket, t.in.Name().GcsObjectName()) +} + +func (t *FileStreamingWritesTest) TestSyncPendingBufferedWritesForNonZonalBucketsDoesNotUpdateSrcSize() { + t.createBufferedWriteHandler() + gcsSynced, err := t.in.Write(t.ctx, []byte("foobar"), 0, WriteMode) + assert.NoError(t.T(), err) + assert.False(t.T(), gcsSynced) + assert.Equal(t.T(), uint64(0), t.in.src.Size) + + gcsSynced, err = t.in.SyncPendingBufferedWrites(context.Background()) + + require.NoError(t.T(), err) + assert.False(t.T(), gcsSynced) + assert.Equal(t.T(), uint64(0), t.in.src.Size) +} + +func (t *FileStreamingWritesTest) TestOutOfOrderWritesToLocalFileFallBackToTempFile() { + testCases := []struct { + name string + offset int64 + expectedContent string + }{ + { + name: "ahead_of_current_offset", + offset: 5, + expectedContent: "taco\x00hello", + }, + { + name: "zero_offset", + offset: 0, + expectedContent: "hello", + }, + { + name: "before_current_offset", + offset: 2, + expectedContent: "tahello", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.createBufferedWriteHandler() + assert.True(t.T(), t.in.IsLocal()) + createTime := t.clock.Now() + t.clock.AdvanceTime(15 * time.Minute) + // Sequential Write at offset 0 + gcsSynced, err := t.in.Write(t.ctx, []byte("taco"), 0, WriteMode) + require.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) + require.NotNil(t.T(), t.in.bwh) + // validate attributes. + attrs, err := t.in.Attributes(t.ctx, true) + require.Nil(t.T(), err) + assert.WithinDuration(t.T(), attrs.Mtime, createTime, 0) + assert.Equal(t.T(), uint64(4), attrs.Size) + + // Out of order write. + mtime := t.clock.Now() + gcsSynced, err = t.in.Write(t.ctx, []byte("hello"), tc.offset, WriteMode) + require.Nil(t.T(), err) + assert.True(t.T(), gcsSynced) + + // Ensure bwh cleared and temp file created. + assert.Nil(t.T(), t.in.bwh) + assert.NotNil(t.T(), t.in.content) + // The inode should agree about the new mtime and size. + attrs, err = t.in.Attributes(t.ctx, true) + require.Nil(t.T(), err) + assert.Equal(t.T(), uint64(len(tc.expectedContent)), attrs.Size) + assert.WithinDuration(t.T(), attrs.Mtime, mtime, 0) + // sync file and validate content + gcsSynced, err = t.in.Sync(t.ctx) + require.Nil(t.T(), err) + assert.True(t.T(), gcsSynced) + // Read the object's contents. + contents, err := storageutil.ReadObject(t.ctx, t.bucket, t.in.Name().GcsObjectName()) + assert.Nil(t.T(), err) + assert.Equal(t.T(), tc.expectedContent, string(contents)) + }) + } +} + +func (t *FileStreamingWritesTest) TestOutOfOrderWriteFollowedByOrderedWrite() { + t.createBufferedWriteHandler() + assert.True(t.T(), t.in.IsLocal()) + createTime := t.in.mtimeClock.Now() + // Out of order write. + gcsSynced, err := t.in.Write(t.ctx, []byte("taco"), 6, WriteMode) + require.Nil(t.T(), err) + assert.True(t.T(), gcsSynced) + // Ensure bwh cleared and temp file created. + assert.Nil(t.T(), t.in.bwh) + assert.NotNil(t.T(), t.in.content) + // validate attributes. + attrs, err := t.in.Attributes(t.ctx, true) + require.Nil(t.T(), err) + assert.WithinDuration(t.T(), attrs.Mtime, createTime, 0) + assert.Equal(t.T(), uint64(10), attrs.Size) + + // Ordered write. + mtime := t.clock.Now() + gcsSynced, err = t.in.Write(t.ctx, []byte("hello"), 0, WriteMode) + require.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) + + // Ensure bwh not re-created. + assert.Nil(t.T(), t.in.bwh) + // The inode should agree about the new mtime and size. + attrs, err = t.in.Attributes(t.ctx, true) + require.Nil(t.T(), err) + assert.Equal(t.T(), uint64(len("hello\x00taco")), attrs.Size) + assert.WithinDuration(t.T(), attrs.Mtime, mtime, 0) + // sync file and validate content + gcsSynced, err = t.in.Sync(t.ctx) + require.Nil(t.T(), err) + assert.True(t.T(), gcsSynced) + // Read the object's contents. + contents, err := storageutil.ReadObject(t.ctx, t.bucket, t.in.Name().GcsObjectName()) + require.NoError(t.T(), err) + assert.Equal(t.T(), "hello\x00taco", string(contents)) +} + +func (t *FileStreamingWritesTest) TestOutOfOrderWritesOnClobberedFileThrowsError() { + t.createBufferedWriteHandler() + gcsSynced, err := t.in.Write(t.ctx, []byte("hi"), 0, WriteMode) + require.Nil(t.T(), err) + require.NotNil(t.T(), t.in.bwh) + assert.False(t.T(), gcsSynced) + assert.Equal(t.T(), int64(2), t.in.bwh.WriteFileInfo().TotalSize) + // Clobber the file. + objWritten, err := storageutil.CreateObject(t.ctx, t.bucket, fileName, []byte("taco")) + require.Nil(t.T(), err) + + gcsSynced, err = t.in.Write(t.ctx, []byte("hello"), 10, WriteMode) + + require.Error(t.T(), err) + assert.False(t.T(), gcsSynced) + var fileClobberedError *gcsfuse_errors.FileClobberedError + assert.ErrorAs(t.T(), err, &fileClobberedError) + // Validate Object on GCS not updated. + statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} + objGot, _, err := t.bucket.StatObject(t.ctx, statReq) + require.NoError(t.T(), err) + assert.Equal(t.T(), storageutil.ConvertObjToMinObject(objWritten), objGot) +} + +func (t *FileStreamingWritesTest) TestUnlinkLocalFileBeforeWrite() { + assert.True(t.T(), t.in.IsLocal()) + + // Unlink. + t.in.Unlink() + + assert.True(t.T(), t.in.unlinked) + // Data shouldn't be updated to GCS. + operations.ValidateObjectNotFoundErr(t.ctx, t.T(), t.bucket, t.in.Name().GcsObjectName()) +} + +func (t *FileStreamingWritesTest) TestUnlinkLocalFileAfterWrite() { + assert.True(t.T(), t.in.IsLocal()) + t.createBufferedWriteHandler() + // Write some content. + gcsSynced, err := t.in.Write(t.ctx, []byte("tacos"), 0, WriteMode) + assert.Nil(t.T(), err) + assert.NotNil(t.T(), t.in.bwh) + assert.False(t.T(), gcsSynced) + + // Unlink. + t.in.Unlink() + + assert.True(t.T(), t.in.IsUnlinked()) + // Data shouldn't be updated to GCS. + operations.ValidateObjectNotFoundErr(t.ctx, t.T(), t.bucket, t.in.Name().GcsObjectName()) +} + +func (t *FileStreamingWritesTest) TestUnlinkEmptySyncedFile() { + t.createInode(emptyGCSFile) + assert.False(t.T(), t.in.IsLocal()) + t.createBufferedWriteHandler() + // Write some content to temp file. + gcsSynced, err := t.in.Write(t.ctx, []byte("tacos"), 0, WriteMode) + assert.Nil(t.T(), err) + assert.NotNil(t.T(), t.in.bwh) + assert.False(t.T(), gcsSynced) + + // Unlink. + t.in.Unlink() + + assert.True(t.T(), t.in.unlinked) +} + +func (t *FileStreamingWritesTest) TestWriteToFileAndFlush() { + testCases := []struct { + name string + isLocal bool + }{ + { + name: "local_file", + isLocal: true, + }, + { + name: "synced_empty_file", + isLocal: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + if tc.isLocal { + assert.True(t.T(), t.in.IsLocal()) + } else { + t.createInode(emptyGCSFile) + assert.False(t.T(), t.in.IsLocal()) + } + t.createBufferedWriteHandler() + // Write some content to temp file. + gcsSynced, err := t.in.Write(t.ctx, []byte("tacos"), 0, WriteMode) + assert.Nil(t.T(), err) + assert.NotNil(t.T(), t.in.bwh) + assert.False(t.T(), gcsSynced) + t.clock.AdvanceTime(10 * time.Second) + + err = t.in.Flush(t.ctx) + + require.Nil(t.T(), err) + // Ensure bwh cleared. + assert.Nil(t.T(), t.in.bwh) + // Verify that fileInode is no more local + assert.False(t.T(), t.in.IsLocal()) + // Check attributes. + attrs, err := t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), uint64(len("tacos")), attrs.Size) + assert.Equal(t.T(), t.clock.Now().UTC(), attrs.Mtime.UTC()) + // Validate Object on GCS. + statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} + m, _, err := t.bucket.StatObject(t.ctx, statReq) + require.NoError(t.T(), err) + assert.NotNil(t.T(), m) + assert.Equal(t.T(), t.in.SourceGeneration().Object, m.Generation) + assert.Equal(t.T(), t.in.SourceGeneration().Metadata, m.MetaGeneration) + assert.Equal(t.T(), uint64(len("tacos")), m.Size) + // Mtime metadata is not written for buffered writes. + assert.Equal(t.T(), "", m.Metadata["gcsfuse_mtime"]) + contents, err := storageutil.ReadObject(t.ctx, t.bucket, t.in.Name().GcsObjectName()) + require.NoError(t.T(), err) + assert.Equal(t.T(), "tacos", string(contents)) + }) + } +} + +func (t *FileStreamingWritesTest) TestFlushEmptyFile() { + testCases := []struct { + name string + isLocal bool + }{ + { + name: "local_file", + isLocal: true, + }, + { + name: "synced_empty_file", + isLocal: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + if tc.isLocal { + assert.True(t.T(), t.in.IsLocal()) + } else { + t.createInode(emptyGCSFile) + assert.False(t.T(), t.in.IsLocal()) + } + t.clock.AdvanceTime(10 * time.Second) + t.createBufferedWriteHandler() + + err := t.in.Flush(t.ctx) + + require.Nil(t.T(), err) + // Ensure bwh cleared. + assert.Nil(t.T(), t.in.bwh) + // Verify that fileInode is no more local + assert.False(t.T(), t.in.IsLocal()) + // Check attributes. + attrs, err := t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), uint64(0), attrs.Size) + // For synced file, mtime is updated by SetInodeAttributes call. + if tc.isLocal { + assert.Equal(t.T(), t.clock.Now().UTC(), attrs.Mtime.UTC()) + } + // Validate Object on GCS. + statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} + m, _, err := t.bucket.StatObject(t.ctx, statReq) + require.NoError(t.T(), err) + assert.NotNil(t.T(), m) + assert.Equal(t.T(), t.in.SourceGeneration().Object, m.Generation) + assert.Equal(t.T(), t.in.SourceGeneration().Metadata, m.MetaGeneration) + assert.Equal(t.T(), uint64(0), m.Size) + // Mtime metadata is not written for buffered writes. + assert.Equal(t.T(), "", m.Metadata["gcsfuse_mtime"]) + contents, err := storageutil.ReadObject(t.ctx, t.bucket, t.in.Name().GcsObjectName()) + require.NoError(t.T(), err) + assert.Equal(t.T(), "", string(contents)) + }) + } +} + +func (t *FileStreamingWritesTest) TestFlushClobberedFile() { + testCases := []struct { + name string + isLocal bool + }{ + { + name: "local_file", + isLocal: true, + }, + { + name: "synced_empty_file", + isLocal: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + if tc.isLocal { + assert.True(t.T(), t.in.IsLocal()) + } else { + t.createInode(emptyGCSFile) + assert.False(t.T(), t.in.IsLocal()) + } + t.createBufferedWriteHandler() + t.clock.AdvanceTime(10 * time.Second) + // Clobber the file. + objWritten, err := storageutil.CreateObject(t.ctx, t.bucket, fileName, []byte("taco")) + require.Nil(t.T(), err) + + err = t.in.Flush(t.ctx) + + require.Error(t.T(), err) + var fileClobberedError *gcsfuse_errors.FileClobberedError + assert.ErrorAs(t.T(), err, &fileClobberedError) + // Validate Object on GCS not updated. + statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} + objGot, _, err := t.bucket.StatObject(t.ctx, statReq) + assert.Nil(t.T(), err) + assert.Equal(t.T(), storageutil.ConvertObjToMinObject(objWritten), objGot) + }) + } +} + +func (t *FileStreamingWritesTest) TestWriteToFileAndSync() { + testCases := []struct { + name string + isLocal bool + }{ + { + name: "local_file", + isLocal: true, + }, + { + name: "synced_empty_file", + isLocal: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + if tc.isLocal { + assert.True(t.T(), t.in.IsLocal()) + } else { + t.createInode(emptyGCSFile) + assert.False(t.T(), t.in.IsLocal()) + } + t.createBufferedWriteHandler() + // Write some content to temp file. + gcsSynced, err := t.in.Write(t.ctx, []byte("tacos"), 0, WriteMode) + assert.Nil(t.T(), err) + assert.NotNil(t.T(), t.in.bwh) + assert.False(t.T(), gcsSynced) + t.clock.AdvanceTime(10 * time.Second) + + gcsSynced, err = t.in.Sync(t.ctx) + + require.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) + // Ensure bwh not cleared. + assert.NotNil(t.T(), t.in.bwh) + // Validate Object not written on GCS. + statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} + m, _, err := t.bucket.StatObject(t.ctx, statReq) + if tc.isLocal { + require.Error(t.T(), err) + var notFoundErr *gcs.NotFoundError + assert.ErrorAs(t.T(), err, ¬FoundErr) + } else { + require.NoError(t.T(), err) + assert.NotNil(t.T(), m) + assert.Equal(t.T(), uint64(0), m.Size) + } + }) + } +} + +func (t *FileStreamingWritesTest) TestSourceGenerationSizeForLocalFileIsReflected() { + t.createBufferedWriteHandler() + assert.True(t.T(), t.in.IsLocal()) + gcsSynced, err := t.in.Write(context.Background(), []byte(setup.GenerateRandomString(5)), 0, WriteMode) + require.NoError(t.T(), err) + assert.False(t.T(), gcsSynced) + + sg := t.in.SourceGeneration() + assert.Nil(t.T(), t.backingObj) + assert.EqualValues(t.T(), 0, sg.Object) + assert.EqualValues(t.T(), 0, sg.Metadata) + assert.EqualValues(t.T(), 5, sg.Size) +} + +func (t *FileStreamingWritesTest) TestSourceGenerationSizeForSyncedFileIsReflected() { + t.createInode(emptyGCSFile) + assert.False(t.T(), t.in.IsLocal()) + t.createBufferedWriteHandler() + gcsSynced, err := t.in.Write(context.Background(), []byte(setup.GenerateRandomString(5)), 0, WriteMode) + require.NoError(t.T(), err) + assert.False(t.T(), gcsSynced) + + sg := t.in.SourceGeneration() + assert.EqualValues(t.T(), t.backingObj.Generation, sg.Object) + assert.EqualValues(t.T(), t.backingObj.MetaGeneration, sg.Metadata) + assert.EqualValues(t.T(), 5, sg.Size) +} + +func (t *FileStreamingWritesTest) TestTruncateOnFileUsingTempFileDoesNotRecreatesBWH() { + t.createBufferedWriteHandler() + assert.True(t.T(), t.in.IsLocal()) + // Out of order write. + gcsSynced, err := t.in.Write(t.ctx, []byte("taco"), 2, WriteMode) + require.Nil(t.T(), err) + assert.True(t.T(), gcsSynced) + // Ensure bwh cleared and temp file created. + assert.Nil(t.T(), t.in.bwh) + assert.NotNil(t.T(), t.in.content) + + gcsSynced, err = t.in.Truncate(t.ctx, 10) + require.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) + + // Ensure bwh not re-created. + assert.Nil(t.T(), t.in.bwh) + // The inode should agree about the new size. + attrs, err := t.in.Attributes(t.ctx, true) + require.Nil(t.T(), err) + assert.Equal(t.T(), uint64(10), attrs.Size) + // sync file and validate content + gcsSynced, err = t.in.Sync(t.ctx) + require.Nil(t.T(), err) + assert.True(t.T(), gcsSynced) + // Read the object's contents. + contents, err := storageutil.ReadObject(t.ctx, t.bucket, t.in.Name().GcsObjectName()) + require.NoError(t.T(), err) + assert.Equal(t.T(), "\x00\x00taco\x00\x00\x00\x00", string(contents)) +} + +func (t *FileStreamingWritesTest) TestDeRegisterFileHandle() { + tbl := []struct { + name string + readonly bool + currentVal int32 + expectedVal int32 + isBwhNil bool + }{ + { + name: "ReadOnlyHandle", + readonly: true, + currentVal: 10, + expectedVal: 10, + isBwhNil: false, + }, + { + name: "NonZeroCurrentValueForWriteHandle", + readonly: false, + currentVal: 10, + expectedVal: 9, + isBwhNil: false, + }, + { + name: "LastWriteHandleToDeregister", + readonly: false, + currentVal: 1, + expectedVal: 0, + isBwhNil: true, + }, + } + for _, tc := range tbl { + t.Run(tc.name, func() { + t.in.config = &cfg.Config{Write: *getWriteConfig()} + t.in.writeHandleCount = tc.currentVal + t.createBufferedWriteHandler() + + t.in.DeRegisterFileHandle(tc.readonly) + + assert.Equal(t.T(), tc.expectedVal, t.in.writeHandleCount) + if tc.isBwhNil { + assert.Nil(t.T(), t.in.bwh) + } else { + assert.NotNil(t.T(), t.in.bwh) + } + }) + } +} + +// FakeBufferedWriteHandler is a test double for BufferedWriteHandler. +type FakeBufferedWriteHandler struct { + WriteFunc func(data []byte, offset int64) error + FlushFunc func() (*gcs.MinObject, error) +} + +func (t *FakeBufferedWriteHandler) Write(ctx context.Context, data []byte, offset int64) error { + if t.WriteFunc != nil { + return t.WriteFunc(data, offset) + } + return nil +} + +func (t *FakeBufferedWriteHandler) Flush(ctx context.Context) (*gcs.MinObject, error) { + if t.FlushFunc != nil { + return t.FlushFunc() + } + return nil, nil +} + +func (t *FakeBufferedWriteHandler) WriteFileInfo() bufferedwrites.WriteFileInfo { + return bufferedwrites.WriteFileInfo{ + TotalSize: 0, + Mtime: time.Time{}, + } +} + +func (t *FakeBufferedWriteHandler) Sync(ctx context.Context) (*gcs.MinObject, error) { return nil, nil } +func (t *FakeBufferedWriteHandler) SetMtime(_ time.Time) {} +func (t *FakeBufferedWriteHandler) Truncate(_ int64) error { return nil } +func (t *FakeBufferedWriteHandler) Destroy() error { return nil } +func (t *FakeBufferedWriteHandler) Unlink() {} + +func (t *FakeBufferedWriteHandler) SetTotalSize() {} + +func (t *FileStreamingWritesTest) TestWriteUsingBufferedWritesFails() { + t.createBufferedWriteHandler() + assert.True(t.T(), t.in.IsLocal()) + writeErr := errors.New("write error") + t.in.bwh = &FakeBufferedWriteHandler{ + WriteFunc: func(data []byte, offset int64) error { + return writeErr + }, + } + + gcsSynced, err := t.in.Write(context.Background(), []byte("hello"), 0, WriteMode) + + require.Error(t.T(), err) + assert.False(t.T(), gcsSynced) + assert.Regexp(t.T(), writeErr.Error(), err.Error()) +} diff --git a/internal/fs/inode/file_test.go b/internal/fs/inode/file_test.go index 31fa292e42..10ad23da96 100644 --- a/internal/fs/inode/file_test.go +++ b/internal/fs/inode/file_test.go @@ -15,6 +15,7 @@ package inode import ( + "errors" "fmt" "io" "math" @@ -24,23 +25,27 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/fake" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/contentcache" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/gcsfuse_errors" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/jacobsa/fuse/fuseops" "github.com/jacobsa/syncutil" + "github.com/jacobsa/timeutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" "golang.org/x/net/context" "golang.org/x/sync/semaphore" - - "github.com/googlecloudplatform/gcsfuse/v2/internal/contentcache" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" - "github.com/jacobsa/fuse/fuseops" - . "github.com/jacobsa/ogletest" - "github.com/jacobsa/timeutil" ) -func TestFile(t *testing.T) { RunTests(t) } - //////////////////////////////////////////////////////////////////////// // Boilerplate //////////////////////////////////////////////////////////////////////// @@ -52,8 +57,14 @@ const fileInodeID = 17 const fileName = "foo/bar" const fileMode os.FileMode = 0641 const Delta = 30 * time.Minute +const LocalFile = "Local" +const EmptyGCSFile = "EmptyGCS" + +var AppendMode = util.NewOpenMode(util.WriteOnly, util.O_APPEND) +var WriteMode = util.NewOpenMode(util.WriteOnly, 0) type FileTest struct { + suite.Suite ctx context.Context bucket gcs.Bucket clock timeutil.SimulatedClock @@ -62,19 +73,32 @@ type FileTest struct { backingObj *gcs.MinObject in *FileInode + + bucketType gcs.BucketType } -var _ SetUpInterface = &FileTest{} -var _ TearDownInterface = &FileTest{} +func TestFileTestSuite(t *testing.T) { + t.Run("NonZonal", func(t *testing.T) { + suite.Run(t, &FileTest{bucketType: gcs.BucketType{Zonal: false}}) + }) + t.Run("Zonal", func(t *testing.T) { + suite.Run(t, &FileTest{bucketType: gcs.BucketType{Zonal: true}}) + }) + t.Run("Pirlo", func(t *testing.T) { + suite.Run(t, &FileTest{bucketType: gcs.BucketType{Pirlo: gcs.PirloStateRapidWritesEnabled}}) + }) +} -func init() { RegisterTestSuite(&FileTest{}) } +func (t *FileTest) SetupSubTest() { + t.SetupTest() +} -func (t *FileTest) SetUp(ti *TestInfo) { +func (t *FileTest) SetupTest() { // Enabling invariant check for all tests. syncutil.EnableInvariantChecking() - t.ctx = ti.Ctx + t.ctx = context.Background() t.clock.SetTime(time.Date(2012, 8, 15, 22, 56, 0, 0, time.Local)) - t.bucket = fake.NewFakeBucket(&t.clock, "some_bucket", gcs.NonHierarchical) + t.bucket = fake.NewFakeBucket(&t.clock, "some_bucket", t.bucketType) // Set up the backing object. var err error @@ -87,30 +111,42 @@ func (t *FileTest) SetUp(ti *TestInfo) { []byte(t.initialContents)) t.backingObj = storageutil.ConvertObjToMinObject(object) - AssertEq(nil, err) + assert.Nil(t.T(), err) // Create the inode. t.createInode() } -func (t *FileTest) TearDown() { +func (t *FileTest) TearDownTest() { t.in.Unlock() } +func (t *FileTest) createInodeWithEmptyObject() { + object, err := storageutil.CreateObject( + t.ctx, + t.bucket, + fileName, + []byte{}) + t.backingObj = storageutil.ConvertObjToMinObject(object) + + assert.Nil(t.T(), err) + + // Create the inode. + t.createInode() +} + func (t *FileTest) createInode() { t.createInodeWithLocalParam(fileName, false) } func (t *FileTest) createInodeWithLocalParam(fileName string, local bool) { - if t.in != nil { - t.in.Unlock() - } - name := NewFileName( NewRootName(""), fileName, ) syncerBucket := gcsx.NewSyncerBucket( - 1, // Append threshold + /*appendThreshold=*/ 1, + chunkRetryDeadlineSecs, + chunkTransferTimeoutSecs, ".gcsfuse_tmp/", t.bucket) @@ -131,45 +167,248 @@ func (t *FileTest) createInodeWithLocalParam(fileName string, local bool) { contentcache.New("", &t.clock), &t.clock, local, - &cfg.WriteConfig{}, - semaphore.NewWeighted(math.MaxInt64)) + &cfg.Config{}, + semaphore.NewWeighted(math.MaxInt64), + nil, + tracing.NewNoopTracer(), + metrics.NewNoopMetrics()) t.in.Lock() } +func (t *FileTest) createBufferedWriteHandler(shouldInitialize bool, openMode util.OpenMode) { + // Initialize BWH for local inode created above. + initialized, err := t.in.InitBufferedWriteHandlerIfEligible(t.ctx, openMode) + require.NoError(t.T(), err) + assert.Equal(t.T(), shouldInitialize, initialized) + if shouldInitialize { + assert.NotNil(t.T(), t.in.bwh) + } +} + +func (t *FileTest) validateMrdInstanceMinObject() { + t.T().Helper() + // Validate only for zonal buckets + if t.in.bucket.BucketType().Zonal { + // Validate MinObject in inode and MRDInstance points to different copy of MinObject. + assert.NotSame(t.T(), &t.in.src, t.in.mrdInstance.GetMinObject()) + // Validate MinObject in MRDInstance is equal to the MinObject in inode. + assert.Equal(t.T(), &t.in.src, t.in.mrdInstance.GetMinObject()) + } +} + +func (t *FileTest) validateMrdWrapperMinObject() { + t.T().Helper() + // Validate only for zonal buckets + if t.in.bucket.BucketType().Zonal { + // Validate MinObject in inode and MRDWrapper points to different copy of MinObject. + assert.NotSame(t.T(), &t.in.src, t.in.MRDWrapper.GetMinObject()) + // Validate MinObject in MRDWrapper is equal to the MinObject in inode. + assert.Equal(t.T(), &t.in.src, t.in.MRDWrapper.GetMinObject()) + } +} + //////////////////////////////////////////////////////////////////////// // Tests //////////////////////////////////////////////////////////////////////// -func (t *FileTest) ID() { - ExpectEq(fileInodeID, t.in.ID()) +func (t *FileTest) TestID() { + assert.Equal(t.T(), fileInodeID, int(t.in.ID())) } -func (t *FileTest) Name() { - ExpectEq(fileName, t.in.Name().GcsObjectName()) +func (t *FileTest) TestName() { + assert.Equal(t.T(), fileName, t.in.Name().GcsObjectName()) } -func (t *FileTest) InitialSourceGeneration() { +func (t *FileTest) TestAreBufferedWritesSupported() { + finalizedTime := time.Date(2025, time.June, 18, 23, 30, 0, 0, time.UTC) + unFinalizedTime := time.Time{} + nonNilContents := "taco" + testCases := []struct { + name string + content string + openMode util.OpenMode + bucketType gcs.BucketType + finalized time.Time + supported bool + }{ + { + name: "AppendToFinalizedObjOnZB", + content: nonNilContents, + bucketType: gcs.BucketType{Zonal: true}, + finalized: finalizedTime, + openMode: AppendMode, + supported: false, + }, + { + name: "AppendToUnfinalizedObjOnZB", + content: nonNilContents, + bucketType: gcs.BucketType{Zonal: true}, + finalized: unFinalizedTime, + openMode: AppendMode, + supported: true, + }, + { + name: "AppendToObjOnNonZB", + content: nonNilContents, + bucketType: gcs.BucketType{}, + finalized: finalizedTime, + openMode: AppendMode, + supported: false, + }, + { + name: "WriteToObjOnNonZB", + content: nonNilContents, + bucketType: gcs.BucketType{}, + finalized: finalizedTime, + openMode: WriteMode, + supported: false, + }, + { + name: "AppendToUnfinalizedObjOnPirloWithRapidWritesEnabled", + content: nonNilContents, + bucketType: gcs.BucketType{Pirlo: gcs.PirloStateRapidWritesEnabled}, + finalized: unFinalizedTime, + openMode: AppendMode, + supported: true, + }, + { + name: "AppendToUnfinalizedObjOnPirloWithRapidWritesDisabled", + content: nonNilContents, + bucketType: gcs.BucketType{Pirlo: gcs.PirloStateRapidWritesDisabled}, + finalized: unFinalizedTime, + openMode: AppendMode, + supported: false, + }, + { + name: "WriteToEmptyObj", + content: "", + bucketType: gcs.BucketType{}, + finalized: finalizedTime, + openMode: WriteMode, + supported: true, + }, + } + for _, tc := range testCases { + t.bucket = fake.NewFakeBucket(&t.clock, "some_bucket", tc.bucketType) + // Set up the backing object. + var err error + t.initialContents = tc.content + object, err := storageutil.CreateObject( + t.ctx, + t.bucket, + fileName, + []byte(t.initialContents)) + assert.Nil(t.T(), err) + object.Finalized = tc.finalized + t.backingObj = storageutil.ConvertObjToMinObject(object) + t.createInode() + t.in.config.Write.EnableRapidAppends = true + + isSupported := t.in.areBufferedWritesSupported(tc.openMode, object) + + assert.Equal(t.T(), tc.supported, isSupported) + } +} + +func (t *FileTest) TestInitialSourceGeneration() { sg := t.in.SourceGeneration() - ExpectEq(t.backingObj.Generation, sg.Object) - ExpectEq(t.backingObj.MetaGeneration, sg.Metadata) + assert.Equal(t.T(), t.backingObj.Generation, sg.Object) + assert.Equal(t.T(), t.backingObj.MetaGeneration, sg.Metadata) + assert.Equal(t.T(), t.backingObj.Size, sg.Size) +} + +func (t *FileTest) TestSourceGenerationSizeAfterWriteDoesNotChange() { + gcsSynced, err := t.in.Write(context.Background(), []byte(setup.GenerateRandomString(5)), 0, WriteMode) + require.NoError(t.T(), err) + assert.False(t.T(), gcsSynced) + + sg := t.in.SourceGeneration() + + assert.Equal(t.T(), t.backingObj.Generation, sg.Object) + assert.Equal(t.T(), t.backingObj.MetaGeneration, sg.Metadata) + assert.Equal(t.T(), t.backingObj.Size, sg.Size) +} + +func (t *FileTest) TestSourceGenerationIsAuthoritativeReturnsTrue() { + assert.True(t.T(), t.in.SourceGenerationIsAuthoritative()) +} + +func (t *FileTest) TestSourceGenerationIsAuthoritativeReturnsFalseAfterWrite() { + gcsSynced, err := t.in.Write(t.ctx, []byte("taco"), 0, WriteMode) + assert.NoError(t.T(), err) + assert.False(t.T(), gcsSynced) + + assert.False(t.T(), t.in.SourceGenerationIsAuthoritative()) +} + +func (t *FileTest) TestSyncPendingBufferedWritesReturnsNilAndNoOpForNonStreamingWrites() { + contents, err := storageutil.ReadObject(t.ctx, t.bucket, t.in.Name().GcsObjectName()) + require.NoError(t.T(), err) + assert.Equal(t.T(), t.initialContents, string(contents)) + gcsSynced, err := t.in.Write(t.ctx, []byte("bar"), 0, WriteMode) + assert.NoError(t.T(), err) + assert.False(t.T(), gcsSynced) + + gcsSynced, err = t.in.SyncPendingBufferedWrites(context.Background()) + + require.NoError(t.T(), err) + assert.False(t.T(), gcsSynced) + contents, err = storageutil.ReadObject(t.ctx, t.bucket, t.in.Name().GcsObjectName()) + require.NoError(t.T(), err) + assert.Equal(t.T(), t.initialContents, string(contents)) +} + +func (t *FileTest) TestAttributes_Clobbered_WithClobberCheckTrue() { + // Simulate a clobbered file by creating a new object with the same name, + // which will have a new generation. + _, err := storageutil.CreateObject( + t.ctx, + t.bucket, + t.in.Name().GcsObjectName(), + []byte("new clobbering content")) + require.NoError(t.T(), err) + + attrs, err := t.in.Attributes(t.ctx, true) + + require.NoError(t.T(), err) + // Since clobberCheck is true and the generation has changed, + // Nlink should be 0. + assert.Equal(t.T(), uint32(0), attrs.Nlink) } -func (t *FileTest) InitialAttributes() { - attrs, err := t.in.Attributes(t.ctx) - AssertEq(nil, err) +func (t *FileTest) TestAttributes_Clobbered_WithClobberCheckFalse() { + // Simulate a clobbered file by creating a new object with the same name, + // which will have a new generation. + _, err := storageutil.CreateObject( + t.ctx, + t.bucket, + t.in.Name().GcsObjectName(), + []byte("new clobbering content")) + require.NoError(t.T(), err) + + attrs, err := t.in.Attributes(t.ctx, false) - ExpectEq(len(t.initialContents), attrs.Size) - ExpectEq(1, attrs.Nlink) - ExpectEq(uid, attrs.Uid) - ExpectEq(gid, attrs.Gid) - ExpectEq(fileMode, attrs.Mode) - ExpectThat(attrs.Atime, timeutil.TimeEq(t.backingObj.Updated)) - ExpectThat(attrs.Ctime, timeutil.TimeEq(t.backingObj.Updated)) - ExpectThat(attrs.Mtime, timeutil.TimeEq(t.backingObj.Updated)) + require.NoError(t.T(), err) + // Since clobberCheck is false, Nlink should be 1. + assert.Equal(t.T(), uint32(1), attrs.Nlink) } -func (t *FileTest) InitialAttributes_MtimeFromObjectMetadata_Gcsfuse() { +func (t *FileTest) TestInitialAttributes() { + attrs, err := t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + + assert.Equal(t.T(), uint64(len(t.initialContents)), attrs.Size) + assert.Equal(t.T(), uint32(1), attrs.Nlink) + assert.Equal(t.T(), uint32(uid), attrs.Uid) + assert.Equal(t.T(), uint32(gid), attrs.Gid) + assert.Equal(t.T(), fileMode, attrs.Mode) + assert.Equal(t.T(), attrs.Atime, t.backingObj.Updated) + assert.Equal(t.T(), attrs.Ctime, t.backingObj.Updated) + assert.Equal(t.T(), attrs.Mtime, t.backingObj.Updated) +} + +func (t *FileTest) TestInitialAttributes_MtimeFromObjectMetadata_Gcsfuse() { // Set up an explicit mtime on the backing object and re-create the inode. if t.backingObj.Metadata == nil { t.backingObj.Metadata = make(map[string]string) @@ -181,13 +420,13 @@ func (t *FileTest) InitialAttributes_MtimeFromObjectMetadata_Gcsfuse() { t.createInode() // Ask it for its attributes. - attrs, err := t.in.Attributes(t.ctx) - AssertEq(nil, err) + attrs, err := t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) - ExpectThat(attrs.Mtime, timeutil.TimeEq(mtime)) + assert.Equal(t.T(), attrs.Mtime, mtime) } -func (t *FileTest) InitialAttributes_MtimeFromObjectMetadata_Gsutil() { +func (t *FileTest) TestInitialAttributes_MtimeFromObjectMetadata_Gsutil() { // Set up an explicit mtime on the backing object and re-create the inode. if t.backingObj.Metadata == nil { t.backingObj.Metadata = make(map[string]string) @@ -199,13 +438,13 @@ func (t *FileTest) InitialAttributes_MtimeFromObjectMetadata_Gsutil() { t.createInode() // Ask it for its attributes. - attrs, err := t.in.Attributes(t.ctx) - AssertEq(nil, err) + attrs, err := t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) - ExpectThat(attrs.Mtime.UTC(), timeutil.TimeEq(mtime)) + assert.Equal(t.T(), attrs.Mtime.UTC(), mtime) } -func (t *FileTest) InitialAttributes_MtimeFromObjectMetadata_GcsfuseOutranksGsutil() { +func (t *FileTest) TestInitialAttributes_MtimeFromObjectMetadata_GcsfuseOutranksGsutil() { // Set up an explicit mtime on the backing object and re-create the inode. if t.backingObj.Metadata == nil { t.backingObj.Metadata = make(map[string]string) @@ -220,14 +459,14 @@ func (t *FileTest) InitialAttributes_MtimeFromObjectMetadata_GcsfuseOutranksGsut t.createInode() // Ask it for its attributes. - attrs, err := t.in.Attributes(t.ctx) - AssertEq(nil, err) + attrs, err := t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) - ExpectThat(attrs.Mtime, timeutil.TimeEq(canonicalMtime)) + assert.Equal(t.T(), attrs.Mtime, canonicalMtime) } -func (t *FileTest) Read() { - AssertEq("taco", t.initialContents) +func (t *FileTest) TestRead() { + assert.Equal(t.T(), "taco", t.initialContents) // Make several reads, checking the expected contents. testCases := []struct { @@ -270,26 +509,28 @@ func (t *FileTest) Read() { err = nil } - AssertEq(nil, err, "%s", desc) - ExpectEq(tc.expected, string(data), "%s", desc) + assert.Nil(t.T(), err, "%s", desc) + assert.Equal(t.T(), tc.expected, string(data), "%s", desc) } } -func (t *FileTest) Write() { +func (t *FileTest) TestWrite() { var err error - AssertEq("taco", t.initialContents) + assert.Equal(t.T(), "taco", t.initialContents) // Overwite a byte. - err = t.in.Write(t.ctx, []byte("p"), 0) - AssertEq(nil, err) + gcsSynced, err := t.in.Write(t.ctx, []byte("p"), 0, WriteMode) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) // Add some data at the end. t.clock.AdvanceTime(time.Second) writeTime := t.clock.Now() - err = t.in.Write(t.ctx, []byte("burrito"), 4) - AssertEq(nil, err) + gcsSynced, err = t.in.Write(t.ctx, []byte("burrito"), 4, WriteMode) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) t.clock.AdvanceTime(time.Second) @@ -301,29 +542,30 @@ func (t *FileTest) Write() { err = nil } - AssertEq(nil, err) - ExpectEq("pacoburrito", string(buf[:n])) + assert.Nil(t.T(), err) + assert.Equal(t.T(), "pacoburrito", string(buf[:n])) // Check attributes. - attrs, err := t.in.Attributes(t.ctx) - AssertEq(nil, err) + attrs, err := t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) - ExpectEq(len("pacoburrito"), attrs.Size) - ExpectThat(attrs.Mtime, timeutil.TimeEq(writeTime)) + assert.Equal(t.T(), uint64(len("pacoburrito")), attrs.Size) + assert.Equal(t.T(), attrs.Mtime, writeTime) } -func (t *FileTest) Truncate() { +func (t *FileTest) TestTruncate() { var attrs fuseops.InodeAttributes var err error - AssertEq("taco", t.initialContents) + assert.Equal(t.T(), "taco", t.initialContents) // Truncate downward. t.clock.AdvanceTime(time.Second) truncateTime := t.clock.Now() - err = t.in.Truncate(t.ctx, 2) - AssertEq(nil, err) + gcsSynced, err := t.in.Truncate(t.ctx, 2) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) t.clock.AdvanceTime(time.Second) @@ -335,279 +577,508 @@ func (t *FileTest) Truncate() { err = nil } - AssertEq(nil, err) - ExpectEq("ta", string(buf[:n])) + assert.Nil(t.T(), err) + assert.Equal(t.T(), "ta", string(buf[:n])) // Check attributes. - attrs, err = t.in.Attributes(t.ctx) - AssertEq(nil, err) + attrs, err = t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) - ExpectEq(len("ta"), attrs.Size) - ExpectThat(attrs.Mtime, timeutil.TimeEq(truncateTime)) + assert.Equal(t.T(), uint64(len("ta")), attrs.Size) + assert.Equal(t.T(), attrs.Mtime, truncateTime) } -func (t *FileTest) WriteThenSync() { - var attrs fuseops.InodeAttributes - var err error - - AssertEq("taco", t.initialContents) - - // Overwite a byte. - t.clock.AdvanceTime(time.Second) - writeTime := t.clock.Now() - - err = t.in.Write(t.ctx, []byte("p"), 0) - AssertEq(nil, err) - - t.clock.AdvanceTime(time.Second) - - // Sync. - err = t.in.Sync(t.ctx) - AssertEq(nil, err) - - // The generation should have advanced. - ExpectLt(t.backingObj.Generation, t.in.SourceGeneration().Object) +func (t *FileTest) TestTruncateNegative() { + assert.Equal(t.T(), "taco", t.initialContents) - // Stat the current object in the bucket. - statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} - m, _, err := t.bucket.StatObject(t.ctx, statReq) - - AssertEq(nil, err) - AssertNe(nil, m) - ExpectEq(t.in.SourceGeneration().Object, m.Generation) - ExpectEq(t.in.SourceGeneration().Metadata, m.MetaGeneration) - ExpectEq(len("paco"), m.Size) - ExpectEq( - writeTime.UTC().Format(time.RFC3339Nano), - m.Metadata["gcsfuse_mtime"]) - - // Read the object's contents. - contents, err := storageutil.ReadObject(t.ctx, t.bucket, t.in.Name().GcsObjectName()) + // Truncate neagtive. + gcsSynced, err := t.in.Truncate(t.ctx, -1) - AssertEq(nil, err) - ExpectEq("paco", string(contents)) - - // Check attributes. - attrs, err = t.in.Attributes(t.ctx) - AssertEq(nil, err) - - ExpectEq(len("paco"), attrs.Size) - ExpectThat(attrs.Mtime, timeutil.TimeEq(writeTime.UTC())) + require.Error(t.T(), err) + assert.False(t.T(), gcsSynced) } -func (t *FileTest) WriteToLocalFileThenSync() { - var attrs fuseops.InodeAttributes - var err error - // Create a local file inode. - t.createInodeWithLocalParam("test", true) - // Create a temp file for the local inode created above. - err = t.in.CreateEmptyTempFile() - AssertEq(nil, err) - // Write some content to temp file. - t.clock.AdvanceTime(time.Second) - writeTime := t.clock.Now() - err = t.in.Write(t.ctx, []byte("tacos"), 0) - AssertEq(nil, err) - t.clock.AdvanceTime(time.Second) - - // Sync. - err = t.in.Sync(t.ctx) - - AssertEq(nil, err) - // Verify that fileInode is no more local - AssertFalse(t.in.IsLocal()) - // Stat the current object in the bucket. - statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} - m, _, err := t.bucket.StatObject(t.ctx, statReq) - AssertEq(nil, err) - AssertNe(nil, m) - ExpectEq(t.in.SourceGeneration().Object, m.Generation) - ExpectEq(t.in.SourceGeneration().Metadata, m.MetaGeneration) - ExpectEq(len("tacos"), m.Size) - ExpectEq( - writeTime.UTC().Format(time.RFC3339Nano), - m.Metadata["gcsfuse_mtime"]) - // Read the object's contents. - contents, err := storageutil.ReadObject(t.ctx, t.bucket, t.in.Name().GcsObjectName()) - AssertEq(nil, err) - ExpectEq("tacos", string(contents)) - // Check attributes. - attrs, err = t.in.Attributes(t.ctx) - AssertEq(nil, err) - ExpectEq(len("tacos"), attrs.Size) - ExpectThat(attrs.Mtime, timeutil.TimeEq(writeTime.UTC())) +func (t *FileTest) TestDestroy_MrdInstanceDestroyed() { + if !t.in.bucket.BucketType().Zonal { + return + } + // Manually initialize MRD pool since FileInode.Read doesn't use it directly. + mi := t.in.GetMRDInstance() + require.NotNil(t.T(), mi) + // Perform a read on MrdInstance to trigger pool creation. + buf := make([]byte, 1) + _, err := mi.Read(t.ctx, buf, 0, metrics.NewNoopMetrics()) + require.NoError(t.T(), err) + // Verify pool is initialized. + assert.Greater(t.T(), int(mi.Size()), 0) + + // Destroy the inode. + err = t.in.Destroy() + + require.NoError(t.T(), err) + // Verify MRD instance is destroyed (pool closed and set to nil). + assert.Equal(t.T(), uint64(0), mi.Size()) } -func (t *FileTest) SyncEmptyLocalFile() { - var attrs fuseops.InodeAttributes - var err error - // Create a local file inode. - t.createInodeWithLocalParam("test", true) - creationTime := t.clock.Now() - // Create a temp file for the local inode created above. - err = t.in.CreateEmptyTempFile() - AssertEq(nil, err) +func (t *FileTest) TestWriteThenSync() { + testcases := []struct { + name string + callSync bool + }{ + { + name: "sync", + callSync: true, + }, + { + name: "flush", + callSync: false, + }, + } - // Sync. - err = t.in.Sync(t.ctx) + for _, tc := range testcases { + t.Run(tc.name, func() { + var attrs fuseops.InodeAttributes + var err error - AssertEq(nil, err) - // Verify that fileInode is no more local - AssertFalse(t.in.IsLocal()) - // Stat the current object in the bucket. - statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} - m, _, err := t.bucket.StatObject(t.ctx, statReq) - AssertEq(nil, err) - AssertNe(nil, m) - ExpectEq(t.in.SourceGeneration().Object, m.Generation) - ExpectEq(t.in.SourceGeneration().Metadata, m.MetaGeneration) - ExpectEq(0, m.Size) - // Validate the mtime. - mtimeInBucket, ok := m.Metadata["gcsfuse_mtime"] - AssertTrue(ok) - mtime, _ := time.Parse(time.RFC3339Nano, mtimeInBucket) - ExpectThat(mtime, timeutil.TimeNear(creationTime, Delta)) - // Read the object's contents. - contents, err := storageutil.ReadObject(t.ctx, t.bucket, t.in.Name().GcsObjectName()) - AssertEq(nil, err) - ExpectEq("", string(contents)) - // Check attributes. - attrs, err = t.in.Attributes(t.ctx) - AssertEq(nil, err) - ExpectEq(0, attrs.Size) -} + assert.Equal(t.T(), "taco", t.initialContents) -func (t *FileTest) AppendThenSync() { - var attrs fuseops.InodeAttributes - var err error + // Overwrite a byte. + t.clock.AdvanceTime(time.Second) + writeTime := t.clock.Now() - AssertEq("taco", t.initialContents) + gcsSynced, err := t.in.Write(t.ctx, []byte("p"), 0, WriteMode) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) - // Append some data. - t.clock.AdvanceTime(time.Second) - writeTime := t.clock.Now() + t.clock.AdvanceTime(time.Second) - err = t.in.Write(t.ctx, []byte("burrito"), int64(len("taco"))) - AssertEq(nil, err) + if tc.callSync { + gcsSynced, err := t.in.Sync(t.ctx) + require.NoError(t.T(), err) + assert.True(t.T(), gcsSynced) + } else { + err = t.in.Flush(t.ctx) + assert.Nil(t.T(), err) + } - t.clock.AdvanceTime(time.Second) + // The generation should have advanced. + assert.Less(t.T(), t.backingObj.Generation, t.in.SourceGeneration().Object) - // Sync. - err = t.in.Sync(t.ctx) - AssertEq(nil, err) + t.validateMrdWrapperMinObject() + t.validateMrdInstanceMinObject() - // The generation should have advanced. - ExpectLt(t.backingObj.Generation, t.in.SourceGeneration().Object) + // Stat the current object in the bucket. + statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} + m, _, err := t.bucket.StatObject(t.ctx, statReq) - // Stat the current object in the bucket. - statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} - m, _, err := t.bucket.StatObject(t.ctx, statReq) + require.NoError(t.T(), err) + assert.NotNil(t.T(), m) + assert.Equal(t.T(), t.in.SourceGeneration().Object, m.Generation) + assert.Equal(t.T(), t.in.SourceGeneration().Metadata, m.MetaGeneration) + assert.Equal(t.T(), t.in.SourceGeneration().Size, m.Size) + assert.Equal(t.T(), uint64(len("paco")), m.Size) + assert.Equal(t.T(), + writeTime.UTC().Format(time.RFC3339Nano), + m.Metadata["gcsfuse_mtime"]) - AssertEq(nil, err) - AssertNe(nil, m) - ExpectEq(t.in.SourceGeneration().Object, m.Generation) - ExpectEq(t.in.SourceGeneration().Metadata, m.MetaGeneration) - ExpectEq(len("tacoburrito"), m.Size) - ExpectEq( - writeTime.UTC().Format(time.RFC3339Nano), - m.Metadata["gcsfuse_mtime"]) + // Read the object's contents. + contents, err := storageutil.ReadObject(t.ctx, t.bucket, t.in.Name().GcsObjectName()) - // Read the object's contents. - contents, err := storageutil.ReadObject(t.ctx, t.bucket, t.in.Name().GcsObjectName()) + require.NoError(t.T(), err) + assert.Equal(t.T(), "paco", string(contents)) - AssertEq(nil, err) - ExpectEq("tacoburrito", string(contents)) + // Check attributes. + attrs, err = t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) - // Check attributes. - attrs, err = t.in.Attributes(t.ctx) - AssertEq(nil, err) - - ExpectEq(len("tacoburrito"), attrs.Size) - ExpectThat(attrs.Mtime, timeutil.TimeEq(writeTime.UTC())) + assert.Equal(t.T(), uint64(len("paco")), attrs.Size) + assert.Equal(t.T(), attrs.Mtime, writeTime.UTC()) + }) + } } -func (t *FileTest) TruncateDownwardThenSync() { - var attrs fuseops.InodeAttributes - var err error - - // Truncate downward. - t.clock.AdvanceTime(time.Second) - truncateTime := t.clock.Now() - - err = t.in.Truncate(t.ctx, 2) - AssertEq(nil, err) - - t.clock.AdvanceTime(time.Second) - - // Sync. - err = t.in.Sync(t.ctx) - AssertEq(nil, err) +func (t *FileTest) TestWriteToLocalFileThenSync() { + testcases := []struct { + name string + callSync bool + }{ + { + name: "sync", + callSync: true, + }, + { + name: "flush", + callSync: false, + }, + } - // The generation should have advanced. - ExpectLt(t.backingObj.Generation, t.in.SourceGeneration().Object) + for _, tc := range testcases { + t.Run(tc.name, func() { + var attrs fuseops.InodeAttributes + var err error + // Create a local file inode. + t.createInodeWithLocalParam("test", true) + // Create a temp file for the local inode created above. + err = t.in.CreateEmptyTempFile(t.ctx) + assert.Nil(t.T(), err) + // Write some content to temp file. + t.clock.AdvanceTime(time.Second) + writeTime := t.clock.Now() + gcsSynced, err := t.in.Write(t.ctx, []byte("tacos"), 0, WriteMode) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) + t.clock.AdvanceTime(time.Second) + + if tc.callSync { + gcsSynced, err := t.in.Sync(t.ctx) + require.NoError(t.T(), err) + assert.True(t.T(), gcsSynced) + } else { + err = t.in.Flush(t.ctx) + assert.Nil(t.T(), err) + } + + // Verify that fileInode is no more local + assert.False(t.T(), t.in.IsLocal()) + // Stat the current object in the bucket. + statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} + m, _, err := t.bucket.StatObject(t.ctx, statReq) + require.NoError(t.T(), err) + assert.NotNil(t.T(), m) + assert.Equal(t.T(), t.in.SourceGeneration().Object, m.Generation) + assert.Equal(t.T(), t.in.SourceGeneration().Metadata, m.MetaGeneration) + assert.Equal(t.T(), t.in.SourceGeneration().Size, m.Size) + assert.Equal(t.T(), uint64(len("tacos")), m.Size) + assert.Equal(t.T(), + writeTime.UTC().Format(time.RFC3339Nano), + m.Metadata["gcsfuse_mtime"]) + t.validateMrdWrapperMinObject() + t.validateMrdInstanceMinObject() + // Read the object's contents. + contents, err := storageutil.ReadObject(t.ctx, t.bucket, t.in.Name().GcsObjectName()) + require.NoError(t.T(), err) + assert.Equal(t.T(), "tacos", string(contents)) + // Check attributes. + attrs, err = t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), uint64(len("tacos")), attrs.Size) + assert.Equal(t.T(), attrs.Mtime, writeTime.UTC()) + }) + } +} - // Stat the current object in the bucket. - statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} - m, _, err := t.bucket.StatObject(t.ctx, statReq) +func (t *FileTest) TestSyncEmptyLocalFile() { + testcases := []struct { + name string + callSync bool + }{ + { + name: "sync", + callSync: true, + }, + { + name: "flush", + callSync: false, + }, + } - AssertEq(nil, err) - AssertNe(nil, m) - ExpectEq(t.in.SourceGeneration().Object, m.Generation) - ExpectEq(t.in.SourceGeneration().Metadata, m.MetaGeneration) - ExpectEq(2, m.Size) - ExpectEq( - truncateTime.UTC().Format(time.RFC3339Nano), - m.Metadata["gcsfuse_mtime"]) + for _, tc := range testcases { + t.Run(tc.name, func() { + var attrs fuseops.InodeAttributes + var err error + // Create a local file inode. + t.createInodeWithLocalParam("test", true) + creationTime := t.clock.Now() + // Create a temp file for the local inode created above. + err = t.in.CreateEmptyTempFile(t.ctx) + assert.Nil(t.T(), err) + + if tc.callSync { + gcsSynced, err := t.in.Sync(t.ctx) + require.NoError(t.T(), err) + assert.True(t.T(), gcsSynced) + } else { + err = t.in.Flush(t.ctx) + assert.Nil(t.T(), err) + } + + // Verify that fileInode is no more local + assert.False(t.T(), t.in.IsLocal()) + // Stat the current object in the bucket. + statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} + m, _, err := t.bucket.StatObject(t.ctx, statReq) + require.NoError(t.T(), err) + assert.NotNil(t.T(), m) + assert.Equal(t.T(), t.in.SourceGeneration().Object, m.Generation) + assert.Equal(t.T(), t.in.SourceGeneration().Metadata, m.MetaGeneration) + assert.Equal(t.T(), t.in.SourceGeneration().Size, m.Size) + assert.Equal(t.T(), uint64(0), m.Size) + t.validateMrdWrapperMinObject() + t.validateMrdInstanceMinObject() + // Validate the mtime. + mtimeInBucket, ok := m.Metadata["gcsfuse_mtime"] + assert.True(t.T(), ok) + mtime, _ := time.Parse(time.RFC3339Nano, mtimeInBucket) + assert.WithinDuration(t.T(), mtime, creationTime, Delta) + // Read the object's contents. + contents, err := storageutil.ReadObject(t.ctx, t.bucket, t.in.Name().GcsObjectName()) + require.NoError(t.T(), err) + assert.Equal(t.T(), "", string(contents)) + // Check attributes. + attrs, err = t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), uint64(0), attrs.Size) + }) + } +} - // Check attributes. - attrs, err = t.in.Attributes(t.ctx) - AssertEq(nil, err) +func (t *FileTest) TestAppendThenSync() { + testcases := []struct { + name string + callSync bool + }{ + { + name: "sync", + callSync: true, + }, + { + name: "flush", + callSync: false, + }, + } - ExpectEq(2, attrs.Size) - ExpectThat(attrs.Mtime, timeutil.TimeEq(truncateTime.UTC())) + for _, tc := range testcases { + t.Run(tc.name, func() { + var attrs fuseops.InodeAttributes + var err error + + assert.Equal(t.T(), "taco", t.initialContents) + + // Append some data. + t.clock.AdvanceTime(time.Second) + writeTime := t.clock.Now() + + gcsSynced, err := t.in.Write(t.ctx, []byte("burrito"), int64(len("taco")), AppendMode) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) + + t.clock.AdvanceTime(time.Second) + + if tc.callSync { + gcsSynced, err := t.in.Sync(t.ctx) + require.NoError(t.T(), err) + assert.True(t.T(), gcsSynced) + } else { + err = t.in.Flush(t.ctx) + assert.Nil(t.T(), err) + } + + // The generation should have advanced. + assert.Less(t.T(), t.backingObj.Generation, t.in.SourceGeneration().Object) + + // Stat the current object in the bucket. + statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} + m, _, err := t.bucket.StatObject(t.ctx, statReq) + + require.NoError(t.T(), err) + assert.NotNil(t.T(), m) + assert.Equal(t.T(), t.in.SourceGeneration().Object, m.Generation) + assert.Equal(t.T(), t.in.SourceGeneration().Metadata, m.MetaGeneration) + assert.Equal(t.T(), t.in.SourceGeneration().Size, m.Size) + assert.Equal(t.T(), uint64(len("tacoburrito")), m.Size) + assert.Equal(t.T(), + writeTime.UTC().Format(time.RFC3339Nano), + m.Metadata["gcsfuse_mtime"]) + t.validateMrdWrapperMinObject() + t.validateMrdInstanceMinObject() + + // Read the object's contents. + contents, err := storageutil.ReadObject(t.ctx, t.bucket, t.in.Name().GcsObjectName()) + + require.NoError(t.T(), err) + assert.Equal(t.T(), "tacoburrito", string(contents)) + + // Check attributes. + attrs, err = t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + + assert.Equal(t.T(), uint64(len("tacoburrito")), attrs.Size) + assert.Equal(t.T(), attrs.Mtime, writeTime.UTC()) + }) + } } -func (t *FileTest) TruncateUpwardThenSync() { - var attrs fuseops.InodeAttributes +func (t *FileTest) TestAppendToUnfinalizedObjInZB() { + // Set up the Zonal Bucket + t.bucket = fake.NewFakeBucket(&t.clock, "some_bucket", gcs.BucketType{Zonal: true}) + // Set up the backing unfinalized object. var err error + t.initialContents = "lychee" + object, err := storageutil.CreateObject( + t.ctx, + t.bucket, + fileName, + []byte(t.initialContents)) + assert.Nil(t.T(), err) + object.Finalized = time.Time{} + t.backingObj = storageutil.ConvertObjToMinObject(object) + t.createInode() + t.in.config = &cfg.Config{Write: *getWriteConfigWithEnabledRapidAppends()} + assert.Nil(t.T(), t.in.content) + t.createBufferedWriteHandler(true, AppendMode) + assert.NotNil(t.T(), t.in.bwh) - AssertEq(4, len(t.initialContents)) - - // Truncate upward. - t.clock.AdvanceTime(time.Second) - truncateTime := t.clock.Now() - - err = t.in.Truncate(t.ctx, 6) - AssertEq(nil, err) - - t.clock.AdvanceTime(time.Second) - - // Sync. - err = t.in.Sync(t.ctx) - AssertEq(nil, err) + gcsSynced, err := t.in.Write(t.ctx, []byte("juice"), int64(len(t.initialContents)), AppendMode) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) - // The generation should have advanced. - ExpectLt(t.backingObj.Generation, t.in.SourceGeneration().Object) + gcsSynced, err = t.in.Sync(t.ctx) + require.NoError(t.T(), err) + assert.True(t.T(), gcsSynced) - // Stat the current object in the bucket. - statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} - m, _, err := t.bucket.StatObject(t.ctx, statReq) - ExpectEq( - truncateTime.UTC().Format(time.RFC3339Nano), - m.Metadata["gcsfuse_mtime"]) + // Read the object contents through back-door. + contents, err := storageutil.ReadObject(t.ctx, t.bucket, t.in.Name().GcsObjectName()) + require.NoError(t.T(), err) + assert.Equal(t.T(), "lycheejuice", string(contents)) +} - AssertEq(nil, err) - AssertNe(nil, m) - ExpectEq(t.in.SourceGeneration().Object, m.Generation) - ExpectEq(t.in.SourceGeneration().Metadata, m.MetaGeneration) - ExpectEq(6, m.Size) +func (t *FileTest) TestTruncateDownwardThenSync() { + testcases := []struct { + name string + callSync bool + }{ + { + name: "sync", + callSync: true, + }, + { + name: "flush", + callSync: false, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func() { + var attrs fuseops.InodeAttributes + var err error + + // Truncate downward. + t.clock.AdvanceTime(time.Second) + truncateTime := t.clock.Now() + + gcsSynced, err := t.in.Truncate(t.ctx, 2) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) + + t.clock.AdvanceTime(time.Second) + + if tc.callSync { + gcsSynced, err := t.in.Sync(t.ctx) + require.NoError(t.T(), err) + assert.True(t.T(), gcsSynced) + } else { + err = t.in.Flush(t.ctx) + assert.Nil(t.T(), err) + } + + // The generation should have advanced. + assert.Less(t.T(), t.backingObj.Generation, t.in.SourceGeneration().Object) + + t.validateMrdWrapperMinObject() + t.validateMrdInstanceMinObject() + + // Stat the current object in the bucket. + statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} + m, _, err := t.bucket.StatObject(t.ctx, statReq) + + require.NoError(t.T(), err) + assert.NotNil(t.T(), m) + assert.Equal(t.T(), t.in.SourceGeneration().Object, m.Generation) + assert.Equal(t.T(), t.in.SourceGeneration().Metadata, m.MetaGeneration) + assert.Equal(t.T(), t.in.SourceGeneration().Size, m.Size) + assert.Equal(t.T(), uint64(2), m.Size) + assert.Equal(t.T(), + truncateTime.UTC().Format(time.RFC3339Nano), + m.Metadata["gcsfuse_mtime"]) + + // Check attributes. + attrs, err = t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + + assert.Equal(t.T(), uint64(2), attrs.Size) + assert.Equal(t.T(), attrs.Mtime, truncateTime.UTC()) + }) + } +} - // Check attributes. - attrs, err = t.in.Attributes(t.ctx) - AssertEq(nil, err) +func (t *FileTest) TestTruncateUpwardThenFlush() { + testcases := []struct { + name string + callSync bool + }{ + { + name: "sync", + callSync: true, + }, + { + name: "flush", + callSync: false, + }, + } - ExpectEq(6, attrs.Size) - ExpectThat(attrs.Mtime, timeutil.TimeEq(truncateTime.UTC())) + for _, tc := range testcases { + t.Run(tc.name, func() { + var attrs fuseops.InodeAttributes + var err error + + assert.Equal(t.T(), 4, len(t.initialContents)) + + // Truncate upward. + t.clock.AdvanceTime(time.Second) + truncateTime := t.clock.Now() + + gcsSynced, err := t.in.Truncate(t.ctx, 6) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) + + t.clock.AdvanceTime(time.Second) + + if tc.callSync { + gcsSynced, err := t.in.Sync(t.ctx) + require.NoError(t.T(), err) + assert.True(t.T(), gcsSynced) + } else { + err = t.in.Flush(t.ctx) + assert.Nil(t.T(), err) + } + + // The generation should have advanced. + assert.Less(t.T(), t.backingObj.Generation, t.in.SourceGeneration().Object) + + t.validateMrdWrapperMinObject() + t.validateMrdInstanceMinObject() + + // Stat the current object in the bucket. + statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} + m, _, err := t.bucket.StatObject(t.ctx, statReq) + + require.NoError(t.T(), err) + assert.NotNil(t.T(), m) + assert.Equal(t.T(), + truncateTime.UTC().Format(time.RFC3339Nano), + m.Metadata["gcsfuse_mtime"]) + assert.Equal(t.T(), t.in.SourceGeneration().Object, m.Generation) + assert.Equal(t.T(), t.in.SourceGeneration().Metadata, m.MetaGeneration) + assert.Equal(t.T(), t.in.SourceGeneration().Size, m.Size) + assert.Equal(t.T(), uint64(6), m.Size) + + // Check attributes. + attrs, err = t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + + assert.Equal(t.T(), uint64(6), attrs.Size) + assert.Equal(t.T(), attrs.Mtime, truncateTime.UTC()) + }) + } } func (t *FileTest) TestTruncateUpwardForLocalFileShouldUpdateLocalFileAttributes() { @@ -615,25 +1086,27 @@ func (t *FileTest) TestTruncateUpwardForLocalFileShouldUpdateLocalFileAttributes var attrs fuseops.InodeAttributes // Create a local file inode. t.createInodeWithLocalParam("test", true) - err = t.in.CreateEmptyTempFile() - AssertEq(nil, err) + // Create a temp file for the local inode created above. + err = t.in.CreateEmptyTempFile(t.ctx) + assert.Nil(t.T(), err) // Fetch the attributes and check if the file is empty. - attrs, err = t.in.Attributes(t.ctx) - AssertEq(nil, err) - AssertEq(0, attrs.Size) + attrs, err = t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), uint64(0), attrs.Size) - err = t.in.Truncate(t.ctx, 6) + gcsSynced, err := t.in.Truncate(t.ctx, 6) - AssertEq(nil, err) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) // The inode should return the new size. - attrs, err = t.in.Attributes(t.ctx) - AssertEq(nil, err) - AssertEq(6, attrs.Size) + attrs, err = t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), uint64(6), attrs.Size) // Data shouldn't be updated to GCS. statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} _, _, err = t.bucket.StatObject(t.ctx, statReq) - AssertNe(nil, err) - AssertEq("gcs.NotFoundError: object test not found", err.Error()) + require.Error(t.T(), err) + assert.Equal(t.T(), "gcs.NotFoundError: object test not found", err.Error()) } func (t *FileTest) TestTruncateDownwardForLocalFileShouldUpdateLocalFileAttributes() { @@ -641,64 +1114,287 @@ func (t *FileTest) TestTruncateDownwardForLocalFileShouldUpdateLocalFileAttribut var attrs fuseops.InodeAttributes // Create a local file inode. t.createInodeWithLocalParam("test", true) - err = t.in.CreateEmptyTempFile() - AssertEq(nil, err) + // Create a temp file for the local inode created above. + err = t.in.CreateEmptyTempFile(t.ctx) + assert.Nil(t.T(), err) // Write some data to the local file. - err = t.in.Write(t.ctx, []byte("burrito"), 0) - AssertEq(nil, err) + gcsSynced, err := t.in.Write(t.ctx, []byte("burrito"), 0, WriteMode) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) // Validate the new data is written correctly. - attrs, err = t.in.Attributes(t.ctx) - AssertEq(nil, err) - AssertEq(7, attrs.Size) + attrs, err = t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), uint64(7), attrs.Size) - err = t.in.Truncate(t.ctx, 2) + gcsSynced, err = t.in.Truncate(t.ctx, 2) - AssertEq(nil, err) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) // The inode should return the new size. - attrs, err = t.in.Attributes(t.ctx) - AssertEq(nil, err) - AssertEq(2, attrs.Size) + attrs, err = t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), uint64(2), attrs.Size) // Data shouldn't be updated to GCS. statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} _, _, err = t.bucket.StatObject(t.ctx, statReq) - AssertNe(nil, err) - AssertEq("gcs.NotFoundError: object test not found", err.Error()) + require.Error(t.T(), err) + assert.Equal(t.T(), "gcs.NotFoundError: object test not found", err.Error()) } -func (t *FileTest) Sync_Clobbered() { - var err error +func (t *FileTest) TestTruncateUpwardForLocalFileWhenStreamingWritesAreEnabled() { + tbl := []struct { + name string + performWrite bool + }{ + { + name: "WithWrite", + performWrite: true, + }, + { + name: "WithOutWrite", + performWrite: false, + }, + } + for _, tc := range tbl { + t.Run(tc.name, func() { + // Create a local file inode. + t.createInodeWithLocalParam("test", true) + t.in.config = &cfg.Config{Write: *getWriteConfig()} + // Fetch the attributes and check if the file is empty. + attrs, err := t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), uint64(0), attrs.Size) + + if tc.performWrite { + t.createBufferedWriteHandler(true, WriteMode) + gcsSynced, err := t.in.Write(t.ctx, []byte("hi"), 0, WriteMode) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) + assert.Equal(t.T(), int64(2), t.in.bwh.WriteFileInfo().TotalSize) + // Fetch the attributes and check if the file size reflects the write. + attrs, err := t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), uint64(2), attrs.Size) + } + t.createBufferedWriteHandler(!tc.performWrite, WriteMode) + + gcsSynced, err := t.in.Truncate(t.ctx, 10) + + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) + // The inode should return the new size. + attrs, err = t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), uint64(10), attrs.Size) + // Data shouldn't be updated to GCS. + statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} + _, _, err = t.bucket.StatObject(t.ctx, statReq) + require.Error(t.T(), err) + assert.Equal(t.T(), "gcs.NotFoundError: object test not found", err.Error()) + }) + } +} - // Truncate downward. - err = t.in.Truncate(t.ctx, 2) - AssertEq(nil, err) +func (t *FileTest) TestTruncateUpwardForEmptyGCSFileWhenStreamingWritesAreEnabled() { + tbl := []struct { + name string + performWrite bool + }{ + { + name: "WithWrite", + performWrite: true, + }, + { + name: "WithOutWrite", + performWrite: false, + }, + } + for _, tc := range tbl { + t.Run(tc.name, func() { + t.createInodeWithEmptyObject() + t.in.config = &cfg.Config{Write: *getWriteConfig()} + + // Fetch the attributes and check if the file is empty. + attrs, err := t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), uint64(0), attrs.Size) + + if tc.performWrite { + t.createBufferedWriteHandler(true, WriteMode) + gcsSynced, err := t.in.Write(t.ctx, []byte("hi"), 0, WriteMode) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) + assert.Equal(t.T(), int64(2), t.in.bwh.WriteFileInfo().TotalSize) + // Fetch the attributes and check if the file size reflects the write. + attrs, err := t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), uint64(2), attrs.Size) + } + t.createBufferedWriteHandler(!tc.performWrite, WriteMode) + + gcsSynced, err := t.in.Truncate(t.ctx, 10) + + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) + // The inode should return the new size. + attrs, err = t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), uint64(10), attrs.Size) + // Data shouldn't be updated to GCS. + statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} + minObject, _, err := t.bucket.StatObject(t.ctx, statReq) + require.NoError(t.T(), err) + assert.Equal(t.T(), uint64(0), minObject.Size) + }) + } +} + +func (t *FileTest) TestTruncateDownwardWhenStreamingWritesAreEnabled() { + tbl := []struct { + name string + fileType string + truncateSize int64 + }{ + { + name: "LocalFileTruncateToNonZero", + fileType: LocalFile, + truncateSize: 2, + }, + { + name: "LocalFileTruncateToZero", + fileType: LocalFile, + truncateSize: 0, + }, + { + name: "EmptyGCSFileTruncateToNonZero", + fileType: EmptyGCSFile, + truncateSize: 2, + }, + { + name: "EmptyGCSFileTruncateToZero", + fileType: EmptyGCSFile, + truncateSize: 0, + }, + } + for _, tc := range tbl { + t.Run(tc.name, func() { + if tc.fileType == LocalFile { + t.createInodeWithLocalParam("test", true) + } + if tc.fileType == EmptyGCSFile { + t.createInodeWithEmptyObject() + } + t.in.config = &cfg.Config{Write: *getWriteConfig()} + // Fetch the attributes and check if the file is empty. + attrs, err := t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), uint64(0), attrs.Size) + + t.createBufferedWriteHandler(true, WriteMode) + gcsSynced, err := t.in.Write(t.ctx, []byte("hihello"), 0, WriteMode) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) + assert.Equal(t.T(), int64(7), t.in.bwh.WriteFileInfo().TotalSize) + // Fetch the attributes and check if the file size reflects the write. + attrs, err = t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), uint64(7), attrs.Size) + gcsSynced, err = t.in.Truncate(t.ctx, tc.truncateSize) + + require.NoError(t.T(), err) + assert.True(t.T(), gcsSynced) + t.createBufferedWriteHandler(false, WriteMode) + }) + } +} +func (t *FileTest) TestSyncFlush_Clobbered() { + testcases := []struct { + name string + callSync bool + }{ + { + name: "sync", + callSync: true, + }, + { + name: "flush", + callSync: false, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func() { + var err error + + // Truncate downward. + gcsSynced, err := t.in.Truncate(t.ctx, 2) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) + + // Clobber the backing object. + newObj, err := storageutil.CreateObject( + t.ctx, + t.bucket, + t.in.Name().GcsObjectName(), + []byte("burrito")) + + assert.Nil(t.T(), err) + + if tc.callSync { + var gcsSynced bool + // Sync. The call should not succeed, and we expect a FileClobberedError. + gcsSynced, err = t.in.Sync(t.ctx) + require.Error(t.T(), err) + assert.False(t.T(), gcsSynced) + } else { + // Flush. The call should not succeed, and we expect a FileClobberedError. + err = t.in.Flush(t.ctx) + } + + t.validateMrdWrapperMinObject() + t.validateMrdInstanceMinObject() + + // Check if the error is a FileClobberedError + var fcErr *gcsfuse_errors.FileClobberedError + assert.True(t.T(), errors.As(err, &fcErr), "expected FileClobberedError but got %v", err) + assert.Equal(t.T(), t.backingObj.Generation, t.in.SourceGeneration().Object) + assert.Equal(t.T(), t.backingObj.MetaGeneration, t.in.SourceGeneration().Metadata) + assert.Equal(t.T(), t.backingObj.Size, t.in.SourceGeneration().Size) + + // The object in the bucket should not have been changed. + statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} + m, _, err := t.bucket.StatObject(t.ctx, statReq) + + require.NoError(t.T(), err) + assert.NotNil(t.T(), m) + assert.Equal(t.T(), newObj.Generation, m.Generation) + assert.Equal(t.T(), newObj.Size, m.Size) + }) + } +} + +func (t *FileTest) TestOpenReader_ThrowsFileClobberedError() { + // Modify the file locally. + gcsSynced, err := t.in.Truncate(t.ctx, 2) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) // Clobber the backing object. - newObj, err := storageutil.CreateObject( + _, err = storageutil.CreateObject( t.ctx, t.bucket, t.in.Name().GcsObjectName(), []byte("burrito")) + assert.Nil(t.T(), err) - AssertEq(nil, err) - - // Sync. The call should succeed, but nothing should change. - err = t.in.Sync(t.ctx) - - AssertEq(nil, err) - ExpectEq(t.backingObj.Generation, t.in.SourceGeneration().Object) - ExpectEq(t.backingObj.MetaGeneration, t.in.SourceGeneration().Metadata) + _, err = t.in.openReader(t.ctx) - // The object in the bucket should not have been changed. - statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} - m, _, err := t.bucket.StatObject(t.ctx, statReq) - - AssertEq(nil, err) - AssertNe(nil, m) - ExpectEq(newObj.Generation, m.Generation) - ExpectEq(newObj.Size, m.Size) + // assert error is not nil. + var fcErr *gcsfuse_errors.FileClobberedError + assert.True(t.T(), errors.As(err, &fcErr), "expected FileClobberedError but got %v", err) } -func (t *FileTest) SetMtime_ContentNotFaultedIn() { +func (t *FileTest) TestSetMtime_ContentNotFaultedIn() { var err error var attrs fuseops.InodeAttributes @@ -706,92 +1402,97 @@ func (t *FileTest) SetMtime_ContentNotFaultedIn() { mtime := time.Now().UTC().Add(123*time.Second).AddDate(0, 0, 0) err = t.in.SetMtime(t.ctx, mtime) - AssertEq(nil, err) + assert.Nil(t.T(), err) // The inode should agree about the new mtime. - attrs, err = t.in.Attributes(t.ctx) + attrs, err = t.in.Attributes(t.ctx, true) - AssertEq(nil, err) - ExpectThat(attrs.Mtime, timeutil.TimeEq(mtime)) + require.NoError(t.T(), err) + assert.Equal(t.T(), attrs.Mtime, mtime) // The inode should have added the mtime to the backing object's metadata. statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} m, _, err := t.bucket.StatObject(t.ctx, statReq) - AssertEq(nil, err) - AssertNe(nil, m) - ExpectEq( + require.NoError(t.T(), err) + assert.NotNil(t.T(), m) + assert.Equal(t.T(), mtime.UTC().Format(time.RFC3339Nano), m.Metadata["gcsfuse_mtime"]) } -func (t *FileTest) SetMtime_ContentClean() { +func (t *FileTest) TestSetMtime_ContentClean() { var err error var attrs fuseops.InodeAttributes // Cause the content to be faulted in. _, err = t.in.Read(t.ctx, make([]byte, 1), 0) - AssertEq(nil, err) + assert.Nil(t.T(), err) // Set mtime. mtime := time.Now().UTC().Add(123*time.Second).AddDate(0, 0, 0) err = t.in.SetMtime(t.ctx, mtime) - AssertEq(nil, err) + assert.Nil(t.T(), err) // The inode should agree about the new mtime. - attrs, err = t.in.Attributes(t.ctx) + attrs, err = t.in.Attributes(t.ctx, true) - AssertEq(nil, err) - ExpectThat(attrs.Mtime, timeutil.TimeEq(mtime)) + require.NoError(t.T(), err) + assert.Equal(t.T(), attrs.Mtime, mtime) // The inode should have added the mtime to the backing object's metadata. statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} m, _, err := t.bucket.StatObject(t.ctx, statReq) - AssertEq(nil, err) - AssertNe(nil, m) - ExpectEq( + require.NoError(t.T(), err) + assert.NotNil(t.T(), m) + assert.Equal(t.T(), mtime.UTC().Format(time.RFC3339Nano), m.Metadata["gcsfuse_mtime"]) } -func (t *FileTest) SetMtime_ContentDirty() { +func (t *FileTest) TestSetMtime_ContentDirty() { var err error var attrs fuseops.InodeAttributes // Dirty the content. - err = t.in.Write(t.ctx, []byte("a"), 0) - AssertEq(nil, err) + gcsSynced, err := t.in.Write(t.ctx, []byte("a"), 0, WriteMode) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) // Set mtime. mtime := time.Now().UTC().Add(123 * time.Second) err = t.in.SetMtime(t.ctx, mtime) - AssertEq(nil, err) + assert.Nil(t.T(), err) // The inode should agree about the new mtime. - attrs, err = t.in.Attributes(t.ctx) + attrs, err = t.in.Attributes(t.ctx, true) - AssertEq(nil, err) - ExpectThat(attrs.Mtime, timeutil.TimeEq(mtime)) + require.NoError(t.T(), err) + assert.Equal(t.T(), attrs.Mtime, mtime) // Sync. - err = t.in.Sync(t.ctx) - AssertEq(nil, err) + gcsSynced, err = t.in.Sync(t.ctx) + require.NoError(t.T(), err) + assert.True(t.T(), gcsSynced) // Now the object in the bucket should have the appropriate mtime. statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} m, _, err := t.bucket.StatObject(t.ctx, statReq) - AssertEq(nil, err) - AssertNe(nil, m) - ExpectEq( + t.validateMrdWrapperMinObject() + t.validateMrdInstanceMinObject() + + require.NoError(t.T(), err) + assert.NotNil(t.T(), m) + assert.Equal(t.T(), mtime.UTC().Format(time.RFC3339Nano), m.Metadata["gcsfuse_mtime"]) } -func (t *FileTest) SetMtime_SourceObjectGenerationChanged() { +func (t *FileTest) TestSetMtime_SourceObjectGenerationChanged() { var err error // Clobber the backing object. @@ -801,24 +1502,24 @@ func (t *FileTest) SetMtime_SourceObjectGenerationChanged() { t.in.Name().GcsObjectName(), []byte("burrito")) - AssertEq(nil, err) + assert.Nil(t.T(), err) // Set mtime. mtime := time.Now().UTC().Add(123 * time.Second) err = t.in.SetMtime(t.ctx, mtime) - AssertEq(nil, err) + assert.Nil(t.T(), err) // The object in the bucket should not have been changed. statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} m, _, err := t.bucket.StatObject(t.ctx, statReq) - AssertEq(nil, err) - AssertNe(nil, m) - ExpectEq(newObj.Generation, m.Generation) - ExpectEq(0, len(m.Metadata)) + require.NoError(t.T(), err) + assert.NotNil(t.T(), m) + assert.Equal(t.T(), newObj.Generation, m.Generation) + assert.Equal(t.T(), 0, len(m.Metadata)) } -func (t *FileTest) SetMtime_SourceObjectMetaGenerationChanged() { +func (t *FileTest) TestSetMtime_SourceObjectMetaGenerationChanged() { var err error // Update the backing object. @@ -830,186 +1531,457 @@ func (t *FileTest) SetMtime_SourceObjectMetaGenerationChanged() { ContentLanguage: &lang, }) - AssertEq(nil, err) + assert.Nil(t.T(), err) // Set mtime. mtime := time.Now().UTC().Add(123 * time.Second) err = t.in.SetMtime(t.ctx, mtime) - AssertEq(nil, err) + assert.Nil(t.T(), err) // The object in the bucket should not have been changed. statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} m, _, err := t.bucket.StatObject(t.ctx, statReq) - AssertEq(nil, err) - AssertNe(nil, m) - ExpectEq(newObj.Generation, m.Generation) - ExpectEq(newObj.MetaGeneration, m.MetaGeneration) + require.NoError(t.T(), err) + assert.NotNil(t.T(), m) + assert.Equal(t.T(), newObj.Generation, m.Generation) + assert.Equal(t.T(), newObj.MetaGeneration, m.MetaGeneration) } -func (t *FileTest) TestSetMtimeForLocalFileShouldUpdateLocalFileAttributes() { +func (t *FileTest) TestSetMtimeForUnlinkedFileIsNoOp() { + t.in.unlinked = true + beforeUpdateAttr, err := t.in.Attributes(t.ctx, true) + require.Nil(t.T(), err) + mtime := beforeUpdateAttr.Mtime.UTC().Add(123 * time.Second) + + // Set mtime. + err = t.in.SetMtime(t.ctx, mtime) + + require.Nil(t.T(), err) + afterUpdateAttr, err := t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.NotEqual(t.T(), mtime, afterUpdateAttr.Mtime) + assert.Equal(t.T(), beforeUpdateAttr.Mtime, afterUpdateAttr.Mtime) +} + +func (t *FileTest) TestTestSetMtimeForLocalFileShouldUpdateLocalFileAttributes() { var err error var attrs fuseops.InodeAttributes + // Create a local file inode. t.createInodeWithLocalParam("test", true) createTime := t.in.mtimeClock.Now() - err = t.in.CreateEmptyTempFile() - AssertEq(nil, err) + // Create a temp file for the local inode created above. + err = t.in.CreateEmptyTempFile(t.ctx) + assert.Nil(t.T(), err) // Validate the attributes on an empty file. - attrs, err = t.in.Attributes(t.ctx) - AssertEq(nil, err) - ExpectThat(attrs.Mtime, timeutil.TimeNear(createTime, Delta)) + attrs, err = t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.WithinDuration(t.T(), attrs.Mtime, createTime, Delta) // Set mtime. mtime := time.Now().UTC().Add(123 * time.Second) err = t.in.SetMtime(t.ctx, mtime) - AssertEq(nil, err) + assert.Nil(t.T(), err) // The inode should agree about the new mtime. - attrs, err = t.in.Attributes(t.ctx) - AssertEq(nil, err) - ExpectThat(attrs.Mtime, timeutil.TimeEq(mtime)) - ExpectThat(attrs.Ctime, timeutil.TimeEq(mtime)) - ExpectThat(attrs.Atime, timeutil.TimeEq(mtime)) + attrs, err = t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), attrs.Mtime, mtime) + assert.Equal(t.T(), attrs.Ctime, mtime) + assert.Equal(t.T(), attrs.Atime, mtime) // Data shouldn't be updated to GCS. statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} _, _, err = t.bucket.StatObject(t.ctx, statReq) - AssertNe(nil, err) - AssertEq("gcs.NotFoundError: object test not found", err.Error()) + require.Error(t.T(), err) + assert.Equal(t.T(), "gcs.NotFoundError: object test not found", err.Error()) +} + +func (t *FileTest) TestSetMtimeForLocalFileWhenStreamingWritesAreEnabled() { + var err error + var attrs fuseops.InodeAttributes + + // Create a local file inode. + t.createInodeWithLocalParam("test", true) + t.in.config = &cfg.Config{Write: *getWriteConfig()} + t.createBufferedWriteHandler(true, WriteMode) + + // Set mtime. + mtime := time.Now().UTC().Add(123 * time.Second) + err = t.in.SetMtime(t.ctx, mtime) + + assert.Nil(t.T(), err) + // The inode should agree about the new mtime. + attrs, err = t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), attrs.Mtime, mtime) + assert.Equal(t.T(), attrs.Ctime, mtime) + assert.Equal(t.T(), attrs.Atime, mtime) } -func (t *FileTest) ContentEncodingGzip() { +func (t *FileTest) TestContentEncodingGzip() { // Set up an explicit content-encoding on the backing object and re-create the inode. contentEncoding := "gzip" t.backingObj.ContentEncoding = contentEncoding t.createInode() - AssertEq(contentEncoding, t.in.Source().ContentEncoding) - AssertTrue(t.in.Source().HasContentEncodingGzip()) + assert.Equal(t.T(), contentEncoding, t.in.Source().ContentEncoding) + assert.True(t.T(), t.in.Source().HasContentEncodingGzip()) } -func (t *FileTest) ContentEncodingNone() { +func (t *FileTest) TestContentEncodingNone() { // Set up an explicit content-encoding on the backing object and re-create the inode. contentEncoding := "" t.backingObj.ContentEncoding = contentEncoding t.createInode() - AssertEq(contentEncoding, t.in.Source().ContentEncoding) - AssertFalse(t.in.Source().HasContentEncodingGzip()) + assert.Equal(t.T(), contentEncoding, t.in.Source().ContentEncoding) + assert.False(t.T(), t.in.Source().HasContentEncodingGzip()) } -func (t *FileTest) ContentEncodingOther() { +func (t *FileTest) TestContentEncodingOther() { // Set up an explicit content-encoding on the backing object and re-create the inode. contentEncoding := "other" t.backingObj.ContentEncoding = contentEncoding t.createInode() - AssertEq(contentEncoding, t.in.Source().ContentEncoding) - AssertFalse(t.in.Source().HasContentEncodingGzip()) + assert.Equal(t.T(), contentEncoding, t.in.Source().ContentEncoding) + assert.False(t.T(), t.in.Source().HasContentEncodingGzip()) } -func (t *FileTest) TestCheckInvariantsShouldNotThrowExceptionForLocalFiles() { +func (t *FileTest) TestTestCheckInvariantsShouldNotThrowExceptionForLocalFiles() { t.createInodeWithLocalParam("test", true) - AssertNe(nil, t.in) + assert.NotNil(t.T(), t.in) } -func (t *FileTest) TestCreateEmptyTempFileShouldCreateEmptyFile() { - err := t.in.CreateEmptyTempFile() +func (t *FileTest) TestCreateEmptyTempFile() { + err := t.in.CreateEmptyTempFile(t.ctx) - AssertEq(nil, err) - AssertNe(nil, t.in.content) + assert.Nil(t.T(), err) + assert.NotNil(t.T(), t.in.content) // Validate that file size is 0. sr, err := t.in.content.Stat() - AssertEq(nil, err) - AssertEq(0, sr.Size) + require.NoError(t.T(), err) + assert.Equal(t.T(), int64(0), sr.Size) } -func (t *FileTest) UnlinkLocalFile() { +func (t *FileTest) TestCreateEmptyTempFileWhenBWHIsNotNil() { + testCases := []struct { + name string + isLocal bool + }{ + { + name: "ShouldNotCreateForEmptyGCSFile", + isLocal: false, + }, + { + name: "ShouldNotCreateForLocalFile", + isLocal: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + if tc.isLocal { + t.createInodeWithLocalParam("test", true) + } else { + t.createInodeWithEmptyObject() + } + t.in.config = &cfg.Config{Write: *getWriteConfig()} + t.createBufferedWriteHandler(true, WriteMode) + + err := t.in.CreateEmptyTempFile(t.ctx) + + assert.Nil(t.T(), err) + assert.Nil(t.T(), t.in.content) + }) + } +} + +func (t *FileTest) TestInitBufferedWriteHandlerIfEligibleShouldNotCreateBWHNonEmptySyncedFile() { + // Enabling buffered writes. + t.in.config = &cfg.Config{Write: *getWriteConfig()} + + initialized, err := t.in.InitBufferedWriteHandlerIfEligible(t.ctx, WriteMode) + + assert.NoError(t.T(), err) + assert.Nil(t.T(), t.in.bwh) + assert.False(t.T(), initialized) +} + +func (t *FileTest) TestUnlinkLocalFile() { var err error // Create a local file inode. t.createInodeWithLocalParam("test", true) // Create a temp file for the local inode created above. - err = t.in.CreateEmptyTempFile() - AssertEq(nil, err) + err = t.in.CreateEmptyTempFile(t.ctx) + assert.Nil(t.T(), err) // Unlink. t.in.Unlink() // Verify that fileInode is now unlinked - AssertTrue(t.in.IsUnlinked()) + assert.True(t.T(), t.in.IsUnlinked()) // Data shouldn't be updated to GCS. statReq := &gcs.StatObjectRequest{Name: t.in.Name().GcsObjectName()} _, _, err = t.bucket.StatObject(t.ctx, statReq) - AssertNe(nil, err) - AssertEq("gcs.NotFoundError: object test not found", err.Error()) + require.Error(t.T(), err) + assert.Equal(t.T(), "gcs.NotFoundError: object test not found", err.Error()) } -func (t *FileTest) ReadLocalFileWhenStreamingWritesAreEnabled() { - // Create a local file inode. - t.createInodeWithLocalParam("test", true) - t.in.writeConfig = getWriteConfig() - err := t.in.Write(t.ctx, []byte("hi"), 0) - AssertEq(nil, err) - AssertEq(2, t.in.bwh.WriteFileInfo().TotalSize) +func (t *FileTest) TestReadFileWhenStreamingWritesAreEnabled() { + tbl := []struct { + name string + fileType string + performWrite bool + }{ + { + name: "LocalFileWithWrite", + fileType: LocalFile, + performWrite: true, + }, + { + name: "LocalFileWithOutWrite", + fileType: LocalFile, + performWrite: false, + }, + { + name: "EmptyGCSFileWithWrite", + fileType: EmptyGCSFile, + performWrite: true, + }, + } + for _, tc := range tbl { + t.Run(tc.name, func() { + if tc.fileType == LocalFile { + // Create a local file inode. + t.createInodeWithLocalParam("test", true) + t.in.config = &cfg.Config{Write: *getWriteConfig()} + t.createBufferedWriteHandler(true, WriteMode) + } + + if tc.fileType == EmptyGCSFile { + t.createInodeWithEmptyObject() + t.in.config = &cfg.Config{Write: *getWriteConfig()} + } + + if tc.performWrite { + t.createBufferedWriteHandler(tc.fileType != LocalFile, WriteMode) + gcsSynced, err := t.in.Write(t.ctx, []byte("hi"), 0, WriteMode) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) + assert.Equal(t.T(), int64(2), t.in.bwh.WriteFileInfo().TotalSize) + } + data := make([]byte, len("hi")) + // Flush is required before reading an object for which BWH is open. + assert.NoError(t.T(), t.in.Flush(context.Background())) + + n, err := t.in.Read(t.ctx, data, 0) + + if tc.performWrite { + assert.Equal(t.T(), len(data), n) + require.NoError(t.T(), err) + } else { + assert.Equal(t.T(), 0, n) + require.Error(t.T(), err) + assert.ErrorIs(t.T(), err, io.EOF) + } + }) + } +} + +func (t *FileTest) TestReadEmptyGCSFileWhenStreamingWritesAreNotInProgress() { + t.createInodeWithEmptyObject() + t.in.config = &cfg.Config{Write: *getWriteConfig()} data := make([]byte, 10) n, err := t.in.Read(t.ctx, data, 0) - AssertEq(0, n) - AssertNe(nil, err) - AssertEq("cannot read a local file when upload in progress", err.Error()) + assert.Equal(t.T(), 0, n) + require.Error(t.T(), err) + assert.Contains(t.T(), err.Error(), "EOF") } -func (t *FileTest) WriteToLocalFileWithInvalidConfigWhenStreamingWritesAreEnabled() { +func (t *FileTest) TestInitBufferedWriteHandlerWithInvalidConfigWhenStreamingWritesAreEnabled() { // Create a local file inode. t.createInodeWithLocalParam("test", true) - t.in.writeConfig.ExperimentalEnableStreamingWrites = true - AssertEq(nil, t.in.bwh) + t.in.config = &cfg.Config{Write: cfg.WriteConfig{EnableStreamingWrites: true}} - err := t.in.Write(t.ctx, []byte("hi"), 0) + initialized, err := t.in.InitBufferedWriteHandlerIfEligible(t.ctx, WriteMode) - AssertNe(nil, err) - AssertTrue(strings.Contains(err.Error(), "invalid configuration")) + assert.True(t.T(), strings.Contains(err.Error(), "invalid configuration")) + assert.False(t.T(), initialized) + assert.Nil(t.T(), t.in.bwh) } -func (t *FileTest) WriteToLocalFileWhenStreamingWritesAreEnabled() { +func (t *FileTest) TestWriteToLocalFileWhenStreamingWritesAreEnabled() { // Create a local file inode. t.createInodeWithLocalParam("test", true) - t.in.writeConfig = getWriteConfig() - AssertEq(nil, t.in.bwh) + t.in.config = &cfg.Config{Write: *getWriteConfig()} + t.createBufferedWriteHandler(true, WriteMode) - err := t.in.Write(t.ctx, []byte("hi"), 0) + gcsSynced, err := t.in.Write(t.ctx, []byte("hi"), 0, WriteMode) - AssertEq(nil, err) - AssertNe(nil, t.in.bwh) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) + assert.NotNil(t.T(), t.in.bwh) writeFileInfo := t.in.bwh.WriteFileInfo() - AssertEq(2, writeFileInfo.TotalSize) + assert.Equal(t.T(), int64(2), writeFileInfo.TotalSize) } -func (t *FileTest) MultipleWritesToLocalFileWhenStreamingWritesAreEnabled() { +func (t *FileTest) TestMultipleWritesToLocalFileWhenStreamingWritesAreEnabled() { // Create a local file inode. t.createInodeWithLocalParam("test", true) - t.in.writeConfig = getWriteConfig() - AssertEq(nil, t.in.bwh) + createTime := t.in.mtimeClock.Now() + t.in.config = &cfg.Config{Write: *getWriteConfig()} + t.createBufferedWriteHandler(true, WriteMode) + + gcsSynced, err := t.in.Write(t.ctx, []byte("hi"), 0, WriteMode) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) + assert.NotNil(t.T(), t.in.bwh) + assert.Equal(t.T(), int64(2), t.in.bwh.WriteFileInfo().TotalSize) + + gcsSynced, err = t.in.Write(t.ctx, []byte("hello"), 2, WriteMode) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) + assert.Equal(t.T(), int64(7), t.in.bwh.WriteFileInfo().TotalSize) + // The inode should agree about the new mtime. + attrs, err := t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), uint64(7), attrs.Size) + assert.WithinDuration(t.T(), attrs.Mtime, createTime, Delta) +} + +func (t *FileTest) TestWriteToEmptyGCSFileWhenStreamingWritesAreEnabled() { + t.createInodeWithEmptyObject() + t.in.config = &cfg.Config{Write: *getWriteConfig()} + createTime := t.in.mtimeClock.Now() + t.createBufferedWriteHandler(true, WriteMode) + + gcsSynced, err := t.in.Write(t.ctx, []byte("hi"), 0, WriteMode) + + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) + assert.NotNil(t.T(), t.in.bwh) + writeFileInfo := t.in.bwh.WriteFileInfo() + assert.Equal(t.T(), int64(2), writeFileInfo.TotalSize) + // The inode should agree about the new mtime. + attrs, err := t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), uint64(2), attrs.Size) + assert.WithinDuration(t.T(), attrs.Mtime, createTime, Delta) +} + +func (t *FileTest) TestSetMtimeOnEmptyGCSFileWhenStreamingWritesAreEnabled() { + t.createInodeWithEmptyObject() + t.in.config = &cfg.Config{Write: *getWriteConfig()} + assert.Nil(t.T(), t.in.bwh) + + // This test checks if the mtime is updated to GCS. Since test framework + // doesn't support t.run, calling the test method here directly. + t.TestSetMtime_ContentNotFaultedIn() + // bufferedWritesHandler shouldn't get initialized. + assert.Nil(t.T(), t.in.bwh) +} + +func (t *FileTest) TestSetMtimeOnEmptyGCSFileAfterWritesWhenStreamingWritesAreEnabled() { + t.createInodeWithEmptyObject() + t.in.config = &cfg.Config{Write: *getWriteConfig()} + t.createBufferedWriteHandler(true, WriteMode) + // Initiate write call. + gcsSynced, err := t.in.Write(t.ctx, []byte("hi"), 0, WriteMode) + assert.Nil(t.T(), err) + assert.False(t.T(), gcsSynced) + assert.NotNil(t.T(), t.in.bwh) + writeFileInfo := t.in.bwh.WriteFileInfo() + assert.Equal(t.T(), int64(2), writeFileInfo.TotalSize) + + // Set mtime. + mtime := time.Now().UTC().Add(123 * time.Second) + err = t.in.SetMtime(t.ctx, mtime) - err := t.in.Write(t.ctx, []byte("hi"), 0) - AssertEq(nil, err) - AssertNe(nil, t.in.bwh) - AssertEq(2, t.in.bwh.WriteFileInfo().TotalSize) + assert.Nil(t.T(), err) + // The inode should agree about the new mtime. + attrs, err := t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), attrs.Mtime, mtime) + assert.Equal(t.T(), attrs.Ctime, mtime) + assert.Equal(t.T(), attrs.Atime, mtime) +} - err = t.in.Write(t.ctx, []byte("hello"), 0) - AssertEq(nil, err) - AssertEq(7, t.in.bwh.WriteFileInfo().TotalSize) +func (t *FileTest) TestUpdateSize() { + var err error + var attrs fuseops.InodeAttributes + // Check initial size. + attrs, err = t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), uint64(len(t.initialContents)), attrs.Size) + + // Update size. + newSize := uint64(100) + t.in.UpdateSize(newSize) + + // Check new size. + attrs, err = t.in.Attributes(t.ctx, true) + require.NoError(t.T(), err) + assert.Equal(t.T(), newSize, attrs.Size) +} + +func (t *FileTest) TestRegisterFileHandle() { + tbl := []struct { + name string + readonly bool + currentVal int32 + expectedVal int32 + }{ + { + name: "ReadOnlyHandle", + readonly: true, + currentVal: 0, + expectedVal: 0, + }, + { + name: "ZeroCurrentValueForWriteHandle", + readonly: false, + currentVal: 0, + expectedVal: 1, + }, + { + name: "NonZeroCurrentValueForWriteHandle", + readonly: false, + currentVal: 5, + expectedVal: 6, + }, + } + for _, tc := range tbl { + t.Run(tc.name, func() { + t.in.writeHandleCount = tc.currentVal + + t.in.RegisterFileHandle(tc.readonly) + + assert.Equal(t.T(), tc.expectedVal, t.in.writeHandleCount) + }) + } } func getWriteConfig() *cfg.WriteConfig { return &cfg.WriteConfig{ - MaxBlocksPerFile: 10, - BlockSizeMb: 10, - ExperimentalEnableStreamingWrites: true, + MaxBlocksPerFile: 10, + BlockSizeMb: 1, + EnableStreamingWrites: true, + } +} + +func getWriteConfigWithEnabledRapidAppends() *cfg.WriteConfig { + return &cfg.WriteConfig{ + MaxBlocksPerFile: 10, + BlockSizeMb: 1, + EnableStreamingWrites: true, + EnableRapidAppends: true, } } diff --git a/internal/fs/inode/hns_dir_test.go b/internal/fs/inode/hns_dir_test.go index 5d7de3681a..ead0bb93c4 100644 --- a/internal/fs/inode/hns_dir_test.go +++ b/internal/fs/inode/hns_dir_test.go @@ -22,96 +22,174 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/metadata" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/caching" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + storagemock "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/mock" "github.com/jacobsa/fuse/fuseops" "github.com/jacobsa/fuse/fuseutil" "github.com/jacobsa/timeutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "golang.org/x/sync/semaphore" "golang.org/x/net/context" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" ) -type HNSDirTest struct { +type hnsDirTest struct { suite.Suite - ctx context.Context - bucket gcsx.SyncerBucket - in DirInode - mockBucket *storage.TestifyMockBucket - typeCache metadata.TypeCache - fixedTime timeutil.SimulatedClock + ctx context.Context + bucket gcsx.SyncerBucket + in DirInode + mockBucket *storagemock.TestifyMockBucket + typeCache metadata.TypeCache + fixedTime timeutil.SimulatedClock + config *cfg.Config + parInodeCtx context.Context } -func TestHNSDirSuite(testSuite *testing.T) { suite.Run(testSuite, new(HNSDirTest)) } +type HNSDirTest struct { + hnsDirTest +} -func (t *HNSDirTest) SetupTest() { +type NonHNSDirTest struct { + hnsDirTest +} + +func TestHNSDirSuiteWithHierarchicalBucket(testSuite *testing.T) { + suite.Run(testSuite, &HNSDirTest{}) +} + +func TestHNSDirSuiteWithNonHierarchicalBucket(testSuite *testing.T) { + suite.Run(testSuite, &NonHNSDirTest{}) +} + +func (t *hnsDirTest) setupTestSuite(hierarchical bool) { t.ctx = context.Background() - t.mockBucket = new(storage.TestifyMockBucket) + t.mockBucket = new(storagemock.TestifyMockBucket) + t.mockBucket.On("BucketType").Return(gcs.BucketType{Hierarchical: hierarchical}) t.bucket = gcsx.NewSyncerBucket( - 1, + /*appendThreshold=*/ 1, + chunkRetryDeadlineSecs, + chunkTransferTimeoutSecs, ".gcsfuse_tmp/", t.mockBucket) t.resetDirInode(false, false, true) } -func (t *HNSDirTest) resetDirInode(implicitDirs, enableNonexistentTypeCache, enableManagedFoldersListing bool) { +func (t *HNSDirTest) SetupTest() { + t.setupTestSuite(true) +} + +func (t *NonHNSDirTest) SetupTest() { + t.setupTestSuite(false) +} + +func (t *hnsDirTest) resetDirInode(implicitDirs, enableNonexistentTypeCache, enableManagedFoldersListing bool) { t.resetDirInodeWithTypeCacheConfigs(implicitDirs, enableNonexistentTypeCache, enableManagedFoldersListing, 4, typeCacheTTL) } -func (t *HNSDirTest) resetDirInodeWithTypeCacheConfigs(implicitDirs, enableNonexistentTypeCache, enableManagedFoldersListing bool, typeCacheMaxSizeMB int64, typeCacheTTL time.Duration) { +func (t *hnsDirTest) resetDirInodeWithTypeCacheConfigs(implicitDirs, enableNonexistentTypeCache, enableManagedFoldersListing bool, typeCacheMaxSizeMB int64, typeCacheTTL time.Duration) { t.fixedTime.SetTime(time.Date(2024, 7, 22, 2, 15, 0, 0, time.Local)) + t.config = &cfg.Config{ + List: cfg.ListConfig{EnableEmptyManagedFolders: enableManagedFoldersListing}, + MetadataCache: cfg.MetadataCacheConfig{TypeCacheMaxSizeMb: typeCacheMaxSizeMB}, + EnableHns: true, + EnableUnsupportedPathSupport: true, + EnableTypeCacheDeprecation: isTypeCacheDeprecationEnabled, + } + + t.parInodeCtx = context.Background() t.in = NewDirInode( dirInodeID, NewDirName(NewRootName(""), dirInodeName), + t.parInodeCtx, fuseops.InodeAttributes{ Uid: uid, Gid: gid, Mode: dirMode, }, implicitDirs, - enableManagedFoldersListing, enableNonexistentTypeCache, typeCacheTTL, &t.bucket, &t.fixedTime, &t.fixedTime, - typeCacheMaxSizeMB, - true, + semaphore.NewWeighted(10), + t.config, ) d := t.in.(*dirInode) assert.NotNil(t.T(), d) t.typeCache = d.cache - assert.NotNil(t.T(), t.typeCache) + if !d.IsTypeCacheDeprecated() { + assert.NotNil(t.T(), t.typeCache) + } else { + assert.Nil(t.T(), t.typeCache) + } //Lock dir Inode t.in.Lock() } -func (t *HNSDirTest) createDirInode(dirInodeName string) DirInode { +func (t *hnsDirTest) createDirInode(dirInodeName string) DirInode { + return t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, isTypeCacheDeprecationEnabled) +} + +func (t *hnsDirTest) createDirInodeWithTypeCacheDeprecationFlag(dirInodeName string, isTypeCacheDeprecated bool) DirInode { + config := &cfg.Config{ + List: cfg.ListConfig{EnableEmptyManagedFolders: false}, + MetadataCache: cfg.MetadataCacheConfig{ + TypeCacheMaxSizeMb: 4, + TtlSecs: 60, + }, + EnableHns: true, + EnableUnsupportedPathSupport: true, + EnableTypeCacheDeprecation: isTypeCacheDeprecated, + } + + parInode := NewDirInode( + 1, + NewRootName(""), + nil, + fuseops.InodeAttributes{ + Uid: uid, + Gid: gid, + Mode: dirMode, + }, + false, + true, + typeCacheTTL, + &t.bucket, + &t.fixedTime, + &t.fixedTime, + semaphore.NewWeighted(10), + config, + ) + return NewDirInode( 5, NewDirName(NewRootName(""), dirInodeName), + parInode.Context(), fuseops.InodeAttributes{ Uid: uid, Gid: gid, Mode: dirMode, }, false, - false, true, typeCacheTTL, &t.bucket, &t.fixedTime, &t.fixedTime, - 4, - false, + semaphore.NewWeighted(10), + config, ) } @@ -119,6 +197,10 @@ func (t *HNSDirTest) TearDownTest() { t.in.Unlock() } +func (t *NonHNSDirTest) TearDownTest() { + t.in.Unlock() +} + func (t *HNSDirTest) TestShouldFindExplicitHNSFolder() { const name = "qux" dirName := path.Join(dirInodeName, name) + "/" @@ -128,7 +210,7 @@ func (t *HNSDirTest) TestShouldFindExplicitHNSFolder() { t.mockBucket.On("GetFolder", mock.Anything, mock.Anything).Return(folder, nil) // Look up with the name. - result, err := findExplicitFolder(t.ctx, &t.bucket, NewDirName(t.in.Name(), name)) + result, err := findExplicitFolder(t.ctx, &t.bucket, NewDirName(t.in.Name(), name), false) t.mockBucket.AssertExpectations(t.T()) assert.Nil(t.T(), err) @@ -142,7 +224,7 @@ func (t *HNSDirTest) TestShouldReturnNilWhenGCSFolderNotFoundForInHNS() { t.mockBucket.On("GetFolder", mock.Anything, mock.Anything).Return(nil, notFoundErr) // Look up with the name. - result, err := findExplicitFolder(t.ctx, &t.bucket, NewDirName(t.in.Name(), "not-present")) + result, err := findExplicitFolder(t.ctx, &t.bucket, NewDirName(t.in.Name(), "not-present"), false) t.mockBucket.AssertExpectations(t.T()) assert.Nil(t.T(), err) @@ -158,10 +240,12 @@ func (t *HNSDirTest) TestLookUpChildWithConflictMarkerName() { statObjectRequest := gcs.StatObjectRequest{ Name: path.Join(dirInodeName, name), } + getFolderRequest := gcs.GetFolderRequest{ + Name: dirName, + } object := gcs.MinObject{Name: dirName} - t.mockBucket.On("GetFolder", mock.Anything, dirName).Return(folder, nil) + t.mockBucket.On("GetFolder", mock.Anything, &getFolderRequest).Return(folder, nil) t.mockBucket.On("StatObject", mock.Anything, &statObjectRequest).Return(&object, &gcs.ExtendedObjectAttributes{}, nil) - t.mockBucket.On("BucketType").Return(gcs.Hierarchical) c, err := t.in.LookUpChild(t.ctx, name+"\n") @@ -178,8 +262,9 @@ func (t *HNSDirTest) TestLookUpChildShouldCheckOnlyForExplicitHNSDirectory() { Name: dirName, } t.mockBucket.On("GetFolder", mock.Anything, mock.Anything).Return(folder, nil) - t.mockBucket.On("BucketType").Return(gcs.Hierarchical) - t.typeCache.Insert(t.fixedTime.Now().Add(time.Minute), name, metadata.ExplicitDirType) + if !t.in.IsTypeCacheDeprecated() { + t.typeCache.Insert(t.fixedTime.Now().Add(time.Minute), name, metadata.ExplicitDirType) + } // Look up with the proper name. result, err := t.in.LookUpChild(t.ctx, name) @@ -188,7 +273,9 @@ func (t *HNSDirTest) TestLookUpChildShouldCheckOnlyForExplicitHNSDirectory() { assert.Nil(t.T(), err) assert.Equal(t.T(), dirName, result.FullName.GcsObjectName()) assert.Equal(t.T(), dirName, result.Folder.Name) - assert.Equal(t.T(), metadata.ExplicitDirType, t.typeCache.Get(t.fixedTime.Now(), name)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.typeCache.Get(t.fixedTime.Now(), name)) + } } func (t *HNSDirTest) TestLookUpChildShouldCheckForHNSDirectoryWhenTypeNotPresent() { @@ -201,8 +288,9 @@ func (t *HNSDirTest) TestLookUpChildShouldCheckForHNSDirectoryWhenTypeNotPresent t.mockBucket.On("GetFolder", mock.Anything, mock.Anything).Return(folder, nil) notFoundErr := &gcs.NotFoundError{Err: errors.New("storage: object doesn't exist")} t.mockBucket.On("StatObject", mock.Anything, mock.Anything).Return(nil, nil, notFoundErr) - t.mockBucket.On("BucketType").Return(gcs.Hierarchical) - assert.Equal(t.T(), metadata.UnknownType, t.typeCache.Get(t.fixedTime.Now(), name)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.UnknownType, t.typeCache.Get(t.fixedTime.Now(), name)) + } // Look up with the proper name. result, err := t.in.LookUpChild(t.ctx, name) @@ -210,7 +298,9 @@ func (t *HNSDirTest) TestLookUpChildShouldCheckForHNSDirectoryWhenTypeNotPresent assert.Nil(t.T(), err) assert.Equal(t.T(), dirName, result.FullName.GcsObjectName()) assert.Equal(t.T(), dirName, result.Folder.Name) - assert.Equal(t.T(), metadata.ExplicitDirType, t.typeCache.Get(t.fixedTime.Now(), name)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.typeCache.Get(t.fixedTime.Now(), name)) + } } func (t *HNSDirTest) TestLookUpChildShouldCheckForHNSDirectoryWhenTypeIsRegularFileType() { @@ -228,7 +318,9 @@ func (t *HNSDirTest) TestLookUpChildShouldCheckForHNSDirectoryWhenTypeIsRegularF CacheControl: "some-value", } t.mockBucket.On("StatObject", mock.Anything, mock.Anything).Return(minObject, attrs, nil) - t.typeCache.Insert(t.fixedTime.Now().Add(time.Minute), name, metadata.RegularFileType) + if !t.in.IsTypeCacheDeprecated() { + t.typeCache.Insert(t.fixedTime.Now().Add(time.Minute), name, metadata.RegularFileType) + } // Look up with the proper name. result, err := t.in.LookUpChild(t.ctx, name) @@ -238,7 +330,9 @@ func (t *HNSDirTest) TestLookUpChildShouldCheckForHNSDirectoryWhenTypeIsRegularF assert.Equal(t.T(), fileName, result.MinObject.Name) assert.Equal(t.T(), int64(2), result.MinObject.Generation) assert.Equal(t.T(), int64(1), result.MinObject.MetaGeneration) - assert.Equal(t.T(), metadata.RegularFileType, t.typeCache.Get(t.fixedTime.Now(), name)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.RegularFileType, t.typeCache.Get(t.fixedTime.Now(), name)) + } } func (t *HNSDirTest) TestLookUpChildShouldCheckForHNSDirectoryWhenTypeIsSymlinkType() { @@ -257,8 +351,9 @@ func (t *HNSDirTest) TestLookUpChildShouldCheckForHNSDirectoryWhenTypeIsSymlinkT CacheControl: "some-value", } t.mockBucket.On("StatObject", mock.Anything, mock.Anything).Return(minObject, attrs, nil) - t.mockBucket.On("BucketType").Return(gcs.Hierarchical) - t.typeCache.Insert(t.fixedTime.Now().Add(time.Minute), name, metadata.SymlinkType) + if !t.in.IsTypeCacheDeprecated() { + t.typeCache.Insert(t.fixedTime.Now().Add(time.Minute), name, metadata.SymlinkType) + } // Look up with the proper name. result, err := t.in.LookUpChild(t.ctx, name) @@ -267,12 +362,16 @@ func (t *HNSDirTest) TestLookUpChildShouldCheckForHNSDirectoryWhenTypeIsSymlinkT assert.Equal(t.T(), fileName, result.MinObject.Name) assert.Equal(t.T(), int64(2), result.MinObject.Generation) assert.Equal(t.T(), int64(1), result.MinObject.MetaGeneration) - assert.Equal(t.T(), metadata.SymlinkType, t.typeCache.Get(t.fixedTime.Now(), name)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.SymlinkType, t.typeCache.Get(t.fixedTime.Now(), name)) + } } func (t *HNSDirTest) TestLookUpChildShouldCheckForHNSDirectoryWhenTypeIsNonExistentType() { const name = "file_type" - t.typeCache.Insert(t.fixedTime.Now().Add(time.Minute), name, metadata.NonexistentType) + if !t.in.IsTypeCacheDeprecated() { + t.typeCache.Insert(t.fixedTime.Now().Add(time.Minute), name, metadata.NonexistentType) + } // Look up with the proper name. result, err := t.in.LookUpChild(t.ctx, name) @@ -286,13 +385,31 @@ func (t *HNSDirTest) TestRenameFolderWithGivenName() { dirName = "qux" renameDirName = "rename" ) + folderInode := NewDirInode( + 7, + NewDirName(NewRootName(""), path.Join(dirInodeName, dirName)), + nil, + fuseops.InodeAttributes{ + Uid: uid, + Gid: gid, + Mode: dirMode, + }, + false, + true, + typeCacheTTL, + &t.bucket, + &t.fixedTime, + &t.fixedTime, + semaphore.NewWeighted(10), + t.config, + ) folderName := path.Join(dirInodeName, dirName) + "/" renameFolderName := path.Join(dirInodeName, renameDirName) + "/" renameFolder := gcs.Folder{Name: renameFolderName} t.mockBucket.On("RenameFolder", t.ctx, folderName, renameFolderName).Return(&renameFolder, nil) // Attempt to rename the folder. - f, err := t.in.RenameFolder(t.ctx, folderName, renameFolderName) + f, err := t.in.RenameFolder(t.ctx, folderName, renameFolderName, folderInode) t.mockBucket.AssertExpectations(t.T()) assert.NoError(t.T(), err) @@ -307,19 +424,90 @@ func (t *HNSDirTest) TestRenameFolderWithNonExistentSourceFolder() { dirName = "qux" renameDirName = "rename" ) + folderInode := NewDirInode( + 7, + NewDirName(NewRootName(""), path.Join(dirInodeName, dirName)), + nil, + fuseops.InodeAttributes{ + Uid: uid, + Gid: gid, + Mode: dirMode, + }, + false, + true, + typeCacheTTL, + &t.bucket, + &t.fixedTime, + &t.fixedTime, + semaphore.NewWeighted(10), + t.config, + ) folderName := path.Join(dirInodeName, dirName) + "/" renameFolderName := path.Join(dirInodeName, renameDirName) + "/" t.mockBucket.On("RenameFolder", t.ctx, folderName, renameFolderName).Return(nil, &gcs.NotFoundError{}) // Attempt to rename the folder. - f, err := t.in.RenameFolder(t.ctx, folderName, renameFolderName) + f, err := t.in.RenameFolder(t.ctx, folderName, renameFolderName, folderInode) t.mockBucket.AssertExpectations(t.T()) assert.True(t.T(), errors.As(err, ¬FoundErr)) assert.Nil(t.T(), f) } -func (t *HNSDirTest) TestDeleteChildDir_WhenImplicitDirFlagTrueOnNonHNSBucket() { +func (t *HNSDirTest) TestRenameFileWithGivenName() { + const ( + fileName = "qux" + renameFileName = "rename" + ) + oldObjName := path.Join(dirInodeName, fileName) + newObjName := path.Join(dirInodeName, renameFileName) + var metaGeneration int64 = 0 + moveObjectReq := gcs.MoveObjectRequest{ + SrcName: oldObjName, + DstName: newObjName, + SrcGeneration: 0, + SrcMetaGenerationPrecondition: &metaGeneration, + } + oldObj := gcs.MinObject{Name: oldObjName} + newObj := gcs.Object{Name: newObjName} + t.mockBucket.On("MoveObject", t.ctx, &moveObjectReq).Return(&newObj, nil) + + // Attempt to rename the file. + f, err := t.in.RenameFile(t.ctx, &oldObj, path.Join(dirInodeName, renameFileName)) + + t.mockBucket.AssertExpectations(t.T()) + // Verify the renamed file exists. + assert.NoError(t.T(), err) + assert.Equal(t.T(), newObjName, f.Name) +} + +func (t *HNSDirTest) TestRenameFileWithNonExistentSourceFile() { + const ( + fileName = "qux" + renameFileName = "rename" + ) + oldObjName := path.Join(dirInodeName, fileName) + newObjName := path.Join(dirInodeName, renameFileName) + var metaGeneration int64 = 0 + moveObjectReq := gcs.MoveObjectRequest{ + SrcName: oldObjName, + DstName: newObjName, + SrcGeneration: 0, + SrcMetaGenerationPrecondition: &metaGeneration, + } + oldObj := gcs.MinObject{Name: oldObjName} + var notFoundErr *gcs.NotFoundError + t.mockBucket.On("MoveObject", t.ctx, &moveObjectReq).Return(nil, &gcs.NotFoundError{}) + + // Attempt to rename the file. + f, err := t.in.RenameFile(t.ctx, &oldObj, newObjName) + + t.mockBucket.AssertExpectations(t.T()) + assert.True(t.T(), errors.As(err, ¬FoundErr)) + assert.Nil(t.T(), f) +} + +func (t *NonHNSDirTest) TestDeleteChildDir_WhenImplicitDirFlagTrueOnNonHNSBucket() { const folderName = "folder" dirName := path.Join(dirInodeName, folderName) + "/" dirIn := t.createDirInode(dirName) @@ -331,14 +519,13 @@ func (t *HNSDirTest) TestDeleteChildDir_WhenImplicitDirFlagTrueOnNonHNSBucket() assert.NoError(t.T(), err) // Ensure no error occurred } -func (t *HNSDirTest) TestDeleteChildDir_WhenImplicitDirFlagFalseAndNonHNSBucket_DeleteObjectGiveSuccess() { +func (t *NonHNSDirTest) TestDeleteChildDir_WhenImplicitDirFlagFalseAndNonHNSBucket_DeleteObjectGiveSuccess() { const name = "dir" dirName := path.Join(dirInodeName, name) + "/" deleteObjectReq := gcs.DeleteObjectRequest{ Name: dirName, Generation: 0, } - t.mockBucket.On("BucketType").Return(gcs.NonHierarchical) t.mockBucket.On("DeleteObject", t.ctx, &deleteObjectReq).Return(nil) dirIn := t.createDirInode(dirName) @@ -346,17 +533,18 @@ func (t *HNSDirTest) TestDeleteChildDir_WhenImplicitDirFlagFalseAndNonHNSBucket_ t.mockBucket.AssertExpectations(t.T()) assert.NoError(t.T(), err) - assert.Equal(t.T(), metadata.Type(0), t.typeCache.Get(t.fixedTime.Now(), dirName)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.Type(0), t.typeCache.Get(t.fixedTime.Now(), dirName)) + } assert.False(t.T(), dirIn.IsUnlinked()) } -func (t *HNSDirTest) TestDeleteChildDir_WithImplicitDirFlagFalseAndNonHNSBucket_DeleteObjectThrowAnError() { +func (t *NonHNSDirTest) TestDeleteChildDir_WithImplicitDirFlagFalseAndNonHNSBucket_DeleteObjectThrowAnError() { const name = "folder" dirName := path.Join(dirInodeName, name) + "/" deleteObjectReq := gcs.DeleteObjectRequest{ Name: dirName, Generation: 0, } - t.mockBucket.On("BucketType").Return(gcs.NonHierarchical) t.mockBucket.On("DeleteObject", t.ctx, &deleteObjectReq).Return(fmt.Errorf("mock error")) dirIn := t.createDirInode(dirName) @@ -375,7 +563,6 @@ func (t *HNSDirTest) TestDeleteChildDir_WithImplicitDirFlagFalseAndBucketTypeIsH Name: dirName, Generation: 0, } - t.mockBucket.On("BucketType").Return(gcs.Hierarchical) t.mockBucket.On("DeleteObject", t.ctx, &deleteObjectReq).Return(nil) t.mockBucket.On("DeleteFolder", t.ctx, dirName).Return(fmt.Errorf("mock error")) dirIn := t.createDirInode(dirName) @@ -395,7 +582,6 @@ func (t *HNSDirTest) TestDeleteChildDir_WithImplicitDirFlagFalseAndBucketTypeIsH Name: dirName, Generation: 0, } - t.mockBucket.On("BucketType").Return(gcs.Hierarchical) t.mockBucket.On("DeleteObject", t.ctx, &deleteObjectReq).Return(fmt.Errorf("mock error")) t.mockBucket.On("DeleteFolder", t.ctx, dirName).Return(nil) dirIn := t.createDirInode(dirName) @@ -405,7 +591,9 @@ func (t *HNSDirTest) TestDeleteChildDir_WithImplicitDirFlagFalseAndBucketTypeIsH t.mockBucket.AssertExpectations(t.T()) assert.NoError(t.T(), err) - assert.Equal(t.T(), metadata.Type(0), t.typeCache.Get(t.fixedTime.Now(), dirName)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.Type(0), t.typeCache.Get(t.fixedTime.Now(), dirName)) + } assert.True(t.T(), dirIn.IsUnlinked()) } @@ -416,7 +604,6 @@ func (t *HNSDirTest) TestDeleteChildDir_WithImplicitDirFlagFalseAndBucketTypeIsH Name: dirName, Generation: 0, } - t.mockBucket.On("BucketType").Return(gcs.Hierarchical) t.mockBucket.On("DeleteObject", t.ctx, &deleteObjectReq).Return(fmt.Errorf("mock error")) t.mockBucket.On("DeleteFolder", t.ctx, dirName).Return(fmt.Errorf("mock delete folder error")) dirIn := t.createDirInode(dirName) @@ -434,7 +621,6 @@ func (t *HNSDirTest) TestDeleteChildDir_WithImplicitDirFlagFalseAndBucketTypeIsH func (t *HNSDirTest) TestCreateChildDirWhenBucketTypeIsHNSWithFailure() { const name = "folder" dirName := path.Join(dirInodeName, name) + "/" - t.mockBucket.On("BucketType").Return(gcs.Hierarchical) t.mockBucket.On("CreateFolder", t.ctx, dirName).Return(nil, fmt.Errorf("mock error")) result, err := t.in.CreateChildDir(t.ctx, name) @@ -442,14 +628,15 @@ func (t *HNSDirTest) TestCreateChildDirWhenBucketTypeIsHNSWithFailure() { t.mockBucket.AssertExpectations(t.T()) assert.NotNil(t.T(), err) assert.Nil(t.T(), result) - assert.Equal(t.T(), metadata.Type(0), t.typeCache.Get(t.fixedTime.Now(), dirName)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.Type(0), t.typeCache.Get(t.fixedTime.Now(), dirName)) + } } func (t *HNSDirTest) TestCreateChildDirWhenBucketTypeIsHNSWithSuccess() { const name = "folder" dirName := path.Join(dirInodeName, name) + "/" folder := gcs.Folder{Name: dirName} - t.mockBucket.On("BucketType").Return(gcs.Hierarchical) t.mockBucket.On("CreateFolder", t.ctx, dirName).Return(&folder, nil) result, err := t.in.CreateChildDir(t.ctx, name) @@ -459,15 +646,16 @@ func (t *HNSDirTest) TestCreateChildDirWhenBucketTypeIsHNSWithSuccess() { assert.NotNil(t.T(), result) assert.Equal(t.T(), dirName, result.Folder.Name) assert.Equal(t.T(), dirName, result.FullName.objectName) - assert.Equal(t.T(), metadata.ExplicitDirType, t.typeCache.Get(t.fixedTime.Now(), name)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.typeCache.Get(t.fixedTime.Now(), name)) + } } -func (t *HNSDirTest) TestCreateChildDirWhenBucketTypeIsNonHNSWithFailure() { +func (t *NonHNSDirTest) TestCreateChildDirWhenBucketTypeIsNonHNSWithFailure() { const name = "folder" var preCond int64 dirName := path.Join(dirInodeName, name) + "/" createObjectReq := gcs.CreateObjectRequest{Name: dirName, Contents: strings.NewReader(""), GenerationPrecondition: &preCond} - t.mockBucket.On("BucketType").Return(gcs.NonHierarchical) t.mockBucket.On("CreateObject", t.ctx, &createObjectReq).Return(nil, fmt.Errorf("mock error")) result, err := t.in.CreateChildDir(t.ctx, name) @@ -475,16 +663,17 @@ func (t *HNSDirTest) TestCreateChildDirWhenBucketTypeIsNonHNSWithFailure() { t.mockBucket.AssertExpectations(t.T()) assert.NotNil(t.T(), err) assert.Nil(t.T(), result) - assert.Equal(t.T(), metadata.Type(0), t.typeCache.Get(t.fixedTime.Now(), dirName)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.Type(0), t.typeCache.Get(t.fixedTime.Now(), dirName)) + } } -func (t *HNSDirTest) TestCreateChildDirWhenBucketTypeIsNonHNSWithSuccess() { +func (t *NonHNSDirTest) TestCreateChildDirWhenBucketTypeIsNonHNSWithSuccess() { const name = "folder" dirName := path.Join(dirInodeName, name) + "/" var preCond int64 createObjectReq := gcs.CreateObjectRequest{Name: dirName, Contents: strings.NewReader(""), GenerationPrecondition: &preCond} object := gcs.Object{Name: dirName} - t.mockBucket.On("BucketType").Return(gcs.NonHierarchical) t.mockBucket.On("CreateObject", t.ctx, &createObjectReq).Return(&object, nil) result, err := t.in.CreateChildDir(t.ctx, name) @@ -494,7 +683,52 @@ func (t *HNSDirTest) TestCreateChildDirWhenBucketTypeIsNonHNSWithSuccess() { assert.NotNil(t.T(), result) assert.Equal(t.T(), dirName, result.MinObject.Name) assert.Equal(t.T(), dirName, result.FullName.objectName) - assert.Equal(t.T(), metadata.ExplicitDirType, t.typeCache.Get(t.fixedTime.Now(), name)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.typeCache.Get(t.fixedTime.Now(), name)) + } +} + +func (t *HNSDirTest) TestDeleteObjects() { + // Arrange + objectNames := []string{"dir1/file1.txt", "dir2/"} + t.mockBucket.On("DeleteObject", t.ctx, &gcs.DeleteObjectRequest{Name: "dir1/file1.txt"}).Return(nil) + // Mock for recursive deletion of dir2/ + listReq := &gcs.ListObjectsRequest{ + Prefix: "dir2/", + MaxResults: MaxResultsForListObjectsCall, + Delimiter: "/", + ContinuationToken: "", + IncludeFoldersAsPrefixes: true, + } + listResp := &gcs.Listing{ + MinObjects: []*gcs.MinObject{ + {Name: "dir2/file2.txt"}, + }, + CollapsedRuns: []string{"dir2/subdir/"}, + } + t.mockBucket.On("ListObjects", mock.Anything, listReq).Return(listResp, nil) + t.mockBucket.On("DeleteObject", mock.Anything, &gcs.DeleteObjectRequest{Name: "dir2/file2.txt"}).Return(nil) + // Mock for recursive call on subdir/ + listReqSubdir := &gcs.ListObjectsRequest{ + Prefix: "dir2/subdir/", + MaxResults: MaxResultsForListObjectsCall, + Delimiter: "/", + ContinuationToken: "", + IncludeFoldersAsPrefixes: true, + } + listRespSubdir := &gcs.Listing{} + t.mockBucket.On("ListObjects", mock.Anything, listReqSubdir).Return(listRespSubdir, nil) + t.mockBucket.On("DeleteObject", mock.Anything, &gcs.DeleteObjectRequest{Name: "dir2/subdir/"}).Return(nil) + t.mockBucket.On("DeleteFolder", mock.Anything, "dir2/subdir/").Return(nil) + t.mockBucket.On("DeleteObject", mock.Anything, &gcs.DeleteObjectRequest{Name: "dir2/"}).Return(nil) + t.mockBucket.On("DeleteFolder", mock.Anything, "dir2/").Return(nil) + + // Act + err := t.in.DeleteObjects(t.ctx, objectNames) + + // Assert + assert.NoError(t.T(), err) + t.mockBucket.AssertExpectations(t.T()) } func (t *HNSDirTest) TestReadEntriesInHierarchicalBucket() { @@ -523,44 +757,230 @@ func (t *HNSDirTest) TestReadEntriesInHierarchicalBucket() { Prefix: dirInodeName, Delimiter: "/", IncludeFoldersAsPrefixes: true, - IncludeTrailingDelimiter: true, + IncludeTrailingDelimiter: false, MaxResults: 5000, ProjectionVal: gcs.NoAcl, } - t.mockBucket.On("BucketType").Return(gcs.Hierarchical) t.mockBucket.On("ListObjects", t.ctx, &listObjectReq).Return(&listing, nil) - entries, _, err := t.in.ReadEntries(t.ctx, tok) + entries, _, _, err := t.in.ReadEntries(t.ctx, tok) t.mockBucket.AssertExpectations(t.T()) assert.NoError(t.T(), err) assert.Equal(t.T(), 6, len(entries)) - for i := 0; i < 6; i++ { + for i := range 6 { switch entries[i].Name { case folder1: assert.Equal(t.T(), folder1, entries[i].Name) assert.Equal(t.T(), fuseutil.DT_Directory, entries[i].Type) - assert.Equal(t.T(), metadata.ExplicitDirType, t.typeCache.Get(t.in.(*dirInode).cacheClock.Now(), folder1)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.typeCache.Get(t.in.(*dirInode).cacheClock.Now(), folder1)) + } case folder2: assert.Equal(t.T(), folder2, entries[i].Name) assert.Equal(t.T(), fuseutil.DT_Directory, entries[i].Type) - assert.Equal(t.T(), metadata.ExplicitDirType, t.typeCache.Get(t.in.(*dirInode).cacheClock.Now(), folder2)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.typeCache.Get(t.in.(*dirInode).cacheClock.Now(), folder2)) + } case implicitDir: assert.Equal(t.T(), implicitDir, entries[i].Name) assert.Equal(t.T(), fuseutil.DT_Directory, entries[i].Type) - assert.Equal(t.T(), metadata.ExplicitDirType, t.typeCache.Get(t.in.(*dirInode).cacheClock.Now(), implicitDir)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.ExplicitDirType, t.typeCache.Get(t.in.(*dirInode).cacheClock.Now(), implicitDir)) + } case file1: assert.Equal(t.T(), file1, entries[i].Name) assert.Equal(t.T(), fuseutil.DT_File, entries[i].Type) - assert.Equal(t.T(), metadata.RegularFileType, t.typeCache.Get(t.in.(*dirInode).cacheClock.Now(), file1)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.RegularFileType, t.typeCache.Get(t.in.(*dirInode).cacheClock.Now(), file1)) + } case file2: assert.Equal(t.T(), file2, entries[i].Name) assert.Equal(t.T(), fuseutil.DT_File, entries[i].Type) - assert.Equal(t.T(), metadata.RegularFileType, t.typeCache.Get(t.in.(*dirInode).cacheClock.Now(), file2)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.RegularFileType, t.typeCache.Get(t.in.(*dirInode).cacheClock.Now(), file2)) + } case file3: assert.Equal(t.T(), file3, entries[i].Name) assert.Equal(t.T(), fuseutil.DT_File, entries[i].Type) - assert.Equal(t.T(), metadata.RegularFileType, t.typeCache.Get(t.in.(*dirInode).cacheClock.Now(), file3)) + if !t.in.IsTypeCacheDeprecated() { + assert.Equal(t.T(), metadata.RegularFileType, t.typeCache.Get(t.in.(*dirInode).cacheClock.Now(), file3)) + } } } } + +func (t *NonHNSDirTest) TestDeleteChildDir_TypeCacheDeprecated() { + testCases := []struct { + name string + isImplicitDir bool + onlyDeleteFromCache bool + }{ + { + name: "ImplicitDir", + isImplicitDir: true, + onlyDeleteFromCache: true, + }, + { + name: "ExplicitDir", + isImplicitDir: false, + onlyDeleteFromCache: false, + }, + } + + for _, tc := range testCases { + t.T().Run(tc.name, func(st *testing.T) { + // Enable type cache deprecation + t.config = &cfg.Config{ + EnableTypeCacheDeprecation: true, + } + dirInode := NewDirInode( + dirInodeID, + NewDirName(NewRootName(""), dirInodeName), + t.parInodeCtx, + fuseops.InodeAttributes{ + Uid: uid, + Gid: gid, + Mode: dirMode, + }, + true, // implicitDirs + false, // enableNonexistentTypeCache + typeCacheTTL, + &t.bucket, + &t.fixedTime, + &t.fixedTime, + semaphore.NewWeighted(10), + t.config, + ) + dirName := path.Join(dirInodeName, tc.name) + "/" + // Expectation: DeleteObject called with OnlyDeleteFromCache + expectedReq := &gcs.DeleteObjectRequest{ + Name: dirName, + Generation: 0, + OnlyDeleteFromCache: tc.onlyDeleteFromCache, + } + t.mockBucket.On("DeleteObject", t.ctx, expectedReq).Return(nil) + + err := dirInode.DeleteChildDir(t.ctx, tc.name, tc.isImplicitDir, nil) + + assert.NoError(st, err) + t.mockBucket.AssertExpectations(st) + }) + } +} + +func (t *HNSDirTest) TestLookUpChild_TypeCacheDeprecated_File() { + t.in.Unlock() + t.in = t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, true) + t.in.Lock() + const name = "file" + objName := path.Join(dirInodeName, name) + minObject := &gcs.MinObject{ + Name: objName, + MetaGeneration: int64(1), + Generation: int64(2), + } + attrs := &gcs.ExtendedObjectAttributes{} + // Mock StatObject for file lookup + t.mockBucket.On("StatObject", mock.Anything, mock.Anything).Return(minObject, attrs, nil) + // Mock GetFolder for dir lookup (should return not found or nil) + notFoundErr := &gcs.NotFoundError{Err: errors.New("not found")} + t.mockBucket.On("GetFolder", mock.Anything, mock.Anything).Return(nil, notFoundErr) + + entry, err := t.in.LookUpChild(t.ctx, name) + + require.NoError(t.T(), err) + require.NotNil(t.T(), entry) + assert.Equal(t.T(), objName, entry.FullName.GcsObjectName()) + assert.Equal(t.T(), metadata.RegularFileType, entry.Type()) +} + +func (t *HNSDirTest) TestLookUpChild_TypeCacheDeprecated_Folder() { + t.in.Unlock() + t.in = t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, true) + t.in.Lock() + const name = "folder" + folderName := path.Join(dirInodeName, name) + "/" + folder := &gcs.Folder{Name: folderName} + // Mock GetFolder for dir lookup + t.mockBucket.On("GetFolder", mock.Anything, mock.Anything).Return(folder, nil) + // Mock StatObject for file lookup (should return not found) + notFoundErr := &gcs.NotFoundError{Err: errors.New("not found")} + t.mockBucket.On("StatObject", mock.Anything, mock.Anything).Return(nil, nil, notFoundErr) + + entry, err := t.in.LookUpChild(t.ctx, name) + + require.NoError(t.T(), err) + require.NotNil(t.T(), entry) + assert.Equal(t.T(), folderName, entry.FullName.GcsObjectName()) + assert.Equal(t.T(), metadata.ExplicitDirType, entry.Type()) +} + +func (t *HNSDirTest) TestLookUpChild_TypeCacheDeprecated_CacheMiss() { + in := t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, true) + const name = "file" + objName := path.Join(dirInodeName, name) + dirObjName := objName + "/" + cacheMissErr := &caching.CacheMissError{} + // Expect cache lookup for file -> CacheMiss + t.mockBucket.On("StatObject", mock.Anything, mock.MatchedBy(func(req *gcs.StatObjectRequest) bool { + return req.Name == objName && req.FetchOnlyFromCache == true + })).Return(nil, nil, cacheMissErr).Once() + // Expect cache lookup for dir -> CacheMiss + t.mockBucket.On("GetFolder", mock.Anything, mock.MatchedBy(func(req *gcs.GetFolderRequest) bool { + return req.Name == dirObjName && req.FetchOnlyFromCache == true + })).Return(nil, cacheMissErr).Once() + // Expect actual lookup for file -> Success + minObject := &gcs.MinObject{ + Name: objName, + Generation: 1, + MetaGeneration: 1, + Size: 100, + } + t.mockBucket.On("StatObject", mock.Anything, mock.MatchedBy(func(req *gcs.StatObjectRequest) bool { + return req.Name == objName && req.FetchOnlyFromCache == false + })).Return(minObject, &gcs.ExtendedObjectAttributes{}, nil).Once() + // Expect actual lookup for dir -> NotFound + t.mockBucket.On("GetFolder", mock.Anything, mock.MatchedBy(func(req *gcs.GetFolderRequest) bool { + return req.Name == dirObjName && req.FetchOnlyFromCache == false + })).Return(nil, &gcs.NotFoundError{}).Once() + + in.Lock() + entry, err := in.LookUpChild(t.ctx, name) + in.Unlock() + + require.NoError(t.T(), err) + require.NotNil(t.T(), entry) + assert.Equal(t.T(), objName, entry.FullName.GcsObjectName()) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *HNSDirTest) TestLookUpChild_TypeCacheDeprecated_CacheHit() { + in := t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, true) + const name = "file" + objName := path.Join(dirInodeName, name) + dirObjName := objName + "/" + // Expect cache lookup for file -> Success + minObject := &gcs.MinObject{ + Name: objName, + Generation: 1, + MetaGeneration: 1, + Size: 100, + } + t.mockBucket.On("StatObject", mock.Anything, mock.MatchedBy(func(req *gcs.StatObjectRequest) bool { + return req.Name == objName && req.FetchOnlyFromCache == true + })).Return(minObject, &gcs.ExtendedObjectAttributes{}, nil).Once() + // Expect cache lookup for dir -> NotFound (nil, nil) + t.mockBucket.On("GetFolder", mock.Anything, mock.MatchedBy(func(req *gcs.GetFolderRequest) bool { + return req.Name == dirObjName && req.FetchOnlyFromCache == true + })).Return(nil, &gcs.NotFoundError{}).Once() + + in.Lock() + entry, err := in.LookUpChild(t.ctx, name) + in.Unlock() + + require.NoError(t.T(), err) + require.NotNil(t.T(), entry) + assert.Equal(t.T(), objName, entry.FullName.GcsObjectName()) + t.mockBucket.AssertExpectations(t.T()) +} diff --git a/internal/fs/inode/inode.go b/internal/fs/inode/inode.go index db5bcd6320..b700d53e3e 100644 --- a/internal/fs/inode/inode.go +++ b/internal/fs/inode/inode.go @@ -17,7 +17,7 @@ package inode import ( "sync" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" "github.com/jacobsa/fuse/fuseops" "golang.org/x/net/context" ) @@ -41,7 +41,10 @@ type Inode interface { IncrementLookupCount() // Return up to date attributes for this inode. - Attributes(ctx context.Context) (fuseops.InodeAttributes, error) + // The `clobberedCheck` parameter controls whether this function performs a + // remote check to see if the backing GCS object has been modified by another + // process. + Attributes(ctx context.Context, clobberedCheck bool) (fuseops.InodeAttributes, error) // Decrement the lookup count for the inode by the given amount. // @@ -53,8 +56,14 @@ type Inode interface { // Clean up any local resources used by the inode, putting it into an // indeterminate state where no method should be called except Unlock. // + // Update the size of the inode. + UpdateSize(size uint64) + // This method may block. Errors are for logging purposes only. Destroy() (err error) + + // Unlink operation marks the inode as unlinked/deleted. + Unlink() } // An inode owned by a gcs bucket. @@ -81,28 +90,41 @@ type GenerationBackedInode interface { type Generation struct { Object int64 Metadata int64 + Size uint64 } -// Compare returns -1, 0, or 1 according to whether g is less than, equal to, or greater -// than other. -func (g Generation) Compare(other Generation) int { +// Compare returns -1, 0, or 1 according to whether src is less than, equal to, +// or greater than existing. It returns 2 if only the size is greater. +// Note: Ordering matters here, latest represents the object fetched from GCS +// and current represents inode cached object's generation. +func (latest Generation) Compare(current Generation) int { // Compare first on object generation number. switch { - case g.Object < other.Object: + case latest.Object < current.Object: return -1 - case g.Object > other.Object: + case latest.Object > current.Object: return 1 } // Break ties on meta-generation. switch { - case g.Metadata < other.Metadata: + case latest.Metadata < current.Metadata: return -1 - case g.Metadata > other.Metadata: + case latest.Metadata > current.Metadata: return 1 } + // Break ties on object size. + // Because objects in zonal buckets can be appended without altering their + // generation or metageneration, the following case applies exclusively to + // zonal buckets. + if latest.Size > current.Size { + return 2 + } + // We ignore latest.Size < current.Size case as little staleness is expected + // on the GCS object's size. + return 0 } diff --git a/internal/fs/inode/inode_test.go b/internal/fs/inode/inode_test.go new file mode 100644 index 0000000000..bdfd56f5ec --- /dev/null +++ b/internal/fs/inode/inode_test.go @@ -0,0 +1,87 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inode_test + +import ( + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/inode" + "github.com/stretchr/testify/assert" +) + +func TestGenerationCompare(t *testing.T) { + testCases := []struct { + name string + latest inode.Generation + current inode.Generation + expected int + }{ + { + name: "latest.Object > current.Object", + latest: inode.Generation{Object: 2, Metadata: 1, Size: 100}, + current: inode.Generation{Object: 1, Metadata: 1, Size: 100}, + expected: 1, + }, + { + name: "latest.Object < current.Object", + latest: inode.Generation{Object: 1, Metadata: 1, Size: 100}, + current: inode.Generation{Object: 2, Metadata: 1, Size: 100}, + expected: -1, + }, + { + name: "latest.Object == current.Object, latest.Metadata < current.Metadata", + latest: inode.Generation{Object: 1, Metadata: 1, Size: 100}, + current: inode.Generation{Object: 1, Metadata: 2, Size: 100}, + expected: -1, + }, + { + name: "latest.Object == current.Object, latest.Metadata > current.Metadata", + latest: inode.Generation{Object: 1, Metadata: 2, Size: 100}, + current: inode.Generation{Object: 1, Metadata: 1, Size: 100}, + expected: 1, + }, + { + name: "latest.Object == current.Object, latest.Metadata < current.Metadata", + latest: inode.Generation{Object: 1, Metadata: 1, Size: 100}, + current: inode.Generation{Object: 1, Metadata: 2, Size: 100}, + expected: -1, + }, + { + name: "latest.Object == current.Object, latest.Metadata == current.Metadata, latest.Size > current.Size", + latest: inode.Generation{Object: 1, Metadata: 1, Size: 200}, + current: inode.Generation{Object: 1, Metadata: 1, Size: 100}, + expected: 2, + }, + { + name: "latest.Object == current.Object, latest.Metadata == current.Metadata, latest.Size < current.Size", + latest: inode.Generation{Object: 1, Metadata: 1, Size: 100}, + current: inode.Generation{Object: 1, Metadata: 1, Size: 200}, + expected: 0, + }, + { + name: "latest.Object == current.Object, latest.Metadata == current.Metadata, latest.Size == current.Size", + latest: inode.Generation{Object: 1, Metadata: 1, Size: 100}, + current: inode.Generation{Object: 1, Metadata: 1, Size: 100}, + expected: 0, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual := tc.latest.Compare(tc.current) + assert.Equal(t, tc.expected, actual) + }) + } +} diff --git a/internal/fs/inode/name.go b/internal/fs/inode/name.go index 289e2a4356..2949e023dd 100644 --- a/internal/fs/inode/name.go +++ b/internal/fs/inode/name.go @@ -15,6 +15,7 @@ package inode import ( + "errors" "fmt" "strings" ) @@ -123,3 +124,26 @@ func (name Name) IsDirectChildOf(parent Name) bool { cleanDiff := strings.TrimSuffix(diff, "/") return !strings.Contains(cleanDiff, "/") } + +// ParentName returns the Name of the parent directory of the current Name. +func (name Name) ParentName() (Name, error) { + if name.IsBucketRoot() { + return Name{}, errors.New("root has no parent") + } + + objectName := strings.TrimSuffix(name.objectName, "/") // normalize for dir or file + lastSlash := strings.LastIndex(objectName, "/") + if lastSlash == -1 { + // Direct child of bucket root + return Name{ + bucketName: name.bucketName, + objectName: "", + }, nil + } + + parentObjectName := objectName[:lastSlash+1] // include trailing slash for dir + return Name{ + bucketName: name.bucketName, + objectName: parentObjectName, + }, nil +} diff --git a/internal/fs/inode/name_test.go b/internal/fs/inode/name_test.go index d8f801ceec..28c3eb6532 100644 --- a/internal/fs/inode/name_test.go +++ b/internal/fs/inode/name_test.go @@ -17,7 +17,8 @@ package inode_test import ( "testing" - "github.com/googlecloudplatform/gcsfuse/v2/internal/fs/inode" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/inode" + . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/ogletest" ) @@ -106,3 +107,46 @@ func TestNameAsMapKey(t *testing.T) { _, ok := count[bar] ExpectFalse(ok) } + +func TestParentName(t *testing.T) { + for _, bucketName := range []string{"", "bucketx"} { + // Setup + root := inode.NewRootName(bucketName) // "" + foo := inode.NewDirName(root, "foo") // "foo/" + bar := inode.NewDirName(foo, "bar") // "foo/bar/" + baz := inode.NewFileName(root, "baz") // "baz" + qux := inode.NewFileName(bar, "qux") // "foo/bar/qux" + // Test cases + testCases := []struct { + name inode.Name + expectedParentName inode.Name + }{ + {name: foo, expectedParentName: root}, + {name: bar, expectedParentName: foo}, + {name: baz, expectedParentName: root}, + {name: qux, expectedParentName: bar}, + } + + for _, tc := range testCases { + parent, err := tc.name.ParentName() + + ExpectEq(nil, err) + ExpectEq(tc.expectedParentName.GcsObjectName(), parent.GcsObjectName()) + ExpectEq(tc.expectedParentName.LocalName(), parent.LocalName()) + } + } +} + +func TestParentNameReturnsErrorOnBucketRoot(t *testing.T) { + for _, bucketName := range []string{"", "bucketx"} { + // Setup + root := inode.NewRootName(bucketName) // "" + + // Call ParentName on bucket root + _, err := root.ParentName() + + // Expect an error + ExpectNe(nil, err) + ExpectThat(err, Error(HasSubstr("root has no parent"))) + } +} diff --git a/internal/fs/inode/recursive_cancellation_test.go b/internal/fs/inode/recursive_cancellation_test.go new file mode 100644 index 0000000000..7d33c90808 --- /dev/null +++ b/internal/fs/inode/recursive_cancellation_test.go @@ -0,0 +1,107 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inode + +import ( + "context" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/jacobsa/fuse/fuseops" + "github.com/jacobsa/timeutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "golang.org/x/sync/semaphore" +) + +type RecursiveCancellationTest struct { + suite.Suite + bucket gcsx.SyncerBucket + fake gcs.Bucket + clock timeutil.SimulatedClock + config *cfg.Config +} + +func (t *RecursiveCancellationTest) SetupTest() { + t.clock.SetTime(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + t.fake = fake.NewFakeBucket(&t.clock, "some_bucket", gcs.BucketType{}) + t.bucket = gcsx.NewSyncerBucket( + /*appendThreshold=*/ 1, + /*chunkRetryDeadlineSecs=*/ 120, + /*chunkTransferTimeoutSecs=*/ 10, + ".gcsfuse_tmp/", t.fake, + ) + t.config = &cfg.Config{ + MetadataCache: cfg.MetadataCacheConfig{ + EnableMetadataPrefetch: true, + TypeCacheMaxSizeMb: 400, + StatCacheMaxSizeMb: 400, + TtlSecs: 60, + MetadataPrefetchEntriesLimit: 5000, + }, + } +} + +func (t *RecursiveCancellationTest) createDirInode(name Name, parentCtx context.Context) *dirInode { + in := NewDirInode( + fuseops.RootInodeID+1, // ID doesn't matter much + name, + parentCtx, + fuseops.InodeAttributes{Mode: dirMode}, + true, // implicitDirs + false, + time.Minute, + &t.bucket, + &t.clock, + &t.clock, + semaphore.NewWeighted(10), + t.config, + ) + return in.(*dirInode) +} + +func (t *RecursiveCancellationTest) TestRecursiveCancellation() { + // Root dir + rootDir := t.createDirInode(NewRootName(""), nil) + + // Child dir + childName := NewDirName(rootDir.Name(), "child/") + childDir := t.createDirInode(childName, rootDir.Context()) + + // Grandchild dir + grandChildName := NewDirName(childDir.Name(), "grandchild/") + grandChildDir := t.createDirInode(grandChildName, childDir.Context()) + + // Start prefetches (simulated by checking if contexts are active) + assert.NoError(t.T(), rootDir.Context().Err()) + assert.NoError(t.T(), childDir.Context().Err()) + assert.NoError(t.T(), grandChildDir.Context().Err()) + + // Cancel root + rootDir.CancelSubdirectoryPrefetches() + + // Verify cancellation propagated + assert.ErrorIs(t.T(), rootDir.Context().Err(), context.Canceled) + assert.ErrorIs(t.T(), childDir.Context().Err(), context.Canceled) + assert.ErrorIs(t.T(), grandChildDir.Context().Err(), context.Canceled) +} + +func TestRecursiveCancellationSuite(t *testing.T) { + suite.Run(t, new(RecursiveCancellationTest)) +} diff --git a/internal/fs/inode/symlink.go b/internal/fs/inode/symlink.go index eaa3a1667d..c01288342d 100644 --- a/internal/fs/inode/symlink.go +++ b/internal/fs/inode/symlink.go @@ -15,9 +15,15 @@ package inode import ( + "errors" + "fmt" + "io" "sync" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/gcsfuse_errors" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "github.com/jacobsa/fuse/fuseops" "golang.org/x/net/context" ) @@ -25,16 +31,25 @@ import ( // When this custom metadata key is present in an object record, it is to be // treated as a symlink. For use in testing only; other users should detect // this with IsSymlink. +// Note: SymlinkMetadataKey is deprecated in favor of StandardSymlinkMetadataKey +// and retained solely for backward compatibility. const SymlinkMetadataKey = "gcsfuse_symlink_target" +const StandardSymlinkMetadataKey = "goog-reserved-file-is-symlink" // IsSymlink Does the supplied object represent a symlink inode? func IsSymlink(m *gcs.MinObject) bool { if m == nil { return false } - - _, ok := m.Metadata[SymlinkMetadataKey] - return ok + // 1. Check legacy/custom key presence + if _, ok := m.Metadata[SymlinkMetadataKey]; ok { + return true + } + // 2. Check standard reserved key value + if val, ok := m.Metadata[StandardSymlinkMetadataKey]; ok { + return val == "true" + } + return false } type SymlinkInode struct { @@ -44,9 +59,11 @@ type SymlinkInode struct { id fuseops.InodeID name Name + bucket *gcsx.SyncerBucket sourceGeneration Generation attrs fuseops.InodeAttributes target string + metadata map[string]string ///////////////////////// // Mutable state @@ -64,17 +81,21 @@ var _ Inode = &SymlinkInode{} // // REQUIRES: IsSymlink(o) func NewSymlinkInode( + ctx context.Context, id fuseops.InodeID, name Name, + bucket *gcsx.SyncerBucket, m *gcs.MinObject, - attrs fuseops.InodeAttributes) (s *SymlinkInode) { + attrs fuseops.InodeAttributes) (s *SymlinkInode, err error) { // Create the inode. s = &SymlinkInode{ - id: id, - name: name, + id: id, + name: name, + bucket: bucket, sourceGeneration: Generation{ Object: m.Generation, Metadata: m.MetaGeneration, + Size: m.Size, }, attrs: fuseops.InodeAttributes{ Nlink: 1, @@ -85,13 +106,85 @@ func NewSymlinkInode( Ctime: m.Updated, Mtime: m.Updated, }, - target: m.Metadata[SymlinkMetadataKey], + metadata: m.Metadata, } // Set up lookup counting. s.lc.Init(id) - return + s.target, err = s.resolveSymlinkTarget(ctx) + if err != nil { + return nil, err + } + + return s, nil +} + +//////////////////////////////////////////////////////////////////////// +// Helpers +//////////////////////////////////////////////////////////////////////// + +// Open a reader for the Symlink GCS object. +func (s *SymlinkInode) openReader(ctx context.Context) (io.ReadCloser, error) { + rc, err := s.bucket.NewReaderWithReadHandle( + ctx, + &gcs.ReadObjectRequest{ + Name: s.name.GcsObjectName(), + Generation: s.sourceGeneration.Object, + }) + + if err != nil { + // If the object with requested generation doesn't exist in GCS, it indicates + // a file clobbering scenario. This likely occurred because the file was + // modified/deleted leading to different generation number. + var notFoundError *gcs.NotFoundError + if errors.As(err, ¬FoundError) { + err = &gcsfuse_errors.FileClobberedError{ + Err: err, + ObjectName: s.name.GcsObjectName(), + } + } + err = fmt.Errorf("NewReader: %w", err) + } + return rc, err +} + +// resolveSymlinkTarget retrieves the target path of the symlink. +// +// It handles two types of symlinks: +// 1. Standard Symlinks: Identified by the "goog-reserved-file-is-symlink" metadata key. +// The target path is stored as the object's content. +// 2. Legacy Symlinks: Identified by the "gcsfuse_symlink_target" metadata key. +// The target path is stored directly in the metadata value. +func (s *SymlinkInode) resolveSymlinkTarget(ctx context.Context) (string, error) { + // Check for standard symlink representation where the target is stored in the object content. + if val, ok := s.metadata[StandardSymlinkMetadataKey]; ok && val == "true" { + rc, err := s.openReader(ctx) + if err != nil { + return "", fmt.Errorf("openReader: %w", err) + } + defer func() { + closeErr := rc.Close() + if closeErr != nil { + logger.Warnf("Error closing reader for symlink object %q: %v", s.name.GcsObjectName(), closeErr) + } + }() + + content, err := io.ReadAll(rc) + if err != nil { + return "", fmt.Errorf("ReadAll: %w", err) + } + return string(content), nil + } + + // Check for legacy symlink representation where the target is stored in metadata. + if target, ok := s.metadata[SymlinkMetadataKey]; ok { + return target, nil + } + + // If none of the metadata keys are present, return error since target of symlink + // cannot be resolved. + return "", fmt.Errorf("symlink target could not be resolved") } //////////////////////////////////////////////////////////////////////// @@ -121,6 +214,12 @@ func (s *SymlinkInode) SourceGeneration() Generation { return s.sourceGeneration } +func (s *SymlinkInode) UpdateSize(size uint64) { + // The size of a symlink is its target's length, not the backing object's size. + // However, to keep generation info consistent, we update it. + s.sourceGeneration.Size = size +} + // LOCKS_REQUIRED(s.mu) func (s *SymlinkInode) IncrementLookupCount() { s.lc.Inc() @@ -139,7 +238,7 @@ func (s *SymlinkInode) Destroy() (err error) { } func (s *SymlinkInode) Attributes( - ctx context.Context) (attrs fuseops.InodeAttributes, err error) { + ctx context.Context, clobberedCheck bool) (attrs fuseops.InodeAttributes, err error) { attrs = s.attrs return } @@ -149,3 +248,23 @@ func (s *SymlinkInode) Target() (target string) { target = s.target return } + +func (s *SymlinkInode) Unlink() { +} + +// Bucket returns the bucket that owns this inode. +func (s *SymlinkInode) Bucket() *gcsx.SyncerBucket { + return s.bucket +} + +// Source returns the MinObject from which this inode was created. +func (s *SymlinkInode) Source() *gcs.MinObject { + return &gcs.MinObject{ + Name: s.name.GcsObjectName(), + Generation: s.sourceGeneration.Object, + MetaGeneration: s.sourceGeneration.Metadata, + Size: s.sourceGeneration.Size, + Metadata: s.metadata, + Updated: s.attrs.Mtime, + } +} diff --git a/internal/fs/inode/symlink_internal_test.go b/internal/fs/inode/symlink_internal_test.go new file mode 100644 index 0000000000..7158506e7a --- /dev/null +++ b/internal/fs/inode/symlink_internal_test.go @@ -0,0 +1,271 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inode + +import ( + "context" + "errors" + "io" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/gcsfuse_errors" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/jacobsa/fuse/fuseops" + "github.com/jacobsa/timeutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type SymlinkInternalTest struct { + suite.Suite + ctx context.Context + bucket gcs.Bucket + clock timeutil.SimulatedClock +} + +func TestSymlinkInternalTest(t *testing.T) { + suite.Run(t, new(SymlinkInternalTest)) +} + +func (t *SymlinkInternalTest) SetupTest() { + t.ctx = context.Background() + t.clock.SetTime(time.Date(2012, 8, 15, 22, 56, 0, 0, time.Local)) + t.bucket = fake.NewFakeBucket(&t.clock, "some_bucket", gcs.BucketType{}) +} + +func (t *SymlinkInternalTest) createSymlinkInode(name string, target string, legacy bool) *SymlinkInode { + objName := name + // Create object in bucket + var content string + if !legacy { + content = target + } + o, err := storageutil.CreateObject(t.ctx, t.bucket, objName, []byte(content)) + require.NoError(t.T(), err) + + m := storageutil.ConvertObjToMinObject(o) + // For standard symlink, we set the metadata key + if m.Metadata == nil { + m.Metadata = make(map[string]string) + } + if legacy { + m.Metadata[SymlinkMetadataKey] = target + } else { + m.Metadata[StandardSymlinkMetadataKey] = "true" + } + syncerBucket := gcsx.NewSyncerBucket( + /*appendThreshold=*/ 1, + /*chunkRetryDeadlineSecs=*/ 120, + /*chunkTransferTimeoutSecs=*/ 10, + ".gcsfuse_tmp/", + t.bucket, + ) + + s, err := NewSymlinkInode( + t.ctx, + fuseops.InodeID(1), + NewFileName(NewRootName(""), name), + &syncerBucket, + m, + fuseops.InodeAttributes{}, + ) + require.NoError(t.T(), err) + return s +} + +func (t *SymlinkInternalTest) TestOpenReader() { + target := "target_file" + s := t.createSymlinkInode("foo", target, false) + + rc, err := s.openReader(t.ctx) + require.NoError(t.T(), err) + defer func() { + require.NoError(t.T(), rc.Close()) + }() + + content, err := io.ReadAll(rc) + require.NoError(t.T(), err) + assert.Equal(t.T(), target, string(content)) +} + +func (t *SymlinkInternalTest) TestOpenReader_Clobbered() { + target := "target_file" + s := t.createSymlinkInode("foo", target, false) + // Clobber the object in GCS (update it, changing generation) + _, err := storageutil.CreateObject(t.ctx, t.bucket, "foo", []byte("new_target")) + require.NoError(t.T(), err) + + _, err = s.openReader(t.ctx) + + require.Error(t.T(), err) + var clobberedErr *gcsfuse_errors.FileClobberedError + assert.True(t.T(), errors.As(err, &clobberedErr)) +} + +func (t *SymlinkInternalTest) TestResolveSymlinkTarget_Standard() { + target := "target_file" + s := t.createSymlinkInode("foo", target, false) + + resolvedTarget, err := s.resolveSymlinkTarget(t.ctx) + + require.NoError(t.T(), err) + assert.Equal(t.T(), target, resolvedTarget) +} + +func (t *SymlinkInternalTest) TestResolveSymlinkTarget_Legacy() { + target := "target_file" + objName := "legacy_symlink" + s := t.createSymlinkInode(objName, target, true) + + resolvedTarget, err := s.resolveSymlinkTarget(t.ctx) + + require.NoError(t.T(), err) + assert.Equal(t.T(), target, resolvedTarget) +} + +func (t *SymlinkInternalTest) TestResolveSymlinkTarget_Clobbered() { + target := "target_file" + s := t.createSymlinkInode("foo", target, false) + // Clobber the object in GCS (update it, changing generation) + _, err := storageutil.CreateObject(t.ctx, t.bucket, "foo", []byte("new_target")) + require.NoError(t.T(), err) + + _, err = s.resolveSymlinkTarget(t.ctx) + + require.Error(t.T(), err) + var clobberedErr *gcsfuse_errors.FileClobberedError + assert.True(t.T(), errors.As(err, &clobberedErr)) +} + +func (t *SymlinkInternalTest) TestNewSymlinkInode_Legacy() { + target := "target_file" + m := &gcs.MinObject{ + Name: "legacy_symlink", + Metadata: map[string]string{ + SymlinkMetadataKey: target, + }, + } + syncerBucket := gcsx.NewSyncerBucket( + /*appendThreshold=*/ 1, + /*chunkRetryDeadlineSecs=*/ 120, + /*chunkTransferTimeoutSecs=*/ 10, + ".gcsfuse_tmp/", + t.bucket, + ) + + s, err := NewSymlinkInode( + t.ctx, + fuseops.InodeID(1), + NewFileName(NewRootName(""), m.Name), + &syncerBucket, + m, + fuseops.InodeAttributes{}, + ) + + require.NoError(t.T(), err) + assert.Equal(t.T(), target, s.Target()) +} + +func (t *SymlinkInternalTest) TestNewSymlinkInode_Standard() { + target := "target_file" + objName := "standard_symlink" + o, err := storageutil.CreateObject(t.ctx, t.bucket, objName, []byte(target)) + require.NoError(t.T(), err) + m := storageutil.ConvertObjToMinObject(o) + if m.Metadata == nil { + m.Metadata = make(map[string]string) + } + m.Metadata[StandardSymlinkMetadataKey] = "true" + syncerBucket := gcsx.NewSyncerBucket( + /*appendThreshold=*/ 1, + /*chunkRetryDeadlineSecs=*/ 120, + /*chunkTransferTimeoutSecs=*/ 10, + ".gcsfuse_tmp/", + t.bucket, + ) + + s, err := NewSymlinkInode( + t.ctx, + fuseops.InodeID(1), + NewFileName(NewRootName(""), m.Name), + &syncerBucket, + m, + fuseops.InodeAttributes{}, + ) + + require.NoError(t.T(), err) + assert.Equal(t.T(), target, s.Target()) +} + +func (t *SymlinkInternalTest) TestNewSymlinkInode_Standard_ReadError() { + objName := "missing_symlink" + // Object does not exist in bucket + m := &gcs.MinObject{ + Name: objName, + Metadata: map[string]string{ + StandardSymlinkMetadataKey: "true", + }, + Generation: 1, + } + syncerBucket := gcsx.NewSyncerBucket( + /*appendThreshold=*/ 1, + /*chunkRetryDeadlineSecs=*/ 120, + /*chunkTransferTimeoutSecs=*/ 10, + ".gcsfuse_tmp/", + t.bucket, + ) + + _, err := NewSymlinkInode( + t.ctx, + fuseops.InodeID(1), + NewFileName(NewRootName(""), m.Name), + &syncerBucket, + m, + fuseops.InodeAttributes{}, + ) + + require.Error(t.T(), err) +} + +func (t *SymlinkInternalTest) TestNewSymlinkInode_InvalidMetadata() { + m := &gcs.MinObject{ + Name: "invalid_symlink", + Metadata: map[string]string{}, + } + syncerBucket := gcsx.NewSyncerBucket( + /*appendThreshold=*/ 1, + /*chunkRetryDeadlineSecs=*/ 120, + /*chunkTransferTimeoutSecs=*/ 10, + ".gcsfuse_tmp/", + t.bucket, + ) + + _, err := NewSymlinkInode( + t.ctx, + fuseops.InodeID(1), + NewFileName(NewRootName(""), m.Name), + &syncerBucket, + m, + fuseops.InodeAttributes{}, + ) + + require.Error(t.T(), err) + assert.Contains(t.T(), err.Error(), "symlink target could not be resolved") +} diff --git a/internal/fs/inode/symlink_test.go b/internal/fs/inode/symlink_test.go index bf29cbb894..9d75f24009 100644 --- a/internal/fs/inode/symlink_test.go +++ b/internal/fs/inode/symlink_test.go @@ -15,11 +15,20 @@ package inode_test import ( + "context" + "os" "testing" + "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/fs/inode" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/jacobsa/fuse/fuseops" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/inode" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" . "github.com/jacobsa/ogletest" + "github.com/jacobsa/timeutil" ) func TestSymlink(t *testing.T) { RunTests(t) } @@ -29,6 +38,7 @@ func TestSymlink(t *testing.T) { RunTests(t) } //////////////////////////////////////////////////////////////////////// type SymlinkTest struct { + bucket *gcsx.SyncerBucket } var _ SetUpInterface = &CoreTest{} @@ -36,6 +46,15 @@ var _ TearDownInterface = &CoreTest{} func init() { RegisterTestSuite(&SymlinkTest{}) } +func (t *SymlinkTest) SetUp(ti *TestInfo) { + bucket := gcsx.NewSyncerBucket( + /*appendThreshold=*/ 1, + /*chunkRetryDeadlineSecs=*/ 120, + /*chunkTransferTimeoutSecs=*/ 10, + ".gcsfuse_tmp/", fake.NewFakeBucket(timeutil.RealClock(), "some-bucket", gcs.BucketType{})) + t.bucket = &bucket +} + //////////////////////////////////////////////////////////////////////// // Tests //////////////////////////////////////////////////////////////////////// @@ -60,6 +79,111 @@ func (t *SymlinkTest) TestIsSymLinkWhenMetadataKeyIsNotPresent() { AssertEq(false, inode.IsSymlink(&m)) } +func (t *SymlinkTest) TestIsSymLinkWhenStandardMetadataKeyIsPresent() { + metadata := map[string]string{ + inode.StandardSymlinkMetadataKey: "true", + } + m := gcs.MinObject{ + Name: "test", + Metadata: metadata, + } + + AssertEq(true, inode.IsSymlink(&m)) +} + +func (t *SymlinkTest) TestIsSymLinkWhenStandardMetadataKeyIsFalse() { + metadata := map[string]string{ + inode.StandardSymlinkMetadataKey: "false", + } + m := gcs.MinObject{ + Name: "test", + Metadata: metadata, + } + + AssertEq(false, inode.IsSymlink(&m)) +} + func (t *SymlinkTest) TestIsSymLinkForNilObject() { AssertEq(false, inode.IsSymlink(nil)) } + +func (t *SymlinkTest) TestAttributes() { + metadata := map[string]string{ + inode.SymlinkMetadataKey: "target", + } + m := &gcs.MinObject{ + Name: "test", + Metadata: metadata, + } + attrs := fuseops.InodeAttributes{ + Uid: 1001, + Gid: 1002, + Mode: 0777 | os.ModeSymlink, + } + name := inode.NewFileName(inode.NewRootName("some-bucket"), m.Name) + s, err := inode.NewSymlinkInode(context.Background(), fuseops.InodeID(42), name, t.bucket, m, attrs) + AssertEq(nil, err) + tests := []struct { + name string + clobberedCheck bool + }{ + {"WithClobberedCheckFalse", false}, + {"WithClobberedCheckTrue", true}, + } + + for _, tt := range tests { + // Call Attributes + extracted, err := s.Attributes(context.TODO(), tt.clobberedCheck) + + // Check expected values + AssertEq(nil, err) + ExpectEq(uint32(1), extracted.Nlink) + ExpectEq(attrs.Uid, extracted.Uid) + ExpectEq(attrs.Gid, extracted.Gid) + ExpectEq(attrs.Mode, extracted.Mode) + } +} + +func (t *SymlinkTest) TestUpdateSize() { + m := &gcs.MinObject{ + Name: "test", + Generation: 1, + MetaGeneration: 2, + Size: 100, + Metadata: map[string]string{inode.SymlinkMetadataKey: "target"}, + } + attrs := fuseops.InodeAttributes{} + name := inode.NewFileName(inode.NewRootName("some-bucket"), m.Name) + s, err := inode.NewSymlinkInode(context.Background(), fuseops.InodeID(42), name, t.bucket, m, attrs) + AssertEq(nil, err) + + s.UpdateSize(200) + + AssertEq(uint64(200), s.SourceGeneration().Size) +} + +func (t *SymlinkTest) TestSource() { + obj, err := storageutil.CreateObject( + context.Background(), + t.bucket, + "test", // The name of the object in GCS + []byte("target_path"), + ) + AssertEq(nil, err) + m := storageutil.ConvertObjToMinObject(obj) + m.Metadata = map[string]string{inode.StandardSymlinkMetadataKey: "true"} + m.Updated = time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) // Explicitly set Updated time for consistent testing. + attrs := fuseops.InodeAttributes{} + name := inode.NewFileName(inode.NewRootName("some-bucket"), m.Name) + s, err := inode.NewSymlinkInode(context.Background(), fuseops.InodeID(42), name, t.bucket, m, attrs) + AssertEq(nil, err) + + source := s.Source() + + AssertEq(m.Name, source.Name) + AssertEq(m.Generation, source.Generation) + AssertEq(m.MetaGeneration, source.MetaGeneration) + AssertEq(m.Size, source.Size) + AssertEq(m.Metadata, source.Metadata) + AssertEq(0, m.Updated.Compare(source.Updated)) +} diff --git a/internal/fs/kernel_list_cache_inifinite_ttl_test.go b/internal/fs/kernel_list_cache_inifinite_ttl_test.go index d14b1b7af1..e5d11a6fdf 100644 --- a/internal/fs/kernel_list_cache_inifinite_ttl_test.go +++ b/internal/fs/kernel_list_cache_inifinite_ttl_test.go @@ -23,12 +23,26 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/common" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) +func SkipTestForUnsupportedKernelVersion(t *testing.T) { + // TODO: b/384648943 make this part of fsTest.SetUpTestSuite() after post fs + // tests are fully migrated to stretchr/testify. + t.Helper() + unsupported, err := common.IsKLCacheEvictionUnSupported() + assert.NoError(t, err) + if unsupported { + t.SkipNow() + } +} + type KernelListCacheTestWithInfiniteTtl struct { suite.Suite fsTest @@ -46,10 +60,13 @@ func (t *KernelListCacheTestWithInfiniteTtl) SetupSuite() { }, } t.serverCfg.RenameDirLimit = 10 + t.serverCfg.MetricHandle = metrics.NewNoopMetrics() + t.serverCfg.TraceHandle = tracing.NewNoopTracer() t.fsTest.SetUpTestSuite() } func TestKernelListCacheTestInfiniteTtlSuite(t *testing.T) { + SkipTestForUnsupportedKernelVersion(t) suite.Run(t, new(KernelListCacheTestWithInfiniteTtl)) } diff --git a/internal/fs/kernel_list_cache_test.go b/internal/fs/kernel_list_cache_test.go index 35fb575b2f..b7e302bdd6 100644 --- a/internal/fs/kernel_list_cache_test.go +++ b/internal/fs/kernel_list_cache_test.go @@ -30,8 +30,10 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -114,10 +116,13 @@ func (t *KernelListCacheTestWithPositiveTtl) SetupSuite() { }, } t.serverCfg.RenameDirLimit = 10 + t.serverCfg.MetricHandle = metrics.NewNoopMetrics() + t.serverCfg.TraceHandle = tracing.NewNoopTracer() t.fsTest.SetUpTestSuite() } func TestKernelListCacheTestWithPositiveTtlSuite(t *testing.T) { + SkipTestForUnsupportedKernelVersion(t) suite.Run(t, new(KernelListCacheTestWithPositiveTtl)) } @@ -132,7 +137,7 @@ func (t *KernelListCacheTestWithPositiveTtl) Test_Parallel_OpenDirAndLookUpInode go func() { defer wg.Done() - for i := 0; i < iterationsPerGoroutine; i++ { + for range iterationsPerGoroutine { f, err := os.Open(path.Join(mntDir, "explicitDir")) assert.Nil(t.T(), err) @@ -142,7 +147,7 @@ func (t *KernelListCacheTestWithPositiveTtl) Test_Parallel_OpenDirAndLookUpInode }() go func() { defer wg.Done() - for i := 0; i < iterationsPerGoroutine; i++ { + for range iterationsPerGoroutine { _, err := os.Stat(path.Join(mntDir, "explicitDir")) assert.Nil(t.T(), err) } @@ -174,11 +179,11 @@ func (t *KernelListCacheTestWithPositiveTtl) Test_Concurrent_ReadDir() { dirPath := path.Join(mntDir, "explicitDir") - for i := 0; i < goroutineCount; i++ { + for range goroutineCount { go func() { defer wg.Done() - for j := 0; j < iterationsPerGoroutine; j++ { + for range iterationsPerGoroutine { f, err := os.Open(dirPath) assert.Nil(t.T(), err) @@ -219,7 +224,7 @@ func (t *KernelListCacheTestWithPositiveTtl) Test_Parallel_ReadDirAndFileOperati // Goroutine 1: Repeatedly calls Readdir go func() { defer wg.Done() - for i := 0; i < iterationsPerGoroutine; i++ { // Adjust iteration count if needed + for range iterationsPerGoroutine { // Adjust iteration count if needed f, err := os.Open(dirPath) assert.Nil(t.T(), err) @@ -234,7 +239,7 @@ func (t *KernelListCacheTestWithPositiveTtl) Test_Parallel_ReadDirAndFileOperati // Goroutine 2: Creates and deletes files go func() { defer wg.Done() - for i := 0; i < iterationsPerGoroutine; i++ { // Adjust iteration count if needed + for range iterationsPerGoroutine { // Adjust iteration count if needed filePath := path.Join(dirPath, "tmp_file.txt") renamedFilePath := path.Join(dirPath, "renamed_tmp_file.txt") @@ -283,7 +288,7 @@ func (t *KernelListCacheTestWithPositiveTtl) Test_Parallel_ReadDirAndDirOperatio // Goroutine 1: Repeatedly calls Readdir go func() { defer wg.Done() - for i := 0; i < iterationsPerGoroutine; i++ { + for range iterationsPerGoroutine { f, err := os.Open(parentDir) assert.Nil(t.T(), err) @@ -298,7 +303,7 @@ func (t *KernelListCacheTestWithPositiveTtl) Test_Parallel_ReadDirAndDirOperatio // Goroutine 2: Creates and deletes directories go func() { defer wg.Done() - for i := 0; i < iterationsPerGoroutine; i++ { + for range iterationsPerGoroutine { dirPath := path.Join(parentDir, "test_dir") renamedDirPath := path.Join(parentDir, "renamed_test_dir") diff --git a/internal/fs/kernel_list_cache_zero_ttl_test.go b/internal/fs/kernel_list_cache_zero_ttl_test.go index 26d3207d9e..e760c21792 100644 --- a/internal/fs/kernel_list_cache_zero_ttl_test.go +++ b/internal/fs/kernel_list_cache_zero_ttl_test.go @@ -21,7 +21,9 @@ import ( "path" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -44,10 +46,13 @@ func (t *KernelListCacheTestWithZeroTtl) SetupSuite() { }, } t.serverCfg.RenameDirLimit = 10 + t.serverCfg.MetricHandle = metrics.NewNoopMetrics() + t.serverCfg.TraceHandle = tracing.NewNoopTracer() t.fsTest.SetUpTestSuite() } func TestKernelListCacheTestZeroTtlSuite(t *testing.T) { + SkipTestForUnsupportedKernelVersion(t) suite.Run(t, new(KernelListCacheTestWithZeroTtl)) } diff --git a/internal/fs/local_file_test.go b/internal/fs/local_file_test.go index 117aba2712..2165847d46 100644 --- a/internal/fs/local_file_test.go +++ b/internal/fs/local_file_test.go @@ -26,15 +26,20 @@ import ( "path/filepath" "reflect" "strings" + "syscall" + "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/fs/inode" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/inode" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" "github.com/jacobsa/fuse/fusetesting" - . "github.com/jacobsa/ogletest" - "github.com/jacobsa/timeutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" ) // ////////////////////////////////////////////////////////////////////// @@ -52,13 +57,14 @@ type LocalFileTest struct { // fsTest has f1 *osFile and f2 *osFile which we will reuse here. f3 *os.File fsTest + suite.Suite } -func init() { - RegisterTestSuite(&LocalFileTest{}) +func TestLocalFileTest(t *testing.T) { + suite.Run(t, &LocalFileTest{}) } -func (t *LocalFileTest) SetUpTestSuite() { +func (t *LocalFileTest) SetupSuite() { t.serverCfg.ImplicitDirectories = true t.serverCfg.NewConfig = &cfg.Config{ Write: cfg.WriteConfig{ @@ -67,10 +73,14 @@ func (t *LocalFileTest) SetUpTestSuite() { t.fsTest.SetUpTestSuite() } -func (t *LocalFileTest) TearDown() { +func (t *LocalFileTest) TearDownSuite() { + t.fsTest.TearDownTestSuite() +} + +func (t *LocalFileTest) TearDownTest() { // Close t.f3 in case of test failure. if t.f3 != nil { - AssertEq(nil, t.f3.Close()) + assert.NoError(t.T(), t.f3.Close()) t.f3 = nil } @@ -82,41 +92,46 @@ func (t *LocalFileTest) TearDown() { // Helpers // ////////////////////////////////////////////////////////////////////// func (t *LocalFileTest) createLocalFile(fileName string) (filePath string, f *os.File) { + t.T().Helper() // Creating a file shouldn't create file on GCS. filePath = path.Join(mntDir, fileName) - f, err := os.Create(filePath) + f, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC|syscall.O_DIRECT, 0655) - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateObjectNotFoundErr(fileName) return } func (t *LocalFileTest) verifyLocalFileEntry(entry os.DirEntry, fileName string, size int) { - AssertEq(false, entry.IsDir()) - AssertEq(fileName, entry.Name()) + t.T().Helper() + assert.False(t.T(), entry.IsDir()) + assert.Equal(t.T(), fileName, entry.Name()) fileInfo, err := entry.Info() - AssertEq(nil, err) - AssertEq(size, fileInfo.Size()) + require.NoError(t.T(), err) + assert.EqualValues(t.T(), size, fileInfo.Size()) } func (t *LocalFileTest) verifyDirectoryEntry(entry os.DirEntry, dirName string) { - AssertEq(true, entry.IsDir()) - AssertEq(dirName, entry.Name()) + t.T().Helper() + assert.True(t.T(), entry.IsDir()) + assert.Equal(t.T(), dirName, entry.Name()) } func (t *LocalFileTest) readDirectory(dirPath string) (entries []os.DirEntry) { + t.T().Helper() entries, err := os.ReadDir(dirPath) - AssertEq(nil, err) + require.NoError(t.T(), err) return } func (t *LocalFileTest) validateObjectNotFoundErr(fileName string) { + t.T().Helper() var notFoundErr *gcs.NotFoundError _, err := storageutil.ReadObject(ctx, bucket, fileName) - ExpectTrue(errors.As(err, ¬FoundErr)) + assert.True(t.T(), errors.As(err, ¬FoundErr)) } func (t *LocalFileTest) closeLocalFile(f **os.File) error { @@ -126,23 +141,26 @@ func (t *LocalFileTest) closeLocalFile(f **os.File) error { } func (t *LocalFileTest) closeFileAndValidateObjectContents(f **os.File, fileName string, contents string) { + t.T().Helper() err := t.closeLocalFile(f) - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateObjectContents(fileName, contents) } func (t *LocalFileTest) validateObjectContents(fileName string, contents string) { + t.T().Helper() contentBytes, err := storageutil.ReadObject(ctx, bucket, fileName) - AssertEq(nil, err) - ExpectEq(contents, string(contentBytes)) + require.NoError(t.T(), err) + assert.Equal(t.T(), contents, string(contentBytes)) } func (t *LocalFileTest) newFileShouldGetSyncedToGCSAtClose(fileName string) { + t.T().Helper() // Create a local file. _, t.f1 = t.createLocalFile(fileName) // Writing contents to local file shouldn't create file on GCS. _, err := t.f1.Write([]byte(FileContents)) - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateObjectNotFoundErr(fileName) // Close the file and validate if the file is created on GCS. @@ -150,37 +168,37 @@ func (t *LocalFileTest) newFileShouldGetSyncedToGCSAtClose(fileName string) { // Validate object attributes non-nil and non-empty. minObject, extendedAttr, err := bucket.StatObject(ctx, &gcs.StatObjectRequest{Name: fileName, ForceFetchFromGcs: true, ReturnExtendedObjectAttributes: true}) - AssertEq(nil, err) - AssertNe(nil, extendedAttr) - AssertNe(nil, minObject) - ExpectFalse(reflect.DeepEqual(*extendedAttr, gcs.ExtendedObjectAttributes{})) - ExpectFalse(reflect.DeepEqual(*minObject, gcs.MinObject{})) + require.NoError(t.T(), err) + require.NotNil(t.T(), extendedAttr) + require.NotNil(t.T(), minObject) + assert.False(t.T(), reflect.DeepEqual(*extendedAttr, gcs.ExtendedObjectAttributes{})) + assert.False(t.T(), reflect.DeepEqual(*minObject, gcs.MinObject{})) } func (t *LocalFileTest) validateNoFileOrDirError(filename string) { _, err := os.Stat(path.Join(mntDir, filename)) - AssertNe(nil, err) - AssertTrue(strings.Contains(err.Error(), "no such file or directory")) + require.Error(t.T(), err) + assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) } //////////////////////////////////////////////////////////////////////// // Tests //////////////////////////////////////////////////////////////////////// -func (t *LocalFileTest) NewFileShouldNotGetSyncedToGCSTillClose() { +func (t *LocalFileTest) TestNewFileShouldNotGetSyncedToGCSTillClose() { t.newFileShouldGetSyncedToGCSAtClose(FileName) } -func (t *LocalFileTest) NewFileUnderExplicitDirectoryShouldNotGetSyncedToGCSTillClose() { +func (t *LocalFileTest) TestNewFileUnderExplicitDirectoryShouldNotGetSyncedToGCSTillClose() { err := os.Mkdir(path.Join(mntDir, "explicit"), dirPerms) - AssertEq(nil, err) + require.NoError(t.T(), err) t.newFileShouldGetSyncedToGCSAtClose("explicit/foo") } -func (t *LocalFileTest) NewFileUnderImplicitDirectoryShouldNotGetSyncedToGCSTillClose() { - AssertEq( - nil, +func (t *LocalFileTest) TestNewFileUnderImplicitDirectoryShouldNotGetSyncedToGCSTillClose() { + require.NoError( + t.T(), t.createObjects( map[string]string{ // File @@ -190,108 +208,108 @@ func (t *LocalFileTest) NewFileUnderImplicitDirectoryShouldNotGetSyncedToGCSTill t.newFileShouldGetSyncedToGCSAtClose("implicitFoo/foo") } -func (t *LocalFileTest) StatOnLocalFile() { +func (t *LocalFileTest) TestStatOnLocalFile() { // Create a local file. var filePath string filePath, t.f1 = t.createLocalFile(FileName) // Stat the local file. fi, err := os.Stat(filePath) - AssertEq(nil, err) - ExpectEq(path.Base(filePath), fi.Name()) - ExpectEq(0, fi.Size()) - ExpectEq(filePerms, fi.Mode()) + require.NoError(t.T(), err) + assert.Equal(t.T(), path.Base(filePath), fi.Name()) + assert.EqualValues(t.T(), 0, fi.Size()) + assert.Equal(t.T(), filePerms, fi.Mode()) // Writing contents to local file shouldn't create file on GCS. _, err = t.f1.Write([]byte(FileContents)) - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateObjectNotFoundErr(FileName) // Stat the local file again to check if new contents are written. fi, err = os.Stat(filePath) - AssertEq(nil, err) - ExpectEq(path.Base(filePath), fi.Name()) - ExpectEq(10, fi.Size()) - ExpectEq(filePerms, fi.Mode()) + require.NoError(t.T(), err) + assert.Equal(t.T(), path.Base(filePath), fi.Name()) + assert.EqualValues(t.T(), 10, fi.Size()) + assert.Equal(t.T(), filePerms, fi.Mode()) // Close the file and validate if the file is created on GCS. t.closeFileAndValidateObjectContents(&t.f1, FileName, FileContents) } -func (t *LocalFileTest) StatOnLocalFileWithConflictingFileNameSuffix() { +func (t *LocalFileTest) TestStatOnLocalFileWithConflictingFileNameSuffix() { // Create a local file. var filePath string filePath, t.f1 = t.createLocalFile(FileName) // Stat the local file. fi, err := os.Stat(filePath + inode.ConflictingFileNameSuffix) - AssertEq(nil, err) - ExpectEq(path.Base(filePath)+inode.ConflictingFileNameSuffix, fi.Name()) - ExpectEq(0, fi.Size()) - ExpectEq(filePerms, fi.Mode()) + require.NoError(t.T(), err) + assert.Equal(t.T(), path.Base(filePath)+inode.ConflictingFileNameSuffix, fi.Name()) + assert.EqualValues(t.T(), 0, fi.Size()) + assert.Equal(t.T(), filePerms, fi.Mode()) // Close the file and validate if the file is created on GCS. t.closeFileAndValidateObjectContents(&t.f1, FileName, "") } -func (t *LocalFileTest) StatOnUnlinkedLocalFile() { +func (t *LocalFileTest) TestStatOnUnlinkedLocalFile() { // Create a local file. var filePath string filePath, t.f1 = t.createLocalFile(FileName) // unlink the local file. err := os.Remove(filePath) - AssertEq(nil, err) + require.NoError(t.T(), err) // Stat the local file and validate error. t.validateNoFileOrDirError(FileName) // Close the file and validate that file is not created on GCS. err = t.closeLocalFile(&t.f1) - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateObjectNotFoundErr(FileName) } -func (t *LocalFileTest) TruncateLocalFile() { +func (t *LocalFileTest) TestTruncateLocalFile() { // Create a local file. var filePath string filePath, t.f1 = t.createLocalFile(FileName) // Writing contents to local file . _, err := t.f1.Write([]byte(FileContents)) - AssertEq(nil, err) + require.NoError(t.T(), err) // Stat the file to validate if new contents are written. fi, err := os.Stat(filePath) - AssertEq(nil, err) - ExpectEq(path.Base(filePath), fi.Name()) - ExpectEq(10, fi.Size()) - ExpectEq(filePerms, fi.Mode()) + require.NoError(t.T(), err) + assert.Equal(t.T(), path.Base(filePath), fi.Name()) + assert.EqualValues(t.T(), 10, fi.Size()) + assert.Equal(t.T(), filePerms, fi.Mode()) // Truncate the file to update the file size. err = os.Truncate(filePath, 5) - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateObjectNotFoundErr(FileName) // Stat the file to validate if file is truncated correctly. fi, err = os.Stat(filePath) - AssertEq(nil, err) - ExpectEq(path.Base(filePath), fi.Name()) - ExpectEq(5, fi.Size()) - ExpectEq(filePerms, fi.Mode()) + require.NoError(t.T(), err) + assert.Equal(t.T(), path.Base(filePath), fi.Name()) + assert.EqualValues(t.T(), 5, fi.Size()) + assert.Equal(t.T(), filePerms, fi.Mode()) // Close the file and validate if the file is created on GCS. t.closeFileAndValidateObjectContents(&t.f1, FileName, "tests") } -func (t *LocalFileTest) MultipleWritesToLocalFile() { +func (t *LocalFileTest) TestMultipleWritesToLocalFile() { // Create a local file. _, t.f1 = t.createLocalFile(FileName) // Write some contents to file sequentially. _, err := t.f1.Write([]byte("string1")) - AssertEq(nil, err) + require.NoError(t.T(), err) _, err = t.f1.Write([]byte("string2")) - AssertEq(nil, err) + require.NoError(t.T(), err) _, err = t.f1.Write([]byte("string3")) - AssertEq(nil, err) + require.NoError(t.T(), err) // File shouldn't get created on GCS. t.validateObjectNotFoundErr(FileName) @@ -299,24 +317,24 @@ func (t *LocalFileTest) MultipleWritesToLocalFile() { t.closeFileAndValidateObjectContents(&t.f1, FileName, "string1string2string3") } -func (t *LocalFileTest) RandomWritesToLocalFile() { +func (t *LocalFileTest) TestRandomWritesToLocalFile() { // Create a local file. _, t.f1 = t.createLocalFile(FileName) // Write some contents to file randomly. _, err := t.f1.WriteAt([]byte("string1"), 0) - AssertEq(nil, err) + require.NoError(t.T(), err) _, err = t.f1.WriteAt([]byte("string2"), 2) - AssertEq(nil, err) + require.NoError(t.T(), err) _, err = t.f1.WriteAt([]byte("string3"), 3) - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateObjectNotFoundErr(FileName) // Close the file and validate if the file is created on GCS. t.closeFileAndValidateObjectContents(&t.f1, FileName, "stsstring3") } -func (t *LocalFileTest) TestReadDirWithEmptyLocalFiles() { +func (t *LocalFileTest) TestTestReadDirWithEmptyLocalFiles() { // Create local files. _, t.f1 = t.createLocalFile(FileName) _, t.f2 = t.createLocalFile(FileName2) @@ -325,7 +343,7 @@ func (t *LocalFileTest) TestReadDirWithEmptyLocalFiles() { entries := t.readDirectory(mntDir) // Verify entries received successfully. - AssertEq(2, len(entries)) + require.Equal(t.T(), 2, len(entries)) t.verifyLocalFileEntry(entries[0], FileName, 0) t.verifyLocalFileEntry(entries[1], FileName2, 0) // Close the local files. @@ -337,13 +355,13 @@ func (t *LocalFileTest) TestReadDirWithNonEmptyLocalFile() { // Create local files. _, t.f1 = t.createLocalFile(FileName) _, err := t.f1.WriteString(FileContents) - AssertEq(nil, err) + require.NoError(t.T(), err) // Attempt to list mntDir. entries := t.readDirectory(mntDir) // Verify entries received successfully. - AssertEq(1, len(entries)) + require.Equal(t.T(), 1, len(entries)) t.verifyLocalFileEntry(entries[0], FileName, 10) // Close the local files. t.closeFileAndValidateObjectContents(&t.f1, FileName, FileContents) @@ -351,8 +369,7 @@ func (t *LocalFileTest) TestReadDirWithNonEmptyLocalFile() { func (t *LocalFileTest) TestReadDirForExplicitDirWithLocalFile() { // Create explicit dir with 2 local files. - AssertEq( - nil, + assert.Nil(t.T(), t.createObjects( map[string]string{ "explicitFoo/": "", @@ -364,7 +381,7 @@ func (t *LocalFileTest) TestReadDirForExplicitDirWithLocalFile() { entries := t.readDirectory(path.Join(mntDir, "explicitFoo/")) // Verify entries received successfully. - AssertEq(2, len(entries)) + assert.Equal(t.T(), 2, len(entries)) t.verifyLocalFileEntry(entries[0], FileName, 0) t.verifyLocalFileEntry(entries[1], FileName2, 0) // Close the local files. @@ -374,8 +391,7 @@ func (t *LocalFileTest) TestReadDirForExplicitDirWithLocalFile() { func (t *LocalFileTest) TestReadDirForImplicitDirWithLocalFile() { // Create implicit dir with 2 local files and 1 synced file. - AssertEq( - nil, + require.NoError(t.T(), t.createObjects( map[string]string{ // File @@ -388,7 +404,7 @@ func (t *LocalFileTest) TestReadDirForImplicitDirWithLocalFile() { entries := t.readDirectory(path.Join(mntDir, "implicitFoo/")) // Verify entries received successfully. - AssertEq(3, len(entries)) + require.Equal(t.T(), 3, len(entries)) t.verifyLocalFileEntry(entries[0], "bar", 0) t.verifyLocalFileEntry(entries[1], FileName, 0) t.verifyLocalFileEntry(entries[2], FileName2, 0) @@ -408,8 +424,7 @@ func (t *LocalFileTest) TestRecursiveListingWithLocalFiles() { // - implicitLocalFile --- file // Create implicit dir with 1 local file1 and 1 synced file. - AssertEq( - nil, + require.NoError(t.T(), t.createObjects( map[string]string{ // File @@ -417,8 +432,7 @@ func (t *LocalFileTest) TestRecursiveListingWithLocalFiles() { })) _, t.f1 = t.createLocalFile("implicitFoo/" + implicitLocalFileName) // Create explicit dir with 1 local file. - AssertEq( - nil, + require.NoError(t.T(), t.createObjects( map[string]string{ "explicitFoo/": "", @@ -438,12 +452,12 @@ func (t *LocalFileTest) TestRecursiveListingWithLocalFiles() { } objs, err := os.ReadDir(path) - AssertEq(nil, err) + require.NoError(t.T(), err) // Check if mntDir has correct objects. if path == mntDir { // numberOfObjects = 3 - AssertEq(3, len(objs)) + require.Equal(t.T(), 3, len(objs)) t.verifyDirectoryEntry(objs[0], "explicitFoo") t.verifyLocalFileEntry(objs[1], FileName, 0) t.verifyDirectoryEntry(objs[2], "implicitFoo") @@ -452,14 +466,14 @@ func (t *LocalFileTest) TestRecursiveListingWithLocalFiles() { // Check if mntDir/explicitFoo/ has correct objects. if path == mntDir+"/explicitFoo" { // numberOfObjects = 1 - AssertEq(1, len(objs)) + require.Equal(t.T(), 1, len(objs)) t.verifyLocalFileEntry(objs[0], explicitLocalFileName, 0) } // Check if mntDir/implicitFoo/ has correct objects. if path == mntDir+"/implicitFoo" { // numberOfObjects = 2 - AssertEq(2, len(objs)) + require.Equal(t.T(), 2, len(objs)) t.verifyLocalFileEntry(objs[0], "bar", 0) t.verifyLocalFileEntry(objs[1], implicitLocalFileName, 0) } @@ -467,35 +481,32 @@ func (t *LocalFileTest) TestRecursiveListingWithLocalFiles() { }) // Validate and close the files. - AssertEq(nil, err) + require.NoError(t.T(), err) t.closeFileAndValidateObjectContents(&t.f1, "implicitFoo/"+implicitLocalFileName, "") t.closeFileAndValidateObjectContents(&t.f2, "explicitFoo/"+explicitLocalFileName, "") t.closeFileAndValidateObjectContents(&t.f3, ""+FileName, "") } -func (t *LocalFileTest) TestRenameOfLocalFileFails() { +func (t *LocalFileTest) TestRenameOfLocalFile() { // Create local file with some content. _, t.f1 = t.createLocalFile(FileName) _, err := t.f1.WriteString(FileContents) - AssertEq(nil, err) + require.NoError(t.T(), err) + newName := "newName" // Attempt to rename local file. - err = os.Rename(path.Join(mntDir, FileName), path.Join(mntDir, "newName")) + err = os.Rename(path.Join(mntDir, FileName), path.Join(mntDir, newName)) // Verify rename operation fails. - AssertNe(nil, err) - AssertTrue(strings.Contains(err.Error(), "operation not supported")) - // write more content to local file. - _, err = t.f1.WriteString(FileContents) - AssertEq(nil, err) + require.NoError(t.T(), err) // Close the local file. - t.closeFileAndValidateObjectContents(&t.f1, FileName, FileContents+FileContents) + t.validateObjectContents(newName, FileContents) + t.validateObjectNotFoundErr(FileName) } func (t *LocalFileTest) TestRenameOfDirectoryWithLocalFileFails() { // Create directory foo. - AssertEq( - nil, + require.NoError(t.T(), t.createObjects( map[string]string{ "foo/": "", @@ -504,33 +515,21 @@ func (t *LocalFileTest) TestRenameOfDirectoryWithLocalFileFails() { // Create local file with some content. _, t.f1 = t.createLocalFile("foo/" + FileName) _, err := t.f1.WriteString(FileContents) - AssertEq(nil, err) + require.NoError(t.T(), err) // Attempt to rename directory containing local file. err = os.Rename(path.Join(mntDir, "foo/"), path.Join(mntDir, "bar/")) // Verify rename operation fails. - AssertNe(nil, err) - AssertTrue(strings.Contains(err.Error(), "operation not supported")) + require.Error(t.T(), err) + assert.True(t.T(), strings.Contains(err.Error(), "operation not supported")) // write more content to local file. _, err = t.f1.WriteString(FileContents) - AssertEq(nil, err) + require.NoError(t.T(), err) // Close the local file. t.closeFileAndValidateObjectContents(&t.f1, "foo/"+FileName, FileContents+FileContents) } -func (t *LocalFileTest) TestRenameOfLocalFileSucceedsAfterSync() { - t.TestRenameOfLocalFileFails() - - // Attempt to Rename synced file. - err := os.Rename(path.Join(mntDir, FileName), path.Join(mntDir, "newName")) - - // Validate. - AssertEq(nil, err) - t.validateObjectContents("newName", FileContents+FileContents) - t.validateObjectNotFoundErr(FileName) -} - func (t *LocalFileTest) TestRenameOfDirectoryWithLocalFileSucceedsAfterSync() { t.TestRenameOfDirectoryWithLocalFileFails() @@ -538,7 +537,7 @@ func (t *LocalFileTest) TestRenameOfDirectoryWithLocalFileSucceedsAfterSync() { err := os.Rename(path.Join(mntDir, "foo/"), path.Join(mntDir, "bar/")) // Validate. - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateObjectContents("bar/"+FileName, FileContents+FileContents) t.validateObjectNotFoundErr("foo/" + FileName) t.validateObjectContents("bar/gcsFile", "") @@ -552,16 +551,16 @@ func (t *LocalFileTest) ReadLocalFile() { // Write some contents to file. contents := "string1string2string3" _, err := t.f1.Write([]byte(contents)) - AssertEq(nil, err) + require.NoError(t.T(), err) // File shouldn't get created on GCS. t.validateObjectNotFoundErr(FileName) // Read the local file contents. buf := make([]byte, len(contents)) n, err := t.f1.ReadAt(buf, 0) - AssertEq(nil, err) - AssertEq(len(contents), n) - AssertEq(contents, string(buf)) + require.NoError(t.T(), err) + assert.Equal(t.T(), len(contents), n) + assert.Equal(t.T(), contents, string(buf)) // Close the file and validate if the file is created on GCS. t.closeFileAndValidateObjectContents(&t.f1, FileName, contents) @@ -575,13 +574,13 @@ func (t *LocalFileTest) TestReadDirContainingUnlinkedLocalFiles() { filepath3, t.f3 = t.createLocalFile(FileName + "3") // Unlink local file 3 err := os.Remove(filepath3) - AssertEq(nil, err) + require.NoError(t.T(), err) // Attempt to list mntDir. entries := t.readDirectory(mntDir) // Verify unlinked entries are not listed. - AssertEq(2, len(entries)) + require.Equal(t.T(), 2, len(entries)) t.verifyLocalFileEntry(entries[0], FileName+"1", 0) t.verifyLocalFileEntry(entries[1], FileName+"2", 0) // Close the local files. @@ -589,87 +588,86 @@ func (t *LocalFileTest) TestReadDirContainingUnlinkedLocalFiles() { t.closeFileAndValidateObjectContents(&t.f2, FileName+"2", "") // Verify unlinked file is not written to GCS err = t.closeLocalFile(&t.f3) - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateObjectNotFoundErr(FileName + "3") } func (t *LocalFileTest) TestUnlinkOfLocalFile() { // Create empty local file. - var filepath string - filepath, t.f1 = t.createLocalFile(FileName) + var filePath string + filePath, t.f1 = t.createLocalFile(FileName) // Attempt to unlink local file. - err := os.Remove(filepath) + err := os.Remove(filePath) // Verify unlink operation succeeds. - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateNoFileOrDirError(FileName) err = t.closeLocalFile(&t.f1) - AssertEq(nil, err) + require.NoError(t.T(), err) // Validate file it is not present on GCS. t.validateObjectNotFoundErr(FileName) } func (t *LocalFileTest) TestWriteOnUnlinkedLocalFileSucceeds() { // Create local file and unlink. - var filepath string - filepath, t.f1 = t.createLocalFile(FileName) - err := os.Remove(filepath) + var filePath string + filePath, t.f1 = t.createLocalFile(FileName) + err := os.Remove(filePath) // Verify unlink operation succeeds. - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateNoFileOrDirError(FileName) // Write to unlinked local file. _, err = t.f1.WriteString(FileContents) - AssertEq(nil, err) + require.NoError(t.T(), err) err = t.closeLocalFile(&t.f1) // Validate flush file does not throw error. - AssertEq(nil, err) + require.NoError(t.T(), err) // Validate unlinked file is not written to GCS t.validateObjectNotFoundErr(FileName) } func (t *LocalFileTest) TestSyncOnUnlinkedLocalFile() { // Create local file. - var filepath string - filepath, t.f1 = t.createLocalFile(FileName) + var filePath string + filePath, t.f1 = t.createLocalFile(FileName) // Attempt to unlink local file. - err := os.Remove(filepath) + err := os.Remove(filePath) // Verify unlink operation succeeds. - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateNoFileOrDirError(FileName) // Validate sync operation does not write to GCS after unlink. err = t.f1.Sync() - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateObjectNotFoundErr(FileName) // Close the local file and validate it is not present on GCS. err = t.closeLocalFile(&t.f1) - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateObjectNotFoundErr(FileName) } func (t *LocalFileTest) TestUnlinkOfSyncedLocalFile() { // Create local file and sync to GCS. - var filepath string - filepath, t.f1 = t.createLocalFile(FileName) + var filePath string + filePath, t.f1 = t.createLocalFile(FileName) t.closeFileAndValidateObjectContents(&t.f1, FileName, "") // Attempt to unlink synced file. - err := os.Remove(filepath) + err := os.Remove(filePath) // Verify unlink operation succeeds. - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateNoFileOrDirError(FileName) t.validateObjectNotFoundErr(FileName) } func (t *LocalFileTest) TestRmDirOfDirectoryContainingGCSAndLocalFiles() { // Create explicit directory with one synced and one local file. - AssertEq( - nil, + require.NoError(t.T(), t.createObjects( map[string]string{ // File @@ -682,16 +680,16 @@ func (t *LocalFileTest) TestRmDirOfDirectoryContainingGCSAndLocalFiles() { err := os.RemoveAll(path.Join(mntDir, "explicit")) // Verify rmDir operation succeeds. - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateNoFileOrDirError("explicit/" + explicitLocalFileName) t.validateNoFileOrDirError("explicit/foo") t.validateNoFileOrDirError("explicit") // Validate writing content to unlinked local file does not throw error _, err = t.f1.WriteString(FileContents) - AssertEq(nil, err) + require.NoError(t.T(), err) // Validate flush file throws IO error and does not create object on GCS err = t.closeLocalFile(&t.f1) - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateObjectNotFoundErr("explicit/" + explicitLocalFileName) // Validate synced files are also deleted. t.validateObjectNotFoundErr("explicit/foo") @@ -701,7 +699,7 @@ func (t *LocalFileTest) TestRmDirOfDirectoryContainingGCSAndLocalFiles() { func (t *LocalFileTest) TestRmDirOfDirectoryContainingOnlyLocalFiles() { // Create a directory with two local files. err := os.Mkdir(path.Join(mntDir, "explicit"), dirPerms) - AssertEq(nil, err) + require.NoError(t.T(), err) _, t.f1 = t.createLocalFile("explicit/" + explicitLocalFileName) _, t.f2 = t.createLocalFile("explicit/" + FileName) @@ -709,16 +707,16 @@ func (t *LocalFileTest) TestRmDirOfDirectoryContainingOnlyLocalFiles() { err = os.RemoveAll(path.Join(mntDir, "explicit")) // Verify rmDir operation succeeds. - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateNoFileOrDirError("explicit/" + explicitLocalFileName) t.validateNoFileOrDirError("explicit/" + FileName) t.validateNoFileOrDirError("explicit") // Close the local files and validate they are not present on GCS. err = t.closeLocalFile(&t.f1) - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateObjectNotFoundErr("explicit/" + explicitLocalFileName) err = t.closeLocalFile(&t.f2) - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateObjectNotFoundErr("explicit/" + FileName) // Validate directory is also deleted. t.validateObjectNotFoundErr("explicit/") @@ -726,8 +724,7 @@ func (t *LocalFileTest) TestRmDirOfDirectoryContainingOnlyLocalFiles() { func (t *LocalFileTest) TestRmDirOfDirectoryContainingOnlyGCSFiles() { // Create explicit directory with one synced and one local file. - AssertEq( - nil, + require.NoError(t.T(), t.createObjects( map[string]string{ // File @@ -740,7 +737,7 @@ func (t *LocalFileTest) TestRmDirOfDirectoryContainingOnlyGCSFiles() { err := os.RemoveAll(path.Join(mntDir, "explicit")) // Verify rmDir operation succeeds. - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateNoFileOrDirError("explicit") t.validateNoFileOrDirError("explicit/foo") t.validateNoFileOrDirError("explicit/bar") @@ -756,21 +753,21 @@ func (t *LocalFileTest) TestCreateSymlinkForLocalFile() { filePath, t.f1 = t.createLocalFile(FileName) // Writing contents to local file shouldn't create file on GCS. _, err := t.f1.Write([]byte(FileContents)) - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateObjectNotFoundErr(FileName) // Create the symlink. symlinkName := path.Join(mntDir, "bar") err = os.Symlink(filePath, symlinkName) - AssertEq(nil, err) + require.NoError(t.T(), err) // Read the link. target, err := os.Readlink(symlinkName) - AssertEq(nil, err) - ExpectEq(filePath, target) + require.NoError(t.T(), err) + assert.Equal(t.T(), filePath, target) contents, err := os.ReadFile(symlinkName) - AssertEq(nil, err) - ExpectEq(FileContents, string(contents)) + require.NoError(t.T(), err) + assert.Equal(t.T(), FileContents, string(contents)) t.closeFileAndValidateObjectContents(&t.f1, FileName, FileContents) } @@ -780,56 +777,56 @@ func (t *LocalFileTest) TestReadSymlinkForDeletedLocalFile() { filePath, t.f1 = t.createLocalFile(FileName) // Writing contents to local file shouldn't create file on GCS. _, err := t.f1.Write([]byte(FileContents)) - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateObjectNotFoundErr(FileName) // Create the symlink. symlinkName := path.Join(mntDir, "bar") err = os.Symlink(filePath, symlinkName) - AssertEq(nil, err) + require.NoError(t.T(), err) // Read the link. target, err := os.Readlink(symlinkName) - AssertEq(nil, err) - ExpectEq(filePath, target) + require.NoError(t.T(), err) + assert.Equal(t.T(), filePath, target) // Remove filePath and then close the fileHandle to avoid syncing to GCS. err = os.Remove(filePath) - AssertEq(nil, err) + require.NoError(t.T(), err) err = t.closeLocalFile(&t.f1) - AssertEq(nil, err) + require.NoError(t.T(), err) t.validateObjectNotFoundErr(FileName) // Reading symlink should fail. _, err = os.Stat(symlinkName) - AssertTrue(strings.Contains(err.Error(), "no such file or directory")) + assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) } -func (t *LocalFileTest) AtimeMtimeAndCtime() { +func (t *LocalFileTest) TestAtimeMtimeAndCtime() { createTime := mtimeClock.Now() var filePath string // Create a local file. filePath, t.f1 = t.createLocalFile(FileName) var err error fi, err := os.Stat(filePath) - AssertEq(nil, err) + require.NoError(t.T(), err) // Check if mtime is returned correctly for unsynced file. _, _, mtime := fusetesting.GetTimes(fi) - ExpectThat(mtime, timeutil.TimeNear(createTime, Delta)) + assert.WithinDuration(t.T(), createTime, mtime, Delta) // Write some contents. _, err = t.f1.Write([]byte("test contents")) - AssertEq(nil, err) + require.NoError(t.T(), err) // Stat it. fi, err = os.Stat(filePath) - AssertEq(nil, err) + require.NoError(t.T(), err) // We require only that atime and ctime be "reasonable". atime, ctime, mtime := fusetesting.GetTimes(fi) - ExpectThat(mtime, timeutil.TimeNear(createTime, Delta)) - ExpectThat(atime, timeutil.TimeNear(createTime, Delta)) - ExpectThat(ctime, timeutil.TimeNear(createTime, Delta)) + assert.WithinDuration(t.T(), createTime, mtime, Delta) + assert.WithinDuration(t.T(), createTime, atime, Delta) + assert.WithinDuration(t.T(), createTime, ctime, Delta) } // Create local file inside - test.txt @@ -839,23 +836,22 @@ func (t *LocalFileTest) AtimeMtimeAndCtime() { // Stat that local file. func (t *LocalFileTest) TestStatLocalFileAfterRecreatingItWithSameName() { filePath := path.Join(mntDir, "test.txt") - AssertEq(nil, err) f1, err := os.Create(filePath) - defer AssertEq(nil, f1.Close()) - AssertEq(nil, err) + defer require.NoError(t.T(), f1.Close()) + require.NoError(t.T(), err) _, err = os.Stat(filePath) - AssertEq(nil, err) + require.NoError(t.T(), err) err = os.Remove(filePath) - AssertEq(nil, err) + require.NoError(t.T(), err) f2, err := os.Create(filePath) - AssertEq(nil, err) - defer AssertEq(nil, f2.Close()) + require.NoError(t.T(), err) + defer require.NoError(t.T(), f2.Close()) f, err := os.Stat(filePath) - AssertEq(nil, err) - ExpectEq("test.txt", f.Name()) - ExpectFalse(f.IsDir()) + require.NoError(t.T(), err) + assert.Equal(t.T(), "test.txt", f.Name()) + assert.False(t.T(), f.IsDir()) } func (t *LocalFileTest) TestStatFailsOnNewFileAfterDeletion() { @@ -868,14 +864,15 @@ func (t *LocalFileTest) TestStatFailsOnNewFileAfterDeletion() { }, Logging: cfg.DefaultLoggingConfig(), } + t.serverCfg.MetricHandle = metrics.NewNoopMetrics() + t.serverCfg.TraceHandle = tracing.NewNoopTracer() filePath := path.Join(mntDir, "test.txt") - AssertEq(nil, err) f1, err := os.Create(filePath) - AssertEq(nil, err) - defer AssertEq(nil, f1.Close()) - AssertEq(nil, os.Remove(filePath)) + require.NoError(t.T(), err) + defer assert.Equal(t.T(), nil, f1.Close()) + assert.Equal(t.T(), nil, os.Remove(filePath)) _, err = os.Stat(filePath) - AssertNe(nil, err) + require.Error(t.T(), err) } diff --git a/internal/fs/local_modifications_test.go b/internal/fs/local_modifications_test.go index 4de361ace7..25d9893bb2 100644 --- a/internal/fs/local_modifications_test.go +++ b/internal/fs/local_modifications_test.go @@ -33,10 +33,9 @@ import ( "unicode" "unicode/utf8" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" "github.com/jacobsa/fuse/fusetesting" . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/ogletest" @@ -115,7 +114,7 @@ func interestingLegalNames() (names []string) { } // Most single-byte UTF-8 strings. - for b := byte(0); b < utf8.RuneSelf; b++ { + for b := range byte(utf8.RuneSelf) { switch b { // NULL and '/' are not legal in file names. case 0, '/': @@ -132,7 +131,7 @@ func interestingLegalNames() (names []string) { // All codepoints in Unicode general categories C* (control and special) and // Z* (space), except for: // - // * Cn (non-character and reserved), which is not included in unicode.C. + // * Cn (non-character and reserved), which is large. // * Co (private usage), which is large. // * Cs (surrages), which is large. // * U+0000, which is forbidden in paths by Go @@ -144,6 +143,10 @@ func interestingLegalNames() (names []string) { continue } + if unicode.In(r, unicode.Cn) { + continue + } + if unicode.In(r, unicode.Co) { continue } @@ -1245,6 +1248,7 @@ func (t *DirectoryTest) Rmdir_OpenedForReading() { // We should still be able to stat the open file handle. fi, err := t.f1.Stat() + AssertEq(nil, err) ExpectEq("dir", fi.Name()) // Attempt to read from the directory. Unfortunately we can't implement the @@ -1294,8 +1298,11 @@ func (t *DirectoryTest) CreateHardLink() { path.Join(mntDir, "foo"), path.Join(mntDir, "bar")) + // Kernel behavior changed with: https://github.com/torvalds/linux/commit/8344213571b2ac8caf013cfd3b37bc3467c3a893 + // Older kernels return ENOSYS (function not implemented) + // Newer kernels (6.x+) return EPERM (operation not permitted) AssertNe(nil, err) - ExpectThat(err, Error(HasSubstr("not implemented"))) + ExpectTrue(errors.Is(err, syscall.ENOSYS) || errors.Is(err, syscall.EPERM), "Expected ENOSYS or EPERM, got: %v", err) } func (t *DirectoryTest) Chmod() { @@ -1538,8 +1545,8 @@ func validateObjectAttributes(extendedAttr1, extendedAttr2 *gcs.ExtendedObjectAt ExpectEq(FileContentsSize, minObject2.Size) ExpectNe(minObject1.Generation, minObject2.Generation) ExpectTrue(minObject1.Updated.Before(minObject2.Updated)) - attr1MTime, _ := time.Parse(time.RFC3339Nano, minObject1.Metadata[gcsx.MtimeMetadataKey]) - attr2MTime, _ := time.Parse(time.RFC3339Nano, minObject2.Metadata[gcsx.MtimeMetadataKey]) + attr1MTime, _ := time.Parse(time.RFC3339Nano, minObject1.Metadata[gcs.MtimeMetadataKey]) + attr2MTime, _ := time.Parse(time.RFC3339Nano, minObject2.Metadata[gcs.MtimeMetadataKey]) ExpectTrue(attr1MTime.Before(attr2MTime)) ExpectEq(minObject1.ContentEncoding, minObject2.ContentEncoding) ExpectNe(nil, minObject1.CRC32C) @@ -1584,6 +1591,8 @@ func (t *FileTest) AppendFileOperation_ShouldNotChangeObjectAttributes() { minObject2, extendedAttr2, err := bucket.StatObject(ctx, &gcs.StatObjectRequest{Name: fileName, ForceFetchFromGcs: true, ReturnExtendedObjectAttributes: true}) AssertEq(nil, err) // Validate object attributes are as expected. + // TODO: Validate on Finalized attribute once the default behavior on GCSFuse + // side is to never finalize object. validateObjectAttributes(extendedAttr1, extendedAttr2, minObject1, minObject2) } @@ -1607,6 +1616,8 @@ func (t *FileTest) WriteAtFileOperation_ShouldNotChangeObjectAttributes() { AssertEq(nil, err) // Validate object attributes are as expected. + // TODO: Validate on Finalized attribute once the default behavior on GCSFuse + // side is to never finalize object. validateObjectAttributes(extendedAttr1, extendedAttr2, minObject1, minObject2) } @@ -1876,7 +1887,9 @@ func (t *FileTest) UnlinkFile_StillOpen() { // Create and open a file. f, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE, 0600) AssertEq(nil, err) - defer f.Close() + defer func() { + _ = f.Close() + }() // Write some data into it. n, err := f.Write([]byte("taco")) @@ -2038,7 +2051,9 @@ func (t *FileTest) Chtimes_OpenFile_Clean() { // Open it for reading. f, err := os.Open(p) AssertEq(nil, err) - defer f.Close() + defer func() { + _ = f.Close() + }() // Change its mtime. newMtime := time.Date(2012, 8, 15, 22, 56, 0, 0, time.Local) @@ -2071,7 +2086,9 @@ func (t *FileTest) Chtimes_OpenFile_Dirty() { p := path.Join(mntDir, "foo") f, err := os.Create(p) AssertEq(nil, err) - defer f.Close() + defer func() { + _ = f.Close() + }() // Dirty the file. _, err = f.Write([]byte("taco")) @@ -2160,11 +2177,14 @@ func (t *FileTest) Sync_Clobbered() { var n int // Create a file. - t.f1, err = os.Create(path.Join(mntDir, "foo")) + f, err := os.Create(path.Join(mntDir, "foo")) AssertEq(nil, err) + defer func() { + _ = f.Close() + }() // Dirty the file by giving it some contents. - n, err = t.f1.Write([]byte("taco")) + n, err = f.Write([]byte("taco")) AssertEq(nil, err) AssertEq(4, n) @@ -2181,9 +2201,9 @@ func (t *FileTest) Sync_Clobbered() { // decided to hold back the writes from above until now (in which case the // inode will fail to load the source object), or it may fail silently. // Either way, this should not result in a new generation being created. - err = t.f1.Sync() + err = f.Sync() if err != nil { - ExpectThat(err, Error(HasSubstr("input/output error"))) + ExpectTrue(errors.Is(err, syscall.ESTALE), "err: %v", err) } contents, err := storageutil.ReadObject(ctx, bucket, "foo") @@ -2242,7 +2262,9 @@ func (t *FileTest) Close_Clobbered() { // Create a file. f, err := os.Create(path.Join(mntDir, "foo")) AssertEq(nil, err) - defer f.Close() + defer func() { + _ = f.Close() + }() // Dirty the file by giving it some contents. n, err = f.Write([]byte("taco")) @@ -2262,7 +2284,7 @@ func (t *FileTest) Close_Clobbered() { // faulting in the object's contents on Linux where close may cause cached // writes to be delivered to the file system. But in any case the new // generation should not be replaced. - f.Close() + _ = f.Close() contents, err := storageutil.ReadObject(ctx, bucket, "foo") AssertEq(nil, err) @@ -2499,7 +2521,7 @@ func (t *RenameTest) DirectoryContainingFiles() { err = os.Mkdir(oldPath, 0700) AssertEq(nil, err) - for i := 0; i < int(RenameDirLimit); i++ { + for i := range int(RenameDirLimit) { file := fmt.Sprintf("%s/%d.txt", oldPath, i) err = os.WriteFile(file, []byte("taco"), 0400) AssertEq(nil, err) @@ -2705,7 +2727,9 @@ func (t *RenameTest) IntoFileSystem() { // Create a file outside of our file system. f, err := os.CreateTemp("", "memfs_test") AssertEq(nil, err) - defer f.Close() + defer func() { + _ = f.Close() + }() oldPath := f.Name() defer os.Remove(oldPath) diff --git a/internal/fs/metrics_test.go b/internal/fs/metrics_test.go new file mode 100644 index 0000000000..6bf36b143f --- /dev/null +++ b/internal/fs/metrics_test.go @@ -0,0 +1,1661 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs_test + +import ( + "context" + "os" + "syscall" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/wrappers" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/jacobsa/fuse" + "github.com/jacobsa/fuse/fuseops" + "github.com/jacobsa/fuse/fuseutil" + "github.com/jacobsa/timeutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/metric" +) + +// serverConfigParams holds parameters for creating a test file system. +type serverConfigParams struct { + enableBufferedRead bool + enableNewReader bool + enableFileCache bool + enableSparseFileCache bool + // enableFileCacheForRangeRead controls if the file cache is used for random reads. + enableFileCacheForRangeRead bool + // enableKernelReader controls if the MrdKernelReader is enabled. + enableKernelReader bool + enableParallelDownloads bool + enableParallelDownloadsBlocking bool + enableStreamingWrites bool + writeGlobalMaxBlocks int +} + +func defaultServerConfigParams() *serverConfigParams { + return &serverConfigParams{ + enableBufferedRead: false, + enableNewReader: true, + enableFileCache: false, + enableFileCacheForRangeRead: true, + enableKernelReader: false, + enableParallelDownloads: false, + enableParallelDownloadsBlocking: false, + enableStreamingWrites: false, + writeGlobalMaxBlocks: 1, + } +} + +func createTestFileSystemWithMetrics(ctx context.Context, t *testing.T, params *serverConfigParams, isZonalBucket bool) (gcs.Bucket, fuseutil.FileSystem, metrics.MetricHandle, *metric.ManualReader) { + t.Helper() + origProvider := otel.GetMeterProvider() + t.Cleanup(func() { otel.SetMeterProvider(origProvider) }) + reader := metric.NewManualReader() + provider := metric.NewMeterProvider(metric.WithReader(reader)) + otel.SetMeterProvider(provider) + + mh, err := metrics.NewOTelMetrics(ctx, 1, 100) + require.NoError(t, err, "metrics.NewOTelMetrics") + bucketName := "test-bucket" + bucketType := gcs.BucketType{Hierarchical: false} + if isZonalBucket { + bucketType = gcs.BucketType{Zonal: true} + } + bucket := fake.NewFakeBucket(timeutil.RealClock(), bucketName, bucketType) + serverCfg := &fs.ServerConfig{ + NewConfig: &cfg.Config{ + Write: cfg.WriteConfig{ + GlobalMaxBlocks: int64(params.writeGlobalMaxBlocks), + EnableStreamingWrites: params.enableStreamingWrites, + BlockSizeMb: 1, + MaxBlocksPerFile: 10, + }, + Read: cfg.ReadConfig{ + EnableBufferedRead: params.enableBufferedRead, + GlobalMaxBlocks: 1, + BlockSizeMb: 1, + MaxBlocksPerHandle: 10, + }, + EnableNewReader: params.enableNewReader, + FileSystem: cfg.FileSystemConfig{ + EnableKernelReader: params.enableKernelReader, + }, + }, + MetricHandle: mh, + TraceHandle: tracing.NewNoopTracer(), + CacheClock: &timeutil.SimulatedClock{}, + BucketName: bucketName, + BucketManager: &fakeBucketManager{ + buckets: map[string]gcs.Bucket{ + bucketName: bucket, + }, + }, + SequentialReadSizeMb: 200, + } + + if params.enableFileCache || params.enableSparseFileCache { + cacheDir := t.TempDir() + t.Cleanup(func() { + os.RemoveAll(cacheDir) + }) + serverCfg.NewConfig.CacheDir = cfg.ResolvedPath(cacheDir) + serverCfg.NewConfig.FileCache = cfg.FileCacheConfig{ + MaxSizeMb: 100, + CacheFileForRangeRead: params.enableFileCacheForRangeRead, + ExperimentalEnableChunkCache: params.enableSparseFileCache, + DownloadChunkSizeMb: 1, // 1MB chunks for testing + EnableParallelDownloads: params.enableParallelDownloads, + ParallelDownloadsPerFile: 16, + ExperimentalParallelDownloadsDefaultOn: params.enableParallelDownloadsBlocking, + } + } + + server, err := fs.NewFileSystem(ctx, serverCfg) + require.NoError(t, err, "NewFileSystem") + return bucket, server, mh, reader +} + +func createWithContents(ctx context.Context, t *testing.T, bucket gcs.Bucket, name string, contents string) { + err := storageutil.CreateObjects(ctx, bucket, map[string][]byte{name: []byte(contents)}) + require.NoError(t, err, "CreateObjects") +} + +func waitForMetricsProcessing() { + time.Sleep(5 * time.Millisecond) +} + +func TestLookUpInode_Metrics(t *testing.T) { + testCases := []struct { + name string + fileName string + createFile bool + expectedError error + }{ + { + name: "non-existent file", + fileName: "non_existent", + createFile: false, + expectedError: fuse.ENOENT, + }, + { + name: "existing file", + fileName: "test", + createFile: true, + expectedError: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + content := "test" + if tc.createFile { + createWithContents(ctx, t, bucket, tc.fileName, content) + } + server = wrappers.WithMonitoring(server, mh) + op := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: tc.fileName, + } + + err := server.LookUpInode(ctx, op) + waitForMetricsProcessing() + + assert.Equal(t, tc.expectedError, err) + attrs := attribute.NewSet(attribute.String("fs_op", "LookUpInode")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) + }) + } +} + +func TestReadFile_BufferedReadMetrics(t *testing.T) { + ctx := context.Background() + params := defaultServerConfigParams() + params.enableBufferedRead = true + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, params, false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + lookupOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookupOp) + require.NoError(t, err, "LookUpInode") + openOp := &fuseops.OpenFileOp{ + Inode: lookupOp.Entry.Child, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err, "OpenFile") + readOp := &fuseops.ReadFileOp{ + Inode: lookupOp.Entry.Child, + Handle: openOp.Handle, + Offset: 0, + Dst: make([]byte, len(content)), + } + + err = server.ReadFile(ctx, readOp) + waitForMetricsProcessing() + + require.NoError(t, err, "ReadFile") + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/download_bytes_count", attribute.NewSet(attribute.String("read_type", string(metrics.ReadTypeBufferedAttr))), int64(len(content))) + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/read_bytes_count", attribute.NewSet(), int64(len(content))) + metrics.VerifyHistogramMetric(t, ctx, reader, "buffered_read/read_latency", attribute.NewSet(), uint64(1)) +} + +func TestSequentialReadFile_FileCacheMetrics(t *testing.T) { + ctx := context.Background() + params := defaultServerConfigParams() + params.enableFileCache = true + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, params, false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + lookupOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookupOp) + require.NoError(t, err, "LookUpInode") + openOp := &fuseops.OpenFileOp{ + Inode: lookupOp.Entry.Child, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err, "OpenFile") + readOp := &fuseops.ReadFileOp{ + Inode: lookupOp.Entry.Child, + Handle: openOp.Handle, + Offset: 0, + Dst: make([]byte, len(content)), + } + + // First read should be a cache miss. + err = server.ReadFile(ctx, readOp) + require.NoError(t, err, "ReadFile") + waitForMetricsProcessing() + + // first read is a miss and second is a hit. + metrics.VerifyCounterMetric( + t, ctx, reader, "file_cache/read_count", + attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("read_type", string(metrics.ReadTypeSequentialAttr))), + int64(1), + ) + metrics.VerifyCounterMetric( + t, ctx, reader, "file_cache/read_bytes_count", + attribute.NewSet(attribute.String("read_type", string(metrics.ReadTypeSequentialAttr))), + int64(len(content)), + ) + metrics.VerifyHistogramMetric( + t, ctx, reader, "file_cache/read_latencies", + attribute.NewSet(attribute.Bool("cache_hit", false)), + uint64(1), + ) + + // Subsequent read should be a cache hit. + err = server.ReadFile(ctx, readOp) + require.NoError(t, err, "ReadFile") + waitForMetricsProcessing() + + metrics.VerifyCounterMetric( + t, ctx, reader, "file_cache/read_count", + attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("read_type", string(metrics.ReadTypeSequentialAttr))), + int64(1), + ) + metrics.VerifyCounterMetric( + t, ctx, reader, "file_cache/read_bytes_count", + attribute.NewSet(attribute.String("read_type", string(metrics.ReadTypeSequentialAttr))), + int64(24), + ) + metrics.VerifyHistogramMetric( + t, ctx, reader, "file_cache/read_latencies", + attribute.NewSet(attribute.Bool("cache_hit", true)), + uint64(1), + ) +} + +func TestSequentialReadFile_FileCacheMetrics_DisabledFileCacheForRangeRead(t *testing.T) { + ctx := context.Background() + params := defaultServerConfigParams() + params.enableFileCache = true + params.enableFileCacheForRangeRead = false + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, params, false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test.txt" + fileSize := 100 + content := string(make([]byte, fileSize)) + createWithContents(ctx, t, bucket, fileName, content) + lookupOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookupOp) + require.NoError(t, err, "LookUpInode") + openOp := &fuseops.OpenFileOp{ + Inode: lookupOp.Entry.Child, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err, "OpenFile") + readOp := &fuseops.ReadFileOp{ + Inode: lookupOp.Entry.Child, + Handle: openOp.Handle, + Offset: 0, + Dst: make([]byte, fileSize/2), + } + + // First read should be a cache miss. + err = server.ReadFile(ctx, readOp) + require.NoError(t, err, "ReadFile") + waitForMetricsProcessing() + + // first read is a miss. + metrics.VerifyCounterMetric( + t, ctx, reader, "file_cache/read_count", + attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("read_type", string(metrics.ReadTypeSequentialAttr))), + int64(1), + ) + metrics.VerifyCounterMetric( + t, ctx, reader, "file_cache/read_bytes_count", + attribute.NewSet(attribute.String("read_type", string(metrics.ReadTypeSequentialAttr))), + int64(fileSize/2), + ) + metrics.VerifyHistogramMetric( + t, ctx, reader, "file_cache/read_latencies", + attribute.NewSet(attribute.Bool("cache_hit", false)), + uint64(1), + ) + + // Subsequent read should be a cache hit as sequential reads are always cached. + readOp.Offset = int64(fileSize / 2) + err = server.ReadFile(ctx, readOp) + require.NoError(t, err, "ReadFile") + waitForMetricsProcessing() + + metrics.VerifyCounterMetric( + t, ctx, reader, "file_cache/read_count", + attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("read_type", string(metrics.ReadTypeSequentialAttr))), + int64(1), + ) + metrics.VerifyCounterMetric( + t, ctx, reader, "file_cache/read_bytes_count", + attribute.NewSet(attribute.String("read_type", string(metrics.ReadTypeSequentialAttr))), + int64(fileSize), + ) + metrics.VerifyHistogramMetric( + t, ctx, reader, "file_cache/read_latencies", + attribute.NewSet(attribute.Bool("cache_hit", true)), + uint64(1), + ) +} + +func TestRandomReadFile_FileCacheMetrics(t *testing.T) { + ctx := context.Background() + params := defaultServerConfigParams() + params.enableFileCache = true + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, params, false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + lookupOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookupOp) + require.NoError(t, err, "LookUpInode") + openOp := &fuseops.OpenFileOp{ + Inode: lookupOp.Entry.Child, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err, "OpenFile") + readOp := &fuseops.ReadFileOp{ + Inode: lookupOp.Entry.Child, + Handle: openOp.Handle, + Offset: 3, + Dst: make([]byte, 2), + } + + // First read should be a cache miss. + err = server.ReadFile(ctx, readOp) + require.NoError(t, err, "ReadFile") + waitForMetricsProcessing() + + // first read is a miss and file_cache/read_bytes_count won't be recorded. + metrics.VerifyCounterMetric( + t, ctx, reader, "file_cache/read_count", + attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("read_type", string(metrics.ReadTypeRandomAttr))), + int64(1), + ) + metrics.VerifyHistogramMetric( + t, ctx, reader, "file_cache/read_latencies", + attribute.NewSet(attribute.Bool("cache_hit", false)), + uint64(1), + ) + + // Subsequent read should be a cache hit. + err = server.ReadFile(ctx, readOp) + require.NoError(t, err, "ReadFile") + waitForMetricsProcessing() + + metrics.VerifyCounterMetric( + t, ctx, reader, "file_cache/read_count", + attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("read_type", string(metrics.ReadTypeRandomAttr))), + int64(1), + ) + metrics.VerifyCounterMetric( + t, ctx, reader, "file_cache/read_bytes_count", + attribute.NewSet(attribute.String("read_type", string(metrics.ReadTypeRandomAttr))), + int64(len(readOp.Dst)), // 2 bytes read from file cache + ) + metrics.VerifyHistogramMetric( + t, ctx, reader, "file_cache/read_latencies", + attribute.NewSet(attribute.Bool("cache_hit", true)), + uint64(1), + ) + + // Read at a different offset should be a cache hit since file is downloaded in FileCache. + readOp.Offset = 8 + err = server.ReadFile(ctx, readOp) + require.NoError(t, err, "ReadFile") + waitForMetricsProcessing() + + metrics.VerifyCounterMetric( + t, ctx, reader, "file_cache/read_count", + attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("read_type", string(metrics.ReadTypeRandomAttr))), + int64(2), + ) + metrics.VerifyCounterMetric( + t, ctx, reader, "file_cache/read_bytes_count", + attribute.NewSet(attribute.String("read_type", string(metrics.ReadTypeRandomAttr))), + int64(2*len(readOp.Dst)), + ) + metrics.VerifyHistogramMetric( + t, ctx, reader, "file_cache/read_latencies", + attribute.NewSet(attribute.Bool("cache_hit", true)), + uint64(2), + ) +} + +func TestRandomReadFile_FileCacheMetrics_DisabledFileCacheOnRangedRead(t *testing.T) { + ctx := context.Background() + params := defaultServerConfigParams() + params.enableFileCache = true + params.enableFileCacheForRangeRead = false + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, params, false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + lookupOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookupOp) + require.NoError(t, err, "LookUpInode") + openOp := &fuseops.OpenFileOp{ + Inode: lookupOp.Entry.Child, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err, "OpenFile") + readOp := &fuseops.ReadFileOp{ + Inode: lookupOp.Entry.Child, + Handle: openOp.Handle, + Offset: 3, + Dst: make([]byte, 2), + } + + // This read should be a cache miss as cacheFileForRangeRead is false and it is a random read. + err = server.ReadFile(ctx, readOp) + require.NoError(t, err, "ReadFile") + waitForMetricsProcessing() + + metrics.VerifyCounterMetric( + t, ctx, reader, "file_cache/read_count", + attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("read_type", string(metrics.ReadTypeRandomAttr))), + int64(1), + ) + metrics.VerifyHistogramMetric( + t, ctx, reader, "file_cache/read_latencies", + attribute.NewSet(attribute.Bool("cache_hit", false)), + uint64(1), + ) + + // Subsequent read should also be a cache miss as range read is disabled. + err = server.ReadFile(ctx, readOp) + require.NoError(t, err, "ReadFile") + waitForMetricsProcessing() + + metrics.VerifyCounterMetric( + t, ctx, reader, "file_cache/read_count", + attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("read_type", string(metrics.ReadTypeRandomAttr))), + int64(2), + ) + metrics.VerifyHistogramMetric( + t, ctx, reader, "file_cache/read_latencies", + attribute.NewSet(attribute.Bool("cache_hit", false)), + uint64(2), + ) +} + +func TestSparseReadFile_GCSReadMetrics(t *testing.T) { + ctx := context.Background() + params := defaultServerConfigParams() + params.enableSparseFileCache = true + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, params, false) + server = wrappers.WithMonitoring(server, mh) + fileName := "sparse_test.txt" + // Create a file larger than the chunk size (1MB) to test sparse behavior. + // With a 3MB file and 1MB chunks, reading from the middle should only + // download that chunk, not the entire file. + chunkSize := 1024 * 1024 // 1MB chunk size configured in test + fileSize := 3 * chunkSize + content := string(make([]byte, fileSize)) + createWithContents(ctx, t, bucket, fileName, content) + lookupOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookupOp) + require.NoError(t, err, "LookUpInode") + openOp := &fuseops.OpenFileOp{ + Inode: lookupOp.Entry.Child, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err, "OpenFile") + // Read from the middle chunk (offset 1.5MB) to trigger sparse download + // of only that chunk, not the entire file + readOp := &fuseops.ReadFileOp{ + Inode: lookupOp.Entry.Child, + Handle: openOp.Handle, + Offset: int64(chunkSize + chunkSize/2), // 1.5MB offset + Dst: make([]byte, 100), + } + + // First read triggers sparse download from GCS + err = server.ReadFile(ctx, readOp) + require.NoError(t, err, "ReadFile") + waitForMetricsProcessing() + + // Verify GCS read metrics for sparse download with Random read type. + // Only the chunk containing the read offset should be downloaded (1MB), + // not the entire file (3MB), demonstrating sparse download behavior. + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/read_count", attribute.NewSet(attribute.String("read_type", string(metrics.ReadTypeRandomAttr))), int64(1)) + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/download_bytes_count", attribute.NewSet(attribute.String("read_type", string(metrics.ReadTypeRandomAttr))), int64(chunkSize)) +} + +func TestReadFile_MrdKernelReaderMetrics(t *testing.T) { + ctx := context.Background() + params := defaultServerConfigParams() + params.enableKernelReader = true + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, params, true) + server = wrappers.WithMonitoring(server, mh) + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + lookupOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookupOp) + require.NoError(t, err, "LookUpInode") + openOp := &fuseops.OpenFileOp{ + Inode: lookupOp.Entry.Child, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err, "OpenFile") + readOp := &fuseops.ReadFileOp{ + Inode: lookupOp.Entry.Child, + Handle: openOp.Handle, + Offset: 0, + Dst: make([]byte, len(content)), + } + + err = server.ReadFile(ctx, readOp) + require.NoError(t, err, "ReadFile") + waitForMetricsProcessing() + + require.NoError(t, err, "ReadFile") + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/read_count", attribute.NewSet(attribute.String("read_type", string(metrics.ReadTypeParallelAttr))), int64(1)) + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/download_bytes_count", attribute.NewSet(attribute.String("read_type", string(metrics.ReadTypeParallelAttr))), int64(len(content))) + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/read_bytes_count", attribute.NewSet(), int64(len(content))) + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/request_count", attribute.NewSet(attribute.String("gcs_method", "MultiRangeDownloader::Add")), int64(1)) + metrics.VerifyHistogramMetric(t, ctx, reader, "gcs/request_latencies", attribute.NewSet(attribute.String("gcs_method", "MultiRangeDownloader::Add")), uint64(1)) +} + +func TestReadFile_GCSReaderSequentialReadMetrics(t *testing.T) { + testCases := []struct { + name string + enableNewReader bool + }{ + {"NewReader", true}, + {"OldReader", false}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + params := defaultServerConfigParams() + params.enableNewReader = tc.enableNewReader + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, params, false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + lookupOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookupOp) + require.NoError(t, err, "LookUpInode") + openOp := &fuseops.OpenFileOp{ + Inode: lookupOp.Entry.Child, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err, "OpenFile") + readOp := &fuseops.ReadFileOp{ + Inode: lookupOp.Entry.Child, + Handle: openOp.Handle, + Offset: 0, + Dst: make([]byte, len(content)), + } + + err = server.ReadFile(ctx, readOp) + waitForMetricsProcessing() + + require.NoError(t, err, "ReadFile") + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/read_count", attribute.NewSet(attribute.String("read_type", string(metrics.ReadTypeSequentialAttr))), int64(1)) + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/download_bytes_count", attribute.NewSet(attribute.String("read_type", string(metrics.ReadTypeSequentialAttr))), int64(len(content))) + }) + } +} + +func TestReadFile_GCSReaderRandomReadMetrics(t *testing.T) { + ctx := context.Background() + params := defaultServerConfigParams() + params.enableNewReader = true + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, params, false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + lookupOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookupOp) + require.NoError(t, err, "LookUpInode") + openOp := &fuseops.OpenFileOp{ + Inode: lookupOp.Entry.Child, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err, "OpenFile") + + // Perform a random read at offset 10, 5, 3, and 0 in order. + readOp := &fuseops.ReadFileOp{ + Inode: lookupOp.Entry.Child, + Handle: openOp.Handle, + Offset: 10, + Dst: make([]byte, len(content)), + } + err = server.ReadFile(ctx, readOp) // Sequential read of 2 bytes (12 - 10). + require.NoError(t, err, "ReadFile") + readOp.Offset = 5 + err = server.ReadFile(ctx, readOp) // Sequential read of 7 bytes (12 - 5). + require.NoError(t, err, "ReadFile") + readOp.Offset = 3 + err = server.ReadFile(ctx, readOp) // Random read of 9 bytes (12 - 3). + require.NoError(t, err, "ReadFile") + readOp.Offset = 0 + err = server.ReadFile(ctx, readOp) // Random read of 12 bytes (12 - 0). + require.NoError(t, err, "ReadFile") + waitForMetricsProcessing() + + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/read_count", attribute.NewSet(attribute.String("read_type", string(metrics.ReadTypeRandomAttr))), int64(4)) + metrics.VerifyCounterMetric(t, ctx, reader, "gcs/download_bytes_count", attribute.NewSet(attribute.String("read_type", string(metrics.ReadTypeRandomAttr))), int64(30)) +} + +func TestGetInodeAttributes_Metrics(t *testing.T) { + ctx := context.Background() + _, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + op := &fuseops.GetInodeAttributesOp{ + Inode: fuseops.RootInodeID, + } + + err := server.GetInodeAttributes(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "GetInodeAttributes")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestRemoveXattr_Metrics(t *testing.T) { + ctx := context.Background() + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test" + createWithContents(ctx, t, bucket, fileName, "test") + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + op := &fuseops.RemoveXattrOp{ + Inode: lookUpOp.Entry.Child, + Name: "user.test", + } + + err = server.RemoveXattr(ctx, op) + waitForMetricsProcessing() + + // The operation is not implemented, so we expect an error. + assert.Error(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "Others")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestListXattr_Metrics(t *testing.T) { + ctx := context.Background() + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test" + createWithContents(ctx, t, bucket, fileName, "test") + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + op := &fuseops.ListXattrOp{ + Inode: lookUpOp.Entry.Child, + } + + err = server.ListXattr(ctx, op) + waitForMetricsProcessing() + + // The operation is not implemented, so we expect an error. + assert.NotNil(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "Others")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestSetXattr_Metrics(t *testing.T) { + ctx := context.Background() + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test" + createWithContents(ctx, t, bucket, fileName, "test") + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + op := &fuseops.SetXattrOp{ + Inode: lookUpOp.Entry.Child, + Name: "user.test", + Value: []byte("test"), + } + + err = server.SetXattr(ctx, op) + waitForMetricsProcessing() + + // The operation is not implemented, so we expect an error. + assert.NotNil(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "Others")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestGetXattr_Metrics(t *testing.T) { + ctx := context.Background() + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test" + createWithContents(ctx, t, bucket, fileName, "test") + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + op := &fuseops.GetXattrOp{ + Inode: lookUpOp.Entry.Child, + Name: "user.test", + } + + err = server.GetXattr(ctx, op) + waitForMetricsProcessing() + + // The operation is not implemented, so we expect an error. + assert.NotNil(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "Others")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestFallocate_Metrics(t *testing.T) { + ctx := context.Background() + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test" + createWithContents(ctx, t, bucket, fileName, "test") + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + openOp := &fuseops.OpenFileOp{ + Inode: lookUpOp.Entry.Child, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err) + op := &fuseops.FallocateOp{ + Inode: lookUpOp.Entry.Child, + Handle: openOp.Handle, + Offset: 0, + Length: 10, + Mode: 0, + } + + err = server.Fallocate(ctx, op) + waitForMetricsProcessing() + + // The operation is not implemented, so we expect an error. + assert.Error(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "Others")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestCreateLink_Metrics(t *testing.T) { + ctx := context.Background() + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test" + createWithContents(ctx, t, bucket, fileName, "test") + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + op := &fuseops.CreateLinkOp{ + Parent: fuseops.RootInodeID, + Name: "link", + Target: lookUpOp.Entry.Child, + } + + err = server.CreateLink(ctx, op) + waitForMetricsProcessing() + + // The operation is not implemented, so we expect an error. + assert.Error(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "CreateLink")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestStatFS_Metrics(t *testing.T) { + ctx := context.Background() + _, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + op := &fuseops.StatFSOp{} + + err := server.StatFS(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "Others")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestReleaseFileHandle_Metrics(t *testing.T) { + ctx := context.Background() + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test" + createWithContents(ctx, t, bucket, fileName, "") + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + openOp := &fuseops.OpenFileOp{ + Inode: lookUpOp.Entry.Child, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err) + op := &fuseops.ReleaseFileHandleOp{ + Handle: openOp.Handle, + } + + err = server.ReleaseFileHandle(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "ReleaseFileHandle")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestFlushFile_Metrics(t *testing.T) { + ctx := context.Background() + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test" + createWithContents(ctx, t, bucket, fileName, "") + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + openOp := &fuseops.OpenFileOp{ + Inode: lookUpOp.Entry.Child, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err) + op := &fuseops.FlushFileOp{ + Inode: lookUpOp.Entry.Child, + Handle: openOp.Handle, + } + + err = server.FlushFile(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "FlushFile")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestSyncFile_Metrics(t *testing.T) { + ctx := context.Background() + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test" + createWithContents(ctx, t, bucket, fileName, "") + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookUpOp) + assert.NoError(t, err) + op := &fuseops.SyncFileOp{ + Inode: lookUpOp.Entry.Child, + } + + err = server.SyncFile(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "SyncFile")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestWriteFile_Metrics(t *testing.T) { + ctx := context.Background() + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test" + createWithContents(ctx, t, bucket, fileName, "") + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + openOp := &fuseops.OpenFileOp{ + Inode: lookUpOp.Entry.Child, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err) + op := &fuseops.WriteFileOp{ + Inode: lookUpOp.Entry.Child, + Handle: openOp.Handle, + Offset: 0, + Data: []byte("test"), + } + + err = server.WriteFile(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "WriteFile")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestReadSymlink_Metrics(t *testing.T) { + ctx := context.Background() + _, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + symlinkName := "test" + target := "target" + createSymlinkOp := &fuseops.CreateSymlinkOp{ + Parent: fuseops.RootInodeID, + Name: symlinkName, + Target: target, + } + err := server.CreateSymlink(ctx, createSymlinkOp) + require.NoError(t, err) + op := &fuseops.ReadSymlinkOp{ + Inode: createSymlinkOp.Entry.Child, + } + + err = server.ReadSymlink(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "ReadSymlink")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestReadFile_Metrics(t *testing.T) { + ctx := context.Background() + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + openOp := &fuseops.OpenFileOp{ + Inode: lookUpOp.Entry.Child, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err) + op := &fuseops.ReadFileOp{ + Inode: lookUpOp.Entry.Child, + Handle: openOp.Handle, + Offset: 0, + Dst: make([]byte, len(content)), + } + + err = server.ReadFile(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "ReadFile")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestOpenFile_Metrics(t *testing.T) { + ctx := context.Background() + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test" + createWithContents(ctx, t, bucket, fileName, "test") + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + op := &fuseops.OpenFileOp{ + Inode: lookUpOp.Entry.Child, + } + + err = server.OpenFile(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "OpenFile")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestReleaseDirHandle_Metrics(t *testing.T) { + ctx := context.Background() + _, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + openOp := &fuseops.OpenDirOp{ + Inode: fuseops.RootInodeID, + } + err := server.OpenDir(ctx, openOp) + require.NoError(t, err) + op := &fuseops.ReleaseDirHandleOp{ + Handle: openOp.Handle, + } + + err = server.ReleaseDirHandle(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "ReleaseDirHandle")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestReadDirPlus_Metrics(t *testing.T) { + ctx := context.Background() + _, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + openOp := &fuseops.OpenDirOp{ + Inode: fuseops.RootInodeID, + } + err := server.OpenDir(ctx, openOp) + require.NoError(t, err) + op := &fuseops.ReadDirPlusOp{ + ReadDirOp: fuseops.ReadDirOp{ + Inode: fuseops.RootInodeID, + Handle: openOp.Handle, + Offset: 0, + Dst: make([]byte, 1024), + }, + } + + err = server.ReadDirPlus(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "ReadDirPlus")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestReadDir_Metrics(t *testing.T) { + ctx := context.Background() + _, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + openOp := &fuseops.OpenDirOp{ + Inode: fuseops.RootInodeID, + } + err := server.OpenDir(ctx, openOp) + require.NoError(t, err) + op := &fuseops.ReadDirOp{ + Inode: fuseops.RootInodeID, + Handle: openOp.Handle, + Offset: 0, + Dst: make([]byte, 1024), + } + + err = server.ReadDir(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "ReadDir")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestOpenDir_Metrics(t *testing.T) { + ctx := context.Background() + _, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + op := &fuseops.OpenDirOp{ + Inode: fuseops.RootInodeID, + } + + err := server.OpenDir(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "OpenDir")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestForgetInode_Metrics(t *testing.T) { + ctx := context.Background() + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test" + createWithContents(ctx, t, bucket, fileName, "test") + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + op := &fuseops.ForgetInodeOp{ + Inode: lookUpOp.Entry.Child, + N: 1, + } + + err = server.ForgetInode(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "ForgetInode")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestRename_Metrics(t *testing.T) { + ctx := context.Background() + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + oldName := "old" + newName := "new" + createWithContents(ctx, t, bucket, oldName, "test") + op := &fuseops.RenameOp{ + OldParent: fuseops.RootInodeID, + OldName: oldName, + NewParent: fuseops.RootInodeID, + NewName: newName, + } + + err := server.Rename(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "Rename")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestUnlink_Metrics(t *testing.T) { + ctx := context.Background() + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test" + createWithContents(ctx, t, bucket, fileName, "test") + op := &fuseops.UnlinkOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + + err := server.Unlink(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "Unlink")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestRmDir_Metrics(t *testing.T) { + ctx := context.Background() + _, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + dirName := "test" + mkDirOp := &fuseops.MkDirOp{ + Parent: fuseops.RootInodeID, + Name: dirName, + } + err := server.MkDir(ctx, mkDirOp) + require.NoError(t, err) + op := &fuseops.RmDirOp{ + Parent: fuseops.RootInodeID, + Name: dirName, + } + + err = server.RmDir(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "RmDir")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestCreateSymlink_Metrics(t *testing.T) { + ctx := context.Background() + _, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + op := &fuseops.CreateSymlinkOp{ + Parent: fuseops.RootInodeID, + Name: "test", + Target: "target", + } + + err := server.CreateSymlink(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "CreateSymlink")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestCreateFile_Metrics(t *testing.T) { + ctx := context.Background() + _, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + op := &fuseops.CreateFileOp{ + Parent: fuseops.RootInodeID, + Name: "test", + } + + err := server.CreateFile(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "CreateFile")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestMkNode_Metrics(t *testing.T) { + ctx := context.Background() + _, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + op := &fuseops.MkNodeOp{ + Parent: fuseops.RootInodeID, + Name: "test", + } + + err := server.MkNode(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "MkNode")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestMkDir_Metrics(t *testing.T) { + ctx := context.Background() + _, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + op := &fuseops.MkDirOp{ + Parent: fuseops.RootInodeID, + Name: "test", + } + + err := server.MkDir(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "MkDir")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestSetInodeAttributes_Metrics(t *testing.T) { + ctx := context.Background() + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, defaultServerConfigParams(), false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test" + createWithContents(ctx, t, bucket, fileName, "test") + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + op := &fuseops.SetInodeAttributesOp{ + Inode: lookUpOp.Entry.Child, + } + + err = server.SetInodeAttributes(ctx, op) + waitForMetricsProcessing() + + assert.NoError(t, err) + attrs := attribute.NewSet(attribute.String("fs_op", "SetInodeAttributes")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/ops_count", attrs, 1) + metrics.VerifyHistogramMetric(t, ctx, reader, "fs/ops_latency", attrs, 1) +} + +func TestReadFile_ReadBlockSizesMetric(t *testing.T) { + tests := []struct { + name string + readSizes []int64 + nilBufferReadCount int + expectedSum int64 + expectedCount uint64 + expectedBuckets map[int]uint64 // Maps bucket index to expected count + }{ + { + name: "Reads in bucket > 0 and <= 8KB", + readSizes: []int64{1024, 2048, 4096}, + expectedSum: 7168, + expectedCount: 3, + expectedBuckets: map[int]uint64{ + 1: 3, // 0 < size <= 8192 + }, + }, + { + name: "Reads spanning different buckets - case 1", + readSizes: []int64{5120, 10240, 20480}, // 5KB, 10KB, 20KB + expectedSum: 35840, + expectedCount: 3, + expectedBuckets: map[int]uint64{ + 1: 1, // 0 < 5KB <= 8KB + 2: 1, // 8KB < 10KB <= 16KB + 3: 1, // 16KB < 20KB <= 32KB + }, + }, + { + name: "Reads spanning different buckets - case 2", + readSizes: []int64{10240, 12288, 20480}, // 10KB, 12KB, 20KB + expectedSum: 43008, + expectedCount: 3, + expectedBuckets: map[int]uint64{ + 2: 2, + 3: 1, + }, + }, + { + name: "Reads at exact boundaries", + readSizes: []int64{8192, 16384}, // 8KB, 16KB + expectedSum: 24576, + expectedCount: 2, + expectedBuckets: map[int]uint64{ + 1: 1, // 0 < 8KB <= 8KB + 2: 1, // 8KB < 16KB <= 16KB + }, + }, + { + name: "Nil byte read", + nilBufferReadCount: 1, + expectedSum: 0, + expectedCount: 1, + expectedBuckets: map[int]uint64{ + 0: 1, // size <= 0 + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + params := defaultServerConfigParams() + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, params, false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test.txt" + + // Find the max read size so we can create a file large enough for all read operations. + var maxReadSize int64 + for _, size := range tc.readSizes { + if size > maxReadSize { + maxReadSize = size + } + } + content := make([]byte, maxReadSize) + createWithContents(ctx, t, bucket, fileName, string(content)) + + lookupOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookupOp) + require.NoError(t, err, "LookUpInode") + openOp := &fuseops.OpenFileOp{ + Inode: lookupOp.Entry.Child, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err, "OpenFile") + + // Perform reads with nil buffer + for i := 0; i < tc.nilBufferReadCount; i++ { + readOp := &fuseops.ReadFileOp{ + Inode: lookupOp.Entry.Child, + Handle: openOp.Handle, + Offset: 0, + Dst: nil, + } + err = server.ReadFile(ctx, readOp) + require.NoError(t, err, "ReadFile with nil buffer") + } + + // Perform reads of different sizes + for _, size := range tc.readSizes { + readOp := &fuseops.ReadFileOp{ + Inode: lookupOp.Entry.Child, + Handle: openOp.Handle, + Offset: 0, + Dst: make([]byte, size), + } + err = server.ReadFile(ctx, readOp) + require.NoError(t, err, "ReadFile for size %d", size) + } + + waitForMetricsProcessing() + + metrics.VerifyHistogramFull(t, ctx, reader, "read/block_sizes", attribute.NewSet(), tc.expectedCount, tc.expectedSum, tc.expectedBuckets) + }) + } +} + +func TestStreamingWrites_Fallback_ExistingFile(t *testing.T) { + // Arrange + ctx := context.Background() + params := defaultServerConfigParams() + params.enableStreamingWrites = true + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, params, false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test" + content := "initial content" + createWithContents(ctx, t, bucket, fileName, content) + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + openOp := &fuseops.OpenFileOp{ + Inode: lookUpOp.Entry.Child, + OpenFlags: syscall.O_RDWR, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err) + op := &fuseops.WriteFileOp{ + Inode: lookUpOp.Entry.Child, + Handle: openOp.Handle, + Offset: 0, + Data: []byte("test"), + } + + // Act + err = server.WriteFile(ctx, op) + + // Assert + require.NoError(t, err) + waitForMetricsProcessing() + attrs := attribute.NewSet(attribute.String("write_fallback_reason", "existing_file")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/streaming_write_fallback_count", attrs, 1, metrics.Subset()) +} + +func TestStreamingWrites_Fallback_OutOfOrder(t *testing.T) { + // Arrange + ctx := context.Background() + params := defaultServerConfigParams() + params.enableStreamingWrites = true + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, params, false) + server = wrappers.WithMonitoring(server, mh) + fileName := "test" + createWithContents(ctx, t, bucket, fileName, "") + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := server.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + openOp := &fuseops.OpenFileOp{ + Inode: lookUpOp.Entry.Child, + OpenFlags: syscall.O_WRONLY, + } + err = server.OpenFile(ctx, openOp) + require.NoError(t, err) + // Write at offset 0. + op1 := &fuseops.WriteFileOp{ + Inode: lookUpOp.Entry.Child, + Handle: openOp.Handle, + Offset: 0, + Data: []byte("test"), + } + err = server.WriteFile(ctx, op1) + require.NoError(t, err) + // Write at offset 10 (out of order). + op2 := &fuseops.WriteFileOp{ + Inode: lookUpOp.Entry.Child, + Handle: openOp.Handle, + Offset: 10, + Data: []byte("data"), + } + + // Act + err = server.WriteFile(ctx, op2) + + // Assert + require.NoError(t, err) + waitForMetricsProcessing() + attrs := attribute.NewSet(attribute.String("open_mode", "write_only"), attribute.String("write_fallback_reason", "out_of_order")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/streaming_write_fallback_count", attrs, 1) +} + +func TestStreamingWrites_Fallback_ConcurrencyLimitBreached(t *testing.T) { + // Arrange + ctx := context.Background() + params := defaultServerConfigParams() + params.enableStreamingWrites = true + params.writeGlobalMaxBlocks = 1 + bucket, server, mh, reader := createTestFileSystemWithMetrics(ctx, t, params, false) + server = wrappers.WithMonitoring(server, mh) + fileName1 := "test1" + fileName2 := "test2" + createWithContents(ctx, t, bucket, fileName1, "") + createWithContents(ctx, t, bucket, fileName2, "") + lookupOp1 := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName1, + } + err := server.LookUpInode(ctx, lookupOp1) + require.NoError(t, err) + openOp1 := &fuseops.OpenFileOp{ + Inode: lookupOp1.Entry.Child, + OpenFlags: syscall.O_WRONLY, + } + err = server.OpenFile(ctx, openOp1) + require.NoError(t, err) + lookupOp2 := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName2, + } + err = server.LookUpInode(ctx, lookupOp2) + require.NoError(t, err) + openOp2 := &fuseops.OpenFileOp{ + Inode: lookupOp2.Entry.Child, + OpenFlags: syscall.O_RDWR, + } + err = server.OpenFile(ctx, openOp2) + require.NoError(t, err) + op1 := &fuseops.WriteFileOp{ + Inode: lookupOp1.Entry.Child, + Handle: openOp1.Handle, + Offset: 0, + Data: []byte("test"), + } + err = server.WriteFile(ctx, op1) + require.NoError(t, err) + op2 := &fuseops.WriteFileOp{ + Inode: lookupOp2.Entry.Child, + Handle: openOp2.Handle, + Offset: 0, + Data: []byte("data"), + } + + // Act + err = server.WriteFile(ctx, op2) + + // Assert + require.NoError(t, err) + waitForMetricsProcessing() + attrs := attribute.NewSet(attribute.String("write_fallback_reason", "concurrency_limit_breached")) + metrics.VerifyCounterMetric(t, ctx, reader, "fs/streaming_write_fallback_count", attrs, 1, metrics.Subset()) +} diff --git a/internal/fs/notifier_test.go b/internal/fs/notifier_test.go new file mode 100644 index 0000000000..776175baad --- /dev/null +++ b/internal/fs/notifier_test.go @@ -0,0 +1,135 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A collection of tests to check notifier is invalidating the entry. +package fs_test + +import ( + "os" + "path" + "syscall" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/common" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/jacobsa/fuse" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type NotifierTest struct { + suite.Suite + fsTest +} + +func TestNotifierTestSuite(t *testing.T) { + suite.Run(t, new(NotifierTest)) +} + +func (t *NotifierTest) SetupSuite() { + t.serverCfg.ImplicitDirectories = true + t.serverCfg.InodeAttributeCacheTTL = 1000 * time.Second + t.serverCfg.Notifier = fuse.NewNotifier() + t.serverCfg.NewConfig = &cfg.Config{ + FileSystem: cfg.FileSystemConfig{ + ExperimentalEnableDentryCache: true, + }, + Write: cfg.WriteConfig{ + EnableStreamingWrites: true, + }, + } + t.fsTest.SetUpTestSuite() +} + +func (t *NotifierTest) TearDownTest() { + t.fsTest.TearDown() +} + +func (t *NotifierTest) TearDownSuite() { + t.fsTest.TearDownTestSuite() +} + +func (t *NotifierTest) TestWriteFileWithRootDirParent() { + filePath := path.Join(mntDir, fileName) + // Create a file in GCS. + _, err := storageutil.CreateObject(ctx, bucket, fileName, []byte("initial content")) + require.NoError(t.T(), err) + // Stat file to cache its entry. + _, err = os.Stat(filePath) + require.NoError(t.T(), err) + // Clobber the file in GCS. This changes the object's generation, making + // our file handle stale. + _, err = storageutil.CreateObject(ctx, bucket, fileName, []byte("modified")) + require.NoError(t.T(), err) + + // Attempt to write. + err = common.WriteFile(filePath, "new data") + + // Should return stale file handle error. + require.Error(t.T(), err) + assert.Regexp(t.T(), syscall.ESTALE.Error(), err.Error()) + // Attempt to write again, the entry has now been invalidated. + err = common.WriteFile(filePath, "new data") + assert.NoError(t.T(), err) +} + +func (t *NotifierTest) TestWriteFileWithNonRootDirParent() { + filePath := path.Join(mntDir, "dir/foo") + // Create a file in GCS. + _, err := storageutil.CreateObject(ctx, bucket, "dir/foo", []byte("initial content")) + require.NoError(t.T(), err) + // Stat file to cache its entry. + _, err = os.Stat(filePath) + require.NoError(t.T(), err) + // Clobber the file in GCS. This changes the object's generation, making + // our file handle stale. + _, err = storageutil.CreateObject(ctx, bucket, "dir/foo", []byte("modified")) + require.NoError(t.T(), err) + + // Attempt to write. + err = common.WriteFile(filePath, "new data") + + // Should return stale file handle error. + require.Error(t.T(), err) + assert.Regexp(t.T(), syscall.ESTALE.Error(), err.Error()) + // Attempt to write again, the entry has now been invalidated. + err = common.WriteFile(filePath, "new data") + assert.NoError(t.T(), err) +} + +func (t *NotifierTest) TestReadFileDoNotFailPersistently() { + filePath := path.Join(mntDir, fileName) + // Create a file in GCS. + _, err := storageutil.CreateObject(ctx, bucket, fileName, []byte("initial content")) + require.NoError(t.T(), err) + // Stat file to cache its entry. + _, err = os.Stat(filePath) + require.NoError(t.T(), err) + // Clobber the file in GCS. This changes the object's generation, making + // our file handle stale. + _, err = storageutil.CreateObject(ctx, bucket, fileName, []byte("modified")) + require.NoError(t.T(), err) + + // Attempt to read file. + _, err = common.ReadFile(filePath) + + // Should return error. + assert.NotNil(t.T(), err) + // Attempt to read again, the entry has now been invalidated. + _, err = common.ReadFile(filePath) + assert.NoError(t.T(), err) +} diff --git a/internal/fs/parallel_dirops_test.go b/internal/fs/parallel_dirops_test.go index bc5c8a9537..c031365a31 100644 --- a/internal/fs/parallel_dirops_test.go +++ b/internal/fs/parallel_dirops_test.go @@ -23,7 +23,9 @@ import ( "sync" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -47,6 +49,8 @@ func (t *ParallelDiropsTest) SetupSuite() { DisableParallelDirops: false, }} t.serverCfg.RenameDirLimit = 10 + t.serverCfg.MetricHandle = metrics.NewNoopMetrics() + t.serverCfg.TraceHandle = tracing.NewNoopTracer() t.fsTest.SetUpTestSuite() } diff --git a/internal/fs/read_cache_test.go b/internal/fs/read_cache_test.go index d06d0d0b4b..5b90ee88c2 100644 --- a/internal/fs/read_cache_test.go +++ b/internal/fs/read_cache_test.go @@ -25,9 +25,11 @@ import ( "syscall" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/util" - testutil "github.com/googlecloudplatform/gcsfuse/v2/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + testutil "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" . "github.com/jacobsa/ogletest" ) @@ -56,6 +58,7 @@ func init() { var CacheDir = path.Join(os.Getenv("HOME"), "cache-dir") var FileCacheDir = path.Join(CacheDir, util.FileCache) +var CacheExcludeName = "do_not_cache" // A collection of tests for a file system where the file cache is enabled // with cache-file-for-range-read set to False. @@ -70,9 +73,12 @@ func (t *FileCacheTest) SetUpTestSuite() { MaxSizeMb: FileCacheSizeInMb, CacheFileForRangeRead: false, EnableCrc: true, + ExcludeRegex: CacheExcludeName, }, CacheDir: cfg.ResolvedPath(CacheDir), } + t.serverCfg.MetricHandle = metrics.NewNoopMetrics() + t.serverCfg.TraceHandle = tracing.NewNoopTracer() t.fsTest.SetUpTestSuite() } @@ -138,7 +144,7 @@ func cacheFilePermissionTest(t *fsTest, fileMode os.FileMode) { AssertEq(fileMode, stat.Mode()) } -func writeShouldNotPopulateCache(t *fsTest) { +func writeShouldNotPopulateCache() { objectContent := generateRandomString(DefaultObjectSizeInMb * util.MiB) filePath := path.Join(mntDir, DefaultObjectName) file, err := os.OpenFile(filePath, os.O_RDWR|syscall.O_DIRECT|os.O_CREATE, util.DefaultFilePerm) @@ -189,6 +195,28 @@ func sequentialToRandomReadShouldPopulateCache(t *fsTest) { AssertTrue(reflect.DeepEqual(string(cachedContent[:100]), objectContent[:100])) } +func excludedFileShouldNotPopulateCache(t *fsTest, cacheDir string) { + objectContent := generateRandomString(DefaultObjectSizeInMb * util.MiB) + objects := map[string]string{CacheExcludeName: objectContent} + err := t.createObjects(objects) + AssertEq(nil, err) + filePath := path.Join(mntDir, CacheExcludeName) + file, err := os.OpenFile(filePath, os.O_RDWR|syscall.O_DIRECT, util.DefaultFilePerm) + defer closeFile(file) + AssertEq(nil, err) + + // reading object with name matching the exclude regex should not cache the object into file. + buf := make([]byte, len(objectContent)) + _, err = file.Read(buf) + AssertEq(nil, err) + AssertEq(objectContent, string(buf)) + + objectPath := util.GetObjectPath(bucket.Name(), CacheExcludeName) + downloadPath := util.GetDownloadPath(cacheDir, objectPath) + _, err = os.Stat(downloadPath) + AssertTrue(os.IsNotExist(err)) +} + func (t *FileCacheTest) ReadShouldChangeLRU() { objectName1 := DefaultObjectName + "1" objectContent1 := generateRandomString(DefaultObjectSizeInMb * util.MiB) @@ -255,12 +283,16 @@ func (t *FileCacheTest) SequentialToRandomReadShouldPopulateCache() { sequentialToRandomReadShouldPopulateCache(&t.fsTest) } +func (t *FileCacheTest) ExcludedFileShouldNotPopulateCache() { + excludedFileShouldNotPopulateCache(&t.fsTest, FileCacheDir) +} + func (t *FileCacheTest) CacheFilePermission() { cacheFilePermissionTest(&t.fsTest, util.DefaultFilePerm) } func (t *FileCacheTest) WriteShouldNotPopulateCache() { - writeShouldNotPopulateCache(&t.fsTest) + writeShouldNotPopulateCache() } func (t *FileCacheTest) FileSizeGreaterThanCacheSize() { @@ -505,7 +537,7 @@ func (t *FileCacheTest) ConcurrentReadsFromSameFileHandle() { readFunc(0, util.MiB) // read concurrently - for i := 0; i < 5; i++ { + for i := range 5 { wg.Add(1) go readFunc(int64(i)*util.MiB, util.MiB) } @@ -659,7 +691,7 @@ func (t *FileCacheWithCacheForRangeRead) CacheFilePermission() { } func (t *FileCacheWithCacheForRangeRead) WriteShouldNotPopulateCache() { - writeShouldNotPopulateCache(&t.fsTest) + writeShouldNotPopulateCache() } func (t *FileCacheWithCacheForRangeRead) SequentialToRandomReadShouldPopulateCache() { diff --git a/internal/fs/read_dir_plus_test.go b/internal/fs/read_dir_plus_test.go new file mode 100644 index 0000000000..2dd37f3e36 --- /dev/null +++ b/internal/fs/read_dir_plus_test.go @@ -0,0 +1,220 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A collection of tests to check Readdirplus operation. +package fs_test + +import ( + "os" + "path" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/jacobsa/fuse/fusetesting" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +//////////////////////////////////////////////////////////////////////// +// ReadDirPlusTest +//////////////////////////////////////////////////////////////////////// + +type ReadDirPlusTest struct { + suite.Suite + fsTest +} + +func TestReadDirPlusTestSuite(t *testing.T) { + suite.Run(t, new(ReadDirPlusTest)) +} + +func (t *ReadDirPlusTest) SetupSuite() { + t.mountCfg.EnableReaddirplus = true + t.serverCfg.ImplicitDirectories = true + t.serverCfg.InodeAttributeCacheTTL = 500 * time.Millisecond + t.serverCfg.NewConfig = &cfg.Config{ + FileSystem: cfg.FileSystemConfig{ + ExperimentalEnableDentryCache: true, + }, + } + t.fsTest.SetUpTestSuite() +} + +func (t *ReadDirPlusTest) TearDownTest() { + t.fsTest.TearDown() +} + +func (t *ReadDirPlusTest) TearDownSuite() { + t.fsTest.TearDownTestSuite() +} + +func (t *ReadDirPlusTest) TestEmptyDirectory() { + entries, err := fusetesting.ReadDirPlusPicky(mntDir) + + assert.Nil(t.T(), err) + assert.Equal(t.T(), 0, len(entries)) +} + +func (t *ReadDirPlusTest) TestDirectoryWithVariousEntryTypes() { + // Set up contents. + assert.Nil(t.T(), t.createObjects( + map[string]string{ + "file.txt": "taco", + "dir/": "", + "dir/baz": "burrito", + "implicit/file.txt": "content", + })) + // Set up a symlink. + err := os.Symlink("file.txt", path.Join(mntDir, "symlink_to_file")) + assert.Nil(t.T(), err) + expectedEntries := []string{ + "dir", + "file.txt", + "implicit", + "symlink_to_file", + } + + // Read the directory. + entries, err := fusetesting.ReadDirPlusPicky(mntDir) + + assert.Nil(t.T(), err) + assert.Equal(t.T(), len(expectedEntries), len(entries)) + // Check entries. + for i, entry := range entries { + assert.Equal(t.T(), expectedEntries[i], entry.Name()) + switch entry.Name() { + case "file.txt": + assert.False(t.T(), entry.IsDir()) + assert.Equal(t.T(), int64(len("taco")), entry.Size()) + assert.Equal(t.T(), filePerms, entry.Mode()) + case "dir": + assert.True(t.T(), entry.IsDir()) + assert.Equal(t.T(), dirPerms|os.ModeDir, entry.Mode()) + case "implicit": + assert.True(t.T(), entry.IsDir()) + assert.Equal(t.T(), dirPerms|os.ModeDir, entry.Mode()) + case "symlink_to_file": + assert.False(t.T(), entry.IsDir()) + assert.Equal(t.T(), os.ModeSymlink, entry.Mode()&os.ModeType) + default: + assert.FailNow(t.T(), "unexpected entry: %s", entry.Name()) + } + } +} + +// Test that stat after Readdirplus return the same attributes as Readdirplus even if data on GCS has changed +func (t *ReadDirPlusTest) TestStatAfterReaddirplus() { + // Set up contents. + testFileName := "file.txt" + filePath := path.Join(mntDir, testFileName) + initialContent := generateRandomString(10) + updatedContent := generateRandomString(5) + assert.Nil(t.T(), t.createObjects(map[string]string{testFileName: initialContent})) + + // Read the directory with Readdirplus. + _, _ = fusetesting.ReadDirPlusPicky(mntDir) + // Modify the file content in GCS. + assert.Nil(t.T(), t.createObjects(map[string]string{testFileName: updatedContent})) + // Stat the file before entry expires in cache. + // This should return the same attributes as Readdirplus. + fileInfo, err := os.Stat(filePath) + + // Check that the stat returns the old attributes. + assert.Nil(t.T(), err) + assert.Equal(t.T(), testFileName, fileInfo.Name()) + assert.Equal(t.T(), int64(len(initialContent)), fileInfo.Size()) + // Check stat after cache expiry. + // Wait for a duration longer than the metadata cache TTL. + time.Sleep(time.Second) + // Stat the file again. + // This should return the updated attributes. + fileInfo, err = os.Stat(filePath) + // Check that the stat returns the updated attributes. + assert.Nil(t.T(), err) + assert.Equal(t.T(), testFileName, fileInfo.Name()) + assert.Equal(t.T(), int64(len(updatedContent)), fileInfo.Size()) +} + +//////////////////////////////////////////////////////////////////////// +// LocalFileEntriesReadDirPlusTest +//////////////////////////////////////////////////////////////////////// + +type LocalFileEntriesReadDirPlusTest struct { + suite.Suite + fsTest +} + +func TestLocalFileEntriesReadDirPlusTestSuite(t *testing.T) { + suite.Run(t, new(LocalFileEntriesReadDirPlusTest)) +} + +func (t *LocalFileEntriesReadDirPlusTest) SetupSuite() { + t.mountCfg.EnableReaddirplus = true + t.serverCfg.InodeAttributeCacheTTL = 60 * time.Second + t.serverCfg.NewConfig = &cfg.Config{ + FileSystem: cfg.FileSystemConfig{ + ExperimentalEnableDentryCache: true, + }, + Write: cfg.WriteConfig{ + CreateEmptyFile: false, + }} + t.fsTest.SetUpTestSuite() +} + +func (t *LocalFileEntriesReadDirPlusTest) TearDownTest() { + t.fsTest.TearDown() +} + +func (t *LocalFileEntriesReadDirPlusTest) TearDownSuite() { + t.fsTest.TearDownTestSuite() +} + +func (t *LocalFileEntriesReadDirPlusTest) TestDirectoryWithLocalFile() { + // Create a local file that is not yet synced to GCS. + f, err := os.Create(path.Join(mntDir, "local_file")) + assert.Nil(t.T(), err) + defer f.Close() + + // Read the directory. + entries, err := fusetesting.ReadDirPlusPicky(mntDir) + + assert.Nil(t.T(), err) + assert.Equal(t.T(), 1, len(entries)) + // Check the entry for the local file. + entry := entries[0] + assert.Equal(t.T(), "local_file", entry.Name()) + assert.False(t.T(), entry.IsDir()) + assert.Equal(t.T(), filePerms, entry.Mode()) +} + +func (t *LocalFileEntriesReadDirPlusTest) TestDirWithOneLocalAndOneGCSEntry() { + // Create a remote object on GCS + assert.Nil(t.T(), t.createObjects(map[string]string{"gcs_file": "content"})) + // Create a local-only file + f, err := os.Create(path.Join(mntDir, "local_file")) + assert.Nil(t.T(), err) + defer f.Close() + + // Read the directory + entries, err := fusetesting.ReadDirPlusPicky(mntDir) + + assert.Nil(t.T(), err) + assert.Equal(t.T(), 2, len(entries)) + assert.Equal(t.T(), "gcs_file", entries[0].Name()) + assert.False(t.T(), entries[0].IsDir()) + assert.Equal(t.T(), int64(len("content")), entries[0].Size()) + assert.Equal(t.T(), "local_file", entries[1].Name()) + assert.False(t.T(), entries[1].IsDir()) +} diff --git a/internal/fs/read_only_test.go b/internal/fs/read_only_test.go index c3b89246de..01b21a276b 100644 --- a/internal/fs/read_only_test.go +++ b/internal/fs/read_only_test.go @@ -18,7 +18,7 @@ import ( "os" "path" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/ogletest" ) diff --git a/internal/fs/rename_dir_test.go b/internal/fs/rename_dir_test.go new file mode 100644 index 0000000000..524e8452f7 --- /dev/null +++ b/internal/fs/rename_dir_test.go @@ -0,0 +1,317 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs_test + +import ( + "fmt" + "os" + "os/exec" + "path" + "strings" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type RenameDirTests struct { + suite.Suite + fsTest +} + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func (t *RenameDirTests) TestRenameFolderWithSrcDirectoryDoesNotExist() { + oldDirPath := path.Join(mntDir, "foo_not_exist") + newDirPath := path.Join(mntDir, "foo_rename") + + err := os.Rename(oldDirPath, newDirPath) + + assert.Error(t.T(), err) + assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) + _, err = os.Stat(newDirPath) + assert.Error(t.T(), err) + assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) +} + +func (t *RenameDirTests) TestRenameFolderWithDstDirectoryNotEmpty() { + oldDirPath := path.Join(mntDir, "foo") + _, err := os.Stat(oldDirPath) + assert.NoError(t.T(), err) + // In the setup phase, we created file1.txt within the bar directory. + newDirPath := path.Join(mntDir, "bar") + _, err = os.Stat(newDirPath) + assert.NoError(t.T(), err) + + err = os.Rename(oldDirPath, newDirPath) + + assert.Error(t.T(), err) + assert.True(t.T(), strings.Contains(err.Error(), "file exists")) +} + +func (t *RenameDirTests) TestRenameFolderWithEmptySourceDirectory() { + oldDirPath := path.Join(mntDir, "foo", "test2") + _, err := os.Stat(oldDirPath) + assert.NoError(t.T(), err) + newDirPath := path.Join(mntDir, "foo_rename") + _, err = os.Stat(newDirPath) + assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) + + err = os.Rename(oldDirPath, newDirPath) + + assert.NoError(t.T(), err) + _, err = os.Stat(oldDirPath) + assert.Error(t.T(), err) + assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) + _, err = os.Stat(newDirPath) + assert.NoError(t.T(), err) + dirEntries, err := os.ReadDir(newDirPath) + assert.NoError(t.T(), err) + assert.Equal(t.T(), 0, len(dirEntries)) +} + +func (t *RenameDirTests) TestRenameSymlinkToExplicitDir() { + targetDirName := "target_dir" + err := os.Mkdir(path.Join(mntDir, targetDirName), dirPerms) + require.NoError(t.T(), err) + oldPath := path.Join(mntDir, "symlink_old") + err = os.Symlink(targetDirName, oldPath) + require.NoError(t.T(), err) + newPath := path.Join(mntDir, "symlink_new") + + err = os.Rename(oldPath, newPath) + + require.NoError(t.T(), err) + _, err = os.Lstat(oldPath) + assert.Error(t.T(), err) + assert.True(t.T(), os.IsNotExist(err)) + fi, err := os.Lstat(newPath) + require.NoError(t.T(), err) + assert.Equal(t.T(), os.ModeSymlink, fi.Mode()&os.ModeSymlink) + targetRead, err := os.Readlink(newPath) + require.NoError(t.T(), err) + assert.Equal(t.T(), targetDirName, targetRead) +} + +func (t *RenameDirTests) TestRenameFolderWithSourceDirectoryHaveLocalFiles() { + oldDirPath := path.Join(mntDir, "foo", "test") + _, err := os.Stat(oldDirPath) + assert.NoError(t.T(), err) + file, err := os.OpenFile(path.Join(oldDirPath, "file4.txt"), os.O_RDWR|os.O_CREATE, filePerms) + assert.NoError(t.T(), err) + defer file.Close() + newDirPath := path.Join(mntDir, "bar", "foo_rename") + + err = os.Rename(oldDirPath, newDirPath) + + assert.Error(t.T(), err) + // In the logs, we encountered the following error: + // "Rename: operation not supported, can't rename directory 'test' with open files: operation not supported." + // This was translated to an "operation not supported" error at the kernel level. + assert.True(t.T(), strings.Contains(err.Error(), "operation not supported")) +} + +func (t *RenameDirTests) TestRenameFolderWithSameParent() { + oldDirPath := path.Join(mntDir, "foo") + _, err := os.Stat(oldDirPath) + require.NoError(t.T(), err) + newDirPath := path.Join(mntDir, "foo_rename") + _, err = os.Stat(newDirPath) + require.Error(t.T(), err) + assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) + + err = os.Rename(oldDirPath, newDirPath) + + assert.NoError(t.T(), err) + _, err = os.Stat(oldDirPath) + assert.Error(t.T(), err) + assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) + _, err = os.Stat(newDirPath) + assert.NoError(t.T(), err) + dirEntries, err := os.ReadDir(newDirPath) + assert.NoError(t.T(), err) + assert.Equal(t.T(), 5, len(dirEntries)) + actualDirEntries := []dirEntry{} + for _, d := range dirEntries { + actualDirEntries = append(actualDirEntries, dirEntry{ + name: d.Name(), + isDir: d.IsDir(), + }) + } + assert.ElementsMatch(t.T(), actualDirEntries, expectedFooDirEntries) +} + +func (t *RenameDirTests) TestRenameFolderWithExistingEmptyDestDirectory() { + oldDirPath := path.Join(mntDir, "foo", "test") + _, err := os.Stat(oldDirPath) + require.NoError(t.T(), err) + newDirPath := path.Join(mntDir, "foo", "test2") + _, err = os.Stat(newDirPath) + require.NoError(t.T(), err) + + // Go's Rename function does not support renaming a directory into an existing empty directory. + // To achieve this, we call a Python rename function as a workaround. + cmd := exec.Command("python3", "-c", fmt.Sprintf("import os; os.rename('%s', '%s')", oldDirPath, newDirPath)) + _, err = cmd.CombinedOutput() + + assert.NoError(t.T(), err) + _, err = os.Stat(oldDirPath) + assert.Error(t.T(), err) + assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) + _, err = os.Stat(newDirPath) + assert.NoError(t.T(), err) + dirEntries, err := os.ReadDir(newDirPath) + assert.NoError(t.T(), err) + assert.Equal(t.T(), 1, len(dirEntries)) + assert.Equal(t.T(), "file3.txt", dirEntries[0].Name()) + assert.False(t.T(), dirEntries[0].IsDir()) +} + +func (t *RenameDirTests) TestRenameFolderWithDifferentParents() { + oldDirPath := path.Join(mntDir, "foo") + _, err := os.Stat(oldDirPath) + assert.NoError(t.T(), err) + newDirPath := path.Join(mntDir, "bar", "foo_rename") + + err = os.Rename(oldDirPath, newDirPath) + + assert.NoError(t.T(), err) + _, err = os.Stat(oldDirPath) + assert.Error(t.T(), err) + assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) + _, err = os.Stat(newDirPath) + assert.NoError(t.T(), err) + dirEntries, err := os.ReadDir(newDirPath) + assert.NoError(t.T(), err) + assert.Equal(t.T(), 5, len(dirEntries)) + actualDirEntries := []dirEntry{} + for _, d := range dirEntries { + actualDirEntries = append(actualDirEntries, dirEntry{ + name: d.Name(), + isDir: d.IsDir(), + }) + } + assert.ElementsMatch(t.T(), actualDirEntries, expectedFooDirEntries) +} + +func (t *RenameDirTests) TestRenameFolderWithOpenGCSFile() { + oldDirPath := path.Join(mntDir, "bar") + _, err := os.Stat(oldDirPath) + assert.NoError(t.T(), err) + newDirPath := path.Join(mntDir, "bar_rename") + filePath := path.Join(oldDirPath, "file1.txt") + f, err := os.Open(filePath) + require.NoError(t.T(), err) + + err = os.Rename(oldDirPath, newDirPath) + + require.NoError(t.T(), err) + _, err = f.WriteString("test") + assert.Error(t.T(), err) + assert.True(t.T(), strings.Contains(err.Error(), "bad file descriptor")) + assert.NoError(t.T(), f.Close()) + _, err = os.Stat(oldDirPath) + assert.Error(t.T(), err) + assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) + _, err = os.Stat(newDirPath) + assert.NoError(t.T(), err) + dirEntries, err := os.ReadDir(newDirPath) + assert.NoError(t.T(), err) + assert.Equal(t.T(), 1, len(dirEntries)) + assert.Equal(t.T(), "file1.txt", dirEntries[0].Name()) + assert.False(t.T(), dirEntries[0].IsDir()) +} + +// Create directory foo. +// Stat the directory foo. +// Rename directory foo --> foo_rename +// Stat the old directory. +// Stat the new directory. +// Read new directory and validate. +// Create old directory again with same name - foo +// Stat the directory - foo +// Read directory again and validate it is empty. +func (t *RenameDirTests) TestCreateDirectoryWithSameNameAfterRename() { + oldDirPath := path.Join(mntDir, "foo") + _, err := os.Stat(oldDirPath) + require.NoError(t.T(), err) + newDirPath := path.Join(mntDir, "foo_rename") + // Rename directory foo --> foo_rename + err = os.Rename(oldDirPath, newDirPath) + require.NoError(t.T(), err) + // Stat old directory. + _, err = os.Stat(oldDirPath) + require.Error(t.T(), err) + require.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) + // Stat new directory. + _, err = os.Stat(newDirPath) + require.NoError(t.T(), err) + // Read new directory and validate. + dirEntries, err := os.ReadDir(newDirPath) + require.NoError(t.T(), err) + require.Equal(t.T(), 5, len(dirEntries)) + actualDirEntries := []dirEntry{} + for _, d := range dirEntries { + actualDirEntries = append(actualDirEntries, dirEntry{ + name: d.Name(), + isDir: d.IsDir(), + }) + } + require.ElementsMatch(t.T(), actualDirEntries, expectedFooDirEntries) + + // Create old directory again. + err = os.Mkdir(oldDirPath, dirPerms) + + assert.NoError(t.T(), err) + _, err = os.Stat(oldDirPath) + assert.NoError(t.T(), err) + dirEntries, err = os.ReadDir(oldDirPath) + assert.NoError(t.T(), err) + assert.Equal(t.T(), 0, len(dirEntries)) +} + +// Create directory - foo/test2 +// Create local file in directory - foo/test2/test.txt +// Stat the local file - foo/test2/test.txt +// Delete directory - rm -r foo/test2 +// Create directory again - foo/test2 +// Create local file with the same name in directory - foo/test2/test.txt +// Stat the local file - foo/test2/test.txt +func (t *RenameDirTests) TestCreateLocalFileInSamePathAfterDeletingParentDirectory() { + dirPath := path.Join(mntDir, "foo", "test2") + filePath := path.Join(dirPath, "test.txt") + // Create local file in side it. + f1, err := os.Create(filePath) + defer require.NoError(t.T(), f1.Close()) + require.NoError(t.T(), err) + _, err = os.Stat(filePath) + require.NoError(t.T(), err) + // Delete directory rm -r foo/test2 + err = os.RemoveAll(dirPath) + assert.NoError(t.T(), err) + // Create directory again foo/test2 + err = os.Mkdir(dirPath, dirPerms) + assert.NoError(t.T(), err) + + // Create local file again. + f2, err := os.Create(filePath) + defer require.NoError(t.T(), f2.Close()) + + assert.NoError(t.T(), err) + _, err = os.Stat(filePath) + assert.NoError(t.T(), err) +} diff --git a/internal/fs/rename_file_test.go b/internal/fs/rename_file_test.go new file mode 100644 index 0000000000..5fd83a53f2 --- /dev/null +++ b/internal/fs/rename_file_test.go @@ -0,0 +1,139 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs_test + +import ( + "os" + "path" + "strings" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type RenameFileTests struct { + suite.Suite +} + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func (t *RenameFileTests) TestRenameFileWithSrcFileDoesNotExist() { + oldFilePath := path.Join(mntDir, "file") + newFilePath := path.Join(mntDir, "file_rename") + + err := os.Rename(oldFilePath, newFilePath) + + assert.Error(t.T(), err) + assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) + _, err = os.Stat(newFilePath) + assert.Error(t.T(), err) + assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) +} + +func (t *RenameFileTests) TestRenameFileWithDstDestFileExist() { + oldFilePath := path.Join(mntDir, "foo", "file1.txt") + _, err := os.Stat(oldFilePath) + assert.NoError(t.T(), err) + newFilePath := path.Join(mntDir, "foo", "file2.txt") + _, err = os.Stat(newFilePath) + assert.NoError(t.T(), err) + + err = os.Rename(oldFilePath, newFilePath) + + assert.NoError(t.T(), err) + _, err = os.Stat(oldFilePath) + assert.Error(t.T(), err) + assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) + content, err := os.ReadFile(newFilePath) + assert.NoError(t.T(), err) + assert.Equal(t.T(), file1Content, string(content)) +} + +func (t *RenameFileTests) TestRenameFile() { + testCases := []struct { + name string + oldFilePath string + newFilePath string + wantContent string + }{ + { + name: "DifferentParent", + oldFilePath: path.Join(mntDir, "foo", "file1.txt"), + newFilePath: path.Join(mntDir, "bar", "file3.txt"), + wantContent: file1Content, + }, + { + name: "SameParent", + oldFilePath: path.Join(mntDir, "foo", "file2.txt"), + newFilePath: path.Join(mntDir, "foo", "file3.txt"), + wantContent: file2Content, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + // Ensure file exists before renaming. + _, err := os.Stat(tc.oldFilePath) + require.NoError(t.T(), err) + + // Rename the file. + err = os.Rename(tc.oldFilePath, tc.newFilePath) + assert.NoError(t.T(), err) + + // Verify the old file no longer exists. + _, err = os.Stat(tc.oldFilePath) + assert.Error(t.T(), err) + assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) + // Verify the new file exists and has the correct content. + f, err := os.Stat(tc.newFilePath) + assert.NoError(t.T(), err) + assert.Equal(t.T(), path.Base(tc.newFilePath), f.Name()) + content, err := os.ReadFile(tc.newFilePath) + assert.NoError(t.T(), err) + assert.Equal(t.T(), tc.wantContent, string(content)) + }) + } +} + +func (t *RenameFileTests) TestRenameSymlinkToFile() { + // Create a target file for the symlink to point to. + targetPath := path.Join(mntDir, "target") + err := os.WriteFile(targetPath, []byte("taco"), filePerms) + require.NoError(t.T(), err) + // Create the symbolic link that we will rename. + oldPath := path.Join(mntDir, "symlink_old") + err = os.Symlink(targetPath, oldPath) + require.NoError(t.T(), err) + newPath := path.Join(mntDir, "symlink_new") + + err = os.Rename(oldPath, newPath) + + require.NoError(t.T(), err) + // The old path should no longer exist. + _, err = os.Lstat(oldPath) + require.Error(t.T(), err) + assert.True(t.T(), os.IsNotExist(err), "err: %v", err) + // The new path should now be a symlink, having replaced the original file. + fi, err := os.Lstat(newPath) + require.NoError(t.T(), err) + assert.Equal(t.T(), os.ModeSymlink, fi.Mode()&os.ModeSymlink) + // The new symlink should point to the correct target. + targetRead, err := os.Readlink(newPath) + require.NoError(t.T(), err) + assert.Equal(t.T(), targetPath, targetRead) +} diff --git a/internal/fs/server.go b/internal/fs/server.go index ea409182ec..95db23d416 100644 --- a/internal/fs/server.go +++ b/internal/fs/server.go @@ -17,8 +17,8 @@ package fs import ( "fmt" - newcfg "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/fs/wrappers" + newcfg "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/wrappers" "github.com/jacobsa/fuse" "github.com/jacobsa/fuse/fuseutil" "golang.org/x/net/context" @@ -33,8 +33,11 @@ func NewServer(ctx context.Context, cfg *ServerConfig) (fuse.Server, error) { fs = wrappers.WithErrorMapping(fs) if newcfg.IsTracingEnabled(cfg.NewConfig) { - fs = wrappers.WithTracing(fs) + fs = wrappers.WithTracing(fs, cfg.TraceHandle) + } + fs = wrappers.WithMonitoring(fs, cfg.MetricHandle) + if cfg.Notifier != nil { + return fuse.NewServerWithNotifier(cfg.Notifier, fuseutil.NewFileSystemServer(fs)), nil } - fs = wrappers.WithMonitoring(fs) return fuseutil.NewFileSystemServer(fs), nil } diff --git a/internal/fs/stale_file_handle_common_test.go b/internal/fs/stale_file_handle_common_test.go new file mode 100644 index 0000000000..a496533b46 --- /dev/null +++ b/internal/fs/stale_file_handle_common_test.go @@ -0,0 +1,140 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs_test + +import ( + "os" + "path" + "syscall" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +// ////////////////////////////////////////////////////////////////////// +// Boilerplate +// ////////////////////////////////////////////////////////////////////// + +type staleFileHandleCommon struct { + // fsTest has f1 *osFile and f2 *osFile which we will reuse here. + fsTest + suite.Suite +} + +// ////////////////////////////////////////////////////////////////////// +// Helpers +// ////////////////////////////////////////////////////////////////////// + +func commonServerConfig() *cfg.Config { + return &cfg.Config{ + FileSystem: cfg.FileSystemConfig{}, + MetadataCache: cfg.MetadataCacheConfig{ + TtlSecs: 0, + }, + } + +} + +func clobberFile(t *testing.T, content string) { + t.Helper() + _, err := storageutil.CreateObject( + ctx, + bucket, + fileName, + []byte(content)) + assert.NoError(t, err) +} + +func createGCSObject(t *testing.T, content string) *os.File { + t.Helper() + _, err := storageutil.CreateObject( + ctx, + bucket, + fileName, + []byte(content)) + assert.NoError(t, err) + // Open file handle to read or write. + fh, err := os.OpenFile(path.Join(mntDir, fileName), os.O_RDWR|syscall.O_DIRECT, filePerms) + assert.NoError(t, err) + return fh +} + +func (t *staleFileHandleCommon) SetupSuite() { + t.serverCfg.NewConfig = commonServerConfig() + t.fsTest.SetUpTestSuite() +} + +func (t *staleFileHandleCommon) TearDownTest() { + // fsTest Cleanups to clean up mntDir and close t.f1 and t.f2. + t.fsTest.TearDown() +} + +func (t *staleFileHandleCommon) TearDownSuite() { + t.fsTest.TearDownTestSuite() +} + +// ////////////////////////////////////////////////////////////////////// +// Tests +// ////////////////////////////////////////////////////////////////////// + +func (t *staleFileHandleCommon) TestClobberedFileSyncAndCloseThrowsStaleFileHandleError() { + // Dirty the file by giving it some contents. + n, err := t.f1.Write([]byte("taco")) + assert.NoError(t.T(), err) + assert.Equal(t.T(), 4, n) + // Replace the underlying object with a new generation. + clobberFile(t.T(), "foobar") + + err = t.f1.Sync() + + operations.ValidateESTALEError(t.T(), err) + err = t.f1.Close() + operations.ValidateESTALEError(t.T(), err) + // Make f1 nil, so that another attempt is not taken in TearDown to close the + // file. + t.f1 = nil + // Validate that object is not updated with un-synced content. + contents, err := storageutil.ReadObject(ctx, bucket, "foo") + assert.NoError(t.T(), err) + assert.Equal(t.T(), "foobar", string(contents)) +} + +func (t *staleFileHandleCommon) TestFileDeletedLocallySyncAndCloseDoNotThrowError() { + // Dirty the file by giving it some contents. + n, err := t.f1.Write([]byte("foobar")) + assert.NoError(t.T(), err) + assert.Equal(t.T(), 6, n) + // Unlink the file. + err = os.Remove(t.f1.Name()) + // Verify unlink operation succeeds. + assert.NoError(t.T(), err) + operations.ValidateNoFileOrDirError(t.T(), path.Join(mntDir, "foo")) + // Attempt to write to file should not give any error. + n, err = t.f1.Write([]byte("taco")) + assert.Equal(t.T(), 4, n) + assert.NoError(t.T(), err) + + operations.SyncFile(t.f1, t.T()) + operations.CloseFileShouldNotThrowError(t.T(), t.f1) + + // Make f1 nil, so that another attempt is not taken in TearDown to close the + // file. + t.f1 = nil + operations.ValidateObjectNotFoundErr(ctx, t.T(), bucket, "foo") +} diff --git a/internal/fs/stale_file_handle_local_file_test.go b/internal/fs/stale_file_handle_local_file_test.go new file mode 100644 index 0000000000..128ce37233 --- /dev/null +++ b/internal/fs/stale_file_handle_local_file_test.go @@ -0,0 +1,48 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs_test + +import ( + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/stretchr/testify/suite" +) + +// ////////////////////////////////////////////////////////////////////// +// Boilerplate +// ////////////////////////////////////////////////////////////////////// + +type staleFileHandleLocalFile struct { + staleFileHandleCommon +} + +// ////////////////////////////////////////////////////////////////////// +// Helpers +// ////////////////////////////////////////////////////////////////////// + +func (t *staleFileHandleLocalFile) SetupTest() { + // Create a local file. + _, t.f1 = operations.CreateLocalFile(ctx, t.T(), mntDir, bucket, "foo") +} + +// ////////////////////////////////////////////////////////////////////// +// Tests +// ////////////////////////////////////////////////////////////////////// + +// Executes all stale handle tests for local files. +func TestStaleFileHandleLocalFile(t *testing.T) { + suite.Run(t, new(staleFileHandleLocalFile)) +} diff --git a/internal/fs/stale_file_handle_streaming_writes_common_test.go b/internal/fs/stale_file_handle_streaming_writes_common_test.go new file mode 100644 index 0000000000..8af0cf2d7b --- /dev/null +++ b/internal/fs/stale_file_handle_streaming_writes_common_test.go @@ -0,0 +1,87 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs_test + +import ( + "math" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +// ////////////////////////////////////////////////////////////////////// +// Boilerplate +// ////////////////////////////////////////////////////////////////////// + +type staleFileHandleStreamingWritesCommon struct { + // fsTest has f1 *osFile and f2 *osFile which we will reuse here. + fsTest + suite.Suite +} + +// ////////////////////////////////////////////////////////////////////// +// Helpers +// ////////////////////////////////////////////////////////////////////// + +func (t *staleFileHandleStreamingWritesCommon) SetupSuite() { + serverCfg := commonServerConfig() + serverCfg.Write.EnableStreamingWrites = true + serverCfg.Write.BlockSizeMb = 1 + serverCfg.Write.MaxBlocksPerFile = 1 + serverCfg.Write.GlobalMaxBlocks = math.MaxInt + + t.serverCfg.NewConfig = serverCfg + t.mountCfg.DisableWritebackCaching = true + + t.fsTest.SetUpTestSuite() +} + +func (t *staleFileHandleStreamingWritesCommon) TearDownTest() { + // fsTest Cleanups to clean up mntDir and close t.f1 and t.f2. + t.fsTest.TearDown() +} + +func (t *staleFileHandleStreamingWritesCommon) TearDownSuite() { + t.fsTest.TearDownTestSuite() +} + +// ////////////////////////////////////////////////////////////////////// +// Tests +// ////////////////////////////////////////////////////////////////////// + +func (t *staleFileHandleStreamingWritesCommon) TestWriteFileSyncFileClobberedFlushThrowsStaleFileHandleError() { + // Dirty the file by giving it some contents. + data, err := operations.GenerateRandomData(operations.MiB * 4) + assert.NoError(t.T(), err) + _, err = t.f1.WriteAt(data, 0) + assert.NoError(t.T(), err) + err = t.f1.Sync() + assert.NoError(t.T(), err) + // Replace the underlying object with a new generation. + clobberFile(t.T(), "foobar") + + err = t.f1.Close() + + operations.ValidateESTALEError(t.T(), err) + // Make f1 nil, so that another attempt is not taken in TearDown to close the + // file. + t.f1 = nil + // Validate that object is not updated with un-synced content. + contents, err := storageutil.ReadObject(ctx, bucket, "foo") + assert.NoError(t.T(), err) + assert.Equal(t.T(), "foobar", string(contents)) +} diff --git a/internal/fs/stale_file_handle_streaming_writes_local_file_test.go b/internal/fs/stale_file_handle_streaming_writes_local_file_test.go new file mode 100644 index 0000000000..2e4c578ad2 --- /dev/null +++ b/internal/fs/stale_file_handle_streaming_writes_local_file_test.go @@ -0,0 +1,74 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs_test + +import ( + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +// ////////////////////////////////////////////////////////////////////// +// Boilerplate +// ////////////////////////////////////////////////////////////////////// + +type staleFileHandleStreamingWritesLocalFile struct { + staleFileHandleStreamingWritesCommon +} + +// ////////////////////////////////////////////////////////////////////// +// Helpers +// ////////////////////////////////////////////////////////////////////// + +func (t *staleFileHandleStreamingWritesLocalFile) SetupTest() { + // Create a local file. + _, t.f1 = operations.CreateLocalFile(ctx, t.T(), mntDir, bucket, "foo") +} + +// ////////////////////////////////////////////////////////////////////// +// Tests +// ////////////////////////////////////////////////////////////////////// + +func (t *staleFileHandleStreamingWritesLocalFile) TestClobberedWriteFileSyncAndCloseThrowsStaleFileHandleError() { + // Replace the underlying object with a new generation. + clobberFile(t.T(), "foobar") + // Writing to file will return Stale File Handle Error. + data, err := operations.GenerateRandomData(operations.MiB * 4) + assert.NoError(t.T(), err) + + _, err = t.f1.WriteAt(data, 0) + + operations.ValidateESTALEError(t.T(), err) + err = t.f1.Sync() + operations.ValidateESTALEError(t.T(), err) + err = t.f1.Close() + operations.ValidateESTALEError(t.T(), err) + // Make f1 nil, so that another attempt is not taken in TearDown to close the + // file. + t.f1 = nil + // Validate that object is not updated with un-synced content. + contents, err := storageutil.ReadObject(ctx, bucket, "foo") + assert.NoError(t.T(), err) + assert.Equal(t.T(), "foobar", string(contents)) +} + +// Executes all stale handle tests for local files with streaming writes. +func TestStaleFileHandleStreamingWritesLocalFile(t *testing.T) { + ts := new(staleFileHandleStreamingWritesLocalFile) + suite.Run(t, ts) +} diff --git a/internal/fs/stale_file_handle_streaming_writes_synced_file_test.go b/internal/fs/stale_file_handle_streaming_writes_synced_file_test.go new file mode 100644 index 0000000000..7e6371c073 --- /dev/null +++ b/internal/fs/stale_file_handle_streaming_writes_synced_file_test.go @@ -0,0 +1,100 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs_test + +import ( + "os" + "path" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +// ////////////////////////////////////////////////////////////////////// +// Boilerplate +// ////////////////////////////////////////////////////////////////////// + +type staleFileHandleStreamingWritesSyncedFile struct { + staleFileHandleStreamingWritesCommon +} + +// ////////////////////////////////////////////////////////////////////// +// Helpers +// ////////////////////////////////////////////////////////////////////// + +func (t *staleFileHandleStreamingWritesSyncedFile) SetupTest() { + // Create an empty object on bucket. + t.f1 = createGCSObject(t.T(), "") +} + +// ////////////////////////////////////////////////////////////////////// +// Tests +// ////////////////////////////////////////////////////////////////////// + +func (t *staleFileHandleStreamingWritesSyncedFile) TestWriteToClobberedFileThrowsStaleFileHandleError() { + // Replace the underlying object with a new generation. + clobberFile(t.T(), "foobar") + // Writing to file will return Stale File Handle Error. + data, err := operations.GenerateRandomData(operations.MiB * 4) + assert.NoError(t.T(), err) + + _, err = t.f1.WriteAt(data, 0) + + operations.ValidateESTALEError(t.T(), err) + err = t.f1.Sync() + assert.NoError(t.T(), err) + err = t.f1.Close() + assert.NoError(t.T(), err) + // Make f1 nil, so that another attempt is not taken in TearDown to close the + // file. + t.f1 = nil + // Validate that object is not updated with un-synced content. + contents, err := storageutil.ReadObject(ctx, bucket, "foo") + assert.NoError(t.T(), err) + assert.Equal(t.T(), "foobar", string(contents)) +} + +func (t *staleFileHandleStreamingWritesSyncedFile) TestRenameFileWriteThrowsStaleFileHandleError() { + // Rename the object. + err := os.Rename(t.f1.Name(), path.Join(mntDir, "bar")) + assert.NoError(t.T(), err) + // Writing to file will return Stale File Handle Error. + data, err := operations.GenerateRandomData(operations.MiB * 4) + assert.NoError(t.T(), err) + + _, err = t.f1.WriteAt(data, 0) + + operations.ValidateESTALEError(t.T(), err) + err = t.f1.Sync() + assert.NoError(t.T(), err) + err = t.f1.Close() + assert.NoError(t.T(), err) + // Make f1 nil, so that another attempt is not taken in TearDown to close the + // file. + t.f1 = nil + // Validate that object is not updated with un-synced content. + contents, err := storageutil.ReadObject(ctx, bucket, "bar") + assert.NoError(t.T(), err) + assert.Equal(t.T(), "", string(contents)) +} + +// Executes all stale handle tests for gcs synced files with streaming writes. +func TestStaleFileHandleStreamingWritesSyncedFile(t *testing.T) { + ts := new(staleFileHandleStreamingWritesSyncedFile) + suite.Run(t, ts) +} diff --git a/internal/fs/stale_file_handle_synced_file_test.go b/internal/fs/stale_file_handle_synced_file_test.go new file mode 100644 index 0000000000..dd19cc543d --- /dev/null +++ b/internal/fs/stale_file_handle_synced_file_test.go @@ -0,0 +1,132 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs_test + +import ( + "os" + "path" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +// ////////////////////////////////////////////////////////////////////// +// Boilerplate +// ////////////////////////////////////////////////////////////////////// + +type staleFileHandleSyncedFile struct { + staleFileHandleCommon +} + +// ////////////////////////////////////////////////////////////////////// +// Helpers +// ////////////////////////////////////////////////////////////////////// + +func (t *staleFileHandleSyncedFile) SetupTest() { + // Create an object on bucket. + t.f1 = createGCSObject(t.T(), "bar") +} + +// ////////////////////////////////////////////////////////////////////// +// Tests +// ////////////////////////////////////////////////////////////////////// + +func (t *staleFileHandleSyncedFile) TestClobberedFileReadThrowsStaleFileHandleError() { + // Replace the underlying object with a new generation. + clobberFile(t.T(), "foobar") + + buffer := make([]byte, 6) + _, err := t.f1.Read(buffer) + + operations.ValidateESTALEError(t.T(), err) + // Validate that object is updated with new content. + contents, err := storageutil.ReadObject(ctx, bucket, "foo") + assert.NoError(t.T(), err) + assert.Equal(t.T(), "foobar", string(contents)) +} + +func (t *staleFileHandleSyncedFile) TestClobberedFileFirstWriteThrowsStaleFileHandleError() { + // Replace the underlying object with a new generation. + clobberFile(t.T(), "foobar") + + _, err := t.f1.Write([]byte("taco")) + + operations.ValidateESTALEError(t.T(), err) + // Attempt to sync to file should not result in error as we first check if the + // content has been dirtied before clobbered check in Sync flow. + err = t.f1.Sync() + assert.NoError(t.T(), err) + // Validate that object is not updated with new content as write failed. + contents, err := storageutil.ReadObject(ctx, bucket, "foo") + assert.NoError(t.T(), err) + assert.Equal(t.T(), "foobar", string(contents)) +} + +func (t *staleFileHandleSyncedFile) TestRenamedFileWriteThrowsStaleFileHandleError() { + // Dirty the file by giving it some contents. + n, err := t.f1.Write([]byte("foobar")) + assert.NoError(t.T(), err) + assert.Equal(t.T(), 6, n) + // Rename the object. + err = os.Rename(t.f1.Name(), path.Join(mntDir, "bar")) + assert.NoError(t.T(), err) + + // Attempt to write to file should give ESTALE error. + _, err = t.f1.Write([]byte("taco")) + operations.ValidateESTALEError(t.T(), err) + // No error on sync and close because no data was written. + err = t.f1.Sync() + require.NoError(t.T(), err) + err = t.f1.Close() + require.NoError(t.T(), err) + // Make f1 nil, so that another attempt is not taken in TearDown to close the + // file. + t.f1 = nil +} + +func (t *staleFileHandleSyncedFile) TestFileDeletedRemotelySyncAndCloseThrowsStaleFileHandleError() { + // Dirty the file by giving it some contents. + n, err := t.f1.Write([]byte("foobar")) + assert.NoError(t.T(), err) + assert.Equal(t.T(), 6, n) + // Unlink the file. + err = storageutil.DeleteObject(ctx, bucket, "foo") + assert.NoError(t.T(), err) + // Verify unlink operation succeeds. + assert.NoError(t.T(), err) + operations.ValidateObjectNotFoundErr(ctx, t.T(), bucket, "foo") + // Attempt to write to file should not give any error. + n, err = t.f1.Write([]byte("taco")) + assert.Equal(t.T(), 4, n) + assert.NoError(t.T(), err) + + err = t.f1.Sync() + + operations.ValidateESTALEError(t.T(), err) + err = t.f1.Close() + operations.ValidateESTALEError(t.T(), err) + // Make f1 nil, so that another attempt is not taken in TearDown to close the + // file. + t.f1 = nil +} + +// Executes all stale handle tests for gcs synced files. +func TestStaleFileHandleSyncedFile(t *testing.T) { + suite.Run(t, new(staleFileHandleSyncedFile)) +} diff --git a/internal/fs/streaming_writes_common_test.go b/internal/fs/streaming_writes_common_test.go new file mode 100644 index 0000000000..1279806a28 --- /dev/null +++ b/internal/fs/streaming_writes_common_test.go @@ -0,0 +1,155 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Streaming write tests which are common for both local file and synced empty +// object. + +package fs_test + +import ( + "errors" + "os" + "path" + "strings" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type StreamingWritesCommonTest struct { + suite.Suite + fsTest +} + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func (t *StreamingWritesCommonTest) TestUnlinkBeforeWrite() { + // unlink the file. + err := os.Remove(t.f1.Name()) + assert.NoError(t.T(), err) + + // Stat the file and validate file is deleted. + operations.ValidateNoFileOrDirError(t.T(), t.f1.Name()) + // Close the file and validate that file is deleted from GCS. + err = t.f1.Close() + assert.NoError(nil, err) + t.f1 = nil + operations.ValidateObjectNotFoundErr(ctx, t.T(), bucket, fileName) +} + +func (t *StreamingWritesCommonTest) TestUnlinkAfterWrite() { + // Write content to file. + _, err := t.f1.Write([]byte("tacos")) + assert.NoError(t.T(), err) + + t.TestUnlinkBeforeWrite() +} + +func (t *StreamingWritesCommonTest) TestRenameFileWithPendingWrites() { + _, err := t.f1.Write([]byte("tacos")) + assert.NoError(t.T(), err) + newFilePath := path.Join(mntDir, "test.txt") + // Check that new file doesn't exist. + _, err = os.Stat(newFilePath) + assert.Error(t.T(), err) + assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) + + err = os.Rename(t.f1.Name(), newFilePath) + + assert.NoError(t.T(), err) + _, err = os.Stat(t.f1.Name()) + assert.Error(t.T(), err) + assert.True(t.T(), strings.Contains(err.Error(), "no such file or directory")) + content, err := os.ReadFile(newFilePath) + assert.NoError(t.T(), err) + assert.Equal(t.T(), "tacos", string(content)) +} + +func (t *StreamingWritesCommonTest) TestTruncateToLowerSizeSyncsFileToGcs() { + _, err := t.f1.Write([]byte("foobar")) + assert.NoError(t.T(), err) + + err = t.f1.Truncate(3) + + assert.NoError(t.T(), err) + content, err := storageutil.ReadObject(ctx, bucket, fileName) + require.NoError(t.T(), err) + assert.Equal(t.T(), "foobar", string(content)) + err = t.f1.Close() + assert.NoError(t.T(), err) + content, err = storageutil.ReadObject(ctx, bucket, fileName) + require.NoError(t.T(), err) + assert.Equal(t.T(), "foo", string(content)) + t.f1 = nil +} + +func (t *StreamingWritesCommonTest) TestTruncateToLowerSizeSyncsFileToGcsAndDeletingFileDeletesFromGcs() { + _, err := t.f1.Write([]byte("foobar")) + assert.NoError(t.T(), err) + err = t.f1.Truncate(3) + assert.NoError(t.T(), err) + content, err := storageutil.ReadObject(ctx, bucket, fileName) + require.NoError(t.T(), err) + assert.Equal(t.T(), "foobar", string(content)) + + err = os.Remove(t.f1.Name()) + + assert.NoError(t.T(), err) + _, err = storageutil.ReadObject(ctx, bucket, fileName) + require.Error(t.T(), err) + var notFoundErr *gcs.NotFoundError + assert.True(t.T(), errors.As(err, ¬FoundErr)) +} + +func (t *StreamingWritesCommonTest) TestOutOfOrderWriteSyncsFileToGcs() { + _, err := t.f1.Write([]byte("foobar")) + assert.NoError(t.T(), err) + + _, err = t.f1.WriteAt([]byte("foo"), 3) + + assert.NoError(t.T(), err) + content, err := storageutil.ReadObject(ctx, bucket, fileName) + require.NoError(t.T(), err) + assert.Equal(t.T(), "foobar", string(content)) + err = t.f1.Close() + assert.NoError(t.T(), err) + content, err = storageutil.ReadObject(ctx, bucket, fileName) + require.NoError(t.T(), err) + assert.Equal(t.T(), "foofoo", string(content)) + t.f1 = nil +} + +func (t *StreamingWritesCommonTest) TestOutOfOrderWriteSyncsFileToGcsAndDeletingFileDeletesFromGcs() { + _, err := t.f1.Write([]byte("foobar")) + assert.NoError(t.T(), err) + _, err = t.f1.WriteAt([]byte("foo"), 3) + assert.NoError(t.T(), err) + content, err := storageutil.ReadObject(ctx, bucket, fileName) + require.NoError(t.T(), err) + assert.Equal(t.T(), "foobar", string(content)) + + err = os.Remove(t.f1.Name()) + + assert.NoError(t.T(), err) + _, err = storageutil.ReadObject(ctx, bucket, fileName) + require.Error(t.T(), err) + var notFoundErr *gcs.NotFoundError + assert.True(t.T(), errors.As(err, ¬FoundErr)) +} diff --git a/internal/fs/streaming_writes_empty_gcs_object_test.go b/internal/fs/streaming_writes_empty_gcs_object_test.go new file mode 100644 index 0000000000..1e6eb16471 --- /dev/null +++ b/internal/fs/streaming_writes_empty_gcs_object_test.go @@ -0,0 +1,77 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Streaming write tests for synced empty object. + +package fs_test + +import ( + "os" + "path" + "syscall" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type StreamingWritesEmptyGCSObjectTest struct { + StreamingWritesCommonTest +} + +func (t *StreamingWritesEmptyGCSObjectTest) SetupSuite() { + t.serverCfg.NewConfig = &cfg.Config{ + Write: cfg.WriteConfig{ + BlockSizeMb: 1, + CreateEmptyFile: false, + EnableStreamingWrites: true, + GlobalMaxBlocks: 20, + MaxBlocksPerFile: 10, + }, + } + t.mountCfg.DisableWritebackCaching = true + t.fsTest.SetUpTestSuite() +} + +func (t *StreamingWritesEmptyGCSObjectTest) TearDownSuite() { + t.fsTest.TearDownTestSuite() +} +func (t *StreamingWritesEmptyGCSObjectTest) SetupTest() { + // Create an empty object on bucket. + _, err := storageutil.CreateObject( + ctx, + bucket, + fileName, + []byte("")) + assert.Equal(t.T(), nil, err) + // Open file handle to read or write. + t.f1, err = os.OpenFile(path.Join(mntDir, fileName), os.O_RDWR|syscall.O_DIRECT, filePerms) + assert.Equal(t.T(), nil, err) + + // Validate that empty file exists on GCS. + content, err := storageutil.ReadObject(ctx, bucket, fileName) + require.NoError(t.T(), err) + assert.Equal(t.T(), "", string(content)) +} + +func (t *StreamingWritesEmptyGCSObjectTest) TearDownTest() { + t.fsTest.TearDown() +} + +func TestStreamingWritesEmptyObjectTest(t *testing.T) { + suite.Run(t, new(StreamingWritesEmptyGCSObjectTest)) +} diff --git a/internal/fs/streaming_writes_local_file_test.go b/internal/fs/streaming_writes_local_file_test.go new file mode 100644 index 0000000000..5e7059dd78 --- /dev/null +++ b/internal/fs/streaming_writes_local_file_test.go @@ -0,0 +1,115 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Streaming write tests for local file. + +package fs_test + +import ( + "os" + "path" + "syscall" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +const ( + fileName = "foo" +) + +type StreamingWritesLocalFileTest struct { + StreamingWritesCommonTest +} + +func (t *StreamingWritesLocalFileTest) SetupSuite() { + t.serverCfg.NewConfig = &cfg.Config{ + Write: cfg.WriteConfig{ + BlockSizeMb: 1, + CreateEmptyFile: false, + EnableStreamingWrites: true, + GlobalMaxBlocks: 20, + MaxBlocksPerFile: 10, + }, + MetadataCache: cfg.MetadataCacheConfig{TtlSecs: 0}, + } + t.mountCfg.DisableWritebackCaching = true + t.fsTest.SetUpTestSuite() +} + +func (t *StreamingWritesLocalFileTest) TearDownSuite() { + t.fsTest.TearDownTestSuite() +} +func (t *StreamingWritesLocalFileTest) SetupTest() { + // CreateLocalFile creates a local file and validates that object does not + // exist on GCS. + _, t.f1 = operations.CreateLocalFile(ctx, t.T(), mntDir, bucket, fileName) +} + +func (t *StreamingWritesLocalFileTest) TearDownTest() { + t.fsTest.TearDown() +} + +func TestStreamingWritesLocalFileTest(t *testing.T) { + suite.Run(t, new(StreamingWritesLocalFileTest)) +} + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func (t *StreamingWritesLocalFileTest) TestRemoveDirectoryContainingLocalAndEmptyObject() { + // Create explicit directory with one synced and one local file. + explicitDirName := "explicit" + emptyFileName := "emptyFile" + nonEmptyFileName := "nonEmptyFile" + assert.Equal(t.T(), + nil, + t.createObjects( + map[string]string{ + // File + explicitDirName + "/": "", + path.Join(explicitDirName, emptyFileName): "", + path.Join(explicitDirName, nonEmptyFileName): "taco", + })) + // Write content to local and empty gcs file. + _, f1 := operations.CreateLocalFile(ctx, t.T(), mntDir, bucket, path.Join(explicitDirName, fileName)) + _, err := f1.WriteString(FileContents) + assert.NoError(t.T(), err) + f2, err := os.OpenFile(path.Join(mntDir, explicitDirName, emptyFileName), os.O_RDWR|syscall.O_DIRECT, filePerms) + assert.Equal(t.T(), nil, err) + _, err = f2.WriteString(FileContents) + assert.NoError(t.T(), err) + + // Attempt to remove explicit directory. + err = os.RemoveAll(path.Join(mntDir, explicitDirName)) + + // Verify rmDir operation succeeds. + assert.NoError(t.T(), err) + operations.ValidateNoFileOrDirError(t.T(), path.Join(explicitDirName, emptyFileName)) + operations.ValidateNoFileOrDirError(t.T(), path.Join(explicitDirName, nonEmptyFileName)) + operations.ValidateNoFileOrDirError(t.T(), path.Join(explicitDirName, fileName)) + operations.ValidateNoFileOrDirError(t.T(), explicitDirName) + err = operations.CloseLocalFile(t.T(), &f1) + assert.NoError(t.T(), err) + err = f2.Close() + assert.NoError(t.T(), err) + operations.ValidateObjectNotFoundErr(ctx, t.T(), bucket, path.Join(explicitDirName, emptyFileName)) + operations.ValidateObjectNotFoundErr(ctx, t.T(), bucket, path.Join(explicitDirName, nonEmptyFileName)) + operations.ValidateObjectNotFoundErr(ctx, t.T(), bucket, path.Join(explicitDirName, fileName)) + operations.ValidateObjectNotFoundErr(ctx, t.T(), bucket, explicitDirName) +} diff --git a/internal/fs/stress_test.go b/internal/fs/stress_test.go index ac2e89bac7..9ec463bf23 100644 --- a/internal/fs/stress_test.go +++ b/internal/fs/stress_test.go @@ -49,7 +49,7 @@ func forEachName(names []string, f func(string) error) (err error) { firstErr := make(chan error, 1) var wg sync.WaitGroup - for i := 0; i < parallelism; i++ { + for range parallelism { wg.Add(1) go func() { defer wg.Done() @@ -96,7 +96,7 @@ func (t *StressTest) CreateAndReadManyFilesInParallel() { const numFiles = 32 var names []string - for i := 0; i < numFiles; i++ { + for i := range numFiles { names = append(names, fmt.Sprintf("%d", i)) } @@ -148,7 +148,7 @@ func (t *StressTest) TruncateFileManyTimesInParallel() { var size int64 startTime := time.Now() for time.Since(startTime) < desiredDuration { - for i := 0; i < 10; i++ { + for range 10 { size = rand.Int63n(1 << 14) err = f.Truncate(size) if err != nil { @@ -167,7 +167,7 @@ func (t *StressTest) TruncateFileManyTimesInParallel() { const numWorkers = 16 finalSizes := make(chan int64, numWorkers) - for i := 0; i < numWorkers; i++ { + for range numWorkers { group.Go(func() (err error) { err = worker(finalSizes) return diff --git a/internal/fs/tracing_test.go b/internal/fs/tracing_test.go new file mode 100644 index 0000000000..343ec2152e --- /dev/null +++ b/internal/fs/tracing_test.go @@ -0,0 +1,1363 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/suite" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/wrappers" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/jacobsa/fuse/fuseops" + "github.com/jacobsa/fuse/fuseutil" + "github.com/jacobsa/timeutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel/trace" +) + +type TracingTestSuite struct { + suite.Suite + globalExporter *tracetest.InMemoryExporter +} + +func (s *TracingTestSuite) SetupSuite() { + s.globalExporter = newInMemoryExporter() +} + +func (s *TracingTestSuite) SetupSubTest() { + s.globalExporter.Reset() +} + +func createTestFileSystemWithTraces(ctx context.Context, t *testing.T, ignoreInterrupts bool) (gcs.Bucket, fuseutil.FileSystem) { + t.Helper() + + bucketName := "test-bucket" + bucket := fake.NewFakeBucket(timeutil.RealClock(), bucketName, gcs.BucketType{Hierarchical: false}) + serverCfg := &fs.ServerConfig{ + NewConfig: &cfg.Config{ + Write: cfg.WriteConfig{ + GlobalMaxBlocks: 1, + }, + Read: cfg.ReadConfig{ + EnableBufferedRead: false, + }, + EnableNewReader: true, + FileSystem: cfg.FileSystemConfig{ + IgnoreInterrupts: ignoreInterrupts, + }, + Trace: cfg.TraceConfig{ + Exporters: []string{"stdout"}, + SamplingRatio: 1.0, + }, + }, + CacheClock: &timeutil.SimulatedClock{}, + BucketName: bucketName, + BucketManager: &fakeBucketManager{ + buckets: map[string]gcs.Bucket{ + bucketName: bucket, + }, + }, + SequentialReadSizeMb: 200, + TraceHandle: tracing.NewOTELTracer(), + MetricHandle: metrics.NewNoopMetrics(), + } + server, err := fs.NewFileSystem(ctx, serverCfg) + require.NoError(t, err, "NewFileSystem") + return bucket, server +} + +func newInMemoryExporter() *tracetest.InMemoryExporter { + ex := tracetest.NewInMemoryExporter() + otel.SetTracerProvider(sdktrace.NewTracerProvider(sdktrace.WithSyncer(ex))) + return ex +} + +func (s *TracingTestSuite) TestTraceLookupInode() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.inode.lookup"}}, + {"disabled", false, []string{"fs.inode.lookup"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + lookupOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + + err := m.LookUpInode(context.Background(), lookupOp) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceStatFS() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.stat_fs"}}, + {"disabled", false, []string{"fs.stat_fs"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + statFsOp := &fuseops.StatFSOp{} + + err := m.StatFS(context.Background(), statFsOp) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceGetInodeAttributes() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.inode.get_attributes"}}, + {"disabled", false, []string{"fs.inode.get_attributes"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + op := &fuseops.GetInodeAttributesOp{ + Inode: fuseops.RootInodeID, + } + + err := m.GetInodeAttributes(context.Background(), op) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceSetInodeAttributes() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.inode.lookup", "fs.inode.set_attributes"}}, + {"disabled", false, []string{"fs.inode.lookup", "fs.inode.set_attributes"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := m.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + op := &fuseops.SetInodeAttributesOp{ + Inode: lookUpOp.Entry.Child, + } + + err = m.SetInodeAttributes(context.Background(), op) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceForgetInode() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.inode.lookup", "fs.inode.forget"}}, + {"disabled", false, []string{"fs.inode.lookup", "fs.inode.forget"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := m.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + op := &fuseops.ForgetInodeOp{ + Inode: lookUpOp.Entry.Child, + N: 1, + } + + err = m.ForgetInode(ctx, op) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceMkDir() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.dir.mk"}}, + {"disabled", false, []string{"fs.dir.mk"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + op := &fuseops.MkDirOp{ + Parent: fuseops.RootInodeID, + Name: "test", + } + + err := m.MkDir(ctx, op) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceMkNode() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.mknode"}}, + {"disabled", false, []string{"fs.mknode"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + op := &fuseops.MkNodeOp{ + Parent: fuseops.RootInodeID, + Name: "test", + } + + err := m.MkNode(ctx, op) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceCreateFile() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.file.create"}}, + {"disabled", false, []string{"fs.file.create"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + op := &fuseops.CreateFileOp{ + Parent: fuseops.RootInodeID, + Name: "test", + } + + err := m.CreateFile(ctx, op) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceCreateLink() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.inode.lookup", "fs.link.create"}}, + {"disabled", false, []string{"fs.inode.lookup", "fs.link.create"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := m.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + op := &fuseops.CreateLinkOp{ + Parent: fuseops.RootInodeID, + Name: "link", + Target: lookUpOp.Entry.Child, + } + + err = m.CreateLink(ctx, op) + assert.Error(t, err) // The operation is not implemented, so we expect an error. + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceCreateSymlink() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.symlink.create"}}, + {"disabled", false, []string{"fs.symlink.create"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + op := &fuseops.CreateSymlinkOp{ + Parent: fuseops.RootInodeID, + Name: "test", + Target: "target", + } + + err := m.CreateSymlink(ctx, op) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceRename() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.rename"}}, + {"disabled", false, []string{"fs.rename"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + oldName := "old" + newName := "new" + content := "test content" + createWithContents(ctx, t, bucket, oldName, content) + op := &fuseops.RenameOp{ + OldParent: fuseops.RootInodeID, + OldName: oldName, + NewParent: fuseops.RootInodeID, + NewName: newName, + } + + err := m.Rename(ctx, op) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceRmDir() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.dir.mk", "fs.dir.rm"}}, + {"disabled", false, []string{"fs.dir.mk", "fs.dir.rm"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + dirName := "test" + mkDirOp := &fuseops.MkDirOp{ + Parent: fuseops.RootInodeID, + Name: dirName, + } + err := m.MkDir(ctx, mkDirOp) + require.NoError(t, err) + op := &fuseops.RmDirOp{ + Parent: fuseops.RootInodeID, + Name: dirName, + } + + err = m.RmDir(ctx, op) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceUnlink() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.unlink"}}, + {"disabled", false, []string{"fs.unlink"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + op := &fuseops.UnlinkOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + + err := m.Unlink(ctx, op) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceOpenDir() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.dir.open"}}, + {"disabled", false, []string{"fs.dir.open"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + op := &fuseops.OpenDirOp{ + Inode: fuseops.RootInodeID, + } + + err := m.OpenDir(ctx, op) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceReadDir() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.dir.open", "fs.dir.read"}}, + {"disabled", false, []string{"fs.dir.open", "fs.dir.read"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + openOp := &fuseops.OpenDirOp{ + Inode: fuseops.RootInodeID, + } + err := m.OpenDir(ctx, openOp) + require.NoError(t, err) + op := &fuseops.ReadDirOp{ + Inode: fuseops.RootInodeID, + Handle: openOp.Handle, + Offset: 0, + Dst: make([]byte, 1024), + } + + err = m.ReadDir(ctx, op) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceReadDirPlus() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.dir.open", "fs.dir.read_plus"}}, + {"disabled", false, []string{"fs.dir.open", "fs.dir.read_plus"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + openOp := &fuseops.OpenDirOp{ + Inode: fuseops.RootInodeID, + } + err := m.OpenDir(ctx, openOp) + require.NoError(t, err) + op := &fuseops.ReadDirPlusOp{ + ReadDirOp: fuseops.ReadDirOp{ + Inode: fuseops.RootInodeID, + Handle: openOp.Handle, + Offset: 0, + Dst: make([]byte, 1024), + }, + } + + err = m.ReadDirPlus(ctx, op) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceReleaseDirHandle() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.dir.open", "fs.dir.release_handle"}}, + {"disabled", false, []string{"fs.dir.open", "fs.dir.release_handle"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + openOp := &fuseops.OpenDirOp{ + Inode: fuseops.RootInodeID, + } + err := m.OpenDir(ctx, openOp) + require.NoError(t, err) + op := &fuseops.ReleaseDirHandleOp{ + Handle: openOp.Handle, + } + + err = m.ReleaseDirHandle(ctx, op) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceOpenFile() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.inode.lookup", "fs.file.open"}}, + {"disabled", false, []string{"fs.inode.lookup", "fs.file.open"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := m.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + op := &fuseops.OpenFileOp{ + Inode: lookUpOp.Entry.Child, + } + + err = m.OpenFile(ctx, op) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceReadFile() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.inode.lookup", "fs.file.open", "fs.file.read"}}, + {"disabled", false, []string{"fs.inode.lookup", "fs.file.open", "fs.file.read"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := m.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + openOp := &fuseops.OpenFileOp{ + Inode: lookUpOp.Entry.Child, + } + err = m.OpenFile(ctx, openOp) + require.NoError(t, err) + op := &fuseops.ReadFileOp{ + Inode: lookUpOp.Entry.Child, + Handle: openOp.Handle, + Offset: 0, + } + + err = m.ReadFile(ctx, op) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceWriteFile() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.inode.lookup", "fs.file.open", "fs.file.write"}}, + {"disabled", false, []string{"fs.inode.lookup", "fs.file.open", "fs.file.write"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "" + createWithContents(ctx, t, bucket, fileName, content) + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := m.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + openOp := &fuseops.OpenFileOp{ + Inode: lookUpOp.Entry.Child, + } + err = m.OpenFile(ctx, openOp) + require.NoError(t, err) + op := &fuseops.WriteFileOp{ + Inode: lookUpOp.Entry.Child, + Handle: openOp.Handle, + Offset: 0, + Data: []byte("test"), + } + + err = m.WriteFile(ctx, op) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.GreaterOrEqual(t, len(ss), len(tt.spans)) + spanNamesInSS := make(map[string]trace.SpanKind) + for _, s := range ss { + spanNamesInSS[s.Name] = s.SpanKind + } + for _, spanName := range tt.spans { + assert.Contains(t, spanNamesInSS, spanName, "span %s not found", spanName) + assert.Equal(t, trace.SpanKindServer, spanNamesInSS[spanName]) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceSyncFile() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.inode.lookup", "fs.file.sync"}}, + {"disabled", false, []string{"fs.inode.lookup", "fs.file.sync"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := m.LookUpInode(ctx, lookUpOp) + assert.NoError(t, err) + op := &fuseops.SyncFileOp{ + Inode: lookUpOp.Entry.Child, + } + + err = m.SyncFile(ctx, op) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceFlushFile() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.inode.lookup", "fs.file.open", "fs.file.flush"}}, + {"disabled", false, []string{"fs.inode.lookup", "fs.file.open", "fs.file.flush"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := m.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + openOp := &fuseops.OpenFileOp{ + Inode: lookUpOp.Entry.Child, + } + err = m.OpenFile(ctx, openOp) + require.NoError(t, err) + op := &fuseops.FlushFileOp{ + Inode: lookUpOp.Entry.Child, + Handle: openOp.Handle, + } + + err = m.FlushFile(ctx, op) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceReleaseFileHandle() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.inode.lookup", "fs.file.open", "fs.file.release_handle"}}, + {"disabled", false, []string{"fs.inode.lookup", "fs.file.open", "fs.file.release_handle"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := m.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + openOp := &fuseops.OpenFileOp{ + Inode: lookUpOp.Entry.Child, + } + err = m.OpenFile(ctx, openOp) + require.NoError(t, err) + op := &fuseops.ReleaseFileHandleOp{ + Handle: openOp.Handle, + } + + err = m.ReleaseFileHandle(ctx, op) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceReadSymlink() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.symlink.create", "fs.symlink.read"}}, + {"disabled", false, []string{"fs.symlink.create", "fs.symlink.read"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + _, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + symlinkName := "test" + target := "target" + createSymlinkOp := &fuseops.CreateSymlinkOp{ + Parent: fuseops.RootInodeID, + Name: symlinkName, + Target: target, + } + err := m.CreateSymlink(ctx, createSymlinkOp) + require.NoError(t, err) + op := &fuseops.ReadSymlinkOp{ + Inode: createSymlinkOp.Entry.Child, + } + + err = m.ReadSymlink(ctx, op) + require.NoError(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceRemoveXattr() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.inode.lookup", "fs.xattr.remove"}}, + {"disabled", false, []string{"fs.inode.lookup", "fs.xattr.remove"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := m.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + op := &fuseops.RemoveXattrOp{ + Inode: lookUpOp.Entry.Child, + Name: "user.test", + } + + err = m.RemoveXattr(ctx, op) + assert.Error(t, err) // The operation is not implemented, so we expect an error. + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceGetXattr() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.inode.lookup", "fs.xattr.get"}}, + {"disabled", false, []string{"fs.inode.lookup", "fs.xattr.get"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := m.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + op := &fuseops.GetXattrOp{ + Inode: lookUpOp.Entry.Child, + Name: "user.test", + } + + err = m.GetXattr(ctx, op) + assert.NotNil(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceListXattr() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.inode.lookup", "fs.xattr.list"}}, + {"disabled", false, []string{"fs.inode.lookup", "fs.xattr.list"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := m.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + op := &fuseops.ListXattrOp{ + Inode: lookUpOp.Entry.Child, + } + + err = m.ListXattr(ctx, op) + assert.NotNil(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceSetXattr() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.inode.lookup", "fs.xattr.set"}}, + {"disabled", false, []string{"fs.inode.lookup", "fs.xattr.set"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := m.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + op := &fuseops.SetXattrOp{ + Inode: lookUpOp.Entry.Child, + Name: "user.test", + Value: []byte("test"), + } + + err = m.SetXattr(ctx, op) + assert.NotNil(t, err) + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceFallocate() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.inode.lookup", "fs.file.open", "fs.fallocate"}}, + {"disabled", false, []string{"fs.inode.lookup", "fs.file.open", "fs.fallocate"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := m.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + openOp := &fuseops.OpenFileOp{ + Inode: lookUpOp.Entry.Child, + } + err = m.OpenFile(ctx, openOp) + require.NoError(t, err) + op := &fuseops.FallocateOp{ + Inode: lookUpOp.Entry.Child, + Handle: openOp.Handle, + Offset: 0, + Length: 10, + Mode: 0, + } + + err = m.Fallocate(ctx, op) + assert.Error(t, err) // The operation is not implemented, so we expect an error. + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func (s *TracingTestSuite) TestTraceSyncFS() { + ctx := context.Background() + testCases := []struct { + name string + ignoreInterrupts bool + spans []string + }{ + {"enabled", true, []string{"fs.inode.lookup", "fs.sync_fs"}}, + {"disabled", false, []string{"fs.inode.lookup", "fs.sync_fs"}}, + } + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + bucket, server := createTestFileSystemWithTraces(ctx, t, tt.ignoreInterrupts) + m := wrappers.WithTracing(server, tracing.NewOTELTracer()) + ctx := context.Background() + fileName := "test.txt" + content := "test content" + createWithContents(ctx, t, bucket, fileName, content) + lookUpOp := &fuseops.LookUpInodeOp{ + Parent: fuseops.RootInodeID, + Name: fileName, + } + err := m.LookUpInode(ctx, lookUpOp) + require.NoError(t, err) + op := &fuseops.SyncFSOp{ + Inode: lookUpOp.Entry.Child, + } + + err = m.SyncFS(ctx, op) + assert.Error(t, err) // The operation is not implemented, so we expect an error. + + ss := s.globalExporter.GetSpans() + require.Len(t, ss, len(tt.spans)) + for i, spanName := range tt.spans { + assert.Equal(t, spanName, ss[i].Name) + assert.Equal(t, trace.SpanKindServer, ss[i].SpanKind) + } + }) + } +} + +func TestTracingTestSuite(t *testing.T) { + suite.Run(t, new(TracingTestSuite)) +} diff --git a/internal/fs/type_cache_test.go b/internal/fs/type_cache_test.go index 1d368a1710..d4026bb1c3 100644 --- a/internal/fs/type_cache_test.go +++ b/internal/fs/type_cache_test.go @@ -20,19 +20,20 @@ package fs_test import ( "fmt" - "io/fs" "math" "os" "path" "sync" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/metadata" - gcsfusefs "github.com/googlecloudplatform/gcsfuse/v2/internal/fs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" - "github.com/googlecloudplatform/gcsfuse/v2/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata" + gcsfusefs "github.com/googlecloudplatform/gcsfuse/v3/internal/fs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" "github.com/jacobsa/oglematchers" . "github.com/jacobsa/ogletest" @@ -70,9 +71,6 @@ var ( typeCacheMaxSizeMb int64 contentInBytes []byte - - fi fs.FileInfo - err error ) func (t *typeCacheTestCommon) SetUpTestSuite() { @@ -82,6 +80,8 @@ func (t *typeCacheTestCommon) SetUpTestSuite() { TtlSecs: ttlInSeconds, }, } + t.serverCfg.MetricHandle = metrics.NewNoopMetrics() + t.serverCfg.TraceHandle = tracing.NewNoopTracer() // Fill server-cfg from mount-config. func(newConfig *cfg.Config, serverCfg *gcsfusefs.ServerConfig) { @@ -174,7 +174,7 @@ func (t *typeCacheTestCommon) createObjectOnGCS(name string) *gcs.Object { } func (t *typeCacheTestCommon) statAndConfirmIsDir(name string, isDir bool) { - fi, err = os.Stat(name) + fi, err := os.Stat(name) ExpectEq(nil, err) AssertNe(nil, fi) @@ -182,7 +182,7 @@ func (t *typeCacheTestCommon) statAndConfirmIsDir(name string, isDir bool) { } func (t *typeCacheTestCommon) statAndExpectNotADirectoryError(name string) { - _, err = os.Stat(name) + _, err := os.Stat(name) ExpectNe(nil, err) ExpectThat(err, oglematchers.Error(oglematchers.HasSubstr("not a directory"))) @@ -210,7 +210,7 @@ func (t *typeCacheTestCommon) testNoInsertionSupported() { func (t *TypeCacheTestWithMaxSize1MB) TestNoEntryInitially() { // Initially, without any existing object, type-cache // should not contain any entry and os.Stat should fail. - _, err = os.Stat(path.Join(mntDir, foo)) + _, err := os.Stat(path.Join(mntDir, foo)) ExpectNe(nil, err) ExpectThat(err, oglematchers.Error(oglematchers.HasSubstr("no such file or directory"))) @@ -256,7 +256,7 @@ func (t *TypeCacheTestWithMaxSize1MB) TestSizeBasedEviction() { // Increase the object-name size to increase per-entry-size (max allowed is 1024) // to decrease count of objects being created for this, // to reduce the runtime. - for i := 0; i < 99; i++ { + for range 99 { objectNameTemplate += "abcdefjhij" // This makes it length+=10. } nameOfIthObject := func(i int) string { @@ -315,10 +315,7 @@ func (t *TypeCacheTestWithMaxSize1MB) TestSizeBasedEviction() { var batchOffset int for remainingObjectsToBeInserted := numObjectsToBeInserted; remainingObjectsToBeInserted > 0; { - objectsInsertedInThisBatch := maxNumObjectsPerBatch - if objectsInsertedInThisBatch > remainingObjectsToBeInserted { - objectsInsertedInThisBatch = remainingObjectsToBeInserted - } + objectsInsertedInThisBatch := min(maxNumObjectsPerBatch, remainingObjectsToBeInserted) wg.Add(1) go createAndStatBatchOfObjects(batchOffset, objectsInsertedInThisBatch) diff --git a/internal/fs/unsupported_path_test.go b/internal/fs/unsupported_path_test.go new file mode 100644 index 0000000000..1453006cad --- /dev/null +++ b/internal/fs/unsupported_path_test.go @@ -0,0 +1,189 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A collection of tests for a file system for unsupported object names. +package fs_test + +import ( + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/stretchr/testify/suite" +) + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +type UnsupportedPathNameTest struct { + suite.Suite + fsTest +} + +func TestUnsupportedPathNameTestSuite(t *testing.T) { + suite.Run(t, new(UnsupportedPathNameTest)) +} + +func (t *UnsupportedPathNameTest) SetupTest() { + t.serverCfg.ImplicitDirectories = true + t.serverCfg.RenameDirLimit = 10 + t.serverCfg.NewConfig = &cfg.Config{ + EnableUnsupportedPathSupport: true, + EnableAtomicRenameObject: true, + } + t.fsTest.SetUpTestSuite() +} + +func (t *UnsupportedPathNameTest) TearDownTest() { + t.fsTest.TearDown() + t.fsTest.TearDownTestSuite() +} + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func (t *UnsupportedPathNameTest) TestReadDirectory_WithUnsupportedNames() { + // Set up contents. + err := t.createObjects(map[string]string{ + "dir1/sub_dir1//file1": "", + "dir1/sub_dir1/file2": "content", + "dir2//file3": "", + "dir2/file4": "content", + "dir3/./file5": "content", + "dir4/.": "content", + "dir5/..": "content", + "file6": "content", + "//a.txt": "", + "./b.txt": "", + "dir6/.config": "content", + }) + t.Require().NoError(err) + var files []string + var dirs []string + + // Walk the mounted directory. + err = filepath.Walk(mntDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if path == mntDir { + return nil + } + if info.IsDir() { + dirs = append(dirs, info.Name()) + } else { + files = append(files, info.Name()) + } + return nil + }) + + // ReadDir should only show the supported object. + t.Require().NoError(err) + t.Assert().ElementsMatch([]string{"dir1", "dir2", "dir3", "dir4", "dir5", "dir6", "sub_dir1"}, dirs) + t.Assert().ElementsMatch([]string{"file2", "file4", "file6", ".config"}, files) +} + +func (t *UnsupportedPathNameTest) TestCopyDirectory_WithUnsupportedNames() { + err := t.createObjects(map[string]string{ + "src/file1": "content1", + "src//file2": "content2", + "src/./file3": "content3", + "src/ok": "content4", + "src/../file5": "content5", + "src///file5": "content6", + "src/.": "content", + "src/..": "content", + }) + t.Require().NoError(err) + srcPath := path.Join(mntDir, "src") + destPath := path.Join(mntDir, "dest") + + // Execute copy command. + cmd := exec.Command("cp", "-r", srcPath, destPath) + err = cmd.Run() + + t.Require().NoError(err) + // Verify the contents of the destination directory. + entries, err := os.ReadDir(destPath) + t.Require().NoError(err) + // Only supported files and directories should be copied. + t.Require().Len(entries, 2) + t.Assert().Equal("file1", entries[0].Name()) + t.Assert().Equal("ok", entries[1].Name()) +} + +func (t *UnsupportedPathNameTest) TestRenameDirectory_WithUnsupportedNames() { + // Set up contents. + err := t.createObjects(map[string]string{ + "src/file1": "content1", + "src//file2": "content2", + "src/./file3": "content3", + "src/ok/file4": "content4", + "src/.": "content", + "src/..": "content", + }) + t.Require().NoError(err) + srcPath := path.Join(mntDir, "src") + destPath := path.Join(mntDir, "dest") + + // Attempt to rename the directory. + err = os.Rename(srcPath, destPath) + t.Require().NoError(err) + + // The old path should not exist. + _, err = os.Stat(srcPath) + t.Assert().True(os.IsNotExist(err)) + // Verify the contents of the destination directory. + entries, err := os.ReadDir(destPath) + t.Require().NoError(err) + // Only supported files and directories are visible during list. + t.Require().Len(entries, 2) + t.Assert().Equal("file1", entries[0].Name()) + t.Assert().Equal("ok", entries[1].Name()) +} + +func (t *UnsupportedPathNameTest) TestDeleteDirectory_WithUnsupportedNames() { + // Set up contents. + err := t.createObjects(map[string]string{ + "dir_to_delete/file1": "content1", + "dir_to_delete//file2": "content2", + "dir_to_delete/./file3": "content3", + "dir_to_delete/ok": "content4", + "dir_to_delete/../file5": "content5", + "dir_to_delete///file6": "content6", + }) + t.Require().NoError(err) + dirPath := path.Join(mntDir, "dir_to_delete") + // Verify that listing only shows supported files. + entries, err := os.ReadDir(dirPath) + t.Require().NoError(err) + t.Require().Len(entries, 2) + t.Assert().Equal("file1", entries[0].Name()) + t.Assert().Equal("ok", entries[1].Name()) + + // Execute rm -rf command. + cmd := exec.Command("rm", "-rf", dirPath) + err = cmd.Run() + + t.Require().NoError(err) + _, err = os.Stat(dirPath) + t.Assert().Error(err) + t.Assert().True(strings.Contains(err.Error(), "no such file or directory")) +} diff --git a/internal/fs/wrappers/error_mapping.go b/internal/fs/wrappers/error_mapping.go index 40ea9b51e4..fb8e99a13a 100644 --- a/internal/fs/wrappers/error_mapping.go +++ b/internal/fs/wrappers/error_mapping.go @@ -22,12 +22,13 @@ import ( "syscall" "cloud.google.com/go/storage" - "github.com/googleapis/gax-go/v2/apierror" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/gcsfuse_errors" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" "github.com/jacobsa/fuse/fuseops" "github.com/jacobsa/fuse/fuseutil" "google.golang.org/api/googleapi" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) var ( @@ -39,6 +40,12 @@ func errno(err error) error { return nil } + // The object is modified or deleted by a concurrent process. + var clobberedErr *gcsfuse_errors.FileClobberedError + if errors.As(err, &clobberedErr) { + return syscall.ESTALE + } + // Use existing em errno var errno syscall.Errno if errors.As(err, &errno) { @@ -64,14 +71,12 @@ func errno(err error) error { return syscall.EACCES } - // The control client API returns an RPC error code instead of googleapi code. - // Currently, we only have gRPC control client APIs, so we are checking the gRPC status code. - // TODO: Add a check for the HTTP status code when the HTTP client is initiated for control client APIs. - var apiErr *apierror.APIError - if errors.As(err, &apiErr) { - switch apiErr.GRPCStatus().Code() { + if grpcStatus, ok := status.FromError(err); ok { + switch grpcStatus.Code() { case codes.Canceled: return syscall.EINTR + case codes.AlreadyExists: + return syscall.EEXIST case codes.PermissionDenied, codes.Unauthenticated: return syscall.EACCES case codes.NotFound: @@ -270,6 +275,15 @@ func (em *errorMapping) ReadDir( return em.mapError("ReadDir", err) } +func (em *errorMapping) ReadDirPlus( + ctx context.Context, + op *fuseops.ReadDirPlusOp) error { + defer em.handlePanic() + + err := em.wrapped.ReadDirPlus(ctx, op) + return em.mapError("ReadDirPlus", err) +} + func (em *errorMapping) ReleaseDirHandle( ctx context.Context, op *fuseops.ReleaseDirHandleOp) error { @@ -386,3 +400,12 @@ func (em *errorMapping) Fallocate( err := em.wrapped.Fallocate(ctx, op) return em.mapError("Fallocate", err) } + +func (em *errorMapping) SyncFS( + ctx context.Context, + op *fuseops.SyncFSOp) error { + defer em.handlePanic() + + err := em.wrapped.SyncFS(ctx, op) + return em.mapError("SyncFS", err) +} diff --git a/internal/fs/wrappers/error_mapping_test.go b/internal/fs/wrappers/error_mapping_test.go index 9f27be86fb..72047a4112 100644 --- a/internal/fs/wrappers/error_mapping_test.go +++ b/internal/fs/wrappers/error_mapping_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/googleapis/gax-go/v2/apierror" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/gcsfuse_errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "google.golang.org/api/googleapi" @@ -45,6 +46,15 @@ func (testSuite *ErrorMapping) TestPermissionDeniedGrpcApiError() { assert.Equal(testSuite.T(), syscall.EACCES, fsErr) } +func (testSuite *ErrorMapping) TestAlreadyExistGrpcApiError() { + statusErr := status.New(codes.AlreadyExists, "already exist") + apiError, _ := apierror.FromError(statusErr.Err()) + + fsErr := errno(apiError) + + assert.Equal(testSuite.T(), syscall.EEXIST, fsErr) +} + func (testSuite *ErrorMapping) TestNotFoundGrpcApiError() { statusErr := status.New(codes.NotFound, "Not found") apiError, _ := apierror.FromError(statusErr.Err()) @@ -80,3 +90,14 @@ func (testSuite *ErrorMapping) TestUnAuthenticatedHttpGoogleApiError() { assert.Equal(testSuite.T(), syscall.EACCES, fsErr) } + +func (testSuite *ErrorMapping) TestFileClobberedError() { + clobberedErr := &gcsfuse_errors.FileClobberedError{ + Err: fmt.Errorf("some error"), + ObjectName: "foo.txt", + } + + gotErrno := errno(clobberedErr) + + assert.Equal(testSuite.T(), syscall.ESTALE, gotErrno) +} diff --git a/internal/fs/wrappers/monitoring.go b/internal/fs/wrappers/monitoring.go index f84114ad8b..86c3ac0e22 100644 --- a/internal/fs/wrappers/monitoring.go +++ b/internal/fs/wrappers/monitoring.go @@ -17,83 +17,40 @@ package wrappers import ( "context" "errors" - "fmt" "syscall" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" - "github.com/googlecloudplatform/gcsfuse/v2/internal/monitor" - "github.com/googlecloudplatform/gcsfuse/v2/internal/monitor/tags" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" "github.com/jacobsa/fuse/fuseops" "github.com/jacobsa/fuse/fuseutil" - "go.opencensus.io/plugin/ochttp" - "go.opencensus.io/stats" - "go.opencensus.io/stats/view" - "go.opencensus.io/tag" ) const name = "cloud.google.com/gcsfuse" -var ( - opsCountOC = stats.Int64("fs/ops_count", "The number of ops processed by the file system.", monitor.UnitDimensionless) - opsLatencyOC = stats.Int64("fs/ops_latency", "The latency of a file system operation.", monitor.UnitMicroseconds) - opsErrorCountOC = stats.Int64("fs/ops_error_count", "The number of errors generated by file system operation.", monitor.UnitDimensionless) -) - // Error categories const ( - errDevice = "DEVICE_ERROR" - errDirNotEmpty = "DIR_NOT_EMPTY" - errFileExists = "FILE_EXISTS" - errFileDir = "FILE_DIR_ERROR" - errNotImplemented = "NOT_IMPLEMENTED" - errIO = "IO_ERROR" - errInterrupt = "INTERRUPT_ERROR" - errInvalidArg = "INVALID_ARGUMENT" - errInvalidOp = "INVALID_OPERATION" - errMisc = "MISC_ERROR" - errNetwork = "NETWORK_ERROR" - errNoFileOrDir = "NO_FILE_OR_DIR" - errNotADir = "NOT_A_DIR" - errPerm = "PERM_ERROR" - errProcessMgmt = "PROCESS_RESOURCE_MGMT_ERROR" - errTooManyFiles = "TOO_MANY_OPEN_FILES" + errDevice = metrics.FsErrorCategoryDEVICEERRORAttr + errDirNotEmpty = metrics.FsErrorCategoryDIRNOTEMPTYAttr + errFileExists = metrics.FsErrorCategoryFILEEXISTSAttr + errFileDir = metrics.FsErrorCategoryFILEDIRERRORAttr + errNotImplemented = metrics.FsErrorCategoryNOTIMPLEMENTEDAttr + errIO = metrics.FsErrorCategoryIOERRORAttr + errInterrupt = metrics.FsErrorCategoryINTERRUPTERRORAttr + errInvalidArg = metrics.FsErrorCategoryINVALIDARGUMENTAttr + errInvalidOp = metrics.FsErrorCategoryINVALIDOPERATIONAttr + errMisc = metrics.FsErrorCategoryMISCERRORAttr + errNetwork = metrics.FsErrorCategoryNETWORKERRORAttr + errNoFileOrDir = metrics.FsErrorCategoryNOFILEORDIRAttr + errNotADir = metrics.FsErrorCategoryNOTADIRAttr + errPerm = metrics.FsErrorCategoryPERMERRORAttr + errProcessMgmt = metrics.FsErrorCategoryPROCESSRESOURCEMGMTERRORAttr + errTooManyFiles = metrics.FsErrorCategoryTOOMANYOPENFILESAttr ) -// Initialize the metrics. -func init() { - - // Register the view. - if err := view.Register( - &view.View{ - Name: "fs/ops_count", - Measure: opsCountOC, - Description: "The cumulative number of ops processed by the file system.", - Aggregation: view.Sum(), - TagKeys: []tag.Key{tags.FSOp}, - }, - &view.View{ - Name: "fs/ops_error_count", - Measure: opsErrorCountOC, - Description: "The cumulative number of errors generated by file system operations", - Aggregation: view.Sum(), - TagKeys: []tag.Key{tags.FSOp, tags.FSErrCategory}, - }, - &view.View{ - Name: "fs/ops_latency", - Measure: opsLatencyOC, - Description: "The cumulative distribution of file system operation latencies", - Aggregation: ochttp.DefaultLatencyDistribution, - TagKeys: []tag.Key{tags.FSOp}, - }); err != nil { - fmt.Printf("Failed to register metrics for the file system: %v\n", err) - } -} - // categorize maps an error to an error-category. // This helps reduce the cardinality of the labels to less than 30. // This lower number of errors allows the various errors to get piped to Cloud metrics without getting dropped. -func categorize(err error) string { +func categorize(err error) metrics.FsErrorCategory { if err == nil { return "" } @@ -268,58 +225,29 @@ func categorize(err error) string { } // Records file system operation count, failed operation count and the operation latency. -func recordOp(ctx context.Context, method string, start time.Time, fsErr error) { - // Recording opCount. - if err := stats.RecordWithTags( - ctx, - []tag.Mutator{ - tag.Upsert(tags.FSOp, method), - }, - opsCountOC.M(1), - ); err != nil { - // Error in recording opCount. - logger.Errorf("Cannot record file system op: %v", err) - } +func recordOp(ctx context.Context, metricHandle metrics.MetricHandle, method metrics.FsOp, start time.Time, fsErr error) { + metricHandle.FsOpsCount(1, method) // Recording opErrorCount. if fsErr != nil { errCategory := categorize(fsErr) - if err := stats.RecordWithTags( - ctx, - []tag.Mutator{ - tag.Upsert(tags.FSOp, method), - tag.Upsert(tags.FSErrCategory, errCategory), - }, - opsErrorCountOC.M(1), - ); err != nil { - // Error in recording opErrorCount. - logger.Errorf("Cannot record error count of the file system failed operations: %v", err) - } - } - - // Recording opLatency. - if err := stats.RecordWithTags( - ctx, - []tag.Mutator{ - tag.Upsert(tags.FSOp, method), - }, - opsLatencyOC.M(time.Since(start).Microseconds()), - ); err != nil { - // Error in opLatency. - logger.Errorf("Cannot record file system operation latency: %v", err) + metricHandle.FsOpsErrorCount(1, errCategory, method) } + metricHandle.FsOpsLatency(ctx, time.Since(start), method) } // WithMonitoring takes a FileSystem, returns a FileSystem with monitoring // on the counts of requests per API. -func WithMonitoring(fs fuseutil.FileSystem) fuseutil.FileSystem { +func WithMonitoring(fs fuseutil.FileSystem, metricHandle metrics.MetricHandle) fuseutil.FileSystem { return &monitoring{ - wrapped: fs, + wrapped: fs, + metricHandle: metricHandle, } } type monitoring struct { - wrapped fuseutil.FileSystem + wrapped fuseutil.FileSystem + metricHandle metrics.MetricHandle } func (fs *monitoring) Destroy() { @@ -328,125 +256,134 @@ func (fs *monitoring) Destroy() { type wrappedCall func(ctx context.Context) error -func (fs *monitoring) invokeWrapped(ctx context.Context, opName string, w wrappedCall) error { +func (fs *monitoring) invokeWrapped(ctx context.Context, opName metrics.FsOp, w wrappedCall) error { startTime := time.Now() err := w(ctx) - recordOp(ctx, opName, startTime, err) + recordOp(ctx, fs.metricHandle, opName, startTime, err) return err } func (fs *monitoring) StatFS(ctx context.Context, op *fuseops.StatFSOp) error { - return fs.invokeWrapped(ctx, "StatFS", func(ctx context.Context) error { return fs.wrapped.StatFS(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpOthersAttr, func(ctx context.Context) error { return fs.wrapped.StatFS(ctx, op) }) } func (fs *monitoring) LookUpInode(ctx context.Context, op *fuseops.LookUpInodeOp) error { - return fs.invokeWrapped(ctx, "LookUpInode", func(ctx context.Context) error { return fs.wrapped.LookUpInode(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpLookUpInodeAttr, func(ctx context.Context) error { return fs.wrapped.LookUpInode(ctx, op) }) } func (fs *monitoring) GetInodeAttributes(ctx context.Context, op *fuseops.GetInodeAttributesOp) error { - return fs.invokeWrapped(ctx, "GetInodeAttributes", func(ctx context.Context) error { return fs.wrapped.GetInodeAttributes(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpGetInodeAttributesAttr, func(ctx context.Context) error { return fs.wrapped.GetInodeAttributes(ctx, op) }) } func (fs *monitoring) SetInodeAttributes(ctx context.Context, op *fuseops.SetInodeAttributesOp) error { - return fs.invokeWrapped(ctx, "SetInodeAttributes", func(ctx context.Context) error { return fs.wrapped.SetInodeAttributes(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpSetInodeAttributesAttr, func(ctx context.Context) error { return fs.wrapped.SetInodeAttributes(ctx, op) }) } func (fs *monitoring) ForgetInode(ctx context.Context, op *fuseops.ForgetInodeOp) error { - return fs.invokeWrapped(ctx, "ForgetInode", func(ctx context.Context) error { return fs.wrapped.ForgetInode(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpForgetInodeAttr, func(ctx context.Context) error { return fs.wrapped.ForgetInode(ctx, op) }) } func (fs *monitoring) BatchForget(ctx context.Context, op *fuseops.BatchForgetOp) error { - return fs.invokeWrapped(ctx, "BatchForget", func(ctx context.Context) error { return fs.wrapped.BatchForget(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpBatchForgetAttr, func(ctx context.Context) error { return fs.wrapped.BatchForget(ctx, op) }) } func (fs *monitoring) MkDir(ctx context.Context, op *fuseops.MkDirOp) error { - return fs.invokeWrapped(ctx, "MkDir", func(ctx context.Context) error { return fs.wrapped.MkDir(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpMkDirAttr, func(ctx context.Context) error { return fs.wrapped.MkDir(ctx, op) }) } func (fs *monitoring) MkNode(ctx context.Context, op *fuseops.MkNodeOp) error { - return fs.invokeWrapped(ctx, "MkNode", func(ctx context.Context) error { return fs.wrapped.MkNode(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpMkNodeAttr, func(ctx context.Context) error { return fs.wrapped.MkNode(ctx, op) }) } func (fs *monitoring) CreateFile(ctx context.Context, op *fuseops.CreateFileOp) error { - return fs.invokeWrapped(ctx, "CreateFile", func(ctx context.Context) error { return fs.wrapped.CreateFile(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpCreateFileAttr, func(ctx context.Context) error { return fs.wrapped.CreateFile(ctx, op) }) } func (fs *monitoring) CreateLink(ctx context.Context, op *fuseops.CreateLinkOp) error { - return fs.invokeWrapped(ctx, "CreateLink", func(ctx context.Context) error { return fs.wrapped.CreateLink(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpCreateLinkAttr, func(ctx context.Context) error { return fs.wrapped.CreateLink(ctx, op) }) } func (fs *monitoring) CreateSymlink(ctx context.Context, op *fuseops.CreateSymlinkOp) error { - return fs.invokeWrapped(ctx, "CreateSymlink", func(ctx context.Context) error { return fs.wrapped.CreateSymlink(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpCreateSymlinkAttr, func(ctx context.Context) error { return fs.wrapped.CreateSymlink(ctx, op) }) } func (fs *monitoring) Rename(ctx context.Context, op *fuseops.RenameOp) error { - return fs.invokeWrapped(ctx, "Rename", func(ctx context.Context) error { return fs.wrapped.Rename(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpRenameAttr, func(ctx context.Context) error { return fs.wrapped.Rename(ctx, op) }) } func (fs *monitoring) RmDir(ctx context.Context, op *fuseops.RmDirOp) error { - return fs.invokeWrapped(ctx, "RmDir", func(ctx context.Context) error { return fs.wrapped.RmDir(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpRmDirAttr, func(ctx context.Context) error { return fs.wrapped.RmDir(ctx, op) }) } func (fs *monitoring) Unlink(ctx context.Context, op *fuseops.UnlinkOp) error { - return fs.invokeWrapped(ctx, "Unlink", func(ctx context.Context) error { return fs.wrapped.Unlink(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpUnlinkAttr, func(ctx context.Context) error { return fs.wrapped.Unlink(ctx, op) }) } func (fs *monitoring) OpenDir(ctx context.Context, op *fuseops.OpenDirOp) error { - return fs.invokeWrapped(ctx, "OpenDir", func(ctx context.Context) error { return fs.wrapped.OpenDir(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpOpenDirAttr, func(ctx context.Context) error { return fs.wrapped.OpenDir(ctx, op) }) } func (fs *monitoring) ReadDir(ctx context.Context, op *fuseops.ReadDirOp) error { - return fs.invokeWrapped(ctx, "ReadDir", func(ctx context.Context) error { return fs.wrapped.ReadDir(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpReadDirAttr, func(ctx context.Context) error { return fs.wrapped.ReadDir(ctx, op) }) +} + +func (fs *monitoring) ReadDirPlus(ctx context.Context, op *fuseops.ReadDirPlusOp) error { + return fs.invokeWrapped(ctx, metrics.FsOpReadDirPlusAttr, func(ctx context.Context) error { return fs.wrapped.ReadDirPlus(ctx, op) }) } func (fs *monitoring) ReleaseDirHandle(ctx context.Context, op *fuseops.ReleaseDirHandleOp) error { - return fs.invokeWrapped(ctx, "ReleaseDirHandle", func(ctx context.Context) error { return fs.wrapped.ReleaseDirHandle(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpReleaseDirHandleAttr, func(ctx context.Context) error { return fs.wrapped.ReleaseDirHandle(ctx, op) }) } func (fs *monitoring) OpenFile(ctx context.Context, op *fuseops.OpenFileOp) error { - return fs.invokeWrapped(ctx, "OpenFile", func(ctx context.Context) error { return fs.wrapped.OpenFile(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpOpenFileAttr, func(ctx context.Context) error { return fs.wrapped.OpenFile(ctx, op) }) } func (fs *monitoring) ReadFile(ctx context.Context, op *fuseops.ReadFileOp) error { - return fs.invokeWrapped(ctx, "ReadFile", func(ctx context.Context) error { return fs.wrapped.ReadFile(ctx, op) }) + fs.metricHandle.ReadBlockSizes(ctx, int64(len(op.Dst))) + return fs.invokeWrapped(ctx, metrics.FsOpReadFileAttr, func(ctx context.Context) error { return fs.wrapped.ReadFile(ctx, op) }) } func (fs *monitoring) WriteFile(ctx context.Context, op *fuseops.WriteFileOp) error { - return fs.invokeWrapped(ctx, "WriteFile", func(ctx context.Context) error { return fs.wrapped.WriteFile(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpWriteFileAttr, func(ctx context.Context) error { return fs.wrapped.WriteFile(ctx, op) }) } func (fs *monitoring) SyncFile(ctx context.Context, op *fuseops.SyncFileOp) error { - return fs.invokeWrapped(ctx, "SyncFile", func(ctx context.Context) error { return fs.wrapped.SyncFile(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpSyncFileAttr, func(ctx context.Context) error { return fs.wrapped.SyncFile(ctx, op) }) } func (fs *monitoring) FlushFile(ctx context.Context, op *fuseops.FlushFileOp) error { - return fs.invokeWrapped(ctx, "FlushFile", func(ctx context.Context) error { return fs.wrapped.FlushFile(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpFlushFileAttr, func(ctx context.Context) error { return fs.wrapped.FlushFile(ctx, op) }) } func (fs *monitoring) ReleaseFileHandle(ctx context.Context, op *fuseops.ReleaseFileHandleOp) error { - return fs.invokeWrapped(ctx, "ReleaseFileHandle", func(ctx context.Context) error { return fs.wrapped.ReleaseFileHandle(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpReleaseFileHandleAttr, func(ctx context.Context) error { return fs.wrapped.ReleaseFileHandle(ctx, op) }) } func (fs *monitoring) ReadSymlink(ctx context.Context, op *fuseops.ReadSymlinkOp) error { - return fs.invokeWrapped(ctx, "ReadSymlink", func(ctx context.Context) error { return fs.wrapped.ReadSymlink(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpReadSymlinkAttr, func(ctx context.Context) error { return fs.wrapped.ReadSymlink(ctx, op) }) } func (fs *monitoring) RemoveXattr(ctx context.Context, op *fuseops.RemoveXattrOp) error { - return fs.invokeWrapped(ctx, "RemoveXattr", func(ctx context.Context) error { return fs.wrapped.RemoveXattr(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpOthersAttr, func(ctx context.Context) error { return fs.wrapped.RemoveXattr(ctx, op) }) } func (fs *monitoring) GetXattr(ctx context.Context, op *fuseops.GetXattrOp) error { - return fs.invokeWrapped(ctx, "GetXattr", func(ctx context.Context) error { return fs.wrapped.GetXattr(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpOthersAttr, func(ctx context.Context) error { return fs.wrapped.GetXattr(ctx, op) }) } func (fs *monitoring) ListXattr(ctx context.Context, op *fuseops.ListXattrOp) error { - return fs.invokeWrapped(ctx, "ListXattr", func(ctx context.Context) error { return fs.wrapped.ListXattr(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpOthersAttr, func(ctx context.Context) error { return fs.wrapped.ListXattr(ctx, op) }) } func (fs *monitoring) SetXattr(ctx context.Context, op *fuseops.SetXattrOp) error { - return fs.invokeWrapped(ctx, "SetXattr", func(ctx context.Context) error { return fs.wrapped.SetXattr(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpOthersAttr, func(ctx context.Context) error { return fs.wrapped.SetXattr(ctx, op) }) } func (fs *monitoring) Fallocate(ctx context.Context, op *fuseops.FallocateOp) error { - return fs.invokeWrapped(ctx, "Fallocate", func(ctx context.Context) error { return fs.wrapped.Fallocate(ctx, op) }) + return fs.invokeWrapped(ctx, metrics.FsOpOthersAttr, func(ctx context.Context) error { return fs.wrapped.Fallocate(ctx, op) }) +} + +func (fs *monitoring) SyncFS(ctx context.Context, op *fuseops.SyncFSOp) error { + return fs.invokeWrapped(ctx, metrics.FsOpOthersAttr, func(ctx context.Context) error { return fs.wrapped.SyncFS(ctx, op) }) } diff --git a/internal/fs/wrappers/monitoring_test.go b/internal/fs/wrappers/monitoring_test.go index 3480de466d..84459b4414 100644 --- a/internal/fs/wrappers/monitoring_test.go +++ b/internal/fs/wrappers/monitoring_test.go @@ -19,6 +19,7 @@ import ( "syscall" "testing" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" "github.com/stretchr/testify/assert" ) @@ -26,7 +27,7 @@ func TestFsErrStrAndCategory(t *testing.T) { t.Parallel() tests := []struct { fsErr error - expectedCategory string + expectedCategory metrics.FsErrorCategory }{ { fsErr: fmt.Errorf("some random error"), diff --git a/internal/fs/wrappers/tracing.go b/internal/fs/wrappers/tracing.go index c5ada7e519..16629c271f 100644 --- a/internal/fs/wrappers/tracing.go +++ b/internal/fs/wrappers/tracing.go @@ -19,154 +19,160 @@ import ( "github.com/jacobsa/fuse/fuseops" "github.com/jacobsa/fuse/fuseutil" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/codes" - "go.opentelemetry.io/otel/trace" + + "github.com/googlecloudplatform/gcsfuse/v3/tracing" ) -type tracing struct { - wrapped fuseutil.FileSystem - tracer trace.Tracer +type tracedFS struct { + wrapped fuseutil.FileSystem + traceHandle tracing.TraceHandle } // WithTracing wraps a FileSystem and creates a root trace. -func WithTracing(wrapped fuseutil.FileSystem) fuseutil.FileSystem { - return &tracing{ - wrapped: wrapped, - tracer: otel.Tracer(name), +func WithTracing(wrapped fuseutil.FileSystem, traceHandle tracing.TraceHandle) fuseutil.FileSystem { + return &tracedFS{ + wrapped: wrapped, + traceHandle: traceHandle, } } -func (fs *tracing) Destroy() { +func (fs *tracedFS) Destroy() { fs.wrapped.Destroy() } -func (fs *tracing) invokeWrapped(ctx context.Context, opName string, w wrappedCall) error { +func (fs *tracedFS) invokeWrapped(ctx context.Context, opName string, w wrappedCall) error { // Span's SpanKind is set to trace.SpanKindServer since GCSFuse is like a server for the requests that the Kernel sends. - ctx, span := fs.tracer.Start(ctx, opName, trace.WithSpanKind(trace.SpanKindServer)) - defer span.End() + ctx, span := fs.traceHandle.StartServerSpan(ctx, opName) + defer fs.traceHandle.EndSpan(span) err := w(ctx) if err != nil { - span.RecordError(err) - span.SetStatus(codes.Error, err.Error()) + fs.traceHandle.RecordError(span, err) } return err } -func (fs *tracing) StatFS(ctx context.Context, op *fuseops.StatFSOp) error { - return fs.invokeWrapped(ctx, "StatFS", func(ctx context.Context) error { return fs.wrapped.StatFS(ctx, op) }) +func (fs *tracedFS) StatFS(ctx context.Context, op *fuseops.StatFSOp) error { + return fs.invokeWrapped(ctx, tracing.StatFS, func(ctx context.Context) error { return fs.wrapped.StatFS(ctx, op) }) +} + +func (fs *tracedFS) LookUpInode(ctx context.Context, op *fuseops.LookUpInodeOp) error { + return fs.invokeWrapped(ctx, tracing.LookUpInode, func(ctx context.Context) error { return fs.wrapped.LookUpInode(ctx, op) }) +} + +func (fs *tracedFS) GetInodeAttributes(ctx context.Context, op *fuseops.GetInodeAttributesOp) error { + return fs.invokeWrapped(ctx, tracing.GetInodeAttributes, func(ctx context.Context) error { return fs.wrapped.GetInodeAttributes(ctx, op) }) } -func (fs *tracing) LookUpInode(ctx context.Context, op *fuseops.LookUpInodeOp) error { - return fs.invokeWrapped(ctx, "LookUpInode", func(ctx context.Context) error { return fs.wrapped.LookUpInode(ctx, op) }) +func (fs *tracedFS) SetInodeAttributes(ctx context.Context, op *fuseops.SetInodeAttributesOp) error { + return fs.invokeWrapped(ctx, tracing.SetInodeAttributes, func(ctx context.Context) error { return fs.wrapped.SetInodeAttributes(ctx, op) }) } -func (fs *tracing) GetInodeAttributes(ctx context.Context, op *fuseops.GetInodeAttributesOp) error { - return fs.invokeWrapped(ctx, "GetInodeAttributes", func(ctx context.Context) error { return fs.wrapped.GetInodeAttributes(ctx, op) }) +func (fs *tracedFS) ForgetInode(ctx context.Context, op *fuseops.ForgetInodeOp) error { + return fs.invokeWrapped(ctx, tracing.ForgetInode, func(ctx context.Context) error { return fs.wrapped.ForgetInode(ctx, op) }) } -func (fs *tracing) SetInodeAttributes(ctx context.Context, op *fuseops.SetInodeAttributesOp) error { - return fs.invokeWrapped(ctx, "SetInodeAttributes", func(ctx context.Context) error { return fs.wrapped.SetInodeAttributes(ctx, op) }) +func (fs *tracedFS) BatchForget(ctx context.Context, op *fuseops.BatchForgetOp) error { + return fs.invokeWrapped(ctx, tracing.BatchForget, func(ctx context.Context) error { return fs.wrapped.BatchForget(ctx, op) }) } -func (fs *tracing) ForgetInode(ctx context.Context, op *fuseops.ForgetInodeOp) error { - return fs.invokeWrapped(ctx, "ForgetInode", func(ctx context.Context) error { return fs.wrapped.ForgetInode(ctx, op) }) +func (fs *tracedFS) MkDir(ctx context.Context, op *fuseops.MkDirOp) error { + return fs.invokeWrapped(ctx, tracing.MkDir, func(ctx context.Context) error { return fs.wrapped.MkDir(ctx, op) }) } -func (fs *tracing) BatchForget(ctx context.Context, op *fuseops.BatchForgetOp) error { - return fs.invokeWrapped(ctx, "BatchForget", func(ctx context.Context) error { return fs.wrapped.BatchForget(ctx, op) }) +func (fs *tracedFS) MkNode(ctx context.Context, op *fuseops.MkNodeOp) error { + return fs.invokeWrapped(ctx, tracing.MkNode, func(ctx context.Context) error { return fs.wrapped.MkNode(ctx, op) }) } -func (fs *tracing) MkDir(ctx context.Context, op *fuseops.MkDirOp) error { - return fs.invokeWrapped(ctx, "MkDir", func(ctx context.Context) error { return fs.wrapped.MkDir(ctx, op) }) +func (fs *tracedFS) CreateFile(ctx context.Context, op *fuseops.CreateFileOp) error { + return fs.invokeWrapped(ctx, tracing.CreateFile, func(ctx context.Context) error { return fs.wrapped.CreateFile(ctx, op) }) } -func (fs *tracing) MkNode(ctx context.Context, op *fuseops.MkNodeOp) error { - return fs.invokeWrapped(ctx, "MkNode", func(ctx context.Context) error { return fs.wrapped.MkNode(ctx, op) }) +func (fs *tracedFS) CreateLink(ctx context.Context, op *fuseops.CreateLinkOp) error { + return fs.invokeWrapped(ctx, tracing.CreateLink, func(ctx context.Context) error { return fs.wrapped.CreateLink(ctx, op) }) } -func (fs *tracing) CreateFile(ctx context.Context, op *fuseops.CreateFileOp) error { - return fs.invokeWrapped(ctx, "CreateFile", func(ctx context.Context) error { return fs.wrapped.CreateFile(ctx, op) }) +func (fs *tracedFS) CreateSymlink(ctx context.Context, op *fuseops.CreateSymlinkOp) error { + return fs.invokeWrapped(ctx, tracing.CreateSymlink, func(ctx context.Context) error { return fs.wrapped.CreateSymlink(ctx, op) }) } -func (fs *tracing) CreateLink(ctx context.Context, op *fuseops.CreateLinkOp) error { - return fs.invokeWrapped(ctx, "CreateLink", func(ctx context.Context) error { return fs.wrapped.CreateLink(ctx, op) }) +func (fs *tracedFS) Rename(ctx context.Context, op *fuseops.RenameOp) error { + return fs.invokeWrapped(ctx, tracing.Rename, func(ctx context.Context) error { return fs.wrapped.Rename(ctx, op) }) } -func (fs *tracing) CreateSymlink(ctx context.Context, op *fuseops.CreateSymlinkOp) error { - return fs.invokeWrapped(ctx, "CreateSymlink", func(ctx context.Context) error { return fs.wrapped.CreateSymlink(ctx, op) }) +func (fs *tracedFS) RmDir(ctx context.Context, op *fuseops.RmDirOp) error { + return fs.invokeWrapped(ctx, tracing.RmDir, func(ctx context.Context) error { return fs.wrapped.RmDir(ctx, op) }) } -func (fs *tracing) Rename(ctx context.Context, op *fuseops.RenameOp) error { - return fs.invokeWrapped(ctx, "Rename", func(ctx context.Context) error { return fs.wrapped.Rename(ctx, op) }) +func (fs *tracedFS) Unlink(ctx context.Context, op *fuseops.UnlinkOp) error { + return fs.invokeWrapped(ctx, tracing.Unlink, func(ctx context.Context) error { return fs.wrapped.Unlink(ctx, op) }) } -func (fs *tracing) RmDir(ctx context.Context, op *fuseops.RmDirOp) error { - return fs.invokeWrapped(ctx, "RmDir", func(ctx context.Context) error { return fs.wrapped.RmDir(ctx, op) }) +func (fs *tracedFS) OpenDir(ctx context.Context, op *fuseops.OpenDirOp) error { + return fs.invokeWrapped(ctx, tracing.OpenDir, func(ctx context.Context) error { return fs.wrapped.OpenDir(ctx, op) }) } -func (fs *tracing) Unlink(ctx context.Context, op *fuseops.UnlinkOp) error { - return fs.invokeWrapped(ctx, "Unlink", func(ctx context.Context) error { return fs.wrapped.Unlink(ctx, op) }) +func (fs *tracedFS) ReadDir(ctx context.Context, op *fuseops.ReadDirOp) error { + return fs.invokeWrapped(ctx, tracing.ReadDir, func(ctx context.Context) error { return fs.wrapped.ReadDir(ctx, op) }) } -func (fs *tracing) OpenDir(ctx context.Context, op *fuseops.OpenDirOp) error { - return fs.invokeWrapped(ctx, "OpenDir", func(ctx context.Context) error { return fs.wrapped.OpenDir(ctx, op) }) +func (fs *tracedFS) ReadDirPlus(ctx context.Context, op *fuseops.ReadDirPlusOp) error { + return fs.invokeWrapped(ctx, tracing.ReadDirPlus, func(ctx context.Context) error { return fs.wrapped.ReadDirPlus(ctx, op) }) } -func (fs *tracing) ReadDir(ctx context.Context, op *fuseops.ReadDirOp) error { - return fs.invokeWrapped(ctx, "ReadDir", func(ctx context.Context) error { return fs.wrapped.ReadDir(ctx, op) }) +func (fs *tracedFS) ReleaseDirHandle(ctx context.Context, op *fuseops.ReleaseDirHandleOp) error { + return fs.invokeWrapped(ctx, tracing.ReleaseDirHandle, func(ctx context.Context) error { return fs.wrapped.ReleaseDirHandle(ctx, op) }) } -func (fs *tracing) ReleaseDirHandle(ctx context.Context, op *fuseops.ReleaseDirHandleOp) error { - return fs.invokeWrapped(ctx, "ReleaseDirHandle", func(ctx context.Context) error { return fs.wrapped.ReleaseDirHandle(ctx, op) }) +func (fs *tracedFS) OpenFile(ctx context.Context, op *fuseops.OpenFileOp) error { + return fs.invokeWrapped(ctx, tracing.OpenFile, func(ctx context.Context) error { return fs.wrapped.OpenFile(ctx, op) }) } -func (fs *tracing) OpenFile(ctx context.Context, op *fuseops.OpenFileOp) error { - return fs.invokeWrapped(ctx, "OpenFile", func(ctx context.Context) error { return fs.wrapped.OpenFile(ctx, op) }) +func (fs *tracedFS) ReadFile(ctx context.Context, op *fuseops.ReadFileOp) error { + return fs.invokeWrapped(ctx, tracing.ReadFile, func(ctx context.Context) error { return fs.wrapped.ReadFile(ctx, op) }) } -func (fs *tracing) ReadFile(ctx context.Context, op *fuseops.ReadFileOp) error { - return fs.invokeWrapped(ctx, "ReadFile", func(ctx context.Context) error { return fs.wrapped.ReadFile(ctx, op) }) +func (fs *tracedFS) WriteFile(ctx context.Context, op *fuseops.WriteFileOp) error { + return fs.invokeWrapped(ctx, tracing.WriteFile, func(ctx context.Context) error { return fs.wrapped.WriteFile(ctx, op) }) } -func (fs *tracing) WriteFile(ctx context.Context, op *fuseops.WriteFileOp) error { - return fs.invokeWrapped(ctx, "WriteFile", func(ctx context.Context) error { return fs.wrapped.WriteFile(ctx, op) }) +func (fs *tracedFS) SyncFile(ctx context.Context, op *fuseops.SyncFileOp) error { + return fs.invokeWrapped(ctx, tracing.SyncFile, func(ctx context.Context) error { return fs.wrapped.SyncFile(ctx, op) }) } -func (fs *tracing) SyncFile(ctx context.Context, op *fuseops.SyncFileOp) error { - return fs.invokeWrapped(ctx, "SyncFile", func(ctx context.Context) error { return fs.wrapped.SyncFile(ctx, op) }) +func (fs *tracedFS) FlushFile(ctx context.Context, op *fuseops.FlushFileOp) error { + return fs.invokeWrapped(ctx, tracing.FlushFile, func(ctx context.Context) error { return fs.wrapped.FlushFile(ctx, op) }) } -func (fs *tracing) FlushFile(ctx context.Context, op *fuseops.FlushFileOp) error { - return fs.invokeWrapped(ctx, "FlushFile", func(ctx context.Context) error { return fs.wrapped.FlushFile(ctx, op) }) +func (fs *tracedFS) ReleaseFileHandle(ctx context.Context, op *fuseops.ReleaseFileHandleOp) error { + return fs.invokeWrapped(ctx, tracing.ReleaseFileHandle, func(ctx context.Context) error { return fs.wrapped.ReleaseFileHandle(ctx, op) }) } -func (fs *tracing) ReleaseFileHandle(ctx context.Context, op *fuseops.ReleaseFileHandleOp) error { - return fs.invokeWrapped(ctx, "ReleaseFileHandle", func(ctx context.Context) error { return fs.wrapped.ReleaseFileHandle(ctx, op) }) +func (fs *tracedFS) ReadSymlink(ctx context.Context, op *fuseops.ReadSymlinkOp) error { + return fs.invokeWrapped(ctx, tracing.ReadSymlink, func(ctx context.Context) error { return fs.wrapped.ReadSymlink(ctx, op) }) } -func (fs *tracing) ReadSymlink(ctx context.Context, op *fuseops.ReadSymlinkOp) error { - return fs.invokeWrapped(ctx, "ReadSymlink", func(ctx context.Context) error { return fs.wrapped.ReadSymlink(ctx, op) }) +func (fs *tracedFS) RemoveXattr(ctx context.Context, op *fuseops.RemoveXattrOp) error { + return fs.invokeWrapped(ctx, tracing.RemoveXattr, func(ctx context.Context) error { return fs.wrapped.RemoveXattr(ctx, op) }) } -func (fs *tracing) RemoveXattr(ctx context.Context, op *fuseops.RemoveXattrOp) error { - return fs.invokeWrapped(ctx, "RemoveXattr", func(ctx context.Context) error { return fs.wrapped.RemoveXattr(ctx, op) }) +func (fs *tracedFS) GetXattr(ctx context.Context, op *fuseops.GetXattrOp) error { + return fs.invokeWrapped(ctx, tracing.GetXattr, func(ctx context.Context) error { return fs.wrapped.GetXattr(ctx, op) }) } -func (fs *tracing) GetXattr(ctx context.Context, op *fuseops.GetXattrOp) error { - return fs.invokeWrapped(ctx, "GetXattr", func(ctx context.Context) error { return fs.wrapped.GetXattr(ctx, op) }) +func (fs *tracedFS) ListXattr(ctx context.Context, op *fuseops.ListXattrOp) error { + return fs.invokeWrapped(ctx, tracing.ListXattr, func(ctx context.Context) error { return fs.wrapped.ListXattr(ctx, op) }) } -func (fs *tracing) ListXattr(ctx context.Context, op *fuseops.ListXattrOp) error { - return fs.invokeWrapped(ctx, "ListXattr", func(ctx context.Context) error { return fs.wrapped.ListXattr(ctx, op) }) +func (fs *tracedFS) SetXattr(ctx context.Context, op *fuseops.SetXattrOp) error { + return fs.invokeWrapped(ctx, tracing.SetXattr, func(ctx context.Context) error { return fs.wrapped.SetXattr(ctx, op) }) } -func (fs *tracing) SetXattr(ctx context.Context, op *fuseops.SetXattrOp) error { - return fs.invokeWrapped(ctx, "SetXattr", func(ctx context.Context) error { return fs.wrapped.SetXattr(ctx, op) }) +func (fs *tracedFS) Fallocate(ctx context.Context, op *fuseops.FallocateOp) error { + return fs.invokeWrapped(ctx, tracing.Fallocate, func(ctx context.Context) error { return fs.wrapped.Fallocate(ctx, op) }) } -func (fs *tracing) Fallocate(ctx context.Context, op *fuseops.FallocateOp) error { - return fs.invokeWrapped(ctx, "Fallocate", func(ctx context.Context) error { return fs.wrapped.Fallocate(ctx, op) }) +func (fs *tracedFS) SyncFS(ctx context.Context, op *fuseops.SyncFSOp) error { + return fs.invokeWrapped(ctx, tracing.SyncFS, func(ctx context.Context) error { return fs.wrapped.SyncFS(ctx, op) }) } diff --git a/internal/fs/wrappers/tracing_test.go b/internal/fs/wrappers/tracing_test.go index d40794ef9e..1a83b071c1 100644 --- a/internal/fs/wrappers/tracing_test.go +++ b/internal/fs/wrappers/tracing_test.go @@ -25,6 +25,8 @@ import ( sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/sdk/trace/tracetest" "go.opentelemetry.io/otel/trace" + + tracing "github.com/googlecloudplatform/gcsfuse/v3/tracing" ) func newInMemoryExporter(t *testing.T) *tracetest.InMemoryExporter { @@ -103,6 +105,10 @@ func (d dummyFS) ReadDir(_ context.Context, _ *fuseops.ReadDirOp) error { return nil } +func (d dummyFS) ReadDirPlus(_ context.Context, _ *fuseops.ReadDirPlusOp) error { + return nil +} + func (d dummyFS) ReleaseDirHandle(_ context.Context, _ *fuseops.ReleaseDirHandleOp) error { return nil } @@ -155,6 +161,10 @@ func (d dummyFS) Fallocate(_ context.Context, _ *fuseops.FallocateOp) error { return nil } +func (d dummyFS) SyncFS(_ context.Context, _ *fuseops.SyncFSOp) error { + return nil +} + func (d dummyFS) Destroy() {} func TestSpanCreation(t *testing.T) { @@ -162,9 +172,9 @@ func TestSpanCreation(t *testing.T) { t.Cleanup(func() { ex.Reset() }) - m := tracing{ - wrapped: dummyFS{}, - tracer: otel.Tracer("test"), + m := tracedFS{ + wrapped: dummyFS{}, + traceHandle: tracing.NewOTELTracer(), } err := m.StatFS(context.Background(), nil) @@ -172,6 +182,6 @@ func TestSpanCreation(t *testing.T) { ss := ex.GetSpans() require.Len(t, ss, 1) - assert.Equal(t, "StatFS", ss[0].Name) + assert.Equal(t, "fs.stat_fs", ss[0].Name) assert.Equal(t, trace.SpanKindServer, ss[0].SpanKind) } diff --git a/internal/fs/zonal_bucket_test.go b/internal/fs/zonal_bucket_test.go new file mode 100644 index 0000000000..60ecd2b187 --- /dev/null +++ b/internal/fs/zonal_bucket_test.go @@ -0,0 +1,63 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs_test + +import ( + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type ZonalBucketTests struct { + RenameFileTests + fsTest +} + +func TestZonalBucketTests(t *testing.T) { suite.Run(t, new(ZonalBucketTests)) } + +func (t *ZonalBucketTests) SetupSuite() { + t.serverCfg.ImplicitDirectories = false + t.serverCfg.MetricHandle = metrics.NewNoopMetrics() + t.serverCfg.TraceHandle = tracing.NewNoopTracer() + bucketType = gcs.BucketType{Zonal: true} + t.fsTest.SetUpTestSuite() +} + +func (t *ZonalBucketTests) TearDownSuite() { + t.fsTest.TearDownTestSuite() +} + +func (t *ZonalBucketTests) SetupTest() { + err := t.createFolders([]string{"foo/", "bar/", "foo/test2/", "foo/test/"}) + require.NoError(t.T(), err) + + err = t.createObjects( + map[string]string{ + "foo/file1.txt": file1Content, + "foo/file2.txt": file2Content, + "foo/test/file3.txt": "xyz", + "foo/implicit_dir/file3.txt": "xxw", + "bar/file1.txt": "-1234556789", + }) + require.NoError(t.T(), err) +} + +func (t *ZonalBucketTests) TearDownTest() { + t.fsTest.TearDown() +} diff --git a/internal/gcsx/bucket_manager.go b/internal/gcsx/bucket_manager.go index 18435fe0dd..2354070640 100644 --- a/internal/gcsx/bucket_manager.go +++ b/internal/gcsx/bucket_manager.go @@ -21,15 +21,18 @@ import ( "path" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/lru" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/metadata" - "github.com/googlecloudplatform/gcsfuse/v2/internal/canned" - "github.com/googlecloudplatform/gcsfuse/v2/internal/monitor" - "github.com/googlecloudplatform/gcsfuse/v2/internal/ratelimit" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/caching" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata" + "github.com/googlecloudplatform/gcsfuse/v3/internal/canned" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/monitor" + "github.com/googlecloudplatform/gcsfuse/v3/internal/ratelimit" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/caching" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" "github.com/jacobsa/timeutil" ) @@ -39,8 +42,12 @@ type BucketConfig struct { EgressBandwidthLimitBytesPerSecond float64 OpRateLimitHz float64 StatCacheMaxSizeMB uint64 - StatCacheTTL time.Duration - EnableMonitoring bool + // Config for TTL of entries for existing file in stat cache + StatCacheTTL time.Duration + // Config for TTL of entries for non-existing file in stat cache + NegativeStatCacheTTL time.Duration + EnableMonitoring bool + LogSeverity cfg.LogSeverity // Files backed by on object of length at least AppendThreshold that have // only been appended to (i.e. none of the object's contents have been @@ -58,15 +65,29 @@ type BucketConfig struct { // Note that if the process fails or is interrupted the temporary object will // not be cleaned up, so the user must ensure that TmpObjectPrefix is // periodically garbage collected. - AppendThreshold int64 - TmpObjectPrefix string + AppendThreshold int64 + ChunkRetryDeadlineSecs int64 + ChunkTransferTimeoutSecs int64 + TmpObjectPrefix string + + // Disable Initial ListObject API check during the mount operation. + DisableListAccessCheck bool + + // Enable dummy I/O mode for testing purposes, simulated read without + // any data read from GCS. + // All the metadata operations like object listing and stats are real. + DummyIOCfg cfg.DummyIoConfig + + IsTypeCacheDeprecated bool + + ImplicitDir bool } // BucketManager manages the lifecycle of buckets. type BucketManager interface { SetUpBucket( ctx context.Context, - name string, isMultibucketMount bool) (b SyncerBucket, err error) + name string, isMultibucketMount bool, metricHandle metrics.MetricHandle) (b SyncerBucket, err error) // Shuts down the bucket manager and its buckets ShutDown() @@ -155,22 +176,36 @@ func (bm *bucketManager) SetUpBucket( ctx context.Context, name string, isMultibucketMount bool, + metricHandle metrics.MetricHandle, ) (sb SyncerBucket, err error) { var b gcs.Bucket // Set up the appropriate backing bucket. if name == canned.FakeBucketName { b = canned.MakeFakeBucket(ctx) } else { - b = bm.storageHandle.BucketHandle(ctx, name, bm.config.BillingProject) + b, err = bm.storageHandle.BucketHandle(ctx, name, bm.config.BillingProject) + if err != nil { + err = fmt.Errorf("BucketHandle: %w", err) + return + } } - // Enable monitoring. - if bm.config.EnableMonitoring { - b = monitor.NewMonitoringBucket(b) + if bm.config.DummyIOCfg.Enable { + logger.Infof("Enabling dummy I/O mode for bucket %q\n", name) + // Wrap in a dummy I/O bucket, which serves the data without actually going to network (GCS). + b = storage.NewDummyIOBucket(b, storage.DummyIOBucketParams{ + ReaderLatency: bm.config.DummyIOCfg.ReaderLatency, + PerMBLatency: bm.config.DummyIOCfg.PerMbLatency, + }) } - // Enable gcs logs. - b = storage.NewDebugBucket(b) + // Enable monitoring. + b = monitor.NewMonitoringBucket(b, metricHandle) + + if bm.config.LogSeverity == cfg.TraceLogSeverity { + // Enable gcs logs. + b = storage.NewDebugBucket(b) + } // Limit to a requested prefix of the bucket, if any. if bm.config.OnlyDir != "" { @@ -192,7 +227,8 @@ func (bm *bucketManager) SetUpBucket( return } - // Enable cached StatObject results, if appropriate. + // Enable cached StatObject results based on stat cache config. + // Disabling stat cache with below config also disables negative stat cache. if bm.config.StatCacheTTL != 0 && bm.sharedStatCache != nil { var statCache metadata.StatCache if isMultibucketMount { @@ -205,7 +241,10 @@ func (bm *bucketManager) SetUpBucket( bm.config.StatCacheTTL, statCache, timeutil.RealClock(), - b) + b, + bm.config.NegativeStatCacheTTL, + bm.config.IsTypeCacheDeprecated, + bm.config.ImplicitDir) } // Enable content type awareness @@ -218,16 +257,19 @@ func (bm *bucketManager) SetUpBucket( } sb = NewSyncerBucket( bm.config.AppendThreshold, + bm.config.ChunkRetryDeadlineSecs, + bm.config.ChunkTransferTimeoutSecs, bm.config.TmpObjectPrefix, b) // Fetch bucket type from storage layout api and set bucket type. b.BucketType() - // Check whether this bucket works, giving the user a warning early if there - // is some problem. - { - _, err = b.ListObjects(ctx, &gcs.ListObjectsRequest{MaxResults: 1}) + // TODO(b/471129209): Cleanup this code after confirming the GetStorageLayout is sufficient for bucket access checks. + if !bm.config.DisableListAccessCheck { + // Check whether this bucket works, giving the user a warning early if there + // is some problem. + _, err = b.ListObjects(ctx, &gcs.ListObjectsRequest{MaxResults: 1, IncludeFoldersAsPrefixes: true, Delimiter: "/"}) if err != nil { return } diff --git a/internal/gcsx/bucket_manager_test.go b/internal/gcsx/bucket_manager_test.go index cfdbfa6bfe..6ba89fbc22 100644 --- a/internal/gcsx/bucket_manager_test.go +++ b/internal/gcsx/bucket_manager_test.go @@ -16,12 +16,17 @@ package gcsx import ( "context" + "strings" "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "cloud.google.com/go/storage/control/apiv2/controlpb" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" . "github.com/jacobsa/ogletest" + "github.com/stretchr/testify/mock" ) func TestBucketManager(t *testing.T) { RunTests(t) } @@ -37,6 +42,7 @@ type BucketManagerTest struct { bucket gcs.Bucket storageHandle storage.StorageHandle fakeStorage storage.FakeStorage + mockClient *storage.MockStorageControlClient } var _ SetUpInterface = &BucketManagerTest{} @@ -45,12 +51,20 @@ var _ TearDownInterface = &BucketManagerTest{} func init() { RegisterTestSuite(&BucketManagerTest{}) } func (t *BucketManagerTest) SetUp(_ *TestInfo) { - t.fakeStorage = storage.NewFakeStorage() + var err error + t.mockClient = new(storage.MockStorageControlClient) + t.fakeStorage = storage.NewFakeStorageWithMockClient(t.mockClient, cfg.HTTP2) t.storageHandle = t.fakeStorage.CreateStorageHandle() + t.mockClient.On("GetStorageLayout", mock.Anything, mock.Anything, mock.Anything). + Return(&controlpb.StorageLayout{ + HierarchicalNamespace: &controlpb.StorageLayout_HierarchicalNamespace{Enabled: true}, + LocationType: "zone", + }, nil) ctx := context.Background() - t.bucket = t.storageHandle.BucketHandle(ctx, TestBucketName, "") + t.bucket, err = t.storageHandle.BucketHandle(ctx, TestBucketName, "") AssertNe(nil, t.bucket) + AssertEq(nil, err) } func (t *BucketManagerTest) TearDown() { @@ -93,7 +107,7 @@ func (t *BucketManagerTest) TestSetUpBucketMethod() { bm.config = bucketConfig bm.gcCtx = ctx - bucket, err := bm.SetUpBucket(context.Background(), TestBucketName, false) + bucket, err := bm.SetUpBucket(context.Background(), TestBucketName, false, metrics.NewNoopMetrics()) ExpectNe(nil, bucket.Syncer) ExpectEq(nil, err) @@ -117,7 +131,7 @@ func (t *BucketManagerTest) TestSetUpBucketMethod_IsMultiBucketMountTrue() { bm.config = bucketConfig bm.gcCtx = ctx - bucket, err := bm.SetUpBucket(context.Background(), TestBucketName, true) + bucket, err := bm.SetUpBucket(context.Background(), TestBucketName, true, metrics.NewNoopMetrics()) ExpectNe(nil, bucket.Syncer) ExpectEq(nil, err) @@ -141,9 +155,10 @@ func (t *BucketManagerTest) TestSetUpBucketMethodWhenBucketDoesNotExist() { bm.config = bucketConfig bm.gcCtx = ctx - bucket, err := bm.SetUpBucket(context.Background(), invalidBucketName, false) + bucket, err := bm.SetUpBucket(context.Background(), invalidBucketName, false, metrics.NewNoopMetrics()) - ExpectEq("error in iterating through objects: storage: bucket doesn't exist", err.Error()) + AssertNe(nil, err) + ExpectTrue(strings.Contains(err.Error(), "error in iterating through objects: storage: bucket doesn't exist")) ExpectNe(nil, bucket.Syncer) } @@ -165,8 +180,9 @@ func (t *BucketManagerTest) TestSetUpBucketMethodWhenBucketDoesNotExist_IsMultiB bm.config = bucketConfig bm.gcCtx = ctx - bucket, err := bm.SetUpBucket(context.Background(), invalidBucketName, true) + bucket, err := bm.SetUpBucket(context.Background(), invalidBucketName, true, metrics.NewNoopMetrics()) - ExpectEq("error in iterating through objects: storage: bucket doesn't exist", err.Error()) + AssertNe(nil, err) + ExpectTrue(strings.Contains(err.Error(), "error in iterating through objects: storage: bucket doesn't exist")) ExpectNe(nil, bucket.Syncer) } diff --git a/internal/gcsx/client_readers/gcs_reader.go b/internal/gcsx/client_readers/gcs_reader.go new file mode 100644 index 0000000000..36ea14835f --- /dev/null +++ b/internal/gcsx/client_readers/gcs_reader.go @@ -0,0 +1,212 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client_readers + +import ( + "context" + "errors" + "fmt" + "io" + "sync" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" +) + +// ReaderType represents different types of go-sdk gcs readers. +type ReaderType int + +// ReaderType enum values. +const ( + // RangeReaderType corresponds to NewReader method in bucket_handle.go + RangeReaderType ReaderType = iota + + // MultiRangeReaderType corresponds to NewMultiRangeDownloader method in bucket_handle.go + MultiRangeReaderType +) + +type GCSReader struct { + gcsx.Reader + object *gcs.MinObject + bucket gcs.Bucket + + rangeReader *RangeReader + mrr *MultiRangeReader + + // mu synchronizes reads through range reader. + mu sync.Mutex + + // readTypeClassifier tracks the read access pattern (e.g., sequential, random) + // to optimize read strategies. It is shared across different reader layers. + readTypeClassifier *gcsx.ReadTypeClassifier + traceHandle tracing.TraceHandle +} + +type GCSReaderConfig struct { + MetricHandle metrics.MetricHandle + TraceHandle tracing.TraceHandle + MrdWrapper *gcsx.MultiRangeDownloaderWrapper + Config *cfg.Config + ReadTypeClassifier *gcsx.ReadTypeClassifier +} + +func NewGCSReader(obj *gcs.MinObject, bucket gcs.Bucket, config *GCSReaderConfig) *GCSReader { + if config.TraceHandle == nil { + config.TraceHandle = tracing.NewNoopTracer() + } + + return &GCSReader{ + object: obj, + bucket: bucket, + rangeReader: NewRangeReader(obj, bucket, config.Config, config.MetricHandle, config.TraceHandle), + mrr: NewMultiRangeReader(obj, config.MetricHandle, config.TraceHandle, config.MrdWrapper), + readTypeClassifier: config.ReadTypeClassifier, + traceHandle: config.TraceHandle, + } +} + +// Detects whether the read was short or not and returns whether it should be retried or not. +// Reads would only be retried in case of rapid buckets and when the read data was less than requested (& object size) +// and there was no error apart from EOF or short reads. +func shouldRetryForShortRead(err error, bytesRead int, p []byte, offset int64, objectSize uint64, bucketType gcs.BucketType, skipSizeChecks bool) bool { + if !bucketType.IsRapid() { + return false + } + + if bytesRead >= len(p) { + return false + } + + if offset+int64(bytesRead) >= int64(objectSize) && !skipSizeChecks { + return false + } + + if !(err == nil || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, util.ErrShortRead)) { + return false + } + + return true +} + +func (gr *GCSReader) ReaderName() string { + return "gcs_reader" +} + +func (gr *GCSReader) ReadAt(ctx context.Context, readRequest *gcsx.ReadRequest) (readResponse gcsx.ReadResponse, err error) { + + if readRequest.Offset >= int64(gr.object.Size) && !readRequest.SkipSizeChecks { + return readResponse, io.EOF + } else if readRequest.Offset < 0 { + err := fmt.Errorf( + "illegal offset %d for %d byte object", + readRequest.Offset, + gr.object.Size) + return readResponse, err + } + + gcsReaderRequest := &gcsx.GCSReaderRequest{ + Buffer: readRequest.Buffer, + Offset: readRequest.Offset, + EndOffset: readRequest.Offset + int64(len(readRequest.Buffer)), + ReadInfo: &readRequest.ReadInfo, + ForceCreateReader: false, + SkipSizeChecks: readRequest.SkipSizeChecks, + } + + bytesRead, err := gr.read(ctx, gcsReaderRequest) + readResponse.Size = bytesRead + + // Retry reading in case of short read. + if shouldRetryForShortRead(err, bytesRead, readRequest.Buffer, readRequest.Offset, gr.object.Size, gr.bucket.BucketType(), readRequest.SkipSizeChecks) { + gcsReaderRequest.Offset += int64(bytesRead) + gcsReaderRequest.Buffer = readRequest.Buffer[bytesRead:] + gcsReaderRequest.ForceCreateReader = true + var bytesReadOnRetry int + bytesReadOnRetry, err = gr.read(ctx, gcsReaderRequest) + readResponse.Size += bytesReadOnRetry + } + + return readResponse, err +} + +func (gr *GCSReader) read(ctx context.Context, readReq *gcsx.GCSReaderRequest) (bytesRead int, err error) { + // We don't take a lock here to allow random reads to proceed without waiting. + // The read type is re-evaluated for zonal buckets inside the lock if necessary. + reqReaderType := gr.readerType(readReq.ReadType, gr.bucket.BucketType()) + var readResp gcsx.ReadResponse + + // In case readReq.SkipSizeChecks is true, it means requests can be beyond cached object size and hence + // it qualifies for scenario where only MRD must be used (RangeReader is not suitable here). + if reqReaderType == RangeReaderType && !readReq.SkipSizeChecks { + gr.mu.Lock() + + // In case of multiple threads reading parallely, it is possible that many of them might be waiting + // at this lock and hence the earlier calculated value of readerType might not be valid once they + // acquire the lock. Hence, needs to be calculated again. + // We recalculate the read type if the expected offset has changed. This is important for both + // rapid and regional buckets. For rapid buckets, it allows switching to MRD for high-performance + // random reads. For regional buckets, it helps in adjusting the prefetch window for the range + // reader when the read pattern changes. + if readReq.ExpectedOffset != gr.readTypeClassifier.NextExpectedOffset() { + *readReq.ReadInfo = gr.readTypeClassifier.GetReadInfo(readReq.Offset, readReq.SeekRecorded) + reqReaderType = gr.readerType(readReq.ReadType, gr.bucket.BucketType()) + } + // If the readerType is range reader after re calculation, then use range reader. + // Otherwise, fall back to MultiRange Downloader. + if reqReaderType == RangeReaderType { + defer gr.mu.Unlock() + // Calculate the end offset based on previous read requests. + // It will be used if a new range reader needs to be created. + readReq.EndOffset = gr.getEndOffset(readReq.Offset) + readResp, err = gr.rangeReader.ReadAt(ctx, readReq) + return readResp.Size, err + } + gr.mu.Unlock() + } + + readResp, err = gr.mrr.ReadAt(ctx, readReq) + return readResp.Size, err +} + +// readerType specifies the go-sdk interface to use for reads. +func (gr *GCSReader) readerType(readType int64, bucketType gcs.BucketType) ReaderType { + if readType == metrics.ReadTypeRandom && bucketType.IsRapid() { + return MultiRangeReaderType + } + return RangeReaderType +} + +func (gr *GCSReader) getEndOffset(start int64) int64 { + end := start + gr.readTypeClassifier.ComputeSeqPrefetchWindowAndAdjustType() + if end > int64(gr.object.Size) { + end = int64(gr.object.Size) + } + return end +} + +func (gr *GCSReader) Destroy() { + gr.mu.Lock() + defer gr.mu.Unlock() + gr.rangeReader.destroy() + gr.mrr.destroy() +} + +func (gr *GCSReader) CheckInvariants() { + gr.rangeReader.checkInvariants() +} diff --git a/internal/gcsx/client_readers/gcs_reader_test.go b/internal/gcsx/client_readers/gcs_reader_test.go new file mode 100644 index 0000000000..817e23cb41 --- /dev/null +++ b/internal/gcsx/client_readers/gcs_reader_test.go @@ -0,0 +1,576 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client_readers + +import ( + "context" + "io" + "strings" + "sync" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + testUtil "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +const ( + sequentialReadSizeInMb = 22 +) + +//////////////////////////////////////////////////////////////////////// +// Helpers +//////////////////////////////////////////////////////////////////////// + +func (t *gcsReaderTest) readAt(ctx context.Context, req *gcsx.ReadRequest) gcsx.ReadResponse { + t.gcsReader.CheckInvariants() + defer t.gcsReader.CheckInvariants() + readInfo := t.gcsReader.readTypeClassifier.GetReadInfo(req.Offset, false) + resp, err := t.gcsReader.ReadAt(ctx, &gcsx.ReadRequest{ + Buffer: req.Buffer, + Offset: req.Offset, + ReadInfo: readInfo, + }) + require.NoError(t.T(), err) + t.gcsReader.readTypeClassifier.RecordRead(req.Offset, int64(resp.Size)) + return resp +} + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +type gcsReaderTest struct { + suite.Suite + ctx context.Context + object *gcs.MinObject + mockBucket *storage.TestifyMockBucket + gcsReader *GCSReader +} + +func TestGCSReaderTestSuite(t *testing.T) { + suite.Run(t, new(gcsReaderTest)) +} + +func (t *gcsReaderTest) SetupTest() { + t.object = &gcs.MinObject{ + Name: testObject, + Size: 17, + Generation: 1234, + } + t.mockBucket = new(storage.TestifyMockBucket) + t.gcsReader = NewGCSReader(t.object, t.mockBucket, &GCSReaderConfig{ + MetricHandle: metrics.NewNoopMetrics(), + TraceHandle: tracing.NewNoopTracer(), + MrdWrapper: nil, + Config: nil, + ReadTypeClassifier: gcsx.NewReadTypeClassifier(int64(sequentialReadSizeInMb), 0), + }) + t.ctx = context.Background() +} + +func (t *gcsReaderTest) TearDownTest() { + t.gcsReader.Destroy() +} + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func (t *gcsReaderTest) Test_NewGCSReader() { + object := &gcs.MinObject{ + Name: testObject, + Size: 30, + Generation: 4321, + } + + gcsReader := NewGCSReader(object, t.mockBucket, &GCSReaderConfig{ + MetricHandle: metrics.NewNoopMetrics(), + TraceHandle: tracing.NewNoopTracer(), + MrdWrapper: nil, + Config: nil, + ReadTypeClassifier: gcsx.NewReadTypeClassifier(sequentialReadSizeInMb, 0), + }) + + assert.Equal(t.T(), object, gcsReader.object) + assert.Equal(t.T(), t.mockBucket, gcsReader.bucket) + assert.True(t.T(), t.gcsReader.readTypeClassifier.IsReadSequential()) +} + +func (t *gcsReaderTest) Test_ReadAt_InvalidOffset() { + testCases := []struct { + name string + objectSize int + start int + }{ + { + name: "InvalidOffset", + objectSize: 50, + start: 68, + }, + { + name: "NegativeOffset", + objectSize: 100, + start: -50, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.gcsReader.rangeReader.reader = nil + t.object.Size = uint64(tc.objectSize) + buf := make([]byte, tc.objectSize) + + _, err := t.gcsReader.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: int64(tc.start), + }) + + assert.Error(t.T(), err) + }) + } +} + +func (t *gcsReaderTest) Test_ReadAt_ExistingReaderLimitIsLessThanRequestedDataSize() { + t.object.Size = 10 + // Simulate an existing reader. + t.gcsReader.rangeReader.reader = &fake.FakeReader{ReadCloser: getReadCloser([]byte("xxx")), Handle: []byte("fake")} + t.gcsReader.rangeReader.cancel = func() {} + t.gcsReader.rangeReader.start = 2 + t.gcsReader.rangeReader.limit = 5 + content := "verify" + rc := &fake.FakeReader{ReadCloser: getReadCloser([]byte(content))} + expectedHandleInRequest := []byte(t.gcsReader.rangeReader.reader.ReadHandle()) + readObjectRequest := &gcs.ReadObjectRequest{ + Name: t.gcsReader.rangeReader.object.Name, + Generation: t.gcsReader.rangeReader.object.Generation, + Range: &gcs.ByteRange{ + Start: 2, + Limit: t.object.Size, + }, + ReadCompressed: t.gcsReader.rangeReader.object.HasContentEncodingGzip(), + ReadHandle: expectedHandleInRequest, + } + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, readObjectRequest).Return(rc, nil) + t.mockBucket.On("BucketType", mock.Anything).Return(gcs.BucketType{}).Times(3) + requestSize := 6 + buf := make([]byte, requestSize) + + readResponse := t.readAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: 2, + }) + + assert.Equal(t.T(), rc, t.gcsReader.rangeReader.reader) + assert.Equal(t.T(), requestSize, readResponse.Size) + assert.Equal(t.T(), content, string(buf[:readResponse.Size])) + assert.Equal(t.T(), int64(2+requestSize), t.gcsReader.readTypeClassifier.NextExpectedOffset()) + assert.Equal(t.T(), expectedHandleInRequest, t.gcsReader.rangeReader.readHandle) +} + +func (t *gcsReaderTest) Test_ReadAt_ExistingReaderLimitIsLessThanRequestedObjectSize() { + t.object.Size = 5 + // Simulate an existing reader + t.gcsReader.rangeReader.reader = &fake.FakeReader{ReadCloser: getReadCloser([]byte("xxx")), Handle: []byte("fake")} + t.gcsReader.rangeReader.cancel = func() {} + t.gcsReader.rangeReader.start = 0 + t.gcsReader.rangeReader.limit = 3 + content := "abcde" + rc := &fake.FakeReader{ReadCloser: getReadCloser([]byte(content))} + expectedHandleInRequest := t.gcsReader.rangeReader.reader.ReadHandle() + readObjectRequest := &gcs.ReadObjectRequest{ + Name: t.gcsReader.rangeReader.object.Name, + Generation: t.gcsReader.rangeReader.object.Generation, + Range: &gcs.ByteRange{ + Start: uint64(0), + Limit: t.object.Size, + }, + ReadCompressed: t.gcsReader.rangeReader.object.HasContentEncodingGzip(), + ReadHandle: expectedHandleInRequest, + } + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, readObjectRequest).Return(rc, nil) + t.mockBucket.On("BucketType", mock.Anything).Return(gcs.BucketType{}).Times(3) + requestSize := 6 + buf := make([]byte, requestSize) + + readResponse := t.readAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: 0, + }) + + assert.Nil(t.T(), t.gcsReader.rangeReader.reader) + assert.Equal(t.T(), int(t.object.Size), readResponse.Size) + assert.Equal(t.T(), content, string(buf[:readResponse.Size])) + assert.Equal(t.T(), int64(t.object.Size), t.gcsReader.readTypeClassifier.NextExpectedOffset()) + assert.Equal(t.T(), []byte(nil), t.gcsReader.rangeReader.readHandle) +} + +func (t *gcsReaderTest) Test_ReadAt_ExistingReaderIsFine() { + t.object.Size = 6 + content := "xxx" + // Simulate an existing reader + t.gcsReader.rangeReader.reader = &fake.FakeReader{ReadCloser: getReadCloser([]byte(content)), Handle: []byte("fake")} + t.gcsReader.rangeReader.cancel = func() {} + t.gcsReader.rangeReader.start = 2 + t.gcsReader.rangeReader.limit = 5 + requestSize := 3 + t.mockBucket.On("BucketType", mock.Anything).Return(gcs.BucketType{}).Times(3) + buf := make([]byte, requestSize) + + readResponse := t.readAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: 2, + }) + + assert.Equal(t.T(), 3, readResponse.Size) + assert.Equal(t.T(), content, string(buf[:readResponse.Size])) + assert.Equal(t.T(), int64(5), t.gcsReader.readTypeClassifier.NextExpectedOffset()) + assert.Equal(t.T(), []byte("fake"), t.gcsReader.rangeReader.readHandle) +} + +func (t *gcsReaderTest) Test_ExistingReader_WrongOffset() { + testCases := []struct { + name string + readHandle []byte + }{ + { + name: "ReaderHasReadHandle", + readHandle: []byte("fake-handle"), + }, + { + name: "ReaderHasNoReadHandle", + readHandle: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.object.Size = 5 + // Simulate an existing reader. + t.gcsReader.rangeReader.readHandle = tc.readHandle + t.gcsReader.rangeReader.reader = &fake.FakeReader{ + ReadCloser: io.NopCloser(strings.NewReader("xxx")), + Handle: tc.readHandle, + } + t.gcsReader.rangeReader.cancel = func() {} + t.gcsReader.rangeReader.start = 2 + t.gcsReader.rangeReader.limit = 5 + content := "abcde" + rc := &fake.FakeReader{ReadCloser: getReadCloser([]byte(content))} + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, mock.Anything).Return(rc, nil).Times(1) + t.mockBucket.On("BucketType", mock.Anything).Return(gcs.BucketType{}).Times(2) + requestSize := 6 + buf := make([]byte, requestSize) + + readResponse := t.readAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: 0, + }) + + t.mockBucket.AssertExpectations(t.T()) + assert.Nil(t.T(), t.gcsReader.rangeReader.reader) + assert.Equal(t.T(), int(t.object.Size), readResponse.Size) + assert.Equal(t.T(), content, string(buf[:readResponse.Size])) + assert.Equal(t.T(), []byte(nil), t.gcsReader.rangeReader.readHandle) + }) + } +} + +func (t *gcsReaderTest) Test_ReadAt_PropagatesCancellation() { + t.gcsReader = NewGCSReader(t.object, t.mockBucket, &GCSReaderConfig{ + MetricHandle: metrics.NewNoopMetrics(), + TraceHandle: tracing.NewNoopTracer(), + MrdWrapper: nil, + Config: &cfg.Config{FileSystem: cfg.FileSystemConfig{IgnoreInterrupts: false}}, + ReadTypeClassifier: gcsx.NewReadTypeClassifier(sequentialReadSizeInMb, 0), + }) + // Set up a blocking reader + finishRead := make(chan struct{}) + blocking := &blockingReader{c: finishRead} + rc := io.NopCloser(blocking) + // Assign it to the rangeReader + t.gcsReader.rangeReader.reader = &fake.FakeReader{ReadCloser: rc} + t.gcsReader.rangeReader.start = 0 + t.gcsReader.rangeReader.limit = 2 + // Track cancel invocation + cancelCalled := make(chan struct{}) + t.gcsReader.rangeReader.cancel = func() { close(cancelCalled) } + // Controlled context + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + // Channel to track read completion + readReturned := make(chan struct{}) + var err error + t.mockBucket.On("BucketType", mock.Anything).Return(gcs.BucketType{}).Times(3) + req := &gcsx.ReadRequest{ + Buffer: make([]byte, 2), + Offset: 0, + } + go func() { + _, err = t.gcsReader.ReadAt(ctx, req) + + assert.Error(t.T(), err) + + close(readReturned) + }() + + // Wait a bit to ensure ReadAt is blocking + select { + case <-readReturned: + t.T().Fatal("Read returned early — cancellation did not propagate properly.") + case <-time.After(100 * time.Millisecond): + // OK: Still blocked + } + // Cancel the context to trigger cancellation + cancel() + // Expect rr.cancel to be called + select { + case <-cancelCalled: + // Pass + case <-time.After(100 * time.Millisecond): + t.T().Fatal("Expected rr.cancel to be called on ctx cancellation.") + } + // Unblock the reader so the read can complete + close(finishRead) + // Ensure read completes + select { + case <-readReturned: + // Pass + case <-time.After(100 * time.Millisecond): + t.T().Fatal("Expected read to return after unblocking.") + } +} + +// Validates: +// 1. No change in ReadAt behavior based inactiveStreamTimeout readConfig. +// 2. Valid timeout readConfig creates inactiveTimeoutReader instance of storage.Reader. +func (t *gcsReaderTest) Test_ReadAt_WithAndWithoutReadConfig() { + testCases := []struct { + name string + config *cfg.Config + expectInactiveTimeoutReader bool + }{ + { + name: "WithoutReadConfig", + config: nil, + expectInactiveTimeoutReader: false, + }, + { + name: "WithReadConfigAndZeroTimeout", + config: &cfg.Config{Read: cfg.ReadConfig{InactiveStreamTimeout: 0}}, + expectInactiveTimeoutReader: false, + }, + { + name: "WithReadConfigAndPositiveTimeout", + config: &cfg.Config{Read: cfg.ReadConfig{InactiveStreamTimeout: 10 * time.Millisecond}}, + expectInactiveTimeoutReader: true, + }, + } + + objectSize := uint64(20) + readOffset := int64(0) + readLength := 10 // Reading only 10 bytes from the complete object reader. + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.SetupTest() // Resets mockBucket, rr, etc. for each sub-test + defer t.TearDownTest() + + t.gcsReader.rangeReader.config = tc.config + t.gcsReader.rangeReader.reader = nil // Ensure startRead path is taken in ReadAt + t.object.Size = objectSize + // Prepare fake content for the GCS object. + // startRead will attempt to read the entire object [0, objectSize) + // because objectSize is small compared to typical SequentialReadSizeMb. + fakeReaderContent := testUtil.GenerateRandomBytes(int(t.object.Size)) + rc := &fake.FakeReader{ReadCloser: getReadCloser(fakeReaderContent)} + expectedReadObjectRequest := &gcs.ReadObjectRequest{ + Name: t.object.Name, + Generation: t.object.Generation, + Range: &gcs.ByteRange{ + Start: uint64(readOffset), // Read from the beginning + Limit: t.object.Size, // getReadInfo will determine this limit + }, + ReadCompressed: t.object.HasContentEncodingGzip(), + ReadHandle: nil, // No existing read handle + } + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, expectedReadObjectRequest).Return(rc, nil).Once() + t.mockBucket.On("BucketType", mock.Anything).Return(gcs.BucketType{Zonal: false}).Times(2) + buf := make([]byte, readLength) + + objectData := t.readAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: readOffset, + }) + + t.mockBucket.AssertExpectations(t.T()) + assert.Equal(t.T(), readLength, objectData.Size) + assert.Equal(t.T(), fakeReaderContent[:readLength], buf[:objectData.Size]) // Ensure buffer is populated correctly + assert.NotNil(t.T(), t.gcsReader.rangeReader.reader, "Reader should be active as partial data read from the requested range.") + assert.NotNil(t.T(), t.gcsReader.rangeReader.cancel) + assert.Equal(t.T(), int64(readLength), t.gcsReader.rangeReader.start) + assert.Equal(t.T(), int64(t.object.Size), t.gcsReader.rangeReader.limit) + _, isInactiveTimeoutReader := t.gcsReader.rangeReader.reader.(*gcsx.InactiveTimeoutReader) + assert.Equal(t.T(), tc.expectInactiveTimeoutReader, isInactiveTimeoutReader) + }) + } +} + +// This test validates the bug fix where seeks are not updated correctly in case of zonal bucket random reads (b/410904634). +func (t *gcsReaderTest) Test_ReadAt_ValidateZonalRandomReads() { + // Re-initialize GCSReader with initialOffset 13 MiB to force Random read type. + t.gcsReader = NewGCSReader(t.object, t.mockBucket, &GCSReaderConfig{ + MetricHandle: metrics.NewNoopMetrics(), + TraceHandle: tracing.NewNoopTracer(), + MrdWrapper: nil, + Config: nil, + ReadTypeClassifier: gcsx.NewReadTypeClassifier(int64(sequentialReadSizeInMb), 13*MiB), + }) + t.gcsReader.rangeReader.reader = nil + t.gcsReader.mrr.isMRDInUse.Store(false) + t.object.Size = 20 * MiB + t.mockBucket.On("BucketType", mock.Anything).Return(gcs.BucketType{Zonal: true}) + testContent := testUtil.GenerateRandomBytes(int(t.object.Size)) + fakeMRDWrapper, err := gcsx.NewMultiRangeDownloaderWrapper(t.mockBucket, t.object, &cfg.Config{}, nil) + assert.NoError(t.T(), err, "Error in creating MRDWrapper") + t.gcsReader.mrr.mrdWrapper = fakeMRDWrapper + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloader(t.object, testContent), nil).Once() + + readRanges := [][]int{{11 * MiB, 15 * MiB}, {12 * MiB, 14 * MiB}, {10 * MiB, 12 * MiB}, {9 * MiB, 11 * MiB}, {8 * MiB, 10 * MiB}} + // Series of random reads to check if seeks are updated correctly and MRD is invoked always + seeks := 0 + for _, readRange := range readRanges { + buf := make([]byte, readRange[1]-readRange[0]) + + t.readAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: int64(readRange[0]), + }) + + assert.Equal(t.T(), uint64(seeks), t.gcsReader.readTypeClassifier.GetSeeks()) + assert.False(t.T(), t.gcsReader.readTypeClassifier.IsReadSequential()) + assert.Equal(t.T(), int64(readRange[1]), t.gcsReader.readTypeClassifier.NextExpectedOffset()) + seeks++ + } +} + +func (t *gcsReaderTest) Test_ReadAt_ShortReadRetry() { + testCases := []struct { + name string + bucketType gcs.BucketType + }{ + { + name: "ZonalBucket", + bucketType: gcs.BucketType{Zonal: true}, + }, + { + name: "PirloBucket", + bucketType: gcs.BucketType{Pirlo: gcs.PirloStateRapidWritesEnabled}, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func() { + t.SetupTest() + defer t.TearDownTest() + // Re-initialize GCSReader with initialOffset 1 to force Random read type. + t.gcsReader = NewGCSReader(t.object, t.mockBucket, &GCSReaderConfig{ + MetricHandle: metrics.NewNoopMetrics(), + TraceHandle: tracing.NewNoopTracer(), + MrdWrapper: nil, + Config: nil, + ReadTypeClassifier: gcsx.NewReadTypeClassifier(int64(sequentialReadSizeInMb), 1), + }) + t.object.Size = 200 + t.mockBucket.On("BucketType", mock.Anything).Return(tc.bucketType) + testContent := testUtil.GenerateRandomBytes(int(t.object.Size)) + fakeMRDWrapper, err := gcsx.NewMultiRangeDownloaderWrapper(t.mockBucket, t.object, &cfg.Config{}, nil) + require.NoError(t.T(), err) + t.gcsReader.mrr.mrdWrapper = fakeMRDWrapper + buf := make([]byte, t.object.Size-1) + // Rapid buckets will use MRD and get a short read, then retry and get the full read. + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloaderWithShortRead(t.object, testContent), nil).Once() + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloader(t.object, testContent), nil).Once() + + readResponse := t.readAt(t.ctx, &gcsx.ReadRequest{Buffer: buf, Offset: 1}) + + assert.Equal(t.T(), int(t.object.Size)-1, readResponse.Size) + assert.Equal(t.T(), testContent[1:], buf) + assert.Equal(t.T(), int64(t.object.Size), t.gcsReader.readTypeClassifier.NextExpectedOffset()) + t.mockBucket.AssertExpectations(t.T()) + }) + } +} + +func (t *gcsReaderTest) Test_ReadAt_ParallelRandomReads() { + // Re-initialize GCSReader with initialOffset 1 to force Random read type. + t.gcsReader = NewGCSReader(t.object, t.mockBucket, &GCSReaderConfig{ + MetricHandle: metrics.NewNoopMetrics(), + TraceHandle: tracing.NewNoopTracer(), + MrdWrapper: nil, + Config: nil, + ReadTypeClassifier: gcsx.NewReadTypeClassifier(int64(sequentialReadSizeInMb), 1), + }) + + // Setup + t.object.Size = 20 * MiB + testContent := testUtil.GenerateRandomBytes(int(t.object.Size)) + // Mock bucket and MRD + t.mockBucket.On("BucketType", mock.Anything).Return(gcs.BucketType{Zonal: true}) + fakeMRDWrapper, err := gcsx.NewMultiRangeDownloaderWrapper(t.mockBucket, t.object, &cfg.Config{}, nil) + require.NoError(t.T(), err) + t.gcsReader.mrr.mrdWrapper = fakeMRDWrapper + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloader(t.object, testContent), nil) + + // Parallel reads + tasks := []struct { + offset int64 + size int + }{ + {1, 1 * MiB}, + {3 * MiB, 2 * MiB}, + {6 * MiB, 1 * MiB}, + {10 * MiB, 5 * MiB}, + } + + var wg sync.WaitGroup + + for _, task := range tasks { + wg.Add(1) + go func(offset int64, size int) { + defer wg.Done() + buf := make([]byte, size) + // Each goroutine gets its own context. + ctx := context.Background() + objData := t.readAt(ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: offset, + }) + + require.Equal(t.T(), size, objData.Size) + require.Equal(t.T(), testContent[offset:offset+int64(size)], buf) + }(task.offset, task.size) + } + wg.Wait() +} diff --git a/internal/gcsx/client_readers/multi_range_reader.go b/internal/gcsx/client_readers/multi_range_reader.go new file mode 100644 index 0000000000..661307a20e --- /dev/null +++ b/internal/gcsx/client_readers/multi_range_reader.go @@ -0,0 +1,109 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client_readers + +import ( + "context" + "fmt" + "io" + "sync/atomic" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" +) + +type MultiRangeReader struct { + gcsx.GCSReader + + object *gcs.MinObject + + // mrdWrapper points to the wrapper object within inode. + mrdWrapper *gcsx.MultiRangeDownloaderWrapper + + // boolean variable to determine if MRD is being used or not. + isMRDInUse atomic.Bool + + metricHandle metrics.MetricHandle + + traceHandle tracing.TraceHandle +} + +func NewMultiRangeReader(object *gcs.MinObject, metricHandle metrics.MetricHandle, traceHandle tracing.TraceHandle, mrdWrapper *gcsx.MultiRangeDownloaderWrapper) *MultiRangeReader { + if traceHandle == nil { + traceHandle = tracing.NewNoopTracer() + } + return &MultiRangeReader{ + object: object, + metricHandle: metricHandle, + traceHandle: traceHandle, + mrdWrapper: mrdWrapper, + } +} + +// readFromMultiRangeReader reads data from the underlying MultiRangeDownloaderWrapper. +// +// It increments the reference count of the mrdWrapper if it's not already in use. +// It then calls the Read method of the mrdWrapper with the provided parameters. +// +// Parameters: +// - ctx: The context for the read operation. It can be used to cancel the operation or set a timeout. +// - p: The byte slice to read data into. +// - offset: The starting offset for the read operation. +// - end: The ending offset for the read operation. +// - timeout: The maximum duration for the read operation. +// +// Returns: +// - int: The number of bytes read. +// - error: An error if the read operation fails. +func (mrd *MultiRangeReader) readFromMultiRangeReader(ctx context.Context, p []byte, offset, end int64, forceCreateMRD bool) (int, error) { + if mrd.mrdWrapper == nil { + return 0, fmt.Errorf("readFromMultiRangeReader: Invalid MultiRangeDownloaderWrapper") + } + + if mrd.isMRDInUse.CompareAndSwap(false, true) { + mrd.mrdWrapper.IncrementRefCount() + } + + return mrd.mrdWrapper.Read(ctx, p, offset, end, mrd.metricHandle, mrd.traceHandle, forceCreateMRD) +} + +func (mrd *MultiRangeReader) ReadAt(ctx context.Context, req *gcsx.GCSReaderRequest) (gcsx.ReadResponse, error) { + var ( + readResponse gcsx.ReadResponse + err error + ) + + if req.Offset >= int64(mrd.object.Size) && !req.SkipSizeChecks { + err = io.EOF + return readResponse, err + } + + readResponse.Size, err = mrd.readFromMultiRangeReader(ctx, req.Buffer, req.Offset, req.EndOffset, req.ForceCreateReader) + + return readResponse, err +} + +func (mrd *MultiRangeReader) destroy() { + if mrd.isMRDInUse.Load() { + err := mrd.mrdWrapper.DecrementRefCount() + if err != nil { + logger.Errorf("randomReader::Destroy:%v", err) + } + mrd.isMRDInUse.Store(false) + } +} diff --git a/internal/gcsx/client_readers/multi_range_reader_test.go b/internal/gcsx/client_readers/multi_range_reader_test.go new file mode 100644 index 0000000000..0f8c604e8f --- /dev/null +++ b/internal/gcsx/client_readers/multi_range_reader_test.go @@ -0,0 +1,231 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client_readers + +import ( + "context" + "errors" + "io" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + testUtil "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +const ( + TestTimeoutForMultiRangeRead = time.Second +) + +type multiRangeReaderTest struct { + suite.Suite + ctx context.Context + object *gcs.MinObject + mockBucket *storage.TestifyMockBucket + multiRangeReader *MultiRangeReader +} + +func (t *multiRangeReaderTest) readAt(dst []byte, offset int64, skipSizeChecks bool) (gcsx.ReadResponse, error) { + req := &gcsx.GCSReaderRequest{ + Offset: offset, + Buffer: dst, + EndOffset: offset + int64(len(dst)), + SkipSizeChecks: skipSizeChecks, + } + return t.multiRangeReader.ReadAt(t.ctx, req) +} + +func TestMultiRangeReaderTestSuite(t *testing.T) { + suite.Run(t, new(multiRangeReaderTest)) +} + +func (t *multiRangeReaderTest) SetupTest() { + t.mockBucket = new(storage.TestifyMockBucket) + t.ctx = context.Background() + t.object = &gcs.MinObject{ + Name: "testObject", + Size: 17, + Generation: 1234, + } + t.multiRangeReader = NewMultiRangeReader(t.object, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), nil) +} + +func (t *multiRangeReaderTest) TearDownTest() { + t.multiRangeReader.destroy() +} + +func (t *multiRangeReaderTest) Test_ReadFromMultiRangeReader_ReadFull() { + testCases := []struct { + name string + dataSize int + extraSize int + }{ + { + name: "ReadFull", + dataSize: 100, + extraSize: 0, + }, + { + name: "ReadWithLargerBuffer", + dataSize: 100, + extraSize: 10, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.multiRangeReader.isMRDInUse.Store(false) + t.object.Size = uint64(tc.dataSize) + testContent := testUtil.GenerateRandomBytes(int(t.object.Size)) + fakeMRDWrapper, err := gcsx.NewMultiRangeDownloaderWrapper(t.mockBucket, t.object, &cfg.Config{}, nil) + require.NoError(t.T(), err, "Error in creating MRDWrapper") + t.multiRangeReader.mrdWrapper = fakeMRDWrapper + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloaderWithSleep(t.object, testContent, time.Microsecond)).Times(1) + t.mockBucket.On("BucketType", mock.Anything).Return(gcs.BucketType{Zonal: true}).Times(1) + buf := make([]byte, tc.dataSize+tc.extraSize) + + bytesRead, err := t.multiRangeReader.readFromMultiRangeReader(t.ctx, buf, 0, int64(t.object.Size), false) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), tc.dataSize, bytesRead) + assert.Equal(t.T(), testContent[:tc.dataSize], buf[:bytesRead]) + }) + } +} + +func (t *multiRangeReaderTest) Test_ReadFromMultiRangeReader_NilMRDWrapper() { + t.multiRangeReader.mrdWrapper = nil + + bytesRead, err := t.multiRangeReader.readFromMultiRangeReader(t.ctx, make([]byte, t.object.Size), 0, int64(t.object.Size), false) + + assert.Error(t.T(), err) + assert.ErrorContains(t.T(), err, "readFromMultiRangeReader: Invalid MultiRangeDownloaderWrapper") + assert.Equal(t.T(), 0, bytesRead) +} + +func (t *multiRangeReaderTest) Test_ReadFromMultiRangeReader_ReadChunk() { + testCases := []struct { + name string + dataSize int + start int + end int + }{ + { + name: "ReadChunk", + dataSize: 100, + start: 37, + end: 93, + }, + } + + for _, tc := range testCases { + t.object.Size = uint64(tc.dataSize) + testContent := testUtil.GenerateRandomBytes(int(t.object.Size)) + fakeMRDWrapper, err := gcsx.NewMultiRangeDownloaderWrapper(t.mockBucket, t.object, &cfg.Config{}, nil) + require.NoError(t.T(), err, "Error in creating MRDWrapper") + t.multiRangeReader.mrdWrapper = fakeMRDWrapper + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloaderWithSleep(t.object, testContent, time.Microsecond)).Times(1) + t.mockBucket.On("BucketType", mock.Anything).Return(gcs.BucketType{Zonal: true}).Times(1) + buf := make([]byte, tc.end-tc.start) + + bytesRead, err := t.multiRangeReader.readFromMultiRangeReader(t.ctx, buf, int64(tc.start), int64(tc.end), false) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), tc.end-tc.start, bytesRead) + assert.Equal(t.T(), testContent[tc.start:tc.end], buf[:bytesRead]) + } +} + +func (t *multiRangeReaderTest) Test_ReadAt_MRDRead() { + testCases := []struct { + name string + dataSize int + offset int + bytesToRead int + }{ + { + name: "ReadChunk", + dataSize: 100, + offset: 37, + bytesToRead: 43, + }, + { + name: "ReadZeroByte", + dataSize: 100, + offset: 37, + bytesToRead: 0, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.multiRangeReader.isMRDInUse.Store(false) + t.object.Size = uint64(tc.dataSize) + testContent := testUtil.GenerateRandomBytes(int(t.object.Size)) + fakeMRDWrapper, err := gcsx.NewMultiRangeDownloaderWrapper(t.mockBucket, t.object, &cfg.Config{}, nil) + require.NoError(t.T(), err, "Error in creating MRDWrapper") + t.multiRangeReader.mrdWrapper = fakeMRDWrapper + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloaderWithSleep(t.object, testContent, time.Microsecond)).Times(1) + t.mockBucket.On("BucketType", mock.Anything).Return(gcs.BucketType{Zonal: true}).Times(1) + buf := make([]byte, tc.bytesToRead) + + readResponse, err := t.readAt(buf, int64(tc.offset), false) + + t.mockBucket.AssertNotCalled(t.T(), "NewReaderWithReadHandle", mock.Anything) + assert.NoError(t.T(), err) + assert.Equal(t.T(), tc.bytesToRead, readResponse.Size) + assert.Equal(t.T(), testContent[tc.offset:tc.offset+tc.bytesToRead], buf[:readResponse.Size]) + }) + } +} + +func (t *multiRangeReaderTest) Test_ReadAt_SkipSizeChecks() { + t.object.Size = 50 + testContent := testUtil.GenerateRandomBytes(int(t.object.Size)) + fakeMRDWrapper, err := gcsx.NewMultiRangeDownloaderWrapper(t.mockBucket, t.object, &cfg.Config{}, nil) + require.NoError(t.T(), err, "Error in creating MRDWrapper") + t.multiRangeReader.mrdWrapper = fakeMRDWrapper + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloaderWithSleep(t.object, testContent, time.Microsecond)).Once() + buf := make([]byte, 20) + offset := int64(40) + + // Read that starts within the object but extends beyond its size, with SkipSizeChecks=true. + // This should not return io.EOF from MultiRangeReader. + readResponse, err := t.readAt(buf, offset, true) + + assert.NoError(t.T(), err) + // The fake downloader will only return the remaining bytes from the object. + expectedBytesRead := int(t.object.Size) - int(offset) + assert.Equal(t.T(), expectedBytesRead, readResponse.Size) + assert.Equal(t.T(), testContent[offset:t.object.Size], buf[:readResponse.Size]) +} + +func (t *multiRangeReaderTest) Test_ReadAt_InvalidOffset() { + t.object.Size = 50 + + _, err := t.readAt(make([]byte, t.object.Size), 65, false) + + assert.True(t.T(), errors.Is(err, io.EOF), "expected %v error got %v", io.EOF, err) +} diff --git a/internal/gcsx/client_readers/range_reader.go b/internal/gcsx/client_readers/range_reader.go new file mode 100644 index 0000000000..60196cc7dd --- /dev/null +++ b/internal/gcsx/client_readers/range_reader.go @@ -0,0 +1,352 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client_readers + +import ( + "context" + "errors" + "fmt" + "io" + "math" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/gcsfuse_errors" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" +) + +const ( + MiB = 1 << 20 + + // Max read size in bytes for random reads. + // If the average read size (between seeks) is below this number, reads will optimise for random access. + // We will skip forwards in a GCS response at most this many bytes. + maxReadSize = 8 * MiB +) + +type RangeReader struct { + gcsx.GCSReader + object *gcs.MinObject + bucket gcs.Bucket + + // start is the current read offset of the reader. + start int64 + + // limit is the exclusive upper bound up to which the reader can read. + limit int64 + + // If non-nil, an in-flight read request and a function for cancelling it. + // + // INVARIANT: (reader == nil) == (cancel == nil) + reader gcs.StorageReader + + // Stores the handle associated with the previously closed newReader instance. + // This will be used while making the new connection to bypass auth and metadata + // checks. + readHandle []byte + cancel func() + + config *cfg.Config + metricHandle metrics.MetricHandle + traceHandle tracing.TraceHandle +} + +func NewRangeReader(object *gcs.MinObject, bucket gcs.Bucket, config *cfg.Config, metricHandle metrics.MetricHandle, traceHandle tracing.TraceHandle) *RangeReader { + if traceHandle == nil { + traceHandle = tracing.NewNoopTracer() + } + + return &RangeReader{ + object: object, + bucket: bucket, + metricHandle: metricHandle, + traceHandle: traceHandle, + config: config, + start: -1, + limit: -1, + } +} + +func (rr *RangeReader) checkInvariants() { + // INVARIANT: (reader == nil) == (cancel == nil) + if (rr.reader == nil) != (rr.cancel == nil) { + panic(fmt.Sprintf("Mismatch: %v vs. %v", rr.reader == nil, rr.cancel == nil)) + } + + // INVARIANT: start <= limit + if !(rr.start <= rr.limit) { + panic(fmt.Sprintf("Unexpected range: [%d, %d)", rr.start, rr.limit)) + } + + // INVARIANT: limit < 0 implies reader != nil + if rr.limit < 0 && rr.reader != nil { + panic(fmt.Sprintf("Unexpected non-nil reader with limit == %d", rr.limit)) + } +} + +func (rr *RangeReader) destroy() { + // Close out the reader, if we have one. + if rr.reader != nil { + rr.closeReader() + rr.reader = nil + rr.cancel = nil + } +} + +// closeReader fetches the readHandle before closing the reader instance. +func (rr *RangeReader) closeReader() { + rr.readHandle = rr.reader.ReadHandle() + err := rr.reader.Close() + if err != nil { + logger.Warnf("error while closing reader: %v", err) + } +} + +func (rr *RangeReader) ReadAt(ctx context.Context, req *gcsx.GCSReaderRequest) (gcsx.ReadResponse, error) { + var ( + readResponse gcsx.ReadResponse + err error + ) + + if req.ForceCreateReader && rr.reader != nil { + rr.closeReader() + rr.reader = nil + rr.cancel = nil + rr.start = -1 + rr.limit = -1 + } + + readResponse.Size, err = rr.readFromExistingReader(ctx, req) + if errors.Is(err, gcsx.FallbackToAnotherReader) { + readResponse.Size, err = rr.readFromRangeReader(ctx, req.Buffer, req.Offset, req.EndOffset, req.ReadType) + } + return readResponse, err +} + +// readFromRangeReader reads using the NewReader interface of go-sdk. It uses +// the existing reader if available, otherwise makes a call to GCS. +// Before calling this method we have to use invalidateReaderIfMisalignedOrTooSmall to get the reader start at the correct position. +func (rr *RangeReader) readFromRangeReader(ctx context.Context, p []byte, offset int64, end int64, readType int64) (int, error) { + var err error + // If we don't have a reader, start a read operation. + if rr.reader == nil { + err = rr.startRead(ctx, offset, end, readType) + if err != nil { + err = fmt.Errorf("startRead: %w", err) + return 0, err + } + } + + var n int + n, err = rr.readFull(ctx, p) + rr.start += int64(n) + + // Sanity check. + if rr.start > rr.limit { + err = fmt.Errorf("reader returned extra bytes: %d", rr.start-rr.limit) + + // Don't attempt to reuse the reader when it's malfunctioning. + rr.closeReader() + rr.reader = nil + rr.cancel = nil + rr.start = -1 + rr.limit = -1 + + return 0, err + } + + // Are we finished with this reader now? + if rr.start == rr.limit { + rr.closeReader() + rr.reader = nil + rr.cancel = nil + } + + // Handle errors. + switch { + case errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF): + // For a non-empty buffer, ReadFull returns EOF or ErrUnexpectedEOF only + // if the reader peters out early. That's fine, but it means we should + // have hit the limit above. + if rr.reader != nil { + err = fmt.Errorf("range reader returned early by skipping %d bytes: %w", rr.limit-rr.start, util.ErrShortRead) + return 0, err + } + + err = nil + + case err != nil: + // Propagate other errors. + err = fmt.Errorf("readFull: %w", err) + return 0, err + } + + return n, err +} + +// Like io.ReadFull, but deals with the cancellation issues. +// +// REQUIRES: rr.reader != nil +func (rr *RangeReader) readFull(ctx context.Context, p []byte) (int, error) { + if rr.config != nil && !rr.config.FileSystem.IgnoreInterrupts { + // Start a goroutine that will cancel the read operation we block on below if + // the calling context is cancelled, but only if this method has not already + // returned (to avoid souring the reader for the next read if this one is + // successful, since the calling context will eventually be cancelled). + readDone := make(chan struct{}) + defer close(readDone) + + go func() { + select { + case <-readDone: + return + + case <-ctx.Done(): + select { + case <-readDone: + return + + default: + rr.cancel() + } + } + }() + } + + return io.ReadFull(rr.reader, p) +} + +// Ensure that rr.reader is set up for a range for which [start, start+size) is +// a prefix. Irrespective of the size requested, we try to fetch more data +// from GCS defined by SequentialReadSizeMb flag to serve future read requests. +func (rr *RangeReader) startRead(ctx context.Context, start int64, end int64, readType int64) error { + ctx, cancel := context.WithCancel(rr.traceHandle.PropagateTraceContext(context.Background(), ctx)) + var err error + + if rr.config != nil && rr.config.Read.InactiveStreamTimeout > 0 { + rr.reader, err = gcsx.NewInactiveTimeoutReader( + ctx, + rr.bucket, + rr.object, + rr.readHandle, + gcs.ByteRange{ + Start: uint64(start), + Limit: uint64(end), + }, + rr.config.Read.InactiveStreamTimeout) + } else { + rr.reader, err = rr.bucket.NewReaderWithReadHandle( + ctx, + &gcs.ReadObjectRequest{ + Name: rr.object.Name, + Generation: rr.object.Generation, + Range: &gcs.ByteRange{ + Start: uint64(start), + Limit: uint64(end), + }, + ReadCompressed: rr.object.HasContentEncodingGzip(), + ReadHandle: rr.readHandle, + }) + } + + // If a file handle is open locally, but the corresponding object doesn't exist + // in GCS, it indicates a file clobbering scenario. This likely occurred because: + // - The file was deleted in GCS while a local handle was still open. + // - The file content was modified leading to different generation number. + var notFoundError *gcs.NotFoundError + if errors.As(err, ¬FoundError) { + err = &gcsfuse_errors.FileClobberedError{ + Err: fmt.Errorf("NewReader: %w", err), + ObjectName: rr.object.Name, + } + cancel() + return err + } + + if err != nil { + err = fmt.Errorf("NewReaderWithReadHandle: %w", err) + cancel() + return err + } + + rr.cancel = cancel + rr.start = start + rr.limit = end + + requestedDataSize := end - start + metrics.CaptureGCSReadMetrics(rr.metricHandle, metrics.ReadTypeNames[readType], requestedDataSize) + + return nil +} + +// skipBytes attempts to advance the reader position to the given offset without +// discarding the existing reader. If possible, it reads and discards data to +// maintain an active GCS connection, improving throughput for sequential reads. +func (rr *RangeReader) skipBytes(offset int64) { + // When the offset is AFTER the reader position, try to seek forward, within reason. + // This happens when the kernel page cache serves some data. It's very common for + // concurrent reads, often by only a few 128kB fuse read requests. The aim is to + // re-use GCS connection and avoid throwing away already read data. + // For parallel sequential reads to a single file, not throwing away the connections + // is a 15-20x improvement in throughput: 150-200 MiB/s instead of 10 MiB/s. + if rr.reader != nil && rr.start < offset && offset-rr.start < maxReadSize { + bytesToSkip := offset - rr.start + discardedBytes, copyError := io.CopyN(io.Discard, rr.reader, bytesToSkip) + // io.EOF is expected if the reader is shorter than the requested offset to read. + if copyError != nil && !errors.Is(copyError, io.EOF) { + logger.Warnf("Error while skipping reader bytes: %v", copyError) + } + rr.start += discardedBytes + } +} + +// invalidateReaderIfMisalignedOrTooSmall ensures that the existing reader is valid +// for the requested offset and length. If the reader is misaligned (not at the requested +// offset) or cannot serve the full request within its limit, it is closed and discarded. +// +// Parameters: +// - startOffset: the starting byte position of the requested read. +// - endOffset: the ending byte position of the requested read. +func (rr *RangeReader) invalidateReaderIfMisalignedOrTooSmall(startOffset, endOffset int64) { + // If we have an existing reader, but it's positioned at the wrong place, + // clean it up and throw it away. + // We will also clean up the existing reader if it can't serve the entire request. + dataToRead := math.Min(float64(endOffset), float64(rr.object.Size)) + if rr.reader != nil && (rr.start != startOffset || int64(dataToRead) > rr.limit) { + rr.closeReader() + rr.reader = nil + rr.cancel = nil + } +} + +// readFromExistingReader attempts to read data from an existing reader if one is available. +// If a reader exists and the read is successful, the data is returned. +// Otherwise, it returns an error indicating that a fallback to another reader is needed. +func (rr *RangeReader) readFromExistingReader(ctx context.Context, req *gcsx.GCSReaderRequest) (int, error) { + rr.skipBytes(req.Offset) + // Since we are reading from an existing reader, we only need to read what was requested. + endOffset := min(req.Offset + int64(len(req.Buffer))) + + rr.invalidateReaderIfMisalignedOrTooSmall(req.Offset, endOffset) + if rr.reader != nil { + return rr.readFromRangeReader(ctx, req.Buffer, req.Offset, endOffset, req.ReadType) + } + + return 0, gcsx.FallbackToAnotherReader +} diff --git a/internal/gcsx/client_readers/range_reader_test.go b/internal/gcsx/client_readers/range_reader_test.go new file mode 100644 index 0000000000..650471beea --- /dev/null +++ b/internal/gcsx/client_readers/range_reader_test.go @@ -0,0 +1,697 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client_readers + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "testing" + "testing/iotest" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/gcsfuse_errors" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + testUtil "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +const ( + fakeHandleData = "fake-handle" + testObject = "testObject" +) + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +type rangeReaderTest struct { + suite.Suite + ctx context.Context + object *gcs.MinObject + mockBucket *storage.TestifyMockBucket + rangeReader *RangeReader +} + +func TestRangeReaderTestSuite(t *testing.T) { + suite.Run(t, new(rangeReaderTest)) +} + +func (t *rangeReaderTest) SetupTest() { + t.object = &gcs.MinObject{ + Name: testObject, + Size: 17, + Generation: 1234, + } + t.mockBucket = new(storage.TestifyMockBucket) + t.rangeReader = NewRangeReader(t.object, t.mockBucket, nil, metrics.NewNoopMetrics(), tracing.NewNoopTracer()) + t.ctx = context.Background() +} + +func (t *rangeReaderTest) TearDownTest() { + t.rangeReader.destroy() +} + +//////////////////////////////////////////////////////////////////////// +// Helpers +//////////////////////////////////////////////////////////////////////// + +func getReadCloser(content []byte) io.ReadCloser { + r := bytes.NewReader(content) + rc := io.NopCloser(r) + return rc +} + +func getReader(size int) *fake.FakeReader { + testContent := testUtil.GenerateRandomBytes(size) + return &fake.FakeReader{ + ReadCloser: getReadCloser(testContent), + Handle: []byte(fakeHandleData), + } +} + +func (t *rangeReaderTest) readAt(dst []byte, offset int64) (gcsx.ReadResponse, error) { + req := &gcsx.GCSReaderRequest{ + Offset: offset, + Buffer: dst, + EndOffset: offset + int64(len(dst)), + ReadInfo: &gcsx.ReadInfo{}, + } + t.rangeReader.checkInvariants() + defer t.rangeReader.checkInvariants() + return t.rangeReader.ReadAt(t.ctx, req) +} + +func (t *rangeReaderTest) mockNewReaderWithHandleCallForTestBucket(start uint64, limit uint64, rd gcs.StorageReader) { + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(rg *gcs.ReadObjectRequest) bool { + return rg != nil && (*rg.Range).Start == start && (*rg.Range).Limit == limit + })).Return(rd, nil).Once() +} + +//////////////////////////////////////////////////////////////////////// +// Blocking reader +//////////////////////////////////////////////////////////////////////// + +// A reader that blocks until a channel is closed, then returns an error. +type blockingReader struct { + c chan struct{} +} + +func (br *blockingReader) Read([]byte) (int, error) { + <-br.c + return 0, errors.New("blockingReader") +} + +//////////////////////////////////////////////////////////////////////// +// Counting closer +//////////////////////////////////////////////////////////////////////// + +type countingCloser struct { + io.Reader + closeCount int +} + +func (cc *countingCloser) Close() (err error) { + cc.closeCount++ + return +} + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func (t *rangeReaderTest) Test_NewRangeReader() { + object := &gcs.MinObject{ + Name: testObject, + Size: 30, + Generation: 4321, + } + + reader := NewRangeReader(object, t.mockBucket, nil, metrics.NewNoopMetrics(), tracing.NewNoopTracer()) + + assert.Equal(t.T(), object, reader.object) + assert.Equal(t.T(), t.mockBucket, reader.bucket) + assert.Equal(t.T(), metrics.NewNoopMetrics(), reader.metricHandle) + assert.Equal(t.T(), int64(-1), reader.start) + assert.Equal(t.T(), int64(-1), reader.limit) +} + +func (t *rangeReaderTest) Test_CheckInvariants() { + tests := []struct { + name string + setup func() *RangeReader + shouldPanic bool + }{ + { + name: "valid no reader", + setup: func() *RangeReader { + return &RangeReader{ + start: 0, + limit: 10, + reader: nil, + cancel: nil, + } + }, + shouldPanic: false, + }, + { + name: "reader without cancel", + setup: func() *RangeReader { + t.rangeReader.reader = getReader(2) + return &RangeReader{ + start: 0, + limit: 10, + reader: t.rangeReader.reader, + cancel: nil, + } + }, + shouldPanic: true, + }, + { + name: "cancel without reader", + setup: func() *RangeReader { + return &RangeReader{ + start: 0, + limit: 10, + reader: nil, + cancel: func() {}, + } + }, + shouldPanic: true, + }, + { + name: "invalid range", + setup: func() *RangeReader { + return &RangeReader{ + start: 20, + limit: 10, + reader: nil, + cancel: nil, + } + }, + shouldPanic: true, + }, + { + name: "negative limit with valid reader", + setup: func() *RangeReader { + t.rangeReader.reader = getReader(2) + return &RangeReader{ + start: -10, + limit: -5, + reader: t.rangeReader.reader, + cancel: func() {}, + } + }, + shouldPanic: true, + }, + { + name: "negative limit with nil reader", + setup: func() *RangeReader { + return &RangeReader{ + start: -10, + limit: -5, + reader: nil, + cancel: nil, + } + }, + shouldPanic: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func() { + rr := tt.setup() + if tt.shouldPanic { + assert.Panics(t.T(), func() { rr.checkInvariants() }, "Expected panic") + } else { + assert.NotPanics(t.T(), func() { rr.checkInvariants() }, "Expected no panic") + } + }) + } +} + +func (t *rangeReaderTest) Test_Destroy_NonNilReader() { + t.rangeReader.reader = getReader(2) + + t.rangeReader.destroy() + + assert.Nil(t.T(), t.rangeReader.reader) + assert.Nil(t.T(), t.rangeReader.cancel) + assert.Equal(t.T(), []byte(fakeHandleData), t.rangeReader.readHandle) +} + +func (t *rangeReaderTest) Test_ReadAt_ReadFailsWithTimeoutError() { + content := "xxx" + r := iotest.OneByteReader(iotest.TimeoutReader(strings.NewReader(content))) + rc := &fake.FakeReader{ReadCloser: io.NopCloser(r)} + t.mockNewReaderWithHandleCallForTestBucket(0, uint64(len(content)), rc) + buf := make([]byte, len(content)) + + readResponse, err := t.readAt(buf, 0) + + assert.Error(t.T(), err) + assert.Contains(t.T(), err.Error(), "timeout") + assert.Zero(t.T(), readResponse.Size) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *rangeReaderTest) Test_ReadAt_SuccessfulRead() { + offset := int64(0) + size := int64(5) + content := []byte("hello world") + r := &fake.FakeReader{ReadCloser: getReadCloser(content)} + t.mockNewReaderWithHandleCallForTestBucket(uint64(offset), uint64(offset+size), r) + buf := make([]byte, size) + + resp, err := t.readAt(buf, offset) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), int(size), resp.Size) + assert.Equal(t.T(), content[:size], buf[:resp.Size]) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *rangeReaderTest) Test_ReadAt_PartialReadWithEOF() { + offset := int64(0) + size := int64(10) // Shorter than requested + partialReader := io.NopCloser(iotest.ErrReader(io.EOF)) // Simulates early EOF + r := &fake.FakeReader{ReadCloser: partialReader} + t.mockNewReaderWithHandleCallForTestBucket(uint64(offset), uint64(offset+size), r) + buf := make([]byte, size) + + resp, err := t.readAt(buf, offset) + + assert.Error(t.T(), err) + assert.Contains(t.T(), err.Error(), "reader returned early by skipping") + assert.Zero(t.T(), resp.Size) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *rangeReaderTest) Test_ReadAt_StartReadNotFound() { + offset := int64(0) + size := int64(5) + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, mock.Anything).Return(nil, &gcs.NotFoundError{}).Once() + + resp, err := t.readAt(make([]byte, size), offset) + + var fcErr *gcsfuse_errors.FileClobberedError + assert.ErrorAs(t.T(), err, &fcErr) + assert.Zero(t.T(), resp.Size) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *rangeReaderTest) Test_ReadAt_StartReadUnexpectedError() { + offset := int64(0) + size := int64(5) + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, mock.Anything).Return(nil, errors.New("network error")).Once() + + resp, err := t.readAt(make([]byte, size), offset) + + assert.Error(t.T(), err) + assert.Zero(t.T(), resp.Size) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *rangeReaderTest) Test_invalidateReaderIfMisalignedOrTooSmall() { + tests := []struct { + name string + readerSetup func() + offset int64 + bufferSize int + expectIncreaseSeek bool + expectReaderNil bool + }{ + { + name: "InvalidateReaderDueToTooSmall", + readerSetup: func() { + t.rangeReader.reader = &fake.FakeReader{ReadCloser: getReader(100)} + t.rangeReader.start = 200 + t.rangeReader.limit = 250 // too small + t.rangeReader.object.Size = 500 + }, + offset: 200, + bufferSize: 100, + expectIncreaseSeek: false, + expectReaderNil: true, + }, + { + name: "KeepReaderIfValid", + readerSetup: func() { + t.rangeReader.reader = &fake.FakeReader{ReadCloser: getReader(100)} + t.rangeReader.start = 200 + t.rangeReader.limit = 400 + }, + offset: 200, + bufferSize: 100, + expectIncreaseSeek: false, + expectReaderNil: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func() { + tt.readerSetup() + + t.rangeReader.invalidateReaderIfMisalignedOrTooSmall(tt.offset, tt.offset+int64(tt.bufferSize)) + + if tt.expectReaderNil { + assert.Nil(t.T(), t.rangeReader.reader, "rangeReader.reader should be nil") + } else { + assert.NotNil(t.T(), t.rangeReader.reader, "rangeReader.reader should not be nil") + } + }) + } +} + +func (t *rangeReaderTest) Test_ReadFromRangeReader_WhenReaderReturnedMoreData() { + testCases := []struct { + name string + readHandle []byte + }{ + { + name: "GCSReturnedReadHandle", + readHandle: []byte(fakeHandleData), + }, + { + name: "GCSReturnedNoReadHandle", + readHandle: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.rangeReader.start = 0 + t.rangeReader.limit = 6 + testContent := testUtil.GenerateRandomBytes(8) + t.rangeReader.reader = &fake.FakeReader{ + ReadCloser: getReadCloser(testContent), + Handle: tc.readHandle, + } + t.rangeReader.cancel = func() {} + + n, err := t.rangeReader.readFromRangeReader(t.ctx, make([]byte, 10), 0, 10, metrics.ReadTypeUnknown) + + assert.Error(t.T(), err) + assert.Zero(t.T(), n) + assert.Nil(t.T(), t.rangeReader.reader) + assert.Equal(t.T(), int64(-1), t.rangeReader.start) + assert.Equal(t.T(), int64(-1), t.rangeReader.limit) + assert.Equal(t.T(), tc.readHandle, t.rangeReader.readHandle) + }) + } +} + +func (t *rangeReaderTest) Test_ReadAt_PropagatesCancellation() { + t.rangeReader = NewRangeReader(t.object, t.mockBucket, &cfg.Config{FileSystem: cfg.FileSystemConfig{IgnoreInterrupts: false}}, metrics.NewNoopMetrics(), tracing.NewNoopTracer()) + // Set up a blocking reader + finishRead := make(chan struct{}) + blocking := &blockingReader{c: finishRead} + rc := io.NopCloser(blocking) + // Assign it to the rangeReader + t.rangeReader.reader = &fake.FakeReader{ReadCloser: rc} + t.rangeReader.start = 0 + t.rangeReader.limit = 2 + // Track cancel invocation + cancelCalled := make(chan struct{}) + t.rangeReader.cancel = func() { close(cancelCalled) } + // Controlled context + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + // Channel to track read completion + readReturned := make(chan struct{}) + + go func() { + _, _ = t.rangeReader.ReadAt(ctx, &gcsx.GCSReaderRequest{ + Buffer: make([]byte, 2), + Offset: 0, + EndOffset: 2, + ReadInfo: &gcsx.ReadInfo{}, + }) + close(readReturned) + }() + + // Wait a bit to ensure ReadAt is blocking + select { + case <-readReturned: + t.T().Fatal("Read returned early — cancellation did not propagate properly.") + case <-time.After(10 * time.Millisecond): + // OK: Still blocked + } + // Cancel the context to trigger cancellation + cancel() + // Expect rr.cancel to be called + select { + case <-cancelCalled: + // Pass + case <-time.After(100 * time.Millisecond): + t.T().Fatal("Expected rr.cancel to be called on ctx cancellation.") + } + // Unblock the reader so the read can complete + close(finishRead) + // Ensure read completes + select { + case <-readReturned: + // Pass + case <-time.After(100 * time.Millisecond): + t.T().Fatal("Expected read to return after unblocking.") + } +} + +func (t *rangeReaderTest) Test_ReadAt_DoesntPropagateCancellationAfterReturning() { + // Set up a reader that will return three bytes. + content := "xyz" + t.rangeReader.reader = &fake.FakeReader{ReadCloser: getReadCloser([]byte(content))} + t.rangeReader.start = 0 + t.rangeReader.limit = 3 + // Snoop on when cancel is called. + cancelCalled := make(chan struct{}) + t.rangeReader.cancel = func() { close(cancelCalled) } + ctx, cancel := context.WithCancel(context.Background()) + bufSize := 2 + buf := make([]byte, bufSize) + + // Successfully read two bytes using a context whose cancellation we control. + readResponse, err := t.rangeReader.ReadAt(ctx, &gcsx.GCSReaderRequest{ + Buffer: buf, + Offset: 0, + ReadInfo: &gcsx.ReadInfo{}, + EndOffset: 2, + }) + + assert.Nil(t.T(), err) + assert.Equal(t.T(), bufSize, readResponse.Size) + assert.Equal(t.T(), content[:bufSize], string(buf[:readResponse.Size])) + // If we cancel the calling context now, it should not cause the underlying + // read context to be cancelled. + cancel() + select { + case <-time.After(10 * time.Millisecond): + case <-cancelCalled: + t.T().Fatal("Read context unexpectedly cancelled") + } +} + +func (t *rangeReaderTest) Test_ReadFromRangeReader_WhenAllDataFromReaderIsRead() { + testCases := []struct { + name string + readHandle []byte + }{ + { + name: "GCSReturnedReadHandle", + readHandle: []byte(fakeHandleData), + }, + { + name: "GCSReturnedNoReadHandle", + readHandle: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.rangeReader.start = 4 + t.rangeReader.limit = 10 + t.object.Size = 10 + dataSize := 6 + testContent := testUtil.GenerateRandomBytes(int(t.object.Size)) + rc := &fake.FakeReader{ + ReadCloser: getReadCloser(testContent), + Handle: tc.readHandle, + } + t.rangeReader.reader = rc + t.rangeReader.cancel = func() {} + buf := make([]byte, dataSize) + + n, err := t.rangeReader.readFromRangeReader(t.ctx, buf, 4, 10, metrics.ReadTypeUnknown) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), dataSize, n) + // Verify the reader state. + assert.Nil(t.T(), t.rangeReader.reader) + assert.Nil(t.T(), t.rangeReader.cancel) + assert.Equal(t.T(), int64(10), t.rangeReader.start) + assert.Equal(t.T(), int64(10), t.rangeReader.limit) + assert.Equal(t.T(), tc.readHandle, t.rangeReader.readHandle) + }) + } +} + +func (t *rangeReaderTest) Test_ReadFromRangeReader_WhenReaderHasLessDataThanRequested() { + testCases := []struct { + name string + readHandle []byte + }{ + { + name: "GCSReturnedReadHandle", + readHandle: []byte(fakeHandleData), + }, + { + name: "GCSReturnedNoReadHandle", + readHandle: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.rangeReader.start = 0 + t.rangeReader.limit = 6 + dataSize := 6 + testContent := testUtil.GenerateRandomBytes(dataSize) + rc := &fake.FakeReader{ + ReadCloser: getReadCloser(testContent), + Handle: tc.readHandle, + } + t.rangeReader.reader = rc + t.rangeReader.cancel = func() {} + buf := make([]byte, 10) + + n, err := t.rangeReader.readFromRangeReader(t.ctx, buf, 0, 10, metrics.ReadTypeUnknown) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), dataSize, n) + // Verify the reader state. + assert.Nil(t.T(), t.rangeReader.reader) + assert.Nil(t.T(), t.rangeReader.cancel) + assert.Equal(t.T(), int64(dataSize), t.rangeReader.start) + assert.Equal(t.T(), int64(dataSize), t.rangeReader.limit) + assert.Equal(t.T(), tc.readHandle, t.rangeReader.readHandle) + }) + } +} + +func (t *rangeReaderTest) Test_ReadAt_ReaderNotExhausted() { + // Set up a reader that has three bytes left to give. + content := "abc" + cc := &countingCloser{ + Reader: strings.NewReader(content), + } + rc := &fake.FakeReader{ReadCloser: cc} + t.rangeReader.reader = rc + var offset int64 = 1 + t.rangeReader.start = offset + t.rangeReader.limit = 4 + t.rangeReader.cancel = func() {} + var bufSize int64 = 2 + buf := make([]byte, bufSize) + + // Read two bytes. + resp, err := t.readAt(buf, offset) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), content[:bufSize], string(buf[:resp.Size])) + assert.Zero(t.T(), cc.closeCount) + assert.Equal(t.T(), rc, t.rangeReader.reader) + assert.Equal(t.T(), offset+bufSize, t.rangeReader.start) +} + +func (t *rangeReaderTest) Test_ReadAt_ShortRead() { + offset := int64(0) + size := int64(10) + // Create a reader that will return less data than requested + shortContent := []byte("hello") + r := &fake.FakeReader{ReadCloser: getReadCloser(shortContent)} + t.mockNewReaderWithHandleCallForTestBucket(uint64(offset), uint64(offset+size), r) + buf := make([]byte, size) + + resp, err := t.readAt(buf, offset) + + assert.Error(t.T(), err) + assert.Contains(t.T(), err.Error(), "reader returned early by skipping 5 bytes: short read") + assert.ErrorIs(t.T(), err, util.ErrShortRead) + assert.Zero(t.T(), resp.Size) + t.mockBucket.AssertExpectations(t.T()) +} + +// Write a unit test to force recreate a reader and verify that the reader was force created and read was successful +func (t *rangeReaderTest) Test_ReadAt_ForceCreateReader() { + offset := int64(0) + size := int64(10) + readSize := int64(3) + content1 := []byte("first-content") + content2 := []byte("second-content") + + // 1. First reader + r1 := &fake.FakeReader{ReadCloser: getReadCloser(content1)} + t.mockNewReaderWithHandleCallForTestBucket(uint64(offset), uint64(offset+size), r1) + + // 2. Read with forceCreateReader = false (default) + req1 := &gcsx.GCSReaderRequest{ + Offset: offset, + Buffer: make([]byte, readSize), + EndOffset: offset + size, + ReadInfo: &gcsx.ReadInfo{}, + } + resp1, err := t.rangeReader.ReadAt(t.ctx, req1) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), int(readSize), resp1.Size) + assert.Equal(t.T(), content1[:readSize], req1.Buffer) + assert.NotNil(t.T(), t.rangeReader.reader) // Reader should be active + firstReader := t.rangeReader.reader + + // 3. Second reader (will be created due to forceCreateReader = true) + r2 := &fake.FakeReader{ReadCloser: getReadCloser(content2)} + t.mockNewReaderWithHandleCallForTestBucket(uint64(offset+readSize), uint64(offset+size), r2) + readsize2 := int64(4) + + // 4. Read with forceCreateReader = true. The existing reader can serve this + // request, but it will be discarded because ForceCreateReader is true. + req2 := &gcsx.GCSReaderRequest{ + Offset: offset + readSize, + Buffer: make([]byte, readsize2), + EndOffset: offset + size, + ForceCreateReader: true, + ReadInfo: &gcsx.ReadInfo{}, + } + resp2, err := t.rangeReader.ReadAt(t.ctx, req2) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), int(readsize2), resp2.Size) + assert.Equal(t.T(), content2[:readsize2], req2.Buffer) + assert.NotNil(t.T(), t.rangeReader.reader) // New reader should not be nil + secondReader := t.rangeReader.reader + assert.NotEqual(t.T(), firstReader, secondReader) + t.mockBucket.AssertExpectations(t.T()) +} diff --git a/internal/gcsx/append_object_creator.go b/internal/gcsx/compose_object_creator.go similarity index 85% rename from internal/gcsx/append_object_creator.go rename to internal/gcsx/compose_object_creator.go index b82216de5d..58e4d3a6c4 100644 --- a/internal/gcsx/append_object_creator.go +++ b/internal/gcsx/compose_object_creator.go @@ -19,9 +19,10 @@ import ( "errors" "fmt" "io" + "maps" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "golang.org/x/net/context" ) @@ -34,10 +35,10 @@ import ( // // Create guarantees to return *gcs.PreconditionError when the source object // has been clobbered. -func newAppendObjectCreator( +func newComposeObjectCreator( prefix string, bucket gcs.Bucket) (oc objectCreator) { - oc = &appendObjectCreator{ + oc = &composeObjectCreator{ prefix: prefix, bucket: bucket, } @@ -49,12 +50,12 @@ func newAppendObjectCreator( // Implementation //////////////////////////////////////////////////////////////////////// -type appendObjectCreator struct { +type composeObjectCreator struct { prefix string bucket gcs.Bucket } -func (oc *appendObjectCreator) chooseName() (name string, err error) { +func (oc *composeObjectCreator) chooseName() (name string, err error) { // Generate a good 64-bit random number. var buf [8]byte _, err = io.ReadFull(rand.Reader, buf[:]) @@ -79,13 +80,15 @@ func (oc *appendObjectCreator) chooseName() (name string, err error) { } // ObjectName param is present here for consistency between fullObjectCreator -// and appendObjectCreator. ObjectName is not used in append flow since +// and composeObjectCreator. ObjectName is not used in append flow since // srcObject.Name gives the objectName. -func (oc *appendObjectCreator) Create( +func (oc *composeObjectCreator) Create( ctx context.Context, objectName string, srcObject *gcs.Object, mtime *time.Time, + chunkRetryDeadlineSecs int64, + chunkTransferTimeoutSecs int64, r io.Reader) (o *gcs.Object, err error) { // Choose a name for a temporary object. tmpName, err := oc.chooseName() @@ -95,14 +98,9 @@ func (oc *appendObjectCreator) Create( } // Create a temporary object containing the additional contents. - var zero int64 - tmp, err := oc.bucket.CreateObject( - ctx, - &gcs.CreateObjectRequest{ - Name: tmpName, - GenerationPrecondition: &zero, - Contents: r, - }) + req := gcs.NewCreateObjectRequest(nil, tmpName, nil, chunkRetryDeadlineSecs, chunkTransferTimeoutSecs) + req.Contents = r + tmp, err := oc.bucket.CreateObject(ctx, req) if err != nil { err = fmt.Errorf("CreateObject: %w", err) return @@ -125,12 +123,10 @@ func (oc *appendObjectCreator) Create( MetadataMap := make(map[string]string) /* Copy Metadata fields from src object to new object generated by compose. */ - for key, value := range srcObject.Metadata { - MetadataMap[key] = value - } + maps.Copy(MetadataMap, srcObject.Metadata) if mtime != nil { - MetadataMap[MtimeMetadataKey] = mtime.UTC().Format(time.RFC3339Nano) + MetadataMap[gcs.MtimeMetadataKey] = mtime.UTC().Format(time.RFC3339Nano) } // Compose the old contents plus the new over the old. diff --git a/internal/gcsx/append_object_creator_test.go b/internal/gcsx/compose_object_creator_test.go similarity index 88% rename from internal/gcsx/append_object_creator_test.go rename to internal/gcsx/compose_object_creator_test.go index 4d03e7c6ec..e81dd30721 100644 --- a/internal/gcsx/append_object_creator_test.go +++ b/internal/gcsx/compose_object_creator_test.go @@ -22,15 +22,15 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/oglemock" . "github.com/jacobsa/ogletest" "golang.org/x/net/context" ) -func TestAppendObjectCreator(t *testing.T) { RunTests(t) } +func TestComposeObjectCreator(t *testing.T) { RunTests(t) } //////////////////////////////////////////////////////////////////////// // Helpers @@ -38,7 +38,7 @@ func TestAppendObjectCreator(t *testing.T) { RunTests(t) } func deleteReqName(expected string) (m Matcher) { m = NewMatcher( - func(c interface{}) (err error) { + func(c any) (err error) { req, ok := c.(*gcs.DeleteObjectRequest) if !ok { err = fmt.Errorf("which has type %T", c) @@ -63,7 +63,7 @@ func deleteReqName(expected string) (m Matcher) { const prefix = ".gcsfuse_tmp/" -type AppendObjectCreatorTest struct { +type ComposeObjectCreatorTest struct { ctx context.Context bucket storage.MockBucket creator objectCreator @@ -73,26 +73,28 @@ type AppendObjectCreatorTest struct { mtime time.Time } -var _ SetUpInterface = &AppendObjectCreatorTest{} +var _ SetUpInterface = &ComposeObjectCreatorTest{} -func init() { RegisterTestSuite(&AppendObjectCreatorTest{}) } +func init() { RegisterTestSuite(&ComposeObjectCreatorTest{}) } -func (t *AppendObjectCreatorTest) SetUp(ti *TestInfo) { +func (t *ComposeObjectCreatorTest) SetUp(ti *TestInfo) { t.ctx = ti.Ctx // Create the bucket. t.bucket = storage.NewMockBucket(ti.MockController, "bucket") // Create the creator. - t.creator = newAppendObjectCreator(prefix, t.bucket) + t.creator = newComposeObjectCreator(prefix, t.bucket) } -func (t *AppendObjectCreatorTest) call() (o *gcs.Object, err error) { +func (t *ComposeObjectCreatorTest) call() (o *gcs.Object, err error) { o, err = t.creator.Create( t.ctx, t.srcObject.Name, &t.srcObject, &t.mtime, + chunkRetryDeadlineSecs, + chunkTransferTimeoutSecs, strings.NewReader(t.srcContents)) return @@ -102,7 +104,7 @@ func (t *AppendObjectCreatorTest) call() (o *gcs.Object, err error) { // Tests //////////////////////////////////////////////////////////////////////// -func (t *AppendObjectCreatorTest) CallsCreateObject() { +func (t *ComposeObjectCreatorTest) CallsCreateObject() { t.srcContents = "taco" // CreateObject @@ -123,7 +125,7 @@ func (t *AppendObjectCreatorTest) CallsCreateObject() { ExpectEq(t.srcContents, string(b)) } -func (t *AppendObjectCreatorTest) CreateObjectFails() { +func (t *ComposeObjectCreatorTest) CreateObjectFails() { var err error // CreateObject @@ -137,7 +139,7 @@ func (t *AppendObjectCreatorTest) CreateObjectFails() { ExpectThat(err, Error(HasSubstr("taco"))) } -func (t *AppendObjectCreatorTest) CreateObjectReturnsPreconditionError() { +func (t *ComposeObjectCreatorTest) CreateObjectReturnsPreconditionError() { var err error // CreateObject @@ -153,7 +155,7 @@ func (t *AppendObjectCreatorTest) CreateObjectReturnsPreconditionError() { ExpectThat(err, Error(HasSubstr("taco"))) } -func (t *AppendObjectCreatorTest) CallsComposeObjects() { +func (t *ComposeObjectCreatorTest) CallsComposeObjects() { t.srcObject.Name = "foo" t.srcObject.Generation = 17 t.srcObject.MetaGeneration = 23 @@ -205,7 +207,7 @@ func (t *AppendObjectCreatorTest) CallsComposeObjects() { ExpectEq(tmpObject.Generation, src.Generation) } -func (t *AppendObjectCreatorTest) CallsComposeObjectsWithObjectProperties() { +func (t *ComposeObjectCreatorTest) CallsComposeObjectsWithObjectProperties() { t.srcObject.Name = "foo" t.srcObject.Generation = 17 t.srcObject.MetaGeneration = 23 @@ -273,7 +275,7 @@ func (t *AppendObjectCreatorTest) CallsComposeObjectsWithObjectProperties() { ExpectEq(tmpObject.Generation, src.Generation) } -func (t *AppendObjectCreatorTest) ComposeObjectsFails() { +func (t *ComposeObjectCreatorTest) ComposeObjectsFails() { // CreateObject tmpObject := &gcs.Object{ Name: "bar", @@ -297,7 +299,7 @@ func (t *AppendObjectCreatorTest) ComposeObjectsFails() { ExpectThat(err, Error(HasSubstr("taco"))) } -func (t *AppendObjectCreatorTest) ComposeObjectsReturnsPreconditionError() { +func (t *ComposeObjectCreatorTest) ComposeObjectsReturnsPreconditionError() { // CreateObject tmpObject := &gcs.Object{ Name: "bar", @@ -323,7 +325,7 @@ func (t *AppendObjectCreatorTest) ComposeObjectsReturnsPreconditionError() { ExpectThat(err, Error(HasSubstr("taco"))) } -func (t *AppendObjectCreatorTest) ComposeObjectsReturnsNotFoundError() { +func (t *ComposeObjectCreatorTest) ComposeObjectsReturnsNotFoundError() { // CreateObject tmpObject := &gcs.Object{ Name: "bar", @@ -349,7 +351,7 @@ func (t *AppendObjectCreatorTest) ComposeObjectsReturnsNotFoundError() { ExpectThat(err, Error(HasSubstr("taco"))) } -func (t *AppendObjectCreatorTest) CallsDeleteObject() { +func (t *ComposeObjectCreatorTest) CallsDeleteObject() { // CreateObject tmpObject := &gcs.Object{ Name: "bar", @@ -371,7 +373,7 @@ func (t *AppendObjectCreatorTest) CallsDeleteObject() { t.call() } -func (t *AppendObjectCreatorTest) DeleteObjectFails() { +func (t *ComposeObjectCreatorTest) DeleteObjectFails() { // CreateObject tmpObject := &gcs.Object{ Name: "bar", @@ -396,7 +398,7 @@ func (t *AppendObjectCreatorTest) DeleteObjectFails() { ExpectThat(err, Error(HasSubstr("taco"))) } -func (t *AppendObjectCreatorTest) DeleteObjectSucceeds() { +func (t *ComposeObjectCreatorTest) DeleteObjectSucceeds() { // CreateObject tmpObject := &gcs.Object{ Name: "bar", diff --git a/internal/gcsx/content_type_bucket.go b/internal/gcsx/content_type_bucket.go index 093066c218..dc88e41623 100644 --- a/internal/gcsx/content_type_bucket.go +++ b/internal/gcsx/content_type_bucket.go @@ -18,7 +18,7 @@ import ( "mime" "path" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "golang.org/x/net/context" ) @@ -67,3 +67,13 @@ func (b contentTypeBucket) CreateObjectChunkWriter(ctx context.Context, req *gcs // Pass on the request. return b.Bucket.CreateObjectChunkWriter(ctx, req, chunkSize, callBack) } + +func (b contentTypeBucket) CreateAppendableObjectWriter(ctx context.Context, req *gcs.CreateObjectChunkWriterRequest) (gcs.Writer, error) { + // Guess a content type if necessary. + if req.ContentType == "" { + req.ContentType = mime.TypeByExtension(path.Ext(req.Name)) + } + + // Pass on the request. + return b.Bucket.CreateAppendableObjectWriter(ctx, req) +} diff --git a/internal/gcsx/content_type_bucket_test.go b/internal/gcsx/content_type_bucket_test.go index c8a845beb5..d901eabdeb 100644 --- a/internal/gcsx/content_type_bucket_test.go +++ b/internal/gcsx/content_type_bucket_test.go @@ -18,9 +18,9 @@ import ( "strings" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/fake" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "github.com/jacobsa/timeutil" "golang.org/x/net/context" ) @@ -83,7 +83,7 @@ func TestContentTypeBucket_CreateObject(t *testing.T) { for i, tc := range contentTypeBucketTestCases { // Set up a bucket. bucket := gcsx.NewContentTypeBucket( - fake.NewFakeBucket(timeutil.RealClock(), "", gcs.NonHierarchical)) + fake.NewFakeBucket(timeutil.RealClock(), "", gcs.BucketType{})) // Create the object. req := &gcs.CreateObjectRequest{ @@ -108,7 +108,7 @@ func TestContentTypeBucket_CreateObjectChunkWriter(t *testing.T) { for i, tc := range contentTypeBucketTestCases { // Set up a bucket. bucket := gcsx.NewContentTypeBucket( - fake.NewFakeBucket(timeutil.RealClock(), "", gcs.NonHierarchical)) + fake.NewFakeBucket(timeutil.RealClock(), "", gcs.BucketType{})) // Create the object. req := &gcs.CreateObjectRequest{ @@ -129,6 +129,35 @@ func TestContentTypeBucket_CreateObjectChunkWriter(t *testing.T) { } } +func TestContentTypeBucket_CreateAppendableObjectWriter(t *testing.T) { + for i, tc := range contentTypeBucketTestCases { + // Set up a bucket. + bucket := gcsx.NewContentTypeBucket( + fake.NewFakeBucket(timeutil.RealClock(), "", gcs.BucketType{})) + + // Create the object writer request. + req := &gcs.CreateObjectChunkWriterRequest{ + CreateObjectRequest: gcs.CreateObjectRequest{ + Name: tc.name, + ContentType: tc.request, + }, + Offset: 100, + ChunkSize: 1024, + } + + w, err := bucket.CreateAppendableObjectWriter(context.Background(), req) + if err != nil { + t.Fatalf("Test case %d: CreateObjectChunkWriter: %v", i, err) + } + + // Check the content type. + writerImpl := w.(*fake.FakeObjectWriter) + if got, want := writerImpl.ContentType, tc.expected; got != want { + t.Errorf("Test case %d: o.ContentType is %q, want %q", i, got, want) + } + } +} + func TestContentTypeBucket_ComposeObjects(t *testing.T) { var err error ctx := context.Background() @@ -136,7 +165,7 @@ func TestContentTypeBucket_ComposeObjects(t *testing.T) { for i, tc := range contentTypeBucketTestCases { // Set up a bucket. bucket := gcsx.NewContentTypeBucket( - fake.NewFakeBucket(timeutil.RealClock(), "", gcs.NonHierarchical)) + fake.NewFakeBucket(timeutil.RealClock(), "", gcs.BucketType{})) // Create a source object. const srcName = "some_src" diff --git a/internal/gcsx/file_cache_reader.go b/internal/gcsx/file_cache_reader.go new file mode 100644 index 0000000000..dba7d59a7e --- /dev/null +++ b/internal/gcsx/file_cache_reader.go @@ -0,0 +1,267 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx + +import ( + "context" + "errors" + "fmt" + "io" + "sync" + "time" + + "github.com/google/uuid" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/file" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + cacheUtil "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/jacobsa/fuse/fuseops" +) + +const ( + // ReadOp ("readOp") is the value used in read context to store pointer to the read operation. + ReadOp = "readOp" + MiB = 1 << 20 +) + +// FileCacheReader is a reader that attempts to satisfy read requests for a GCS +// object from a local file cache. It is designed to be part of a layered +// reading strategy, where it acts as the first-level cache. +// +// FileCacheReader supports parallel reads. +type FileCacheReader struct { + Reader + object *gcs.MinObject + bucket gcs.Bucket + + // fileCacheHandler is used to get file cache handle and read happens using that. + // This will be nil if the file cache is disabled. + fileCacheHandler *file.CacheHandler + + // cacheFileForRangeRead is also valid for cache workflow, if true, object content + // will be downloaded for random reads as well too. + cacheFileForRangeRead bool + + // To synchronize access to fileCacheHandle. + mu sync.RWMutex + + // fileCacheHandle is used to read from the cached location. It is created on the fly + // using fileCacheHandler for the given object and bucket. + // GUARDED_BY(mu) + fileCacheHandle *file.CacheHandle + + metricHandle metrics.MetricHandle + + traceHandle tracing.TraceHandle + + handleID fuseops.HandleID +} + +func NewFileCacheReader(o *gcs.MinObject, bucket gcs.Bucket, fileCacheHandler *file.CacheHandler, cacheFileForRangeRead bool, metricHandle metrics.MetricHandle, traceHandle tracing.TraceHandle, handleID fuseops.HandleID) *FileCacheReader { + if traceHandle == nil { + traceHandle = tracing.NewNoopTracer() + } + + return &FileCacheReader{ + object: o, + bucket: bucket, + fileCacheHandler: fileCacheHandler, + cacheFileForRangeRead: cacheFileForRangeRead, + metricHandle: metricHandle, + traceHandle: traceHandle, + handleID: handleID, + } +} + +func (fc *FileCacheReader) ReaderName() string { + return "file_cache_reader" +} + +// tryReadingFromFileCache creates the cache handle first if it doesn't exist already +// and then use that handle to read object's content which is cached in local file. +// For the successful read, it returns number of bytes read, and a boolean representing +// cacheHit as true. +// For unsuccessful read, returns cacheHit as false, in this case content +// should be read from GCS. +// And it returns non-nil error in case something unexpected happens during the execution. +// In this case, we must abort the Read operation. +// +// Important: What happens if the file in cache deleted externally? +// That means, we have fileInfo entry in the fileInfoCache for that deleted file. +// (a) If a new fileCacheHandle is created in that case it will return FileNotPresentInCache +// error, given by fileCacheHandler.GetCacheHandle(). +// (b) If there is already an open fileCacheHandle then it means there is an open +// fileHandle to file in cache. So, we will get the correct data from fileHandle +// because Linux does not delete a file until open fileHandle count for a file is zero. +func (fc *FileCacheReader) tryReadingFromFileCache(ctx context.Context, p []byte, offset int64) (int, bool, error) { + if fc.fileCacheHandler == nil { + return 0, false, nil + } + ctx, span := fc.traceHandle.StartSpan(ctx, tracing.FileCacheRead) + + // By default, consider read type random if the offset is non-zero. + isSequential := offset == 0 + + requestID := uuid.New() + logger.Tracef("%.13v <- FileCache(%s:/%s, offset: %d, size: %d handle: %d)", requestID, fc.bucket.Name(), fc.object.Name, offset, len(p), fc.handleID) + + startTime := time.Now() + var bytesRead int + var cacheHit bool + var err error + + defer func() { + executionTime := time.Since(startTime) + var requestOutput string + if err != nil { + requestOutput = fmt.Sprintf("err: %v (%v)", err, executionTime) + } else { + fc.mu.RLock() + if fc.fileCacheHandle != nil { + isSequential = fc.fileCacheHandle.IsSequential(offset) + } + fc.mu.RUnlock() + requestOutput = fmt.Sprintf("OK (isSeq: %t, cacheHit: %t) (%v)", isSequential, cacheHit, executionTime) + } + + logger.Tracef("%.13v -> %s", requestID, requestOutput) + + readType := metrics.ReadTypeRandom + if isSequential { + readType = metrics.ReadTypeSequential + } + captureFileCacheMetrics(ctx, fc.metricHandle, metrics.ReadTypeNames[readType], bytesRead, cacheHit, executionTime) + fc.traceHandle.SetCacheReadAttributes(span, cacheHit, bytesRead) + if err != nil { + fc.traceHandle.RecordError(span, err) + } + fc.traceHandle.EndSpan(span) + }() + + // Create fileCacheHandle if not already. + fc.mu.Lock() + if fc.fileCacheHandle == nil { + fc.fileCacheHandle, err = fc.fileCacheHandler.GetCacheHandle(fc.object, fc.bucket, fc.cacheFileForRangeRead, offset) + if err != nil { + fc.mu.Unlock() + cacheHit = false + bytesRead = 0 + switch { + case errors.Is(err, lru.ErrInvalidEntrySize): + logger.Warnf("tryReadingFromFileCache: while creating CacheHandle: %v", err) + err = nil + return 0, false, nil + case errors.Is(err, cacheUtil.ErrCacheHandleNotRequiredForRandomRead): + // Fall back to GCS if it is a random read, cacheFileForRangeRead is + // false and there doesn't already exist file in cache. + isSequential = false + err = nil + return 0, false, nil + case errors.Is(err, cacheUtil.ErrFileExcludedFromCacheByRegex): + err = nil + return 0, false, nil + default: + err = fmt.Errorf("tryReadingFromFileCache: GetCacheHandle failed: %w", err) + return 0, false, err + } + } + } + fc.mu.Unlock() + + fc.mu.RLock() + // Because we're releasing write lock & then taking a read lock, we need to perform a nil check before accessing + // fileCacheHandle as some other thread could have set it to nil in between. + if fc.fileCacheHandle == nil { + fc.mu.RUnlock() + return 0, false, nil + } + bytesRead, cacheHit, err = fc.fileCacheHandle.Read(ctx, fc.bucket, fc.object, offset, p) + fc.mu.RUnlock() + if err == nil { + return bytesRead, cacheHit, nil + } + + bytesRead = 0 + cacheHit = false + + if cacheUtil.IsCacheHandleInvalid(err) { + fc.mu.Lock() + if fc.fileCacheHandle != nil { + logger.Tracef("Closing cacheHandle:%p for object: %s:/%s", fc.fileCacheHandle, fc.bucket.Name(), fc.object.Name) + closeErr := fc.fileCacheHandle.Close() + if closeErr != nil { + logger.Warnf("tryReadingFromFileCache: close cacheHandle error: %v", closeErr) + } + fc.fileCacheHandle = nil + } + fc.mu.Unlock() + } else if !errors.Is(err, cacheUtil.ErrFallbackToGCS) { + err = fmt.Errorf("tryReadingFromFileCache: while reading via cache: %w", err) + return 0, false, err + } + err = nil + + return 0, false, nil +} + +func (fc *FileCacheReader) ReadAt(ctx context.Context, req *ReadRequest) (ReadResponse, error) { + var readResponse ReadResponse + + if req.Offset >= int64(fc.object.Size) { + return readResponse, io.EOF + } + + // Note: If we are reading the file for the first time and read type is sequential + // then the file cache behavior is write-through i.e. data is first read from + // GCS, cached in file and then served from that file. But the cacheHit is + // false in that case. + bytesRead, cacheHit, err := fc.tryReadingFromFileCache(ctx, req.Buffer, req.Offset) + if err != nil { + return readResponse, fmt.Errorf("ReadAt: while reading from cache: %w", err) + } + // Data was served from cache. + if cacheHit || bytesRead == len(req.Buffer) || (bytesRead < len(req.Buffer) && uint64(req.Offset)+uint64(bytesRead) == fc.object.Size) { + readResponse.Size = bytesRead + return readResponse, nil + } + + // The cache is unable to serve data and requires a fallback to another reader. + return readResponse, FallbackToAnotherReader +} + +func captureFileCacheMetrics(ctx context.Context, metricHandle metrics.MetricHandle, readType metrics.ReadType, readDataSize int, cacheHit bool, readLatency time.Duration) { + metricHandle.FileCacheReadCount(1, cacheHit, readType) + metricHandle.FileCacheReadBytesCount(int64(readDataSize), readType) + metricHandle.FileCacheReadLatencies(ctx, readLatency, cacheHit) +} + +func (fc *FileCacheReader) Destroy() { + fc.mu.Lock() + defer fc.mu.Unlock() + if fc.fileCacheHandle != nil { + logger.Tracef("Closing cacheHandle:%p for object: %s:/%s", fc.fileCacheHandle, fc.bucket.Name(), fc.object.Name) + err := fc.fileCacheHandle.Close() + if err != nil { + logger.Warnf("fc.Destroy(): while closing cacheFileHandle: %v", err) + } + fc.fileCacheHandle = nil + } +} + +func (fc *FileCacheReader) CheckInvariants() { +} diff --git a/internal/gcsx/file_cache_reader_test.go b/internal/gcsx/file_cache_reader_test.go new file mode 100644 index 0000000000..3601bf6574 --- /dev/null +++ b/internal/gcsx/file_cache_reader_test.go @@ -0,0 +1,910 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "path" + "sync" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/util/diskutil" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/file" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/file/downloader" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + testutil "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +const ( + testObject = "testObject" + testObject_unfinalized = "testObject_unfinalized" + sequentialReadSizeInMb = 22 + sequentialReadSizeInBytes = sequentialReadSizeInMb * MiB + cacheMaxSize = 2 * sequentialReadSizeInMb * MiB +) + +type fileCacheReaderTest struct { + suite.Suite + ctx context.Context + object *gcs.MinObject + unfinalized_object *gcs.MinObject + mockBucket *storage.TestifyMockBucket + cacheDir string + jobManager *downloader.JobManager + cacheHandler *file.CacheHandler + reader *FileCacheReader + reader_unfinalized_object *FileCacheReader + bucketType gcs.BucketType +} + +func TestNonZonalBucketFileCacheReaderTestSuite(t *testing.T) { + nonZonalBucketFileCacheReaderTestSuite := &fileCacheReaderTest{ + bucketType: gcs.BucketType{}} + suite.Run(t, nonZonalBucketFileCacheReaderTestSuite) +} + +func TestZonalFileCacheReaderTestSuite(t *testing.T) { + zonalBucketFileCacheReaderTestSuite := &fileCacheReaderTest{ + bucketType: gcs.BucketType{Zonal: true, Hierarchical: true}} + suite.Run(t, zonalBucketFileCacheReaderTestSuite) +} + +func (t *fileCacheReaderTest) SetupTest() { + t.object = &gcs.MinObject{ + Name: testObject, + Size: 17, + Generation: 1234, + Finalized: time.Date(2025, time.June, 27, 07, 22, 30, 0, time.UTC), + } + t.unfinalized_object = &gcs.MinObject{ + Name: testObject_unfinalized, + Size: 17, + Generation: 1234, + } + t.mockBucket = new(storage.TestifyMockBucket) + t.cacheDir = path.Join(os.Getenv("HOME"), "test_cache_dir") + lruCache := lru.NewCache(cacheMaxSize) + fileCacheConfig := &cfg.FileCacheConfig{EnableCrc: false} + cacheDirVolumeBlockSize := diskutil.GetVolumeBlockSize(t.cacheDir) + t.jobManager = downloader.NewJobManager(lruCache, util.DefaultFilePerm, util.DefaultDirPerm, t.cacheDir, sequentialReadSizeInMb, fileCacheConfig, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), cacheDirVolumeBlockSize) + t.cacheHandler = file.NewCacheHandler(lruCache, t.jobManager, t.cacheDir, util.DefaultFilePerm, util.DefaultDirPerm, "", "", false, cacheDirVolumeBlockSize) + t.reader = NewFileCacheReader(t.object, t.mockBucket, t.cacheHandler, true, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), 0) + t.reader_unfinalized_object = NewFileCacheReader(t.unfinalized_object, t.mockBucket, t.cacheHandler, true, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), 0) + t.ctx = context.Background() +} + +func (t *fileCacheReaderTest) TearDownTest() { + err := os.RemoveAll(t.cacheDir) + if err != nil { + t.T().Logf("Failed to clean up test cache directory '%s': %v", t.cacheDir, err) + } + t.reader.Destroy() +} + +func (t *fileCacheReaderTest) mockNewReaderWithHandleCallForTestBucket(limit uint64, rd gcs.StorageReader) { + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(rg *gcs.ReadObjectRequest) bool { + return rg != nil && (*rg.Range).Start == 0 && (*rg.Range).Limit == limit + })).Return(rd, nil).Once() +} + +func getReadCloser(content []byte) io.ReadCloser { + r := bytes.NewReader(content) + rc := io.NopCloser(r) + return rc +} + +func (t *fileCacheReaderTest) TestNewFileCacheReader() { + reader := NewFileCacheReader(t.object, t.mockBucket, t.cacheHandler, true, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), 0) + + assert.NotNil(t.T(), reader) + assert.Equal(t.T(), t.object, reader.object) + assert.Equal(t.T(), t.mockBucket, reader.bucket) + assert.Equal(t.T(), t.cacheHandler, reader.fileCacheHandler) + assert.True(t.T(), reader.cacheFileForRangeRead) + assert.NotNil(t.T(), reader.metricHandle) + assert.NotNil(t.T(), reader.traceHandle) + assert.Nil(t.T(), reader.fileCacheHandle) +} + +func (t *fileCacheReaderTest) Test_ReadAt_NilFileCacheHandlerThrowFallBackError() { + reader := NewFileCacheReader(t.object, t.mockBucket, nil, true, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), 0) + + readResponse, err := reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: make([]byte, 10), + Offset: 0, + }) + + assert.True(t.T(), errors.Is(err, FallbackToAnotherReader), "expected %v error got %v", FallbackToAnotherReader, err) + assert.Zero(t.T(), readResponse.Size) +} + +func (t *fileCacheReaderTest) Test_ReadAt_FileSizeIsGreaterThanCacheSize() { + t.object.Size = cacheMaxSize + 5 + t.mockBucket.On("Name").Return("test-bucket") + + readResponse, err := t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: make([]byte, t.object.Size), + Offset: 0, + }) + + assert.True(t.T(), errors.Is(err, FallbackToAnotherReader), "expected %v error got %v", FallbackToAnotherReader, err) + assert.Zero(t.T(), readResponse.Size) +} + +func (t *fileCacheReaderTest) Test_ReadAt_OffsetGreaterThanFileSizeWillReturnEOF() { + offset := t.object.Size + 10 + + readResponse, err := t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: make([]byte, 10), + Offset: int64(offset), + }) + + assert.True(t.T(), errors.Is(err, io.EOF), "expected %v error got %v", io.EOF, err) + assert.Zero(t.T(), readResponse.Size) +} + +func (t *fileCacheReaderTest) Test_tryReadingFromFileCache_CacheHit() { + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(t.object.Size, rd) + t.mockBucket.On("Name").Return("test-bucket") + t.mockBucket.On("BucketType").Return(t.bucketType) + buf := make([]byte, t.object.Size) + // First read will be a cache miss. + n, cacheHit, err := t.reader.tryReadingFromFileCache(t.ctx, buf, 0) + assert.NoError(t.T(), err) + assert.False(t.T(), cacheHit) + assert.Equal(t.T(), n, len(buf)) + + // Second read will be a cache hit. + n, cacheHit, err = t.reader.tryReadingFromFileCache(t.ctx, buf, 0) + + assert.NoError(t.T(), err) + assert.True(t.T(), cacheHit) + assert.Equal(t.T(), n, len(buf)) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *fileCacheReaderTest) Test_tryReadingFromFileCache_SequentialSubsequentReadOffsetLessThanReadChunkSize() { + t.object.Size = 20 * util.MiB + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(t.object.Size, rd) + t.mockBucket.On("Name").Return("test-bucket") + t.mockBucket.On("BucketType").Return(t.bucketType) + start1 := 0 + end1 := util.MiB + require.Less(t.T(), end1, int(t.object.Size)) + // First call from offset 0 - sequential read + buf := make([]byte, end1-start1) + _, cacheHit, err := t.reader.tryReadingFromFileCache(t.ctx, buf, int64(start1)) + assert.NoError(t.T(), err) + assert.False(t.T(), cacheHit) + assert.Equal(t.T(), buf, testContent[start1:end1]) + start2 := 3*util.MiB + 4 + end2 := start2 + util.MiB + buf2 := make([]byte, end2-start2) + + // Assuming start2 offset download in progress + _, cacheHit, err = t.reader.tryReadingFromFileCache(t.ctx, buf2, int64(start2)) + + assert.NoError(t.T(), err) + assert.True(t.T(), cacheHit) + assert.Equal(t.T(), buf2, testContent[start2:end2]) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *fileCacheReaderTest) Test_ReadAt_SequentialRangeRead() { + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(t.object.Size, rd) + t.mockBucket.On("Name").Return("test-bucket") + t.mockBucket.On("BucketType").Return(t.bucketType) + start := 0 + end := 10 + require.Less(t.T(), end, int(t.object.Size)) + buf := make([]byte, end-start) + + readResponse, err := t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf, + Offset: int64(start), + }) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), testContent[start:end], buf[:readResponse.Size]) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *fileCacheReaderTest) Test_ReadAt_RandomReadNotStartWithZeroOffsetWhenCacheForRangeReadIsFalse() { + t.reader.cacheFileForRangeRead = false + start := 5 + end := 10 + t.mockBucket.On("Name").Return("test-bucket") + buf := make([]byte, end-start) + readResponse, err := t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf, + Offset: int64(start), + }) + assert.True(t.T(), errors.Is(err, FallbackToAnotherReader), "expected %v error got %v", FallbackToAnotherReader, err) + assert.Zero(t.T(), readResponse.Size) + job := t.jobManager.CreateJobIfNotExists(t.object, t.mockBucket) + jobStatus := job.GetStatus() + assert.True(t.T(), jobStatus.Name == downloader.NotStarted) + + readResponse, err = t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf, + Offset: int64(start), + }) + + assert.True(t.T(), errors.Is(err, FallbackToAnotherReader), "expected %v error got %v", FallbackToAnotherReader, err) + assert.Zero(t.T(), readResponse.Size) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *fileCacheReaderTest) Test_ReadAt_RandomReadNotStartWithZeroOffsetWhenCacheForRangeReadIsTrue() { + t.reader.cacheFileForRangeRead = true + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + start := 5 + end := 10 + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + // Mock for download job's NewReader call + t.mockNewReaderWithHandleCallForTestBucket(t.object.Size, rd) + t.mockBucket.On("Name").Return("test-bucket") + t.mockBucket.On("BucketType").Return(t.bucketType) + buf := make([]byte, end-start) + + readResponse, err := t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf, + Offset: int64(start), + }) + + assert.True(t.T(), errors.Is(err, FallbackToAnotherReader), "expected %v error got %v", FallbackToAnotherReader, err) + assert.Zero(t.T(), readResponse.Size) + job := t.jobManager.GetJob(t.object.Name, t.mockBucket.Name()) + assert.True(t.T(), job == nil || job.GetStatus().Name == downloader.Downloading) + assert.NotNil(t.T(), t.reader.fileCacheHandle) +} + +func (t *fileCacheReaderTest) Test_ReadAt_SequentialToRandomSubsequentReadOffsetMoreThanReadChunkSize() { + t.object.Size = 20 * util.MiB + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + // Mock for download job's NewReader call + t.mockNewReaderWithHandleCallForTestBucket(t.object.Size, rd) + t.mockBucket.On("Name").Return("test-bucket") + t.mockBucket.On("BucketType").Return(t.bucketType) + start1 := 0 + end1 := util.MiB + require.Less(t.T(), end1, int(t.object.Size)) + // First call from offset 0 - sequential read + buf := make([]byte, end1-start1) + readResponse, err := t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf, + Offset: int64(start1), + }) + // Served from file cache + assert.NoError(t.T(), err) + assert.Equal(t.T(), testContent[start1:end1], buf[:readResponse.Size]) + start2 := 16*util.MiB + 4 + end2 := start2 + util.MiB + buf2 := make([]byte, end2-start2) + + readResponse, err = t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf2, + Offset: int64(start2), + }) + + // Assuming a download with a start offset of start2 is in progress, a fallback to another reader will be required. + assert.True(t.T(), errors.Is(err, FallbackToAnotherReader), "expected %v error got %v", FallbackToAnotherReader, err) + assert.Zero(t.T(), readResponse.Size) + job := t.jobManager.GetJob(t.object.Name, t.mockBucket.Name()) + assert.True(t.T(), job == nil || job.GetStatus().Name == downloader.Downloading) + assert.NotNil(t.T(), t.reader.fileCacheHandle) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *fileCacheReaderTest) Test_ReadAt_SequentialToRandomSubsequentReadOffsetLessThanPrevious() { + t.object.Size = 20 * util.MiB + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(t.object.Size, rd) + t.mockBucket.On("Name").Return("test-bucket") + t.mockBucket.On("BucketType").Return(t.bucketType) + start1 := 0 + end1 := util.MiB + require.Less(t.T(), end1, int(t.object.Size)) + // First call from offset 0 - sequential read + buf := make([]byte, end1-start1) + readResponse, err := t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf, + Offset: int64(start1), + }) + // Served from file cache + assert.NoError(t.T(), err) + assert.Equal(t.T(), testContent[start1:end1], buf[:readResponse.Size]) + start2 := 16*util.MiB + 4 + end2 := start2 + util.MiB + buf2 := make([]byte, end2-start2) + // Assuming a download with a start offset of start2 is in progress, a fallback to another reader will be required. + readResponse, err = t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf2, + Offset: int64(start2), + }) + assert.True(t.T(), errors.Is(err, FallbackToAnotherReader), "expected %v error got %v", FallbackToAnotherReader, err) + assert.Zero(t.T(), readResponse.Size) + // Assuming start3 offset is downloaded + start3 := 4 * util.MiB + end3 := start3 + util.MiB + buf3 := make([]byte, end3-start3) + + readResponse, err = t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf3, + Offset: int64(start3), + }) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), testContent[start3:end3], buf3) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *fileCacheReaderTest) Test_ReadAt_CacheMissDueToInvalidJob() { + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + rc1 := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(t.object.Size, rc1) + t.mockBucket.On("Name").Return("test-bucket") + t.mockBucket.On("BucketType").Return(t.bucketType) + buf := make([]byte, t.object.Size) + readResponse, err := t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf, + Offset: 0, + }) + assert.NoError(t.T(), err) + assert.Equal(t.T(), testContent, buf[:readResponse.Size]) + job := t.jobManager.GetJob(t.object.Name, t.mockBucket.Name()) + if job != nil { + jobStatus := job.GetStatus().Name + assert.True(t.T(), jobStatus == downloader.Downloading || jobStatus == downloader.Completed, fmt.Sprintf("the actual status is %v", jobStatus)) + } + err = t.reader.fileCacheHandler.InvalidateCache(t.object.Name, t.mockBucket.Name()) + assert.NoError(t.T(), err) + + readResponse, err = t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf, + Offset: 0, + }) + + // As job is invalidated Need to get served from GCS reader + assert.True(t.T(), errors.Is(err, FallbackToAnotherReader), "expected %v error got %v", FallbackToAnotherReader, err) + assert.Zero(t.T(), readResponse.Size) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *fileCacheReaderTest) Test_ReadAt_CachePopulatedAndThenCacheMissDueToInvalidJob() { + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + // First successful read with cache + rd1 := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(t.object.Size, rd1) + t.mockBucket.On("Name").Return("test-bucket") + t.mockBucket.On("BucketType").Return(t.bucketType) + buf := make([]byte, t.object.Size) + readResponse, err := t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf, + Offset: 0, + }) + assert.NoError(t.T(), err) + assert.Equal(t.T(), testContent, buf) + job := t.jobManager.GetJob(t.object.Name, t.mockBucket.Name()) + if job != nil { + jobStatus := job.GetStatus().Name + assert.True(t.T(), jobStatus == downloader.Downloading || jobStatus == downloader.Completed, fmt.Sprintf("the actual status is %v", jobStatus)) + } + assert.NotNil(t.T(), t.reader.fileCacheHandle) + // Invalidate the cache to simulate cache miss + err = t.reader.fileCacheHandler.InvalidateCache(t.object.Name, t.mockBucket.Name()) + assert.NoError(t.T(), err) + readResponse, err = t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: make([]byte, t.object.Size), + Offset: 0, + }) + assert.True(t.T(), errors.Is(err, FallbackToAnotherReader), "expected %v error got %v", FallbackToAnotherReader, err) + assert.Zero(t.T(), readResponse.Size) + assert.Nil(t.T(), t.reader.fileCacheHandle) + rd2 := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(t.object.Size, rd2) + clear(buf) + + readResponse, err = t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf, + Offset: 0, + }) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), testContent, buf[:readResponse.Size]) + assert.NotNil(t.T(), t.reader.fileCacheHandle) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *fileCacheReaderTest) Test_ReadAt_CachePopulatedAndThenCacheMissDueToInvalidFileHandleAfterThenCacheHitWithNewFileCacheHandle() { + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(t.object.Size, rd) + t.mockBucket.On("Name").Return("test-bucket") + t.mockBucket.On("BucketType").Return(t.bucketType) + buf := make([]byte, t.object.Size) + readResponse, err := t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf, + Offset: 0, + }) + assert.NoError(t.T(), err) + assert.Equal(t.T(), testContent, buf[:readResponse.Size]) + assert.NotNil(t.T(), t.reader.fileCacheHandle) + err = t.reader.fileCacheHandle.Close() + assert.NoError(t.T(), err) + readResponse, err = t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: make([]byte, t.object.Size), + Offset: 0, + }) + assert.True(t.T(), errors.Is(err, FallbackToAnotherReader), "expected %v error got %v", FallbackToAnotherReader, err) + assert.Zero(t.T(), readResponse.Size) + assert.Nil(t.T(), t.reader.fileCacheHandle) + clear(buf) + + readResponse, err = t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf, + Offset: 0, + }) + + // Reading from file cache with new file cache handle. + assert.NoError(t.T(), err) + assert.Equal(t.T(), testContent, buf[:readResponse.Size]) + assert.NotNil(t.T(), t.reader.fileCacheHandle) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *fileCacheReaderTest) Test_ReadAt_IfCacheFileGetsDeleted() { + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(t.object.Size, rd) + t.mockBucket.On("Name").Return("test-bucket") + t.mockBucket.On("BucketType").Return(t.bucketType) + buf := make([]byte, t.object.Size) + readResponse, err := t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf, + Offset: 0, + }) + assert.NoError(t.T(), err) + assert.Equal(t.T(), testContent, buf[:readResponse.Size]) + assert.NotNil(t.T(), t.reader.fileCacheHandle) + err = t.reader.fileCacheHandle.Close() + assert.NoError(t.T(), err) + t.reader.fileCacheHandle = nil + // Delete the local cache file. + filePath := util.GetDownloadPath(t.cacheDir, util.GetObjectPath(t.mockBucket.Name(), t.object.Name)) + err = os.Remove(filePath) + assert.NoError(t.T(), err) + + readResponse, err = t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: make([]byte, t.object.Size), + Offset: 0, + }) + + assert.True(t.T(), errors.Is(err, util.ErrFileNotPresentInCache)) + assert.Zero(t.T(), readResponse.Size) +} + +func (t *fileCacheReaderTest) Test_ReadAt_IfCacheFileGetsDeletedWithCacheHandleOpen() { + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(t.object.Size, rd) + t.mockBucket.On("Name").Return("test-bucket") + t.mockBucket.On("BucketType").Return(t.bucketType) + buf := make([]byte, t.object.Size) + readResponse, err := t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf, + Offset: 0, + }) + assert.NoError(t.T(), err) + assert.Equal(t.T(), testContent, buf[:readResponse.Size]) + assert.NotNil(t.T(), t.reader.fileCacheHandle) + // Delete the local cache file. + filePath := util.GetDownloadPath(t.cacheDir, util.GetObjectPath(t.mockBucket.Name(), t.object.Name)) + err = os.Remove(filePath) + assert.NoError(nil, err) + clear(buf) + + // Read via cache only, as we have old fileHandle open and linux + // doesn't delete the file until the fileHandle count for the file is zero. + readResponse, err = t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf, + Offset: 0, + }) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), testContent, buf[:readResponse.Size]) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *fileCacheReaderTest) Test_ReadAt_FailedJobNextReadCreatesNewJobAndCacheHit() { + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + // First NewReaderWithReadHandle call fails, simulating a failed attempt to read from GCS. + // This triggers a fallback to GCS reader. + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, mock.Anything).Return(nil, errors.New("")).Once() + t.mockBucket.On("Name").Return("test-bucket") + t.mockBucket.On("BucketType").Return(t.bucketType) + // First ReadAt call: + // - Should result in a FallbackToAnotherReader error. + // - No data should be returned. + // - The job should be marked as failed (if jobManager is functioning correctly). + readResponse, err := t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: make([]byte, t.object.Size), + Offset: 0, + }) + assert.True(t.T(), errors.Is(err, FallbackToAnotherReader), "expected %v error got %v", FallbackToAnotherReader, err) + assert.Zero(t.T(), readResponse.Size) + job := t.jobManager.GetJob(t.object.Name, t.mockBucket.Name()) + assert.True(t.T(), job == nil || job.GetStatus().Name == downloader.Failed) + rc := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(t.object.Size, rc) + buf := make([]byte, t.object.Size) + // Second ReadAt call: The file cache should be populated as a result of this successful read. + readResponse, err = t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf, + Offset: 0, + }) + assert.NoError(t.T(), err) + assert.Equal(t.T(), testContent, buf[:readResponse.Size]) + assert.NotNil(t.T(), t.reader.fileCacheHandle) + clear(buf) + + // Third ReadAt call: Should be served directly from the file cache. + readResponse, err = t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf, + Offset: 0, + }) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), testContent, buf[:readResponse.Size]) + assert.NotNil(t.T(), t.reader.fileCacheHandle) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *fileCacheReaderTest) Test_ReadAt_NegativeOffsetShouldThrowError() { + t.mockBucket.On("Name").Return("test-bucket") + + readResponse, err := t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: make([]byte, 10), + Offset: -1, + }) + + assert.Error(t.T(), err) + assert.Zero(t.T(), readResponse.Size) +} + +func (t *fileCacheReaderTest) Test_ReadAt_OffsetBeyondObjectSizeShouldThrowEOFError() { + readResponse, err := t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: make([]byte, 10), + Offset: int64(t.object.Size) + 1, + }) + + assert.Error(t.T(), err) + assert.Zero(t.T(), readResponse.Size) + assert.ErrorIs(t.T(), err, io.EOF) +} + +func (t *fileCacheReaderTest) skipForNonZonalBucket() { + t.T().Helper() + + if !t.bucketType.Zonal { + t.T().Skipf("Skipping test for non-zonal bucket type") + } +} + +func (t *fileCacheReaderTest) fullyReadOriginalSizeOfUnfinalizedObject(origObjectSize uint64) { + t.T().Helper() + t.mockBucket.On("Name").Return("test-bucket") + testContent := testutil.GenerateRandomBytes(int(t.unfinalized_object.Size)) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(t.unfinalized_object.Size, rd) + t.mockBucket.On("BucketType").Return(t.bucketType) + readResponse, err := t.reader_unfinalized_object.ReadAt(t.ctx, &ReadRequest{ + Buffer: make([]byte, origObjectSize), + Offset: 0, + }) + assert.NoError(t.T(), err) + assert.Equal(t.T(), int(origObjectSize), readResponse.Size) +} + +func (t *fileCacheReaderTest) waitForDownloadJobToFinish(obj *gcs.MinObject) { + t.T().Helper() + + job := t.jobManager.GetJob(obj.Name, t.mockBucket.Name()) + if job != nil { + for job.GetStatus().Name == downloader.Downloading { + time.Sleep(10 * time.Millisecond) // Poll the job status. + } + } +} + +func (t *fileCacheReaderTest) Test_ReadAt_UnfinalizedObjectReadFromOffsetBeyondCachedSizeAfterSizeIncreasedShouldThrowFallbackError() { + t.skipForNonZonalBucket() + origObjectSize := t.unfinalized_object.Size + // First read, which may start a background download job. + t.fullyReadOriginalSizeOfUnfinalizedObject(origObjectSize) + // Wait for the background download job to complete to avoid a data race. + // This is needed to avoid the race condition on the size of t.unfinalized_object later on. + t.waitForDownloadJobToFinish(t.unfinalized_object) + + // Resize the object, and read beyond previously cached size and within new object size. + cachedObjectSize := int64(origObjectSize) + objectSizeIncrease := uint64(10) + newObjectSize := origObjectSize + objectSizeIncrease + t.unfinalized_object.Size = newObjectSize + readResponse, err := t.reader_unfinalized_object.ReadAt(t.ctx, &ReadRequest{ + Buffer: make([]byte, objectSizeIncrease), + Offset: cachedObjectSize, + }) + + assert.Error(t.T(), err) + assert.Zero(t.T(), readResponse.Size) + assert.ErrorIs(t.T(), err, FallbackToAnotherReader) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *fileCacheReaderTest) Test_ReadAt_UnfinalizedObjectReadFromOffsetBeyondObjectSizeAfterSizeIncreasedShouldThrowEOFError() { + t.skipForNonZonalBucket() + origObjectSize := t.unfinalized_object.Size + // First read, which may start a background download job. + t.fullyReadOriginalSizeOfUnfinalizedObject(origObjectSize) + // Wait for the background download job to complete to avoid a data race. + // This is needed to avoid the race condition on the size of t.unfinalized_object later on. + t.waitForDownloadJobToFinish(t.unfinalized_object) + + // Resize the object, and read beyond new object size. + objectSizeIncrease := uint64(10) + newObjectSize := origObjectSize + objectSizeIncrease + t.unfinalized_object.Size = newObjectSize + readResponse, err := t.reader_unfinalized_object.ReadAt(t.ctx, &ReadRequest{ + Buffer: make([]byte, 1), + Offset: int64(newObjectSize), + }) + + assert.Error(t.T(), err) + assert.Zero(t.T(), readResponse.Size) + assert.ErrorIs(t.T(), err, io.EOF) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *fileCacheReaderTest) Test_ReadAt_UnfinalizedObjectReadFromOffsetBelowCachedSizeAndReadBeyondCachedSizeWithIncreasedObjectSizeShouldThrowFallbackError() { + t.skipForNonZonalBucket() + origObjectSize := t.unfinalized_object.Size + // First read, which may start a background download job. + t.fullyReadOriginalSizeOfUnfinalizedObject(origObjectSize) + // Wait for the background download job to complete to avoid a data race. + // This is needed to avoid the race condition on the size of t.unfinalized_object later on. + t.waitForDownloadJobToFinish(t.unfinalized_object) + + // Resize the object, and read from below previously cached size and within new object size. + cachedObjectSize := int64(origObjectSize) + objectSizeIncrease := uint64(10) + newObjectSize := origObjectSize + objectSizeIncrease + t.unfinalized_object.Size = newObjectSize + readResponse, err := t.reader_unfinalized_object.ReadAt(t.ctx, &ReadRequest{ + Buffer: make([]byte, cachedObjectSize+1), + Offset: cachedObjectSize / 2, + }) + + assert.Error(t.T(), err) + assert.Zero(t.T(), readResponse.Size) + assert.ErrorIs(t.T(), err, FallbackToAnotherReader) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *fileCacheReaderTest) Test_ReadAt_UnfinalizedObjectReadFromOffsetBelowCachedSizeAndReadBeyondObjectSizeWithIncreasedObjectSizeShouldThrowFallbackError() { + t.skipForNonZonalBucket() + origObjectSize := t.unfinalized_object.Size + // First read, which may start a background download job. + t.fullyReadOriginalSizeOfUnfinalizedObject(origObjectSize) + // Wait for the background download job to complete to avoid a data race. + // This is needed to avoid the race condition on the size of t.unfinalized_object later on. + t.waitForDownloadJobToFinish(t.unfinalized_object) + + // Resize the object, and read from below cached size and beyond this new object size. + cachedObjectSize := int64(origObjectSize) + objectSizeIncrease := uint64(10) + newObjectSize := origObjectSize + objectSizeIncrease + t.unfinalized_object.Size = newObjectSize + readResponse, err := t.reader_unfinalized_object.ReadAt(t.ctx, &ReadRequest{ + Buffer: make([]byte, newObjectSize), + Offset: cachedObjectSize / 2, + }) + + assert.Error(t.T(), err) + assert.Zero(t.T(), readResponse.Size) + assert.ErrorIs(t.T(), err, FallbackToAnotherReader) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *fileCacheReaderTest) Test_ReadAt_UnfinalizedObjectReadFromOffsetBelowCachedSizeAndReadBeyondCachedSizeShouldNotThrowError() { + t.skipForNonZonalBucket() + origObjectSize := t.unfinalized_object.Size + t.fullyReadOriginalSizeOfUnfinalizedObject(origObjectSize) + + cachedObjectSize := int64(origObjectSize) + readResponse, err := t.reader_unfinalized_object.ReadAt(t.ctx, &ReadRequest{ + Buffer: make([]byte, cachedObjectSize+1), + Offset: cachedObjectSize / 2, + }) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), int(cachedObjectSize-cachedObjectSize/2), readResponse.Size) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *fileCacheReaderTest) Test_ReadAt_FinalizedObjectReadFromOffsetBelowCachedSizeAndReadBeyondCachedSizeShouldNotThrowError() { + t.mockBucket.On("Name").Return("test-bucket") + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(t.object.Size, rd) + t.mockBucket.On("BucketType").Return(t.bucketType) + readResponse, err := t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: make([]byte, t.object.Size), + Offset: 0, + }) + assert.NoError(t.T(), err) + assert.Equal(t.T(), int(t.object.Size), readResponse.Size) + + readResponse, err = t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: make([]byte, t.object.Size+1), + Offset: int64(t.object.Size) / 2, + }) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), int(t.object.Size-t.object.Size/2), readResponse.Size) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *fileCacheReaderTest) Test_Destroy_NonNilCacheHandle() { + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(t.object.Size, rd) + t.mockBucket.On("Name").Return("test-bucket") + t.mockBucket.On("BucketType").Return(t.bucketType) + buf := make([]byte, t.object.Size) + readResponse, err := t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf, + Offset: 0, + }) + assert.NoError(t.T(), err) + assert.Equal(t.T(), testContent, buf[:readResponse.Size]) + assert.NotNil(t.T(), t.reader.fileCacheHandle) + + t.reader.Destroy() + + assert.Nil(t.T(), t.reader.fileCacheHandle) +} + +func (t *fileCacheReaderTest) Test_Destroy_NilCacheHandle() { + t.reader.fileCacheHandler = nil + + t.reader.Destroy() + + assert.Nil(nil, t.reader.fileCacheHandle) +} + +func (t *fileCacheReaderTest) Test_Concurrent_ReadAt() { + // Setup + t.object.Size = 100 + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + // Mock NewReaderWithReadHandle for the downloader job, which is triggered by GetCacheHandle. + // This should only be called once across all concurrent reads. + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, mock.Anything).Return(rd, nil).Once() + t.mockBucket.On("Name").Return("test-bucket") + t.mockBucket.On("BucketType").Return(t.bucketType) + var wg sync.WaitGroup + numGoroutines := 5 + + // Act + for i := 0; i < numGoroutines; i++ { + wg.Add(1) + go func() { + defer wg.Done() + buf := make([]byte, t.object.Size) + readResponse, err := t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf, + Offset: 0, + }) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), int(t.object.Size), readResponse.Size) + assert.Equal(t.T(), testContent, buf) + }() + } + // Wait for all goroutines or timeout + done := make(chan bool) + go func() { + wg.Wait() + close(done) + }() + + select { + case <-done: + // Final Assertions + t.mockBucket.AssertExpectations(t.T()) + assert.NotNil(t.T(), t.reader.fileCacheHandle) + case <-time.After(10 * time.Second): + assert.Fail(t.T(), "Test timed out, potential deadlock.") + } +} + +func (t *fileCacheReaderTest) Test_Concurrent_ReadAt_And_Destroy() { + // Setup + t.object.Size = 100 + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(t.object.Size, rd) + t.mockBucket.On("Name").Return("test-bucket") + t.mockBucket.On("BucketType").Return(t.bucketType) + var wg sync.WaitGroup + numGoroutines := 20 + wg.Add(numGoroutines) + + // Act: Concurrently try to read and destroy from a bunch of goroutines. + // Reader goroutines + for i := 0; i < numGoroutines/2; i++ { + go func() { + defer wg.Done() + // This read should not cause a panic, even if Destroy() nils out the handle + // in the middle of the operation. + buf := make([]byte, t.object.Size) + _, err := t.reader.ReadAt(t.ctx, &ReadRequest{ + Buffer: buf, + Offset: 0, + }) + + // Assert: The read might fail with a fallback error or succeed if it completes before Destroy. + // The key is that it doesn't panic. + assert.True(t.T(), err == nil || errors.Is(err, FallbackToAnotherReader)) + }() + } + // Destroyer goroutines + for i := 0; i < numGoroutines/2; i++ { + go func() { + defer wg.Done() + t.reader.Destroy() + }() + } + + wg.Wait() +} diff --git a/internal/gcsx/garbage_collect.go b/internal/gcsx/garbage_collect.go index 896bd18163..db842fa746 100644 --- a/internal/gcsx/garbage_collect.go +++ b/internal/gcsx/garbage_collect.go @@ -19,12 +19,12 @@ import ( "sync/atomic" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" "golang.org/x/net/context" "golang.org/x/sync/errgroup" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" ) func garbageCollectOnce( diff --git a/internal/gcsx/inactive_timeout_reader.go b/internal/gcsx/inactive_timeout_reader.go new file mode 100644 index 0000000000..5b8df947b8 --- /dev/null +++ b/internal/gcsx/inactive_timeout_reader.go @@ -0,0 +1,245 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx + +import ( + "errors" + "fmt" + "time" + + storagev2 "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/clock" + "github.com/googlecloudplatform/gcsfuse/v3/internal/locker" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "golang.org/x/net/context" +) + +// InactiveTimeoutReader is a wrapper over gcs.StorageReader that automatically +// closes the wrapped GCS reader connection after a specified period of +// inactivity (timeout). When a read operation is attempted on an inactive +// (closed) reader, it automatically attempts to reconnect using the last known +// read handle and the appropriate offset based on bytes previously read. +// +// This is useful for managing resources, especially when dealing with many +// potentially inactive or idle readers. +// +// Important notes: +// - Inactivity Timer: A background goroutine monitors read activity. If no +// Read calls occur within the timeout duration, the underlying gcsReader is +// closed. +// - Due to the activity check happens periodically (every timeout duration), the +// actual reader connection closure can happen anywhere b/w timeout and 2 * timeout +// after the very last read operation, depending on when the last read occurred +// relative to the background routine wake-up cycle. +// - Thread Safety: The reader is safe for concurrent use by multiple goroutines, +// protected by an internal mutex. +type InactiveTimeoutReader struct { + object *gcs.MinObject + bucket gcs.Bucket + + // The underlying GCS storage reader; nil if closed due to inactivity. + gcsReader gcs.StorageReader + + // Total number of bytes successfully read so far. + seen uint64 + + // Requested range [start, end) from this reader. + reqRange gcs.ByteRange + + // The read handle used for efficient reconnection for zonal bucket. + readHandle []byte + + // Derived from the parent context, used for creating new readers and monitoring cancellation. + ctx context.Context + cancel context.CancelFunc + + // Mutex protecting internal state (mainly gcsReader & isActive). + mu locker.Locker + + // Flag set by Read and reset by the monitor goroutine to track activity within the timeout window. + isActive bool +} + +var ( + ErrZeroInactivityTimeout = errors.New("ErrZeroInactivityTimeout") +) + +// NewInactiveTimeoutReader creates a new gcs.StorageReader that wraps an +// underlying GCS reader. It attempts to create the initial reader using the +// provided parameters. If successful, it starts a background goroutine to monitor +// for inactivity based on the specified timeout. +// +// If the timeout duration is zero, it returns (nil, ErrZeroInactivityTimeout) as a zero timeout +// defeats the purpose of this wrapper. +func NewInactiveTimeoutReader(ctx context.Context, bucket gcs.Bucket, object *gcs.MinObject, readHandle []byte, byteRange gcs.ByteRange, timeout time.Duration) (gcs.StorageReader, error) { + return NewInactiveTimeoutReaderWithClock(ctx, bucket, object, readHandle, byteRange, timeout, clock.RealClock{}) +} + +func NewInactiveTimeoutReaderWithClock(ctx context.Context, bucket gcs.Bucket, object *gcs.MinObject, readHandle []byte, byteRange gcs.ByteRange, timeout time.Duration, clock clock.Clock) (gcs.StorageReader, error) { + if timeout == 0 { + return nil, ErrZeroInactivityTimeout + } + + itr := &InactiveTimeoutReader{ + object: object, + bucket: bucket, + reqRange: byteRange, + readHandle: readHandle, + mu: locker.New("InactiveTimeoutReader: "+object.Name, func() {}), + isActive: false, + } + itr.ctx, itr.cancel = context.WithCancel(ctx) + + var err error + if itr.gcsReader, err = itr.createGCSReader(); err != nil { + return nil, err + } + + // Start the background periodic routine. + go itr.monitor(clock, timeout) + + return itr, nil +} + +// createGCSReader is a helper method to create the underlined reader from itr.start + itr.seen offset. +func (itr *InactiveTimeoutReader) createGCSReader() (gcs.StorageReader, error) { + reader, err := itr.bucket.NewReaderWithReadHandle( + itr.ctx, + &gcs.ReadObjectRequest{ + Name: itr.object.Name, + Generation: itr.object.Generation, + Range: &gcs.ByteRange{ + Start: itr.reqRange.Start + itr.seen, + Limit: itr.reqRange.Limit, + }, + ReadCompressed: itr.object.HasContentEncodingGzip(), + ReadHandle: itr.readHandle, + }) + if err != nil { + return nil, fmt.Errorf("NewReaderWithReadHandle: %w", err) + } + return reader, nil +} + +// Read implements io.Reader interface. +// +// If the underlying reader has been closed due to inactivity, Read automatically +// attempts to reconnect using the stored read handle and the correct offset +// (start + bytes previously seen). If reconnection fails, the error is returned. +// +// Each successful Read call marks the reader as active, resetting the inactivity timer +// in the background monitor. This method is thread-safe. +// +// Calling Read() after explicitly calling Close() is not supported and will +// lead to undefined behavior. +func (itr *InactiveTimeoutReader) Read(p []byte) (n int, err error) { + itr.mu.Lock() + defer itr.mu.Unlock() + + itr.isActive = true + + if itr.gcsReader == nil { + if itr.gcsReader, err = itr.createGCSReader(); err != nil { + return 0, err + } + } + + n, err = itr.gcsReader.Read(p) + itr.seen += uint64(n) + return n, err +} + +// Close explicitly closes the underlying gcs.StorageReader if it's currently open. +// It also signals the background monitor goroutine to stop. +// Returns an error if closing the underlying reader fails. +func (itr *InactiveTimeoutReader) Close() (err error) { + itr.mu.Lock() + defer itr.mu.Unlock() + + // Signal background periodic routine to stop. + itr.cancel() + + if itr.gcsReader == nil { + return nil + } + + err = itr.gcsReader.Close() + itr.gcsReader = nil + if err != nil { + return fmt.Errorf("Close reader: %w", err) + } + return nil +} + +// ReadHandle returns the read handle associated with the underlying GCS reader. +// If the reader has been closed due to inactivity, it returns the handle +// stored from the last active reader. +func (itr *InactiveTimeoutReader) ReadHandle() (rh storagev2.ReadHandle) { + itr.mu.Lock() + defer itr.mu.Unlock() + + if itr.gcsReader == nil { + return itr.readHandle + } + + return itr.gcsReader.ReadHandle() +} + +// monitor runs in a background goroutine, and checks for inactivity. +func (itr *InactiveTimeoutReader) monitor(clock clock.Clock, timeout time.Duration) { + timer := clock.After(timeout) + for { + select { + case <-timer: + itr.handleTimeout() + timer = clock.After(timeout) + case <-itr.ctx.Done(): + return + } + } +} + +// handleTimeout is called when the inactivity timer fires. It acquires the +// reader's lock, checks the activity state, and takes appropriate action. +// If the reader was marked as active since the last check, it resets the +// activity flag. If the reader was inactive, it closes the underlying GCS reader. +// It always returns a new timer channel for the next fire. +func (itr *InactiveTimeoutReader) handleTimeout() { + itr.mu.Lock() + defer itr.mu.Unlock() + + if itr.isActive { + itr.isActive = false + } else { + itr.closeGCSReader() + } +} + +// closeGCSReader closes the wrapped gcsReader, itr.mu.Lock() should be taken +// before calling this. +func (itr *InactiveTimeoutReader) closeGCSReader() { + if itr.gcsReader == nil { + return + } + + // Not printing the timeout explicitly, as can be refer from the code/config. + logger.Tracef("Closing reader for object %q due to inactivity.\n", itr.object.Name) + itr.readHandle = itr.gcsReader.ReadHandle() + if err := itr.gcsReader.Close(); err != nil { + logger.Warnf("Error closing inactive reader for object %q: %v", itr.object.Name, err) + } + itr.gcsReader = nil +} diff --git a/internal/gcsx/inactive_timeout_reader_test.go b/internal/gcsx/inactive_timeout_reader_test.go new file mode 100644 index 0000000000..c58dd0a904 --- /dev/null +++ b/internal/gcsx/inactive_timeout_reader_test.go @@ -0,0 +1,378 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx + +import ( + "bytes" + "context" + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/clock" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type InactiveTimeoutReaderTestSuite struct { + suite.Suite + ctx context.Context + mockBucket *storage.TestifyMockBucket + object *gcs.MinObject + reader gcs.StorageReader // The reader under test + + // Fields to hold results from setup for individual tests + initialData []byte + readHandle []byte + initialFakeReader *fake.FakeReader + timeout time.Duration + simulatedClock *clock.SimulatedClock +} + +func TestInactiveTimeoutReaderTestSuite(t *testing.T) { + suite.Run(t, new(InactiveTimeoutReaderTestSuite)) +} + +func (s *InactiveTimeoutReaderTestSuite) SetupTest() { + s.ctx = context.Background() + s.mockBucket = new(storage.TestifyMockBucket) + + // Default values, can be overridden by tests before calling setupReader + s.initialData = []byte("default data") + s.timeout = 100 * time.Millisecond + s.object = &gcs.MinObject{Name: "test-object", Generation: 123, Size: uint64(len(s.initialData))} + s.reader = nil // Reset reader before each test + s.initialFakeReader = nil + s.simulatedClock = clock.NewSimulatedClock(time.Date(2020, time.January, 1, 12, 0, 0, 0, time.UTC)) +} + +func (s *InactiveTimeoutReaderTestSuite) TearDownTest() { + if s.reader == nil { + return + } + + // Close the wrapper reader, not the potentially nil internal one + s.reader.Close() + s.reader = nil + s.mockBucket.AssertExpectations(s.T()) +} + +// setupReader is a helper within the suite to create the reader under test. +// Tests should call this after setting specific suite properties like initialData or timeout. +func (s *InactiveTimeoutReaderTestSuite) setupReader() { + s.object.Size = uint64(len(s.initialData)) // Ensure object size matches data + readCloser := getReadCloser(s.initialData) + s.initialFakeReader = &fake.FakeReader{ReadCloser: readCloser, Handle: s.readHandle} + + readObjectRequest := &gcs.ReadObjectRequest{ + Name: s.object.Name, + Generation: s.object.Generation, + Range: &gcs.ByteRange{ + Start: uint64(0), + Limit: s.object.Size, + }, + ReadCompressed: s.object.HasContentEncodingGzip(), + ReadHandle: s.readHandle, + } + s.mockBucket.On("NewReaderWithReadHandle", mock.Anything, readObjectRequest).Return(s.initialFakeReader, nil).Times(1) + + var err error + // Use NewInactiveTimeoutReader directly as NewStorageReaderWithInactiveTimeout is deprecated. + s.reader, err = NewInactiveTimeoutReaderWithClock(s.ctx, s.mockBucket, s.object, s.readHandle, gcs.ByteRange{Start: uint64(0), Limit: s.object.Size}, s.timeout, s.simulatedClock) + time.Sleep(5 * time.Millisecond) // Allow time to schedule and create a timer. + s.Require().Nil(err) + s.Require().NotNil(s.reader) +} + +func (s *InactiveTimeoutReaderTestSuite) Test_NewInactiveTimeoutReader_InitialReadError() { + s.initialData = make([]byte, 100) // Size doesn't matter here + s.object = &gcs.MinObject{Name: "fail-object", Generation: 456, Size: 100} + s.timeout = 100 * time.Millisecond + initialErr := errors.New("initial connection failed") + // Expect the initial NewReader call to fail + s.mockBucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(req *gcs.ReadObjectRequest) bool { + return req.Name == s.object.Name && + req.Generation == s.object.Generation && + req.Range.Start == 0 && + req.Range.Limit == 100 + })).Return(nil, initialErr).Once() + + _, err := NewInactiveTimeoutReader(s.ctx, s.mockBucket, s.object, []byte{}, gcs.ByteRange{Start: 0, Limit: 100}, s.timeout) + + s.Error(err) + s.ErrorIs(err, initialErr) // Should be the exact error from the bucket +} + +func (s *InactiveTimeoutReaderTestSuite) Test_NewInactiveTimeoutReader_ZeroTimeoutError() { + s.initialData = []byte("zero timeout") + s.timeout = 0 // Zero timeout + + _, err := NewInactiveTimeoutReader(s.ctx, s.mockBucket, s.object, []byte{}, gcs.ByteRange{Start: 0, Limit: 100}, s.timeout) + + s.Error(err) + s.ErrorIs(err, ErrZeroInactivityTimeout) +} + +func (s *InactiveTimeoutReaderTestSuite) Test_Read_InitialReadNoError() { + s.initialData = []byte("hello world") + s.timeout = 100 * time.Millisecond + s.setupReader() + buf := make([]byte, 5) + + n, err := s.reader.Read(buf) + + s.NoError(err) + s.Equal(5, n) + s.Equal("hello", string(buf[:n])) +} + +func (s *InactiveTimeoutReaderTestSuite) Test_NoReadCloserWithinTimeout() { + s.initialData = []byte("hello world!") + s.timeout = 100 * time.Millisecond + s.setupReader() + buf := make([]byte, 6) + n1, err1 := s.reader.Read(buf) + s.NoError(err1) + s.Equal(6, n1) + s.simulatedClock.AdvanceTime(s.timeout / 2) + // Allow some time to routine incase timer fired in half timeout. + time.Sleep(5 * time.Millisecond) + + n2, err2 := s.reader.Read(buf) + + inactiveReader := s.reader.(*InactiveTimeoutReader) + s.True(inactiveReader.isActive) + s.NoError(err2) + s.Equal(6, n2) + s.Equal("world!", string(buf[:n2])) +} + +func (s *InactiveTimeoutReaderTestSuite) Test_ReadFull_Succeeds() { + buf := make([]byte, 16) + s.initialData = []byte("hello world!") + s.timeout = 100 * time.Millisecond + s.setupReader() + + n, err := s.reader.Read(buf) + + s.NoError(err) + s.Equal(12, n) +} + +func (s *InactiveTimeoutReaderTestSuite) Test_Read_ReconnectFails() { + buf := make([]byte, 5) + s.initialData = []byte("reconnect failure") + s.timeout = 50 * time.Millisecond + s.setupReader() + n, err := s.reader.Read(buf) + s.Require().NoError(err) + s.Require().Equal(5, n) + // First timeout fire will make the reader inactive. + s.simulatedClock.AdvanceTime(s.timeout + time.Millisecond) + // Wait for the monitor routine to make the read inactive. + require.Eventually(s.T(), func() bool { + rr := s.reader.(*InactiveTimeoutReader) + rr.mu.Lock() + defer rr.mu.Unlock() + return !rr.isActive + }, time.Second, 10*time.Millisecond, "Monitor did mark the reader inactive in time") + // 2nd fire will close the inactive reader. + s.simulatedClock.AdvanceTime(s.timeout + time.Millisecond) + // Wait for the monitor routine to close the wrapped reader. + require.Eventually(s.T(), func() bool { + rr := s.reader.(*InactiveTimeoutReader) + rr.mu.Lock() + defer rr.mu.Unlock() + return (rr.gcsReader == nil) + }, time.Second, 10*time.Millisecond, "Monitor did not close the reader in time") + reconnectErr := errors.New("failed to create new reader") + expectedReadHandle := s.initialFakeReader.Handle + s.mockBucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(req *gcs.ReadObjectRequest) bool { + return req.Name == s.object.Name && + req.Range.Start == 5 && // Expect reconnect from offset 5 + bytes.Equal(req.ReadHandle, expectedReadHandle) + })).Return(nil, reconnectErr).Times(1) // Expect only one call for the first failed attempt + + nFail, errFail := s.reader.Read(buf) + + // First failed reconnect attempt + s.ErrorIs(errFail, reconnectErr) + s.Equal(0, nFail) +} + +func (s *InactiveTimeoutReaderTestSuite) Test_Read_TimeoutAndSuccessfulReconnect() { + s.initialData = []byte("abcdefghijklmnopqrstuvwxyz") + s.timeout = 50 * time.Second + s.setupReader() + buf := make([]byte, 10) + n, err := s.reader.Read(buf) + s.Require().NoError(err) + s.Require().Equal(10, n) + s.Equal("abcdefghij", string(buf[:n])) + // First timeout fire will make the reader inactive. + s.simulatedClock.AdvanceTime(s.timeout + time.Millisecond) + // Wait for the monitor routine to make the read inactive. + require.Eventually(s.T(), func() bool { + rr := s.reader.(*InactiveTimeoutReader) + rr.mu.Lock() + defer rr.mu.Unlock() + return !rr.isActive + }, time.Second, 10*time.Millisecond, "Monitor did mark the reader inactive in time") + // 2nd fire will close the inactive reader. + s.simulatedClock.AdvanceTime(s.timeout + time.Millisecond) + // Wait for the monitor routine to close the wrapped reader. + require.Eventually(s.T(), func() bool { + rr := s.reader.(*InactiveTimeoutReader) + rr.mu.Lock() + defer rr.mu.Unlock() + return (rr.gcsReader == nil) + }, time.Second, 10*time.Millisecond, "Monitor did not close the reader in time") + expectedReadHandleAfterClose := s.initialFakeReader.Handle // The handle that should be stored after close + reconnectReadObjectRequest := &gcs.ReadObjectRequest{ + Name: s.object.Name, + Generation: s.object.Generation, + Range: &gcs.ByteRange{ + Start: uint64(10), // Expect reconnect from offset 10 + Limit: s.object.Size, + }, + ReadCompressed: s.object.HasContentEncodingGzip(), + ReadHandle: expectedReadHandleAfterClose, // Expect the stored handle to be used for reconnect + } + // Use the same initialFakeReader for simplicity, as it tracks the read offset internally. + s.mockBucket.On("NewReaderWithReadHandle", mock.Anything, reconnectReadObjectRequest).Return(s.initialFakeReader, nil).Times(1) + bufReconnect := make([]byte, 5) + + // Read after timeout (should trigger reconnect) + nReconnect, errReconnect := s.reader.Read(bufReconnect) + + s.Nil(errReconnect) + s.Equal(5, nReconnect) + s.Equal("klmno", string(bufReconnect[:nReconnect])) +} + +func (s *InactiveTimeoutReaderTestSuite) Test_Close_ExplicitClose() { + s.initialData = []byte("close me") + s.timeout = 100 * time.Millisecond + s.setupReader() + + err := s.reader.Close() + + s.NoError(err) + s.Nil(s.reader.(*InactiveTimeoutReader).gcsReader) + s.reader = nil // Prevent TearDownTest from closing again +} + +func (s *InactiveTimeoutReaderTestSuite) Test_handleTimeout_InactiveClose() { + s.initialData = []byte("simple close test") + s.timeout = 50 * time.Millisecond + s.readHandle = []byte("handle-before-close") + s.setupReader() // Sets up s.reader and s.initialFakeReader + expectedHandleAfterClose := []byte("handle-after-close") + s.initialFakeReader.Handle = expectedHandleAfterClose + itr := s.reader.(*InactiveTimeoutReader) + itr.isActive = false // Simulate inactivity + + itr.handleTimeout() + + s.Nil(itr.gcsReader) + s.False(itr.isActive, "isActive should remain false") + s.Equal(expectedHandleAfterClose, itr.readHandle, "readHandle should be updated from closed reader") +} + +func (s *InactiveTimeoutReaderTestSuite) Test_handleTimeout_ActiveBecomeInactive() { + s.initialData = []byte("simple close test") + s.timeout = 50 * time.Millisecond + s.readHandle = []byte("handle-before-close") + s.setupReader() // Sets up s.reader and s.initialFakeReader + expectedHandleAfterClose := []byte("handle-after-close") + s.initialFakeReader.Handle = expectedHandleAfterClose + itr := s.reader.(*InactiveTimeoutReader) + itr.isActive = true + + itr.handleTimeout() + + s.NotNil(itr.gcsReader) + s.False(itr.isActive, "isActive become false") +} + +func (s *InactiveTimeoutReaderTestSuite) Test_closeGCSReader_NilReader() { + s.initialData = []byte("simple close test") + s.timeout = 50 * time.Millisecond + s.readHandle = []byte("handle-before-close") + s.setupReader() // Sets up s.reader and s.initialFakeReader + itr := s.reader.(*InactiveTimeoutReader) + itr.gcsReader = nil + + itr.closeGCSReader() + + s.Nil(itr.gcsReader) + s.Equal(s.readHandle, itr.readHandle) +} + +func (s *InactiveTimeoutReaderTestSuite) Test_closeGCSReader_NonNilReader() { + s.initialData = []byte("simple close test") + s.timeout = 50 * time.Millisecond + s.readHandle = []byte("handle-before-close") + s.setupReader() // Sets up s.reader and s.initialFakeReader + expectedHandleAfterClose := []byte("handle-after-close") + s.initialFakeReader.Handle = expectedHandleAfterClose + itr := s.reader.(*InactiveTimeoutReader) + + itr.closeGCSReader() + + s.Nil(itr.gcsReader) + s.Equal(expectedHandleAfterClose, itr.readHandle, "readHandle should be updated from closed reader") +} + +func (s *InactiveTimeoutReaderTestSuite) TestRaceCondition() { + var wg sync.WaitGroup + wg.Add(2) + s.initialData = []byte(strings.Repeat("abc", 1000)) + s.timeout = time.Second + s.readHandle = []byte("handle-before-close") + s.setupReader() // Sets up s.reader and s.initialFakeReader + + // Read() + go func() { + defer wg.Done() + // Read the complete object with buffer size 100. + for offset := 0; offset < int(s.object.Size); offset += 100 { + rc := &fake.FakeReader{ReadCloser: getReadCloser(s.initialData[offset:])} + s.mockBucket.On("NewReaderWithReadHandle", mock.Anything, mock.Anything).Return(rc, nil).Maybe() + buf := make([]byte, 100) + + n, err := s.reader.Read(buf) + + s.NoError(err) + s.Equal(100, n) + } + }() + + // Concurrent handleTimeout. + go func() { + defer wg.Done() + for range 1000 { + s.reader.(*InactiveTimeoutReader).handleTimeout() + } + }() + + wg.Wait() +} diff --git a/internal/gcsx/integration_test.go b/internal/gcsx/integration_test.go index 7addfb20fc..dffbe5d060 100644 --- a/internal/gcsx/integration_test.go +++ b/internal/gcsx/integration_test.go @@ -24,12 +24,12 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/fake" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" "golang.org/x/net/context" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/ogletest" "github.com/jacobsa/timeutil" @@ -79,17 +79,21 @@ func init() { RegisterTestSuite(&IntegrationTest{}) } func (t *IntegrationTest) SetUp(ti *TestInfo) { t.ctx = ti.Ctx - t.bucket = fake.NewFakeBucket(&t.clock, "some_bucket", gcs.NonHierarchical) + t.bucket = fake.NewFakeBucket(&t.clock, "some_bucket", gcs.BucketType{}) // Set up a fixed, non-zero time. t.clock.SetTime(time.Date(2012, 8, 15, 22, 56, 0, 0, time.Local)) // Set up the syncer. const appendThreshold = 0 + const chunkRetryDeadlineSecs = 120 + const chunkTransferTimeoutSecs = 10 const tmpObjectPrefix = ".gcsfuse_tmp/" t.syncer = gcsx.NewSyncer( appendThreshold, + chunkRetryDeadlineSecs, + chunkTransferTimeoutSecs, tmpObjectPrefix, t.bucket) } @@ -102,7 +106,7 @@ func (t *IntegrationTest) TearDown() { func (t *IntegrationTest) create(o *gcs.Object) { // Set up a reader. - rc, err := t.bucket.NewReader( + rc, err := t.bucket.NewReaderWithReadHandle( t.ctx, &gcs.ReadObjectRequest{ Name: o.Name, diff --git a/internal/gcsx/mock_random_reader.go b/internal/gcsx/mock_random_reader.go new file mode 100644 index 0000000000..8812790e8e --- /dev/null +++ b/internal/gcsx/mock_random_reader.go @@ -0,0 +1,45 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx + +import ( + "context" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/stretchr/testify/mock" +) + +type MockRandomReader struct { + RandomReader + mock.Mock +} + +func (m *MockRandomReader) ReadAt(ctx context.Context, dst []byte, offset int64) (ObjectData, error) { + args := m.Called(ctx, dst, offset) + return args.Get(0).(ObjectData), args.Error(1) +} + +func (m *MockRandomReader) Object() *gcs.MinObject { + args := m.Called() + return args.Get(0).(*gcs.MinObject) +} + +func (m *MockRandomReader) Destroy() { + m.Called() +} + +func (m *MockRandomReader) CheckInvariants() { + m.Called() +} diff --git a/internal/gcsx/mock_reader.go b/internal/gcsx/mock_reader.go new file mode 100644 index 0000000000..0b2d5db0e9 --- /dev/null +++ b/internal/gcsx/mock_reader.go @@ -0,0 +1,42 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx + +import ( + "context" + + "github.com/stretchr/testify/mock" +) + +type MockReader struct { + mock.Mock +} + +func (m *MockReader) ReaderName() string { + return "mock_reader" +} + +func (m *MockReader) ReadAt(ctx context.Context, req *ReadRequest) (ReadResponse, error) { + args := m.Called(ctx, req) + return args.Get(0).(ReadResponse), args.Error(1) +} + +func (m *MockReader) Destroy() { + m.Called() +} + +func (m *MockReader) CheckInvariants() { + m.Called() +} diff --git a/internal/gcsx/mrd_instance.go b/internal/gcsx/mrd_instance.go new file mode 100644 index 0000000000..1ea55f6b22 --- /dev/null +++ b/internal/gcsx/mrd_instance.go @@ -0,0 +1,414 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx + +import ( + "bytes" + "context" + "fmt" + "io" + "strconv" + "sync" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/monitor" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/jacobsa/fuse/fuseops" +) + +const mrdPoolCloseTimeout = 120 * time.Second + +// MrdInstance manages a pool of Multi-Range Downloader (MRD) instances for a +// single file inode. It handles the lifecycle of the MRD pool, including +// creation, destruction, and caching. +type MrdInstance struct { + // mrdPool holds the pool of MultiRangeDownloader instances. + mrdPool *MRDPool + // inodeId is the ID of the file inode associated with this instance. + inodeId fuseops.InodeID + // object is the GCS object for which the downloaders are created. + object *gcs.MinObject + // bucket is the GCS bucket containing the object. + bucket gcs.Bucket + // refCount tracks the number of active users of this instance. + refCount int64 + // refCountMu protects access to refCount. + refCountMu sync.Mutex + // poolMu protects access to mrdPool. + poolMu sync.RWMutex + // mrdCache is a shared cache for inactive MrdInstance objects. + mrdCache *lru.Cache + // holds config specified by the user using config-file flag and CLI flags. + config *cfg.Config +} + +// NewMrdInstance creates a new MrdInstance for a given GCS object. +func NewMrdInstance(obj *gcs.MinObject, bucket gcs.Bucket, cache *lru.Cache, inodeId fuseops.InodeID, config *cfg.Config) *MrdInstance { + mrdInstance := MrdInstance{ + bucket: bucket, + mrdCache: cache, + inodeId: inodeId, + config: config, + object: obj, + } + return &mrdInstance +} + +// SetMinObject sets the gcs.MinObject stored in the MrdInstance to passed value, only if it's non nil. +// If the generation of the object has changed, it recreates the MRD pool to ensure consistency. +func (mi *MrdInstance) SetMinObject(minObj *gcs.MinObject) error { + if minObj == nil { + return fmt.Errorf("MrdInstance::SetMinObject: Missing MinObject") + } + + mi.poolMu.Lock() + defer mi.poolMu.Unlock() + + oldObj := mi.object + // No need to create a new pool if the pool does not exist. + // If generation matches, just update the object (e.g. for size updates) and return. + if mi.mrdPool == nil || (oldObj != nil && oldObj.Generation == minObj.Generation) { + mi.object = minObj + return nil + } + + // Generations differ, need to create and swap a new pool. + if err := mi.createAndSwapPool(minObj); err != nil { + return fmt.Errorf("MrdInstance::SetMinObject: failed to create and swap pool: %w", err) + } + + return nil +} + +// createAndSwapPool creates a new MRD pool and swaps it with the existing one. +// It also updates the minObject with the passed object & closes the old pool. +// LOCKS_REQUIRED(mi.poolMu). +func (mi *MrdInstance) createAndSwapPool(obj *gcs.MinObject) error { + newPool, err := NewMRDPool(&MRDPoolConfig{PoolSize: int(mi.config.Mrd.PoolSize), object: obj, bucket: mi.bucket}, nil) + if err != nil { + return err + } + + oldPool := mi.mrdPool + mi.mrdPool = newPool + mi.object = obj + + closePoolWithTimeout(oldPool, "MrdInstance::createAndSwapPool", mrdPoolCloseTimeout) + return nil +} + +// GetMinObject returns the gcs.MinObject stored in MrdInstance. Used only for unit testing. +func (mi *MrdInstance) GetMinObject() *gcs.MinObject { + mi.poolMu.RLock() + defer mi.poolMu.RUnlock() + return mi.object +} + +// getMRDEntry returns a valid MRDEntry from the pool. +// It ensures the pool is initialized and the returned entry is in a usable state, +// recreating it if necessary. +func (mi *MrdInstance) getMRDEntry() (*MRDEntry, error) { + // Ensure the pool is initialized. + if err := mi.ensureMRDPool(); err != nil { + return nil, err + } + + mi.poolMu.RLock() + defer mi.poolMu.RUnlock() + if mi.mrdPool == nil { + return nil, fmt.Errorf("MrdInstance::getMRDEntry: MRDPool is nil") + } + + entry := mi.mrdPool.Next() + + // Check if the entry is valid. + entry.mu.RLock() + isValid := entry.mrd != nil && entry.mrd.Error() == nil + entry.mu.RUnlock() + + // Recreate the entry if it is not valid. + if !isValid { + if err := mi.mrdPool.RecreateMRD(entry, nil); err != nil { + return nil, fmt.Errorf("MrdInstance::getMRDEntry: failed to recreate MRD: %w", err) + } + } + return entry, nil +} + +// Read downloads data from the GCS object into the provided buffer starting at the offset. +// It handles the details of selecting a valid MRD entry, locking it, and waiting for the async download to complete. +func (mi *MrdInstance) Read(ctx context.Context, p []byte, offset int64, metrics metrics.MetricHandle) (int, error) { + if len(p) == 0 { + return 0, nil + } + + entry, err := mi.getMRDEntry() + if err != nil { + return 0, err + } + + // Prepare the buffer for the read operation. + // bytes.NewBuffer creates a buffer using p as the initial content. + // Reset() sets the length to 0 but keeps the capacity, allowing writes to fill p. + buffer := bytes.NewBuffer(p) + buffer.Reset() + done := make(chan readResult, 1) + + // Take Read lock before using MRD for reading. + entry.mu.RLock() + if entry.mrd == nil { + entry.mu.RUnlock() + return 0, fmt.Errorf("MrdInstance::Read: mrd is nil") + } + + start := time.Now() + defer monitor.CaptureMultiRangeDownloaderMetrics(ctx, metrics, "MultiRangeDownloader::Add", start) + entry.mrd.Add(buffer, offset, int64(len(p)), func(offsetAddCallback int64, bytesReadAddCallback int64, e error) { + done <- readResult{bytesRead: int(bytesReadAddCallback), err: e} + }) + entry.mu.RUnlock() + + if !mi.config.FileSystem.IgnoreInterrupts { + select { + case <-ctx.Done(): + return 0, ctx.Err() + case res := <-done: + if res.err != nil && res.err != io.EOF { + return res.bytesRead, fmt.Errorf("Error in Add call: %w", res.err) + } + return res.bytesRead, res.err + } + } else { + res := <-done + if res.err != nil && res.err != io.EOF { + return res.bytesRead, fmt.Errorf("Error in Add call: %w", res.err) + } + return res.bytesRead, res.err + } +} + +// ensureMRDPool ensures that the MRD pool is initialized. If the pool +// already exists, this function is a no-op. +func (mi *MrdInstance) ensureMRDPool() (err error) { + // Return early if pool exists. + mi.poolMu.RLock() + if mi.mrdPool != nil { + mi.poolMu.RUnlock() + return + } + mi.poolMu.RUnlock() + + mi.poolMu.Lock() + defer mi.poolMu.Unlock() + + // Re-check under write lock to handle race condition. + if mi.mrdPool != nil { + return + } + + // Creating a new pool. Not reusing any handle while creating a new pool. + mi.mrdPool, err = NewMRDPool(&MRDPoolConfig{PoolSize: int(mi.config.Mrd.PoolSize), object: mi.object, bucket: mi.bucket}, nil) + if err != nil { + err = fmt.Errorf("MrdInstance::ensureMRDPool Error in creating MRDPool: %w", err) + } + return +} + +// RecreateMRD recreates the entire MRD pool. This is typically called by the +// file inode when the backing GCS object's generation changes, invalidating +// all existing downloader instances. +func (mi *MrdInstance) RecreateMRD() error { + mi.poolMu.Lock() + defer mi.poolMu.Unlock() + + obj := mi.object + if obj == nil { + return fmt.Errorf("MrdInstance::RecreateMRD: object is nil") + } + + if err := mi.createAndSwapPool(obj); err != nil { + return fmt.Errorf("MrdInstance::RecreateMRD: failed to create new pool: %w", err) + } + + return nil +} + +// closePool closes all MRD instances in the pool and releases associated resources. +func (mi *MrdInstance) closePool() { + mi.poolMu.Lock() + defer mi.poolMu.Unlock() + pool := mi.mrdPool + mi.mrdPool = nil + closePoolWithTimeout(pool, "MrdInstance::closePool", mrdPoolCloseTimeout) +} + +// closePoolWithTimeout closes the given MRD pool in a separate goroutine with a timeout. +// If closing the pool takes longer than the specified timeout, a warning is logged. +func closePoolWithTimeout(pool *MRDPool, caller string, timeout time.Duration) { + if pool == nil { + return + } + + go func() { + done := make(chan struct{}) + go func() { + defer close(done) + pool.Close() + }() + + timer := time.NewTimer(timeout) + defer timer.Stop() + + select { + case <-done: + case <-timer.C: + var objectName string + if pool.poolConfig != nil && pool.poolConfig.object != nil { + objectName = pool.poolConfig.object.Name + } + logger.Warnf("%s: MRDPool.Close() timed out after %v for object %s", caller, timeout, objectName) + } + }() +} + +// Destroy completely destroys the MrdInstance, cleaning up +// its resources and ensuring it is removed from the cache. This should be +// called when the owning inode is destroyed. +func (mi *MrdInstance) Destroy() { + mi.refCountMu.Lock() + defer mi.refCountMu.Unlock() + + // If it's in use, this indicates a potential lifecycle mismatch between the + // inode and its readers. + if mi.refCount > 0 { + logger.Warnf("MrdInstance::Destroy called on an instance with refCount %d", mi.refCount) + } + + // Remove from cache. + if mi.mrdCache != nil { + mi.mrdCache.Erase(getKey(mi.inodeId)) + } + + // Close the pool. + mi.closePool() +} + +// getKey generates a unique key for the MrdInstance based on its inode ID. +func getKey(id fuseops.InodeID) string { + return strconv.FormatUint(uint64(id), 10) +} + +// IncrementRefCount increases the reference count for the MrdInstance. When the +// instance is actively used (refCount > 0), it is removed from the inactive +// MRD cache to prevent eviction. +func (mi *MrdInstance) IncrementRefCount() { + mi.refCountMu.Lock() + defer mi.refCountMu.Unlock() + mi.refCount++ + + if mi.refCount == 1 && mi.mrdCache != nil { + // Remove from cache + deletedEntry := mi.mrdCache.Erase(getKey(mi.inodeId)) + if deletedEntry != nil { + logger.Tracef("MrdInstance::IncrementRefCount: MrdInstance Inode (%d) erased from cache", mi.inodeId) + } + } +} + +// handleEviction handles the cleanup of the MrdInstance when it is evicted from the cache. +// Race protection: MrdInstance could be reopened (refCount>0) or re-added to cache before eviction. +func (mi *MrdInstance) handleEviction() { + mi.refCountMu.Lock() + defer mi.refCountMu.Unlock() + + // Check if mrdInstance was reopened (refCount>0) - must skip eviction. + if mi.refCount > 0 { + return + } + + // Check if mrdInstance was re-added to cache (refCount went 0→1→0 in between eviction and closure.) + // Lock order: refCountMu -> cache.mu (consistent with Increment/DecrementRefCount) + if mi.mrdCache != nil && mi.mrdCache.LookUpWithoutChangingOrder(getKey(mi.inodeId)) == mi { + return + } + + mi.closePool() +} + +// DecrementRefCount decreases the reference count. When the count drops to zero, the +// instance is considered inactive and is added to the LRU cache for potential +// reuse. If the cache is full, this may trigger the eviction and closure of the +// least recently used MRD instances. +func (mi *MrdInstance) DecrementRefCount() { + mi.refCountMu.Lock() + defer mi.refCountMu.Unlock() + + if mi.refCount <= 0 { + logger.Errorf("MrdInstance::DecrementRefCount: Refcount cannot be negative") + return + } + + mi.refCount-- + // Do nothing if MRDInstance is in use or cache is not enabled. + if mi.refCount > 0 || mi.mrdCache == nil { + return + } + + // Add to cache. + // Lock order: refCountMu -> cache.mu -> poolMu (via Size() inside Insert). + // This is a safe order. + evictedValues, err := mi.mrdCache.Insert(getKey(mi.inodeId), mi) + if err != nil { + logger.Errorf("MrdInstance::DecrementRefCount: Failed to insert MrdInstance for inode (%d) into cache, destroying immediately: %v", mi.inodeId, err) + // The instance could not be inserted into the cache. Since the refCount is 0, + // we must close it now to prevent it from being leaked. + mi.closePool() + return + } + + logger.Tracef("MrdInstance::DecrementRefCount: MrdInstance for inode (%d) added to cache", mi.inodeId) + + // Do not proceed if no eviction happened. + if evictedValues == nil { + return + } + + // Evict outside all locks. + mi.refCountMu.Unlock() + for _, instance := range evictedValues { + mrdInstance, ok := instance.(*MrdInstance) + if !ok { + logger.Errorf("MrdInstance::DecrementRefCount: Invalid value type, expected *MrdInstance, got %T", instance) + } else { + mrdInstance.handleEviction() + } + } + // Reacquire the lock ensuring safe defer's Unlock. + mi.refCountMu.Lock() +} + +// Size returns the number of active MRDs. +func (mi *MrdInstance) Size() uint64 { + mi.poolMu.RLock() + defer mi.poolMu.RUnlock() + if mi.mrdPool != nil { + return mi.mrdPool.Size() + } + return 0 +} diff --git a/internal/gcsx/mrd_instance_test.go b/internal/gcsx/mrd_instance_test.go new file mode 100644 index 0000000000..94138aa414 --- /dev/null +++ b/internal/gcsx/mrd_instance_test.go @@ -0,0 +1,974 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx + +import ( + "bytes" + "context" + "fmt" + "os" + "strconv" + "sync" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/jacobsa/fuse/fuseops" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type MrdInstanceTest struct { + suite.Suite + object *gcs.MinObject + bucket *storage.TestifyMockBucket + cache *lru.Cache + inodeID fuseops.InodeID + config *cfg.Config + mrdInstance *MrdInstance +} + +func TestMrdInstanceTestSuite(t *testing.T) { + suite.Run(t, new(MrdInstanceTest)) +} + +func (t *MrdInstanceTest) SetupTest() { + t.object = &gcs.MinObject{ + Name: "foo", + Size: 1024 * MiB, + Generation: 1234, + } + t.bucket = new(storage.TestifyMockBucket) + t.cache = lru.NewCache(2) // Small cache size for testing eviction + t.inodeID = 100 + t.config = &cfg.Config{Mrd: cfg.MrdConfig{PoolSize: 1}} + + t.mrdInstance = NewMrdInstance(t.object, t.bucket, t.cache, t.inodeID, t.config) +} + +func (t *MrdInstanceTest) TestNewMrdInstance() { + assert.Equal(t.T(), t.object, t.mrdInstance.object) + assert.Equal(t.T(), t.bucket, t.mrdInstance.bucket) + assert.Equal(t.T(), t.cache, t.mrdInstance.mrdCache) + assert.Equal(t.T(), t.inodeID, t.mrdInstance.inodeId) + assert.Equal(t.T(), t.config, t.mrdInstance.config) + assert.Nil(t.T(), t.mrdInstance.mrdPool) + assert.Equal(t.T(), int64(0), t.mrdInstance.refCount) +} + +func (t *MrdInstanceTest) TestRead_Success() { + data := []byte("hello world") + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, data) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + buf := make([]byte, 5) + + n, err := t.mrdInstance.Read(context.Background(), buf, 0, metrics.NewNoopMetrics()) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), 5, n) + assert.Equal(t.T(), "hello", string(buf)) +} + +func (t *MrdInstanceTest) TestRead_InitializesPool() { + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + assert.Nil(t.T(), t.mrdInstance.mrdPool) + buf := make([]byte, 1) + + _, err := t.mrdInstance.Read(context.Background(), buf, 0, metrics.NewNoopMetrics()) + + assert.NoError(t.T(), err) + assert.NotNil(t.T(), t.mrdInstance.mrdPool) +} + +func (t *MrdInstanceTest) TestRead_RecreatesInvalidEntry() { + fakeMRD1 := fake.NewFakeMultiRangeDownloader(t.object, nil) + fakeMRD2 := fake.NewFakeMultiRangeDownloader(t.object, nil) + // Initial creation + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD1, nil).Once() + buf := make([]byte, 1) + _, err := t.mrdInstance.Read(context.Background(), buf, 0, metrics.NewNoopMetrics()) + assert.NoError(t.T(), err) + + // Manually invalidate the entry to simulate a failure + entry := &t.mrdInstance.mrdPool.entries[0] + entry.mrd.Close() // Close it. + // Replace the entry's MRD with one that returns an error. + entry.mu.Lock() + entry.mrd = fake.NewFakeMultiRangeDownloaderWithStatusError(t.object, nil, fmt.Errorf("broken")) + entry.mu.Unlock() + + // Expect recreation + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD2, nil).Once() + + _, err = t.mrdInstance.Read(context.Background(), buf, 0, metrics.NewNoopMetrics()) + + assert.NoError(t.T(), err) + entry.mu.RLock() + assert.Equal(t.T(), fakeMRD2, entry.mrd) + entry.mu.RUnlock() +} + +func (t *MrdInstanceTest) TestRead_EnsureFails() { + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(nil, fmt.Errorf("init error")).Once() + assert.Nil(t.T(), t.mrdInstance.mrdPool) + buf := make([]byte, 1) + + n, err := t.mrdInstance.Read(context.Background(), buf, 0, metrics.NewNoopMetrics()) + + assert.Error(t.T(), err) + assert.Contains(t.T(), err.Error(), "init error") + assert.Equal(t.T(), 0, n) +} + +func (t *MrdInstanceTest) TestRead_RecreationFails() { + fakeMRD1 := fake.NewFakeMultiRangeDownloader(t.object, nil) + // Initial creation. + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD1, nil).Once() + buf := make([]byte, 1) + _, err := t.mrdInstance.Read(context.Background(), buf, 0, metrics.NewNoopMetrics()) + assert.NoError(t.T(), err) + + // Manually invalidate the entry to simulate a failure. + entry := &t.mrdInstance.mrdPool.entries[0] + entry.mrd.Close() // Close it. + // Replace the entry's MRD with one that returns an error. + entry.mu.Lock() + entry.mrd = fake.NewFakeMultiRangeDownloaderWithStatusError(t.object, nil, fmt.Errorf("broken")) + entry.mu.Unlock() + + // Expect recreation failure. + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(nil, fmt.Errorf("recreate error")).Once() + + n, err := t.mrdInstance.Read(context.Background(), buf, 0, metrics.NewNoopMetrics()) + + assert.Error(t.T(), err) + assert.Contains(t.T(), err.Error(), "recreate error") + assert.Equal(t.T(), 0, n) +} + +func (t *MrdInstanceTest) TestRead_EmptyBuffer() { + n, err := t.mrdInstance.Read(context.Background(), []byte{}, 0, metrics.NewNoopMetrics()) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), 0, n) +} + +func (t *MrdInstanceTest) TestRead_ContextCancelled() { + data := []byte("hello world") + fakeMRD := fake.NewFakeMultiRangeDownloaderWithSleep(t.object, data, 100*time.Millisecond) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + ctx, cancel := context.WithCancel(context.Background()) + buf := make([]byte, 5) + + cancel() + n, err := t.mrdInstance.Read(ctx, buf, 0, metrics.NewNoopMetrics()) + + assert.Error(t.T(), err) + assert.Equal(t.T(), context.Canceled, err) + assert.Equal(t.T(), 0, n) +} + +func (t *MrdInstanceTest) TestRead_AddError() { + fakeMRD := fake.NewFakeMultiRangeDownloaderWithSleepAndDefaultError(t.object, []byte("data"), 0, fmt.Errorf("read error")) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + buf := make([]byte, 5) + + n, err := t.mrdInstance.Read(context.Background(), buf, 0, metrics.NewNoopMetrics()) + + assert.Error(t.T(), err) + assert.Contains(t.T(), err.Error(), "read error") + assert.Equal(t.T(), 0, n) +} + +func (t *MrdInstanceTest) TestGetMRDEntry_Success() { + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + + entry, err := t.mrdInstance.getMRDEntry() + + assert.NoError(t.T(), err) + assert.NotNil(t.T(), entry) + assert.Equal(t.T(), fakeMRD, entry.mrd) +} + +func (t *MrdInstanceTest) TestGetMRDEntry_EnsureFails() { + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(nil, fmt.Errorf("init error")).Once() + + entry, err := t.mrdInstance.getMRDEntry() + + assert.Error(t.T(), err) + assert.Nil(t.T(), entry) + assert.Contains(t.T(), err.Error(), "init error") +} + +func (t *MrdInstanceTest) TestGetMRDEntry_RecreatesInvalidMRD() { + fakeMRD1 := fake.NewFakeMultiRangeDownloader(t.object, nil) + fakeMRD2 := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD1, nil).Once() + entry1, err := t.mrdInstance.getMRDEntry() + assert.NoError(t.T(), err) + assert.Equal(t.T(), fakeMRD1, entry1.mrd) + entry1.mrd.Close() + entry1.mu.Lock() + entry1.mrd = fake.NewFakeMultiRangeDownloaderWithStatusError(t.object, nil, fmt.Errorf("broken")) + entry1.mu.Unlock() + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD2, nil).Once() + + // This should force recreation of the MRD. + entry2, err := t.mrdInstance.getMRDEntry() + + assert.NoError(t.T(), err) + assert.Equal(t.T(), fakeMRD2, entry2.mrd) +} + +func (t *MrdInstanceTest) TestRecreateMRD() { + fakeMRD1 := fake.NewFakeMultiRangeDownloader(t.object, nil) + fakeMRD2 := fake.NewFakeMultiRangeDownloader(t.object, nil) + // Initial creation + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD1, nil).Once() + buf := make([]byte, 1) + _, err := t.mrdInstance.Read(context.Background(), buf, 0, metrics.NewNoopMetrics()) + assert.NoError(t.T(), err) + pool1 := t.mrdInstance.mrdPool + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD2, nil).Once() + + // Recreate + err = t.mrdInstance.RecreateMRD() + + assert.NoError(t.T(), err) + assert.NotNil(t.T(), t.mrdInstance.mrdPool) + assert.NotSame(t.T(), pool1, t.mrdInstance.mrdPool) +} + +func (t *MrdInstanceTest) TestDestroy() { + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + buf := make([]byte, 1) + _, err := t.mrdInstance.Read(context.Background(), buf, 0, metrics.NewNoopMetrics()) + assert.NoError(t.T(), err) + assert.NotNil(t.T(), t.mrdInstance.mrdPool) + + t.mrdInstance.Destroy() + + assert.Nil(t.T(), t.mrdInstance.mrdPool) +} + +func (t *MrdInstanceTest) TestIncrementRefCount() { + // Setup: Put something in cache first to verify removal + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + buf := make([]byte, 1) + _, err := t.mrdInstance.Read(context.Background(), buf, 0, metrics.NewNoopMetrics()) + assert.NoError(t.T(), err) + // Manually insert into cache to simulate it being inactive + key := strconv.FormatUint(uint64(t.inodeID), 10) + _, err = t.cache.Insert(key, t.mrdInstance) + assert.NoError(t.T(), err) + assert.NotNil(t.T(), t.cache.LookUpWithoutChangingOrder(key)) + + t.mrdInstance.IncrementRefCount() + + assert.Equal(t.T(), int64(1), t.mrdInstance.refCount) + assert.Nil(t.T(), t.cache.LookUpWithoutChangingOrder(key)) +} + +func (t *MrdInstanceTest) TestDecrementRefCount() { + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + buf := make([]byte, 1) + _, err := t.mrdInstance.Read(context.Background(), buf, 0, metrics.NewNoopMetrics()) + assert.NoError(t.T(), err) + t.mrdInstance.refCount = 1 + + t.mrdInstance.DecrementRefCount() + + assert.Equal(t.T(), int64(0), t.mrdInstance.refCount) + key := strconv.FormatUint(uint64(t.inodeID), 10) + assert.NotNil(t.T(), t.cache.LookUpWithoutChangingOrder(key)) +} + +func (t *MrdInstanceTest) TestDecrementRefCount_Eviction() { + // Fill cache with other items + localMrdInstance := &MrdInstance{mrdPool: &MRDPool{poolConfig: &MRDPoolConfig{PoolSize: 1}, stopCreation: make(chan struct{})}} + _, err := t.cache.Insert("other1", localMrdInstance) + assert.NoError(t.T(), err) + _, err = t.cache.Insert("other2", localMrdInstance) + assert.NoError(t.T(), err) + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + buf := make([]byte, 1) + _, err = t.mrdInstance.Read(context.Background(), buf, 0, metrics.NewNoopMetrics()) + assert.NoError(t.T(), err) + t.mrdInstance.refCount = 1 + + // This should trigger eviction of "other1" (LRU) + t.mrdInstance.DecrementRefCount() + + assert.Equal(t.T(), int64(0), t.mrdInstance.refCount) + key := strconv.FormatUint(uint64(t.inodeID), 10) + assert.NotNil(t.T(), t.cache.LookUpWithoutChangingOrder(key)) + assert.Nil(t.T(), t.cache.LookUpWithoutChangingOrder("other1")) + assert.NotNil(t.T(), t.cache.LookUpWithoutChangingOrder("other2")) +} + +func (t *MrdInstanceTest) TestGetKey() { + testCases := []struct { + inodeID fuseops.InodeID + expected string + }{ + {0, "0"}, + {123, "123"}, + {18446744073709551615, "18446744073709551615"}, // Max uint64 + } + + for _, tc := range testCases { + assert.Equal(t.T(), tc.expected, getKey(tc.inodeID)) + } +} + +func (t *MrdInstanceTest) TestEnsureMRDPool_Success() { + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + assert.Nil(t.T(), t.mrdInstance.mrdPool) + + err := t.mrdInstance.ensureMRDPool() + + assert.NoError(t.T(), err) + assert.NotNil(t.T(), t.mrdInstance.mrdPool) +} + +func (t *MrdInstanceTest) TestEnsureMRDPool_AlreadyExists() { + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + err := t.mrdInstance.ensureMRDPool() + assert.NoError(t.T(), err) + pool := t.mrdInstance.mrdPool + + // Call again + err = t.mrdInstance.ensureMRDPool() + + assert.NoError(t.T(), err) + assert.Equal(t.T(), pool, t.mrdInstance.mrdPool) + t.bucket.AssertExpectations(t.T()) // Should only be called once +} + +func (t *MrdInstanceTest) TestEnsureMRDPool_Failure() { + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(nil, fmt.Errorf("init error")).Once() + assert.Nil(t.T(), t.mrdInstance.mrdPool) + + err := t.mrdInstance.ensureMRDPool() + + assert.Error(t.T(), err) + assert.Nil(t.T(), t.mrdInstance.mrdPool) + assert.Contains(t.T(), err.Error(), "init error") +} + +func (t *MrdInstanceTest) TestSize() { + assert.Equal(t.T(), uint64(0), t.mrdInstance.Size()) + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + buf := make([]byte, 1) + _, err := t.mrdInstance.Read(context.Background(), buf, 0, metrics.NewNoopMetrics()) + assert.NoError(t.T(), err) + + poolSize := t.mrdInstance.Size() + + // Pool size is 1 based on SetupTest config (PoolSize: 1) + assert.Equal(t.T(), uint64(1), poolSize) + t.mrdInstance.Destroy() + assert.Equal(t.T(), uint64(0), t.mrdInstance.Size()) +} + +func (t *MrdInstanceTest) TestDestroy_RemovesFromCache() { + // Manually insert into cache + key := getKey(t.inodeID) + _, err := t.cache.Insert(key, t.mrdInstance) + assert.NoError(t.T(), err) + assert.NotNil(t.T(), t.cache.LookUpWithoutChangingOrder(key)) + + t.mrdInstance.Destroy() + + assert.Nil(t.T(), t.cache.LookUpWithoutChangingOrder(key)) +} + +func (t *MrdInstanceTest) TestDestroy_WithRefCount() { + t.mrdInstance.refCount = 1 + // Should log warning but proceed to destroy + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + err := t.mrdInstance.ensureMRDPool() + assert.NoError(t.T(), err) + assert.NotNil(t.T(), t.mrdInstance.mrdPool) + // Capture logs to verify error message + var buf bytes.Buffer + logger.SetOutput(&buf) + defer logger.SetOutput(os.Stdout) + + t.mrdInstance.Destroy() + + assert.Nil(t.T(), t.mrdInstance.mrdPool) + assert.Contains(t.T(), buf.String(), "MrdInstance::Destroy called on an instance with refCount 1") +} + +func (t *MrdInstanceTest) TestDecrementRefCount_Negative() { + t.mrdInstance.refCount = 0 + // Capture logs to verify error message + var buf bytes.Buffer + logger.SetOutput(&buf) + defer logger.SetOutput(os.Stdout) + + // Should log error and return, not panic. RefCount should remain 0. + t.mrdInstance.DecrementRefCount() + + assert.Equal(t.T(), int64(0), t.mrdInstance.refCount) + assert.Contains(t.T(), buf.String(), "MrdInstance::DecrementRefCount: Refcount cannot be negative") +} + +func (t *MrdInstanceTest) TestDecrementRefCount_CacheInsertFailure() { + // Create a cache with capacity 1 + smallCache := lru.NewCache(1) + // Create instance with pool size 2 (so Size() returns 2). + config := &cfg.Config{Mrd: cfg.MrdConfig{PoolSize: 2}} + mi := NewMrdInstance(t.object, t.bucket, smallCache, t.inodeID, config) + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil) + // Initialize pool. + err := mi.ensureMRDPool() + assert.NoError(t.T(), err) + mi.refCount = 1 + // Capture logs to verify error message + var buf bytes.Buffer + logger.SetOutput(&buf) + defer logger.SetOutput(os.Stdout) + + // This should fail to insert into cache (Size 2 > Cap 1) and should close the pool instantly. + mi.DecrementRefCount() + + assert.Equal(t.T(), int64(0), mi.refCount) + mi.poolMu.RLock() + assert.Nil(t.T(), mi.mrdPool) + mi.poolMu.RUnlock() + assert.Contains(t.T(), buf.String(), "Failed to insert MrdInstance") +} + +func (t *MrdInstanceTest) TestClosePool() { + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + err := t.mrdInstance.ensureMRDPool() + assert.NoError(t.T(), err) + assert.NotNil(t.T(), t.mrdInstance.mrdPool) + + t.mrdInstance.closePool() + + t.mrdInstance.poolMu.RLock() + assert.Nil(t.T(), t.mrdInstance.mrdPool) + t.mrdInstance.poolMu.RUnlock() +} + +func (t *MrdInstanceTest) TestHandleEviction_Resurrected() { + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + // Initialize pool. + err := t.mrdInstance.ensureMRDPool() + assert.NoError(t.T(), err) + // Simulate resurrection (refCount > 0). + t.mrdInstance.refCount = 1 + + t.mrdInstance.handleEviction() + + // Pool should still exist because refCount > 0. + t.mrdInstance.poolMu.RLock() + assert.NotNil(t.T(), t.mrdInstance.mrdPool) + t.mrdInstance.poolMu.RUnlock() +} + +func (t *MrdInstanceTest) TestHandleEviction_ReAddedToCache() { + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + // Initialize pool. + err := t.mrdInstance.ensureMRDPool() + assert.NoError(t.T(), err) + // Add to cache to simulate it being re-added concurrently. + key := getKey(t.inodeID) + _, err = t.cache.Insert(key, t.mrdInstance) + assert.NoError(t.T(), err) + // refCount is 0, but it is in the cache. + t.mrdInstance.refCount = 0 + + t.mrdInstance.handleEviction() + + // Pool should still exist because it's in the cache. + t.mrdInstance.poolMu.RLock() + assert.NotNil(t.T(), t.mrdInstance.mrdPool) + t.mrdInstance.poolMu.RUnlock() +} + +func (t *MrdInstanceTest) TestHandleEviction_SafeToClose() { + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + // Initialize pool. + err := t.mrdInstance.ensureMRDPool() + assert.NoError(t.T(), err) + // Ensure not in cache. + key := getKey(t.inodeID) + t.cache.Erase(key) + // refCount is 0. + t.mrdInstance.refCount = 0 + + t.mrdInstance.handleEviction() + + // Pool should be closed (nil). + t.mrdInstance.poolMu.RLock() + assert.Nil(t.T(), t.mrdInstance.mrdPool) + t.mrdInstance.poolMu.RUnlock() +} + +func (t *MrdInstanceTest) TestCreateAndSwapPool_Success() { + // Setup initial state + initialObj := &gcs.MinObject{Name: "old", Generation: 1} + t.mrdInstance.object = initialObj + // Create an initial pool + fakeMRD1 := fake.NewFakeMultiRangeDownloader(initialObj, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD1, nil).Once() + err := t.mrdInstance.ensureMRDPool() + assert.NoError(t.T(), err) + oldPool := t.mrdInstance.mrdPool + assert.NotNil(t.T(), oldPool) + // Prepare for new pool creation + newObj := &gcs.MinObject{Name: "new", Generation: 2} + fakeMRD2 := fake.NewFakeMultiRangeDownloader(newObj, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD2, nil).Once() + + // Call createAndSwapPool + t.mrdInstance.poolMu.Lock() + err = t.mrdInstance.createAndSwapPool(newObj) + t.mrdInstance.poolMu.Unlock() + + assert.NoError(t.T(), err) + assert.NotSame(t.T(), oldPool, t.mrdInstance.mrdPool) + assert.Equal(t.T(), newObj, t.mrdInstance.object) + assert.Equal(t.T(), fakeMRD2, t.mrdInstance.mrdPool.entries[0].mrd) +} + +func (t *MrdInstanceTest) TestCreateAndSwapPool_Failure() { + // Setup initial state + initialObj := &gcs.MinObject{Name: "old", Generation: 1} + t.mrdInstance.object = initialObj + // Create an initial pool + fakeMRD1 := fake.NewFakeMultiRangeDownloader(initialObj, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD1, nil).Once() + err := t.mrdInstance.ensureMRDPool() + assert.NoError(t.T(), err) + oldPool := t.mrdInstance.mrdPool + assert.NotNil(t.T(), oldPool) + // Prepare for new pool creation failure + newObj := &gcs.MinObject{Name: "new", Generation: 2} + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(nil, fmt.Errorf("creation failed")).Once() + + // Call createAndSwapPool + t.mrdInstance.poolMu.Lock() + err = t.mrdInstance.createAndSwapPool(newObj) + t.mrdInstance.poolMu.Unlock() + + assert.Error(t.T(), err) + assert.Contains(t.T(), err.Error(), "creation failed") + // Verify state remains unchanged + assert.Equal(t.T(), oldPool, t.mrdInstance.mrdPool) + assert.Equal(t.T(), initialObj, t.mrdInstance.object) +} + +func (t *MrdInstanceTest) TestSetMinObject_NilObject() { + err := t.mrdInstance.SetMinObject(nil) + + assert.Error(t.T(), err) + + assert.Contains(t.T(), err.Error(), "Missing MinObject") +} + +func (t *MrdInstanceTest) TestSetMinObject_SameGeneration() { + // Setup + initialObj := t.mrdInstance.GetMinObject() + // Ensure pool exists. + fakeMRD1 := fake.NewFakeMultiRangeDownloader(initialObj, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD1, nil).Once() + err := t.mrdInstance.ensureMRDPool() + assert.NoError(t.T(), err) + t.mrdInstance.poolMu.RLock() + initialPool := t.mrdInstance.mrdPool + t.mrdInstance.poolMu.RUnlock() + // Same generation update (e.g. size change). + newObj := &gcs.MinObject{ + Name: initialObj.Name, + Generation: initialObj.Generation, + Size: initialObj.Size + 100, + } + + err = t.mrdInstance.SetMinObject(newObj) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), newObj, t.mrdInstance.GetMinObject()) + // Pool should not change for same generation. + t.mrdInstance.poolMu.RLock() + assert.Equal(t.T(), initialPool, t.mrdInstance.mrdPool) + t.mrdInstance.poolMu.RUnlock() +} + +func (t *MrdInstanceTest) TestSetMinObject_DifferentGeneration() { + // Setup + initialObj := t.mrdInstance.GetMinObject() + // Ensure pool exists. + fakeMRD1 := fake.NewFakeMultiRangeDownloader(initialObj, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD1, nil).Once() + err := t.mrdInstance.ensureMRDPool() + assert.NoError(t.T(), err) + t.mrdInstance.poolMu.RLock() + initialPool := t.mrdInstance.mrdPool + t.mrdInstance.poolMu.RUnlock() + // New generation + newObj := &gcs.MinObject{ + Name: initialObj.Name, + Generation: initialObj.Generation + 1, + Size: initialObj.Size, + } + // Mock creation of new pool + fakeMRD2 := fake.NewFakeMultiRangeDownloader(newObj, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD2, nil).Once() + + err = t.mrdInstance.SetMinObject(newObj) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), newObj, t.mrdInstance.GetMinObject()) + t.mrdInstance.poolMu.RLock() + assert.NotSame(t.T(), initialPool, t.mrdInstance.mrdPool) + assert.NotNil(t.T(), t.mrdInstance.mrdPool) + t.mrdInstance.poolMu.RUnlock() +} + +func (t *MrdInstanceTest) TestGetMinObject() { + obj := t.mrdInstance.GetMinObject() + + assert.Equal(t.T(), t.object, obj) +} + +func (t *MrdInstanceTest) TestClosePoolWithTimeout_LogWarningOnTimeout() { + // 1. Capture logs. + var buf logBuffer + logger.SetOutput(&buf) + defer logger.SetOutput(os.Stdout) + // 2. Create a pool that blocks on Close(). + // MRDPool.Close() waits on creationWg. We increment it to block Close(). + pool := &MRDPool{ + poolConfig: &MRDPoolConfig{ + object: t.object, + }, + stopCreation: make(chan struct{}), + } + pool.creationWg.Add(1) + + // 3. Call the function. + closePoolWithTimeout(pool, "TestCaller", 10*time.Millisecond) + + // 4. Wait enough time for timeout to trigger. + time.Sleep(50 * time.Millisecond) + // 5. Verify log. + assert.Contains(t.T(), buf.String(), "TestCaller: MRDPool.Close() timed out") + assert.Contains(t.T(), buf.String(), t.object.Name) + // 7. Cleanup: Unblock the pool closure to avoid goroutine leak. + pool.creationWg.Done() +} + +// logBuffer is a thread-safe buffer for capturing logs in tests. +type logBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *logBuffer) Write(p []byte) (n int, err error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *logBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +func (t *MrdInstanceTest) Test_Cache_AddAndRemove() { + key := getKey(t.inodeID) + // Setup: Ensure pool is created + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + err := t.mrdInstance.ensureMRDPool() + assert.NoError(t.T(), err) + + // Act: Open, close, and reopen file + t.mrdInstance.IncrementRefCount() + t.mrdInstance.DecrementRefCount() + assert.NotNil(t.T(), t.cache.LookUpWithoutChangingOrder(key), "Instance should be in cache.") + t.mrdInstance.IncrementRefCount() + + // Assert: Instance reused and removed from cache on reopen + assert.Equal(t.T(), int64(1), t.mrdInstance.refCount) + assert.Nil(t.T(), t.cache.LookUpWithoutChangingOrder(key), "Instance should be removed from cache") + t.mrdInstance.poolMu.RLock() + assert.NotNil(t.T(), t.mrdInstance.mrdPool, "MRD Pool should still exist (reused)") + t.mrdInstance.poolMu.RUnlock() +} + +func (t *MrdInstanceTest) Test_DecrementRefCount_ParallelUpdates() { + // Arrange + const finalRefCount int64 = 0 + maxRefCount := 10 + wg := sync.WaitGroup{} + key := getKey(t.inodeID) + // Ensure pool exists so it can be cached + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + err := t.mrdInstance.ensureMRDPool() + assert.NoError(t.T(), err) + + // Act: Increment refcount in parallel + for i := 0; i < maxRefCount; i++ { + wg.Add(1) + go func() { + t.mrdInstance.IncrementRefCount() + wg.Done() + }() + } + wg.Wait() + // Act: Decrement refcount in parallel + for i := 0; i < maxRefCount; i++ { + wg.Add(1) + go func() { + t.mrdInstance.DecrementRefCount() + wg.Done() + }() + } + wg.Wait() + + // Assert: Final state is refCount=0, MRD pooled in cache + assert.Equal(t.T(), finalRefCount, t.mrdInstance.refCount) + t.mrdInstance.poolMu.RLock() + assert.NotNil(t.T(), t.mrdInstance.mrdPool, "MRD Pool should be pooled in cache") + t.mrdInstance.poolMu.RUnlock() + assert.NotNil(t.T(), t.cache.LookUpWithoutChangingOrder(key), "Instance should be in cache") +} + +func (t *MrdInstanceTest) Test_Cache_EvictionOnOverflow() { + // Arrange: Create 3 instances (cache max is 2) + instances := make([]*MrdInstance, 3) + for i := 0; i < 3; i++ { + obj := &gcs.MinObject{ + Name: fmt.Sprintf("file%d", i), + Size: 100, + Generation: int64(1000 + i), + } + instance := NewMrdInstance(obj, t.bucket, t.cache, fuseops.InodeID(100+i), t.config) + + // Setup mock for pool creation + fakeMRD := fake.NewFakeMultiRangeDownloader(obj, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + err := instance.ensureMRDPool() + assert.NoError(t.T(), err) + + instances[i] = instance + } + + // Act: Open and close all 3 instances (triggers eviction on 3rd) + for i := 0; i < 3; i++ { + instances[i].IncrementRefCount() + instances[i].DecrementRefCount() + } + + // Assert: First instance evicted (LRU), last 2 remain in cache + instances[0].poolMu.RLock() + assert.Nil(t.T(), instances[0].mrdPool, "First instance's MRD Pool should be closed (evicted)") + instances[0].poolMu.RUnlock() + for i := 1; i < 3; i++ { + key := getKey(instances[i].inodeId) + assert.NotNil(t.T(), t.cache.LookUpWithoutChangingOrder(key), "Instance %d should be in cache", i) + instances[i].poolMu.RLock() + assert.NotNil(t.T(), instances[i].mrdPool, "Instance %d MRD Pool should exist (pooled)", i) + instances[i].poolMu.RUnlock() + } +} + +func (t *MrdInstanceTest) Test_Cache_DeletedIfReopened() { + // Arrange: Create 2 instances and fill cache (size 2) + instances := make([]*MrdInstance, 2) + for i := 0; i < 2; i++ { + obj := &gcs.MinObject{ + Name: fmt.Sprintf("file%d", i), + Size: 100, + Generation: int64(1000 + i), + } + instance := NewMrdInstance(obj, t.bucket, t.cache, fuseops.InodeID(100+i), t.config) + + fakeMRD := fake.NewFakeMultiRangeDownloader(obj, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + err := instance.ensureMRDPool() + assert.NoError(t.T(), err) + + instance.IncrementRefCount() + instance.DecrementRefCount() + instances[i] = instance + } + + // Act: Reopen instance 0 -> should remove it from cache + instances[0].IncrementRefCount() + + // Assert: instance 0 will be deleted from cache. + assert.Nil(t.T(), t.cache.LookUpWithoutChangingOrder(getKey(instances[0].inodeId)), "Instance 0 should not be in cache") +} + +func (t *MrdInstanceTest) Test_Cache_ConcurrentAddRemove() { + // Arrange + const numGoroutines = 10 + const numIterations = 100 + wg := sync.WaitGroup{} + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + err := t.mrdInstance.ensureMRDPool() + assert.NoError(t.T(), err) + + // Act: Concurrent open/close cycles from multiple goroutines + for i := 0; i < numGoroutines; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < numIterations; j++ { + t.mrdInstance.IncrementRefCount() + t.mrdInstance.DecrementRefCount() + } + }() + } + wg.Wait() + + // Assert: Final state is refCount=0 (no deadlocks or panics) + assert.Equal(t.T(), int64(0), t.mrdInstance.refCount, "RefCount should be 0 after all operations") + assert.NotNil(t.T(), t.cache.LookUpWithoutChangingOrder(getKey(t.inodeID)), "Instance should be in cache") +} + +func (t *MrdInstanceTest) Test_Cache_Disabled() { + // Arrange: Create instance with nil cache (disabled) + instance := NewMrdInstance(t.object, t.bucket, nil, t.inodeID, t.config) + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + err := instance.ensureMRDPool() + assert.NoError(t.T(), err) + + // Act: Open and close file + instance.IncrementRefCount() + instance.DecrementRefCount() + + // Assert: MRD Pool should be open forever since cache is disabled. + instance.poolMu.RLock() + assert.NotNil(t.T(), instance.mrdPool, "MRD Pool should be open when cache disabled") + instance.poolMu.RUnlock() +} + +func (t *MrdInstanceTest) Test_Cache_EvictionRaceWithRepool() { + // Arrange: Add instance to cache then fill with 2 more to trigger eviction (cache size 2) + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + err := t.mrdInstance.ensureMRDPool() + assert.NoError(t.T(), err) + t.mrdInstance.IncrementRefCount() + t.mrdInstance.DecrementRefCount() + for i := 0; i < 2; i++ { + obj := &gcs.MinObject{ + Name: fmt.Sprintf("file%d", i), + Size: 100, + Generation: int64(1000 + i), + } + instance := NewMrdInstance(obj, t.bucket, t.cache, fuseops.InodeID(200+i), t.config) + + fakeMRD := fake.NewFakeMultiRangeDownloader(obj, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + err := instance.ensureMRDPool() + assert.NoError(t.T(), err) + + instance.IncrementRefCount() + instance.DecrementRefCount() + } + // Verify it was evicted + t.mrdInstance.poolMu.RLock() + assert.Nil(t.T(), t.mrdInstance.mrdPool) + t.mrdInstance.poolMu.RUnlock() + buf := make([]byte, 10) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return( + fake.NewFakeMultiRangeDownloader(t.object, nil), + nil, + ).Once() + + // Act: Access evicted instance (should recreate MRD Pool). Read should recreate it + _, err = t.mrdInstance.Read(context.Background(), buf, 0, metrics.NewNoopMetrics()) + + // Assert: MRD Pool recreated successfully after eviction + assert.NoError(t.T(), err) + t.mrdInstance.poolMu.RLock() + assert.NotNil(t.T(), t.mrdInstance.mrdPool, "MRD Pool should be recreated after eviction") + t.mrdInstance.poolMu.RUnlock() +} + +func (t *MrdInstanceTest) Test_Cache_MultipleEvictions() { + // Arrange: Create small cache (size 2) and 5 instances + smallCache := lru.NewCache(2) + instances := make([]*MrdInstance, 5) + for i := 0; i < 5; i++ { + obj := &gcs.MinObject{ + Name: fmt.Sprintf("file%d", i), + Size: 100, + Generation: int64(1000 + i), + } + instance := NewMrdInstance(obj, t.bucket, smallCache, fuseops.InodeID(300+i), t.config) + + fakeMRD := fake.NewFakeMultiRangeDownloader(obj, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + err := instance.ensureMRDPool() + assert.NoError(t.T(), err) + + instances[i] = instance + } + + // Act: Add all 5 instances (triggers batch eviction of 3) + for i := 0; i < 5; i++ { + instances[i].IncrementRefCount() + instances[i].DecrementRefCount() + } + + // Assert: First 3 evicted, last 2 remain in cache + for i := 0; i < 3; i++ { + instances[i].poolMu.RLock() + assert.Nil(t.T(), instances[i].mrdPool, "Instance %d should be evicted", i) + instances[i].poolMu.RUnlock() + } + for i := 3; i < 5; i++ { + instances[i].poolMu.RLock() + assert.NotNil(t.T(), instances[i].mrdPool, "Instance %d should be in cache", i) + instances[i].poolMu.RUnlock() + } +} diff --git a/internal/gcsx/mrd_kernel_reader.go b/internal/gcsx/mrd_kernel_reader.go new file mode 100644 index 0000000000..9237991d02 --- /dev/null +++ b/internal/gcsx/mrd_kernel_reader.go @@ -0,0 +1,122 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// MrdKernelReader is a reader that uses an MRD Instance to read data from a GCS object. +// This reader is kernel-optimized compared to the GCSReader as it doesn't have complex logic +// to switch between sequential and random read strategies. +type MrdKernelReader struct { + mrdInstanceInUse atomic.Bool + mrdInstance *MrdInstance + metrics metrics.MetricHandle +} + +// NewMrdKernelReader creates a new MrdKernelReader that uses the provided +// MrdInstance to manage MRD connections. +func NewMrdKernelReader(mrdInstance *MrdInstance, metricsHandle metrics.MetricHandle) *MrdKernelReader { + return &MrdKernelReader{ + mrdInstance: mrdInstance, + metrics: metricsHandle, + } +} + +// isShortRead determines what constitutes a short read for retry purposes. +// It returns true if bytesRead < bufferSize and the error is a gRPC OutOfRange error. +func isShortRead(bytesRead int, bufferSize int, err error) bool { + if bytesRead >= bufferSize { + return false + } + + // Even without O_DIRECT, OutOfRange errors can occur during appends from the same mount. + // The kernel tracks the updated file size and allows reads, but the active MRD connection might + // still reference the old object size. We update the local object in the MrdInstance without + // recreating the MRD connection. Reads beyond the old size thus return OutOfRange, which + // we handle as a short read to trigger MRD recreation and retry. + + // Check for gRPC OutOfRange error, handling wrapped errors. + var se interface{ GRPCStatus() *status.Status } + if errors.As(err, &se) { + return se.GRPCStatus().Code() == codes.OutOfRange + } + + return false +} + +// ReadAt reads data into the provided request buffer starting at the specified +// offset. It retrieves an available MRD entry and uses it to download the +// requested byte range. +func (mkr *MrdKernelReader) ReadAt(ctx context.Context, req *ReadRequest) (ReadResponse, error) { + // If the destination buffer is empty, there's nothing to read. + if len(req.Buffer) == 0 { + return ReadResponse{}, nil + } + + // mrdInstance is set to nil in Destroy which will be called only after all active Read operations + // have finished. Hence, not taking RLock to access it. + if mkr.mrdInstance == nil { + return ReadResponse{}, fmt.Errorf("MrdKernelReader: mrdInstance is nil") + } + + if mkr.mrdInstanceInUse.CompareAndSwap(false, true) { + mkr.mrdInstance.IncrementRefCount() + } + + var bytesRead int + defer func() { + metrics.CaptureGCSReadMetrics(mkr.metrics, metrics.ReadTypeParallelAttr, int64(bytesRead)) + mkr.metrics.GcsReadBytesCount(int64(bytesRead)) + }() + + var err error + bytesRead, err = mkr.mrdInstance.Read(ctx, req.Buffer, req.Offset, mkr.metrics) + if isShortRead(bytesRead, len(req.Buffer), err) { + logger.Tracef("Short read detected: read %d bytes out of %d requested. Retrying...", bytesRead, len(req.Buffer)) + if err = mkr.mrdInstance.RecreateMRD(); err != nil { + logger.Warnf("Failed to recreate MRD for short read retry. Will retry with older MRD: %v", err) + } + retryOffset := req.Offset + int64(bytesRead) + retryBuffer := req.Buffer[bytesRead:] + var bytesReadOnRetry int + bytesReadOnRetry, err = mkr.mrdInstance.Read(ctx, retryBuffer, retryOffset, mkr.metrics) + bytesRead += bytesReadOnRetry + } + return ReadResponse{Size: bytesRead}, err +} + +// Destroy cleans up the resources used by the reader, primarily by destroying +// the associated MrdInstance. This should be called when the reader is no +// longer needed. +func (mkr *MrdKernelReader) Destroy() { + // No need to take lock as Destroy will only be called when file handle is being released + // and there will be no read calls at that point. + if mkr.mrdInstance != nil { + if mkr.mrdInstanceInUse.CompareAndSwap(true, false) { + mkr.mrdInstance.DecrementRefCount() + } + mkr.mrdInstance = nil + } +} diff --git a/internal/gcsx/mrd_kernel_reader_test.go b/internal/gcsx/mrd_kernel_reader_test.go new file mode 100644 index 0000000000..d3bcbc88c7 --- /dev/null +++ b/internal/gcsx/mrd_kernel_reader_test.go @@ -0,0 +1,377 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/jacobsa/fuse/fuseops" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type MrdKernelReaderTest struct { + suite.Suite + object *gcs.MinObject + bucket *storage.TestifyMockBucket + cache *lru.Cache + inodeID fuseops.InodeID + config *cfg.Config + mrdInstance *MrdInstance + reader *MrdKernelReader +} + +func TestMrdKernelReaderTestSuite(t *testing.T) { + suite.Run(t, new(MrdKernelReaderTest)) +} + +func (t *MrdKernelReaderTest) SetupTest() { + t.object = &gcs.MinObject{ + Name: "foo", + Size: 1024 * MiB, + Generation: 1234, + } + t.bucket = new(storage.TestifyMockBucket) + t.cache = lru.NewCache(2) + t.inodeID = 100 + t.config = &cfg.Config{Mrd: cfg.MrdConfig{PoolSize: 1}} + + t.mrdInstance = NewMrdInstance(t.object, t.bucket, t.cache, t.inodeID, t.config) + t.reader = NewMrdKernelReader(t.mrdInstance, metrics.NewNoopMetrics()) +} + +func (t *MrdKernelReaderTest) TestNewMrdKernelReader() { + assert.NotNil(t.T(), t.reader) + assert.Equal(t.T(), t.mrdInstance, t.reader.mrdInstance) +} + +func (t *MrdKernelReaderTest) TestReadAt_EmptyBuffer() { + req := &ReadRequest{ + Buffer: []byte{}, + Offset: 0, + } + + resp, err := t.reader.ReadAt(context.Background(), req) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), 0, resp.Size) +} + +func (t *MrdKernelReaderTest) TestReadAt_Success() { + data := []byte("hello world") + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, data) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + req := &ReadRequest{ + Buffer: make([]byte, 5), + Offset: 0, + } + // Verify initial refCount + t.mrdInstance.refCountMu.Lock() + assert.Equal(t.T(), int64(0), t.mrdInstance.refCount) + t.mrdInstance.refCountMu.Unlock() + + resp, err := t.reader.ReadAt(context.Background(), req) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), 5, resp.Size) + assert.Equal(t.T(), "hello", string(req.Buffer)) + // Verify refCount incremented + t.mrdInstance.refCountMu.Lock() + assert.Equal(t.T(), int64(1), t.mrdInstance.refCount) + t.mrdInstance.refCountMu.Unlock() +} + +func (t *MrdKernelReaderTest) TestReadAt_MultipleCalls() { + data := []byte("hello world") + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, data) + // Expect NewMultiRangeDownloader only once because subsequent reads reuse the instance/pool + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + req := &ReadRequest{ + Buffer: make([]byte, 5), + Offset: 0, + } + + // First call + _, err := t.reader.ReadAt(context.Background(), req) + assert.NoError(t.T(), err) + + // Second call + _, err = t.reader.ReadAt(context.Background(), req) + assert.NoError(t.T(), err) + + // Verify refCount is still 1 + t.mrdInstance.refCountMu.Lock() + assert.Equal(t.T(), int64(1), t.mrdInstance.refCount) + t.mrdInstance.refCountMu.Unlock() +} + +func (t *MrdKernelReaderTest) TestReadAt_NilMrdInstance() { + t.reader.mrdInstance = nil + req := &ReadRequest{ + Buffer: make([]byte, 5), + Offset: 0, + } + + resp, err := t.reader.ReadAt(context.Background(), req) + + assert.Error(t.T(), err) + assert.Contains(t.T(), err.Error(), "mrdInstance is nil") + assert.Equal(t.T(), 0, resp.Size) +} + +func (t *MrdKernelReaderTest) TestReadAt_ShortRead_NoRetry() { + data := []byte("hello world") + // MRD returns short read. + fakeMRD := fake.NewFakeMultiRangeDownloaderWithShortRead(t.object, data) + // Expectation: + // 1. Read calls ensureMRDPool -> NewMRDPool -> NewMultiRangeDownloader. Returns fakeMRD. + // 2. Read returns short read. + // 3. isShortRead returns false because err is nil. + // 4. ReadAt returns the short read without retrying. + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + buf := make([]byte, len(data)) + req := &ReadRequest{Buffer: buf, Offset: 0} + + resp, err := t.reader.ReadAt(context.Background(), req) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), 5, resp.Size) // Short read size + assert.Equal(t.T(), "hello", string(buf[:5])) + // Verify refCount incremented (only once) + t.mrdInstance.refCountMu.Lock() + assert.Equal(t.T(), int64(1), t.mrdInstance.refCount) + t.mrdInstance.refCountMu.Unlock() + t.bucket.AssertExpectations(t.T()) +} + +func (t *MrdKernelReaderTest) TestReadAt_OutOfRange_TriggersRetry() { + data := []byte("hello world") + // First MRD returns OutOfRange error. + outOfRangeErr := status.Error(codes.OutOfRange, "Out of range") + fakeMRD1 := fake.NewFakeMultiRangeDownloaderWithSleepAndDefaultError(t.object, []byte{}, 0, outOfRangeErr) + // Second MRD returns full read. + fakeMRD2 := fake.NewFakeMultiRangeDownloader(t.object, data) + // Expectation: + // 1. Initial Read calls ensureMRDPool -> NewMRDPool -> NewMultiRangeDownloader. Returns fakeMRD1. + // 2. Read returns OutOfRange. isShortRead detects this as recoverable. + // 3. ReadAt calls RecreateMRD -> NewMRDPool -> NewMultiRangeDownloader. Returns fakeMRD2. + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD1, nil).Once() + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD2, nil).Once() + // Create the ReadRequest. + buf := make([]byte, len(data)) + req := &ReadRequest{ + Buffer: buf, + Offset: 0, + } + + resp, err := t.reader.ReadAt(context.Background(), req) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), len(data), resp.Size) + assert.Equal(t.T(), string(data), string(buf)) + // Verify refCount incremented + t.mrdInstance.refCountMu.Lock() + assert.Equal(t.T(), int64(1), t.mrdInstance.refCount) + t.mrdInstance.refCountMu.Unlock() + t.bucket.AssertExpectations(t.T()) +} + +func (t *MrdKernelReaderTest) TestReadAt_OutOfRange_RetryFails() { + // First MRD returns OutOfRange error. + outOfRangeErr := status.Error(codes.OutOfRange, "Out of range") + fakeMRD1 := fake.NewFakeMultiRangeDownloaderWithSleepAndDefaultError(t.object, []byte{}, 0, outOfRangeErr) + // Second MRD returns Internal error. + retryErr := status.Error(codes.Internal, "Internal error") + fakeMRD2 := fake.NewFakeMultiRangeDownloaderWithSleepAndDefaultError(t.object, []byte{}, 0, retryErr) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD1, nil).Once() + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD2, nil).Once() + // Create the ReadRequest. + buf := make([]byte, 10) + req := &ReadRequest{ + Buffer: buf, + Offset: 0, + } + + resp, err := t.reader.ReadAt(context.Background(), req) + + // Should return the error from the retry. + assert.ErrorIs(t.T(), err, retryErr) + assert.Equal(t.T(), 0, resp.Size) + t.bucket.AssertExpectations(t.T()) +} + +func TestIsShortRead(t *testing.T) { + testCases := []struct { + name string + bytesRead int + bufferSize int + err error + expected bool + }{ + { + name: "Full read, no error", + bytesRead: 10, + bufferSize: 10, + err: nil, + expected: false, + }, + { + name: "Full read, error", + bytesRead: 10, + bufferSize: 10, + err: errors.New("error"), + expected: false, + }, + { + name: "Short read, no error", + bytesRead: 5, + bufferSize: 10, + err: nil, + expected: false, + }, + { + name: "Short read, OutOfRange", + bytesRead: 0, + bufferSize: 10, + err: status.Error(codes.OutOfRange, "out of range"), + expected: true, + }, + { + name: "Short read, Wrapped OutOfRange", + bytesRead: 0, + bufferSize: 10, + err: fmt.Errorf("wrapped: %w", status.Error(codes.OutOfRange, "out of range")), + expected: true, + }, + { + name: "Short read, Internal error", + bytesRead: 5, + bufferSize: 10, + err: status.Error(codes.Internal, "internal error"), + expected: false, + }, + { + name: "Short read, Generic error", + bytesRead: 5, + bufferSize: 10, + err: errors.New("generic error"), + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, isShortRead(tc.bytesRead, tc.bufferSize, tc.err)) + }) + } +} + +func (t *MrdKernelReaderTest) TestDestroy() { + // Setup state where refCount is incremented + t.reader.mrdInstanceInUse.Store(true) + t.mrdInstance.refCount = 1 + + t.reader.Destroy() + + assert.Nil(t.T(), t.reader.mrdInstance) + assert.False(t.T(), t.reader.mrdInstanceInUse.Load()) + // Verify refCount decremented + t.mrdInstance.refCountMu.Lock() + assert.Equal(t.T(), int64(0), t.mrdInstance.refCount) + t.mrdInstance.refCountMu.Unlock() + // Verify that calling Destroy again doesn't panic + t.reader.Destroy() +} + +func (t *MrdKernelReaderTest) TestReadAt_RecreateMRDFails_RetriesWithOldMRD() { + // First MRD returns OutOfRange error. + outOfRangeErr := status.Error(codes.OutOfRange, "Out of range") + fakeMRD1 := fake.NewFakeMultiRangeDownloaderWithSleepAndDefaultError(t.object, []byte{}, 0, outOfRangeErr) + // Expectation: + // 1. Initial Read calls ensureMRDPool -> NewMRDPool -> NewMultiRangeDownloader. Returns fakeMRD1. + // 2. Read returns OutOfRange. + // 3. ReadAt calls RecreateMRD -> NewMRDPool -> NewMultiRangeDownloader. Returns ERROR. + // 4. ReadAt logs warning and retries with existing pool (fakeMRD1). + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD1, nil).Once() + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(nil, errors.New("recreate failed")).Once() + buf := make([]byte, 10) + req := &ReadRequest{ + Buffer: buf, + Offset: 0, + } + + resp, err := t.reader.ReadAt(context.Background(), req) + + // Assert + // We verify that we didn't get the "recreate failed" error. + assert.ErrorIs(t.T(), err, outOfRangeErr) + assert.NotEqual(t.T(), "recreate failed", err.Error()) + // We expect 0 bytes to be read as OutOfRange returns 0 bytes. + assert.Equal(t.T(), 0, resp.Size) + t.bucket.AssertExpectations(t.T()) +} + +func (t *MrdKernelReaderTest) TestDestroy_NoReadAt() { + // Check initial refCount + t.mrdInstance.refCountMu.Lock() + assert.Equal(t.T(), int64(0), t.mrdInstance.refCount) + t.mrdInstance.refCountMu.Unlock() + // Capture logs + var buf bytes.Buffer + logger.SetOutput(&buf) + defer logger.SetOutput(os.Stdout) + + // Act + t.reader.Destroy() + + // Assert + t.mrdInstance.refCountMu.Lock() + assert.Equal(t.T(), int64(0), t.mrdInstance.refCount) + t.mrdInstance.refCountMu.Unlock() + assert.NotContains(t.T(), buf.String(), "MrdInstance::DecrementRefCount: Refcount cannot be negative") +} + +func (t *MrdKernelReaderTest) TestReadAt_ContextCanceled() { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + req := &ReadRequest{ + Buffer: make([]byte, 10), + Offset: 0, + } + // Use sleep to ensure context cancellation is detected before read completes + fakeMRD := fake.NewFakeMultiRangeDownloaderWithSleep(t.object, []byte("data"), 100*time.Millisecond) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + + resp, err := t.reader.ReadAt(ctx, req) + + assert.ErrorIs(t.T(), err, context.Canceled) + assert.Equal(t.T(), 0, resp.Size) +} diff --git a/internal/gcsx/mrd_pool.go b/internal/gcsx/mrd_pool.go new file mode 100644 index 0000000000..dcfef2d945 --- /dev/null +++ b/internal/gcsx/mrd_pool.go @@ -0,0 +1,237 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/gcsfuse_errors" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" +) + +const ( + smallFileThresholdMiB = 100 + mediumFileThresholdMiB = 500 +) + +// MRDEntry holds a single MultiRangeDownloader instance and a mutex to protect access to it. +type MRDEntry struct { + mrd gcs.MultiRangeDownloader + mu sync.RWMutex +} + +// MRDPoolConfig contains configuration for the MRD pool. +type MRDPoolConfig struct { + // PoolSize is the number of MultiRangeDownloader instances in the pool + PoolSize int + + object *gcs.MinObject + bucket gcs.Bucket + Handle []byte +} + +// MRDPool manages a pool of MultiRangeDownloader instances to allow concurrent downloads. +type MRDPool struct { + poolConfig *MRDPoolConfig + entries []MRDEntry + current atomic.Uint64 + currentSize atomic.Uint64 + ctx context.Context + // stopCreation is used to signal background creation goroutine to stop without + // canceling the context, enabling graceful shutdown. + stopCreation chan struct{} + // creationWg is used to wait for the background creation of MRDs to finish. + creationWg sync.WaitGroup +} + +// determinePoolSize sets the pool size to 1 if the object size is smaller than +// smallFileThresholdMiB. +func (mrdPoolConfig *MRDPoolConfig) determinePoolSize() { + if mrdPoolConfig.object.Size < smallFileThresholdMiB*MiB { + mrdPoolConfig.PoolSize = 1 + return + } + if mrdPoolConfig.object.Size < mediumFileThresholdMiB*MiB { + mrdPoolConfig.PoolSize = 2 + return + } +} + +// NewMRDPool initializes a new MRDPool. +// It creates the first MRD synchronously to ensure immediate availability and starts a background goroutine to create the remaining MRDs. +func NewMRDPool(config *MRDPoolConfig, handle []byte) (*MRDPool, error) { + if config == nil { + return nil, fmt.Errorf("config cannot be nil") + } + p := &MRDPool{ + poolConfig: config, + ctx: context.Background(), + stopCreation: make(chan struct{}), + } + p.poolConfig.determinePoolSize() + logger.Tracef("Initializing MRD Pool with size: %d", p.poolConfig.PoolSize) + p.entries = make([]MRDEntry, p.poolConfig.PoolSize) + + // Create the first MRD synchronously. + mrd, err := config.bucket.NewMultiRangeDownloader(p.ctx, &gcs.MultiRangeDownloaderRequest{ + Name: config.object.Name, + Generation: config.object.Generation, + ReadCompressed: config.object.HasContentEncodingGzip(), + ReadHandle: handle, + }) + if err != nil { + var notFoundError *gcs.NotFoundError + if errors.As(err, ¬FoundError) { + return nil, &gcsfuse_errors.FileClobberedError{ + Err: fmt.Errorf("NewMRDPool: %w", err), + ObjectName: config.object.Name, + } + } + return nil, err + } + p.entries[0].mrd = mrd + p.currentSize.Store(1) + + // Create the rest of the MRDs asynchronously. + if p.poolConfig.PoolSize > 1 { + mrdHandle := mrd.GetHandle() + p.creationWg.Add(1) + go func() { + defer p.creationWg.Done() + p.createRemainingMRDs(mrdHandle) + }() + } + + return p, nil +} + +// createRemainingMRDs creates the remaining MultiRangeDownloader instances in the background. +// It populates the pool entries and increments the currentSize counter. +func (p *MRDPool) createRemainingMRDs(handle []byte) { + for i := 1; i < p.poolConfig.PoolSize; i++ { + // Check if we should stop creating MRDs (graceful shutdown initiated) + select { + case <-p.stopCreation: + return + default: + } + mrd, err := p.poolConfig.bucket.NewMultiRangeDownloader(p.ctx, &gcs.MultiRangeDownloaderRequest{ + Name: p.poolConfig.object.Name, + Generation: p.poolConfig.object.Generation, + ReadCompressed: p.poolConfig.object.HasContentEncodingGzip(), + ReadHandle: handle, + }) + if err == nil { + p.entries[i].mu.Lock() + p.entries[i].mrd = mrd + p.entries[i].mu.Unlock() + } else { + logger.Warnf("Error in creating MRD. Would be retried once before using the MRD") + } + p.currentSize.Add(1) + } +} + +// Next returns the next available MRDEntry from the pool using a round-robin strategy based on the number of currently initialized MRDs. +// Please check returned MRD is non nil and valid (i.e. not in an error state) before using it. +func (p *MRDPool) Next() *MRDEntry { + limit := p.currentSize.Load() + // Use post-increment style to get 0-based index for round-robin. + idx := (p.current.Add(1) - 1) % limit + return &p.entries[idx] +} + +// RecreateMRD attempts to recreate a specific MRDEntry's MultiRangeDownloader. +// It uses a handle from an existing MRD or a fallback handle. +func (p *MRDPool) RecreateMRD(entry *MRDEntry, fallbackHandle []byte) error { + entry.mu.Lock() + defer entry.mu.Unlock() + + var handle []byte + if entry.mrd != nil { + handle = entry.mrd.GetHandle() + } else if fallbackHandle != nil { + handle = fallbackHandle + } else { + for i := 0; i < int(p.currentSize.Load()); i++ { + if &p.entries[i] == entry { + continue + } + // Use TryRLock to avoid deadlock if multiple entries are being recreated simultaneously. + if p.entries[i].mu.TryRLock() { + if p.entries[i].mrd != nil { + handle = p.entries[i].mrd.GetHandle() + p.entries[i].mu.RUnlock() + break + } + p.entries[i].mu.RUnlock() + } + } + } + + mrd, err := p.poolConfig.bucket.NewMultiRangeDownloader(p.ctx, &gcs.MultiRangeDownloaderRequest{ + Name: p.poolConfig.object.Name, + Generation: p.poolConfig.object.Generation, + ReadCompressed: p.poolConfig.object.HasContentEncodingGzip(), + ReadHandle: handle, + }) + + if err == nil { + entry.mrd = mrd + } else { + return fmt.Errorf("Error in recreating MRD: %w", err) + } + return nil +} + +// Close shuts down the MRDPool gracefully. +// It signals background creation to stop, waits for pending creations to finish, +// waits for active downloads on existing MRDs to complete, and then closes all MRDs. +// The context used for MRD creation is never canceled, ensuring in-flight range +// requests complete without interruption. +// It returns a handle from one of the closed MRDs for potential future use. +func (p *MRDPool) Close() (handle []byte) { + // Signal background creation to stop + close(p.stopCreation) + // Wait for background creation to finish + p.creationWg.Wait() + + // Wait for all MRDs to complete their work and close them + for i := range p.entries { + entry := &p.entries[i] + entry.mu.Lock() + if entry.mrd != nil { + // Wait for in-flight downloads to complete + entry.mrd.Wait() + if handle == nil { + handle = entry.mrd.GetHandle() + } + entry.mrd.Close() + entry.mrd = nil + } + entry.mu.Unlock() + } + return +} + +// Return the max size of the pool. +func (p *MRDPool) Size() uint64 { + return uint64(p.poolConfig.PoolSize) +} diff --git a/internal/gcsx/mrd_pool_test.go b/internal/gcsx/mrd_pool_test.go new file mode 100644 index 0000000000..953b71fd6b --- /dev/null +++ b/internal/gcsx/mrd_pool_test.go @@ -0,0 +1,352 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx + +import ( + "context" + "fmt" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/gcsfuse_errors" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type mrdPoolTest struct { + suite.Suite + object *gcs.MinObject + bucket *storage.TestifyMockBucket + poolConfig *MRDPoolConfig +} + +func TestMRDPoolTestSuite(t *testing.T) { + suite.Run(t, new(mrdPoolTest)) +} + +func (t *mrdPoolTest) SetupTest() { + t.object = &gcs.MinObject{ + Name: "foo", + Size: 1024 * MiB, + Generation: 1234, + } + t.bucket = new(storage.TestifyMockBucket) + t.poolConfig = &MRDPoolConfig{ + PoolSize: 4, + object: t.object, + bucket: t.bucket, + } +} + +func (t *mrdPoolTest) TestNewMRDPool_SmallFile() { + t.object.Size = 100 * MiB + t.poolConfig.PoolSize = 4 + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + // Two MRD instances will be created for [100MB to 500MB) files. + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Times(2) + + pool, err := NewMRDPool(t.poolConfig, nil) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), 2, pool.poolConfig.PoolSize) + assert.Len(t.T(), pool.entries, 2) + assert.NotNil(t.T(), pool.entries[0].mrd) +} + +func (t *mrdPoolTest) TestNewMRDPool_LargeFile() { + t.object.Size = 1024 * MiB + t.poolConfig.PoolSize = 2 + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + // Expect calls for initial + async creation + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Times(2) + + pool, err := NewMRDPool(t.poolConfig, nil) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), 2, pool.poolConfig.PoolSize) + pool.creationWg.Wait() // Wait for async creation to finish + assert.Equal(t.T(), uint64(2), pool.currentSize.Load()) + assert.NotNil(t.T(), pool.entries[0].mrd) + assert.NotNil(t.T(), pool.entries[1].mrd) +} + +func (t *mrdPoolTest) TestNewMRDPool_AsyncCreationFailure() { + t.object.Size = 1024 * MiB + t.poolConfig.PoolSize = 2 + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() // First succeeds + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(nil, fmt.Errorf("async error")).Once() // Second fails + + pool, err := NewMRDPool(t.poolConfig, nil) + + assert.NoError(t.T(), err) + pool.creationWg.Wait() // Wait for async creation to finish + assert.Equal(t.T(), uint64(2), pool.currentSize.Load()) + assert.NotNil(t.T(), pool.entries[0].mrd) + assert.Nil(t.T(), pool.entries[1].mrd) +} + +func (t *mrdPoolTest) TestNewMRDPool_FileClobbered() { + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(nil, &gcs.NotFoundError{Err: fmt.Errorf("not found")}).Once() + + pool, err := NewMRDPool(t.poolConfig, nil) + + require.Error(t.T(), err) + assert.Nil(t.T(), pool) + var clobberedErr *gcsfuse_errors.FileClobberedError + assert.ErrorAs(t.T(), err, &clobberedErr) +} + +func (t *mrdPoolTest) TestNewMRDPool_NilConfig() { + pool, err := NewMRDPool(nil, nil) + + assert.ErrorContains(t.T(), err, "config cannot be nil") + + assert.Nil(t.T(), pool) +} + +func (t *mrdPoolTest) TestNewMRDPool_Error() { + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(nil, fmt.Errorf("error")).Once() + + pool, err := NewMRDPool(t.poolConfig, nil) + + assert.Error(t.T(), err) + assert.Nil(t.T(), pool) +} + +func (t *mrdPoolTest) TestNext() { + t.poolConfig.PoolSize = 3 + // Return a new downloader for each call to ensure we get different instances. + fakeMRD1 := fake.NewFakeMultiRangeDownloader(t.object, nil) + fakeMRD2 := fake.NewFakeMultiRangeDownloader(t.object, nil) + fakeMRD3 := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD1, nil).Once() + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD2, nil).Once() + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD3, nil).Once() + pool, err := NewMRDPool(t.poolConfig, nil) + assert.NoError(t.T(), err) + pool.creationWg.Wait() + + // Verify round robin + e1 := pool.Next() + e2 := pool.Next() + e3 := pool.Next() + e4 := pool.Next() + + assert.Same(t.T(), e1.mrd, fakeMRD1) + assert.Same(t.T(), e2.mrd, fakeMRD2) + assert.Same(t.T(), e3.mrd, fakeMRD3) + assert.Same(t.T(), e4.mrd, fakeMRD1) +} + +func (t *mrdPoolTest) TestDeterminePoolSize() { + testCases := []struct { + name string + objectSize uint64 + initialPoolSize int + expectedPoolSize int + }{ + { + name: "SmallFile_BelowThreshold", + objectSize: 50 * MiB, + initialPoolSize: 4, + expectedPoolSize: 1, + }, + { + name: "SmallFile_AtThreshold", + objectSize: smallFileThresholdMiB * MiB, + initialPoolSize: 4, + expectedPoolSize: 2, + }, + { + name: "MediumFile_BetweenThresholds", + objectSize: 300 * MiB, + initialPoolSize: 4, + expectedPoolSize: 2, + }, + { + name: "MediumFile_JustBelowThreshold", + objectSize: (mediumFileThresholdMiB - 1) * MiB, + initialPoolSize: 4, + expectedPoolSize: 2, + }, + { + name: "LargeFile_AtThreshold", + objectSize: mediumFileThresholdMiB * MiB, + initialPoolSize: 4, + expectedPoolSize: 4, + }, + { + name: "LargeFile_AboveThreshold", + objectSize: 2048 * MiB, + initialPoolSize: 4, + expectedPoolSize: 4, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.object.Size = tc.objectSize + t.poolConfig.PoolSize = tc.initialPoolSize + + t.poolConfig.determinePoolSize() + + assert.Equal(t.T(), tc.expectedPoolSize, t.poolConfig.PoolSize) + }) + } +} + +func (t *mrdPoolTest) TestRecreateMRD() { + t.poolConfig.PoolSize = 1 + fakeMRD1 := fake.NewFakeMultiRangeDownloader(t.object, nil) + fakeMRD2 := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD1, nil).Once() + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD2, nil).Once() + pool, err := NewMRDPool(t.poolConfig, nil) + assert.NoError(t.T(), err) + entry := pool.Next() + oldMRD := entry.mrd + + err = pool.RecreateMRD(entry, nil) + + assert.NoError(t.T(), err) + assert.NotSame(t.T(), oldMRD, entry.mrd) +} + +func (t *mrdPoolTest) TestRecreateMRD_UsesFallbackHandle() { + t.poolConfig.PoolSize = 1 + // Initial creation + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + pool, err := NewMRDPool(t.poolConfig, nil) + require.NoError(t.T(), err) + entry := pool.Next() + // Simulate invalid entry + entry.mu.Lock() + entry.mrd = nil + entry.mu.Unlock() + fallbackHandle := []byte("fallback") + // Expectation: Recreate uses fallback handle + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.MatchedBy(func(req *gcs.MultiRangeDownloaderRequest) bool { + return string(req.ReadHandle) == "fallback" + })).Return(fakeMRD, nil).Once() + + err = pool.RecreateMRD(entry, fallbackHandle) + + assert.NoError(t.T(), err) + t.bucket.AssertExpectations(t.T()) +} + +func (t *mrdPoolTest) TestRecreateMRD_UsesPeerHandle() { + t.poolConfig.PoolSize = 2 + // Initial creation + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Times(2) + pool, err := NewMRDPool(t.poolConfig, nil) + require.NoError(t.T(), err) + pool.creationWg.Wait() + // Inject a mock MRD into entry 0 that returns a specific handle + peerHandle := []byte("peer_handle") + mockMRD := fake.NewFakeMultiRangeDownloaderWithHandle(t.object, nil, peerHandle) + pool.entries[0].mu.Lock() + pool.entries[0].mrd = mockMRD + pool.entries[0].mu.Unlock() + // Entry 1 is the one we want to recreate + entryToRecreate := &pool.entries[1] + entryToRecreate.mu.Lock() + entryToRecreate.mrd = nil + entryToRecreate.mu.Unlock() + // Expectation: Recreate uses peer handle from entry 0 + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.MatchedBy(func(req *gcs.MultiRangeDownloaderRequest) bool { + return string(req.ReadHandle) == "peer_handle" + })).Return(fakeMRD, nil).Once() + + err = pool.RecreateMRD(entryToRecreate, nil) + + assert.NoError(t.T(), err) + t.bucket.AssertExpectations(t.T()) +} + +func (t *mrdPoolTest) TestRecreateMRD_Error() { + t.poolConfig.PoolSize = 1 + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + pool, err := NewMRDPool(t.poolConfig, nil) + assert.NoError(t.T(), err) + entry := pool.Next() + oldMRD := entry.mrd + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(nil, fmt.Errorf("recreate error")).Once() // Fail the recreation + + err = pool.RecreateMRD(entry, nil) + + assert.Error(t.T(), err) + assert.Same(t.T(), oldMRD, entry.mrd) // Should remain unchanged on error +} + +func (t *mrdPoolTest) TestClose() { + t.poolConfig.PoolSize = 2 + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Times(2) + pool, err := NewMRDPool(t.poolConfig, nil) + assert.NoError(t.T(), err) + + pool.Close() + + // Verify entries are cleared + for i := 0; i < len(pool.entries); i++ { + assert.Nil(t.T(), pool.entries[i].mrd) + } +} + +func (t *mrdPoolTest) TestClose_ReturnsHandle() { + t.poolConfig.PoolSize = 1 + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + pool, err := NewMRDPool(t.poolConfig, nil) + require.NoError(t.T(), err) + // Inject mock to return handle on Close/GetHandle check + expectedHandle := []byte("handle") + mockMRD := fake.NewFakeMultiRangeDownloaderWithHandle(t.object, nil, expectedHandle) + pool.entries[0].mu.Lock() + pool.entries[0].mrd = mockMRD + pool.entries[0].mu.Unlock() + + handle := pool.Close() + + assert.Equal(t.T(), expectedHandle, handle) +} + +func (t *mrdPoolTest) TestCloseDoesNotCancelDownloaderContext() { + t.poolConfig.PoolSize = 1 + fakeMRD := fake.NewFakeMultiRangeDownloader(t.object, nil) + ctxCh := make(chan context.Context, 1) + t.bucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + ctxCh <- args.Get(0).(context.Context) + }).Return(fakeMRD, nil).Once() + pool, err := NewMRDPool(t.poolConfig, nil) + require.NoError(t.T(), err) + capturedCtx := <-ctxCh + + pool.Close() + + // context.Background() never gets canceled and has no Done channel + require.NotNil(t.T(), capturedCtx) + assert.Nil(t.T(), capturedCtx.Done()) + assert.NoError(t.T(), capturedCtx.Err()) +} diff --git a/internal/gcsx/multi_range_downloader_wrapper.go b/internal/gcsx/multi_range_downloader_wrapper.go new file mode 100644 index 0000000000..34715d5123 --- /dev/null +++ b/internal/gcsx/multi_range_downloader_wrapper.go @@ -0,0 +1,356 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx + +import ( + "bytes" + "errors" + "fmt" + "io" + "sync" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/gcsfuse_errors" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/monitor" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "golang.org/x/net/context" +) + +func NewMultiRangeDownloaderWrapper(bucket gcs.Bucket, object *gcs.MinObject, config *cfg.Config, mrdCache *lru.Cache) (*MultiRangeDownloaderWrapper, error) { + if object == nil { + return nil, fmt.Errorf("NewMultiRangeDownloaderWrapper: Missing MinObject") + } + // In case of a local inode, MRDWrapper would be created with an empty minObject (i.e. with a minObject without any information) + // and when the object is actually created, MRDWrapper would be updated using SetMinObject method. + wrapper := MultiRangeDownloaderWrapper{ + bucket: bucket, + object: object, + config: config, + mrdCache: mrdCache, + } + return &wrapper, nil +} + +type readResult struct { + bytesRead int + err error +} + +type MultiRangeDownloaderWrapper struct { + lru.ValueType // For LRU cache compatibility + + // Holds the object implementing MultiRangeDownloader interface. + Wrapped gcs.MultiRangeDownloader + + // Bucket and object details for MultiRangeDownloader. + // Object should not be nil. + object *gcs.MinObject + bucket gcs.Bucket + + // Refcount is used to determine when to close the MultiRangeDownloader. + refCount int + // Mutex is used to synchronize access over refCount. + mu sync.RWMutex + // GCSFuse mount config. + config *cfg.Config + // MRD Read handle. Would be updated when MRD is being closed so that it can be used + // next time during MRD recreation. + handle []byte + + // MRD cache for LRU-based eviction of inactive MRD instances. + mrdCache *lru.Cache +} + +// SetMinObject sets the gcs.MinObject stored in the wrapper to passed value, only if it's non nil. +func (mrdWrapper *MultiRangeDownloaderWrapper) SetMinObject(minObj *gcs.MinObject) error { + if minObj == nil { + return fmt.Errorf("MultiRangeDownloaderWrapper::SetMinObject: Missing MinObject") + } + mrdWrapper.mu.Lock() + defer mrdWrapper.mu.Unlock() + mrdWrapper.object = minObj + return nil +} + +// wrapperKey generates a unique key for the given MultiRangeDownloaderWrapper. +// Uses the pointer address as the unique identifier, would be safe as long as +// wrapper is uniquely associated with the lifecycle of the FileInode instance. +func wrapperKey(wrapper *MultiRangeDownloaderWrapper) string { + return fmt.Sprintf("%p", wrapper) +} + +// GetMinObject returns the minObject stored in MultiRangeDownloaderWrapper. Used only for unit testing. +func (mrdWrapper *MultiRangeDownloaderWrapper) GetMinObject() *gcs.MinObject { + mrdWrapper.mu.RLock() + defer mrdWrapper.mu.RUnlock() + return mrdWrapper.object +} + +// GetRefCount returns current refcount. +func (mrdWrapper *MultiRangeDownloaderWrapper) GetRefCount() int { + mrdWrapper.mu.RLock() + defer mrdWrapper.mu.RUnlock() + return mrdWrapper.refCount +} + +// IncrementRefCount increments the refcount. +// This method should be called exactly once per user of this wrapper. +// It has to be called before using the MultiRangeDownloader. +// If lru cache is enabled and refCount was 0, the wrapper is removed from the cache. +func (mrdWrapper *MultiRangeDownloaderWrapper) IncrementRefCount() { + mrdWrapper.mu.Lock() + defer mrdWrapper.mu.Unlock() + + mrdWrapper.refCount++ + + // If refCount was 0, remove from cache (file is being reopened) + if mrdWrapper.refCount == 1 && mrdWrapper.mrdCache != nil { + deletedEntry := mrdWrapper.mrdCache.Erase(wrapperKey(mrdWrapper)) + if deletedEntry != nil { + logger.Tracef("MRDWrapper (%s) erased from cache", mrdWrapper.object.Name) + } + } +} + +// DecrementRefCount decrements the refcount. When refCount reaches 0, the wrapper +// with added to the cache for potential reuse. In case, cache is not enabled, MRD +// is closed immediately. +// Returns error on invalid usage. +// This method should be called exactly once per user of this wrapper +// when MultiRangeDownloader is no longer needed & can be cleaned up. +func (mrdWrapper *MultiRangeDownloaderWrapper) DecrementRefCount() (err error) { + mrdWrapper.mu.Lock() + defer mrdWrapper.mu.Unlock() + + if mrdWrapper.refCount <= 0 { + err = fmt.Errorf("MultiRangeDownloaderWrapper DecrementRefCount: Refcount cannot be negative") + return + } + + mrdWrapper.refCount-- + // Do nothing if MRD is in use or cache is not enabled. + if mrdWrapper.refCount > 0 || mrdWrapper.mrdCache == nil { + return + } + + // Cache with refCount 0: add the wrapper to cache and evict overflow wrappers. + evictedValues, err := mrdWrapper.mrdCache.Insert(wrapperKey(mrdWrapper), mrdWrapper) + if err != nil { + logger.Errorf("failed to insert wrapper (%s) into cache: %v", mrdWrapper.object.Name, err) + return + } + logger.Tracef("MRDWrapper (%s) added wrapper to cache", mrdWrapper.object.Name) + + // Do not proceed if no eviction happened. + if evictedValues == nil { + return nil + } + + // Evict outside all locks to avoid deadlock. + mrdWrapper.mu.Unlock() + for _, wrapper := range evictedValues { + mrdWrapper, ok := wrapper.(*MultiRangeDownloaderWrapper) + if !ok { + logger.Errorf("invalid value type, expected MultiRangeDownloaderWrapper, got %T", wrapper) + } else { + mrdWrapper.CloseMRDForEviction() + } + } + // Reacquire the lock ensuring safe defer's Unlock. + mrdWrapper.mu.Lock() + + return nil +} + +// CloseMRDForEviction closes the MRD when evicted from cache. +// This method is called after wrapper was removed from cache for eviction. +// Race protection: wrapper could be reopened (refCount>0) or re-added to cache before eviction. +func (mrdWrapper *MultiRangeDownloaderWrapper) CloseMRDForEviction() { + mrdWrapper.mu.Lock() + defer mrdWrapper.mu.Unlock() + + // Check if wrapper was reopened (refCount>0) - must skip eviction. + if mrdWrapper.refCount > 0 { + return + } + + // Check if wrapper was re-added to cache (refCount went 0→1→0 in between eviction and closure.) + // Lock order: wrapper.mu -> cache.mu (consistent with Increment/DecrementRefCount) + if mrdWrapper.mrdCache != nil && mrdWrapper.mrdCache.LookUpWithoutChangingOrder(wrapperKey(mrdWrapper)) != nil { + return + } + mrdWrapper.closeLocked() +} + +// Ensures that MultiRangeDownloader exists, creating it if it does not exist. +// LOCK_REQUIRED(mrdWrapper.mu.RLock) +func (mrdWrapper *MultiRangeDownloaderWrapper) ensureMultiRangeDownloader(ctx context.Context, traceHandle tracing.TraceHandle, forceRecreateMRD bool) (err error) { + if mrdWrapper.object == nil || mrdWrapper.bucket == nil { + return fmt.Errorf("ensureMultiRangeDownloader error: Missing minObject or bucket") + } + + // Create the MRD if it does not exist. + // In case the existing MRD is unusable due to closed stream, recreate the MRD. + if forceRecreateMRD || mrdWrapper.Wrapped == nil || mrdWrapper.Wrapped.Error() != nil { + // The calling function holds a read lock. To create a new downloader, we need to + // upgrade to a write lock. This is done by releasing the read lock, acquiring + // the write lock, and then using a deferred function to downgrade back to a + // read lock before this function returns. + mrdWrapper.mu.RUnlock() + mrdWrapper.mu.Lock() + defer func() { + mrdWrapper.mu.Unlock() + mrdWrapper.mu.RLock() + }() + // Checking if the mrdWrapper state is same after taking the lock. + if forceRecreateMRD || mrdWrapper.Wrapped == nil || mrdWrapper.Wrapped.Error() != nil { + var mrd gcs.MultiRangeDownloader + var handle []byte + if !forceRecreateMRD { + // Get read handle from MRD if it exists otherwise use the cached read handle + if mrdWrapper.Wrapped != nil { + handle = mrdWrapper.Wrapped.GetHandle() + } else { + handle = mrdWrapper.handle + } + } + ctx = traceHandle.PropagateTraceContext(context.Background(), ctx) + mrd, err = mrdWrapper.bucket.NewMultiRangeDownloader(ctx, &gcs.MultiRangeDownloaderRequest{ + Name: mrdWrapper.object.Name, + Generation: mrdWrapper.object.Generation, + ReadCompressed: mrdWrapper.object.HasContentEncodingGzip(), + ReadHandle: handle, + }) + if err != nil { + var notFoundError *gcs.NotFoundError + if errors.As(err, ¬FoundError) { + return &gcsfuse_errors.FileClobberedError{ + Err: fmt.Errorf("ensureMultiRangeDownloader: %w", err), + ObjectName: mrdWrapper.object.Name, + } + } + return err + } + // Updating mrdWrapper.Wrapped only when MRD creation was successful. + mrdWrapper.Wrapped = mrd + } + } + return +} + +// Reads the data using MultiRangeDownloader. +func (mrdWrapper *MultiRangeDownloaderWrapper) Read(ctx context.Context, buf []byte, startOffset int64, endOffset int64, metricHandle metrics.MetricHandle, traceHandle tracing.TraceHandle, forceCreateMRD bool) (bytesRead int, err error) { + // Bidi Api with 0 as read_limit means no limit whereas we do not want to read anything with empty buffer. + // Hence, handling it separately. + if len(buf) == 0 { + return 0, nil + } + + if traceHandle == nil { + traceHandle = tracing.NewNoopTracer() + } + + mrdWrapper.mu.RLock() + err = mrdWrapper.ensureMultiRangeDownloader(ctx, traceHandle, forceCreateMRD) + if err != nil { + err = fmt.Errorf("MultiRangeDownloaderWrapper::Read: Error in creating MultiRangeDownloader: %w", err) + mrdWrapper.mu.RUnlock() + return + } + + // We will only read what is requested by the client. Hence, capping end to the requested value. + if endOffset > startOffset+int64(len(buf)) { + endOffset = startOffset + int64(len(buf)) + } + + buffer := bytes.NewBuffer(buf) + buffer.Reset() + done := make(chan readResult, 1) + + mu := sync.Mutex{} + defer func() { + mu.Lock() + close(done) + done = nil + mu.Unlock() + }() + + start := time.Now() + mrdWrapper.Wrapped.Add(buffer, startOffset, endOffset-startOffset, func(offsetAddCallback int64, bytesReadAddCallback int64, e error) { + defer func() { + mu.Lock() + if done != nil { + done <- readResult{bytesRead: int(bytesReadAddCallback), err: e} + } + mu.Unlock() + }() + + if e != nil && e != io.EOF { + e = fmt.Errorf("error in Add call: %w", e) + } + }) + mrdWrapper.mu.RUnlock() + + if !mrdWrapper.config.FileSystem.IgnoreInterrupts { + select { + case <-ctx.Done(): + err = ctx.Err() + case res := <-done: + bytesRead = res.bytesRead + err = res.err + } + } else { + res := <-done + bytesRead = res.bytesRead + err = res.err + } + if err != nil { + err = fmt.Errorf("MultiRangeDownloaderWrapper::Read: %w", err) + logger.Error(err.Error()) + } + monitor.CaptureMultiRangeDownloaderMetrics(ctx, metricHandle, "MultiRangeDownloader::Add", start) + + return +} + +// closeLocked closes the MultiRangeDownloader. +// LOCK_REQUIRED(mrdWrapper.mu.Lock) +func (mrdWrapper *MultiRangeDownloaderWrapper) closeLocked() { + if mrdWrapper.Wrapped == nil { + return + } + + // Save handle for potential recreation + mrdWrapper.handle = mrdWrapper.Wrapped.GetHandle() + + // Close the MRD + if err := mrdWrapper.Wrapped.Close(); err != nil { + logger.Warnf("Error closing MRD (%s): %v", mrdWrapper.object.Name, err) + return + } + logger.Tracef("MRDWrapper (%s) closed MRD", mrdWrapper.object.Name) + mrdWrapper.Wrapped = nil +} + +// Size returns the size of the wrapper for LRU cache accounting. +// Later, we can set to the number of MRD instances within the cache. +func (mrdWrapper *MultiRangeDownloaderWrapper) Size() uint64 { + return 1 +} diff --git a/internal/gcsx/multi_range_downloader_wrapper_test.go b/internal/gcsx/multi_range_downloader_wrapper_test.go new file mode 100644 index 0000000000..bfea2c162e --- /dev/null +++ b/internal/gcsx/multi_range_downloader_wrapper_test.go @@ -0,0 +1,666 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx + +import ( + "context" + "fmt" + "io" + "sync" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/gcsfuse_errors" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + testutil "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type mrdWrapperTest struct { + suite.Suite + object *gcs.MinObject + objectData []byte + mockBucket *storage.TestifyMockBucket + mrdWrapper *MultiRangeDownloaderWrapper +} + +func TestMRDWrapperTestSuite(t *testing.T) { + suite.Run(t, new(mrdWrapperTest)) +} + +func (t *mrdWrapperTest) SetupTest() { + var err error + t.object = &gcs.MinObject{ + Name: "foo", + Size: 100, + Generation: 1234, + } + t.objectData = testutil.GenerateRandomBytes(int(t.object.Size)) + // Create the bucket. + t.mockBucket = new(storage.TestifyMockBucket) + t.mrdWrapper, err = NewMultiRangeDownloaderWrapper(t.mockBucket, t.object, &cfg.Config{}, nil) + assert.Nil(t.T(), err, "Error in creating MRDWrapper") + t.mrdWrapper.Wrapped = fake.NewFakeMultiRangeDownloaderWithSleep(t.object, t.objectData, time.Microsecond) + t.mrdWrapper.refCount = 0 +} + +func (t *mrdWrapperTest) Test_IncrementRefCount_ParallelUpdates() { + const finalRefCount int = 1 + wg := sync.WaitGroup{} + for range finalRefCount { + wg.Add(1) + go func() { + t.mrdWrapper.IncrementRefCount() + wg.Done() + }() + } + wg.Wait() + + assert.Equal(t.T(), finalRefCount, t.mrdWrapper.refCount) +} + +func (t *mrdWrapperTest) Test_DecrementRefCount_ParallelUpdates() { + const finalRefCount int = 0 + maxRefCount := 10 + wg := sync.WaitGroup{} + // Incrementing refcount in parallel. + for range maxRefCount { + wg.Add(1) + go func() { + t.mrdWrapper.IncrementRefCount() + wg.Done() + }() + } + wg.Wait() + // Decrementing refcount in parallel. + for range maxRefCount { + wg.Add(1) + go func() { + err := t.mrdWrapper.DecrementRefCount() + assert.Nil(t.T(), err) + wg.Done() + }() + } + wg.Wait() + + assert.Equal(t.T(), finalRefCount, t.mrdWrapper.GetRefCount()) + assert.NotNil(t.T(), t.mrdWrapper.Wrapped) +} + +func (t *mrdWrapperTest) Test_DecrementRefCount_InvalidUse() { + errMsg := "MultiRangeDownloaderWrapper DecrementRefCount: Refcount cannot be negative" + assert.ErrorContains(t.T(), t.mrdWrapper.DecrementRefCount(), errMsg) +} + +func (t *mrdWrapperTest) Test_Read() { + testCases := []struct { + name string + start int + end int + }{ + { + name: "ReadFull", + start: 0, + end: int(t.object.Size), + }, + { + name: "ReadChunk", + start: 10, + end: 10 + int(t.object.Size)/2, + }, + { + name: "ReadEmpty", + start: 10, + end: 10, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + buf := make([]byte, tc.end-tc.start) + t.mrdWrapper.Wrapped = nil + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloaderWithSleep(t.object, t.objectData, time.Microsecond)) + + bytesRead, err := t.mrdWrapper.Read(context.Background(), buf, int64(tc.start), int64(tc.end), metrics.NewNoopMetrics(), tracing.NewNoopTracer(), false) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), tc.end-tc.start, bytesRead) + assert.Equal(t.T(), t.objectData[tc.start:tc.end], buf[:bytesRead]) + }) + } +} + +func (t *mrdWrapperTest) Test_Read_ErrorInCreatingMRD() { + t.mrdWrapper.Wrapped = nil + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(nil, fmt.Errorf("Error in creating MRD")).Once() + + bytesRead, err := t.mrdWrapper.Read(context.Background(), make([]byte, t.object.Size), 0, int64(t.object.Size), metrics.NewNoopMetrics(), tracing.NewNoopTracer(), false) + + assert.ErrorContains(t.T(), err, "MultiRangeDownloaderWrapper::Read: Error in creating MultiRangeDownloader") + assert.Equal(t.T(), 0, bytesRead) +} + +func (t *mrdWrapperTest) Test_Read_ShortRead() { + t.mrdWrapper.Wrapped = nil + // Configure the fake MRD to return a short read. + fakeMRD := fake.NewFakeMultiRangeDownloaderWithShortRead(t.object, t.objectData) + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fakeMRD, nil).Once() + + bytesRead, err := t.mrdWrapper.Read(context.Background(), make([]byte, t.object.Size), 0, int64(t.object.Size), metrics.NewNoopMetrics(), tracing.NewNoopTracer(), false) + + assert.NoError(t.T(), err) + assert.Less(t.T(), bytesRead, int(t.object.Size)) +} + +func (t *mrdWrapperTest) TestReadContextCancelledWithInterruptsEnabled() { + t.mrdWrapper.Wrapped = nil + t.mrdWrapper.config = &cfg.Config{FileSystem: cfg.FileSystemConfig{IgnoreInterrupts: false}} + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloaderWithSleep(t.object, t.objectData, time.Microsecond), nil).Once() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + bytesRead, err := t.mrdWrapper.Read(ctx, make([]byte, t.object.Size), 0, int64(t.object.Size), metrics.NewNoopMetrics(), tracing.NewNoopTracer(), false) + + require.Error(t.T(), err) + assert.ErrorContains(t.T(), err, "context canceled") + assert.Equal(t.T(), 0, bytesRead) +} + +func (t *mrdWrapperTest) TestReadContextCancelledWithInterruptsDisabled() { + t.mrdWrapper.config = &cfg.Config{FileSystem: cfg.FileSystemConfig{IgnoreInterrupts: true}} + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloaderWithSleep(t.object, t.objectData, time.Microsecond), nil).Once() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + bytesRead, err := t.mrdWrapper.Read(ctx, make([]byte, t.object.Size), 0, int64(t.object.Size), metrics.NewNoopMetrics(), tracing.NewNoopTracer(), false) + + require.NoError(t.T(), err) + assert.Equal(t.T(), 100, bytesRead) +} + +func (t *mrdWrapperTest) Test_Read_EOF() { + t.mrdWrapper.Wrapped = nil + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloaderWithSleepAndDefaultError(t.object, t.objectData, time.Microsecond, io.EOF), nil).Once() + + _, err := t.mrdWrapper.Read(context.Background(), make([]byte, t.object.Size), 0, int64(t.object.Size), metrics.NewNoopMetrics(), tracing.NewNoopTracer(), false) + + assert.ErrorIs(t.T(), err, io.EOF) +} + +func (t *mrdWrapperTest) Test_Read_Error() { + t.mrdWrapper.Wrapped = nil + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloaderWithSleepAndDefaultError(t.object, t.objectData, time.Microsecond, fmt.Errorf("Error")), nil).Once() + + bytesRead, err := t.mrdWrapper.Read(context.Background(), make([]byte, t.object.Size), 0, int64(t.object.Size), metrics.NewNoopMetrics(), tracing.NewNoopTracer(), false) + + assert.ErrorContains(t.T(), err, "error in Add call") + assert.Equal(t.T(), 0, bytesRead) +} + +func (t *mrdWrapperTest) Test_NewMultiRangeDownloaderWrapper() { + testCases := []struct { + name string + bucket gcs.Bucket + obj *gcs.MinObject + err error + }{ + { + name: "ValidParameters", + bucket: t.mockBucket, + obj: t.object, + err: nil, + }, + { + name: "NilMinObject", + bucket: t.mockBucket, + obj: nil, + err: fmt.Errorf("NewMultiRangeDownloaderWrapper: Missing MinObject"), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + _, err := NewMultiRangeDownloaderWrapper(tc.bucket, tc.obj, &cfg.Config{}, nil) + if tc.err == nil { + assert.NoError(t.T(), err) + } else { + assert.Error(t.T(), err) + assert.EqualError(t.T(), err, tc.err.Error()) + } + }) + } +} + +func (t *mrdWrapperTest) Test_SetMinObject() { + testCases := []struct { + name string + obj *gcs.MinObject + err error + }{ + { + name: "ValidMinObject", + obj: t.object, + err: nil, + }, + { + name: "NilMinObject", + obj: nil, + err: fmt.Errorf("MultiRangeDownloaderWrapper::SetMinObject: Missing MinObject"), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + err := t.mrdWrapper.SetMinObject(tc.obj) + if tc.err == nil { + assert.NoError(t.T(), err) + } else { + assert.Error(t.T(), err) + assert.EqualError(t.T(), err, tc.err.Error()) + } + }) + } +} + +func (t *mrdWrapperTest) Test_EnsureMultiRangeDownloader() { + testCases := []struct { + name string + obj *gcs.MinObject + bucket gcs.Bucket + err error + }{ + { + name: "ValidMinObject", + obj: t.object, + bucket: t.mockBucket, + err: nil, + }, + { + name: "NilMinObject", + obj: nil, + bucket: t.mockBucket, + err: fmt.Errorf("ensureMultiRangeDownloader error: Missing minObject or bucket"), + }, + { + name: "NilBucket", + obj: t.object, + bucket: nil, + err: fmt.Errorf("ensureMultiRangeDownloader error: Missing minObject or bucket"), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.mrdWrapper.bucket = tc.bucket + t.mrdWrapper.object = tc.obj + t.mrdWrapper.Wrapped = nil + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloaderWithSleep(t.object, t.objectData, time.Microsecond)) + t.mrdWrapper.mu.RLock() + defer t.mrdWrapper.mu.RUnlock() + err := t.mrdWrapper.ensureMultiRangeDownloader(context.Background(), tracing.NewNoopTracer(), false) + if tc.err == nil { + assert.NoError(t.T(), err) + assert.NotNil(t.T(), t.mrdWrapper.Wrapped) + } else { + assert.Error(t.T(), err) + assert.EqualError(t.T(), err, tc.err.Error()) + assert.Nil(t.T(), t.mrdWrapper.Wrapped) + } + }) + } +} + +func (t *mrdWrapperTest) Test_EnsureMultiRangeDownloader_UnusableExistingMRDTriggersRecreation() { + t.mrdWrapper.bucket = t.mockBucket + t.mrdWrapper.object = t.object + t.mrdWrapper.Wrapped = fake.NewFakeMultiRangeDownloaderWithStatusError(t.object, t.objectData, fmt.Errorf("MRD is unusable...")) + + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloaderWithSleep(t.object, t.objectData, time.Microsecond)) + t.mrdWrapper.mu.RLock() + defer t.mrdWrapper.mu.RUnlock() + + err := t.mrdWrapper.ensureMultiRangeDownloader(context.Background(), tracing.NewNoopTracer(), false) + + assert.NoError(t.T(), err) + assert.NotNil(t.T(), t.mrdWrapper.Wrapped) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *mrdWrapperTest) Test_EnsureMultiRangeDownloader_UsableExistingMRDPreventsRecreation() { + t.mrdWrapper.bucket = t.mockBucket + t.mrdWrapper.object = t.object + t.mrdWrapper.Wrapped = fake.NewFakeMultiRangeDownloaderWithStatusError(t.object, t.objectData, nil) + t.mrdWrapper.mu.RLock() + defer t.mrdWrapper.mu.RUnlock() + + err := t.mrdWrapper.ensureMultiRangeDownloader(context.Background(), tracing.NewNoopTracer(), false) + + assert.NoError(t.T(), err) + assert.NotNil(t.T(), t.mrdWrapper.Wrapped) + t.mockBucket.AssertNotCalled(t.T(), "NewMultiRangeDownloader") +} + +func (t *mrdWrapperTest) Test_EnsureMultiRangeDownloader_ForceRecreateMRD() { + t.mrdWrapper.bucket = t.mockBucket + t.mrdWrapper.object = t.object + t.mrdWrapper.Wrapped = nil + // First call to create an MRD. + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloaderWithSleep(t.object, t.objectData, time.Microsecond), nil).Once() + t.mrdWrapper.mu.RLock() + err := t.mrdWrapper.ensureMultiRangeDownloader(context.Background(), tracing.NewNoopTracer(), false) + t.mrdWrapper.mu.RUnlock() + require.NoError(t.T(), err) + initialMRD := t.mrdWrapper.Wrapped + require.NotNil(t.T(), initialMRD) + + // Second call with forceRecreateMRD=true should create a new MRD. + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloaderWithSleep(t.object, t.objectData, time.Microsecond), nil).Once() + t.mrdWrapper.mu.RLock() + err = t.mrdWrapper.ensureMultiRangeDownloader(context.Background(), tracing.NewNoopTracer(), true) + t.mrdWrapper.mu.RUnlock() + + require.NoError(t.T(), err) + assert.NotNil(t.T(), t.mrdWrapper.Wrapped) + assert.NotSame(t.T(), initialMRD, t.mrdWrapper.Wrapped, "A new MRD instance should have been created") + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *mrdWrapperTest) Test_EnsureMultiRangeDownloader_FileClobbered() { + t.mrdWrapper.Wrapped = nil + notFoundErr := &gcs.NotFoundError{Err: fmt.Errorf("not found")} + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(nil, notFoundErr).Once() + + t.mrdWrapper.mu.RLock() + defer t.mrdWrapper.mu.RUnlock() + err := t.mrdWrapper.ensureMultiRangeDownloader(context.Background(), tracing.NewNoopTracer(), false) + + require.Error(t.T(), err) + var clobberedErr *gcsfuse_errors.FileClobberedError + assert.ErrorAs(t.T(), err, &clobberedErr) + assert.Nil(t.T(), t.mrdWrapper.Wrapped) +} + +// mrdWrapperCacheTest inherits from mrdWrapperTest and adds cache functionality. +type mrdWrapperCacheTest struct { + mrdWrapperTest + cache *lru.Cache +} + +func TestMRDWrapperCacheTestSuite(t *testing.T) { + suite.Run(t, new(mrdWrapperCacheTest)) +} + +func (t *mrdWrapperCacheTest) SetupTest() { + t.mrdWrapperTest.SetupTest() + + // Recreate wrapper with cache enabled + t.cache = lru.NewCache(3) + var err error + t.mrdWrapper, err = NewMultiRangeDownloaderWrapper( + t.mockBucket, + t.object, + &cfg.Config{}, + t.cache, + ) + assert.Nil(t.T(), err, "Error in creating MRDWrapper with cache") + t.mrdWrapper.Wrapped = fake.NewFakeMultiRangeDownloaderWithSleep(t.object, t.objectData, time.Microsecond) + t.mrdWrapper.refCount = 0 +} + +func (t *mrdWrapperCacheTest) Test_Cache_AddAndRemove() { + key := wrapperKey(t.mrdWrapper) + + // Act: Open, close, and reopen file + t.mrdWrapper.IncrementRefCount() + err := t.mrdWrapper.DecrementRefCount() + assert.NotNil(t.T(), t.cache.LookUpWithoutChangingOrder(key), "Wrapper should be in cache.") + t.mrdWrapper.IncrementRefCount() + + // Assert: MRD reused and removed from cache on reopen + assert.NoError(t.T(), err) + assert.Equal(t.T(), 1, t.mrdWrapper.refCount) + assert.Nil(t.T(), t.cache.LookUpWithoutChangingOrder(key), "Wrapper should be removed from cache") + assert.NotNil(t.T(), t.mrdWrapper.Wrapped, "MRD should still exist (reused)") +} + +// Override parent test - with cache enabled, MRD stays pooled +func (t *mrdWrapperCacheTest) Test_DecrementRefCount_ParallelUpdates() { + // Arrange + const finalRefCount int = 0 + maxRefCount := 10 + wg := sync.WaitGroup{} + key := wrapperKey(t.mrdWrapper) + + // Act: Increment refcount in parallel + for range maxRefCount { + wg.Add(1) + go func() { + t.mrdWrapper.IncrementRefCount() + wg.Done() + }() + } + wg.Wait() + + // Act: Decrement refcount in parallel + for range maxRefCount { + wg.Add(1) + go func() { + err := t.mrdWrapper.DecrementRefCount() + assert.Nil(t.T(), err) + wg.Done() + }() + } + wg.Wait() + + // Assert: Final state is refCount=0, MRD pooled in cache + assert.Equal(t.T(), finalRefCount, t.mrdWrapper.GetRefCount()) + assert.NotNil(t.T(), t.mrdWrapper.Wrapped, "MRD should be pooled in cache") + assert.NotNil(t.T(), t.cache.LookUpWithoutChangingOrder(key), "Wrapper should be in cache") +} + +func (t *mrdWrapperCacheTest) Test_Cache_EvictionOnOverflow() { + // Arrange: Create 4 wrappers (cache max is 3) + wrappers := make([]*MultiRangeDownloaderWrapper, 4) + for i := range 4 { + obj := &gcs.MinObject{ + Name: fmt.Sprintf("file%d", i), + Size: 100, + Generation: int64(1000 + i), + } + wrapper, err := NewMultiRangeDownloaderWrapper( + t.mockBucket, + obj, + &cfg.Config{}, + t.cache, + ) + assert.NoError(t.T(), err) + wrapper.Wrapped = fake.NewFakeMultiRangeDownloaderWithSleep(obj, t.objectData, time.Microsecond) + wrappers[i] = wrapper + } + + // Act: Open and close all 4 wrappers (triggers eviction on 4th) + for i := range 4 { + wrappers[i].IncrementRefCount() + err := wrappers[i].DecrementRefCount() + assert.NoError(t.T(), err) + } + + // Assert: First wrapper evicted (LRU), last 3 remain in cache + assert.Nil(t.T(), wrappers[0].Wrapped, "First wrapper's MRD should be closed (evicted)") + for i := range wrappers[1:] { + key := wrapperKey(wrappers[i+1]) + assert.NotNil(t.T(), t.cache.LookUpWithoutChangingOrder(key), "Wrapper %d should be in cache", i+1) + assert.NotNil(t.T(), wrappers[i+1].Wrapped, "Wrapper %d MRD should exist (pooled)", i+1) + } +} + +func (t *mrdWrapperCacheTest) Test_Cache_DeletedIfReopened() { + // Arrange: Create 3 wrappers and fill cache + wrappers := make([]*MultiRangeDownloaderWrapper, 3) + for i := range 3 { + obj := &gcs.MinObject{ + Name: fmt.Sprintf("file%d", i), + Size: 100, + Generation: int64(1000 + i), + } + wrapper, err := NewMultiRangeDownloaderWrapper( + t.mockBucket, + obj, + &cfg.Config{}, + t.cache, + ) + assert.NoError(t.T(), err) + wrapper.Wrapped = fake.NewFakeMultiRangeDownloaderWithSleep(obj, t.objectData, time.Microsecond) + wrappers[i] = wrapper + wrappers[i].IncrementRefCount() + err = wrappers[i].DecrementRefCount() + assert.NoError(t.T(), err) + } + + // Act: Reopen wrapper 0 -> should remove it from cache + wrappers[0].IncrementRefCount() + + // Assert: wrapper 0 will be deleted from cache. + assert.Nil(t.T(), t.cache.LookUpWithoutChangingOrder(wrapperKey(wrappers[0])), "Wrapper 0 should not be in cache") +} + +func (t *mrdWrapperCacheTest) Test_Cache_ConcurrentAddRemove() { + // Arrange + const numGoroutines = 10 + const numIterations = 100 + wg := sync.WaitGroup{} + + // Act: Concurrent open/close cycles from multiple goroutines + for range numGoroutines { + wg.Add(1) + go func() { + defer wg.Done() + for range numIterations { + t.mrdWrapper.IncrementRefCount() + err := t.mrdWrapper.DecrementRefCount() + assert.NoError(t.T(), err) + } + }() + } + wg.Wait() + + // Assert: Final state is refCount=0 (no deadlocks or panics) + assert.Equal(t.T(), 0, t.mrdWrapper.refCount, "RefCount should be 0 after all operations") + assert.NotNil(t.T(), t.cache.LookUpWithoutChangingOrder(wrapperKey(t.mrdWrapper)), "Wrapper should be in cache") +} + +func (t *mrdWrapperCacheTest) Test_Cache_Disabled() { + // Arrange: Create wrapper with nil cache (disabled) + wrapper, err := NewMultiRangeDownloaderWrapper( + t.mockBucket, + t.object, + &cfg.Config{}, + nil, // Cache disabled + ) + assert.NoError(t.T(), err) + wrapper.Wrapped = fake.NewFakeMultiRangeDownloaderWithSleep(t.object, t.objectData, time.Microsecond) + + // Act: Open and close file + wrapper.IncrementRefCount() + err = wrapper.DecrementRefCount() + assert.NoError(t.T(), err) + + // Assert: MRD will be open forever since cache is disabled. + assert.NotNil(t.T(), wrapper.Wrapped, "MRD should be open when cache disabled") +} + +func (t *mrdWrapperCacheTest) Test_Cache_EvictionRaceWithRepool() { + // Arrange: Add wrapper to cache then fill with 3 more to trigger eviction + t.mrdWrapper.IncrementRefCount() + err := t.mrdWrapper.DecrementRefCount() + assert.NoError(t.T(), err) + for i := range 3 { + obj := &gcs.MinObject{ + Name: fmt.Sprintf("file%d", i), + Size: 100, + Generation: int64(1000 + i), + } + wrapper, err := NewMultiRangeDownloaderWrapper( + t.mockBucket, + obj, + &cfg.Config{}, + t.cache, + ) + assert.NoError(t.T(), err) + wrapper.Wrapped = fake.NewFakeMultiRangeDownloaderWithSleep(obj, t.objectData, time.Microsecond) + wrapper.IncrementRefCount() + err = wrapper.DecrementRefCount() + assert.NoError(t.T(), err) + } + + // Act: Access evicted wrapper (should recreate MRD) + buf := make([]byte, 10) + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return( + fake.NewFakeMultiRangeDownloaderWithSleep(t.object, t.objectData, time.Microsecond), + nil, + ).Once() + bytesRead, err := t.mrdWrapper.Read(context.Background(), buf, 0, 10, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), false) + + // Assert: MRD recreated successfully after eviction + assert.NoError(t.T(), err) + assert.Equal(t.T(), 10, bytesRead) + assert.NotNil(t.T(), t.mrdWrapper.Wrapped, "MRD should be recreated after eviction") +} + +func (t *mrdWrapperCacheTest) Test_Cache_MultipleEvictions() { + // Arrange: Create small cache (size 2) and 5 wrappers + smallCache := lru.NewCache(2) + wrappers := make([]*MultiRangeDownloaderWrapper, 5) + for i := range 5 { + obj := &gcs.MinObject{ + Name: fmt.Sprintf("file%d", i), + Size: 100, + Generation: int64(1000 + i), + } + wrapper, err := NewMultiRangeDownloaderWrapper( + t.mockBucket, + obj, + &cfg.Config{}, + smallCache, + ) + assert.NoError(t.T(), err) + wrapper.Wrapped = fake.NewFakeMultiRangeDownloaderWithSleep(obj, t.objectData, time.Microsecond) + wrappers[i] = wrapper + } + + // Act: Add all 5 wrappers (triggers batch eviction of 3) + for i := range 5 { + wrappers[i].IncrementRefCount() + err := wrappers[i].DecrementRefCount() + assert.NoError(t.T(), err) + } + + // Assert: First 3 evicted, last 2 remain in cache + for i := range wrappers[:3] { + assert.Nil(t.T(), wrappers[i].Wrapped, "Wrapper %d should be evicted", i) + } + for i := range wrappers[3:] { + assert.NotNil(t.T(), wrappers[i+3].Wrapped, "Wrapper %d should be in cache", i+3) + } +} diff --git a/internal/gcsx/prefix_bucket.go b/internal/gcsx/prefix_bucket.go index ff3a1259ec..8c90691210 100644 --- a/internal/gcsx/prefix_bucket.go +++ b/internal/gcsx/prefix_bucket.go @@ -16,11 +16,10 @@ package gcsx import ( "errors" - "io" "strings" "unicode/utf8" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "golang.org/x/net/context" ) @@ -67,15 +66,15 @@ func (b *prefixBucket) BucketType() gcs.BucketType { return b.wrapped.BucketType() } -func (b *prefixBucket) NewReader( +func (b *prefixBucket) NewReaderWithReadHandle( ctx context.Context, - req *gcs.ReadObjectRequest) (rc io.ReadCloser, err error) { + req *gcs.ReadObjectRequest) (rd gcs.StorageReader, err error) { // Modify the request and call through. mReq := new(gcs.ReadObjectRequest) *mReq = *req mReq.Name = b.wrappedName(req.Name) - rc, err = b.wrapped.NewReader(ctx, mReq) + rd, err = b.wrapped.NewReaderWithReadHandle(ctx, mReq) return } @@ -111,7 +110,21 @@ func (b *prefixBucket) CreateObjectChunkWriter(ctx context.Context, req *gcs.Cre return wc, err } -func (b *prefixBucket) FinalizeUpload(ctx context.Context, w gcs.Writer) (o *gcs.Object, err error) { +func (b *prefixBucket) CreateAppendableObjectWriter(ctx context.Context, req *gcs.CreateObjectChunkWriterRequest) (gcs.Writer, error) { + // Modify the request and call through. + mReq := new(gcs.CreateObjectChunkWriterRequest) + *mReq = *req + mReq.Name = b.wrappedName(req.Name) + + wc, err := b.wrapped.CreateAppendableObjectWriter(ctx, mReq) + if err != nil { + return nil, err + } + + return wc, err +} + +func (b *prefixBucket) FinalizeUpload(ctx context.Context, w gcs.Writer) (o *gcs.MinObject, err error) { o, err = b.wrapped.FinalizeUpload(ctx, w) // Modify the returned object. if o != nil { @@ -120,6 +133,15 @@ func (b *prefixBucket) FinalizeUpload(ctx context.Context, w gcs.Writer) (o *gcs return } +func (b *prefixBucket) FlushPendingWrites(ctx context.Context, w gcs.Writer) (o *gcs.MinObject, err error) { + o, err = b.wrapped.FlushPendingWrites(ctx, w) + // Modify the returned object. + if o != nil { + o.Name = b.localName(o.Name) + } + return +} + func (b *prefixBucket) CopyObject( ctx context.Context, req *gcs.CopyObjectRequest) (o *gcs.Object, err error) { @@ -235,15 +257,34 @@ func (b *prefixBucket) DeleteObject( return } +func (b *prefixBucket) MoveObject(ctx context.Context, req *gcs.MoveObjectRequest) (*gcs.Object, error) { + // Modify the request and call through. + mReq := new(gcs.MoveObjectRequest) + *mReq = *req + mReq.SrcName = b.wrappedName(req.SrcName) + mReq.DstName = b.wrappedName(req.DstName) + + o, err := b.wrapped.MoveObject(ctx, mReq) + + // Modify the returned object. + if o != nil { + o.Name = b.localName(o.Name) + } + + return o, err +} + func (b *prefixBucket) DeleteFolder(ctx context.Context, folderName string) (err error) { mFolderName := b.wrappedName(folderName) return b.wrapped.DeleteFolder(ctx, mFolderName) } -func (b *prefixBucket) GetFolder(ctx context.Context, folderName string) (folder *gcs.Folder, err error) { - mFolderName := b.wrappedName(folderName) +func (b *prefixBucket) GetFolder(ctx context.Context, req *gcs.GetFolderRequest) (folder *gcs.Folder, err error) { + mReq := new(gcs.GetFolderRequest) + *mReq = *req + mReq.Name = b.wrappedName(req.Name) - f, err := b.wrapped.GetFolder(ctx, mFolderName) + f, err := b.wrapped.GetFolder(ctx, mReq) // Modify the returned folder. if f != nil { @@ -277,3 +318,18 @@ func (b *prefixBucket) RenameFolder(ctx context.Context, folderName string, dest return f, err } + +func (b *prefixBucket) NewMultiRangeDownloader( + ctx context.Context, req *gcs.MultiRangeDownloaderRequest) (mrd gcs.MultiRangeDownloader, err error) { + // Modify the request and call through. + mReq := new(gcs.MultiRangeDownloaderRequest) + *mReq = *req + mReq.Name = b.wrappedName(req.Name) + + mrd, err = b.wrapped.NewMultiRangeDownloader(ctx, mReq) + return +} + +func (b *prefixBucket) GCSName(object *gcs.MinObject) string { + return b.wrappedName(b.wrapped.GCSName(object)) +} diff --git a/internal/gcsx/prefix_bucket_test.go b/internal/gcsx/prefix_bucket_test.go index 6b0aaa55f1..fd8205395e 100644 --- a/internal/gcsx/prefix_bucket_test.go +++ b/internal/gcsx/prefix_bucket_test.go @@ -15,61 +15,61 @@ package gcsx_test import ( + "bytes" "errors" "io" "strings" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/fake" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/net/context" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" - . "github.com/jacobsa/oglematchers" - . "github.com/jacobsa/ogletest" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/stretchr/testify/suite" + "github.com/jacobsa/timeutil" ) -func TestPrefixBucket(t *testing.T) { RunTests(t) } - //////////////////////////////////////////////////////////////////////// // Boilerplate //////////////////////////////////////////////////////////////////////// type PrefixBucketTest struct { + suite.Suite ctx context.Context prefix string wrapped gcs.Bucket bucket gcs.Bucket } -var _ SetUpInterface = &PrefixBucketTest{} - -func init() { RegisterTestSuite(&PrefixBucketTest{}) } +func TestPrefixBucket(t *testing.T) { + suite.Run(t, new(PrefixBucketTest)) +} -func (t *PrefixBucketTest) SetUp(ti *TestInfo) { +func (t *PrefixBucketTest) SetupTest() { var err error - t.ctx = ti.Ctx + t.ctx = context.Background() t.prefix = "foo_" - t.wrapped = fake.NewFakeBucket(timeutil.RealClock(), "some_bucket", gcs.NonHierarchical) + t.wrapped = fake.NewFakeBucket(timeutil.RealClock(), "some_bucket", gcs.BucketType{}) t.bucket, err = gcsx.NewPrefixBucket(t.prefix, t.wrapped) - AssertEq(nil, err) + assert.NoError(t.T(), err) } //////////////////////////////////////////////////////////////////////// // Tests //////////////////////////////////////////////////////////////////////// -func (t *PrefixBucketTest) Name() { - ExpectEq(t.wrapped.Name(), t.bucket.Name()) +func (t *PrefixBucketTest) Test_Name() { + assert.Equal(t.T(), t.wrapped.Name(), t.bucket.Name()) } -func (t *PrefixBucketTest) NewReader() { +func (t *PrefixBucketTest) Test_NewReader() { var err error suffix := "taco" name := t.prefix + suffix @@ -77,24 +77,214 @@ func (t *PrefixBucketTest) NewReader() { // Create an object through the back door. _, err = storageutil.CreateObject(t.ctx, t.wrapped, name, []byte(contents)) - AssertEq(nil, err) + assert.Equal(t.T(), nil, err) // Read it through the prefix bucket. - rc, err := t.bucket.NewReader( + rc, err := t.bucket.NewReaderWithReadHandle( t.ctx, &gcs.ReadObjectRequest{ Name: suffix, }) - AssertEq(nil, err) + assert.Equal(t.T(), nil, err) defer rc.Close() actual, err := io.ReadAll(rc) - AssertEq(nil, err) - ExpectEq(contents, string(actual)) + assert.NoError(t.T(), err) + assert.Equal(t.T(), contents, string(actual)) +} + +func (t *PrefixBucketTest) Test_NewReaderWithReadHandle() { + var err error + suffix := "taco" + name := t.prefix + suffix + contents := "foobar" + // Create an object through the back door. + _, err = storageutil.CreateObject(t.ctx, t.wrapped, name, []byte(contents)) + assert.Equal(t.T(), nil, err) + + // Read it through the prefix bucket with read handle. + rc, err := t.bucket.NewReaderWithReadHandle( + t.ctx, + &gcs.ReadObjectRequest{ + Name: suffix, + ReadHandle: []byte("new-handle"), + }) + + assert.Equal(t.T(), nil, err) + defer rc.Close() + actual, err := io.ReadAll(rc) + assert.NoError(t.T(), nil, err) + assert.Equal(t.T(), contents, string(actual)) + assert.Equal(t.T(), string(rc.ReadHandle()), "opaque-handle") +} + +func (t *PrefixBucketTest) Test_NewReaderWithNilReadHandle() { + var err error + suffix := "taco" + name := t.prefix + suffix + contents := "foobar" + // Create an object through the back door. + _, err = storageutil.CreateObject(t.ctx, t.wrapped, name, []byte(contents)) + assert.Equal(t.T(), nil, err) + + // Read it through the prefix bucket with out read handle. + rc, err := t.bucket.NewReaderWithReadHandle( + t.ctx, + &gcs.ReadObjectRequest{ + Name: suffix, + }) + + assert.NoError(t.T(), err) + defer rc.Close() + actual, err := io.ReadAll(rc) + assert.NoError(t.T(), err) + assert.Equal(t.T(), contents, string(actual)) + assert.Equal(t.T(), string(rc.ReadHandle()), "opaque-handle") +} + +func (t *PrefixBucketTest) Test_NewMultiRangeReader_WithFullContentRead() { + var err error + suffix := "taco" + name := t.prefix + suffix + contents := "foobar" + // Create an object through the back door. + _, err = storageutil.CreateObject(t.ctx, t.wrapped, name, []byte(contents)) + assert.Equal(t.T(), nil, err) + + // Read it through the prefix bucket. + mrd, err := t.bucket.NewMultiRangeDownloader( + t.ctx, + &gcs.MultiRangeDownloaderRequest{ + Name: suffix, + }) + + assert.NoError(t.T(), err) + assert.NotNil(t.T(), mrd) + defer func() { + assert.NoError(t.T(), mrd.Close()) + }() + + size := int64(len(contents)) + var outputString string + outputWriter := bytes.NewBufferString(outputString) + mrd.Add(outputWriter, 0, size, func(int64, int64, error) {}) + mrd.Wait() + + assert.Equal(t.T(), contents, outputWriter.String()) +} + +func (t *PrefixBucketTest) Test_NewMultiRangeReader_WithoutWait() { + var err error + suffix := "taco" + name := t.prefix + suffix + contents := "foobar" + // Create an object through the back door. + _, err = storageutil.CreateObject(t.ctx, t.wrapped, name, []byte(contents)) + assert.Equal(t.T(), nil, err) + + // Read it through the prefix bucket with out read handle. + mrd, err := t.bucket.NewMultiRangeDownloader( + t.ctx, + &gcs.MultiRangeDownloaderRequest{ + Name: suffix, + }) + + var outputString string + outputWriter := bytes.NewBufferString(outputString) + + assert.NoError(t.T(), err) + assert.NotNil(t.T(), mrd) + defer func() { + assert.NoError(t.T(), mrd.Close()) + assert.Equal(t.T(), contents, outputWriter.String()) + }() + + size := int64(len(contents)) + mrd.Add(outputWriter, 0, size, func(offset, length int64, err error) {}) +} + +func (t *PrefixBucketTest) Test_NewMultiRangeReader_WithMultipleReads() { + var err error + suffix := "taco" + name := t.prefix + suffix + contents := "foobar" + // Create an object through the back door. + _, err = storageutil.CreateObject(t.ctx, t.wrapped, name, []byte(contents)) + assert.Equal(t.T(), nil, err) + + // Read it through the prefix bucket with out read handle. + mrd, err := t.bucket.NewMultiRangeDownloader( + t.ctx, + &gcs.MultiRangeDownloaderRequest{ + Name: suffix, + }) + + assert.NoError(t.T(), err) + assert.NotNil(t.T(), mrd) + defer func() { + assert.NoError(t.T(), mrd.Close()) + }() + + size := int64(len(contents)) + halfSize := size / 2 + var outputString1 string + outputWriter1 := bytes.NewBufferString(outputString1) + mrd.Add(outputWriter1, 0, halfSize, func(offset, length int64, err error) {}) + + var outputString2 string + outputWriter2 := bytes.NewBufferString(outputString2) + mrd.Add(outputWriter2, halfSize, halfSize, func(offset, length int64, err error) {}) + + mrd.Wait() + + assert.Equal(t.T(), "foo", outputWriter1.String()) + assert.Equal(t.T(), "bar", outputWriter2.String()) +} + +func (t *PrefixBucketTest) Test_NewMultiRangeReader_WithOutOfBoundsReadError() { + var err error + suffix := "taco" + name := t.prefix + suffix + contents := "foobar" + // Create an object through the back door. + _, err = storageutil.CreateObject(t.ctx, t.wrapped, name, []byte(contents)) + assert.Equal(t.T(), nil, err) + + // Read it through the prefix bucket with out read handle. + mrd, err := t.bucket.NewMultiRangeDownloader( + t.ctx, + &gcs.MultiRangeDownloaderRequest{ + Name: suffix, + }) + + assert.NoError(t.T(), err) + assert.NotNil(t.T(), mrd) + defer func() { + assert.Error(t.T(), mrd.Close()) + }() + + size := int64(len(contents)) + var outputString string + outputWriter := bytes.NewBufferString(outputString) + mrd.Add(outputWriter, size+1, 1, func(offset, length int64, err error) {}) } -func (t *PrefixBucketTest) CreateObject() { +func (t *PrefixBucketTest) Test_NewMultiRangeReader_WithNonexistentObjectError() { + var err error + + // Read it through the prefix bucket with out read handle. + mrd, err := t.bucket.NewMultiRangeDownloader( + t.ctx, + &gcs.MultiRangeDownloaderRequest{ + Name: "taco", + }) + + assert.Error(t.T(), err) + assert.Nil(t.T(), mrd) +} + +func (t *PrefixBucketTest) Test_CreateObject() { var err error suffix := "taco" contents := "foobar" @@ -108,17 +298,17 @@ func (t *PrefixBucketTest) CreateObject() { Contents: strings.NewReader(contents), }) - AssertEq(nil, err) - ExpectEq(suffix, o.Name) - ExpectEq("en-GB", o.ContentLanguage) + assert.Equal(t.T(), nil, err) + assert.Equal(t.T(), suffix, o.Name) + assert.Equal(t.T(), "en-GB", o.ContentLanguage) // Read it through the back door. actual, err := storageutil.ReadObject(t.ctx, t.wrapped, t.prefix+suffix) - AssertEq(nil, err) - ExpectEq(contents, string(actual)) + assert.Equal(t.T(), nil, err) + assert.Equal(t.T(), contents, string(actual)) } -func (t *PrefixBucketTest) CreateObjectChunkWriterAndFinalizeUpload() { +func (t *PrefixBucketTest) TestCreateObjectChunkWriterAndFinalizeUpload() { var err error suffix := "taco" content := []byte("foobar") @@ -128,25 +318,112 @@ func (t *PrefixBucketTest) CreateObjectChunkWriterAndFinalizeUpload() { t.ctx, &gcs.CreateObjectRequest{ Name: suffix, - ContentLanguage: "en-GB", + ContentEncoding: "gzip", Contents: nil, }, 1024, nil) - AssertEq(nil, err) + assert.NoError(t.T(), err) _, err = w.Write(content) - AssertEq(nil, err) + assert.NoError(t.T(), err) o, err := t.bucket.FinalizeUpload(t.ctx, w) - AssertEq(nil, err) - ExpectEq(suffix, o.Name) - ExpectEq("en-GB", o.ContentLanguage) + assert.NoError(t.T(), err) + assert.Equal(t.T(), suffix, o.Name) + assert.Equal(t.T(), "gzip", o.ContentEncoding) + // Read it through the back door. + actual, err := storageutil.ReadObject(t.ctx, t.wrapped, t.prefix+suffix) + assert.Equal(t.T(), nil, err) + assert.Equal(t.T(), string(content), string(actual)) +} + +func (t *PrefixBucketTest) TestCreateObjectChunkWriterAndFlushPendingWrites() { + var err error + suffix := "taco" + content := []byte("foobar") + + // Create the object. + w, err := t.bucket.CreateObjectChunkWriter( + t.ctx, + &gcs.CreateObjectRequest{ + Name: suffix, + ContentEncoding: "gzip", + Contents: nil, + }, + 1024, nil) + assert.NoError(t.T(), err) + _, err = w.Write(content) + assert.NoError(t.T(), err) + o, err := t.bucket.FlushPendingWrites(t.ctx, w) + + assert.NoError(t.T(), err) + assert.EqualValues(t.T(), int64(len(content)), o.Size) + assert.Equal(t.T(), suffix, o.Name) // Read it through the back door. actual, err := storageutil.ReadObject(t.ctx, t.wrapped, t.prefix+suffix) - AssertEq(nil, err) - ExpectEq(string(content), string(actual)) + assert.Equal(t.T(), nil, err) + assert.Equal(t.T(), string(content), string(actual)) } -func (t *PrefixBucketTest) CopyObject() { +func (t *PrefixBucketTest) TestCreateAppendableObjectWriterAndFlush() { + var err error + suffix := "taco" + content := []byte("foobar") + + // Create the object writer. + w, err := t.bucket.CreateAppendableObjectWriter( + t.ctx, + &gcs.CreateObjectChunkWriterRequest{ + CreateObjectRequest: gcs.CreateObjectRequest{ + Name: suffix, + }, + ChunkSize: 1024, + Offset: 10, + }) + assert.NoError(t.T(), err) + assert.NotNil(t.T(), w) + _, err = w.Write(content) + assert.NoError(t.T(), err) + o, err := t.bucket.FlushPendingWrites(t.ctx, w) + + assert.NoError(t.T(), err) + assert.EqualValues(t.T(), int64(len(content)), o.Size) + // Read it through the back door. + actual, err := storageutil.ReadObject(t.ctx, t.wrapped, t.prefix+suffix) + assert.Equal(t.T(), nil, err) + assert.Equal(t.T(), string(content), string(actual)) +} + +func (t *PrefixBucketTest) TestCreateAppendableObjectWriterAndClose() { + var err error + suffix := "taco" + content := []byte("foobar") + + // Create the object writer. + w, err := t.bucket.CreateAppendableObjectWriter( + t.ctx, + &gcs.CreateObjectChunkWriterRequest{ + CreateObjectRequest: gcs.CreateObjectRequest{ + Name: suffix, + }, + ChunkSize: 1024, + Offset: 10, + }) + assert.NoError(t.T(), err) + assert.NotNil(t.T(), w) + _, err = w.Write(content) + assert.NoError(t.T(), err) + o, err := t.bucket.FinalizeUpload(t.ctx, w) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), suffix, o.Name) + + // Read it through the back door. + actual, err := storageutil.ReadObject(t.ctx, t.wrapped, t.prefix+suffix) + assert.Equal(t.T(), nil, err) + assert.Equal(t.T(), string(content), string(actual)) +} + +func (t *PrefixBucketTest) Test_CopyObject() { var err error suffix := "taco" name := t.prefix + suffix @@ -154,7 +431,7 @@ func (t *PrefixBucketTest) CopyObject() { // Create an object through the back door. _, err = storageutil.CreateObject(t.ctx, t.wrapped, name, []byte(contents)) - AssertEq(nil, err) + assert.Equal(t.T(), nil, err) // Copy it to a new name. newSuffix := "burrito" @@ -165,16 +442,16 @@ func (t *PrefixBucketTest) CopyObject() { DstName: newSuffix, }) - AssertEq(nil, err) - ExpectEq(newSuffix, o.Name) + assert.Equal(t.T(), nil, err) + assert.Equal(t.T(), newSuffix, o.Name) // Read it through the back door. actual, err := storageutil.ReadObject(t.ctx, t.wrapped, t.prefix+newSuffix) - AssertEq(nil, err) - ExpectEq(contents, string(actual)) + assert.Equal(t.T(), nil, err) + assert.Equal(t.T(), contents, string(actual)) } -func (t *PrefixBucketTest) ComposeObjects() { +func (t *PrefixBucketTest) Test_ComposeObjects() { var err error suffix0 := "taco" @@ -192,7 +469,7 @@ func (t *PrefixBucketTest) ComposeObjects() { t.prefix + suffix1: []byte(contents1), }) - AssertEq(nil, err) + assert.Equal(t.T(), nil, err) // Compose them. newSuffix := "enchilada" @@ -206,16 +483,16 @@ func (t *PrefixBucketTest) ComposeObjects() { }, }) - AssertEq(nil, err) - ExpectEq(newSuffix, o.Name) + assert.Equal(t.T(), nil, err) + assert.Equal(t.T(), newSuffix, o.Name) // Read it through the back door. actual, err := storageutil.ReadObject(t.ctx, t.wrapped, t.prefix+newSuffix) - AssertEq(nil, err) - ExpectEq(contents0+contents1, string(actual)) + assert.Equal(t.T(), nil, err) + assert.Equal(t.T(), contents0+contents1, string(actual)) } -func (t *PrefixBucketTest) StatObject() { +func (t *PrefixBucketTest) Test_StatObject() { var err error suffix := "taco" name := t.prefix + suffix @@ -223,7 +500,7 @@ func (t *PrefixBucketTest) StatObject() { // Create an object through the back door. _, err = storageutil.CreateObject(t.ctx, t.wrapped, name, []byte(contents)) - AssertEq(nil, err) + assert.Equal(t.T(), nil, err) // Stat it. m, _, err := t.bucket.StatObject( @@ -232,13 +509,13 @@ func (t *PrefixBucketTest) StatObject() { Name: suffix, }) - AssertEq(nil, err) - AssertNe(nil, m) - ExpectEq(suffix, m.Name) - ExpectEq(len(contents), m.Size) + assert.Equal(t.T(), nil, err) + assert.NotEqual(t.T(), nil, m) + assert.Equal(t.T(), suffix, m.Name) + assert.Equal(t.T(), uint64(len(contents)), m.Size) } -func (t *PrefixBucketTest) ListObjects_NoOptions() { +func (t *PrefixBucketTest) Test_ListObjects_NoOptions() { var err error // Create a few objects. @@ -252,24 +529,24 @@ func (t *PrefixBucketTest) ListObjects_NoOptions() { "some_other": []byte(""), }) - AssertEq(nil, err) + assert.Equal(t.T(), nil, err) // List. l, err := t.bucket.ListObjects( t.ctx, &gcs.ListObjectsRequest{}) - AssertEq(nil, err) - AssertEq("", l.ContinuationToken) - AssertThat(l.CollapsedRuns, ElementsAre()) + assert.Equal(t.T(), nil, err) + assert.Equal(t.T(), "", l.ContinuationToken) + assert.Empty(t.T(), l.CollapsedRuns) - AssertEq(3, len(l.MinObjects)) - ExpectEq("burrito", l.MinObjects[0].Name) - ExpectEq("enchilada", l.MinObjects[1].Name) - ExpectEq("taco", l.MinObjects[2].Name) + assert.Equal(t.T(), 3, len(l.MinObjects)) + assert.Equal(t.T(), "burrito", l.MinObjects[0].Name) + assert.Equal(t.T(), "enchilada", l.MinObjects[1].Name) + assert.Equal(t.T(), "taco", l.MinObjects[2].Name) } -func (t *PrefixBucketTest) ListObjects_Prefix() { +func (t *PrefixBucketTest) Test_ListObjects_Prefix() { var err error // Create a few objects. @@ -284,7 +561,7 @@ func (t *PrefixBucketTest) ListObjects_Prefix() { "some_other": []byte(""), }) - AssertEq(nil, err) + assert.Equal(t.T(), nil, err) // List, with a prefix. l, err := t.bucket.ListObjects( @@ -293,16 +570,16 @@ func (t *PrefixBucketTest) ListObjects_Prefix() { Prefix: "burrito", }) - AssertEq(nil, err) - AssertEq("", l.ContinuationToken) - AssertThat(l.CollapsedRuns, ElementsAre()) + assert.Equal(t.T(), nil, err) + assert.Equal(t.T(), "", l.ContinuationToken) + assert.Empty(t.T(), l.CollapsedRuns) - AssertEq(2, len(l.MinObjects)) - ExpectEq("burrito0", l.MinObjects[0].Name) - ExpectEq("burrito1", l.MinObjects[1].Name) + assert.Equal(t.T(), 2, len(l.MinObjects)) + assert.Equal(t.T(), "burrito0", l.MinObjects[0].Name) + assert.Equal(t.T(), "burrito1", l.MinObjects[1].Name) } -func (t *PrefixBucketTest) ListObjects_Delimeter() { +func (t *PrefixBucketTest) Test_ListObjects_Delimeter() { var err error // Create a few objects. @@ -317,27 +594,27 @@ func (t *PrefixBucketTest) ListObjects_Delimeter() { "some_other": []byte(""), }) - AssertEq(nil, err) + assert.Equal(t.T(), nil, err) // List, with a delimiter. Make things extra interesting by using a delimiter // that is contained within the bucket prefix. - AssertNe(-1, strings.IndexByte(t.prefix, '_')) + assert.NotEqual(t.T(), -1, strings.IndexByte(t.prefix, '_')) l, err := t.bucket.ListObjects( t.ctx, &gcs.ListObjectsRequest{ Delimiter: "_", }) - AssertEq(nil, err) - AssertEq("", l.ContinuationToken) + assert.Equal(t.T(), nil, err) + assert.Equal(t.T(), "", l.ContinuationToken) - ExpectThat(l.CollapsedRuns, ElementsAre("burrito_", "enchilada_")) + assert.ElementsMatch(t.T(), l.CollapsedRuns, []string{"burrito_", "enchilada_"}) - AssertEq(1, len(l.MinObjects)) - ExpectEq("burrito", l.MinObjects[0].Name) + assert.Equal(t.T(), 1, len(l.MinObjects)) + assert.Equal(t.T(), "burrito", l.MinObjects[0].Name) } -func (t *PrefixBucketTest) ListObjects_PrefixAndDelimeter() { +func (t *PrefixBucketTest) Test_ListObjects_PrefixAndDelimeter() { var err error // Create a few objects. @@ -352,11 +629,11 @@ func (t *PrefixBucketTest) ListObjects_PrefixAndDelimeter() { "some_other": []byte(""), }) - AssertEq(nil, err) + assert.Equal(t.T(), nil, err) // List, with a delimiter and a prefix. Make things extra interesting by // using a delimiter that is contained within the bucket prefix. - AssertNe(-1, strings.IndexByte(t.prefix, '_')) + assert.NotEqual(t.T(), -1, strings.IndexByte(t.prefix, '_')) l, err := t.bucket.ListObjects( t.ctx, &gcs.ListObjectsRequest{ @@ -364,16 +641,16 @@ func (t *PrefixBucketTest) ListObjects_PrefixAndDelimeter() { Prefix: "burrito", }) - AssertEq(nil, err) - AssertEq("", l.ContinuationToken) + assert.Equal(t.T(), nil, err) + assert.Equal(t.T(), "", l.ContinuationToken) - ExpectThat(l.CollapsedRuns, ElementsAre("burrito_")) + assert.ElementsMatch(t.T(), l.CollapsedRuns, []string{"burrito_"}) - AssertEq(1, len(l.MinObjects)) - ExpectEq("burrito", l.MinObjects[0].Name) + assert.Equal(t.T(), 1, len(l.MinObjects)) + assert.Equal(t.T(), "burrito", l.MinObjects[0].Name) } -func (t *PrefixBucketTest) UpdateObject() { +func (t *PrefixBucketTest) Test_UpdateObject() { var err error suffix := "taco" name := t.prefix + suffix @@ -381,7 +658,7 @@ func (t *PrefixBucketTest) UpdateObject() { // Create an object through the back door. _, err = storageutil.CreateObject(t.ctx, t.wrapped, name, []byte(contents)) - AssertEq(nil, err) + assert.Equal(t.T(), nil, err) // Update it. newContentLanguage := "en-GB" @@ -392,12 +669,12 @@ func (t *PrefixBucketTest) UpdateObject() { ContentLanguage: &newContentLanguage, }) - AssertEq(nil, err) - ExpectEq(suffix, o.Name) - ExpectEq(newContentLanguage, o.ContentLanguage) + assert.Equal(t.T(), nil, err) + assert.Equal(t.T(), suffix, o.Name) + assert.Equal(t.T(), newContentLanguage, o.ContentLanguage) } -func (t *PrefixBucketTest) DeleteObject() { +func (t *PrefixBucketTest) Test_DeleteObject() { var err error suffix := "taco" name := t.prefix + suffix @@ -405,7 +682,7 @@ func (t *PrefixBucketTest) DeleteObject() { // Create an object through the back door. _, err = storageutil.CreateObject(t.ctx, t.wrapped, name, []byte(contents)) - AssertEq(nil, err) + assert.Equal(t.T(), nil, err) // Delete it. err = t.bucket.DeleteObject( @@ -414,7 +691,7 @@ func (t *PrefixBucketTest) DeleteObject() { Name: suffix, }) - AssertEq(nil, err) + assert.Equal(t.T(), nil, err) // It should be gone. _, _, err = t.wrapped.StatObject( @@ -424,12 +701,12 @@ func (t *PrefixBucketTest) DeleteObject() { }) var notFoundErr *gcs.NotFoundError - ExpectTrue(errors.As(err, ¬FoundErr)) + assert.True(t.T(), errors.As(err, ¬FoundErr)) } func TestGetFolder_Prefix(t *testing.T) { prefix := "foo_" - wrapped := fake.NewFakeBucket(timeutil.RealClock(), "some_bucket", gcs.NonHierarchical) + wrapped := fake.NewFakeBucket(timeutil.RealClock(), "some_bucket", gcs.BucketType{}) bucket, err := gcsx.NewPrefixBucket(prefix, wrapped) require.Nil(t, err) folderName := "taco" @@ -440,7 +717,7 @@ func TestGetFolder_Prefix(t *testing.T) { result, err := bucket.GetFolder( ctx, - folderName) + &gcs.GetFolderRequest{Name: folderName}) assert.Nil(nil, err) assert.Equal(t, folderName, result.Name) @@ -448,7 +725,7 @@ func TestGetFolder_Prefix(t *testing.T) { func TestDeleteFolder(t *testing.T) { prefix := "foo_" - wrapped := fake.NewFakeBucket(timeutil.RealClock(), "some_bucket", gcs.NonHierarchical) + wrapped := fake.NewFakeBucket(timeutil.RealClock(), "some_bucket", gcs.BucketType{}) bucket, err := gcsx.NewPrefixBucket(prefix, wrapped) require.Nil(t, err) folderName := "taco" @@ -465,7 +742,7 @@ func TestDeleteFolder(t *testing.T) { if assert.Nil(t, err) { _, err = wrapped.GetFolder( ctx, - folderName) + &gcs.GetFolderRequest{Name: folderName}) var notFoundErr *gcs.NotFoundError assert.ErrorAs(t, err, ¬FoundErr) } @@ -477,7 +754,7 @@ func TestRenameFolder(t *testing.T) { old_suffix := "test" name := prefix + old_suffix new_suffix := "new_test" - wrapped := fake.NewFakeBucket(timeutil.RealClock(), "some_bucket", gcs.NonHierarchical) + wrapped := fake.NewFakeBucket(timeutil.RealClock(), "some_bucket", gcs.BucketType{}) bucket, err := gcsx.NewPrefixBucket(prefix, wrapped) require.Nil(t, err) ctx := context.Background() @@ -489,10 +766,10 @@ func TestRenameFolder(t *testing.T) { assert.Equal(t, new_suffix, f.Name) // New folder should get created - _, err = bucket.GetFolder(ctx, new_suffix) + _, err = bucket.GetFolder(ctx, &gcs.GetFolderRequest{Name: new_suffix}) assert.Nil(t, err) // Old folder should be gone. - _, err = bucket.GetFolder(ctx, old_suffix) + _, err = bucket.GetFolder(ctx, &gcs.GetFolderRequest{Name: old_suffix}) var notFoundErr *gcs.NotFoundError assert.True(t, errors.As(err, ¬FoundErr)) } @@ -501,7 +778,7 @@ func TestCreateFolder(t *testing.T) { prefix := "foo_" var err error suffix := "test" - wrapped := fake.NewFakeBucket(timeutil.RealClock(), "some_bucket", gcs.NonHierarchical) + wrapped := fake.NewFakeBucket(timeutil.RealClock(), "some_bucket", gcs.BucketType{}) bucket, err := gcsx.NewPrefixBucket(prefix, wrapped) require.NoError(t, err) ctx := context.Background() @@ -511,6 +788,50 @@ func TestCreateFolder(t *testing.T) { assert.Equal(t, f.Name, suffix) assert.NoError(t, err) // Folder should get created - _, err = bucket.GetFolder(ctx, suffix) + _, err = bucket.GetFolder(ctx, &gcs.GetFolderRequest{Name: suffix}) assert.NoError(t, err) } + +func TestMoveObject(t *testing.T) { + var notFoundErr *gcs.NotFoundError + var err error + prefix := "foo_" + suffix := "test" + wrapped := fake.NewFakeBucket(timeutil.RealClock(), "some_bucket", gcs.BucketType{Hierarchical: true}) + bucket, err := gcsx.NewPrefixBucket(prefix, wrapped) + require.NoError(t, err) + ctx := context.Background() + contents := "foobar" + name := prefix + suffix + // Create an object through the back door. + _, err = storageutil.CreateObject(ctx, wrapped, name, []byte(contents)) + assert.NoError(t, err) + + // Move it to a new name. + newSuffix := "burrito" + o, err := bucket.MoveObject( + ctx, + &gcs.MoveObjectRequest{ + SrcName: suffix, + DstName: newSuffix, + }) + + assert.NoError(t, err) + assert.Equal(t, newSuffix, o.Name) + + newName := prefix + newSuffix + // Read it through the back door. + actual, err := storageutil.ReadObject(ctx, wrapped, newName) + assert.NoError(t, err) + assert.Equal(t, contents, string(actual)) + + // Stat old object. + m, _, err := bucket.StatObject( + ctx, + &gcs.StatObjectRequest{ + Name: suffix, + }) + + assert.True(t, errors.As(err, ¬FoundErr)) + assert.Nil(t, m) +} diff --git a/internal/gcsx/random_reader.go b/internal/gcsx/random_reader.go index 61f2f0de81..42e45be31d 100644 --- a/internal/gcsx/random_reader.go +++ b/internal/gcsx/random_reader.go @@ -15,43 +15,47 @@ package gcsx import ( + "errors" "fmt" "io" - "strings" + "math" + "sync" + "sync/atomic" "time" "github.com/google/uuid" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/file" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/lru" - cacheutil "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/util" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" - "github.com/googlecloudplatform/gcsfuse/v2/internal/monitor" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/file" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + cacheutil "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/gcsfuse_errors" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" "github.com/jacobsa/fuse/fuseops" "golang.org/x/net/context" ) -// MB is 1 Megabyte. (Silly comment to make the lint warning go away) -const MB = 1 << 20 - // Min read size in bytes for random reads. // We will not send a request to GCS for less than this many bytes (unless the // end of the object comes first). -const minReadSize = MB +const minReadSize = MiB // Max read size in bytes for random reads. // If the average read size (between seeks) is below this number, reads will // optimised for random access. // We will skip forwards in a GCS response at most this many bytes. -// About 6 MB of data is buffered anyway, so 8 MB seems like a good round number. -const maxReadSize = 8 * MB +// About 6 MiB of data is buffered anyway, so 8 MiB seems like a good round number. +const maxReadSize = 8 * MiB // Minimum number of seeks before evaluating if the read pattern is random. const minSeeksForRandom = 2 -// "readOp" is the value used in read context to store pointer to the read operation. -const ReadOp = "readOp" +// TODO(b/385826024): Revert timeout to an appropriate value +const TimeoutForMultiRangeRead = time.Hour + +var FallbackToNewRangeReader = errors.New("fallback to new range reader is required") // RandomReader is an object that knows how to read ranges within a particular // generation of a particular GCS object. Optimised for (large) sequential reads. @@ -64,10 +68,12 @@ type RandomReader interface { // Panic if any internal invariants are violated. CheckInvariants() - // ReadAt Matches the semantics of io.ReaderAt, with the addition of context - // support and cache support. It returns a boolean which represent either - // content is read from fileCache (cacheHit = true) or gcs (cacheHit = false) - ReadAt(ctx context.Context, p []byte, offset int64) (n int, cacheHit bool, err error) + // ReadAt returns the data from the requested offset and upto the size of input + // byte array. It either populates input array i.e., p or returns a different + // byte array. In case input array is populated, the same array will be returned + // as part of response. Hence the callers should use the byte array returned + // as part of response always. + ReadAt(ctx context.Context, p []byte, offset int64) (objectData ObjectData, err error) // Return the record for the object to which the reader is bound. Object() (o *gcs.MinObject) @@ -77,19 +83,49 @@ type RandomReader interface { Destroy() } +// ObjectData specifies the response returned as part of ReadAt call. +type ObjectData struct { + // Byte array populated with the requested data. + DataBuf []byte + // Size of the data returned. + Size int + // Specified whether data is served from cache or not. + CacheHit bool +} + +// ReaderType represents different types of go-sdk gcs readers. +// For eg: NewReader and MRD both point to bidi read api. This enum specifies +// the go-sdk type. +type ReaderType int + +// ReaderType enum values. +const ( + // RangeReader corresponds to NewReader method in bucket_handle.go + RangeReader ReaderType = iota + // MultiRangeReader corresponds to NewMultiRangeDownloader method in bucket_handle.go + MultiRangeReader +) + // NewRandomReader create a random reader for the supplied object record that // reads using the given bucket. -func NewRandomReader(o *gcs.MinObject, bucket gcs.Bucket, sequentialReadSizeMb int32, fileCacheHandler *file.CacheHandler, cacheFileForRangeRead bool) RandomReader { +func NewRandomReader(o *gcs.MinObject, bucket gcs.Bucket, sequentialReadSizeMb int32, fileCacheHandler *file.CacheHandler, cacheFileForRangeRead bool, metricHandle metrics.MetricHandle, traceHandle tracing.TraceHandle, mrdWrapper *MultiRangeDownloaderWrapper, config *cfg.Config, handleID fuseops.HandleID) RandomReader { + if traceHandle == nil { + traceHandle = tracing.NewNoopTracer() + } + return &randomReader{ object: o, bucket: bucket, start: -1, limit: -1, - seeks: 0, - totalReadBytes: 0, sequentialReadSizeMb: sequentialReadSizeMb, fileCacheHandler: fileCacheHandler, cacheFileForRangeRead: cacheFileForRangeRead, + mrdWrapper: mrdWrapper, + metricHandle: metricHandle, + traceHandle: traceHandle, + config: config, + handleID: handleID, } } @@ -100,7 +136,7 @@ type randomReader struct { // If non-nil, an in-flight read request and a function for cancelling it. // // INVARIANT: (reader == nil) == (cancel == nil) - reader io.ReadCloser + reader gcs.StorageReader cancel func() // The range of the object that we expect reader to yield, when reader is @@ -113,8 +149,11 @@ type randomReader struct { // reads from cache. start int64 limit int64 - seeks uint64 - totalReadBytes uint64 + seeks atomic.Uint64 + totalReadBytes atomic.Uint64 + + // ReadType of the reader. Will be sequential by default. + readType atomic.Int64 sequentialReadSizeMb int32 @@ -129,6 +168,41 @@ type randomReader struct { // fileCacheHandle is used to read from the cached location. It is created on the fly // using fileCacheHandler for the given object and bucket. fileCacheHandle *file.CacheHandle + + // Stores the handle associated with the previously closed newReader instance. + // This will be used while making the new connection to bypass auth and metadata + // checks. + readHandle []byte + + handleID fuseops.HandleID + + // mrdWrapper points to the wrapper object within inode. + mrdWrapper *MultiRangeDownloaderWrapper + + // boolean variable to determine if MRD is being used or not. + isMRDInUse atomic.Bool + + metricHandle metrics.MetricHandle + + traceHandle tracing.TraceHandle + + config *cfg.Config + + // Specifies the next expected offset for the reads. Used to distinguish between + // sequential and random reads. + expectedOffset atomic.Int64 + + // To synchronize reads served from range reader. + mu sync.Mutex + + // To synchronize access to fileCacheHandle + fileCacheMu sync.RWMutex +} + +type readInfo struct { + readType int64 + expectedOffset int64 + seekRecorded bool } func (rr *randomReader) CheckInvariants() { @@ -167,7 +241,14 @@ func (rr *randomReader) CheckInvariants() { func (rr *randomReader) tryReadingFromFileCache(ctx context.Context, p []byte, offset int64) (n int, cacheHit bool, err error) { - + ctx, span := rr.traceHandle.StartSpan(ctx, tracing.FileCacheRead) + defer func() { + rr.traceHandle.SetCacheReadAttributes(span, cacheHit, n) + if err != nil { + rr.traceHandle.RecordError(span, err) + } + rr.traceHandle.EndSpan(span) + }() if rr.fileCacheHandler == nil { return } @@ -177,8 +258,7 @@ func (rr *randomReader) tryReadingFromFileCache(ctx context.Context, // Request log and start the execution timer. requestId := uuid.New() - readOp := ctx.Value(ReadOp).(*fuseops.ReadFileOp) - logger.Tracef("%.13v <- FileCache(%s:/%s, offset: %d, size: %d handle: %d)", requestId, rr.bucket.Name(), rr.object.Name, offset, len(p), readOp.Handle) + logger.Tracef("%.13v <- FileCache(%s:/%s, offset: %d, size: %d handle: %d)", requestId, rr.bucket.Name(), rr.object.Name, offset, len(p), rr.handleID) startTime := time.Now() // Response log @@ -188,43 +268,56 @@ func (rr *randomReader) tryReadingFromFileCache(ctx context.Context, if err != nil { requestOutput = fmt.Sprintf("err: %v (%v)", err, executionTime) } else { + rr.fileCacheMu.RLock() if rr.fileCacheHandle != nil { isSeq = rr.fileCacheHandle.IsSequential(offset) } + rr.fileCacheMu.RUnlock() requestOutput = fmt.Sprintf("OK (isSeq: %t, hit: %t) (%v)", isSeq, cacheHit, executionTime) } // Here rr.fileCacheHandle will not be nil since we return from the above in those cases. logger.Tracef("%.13v -> %s", requestId, requestOutput) - readType := util.Random + readType := metrics.ReadTypeRandom if isSeq { - readType = util.Sequential + readType = metrics.ReadTypeSequential } - // Capture file cache metrics to be exported via stackdriver - monitor.CaptureFileCacheMetrics(ctx, readType, n, cacheHit, executionTime) + captureFileCacheMetrics(ctx, rr.metricHandle, metrics.ReadTypeNames[readType], n, cacheHit, executionTime) }() // Create fileCacheHandle if not already. + rr.fileCacheMu.Lock() if rr.fileCacheHandle == nil { rr.fileCacheHandle, err = rr.fileCacheHandler.GetCacheHandle(rr.object, rr.bucket, rr.cacheFileForRangeRead, offset) if err != nil { + rr.fileCacheMu.Unlock() // We fall back to GCS if file size is greater than the cache size - if strings.Contains(err.Error(), lru.InvalidEntrySizeErrorMsg) { + if errors.Is(err, lru.ErrInvalidEntrySize) { logger.Warnf("tryReadingFromFileCache: while creating CacheHandle: %v", err) return 0, false, nil - } else if strings.Contains(err.Error(), cacheutil.CacheHandleNotRequiredForRandomReadErrMsg) { + } else if errors.Is(err, cacheutil.ErrCacheHandleNotRequiredForRandomRead) { // Fall back to GCS if it is a random read, cacheFileForRangeRead is // False and there doesn't already exist file in cache. isSeq = false return 0, false, nil + } else if errors.Is(err, cacheutil.ErrFileExcludedFromCacheByRegex) { + // Fall back to GCS if the file is explicitly excluded from cache. + return 0, false, nil } return 0, false, fmt.Errorf("tryReadingFromFileCache: while creating CacheHandle instance: %w", err) } } + rr.fileCacheMu.Unlock() + rr.fileCacheMu.RLock() + if rr.fileCacheHandle == nil { + rr.fileCacheMu.RUnlock() + return + } n, cacheHit, err = rr.fileCacheHandle.Read(ctx, rr.bucket, rr.object, offset, p) + rr.fileCacheMu.RUnlock() if err == nil { return } @@ -233,13 +326,17 @@ func (rr *randomReader) tryReadingFromFileCache(ctx context.Context, n = 0 if cacheutil.IsCacheHandleInvalid(err) { - logger.Tracef("Closing cacheHandle:%p for object: %s:/%s", rr.fileCacheHandle, rr.bucket.Name(), rr.object.Name) - err = rr.fileCacheHandle.Close() - if err != nil { - logger.Warnf("tryReadingFromFileCache: while closing fileCacheHandle: %v", err) + rr.fileCacheMu.Lock() + if rr.fileCacheHandle != nil { + logger.Tracef("Closing cacheHandle:%p for object: %s:/%s", rr.fileCacheHandle, rr.bucket.Name(), rr.object.Name) + err = rr.fileCacheHandle.Close() + if err != nil { + logger.Warnf("tryReadingFromFileCache: while closing fileCacheHandle: %v", err) + } + rr.fileCacheHandle = nil } - rr.fileCacheHandle = nil - } else if !strings.Contains(err.Error(), cacheutil.FallbackToGCSErrMsg) { + rr.fileCacheMu.Unlock() + } else if !errors.Is(err, cacheutil.ErrFallbackToGCS) { err = fmt.Errorf("tryReadingFromFileCache: while reading via cache: %w", err) return } @@ -251,115 +348,74 @@ func (rr *randomReader) tryReadingFromFileCache(ctx context.Context, func (rr *randomReader) ReadAt( ctx context.Context, p []byte, - offset int64) (n int, cacheHit bool, err error) { + offset int64) (objectData ObjectData, err error) { + objectData = ObjectData{ + DataBuf: p, + CacheHit: false, + Size: 0, + } if offset >= int64(rr.object.Size) { err = io.EOF return + } else if offset < 0 { + err = fmt.Errorf( + "illegal offset %d for %d byte object", + offset, + rr.object.Size) + return } // Note: If we are reading the file for the first time and read type is sequential // then the file cache behavior is write-through i.e. data is first read from // GCS, cached in file and then served from that file. But the cacheHit is // false in that case. - n, cacheHit, err = rr.tryReadingFromFileCache(ctx, p, offset) + n, cacheHit, err := rr.tryReadingFromFileCache(ctx, p, offset) if err != nil { err = fmt.Errorf("ReadAt: while reading from cache: %w", err) return } // Data was served from cache. if cacheHit || n == len(p) || (n < len(p) && uint64(offset)+uint64(n) == rr.object.Size) { + objectData.CacheHit = cacheHit + objectData.Size = n return } - for len(p) > 0 { - // Have we blown past the end of the object? - if offset >= int64(rr.object.Size) { - err = io.EOF - return - } + // Not taking any lock for getting reader type to ensure random read requests do not wait. + readInfo := rr.getReadInfo(offset, false) + reqReaderType := readerType(readInfo.readType, rr.bucket.BucketType()) - // When the offset is AFTER the reader position, try to seek forward, within reason. - // This happens when the kernel page cache serves some data. It's very common for - // concurrent reads, often by only a few 128kB fuse read requests. The aim is to - // re-use GCS connection and avoid throwing away already read data. - // For parallel sequential reads to a single file, not throwing away the connections - // is a 15-20x improvement in throughput: 150-200 MB/s instead of 10 MB/s. - if rr.reader != nil && rr.start < offset && offset-rr.start < maxReadSize { - bytesToSkip := int64(offset - rr.start) - p := make([]byte, bytesToSkip) - n, _ := io.ReadFull(rr.reader, p) - rr.start += int64(n) - } + if reqReaderType == RangeReader { + rr.mu.Lock() + expectedOffset := rr.expectedOffset.Load() - // If we have an existing reader but it's positioned at the wrong place, - // clean it up and throw it away. - if rr.reader != nil && rr.start != offset { - rr.reader.Close() - rr.reader = nil - rr.cancel = nil - rr.seeks++ + // Calculating reader type again for zonal buckets in case another read has been served + // since last computation. This is to ensure that we don't use range reader incorrectly + // when MRD should've been used. + if rr.bucket.BucketType().Zonal && readInfo.expectedOffset != expectedOffset { + readInfo = rr.getReadInfo(offset, readInfo.seekRecorded) + reqReaderType = readerType(readInfo.readType, rr.bucket.BucketType()) } - // If we don't have a reader, start a read operation. - if rr.reader == nil { - err = rr.startRead(ctx, offset, int64(len(p))) - if err != nil { - err = fmt.Errorf("startRead: %w", err) - return + if reqReaderType == MultiRangeReader { + rr.mu.Unlock() + } else { + defer rr.mu.Unlock() + + // Check first if we can read using existing reader. if not, create a new range reader + objectData.Size, err = rr.readFromExistingRangeReader(ctx, p, offset) + if errors.Is(err, FallbackToNewRangeReader) { + // reader does not exist and need to be created, get the end offset. + end := rr.getEndOffset(offset) + objectData.Size, err = rr.readFromRangeReader(ctx, p, offset, end, readInfo.readType) } - } - - // Now we have a reader positioned at the correct place. Consume as much from - // it as possible. - var tmp int - tmp, err = rr.readFull(ctx, p) - - n += tmp - p = p[tmp:] - rr.start += int64(tmp) - offset += int64(tmp) - rr.totalReadBytes += uint64(tmp) - - // Sanity check. - if rr.start > rr.limit { - err = fmt.Errorf("reader returned %d too many bytes", rr.start-rr.limit) - - // Don't attempt to reuse the reader when it's behaving wackily. - rr.reader.Close() - rr.reader = nil - rr.cancel = nil - rr.start = -1 - rr.limit = -1 - return } + } - // Are we finished with this reader now? - if rr.start == rr.limit { - rr.reader.Close() - rr.reader = nil - rr.cancel = nil - } - - // Handle errors. - switch { - case err == io.EOF || err == io.ErrUnexpectedEOF: - // For a non-empty buffer, ReadFull returns EOF or ErrUnexpectedEOF only - // if the reader peters out early. That's fine, but it means we should - // have hit the limit above. - if rr.reader != nil { - err = fmt.Errorf("reader returned %d too few bytes", rr.limit-rr.start) - return - } - - err = nil - - case err != nil: - // Propagate other errors. - err = fmt.Errorf("readFull: %w", err) - return - } + if reqReaderType == MultiRangeReader { + objectData.Size, err = rr.readFromMultiRangeReader(ctx, p, offset, offset+int64(len(p)), TimeoutForMultiRangeRead) } return @@ -371,16 +427,29 @@ func (rr *randomReader) Object() (o *gcs.MinObject) { } func (rr *randomReader) Destroy() { + defer func() { + if rr.isMRDInUse.Load() { + err := rr.mrdWrapper.DecrementRefCount() + if err != nil { + logger.Errorf("randomReader::Destroy:%v", err) + } + rr.isMRDInUse.Store(false) + } + }() + // Close out the reader, if we have one. if rr.reader != nil { - err := rr.reader.Close() + rr.mu.Lock() + defer rr.mu.Unlock() + if rr.reader != nil { + rr.closeReader() + } rr.reader = nil rr.cancel = nil - if err != nil { - logger.Warnf("rr.Destroy(): while closing reader: %v", err) - } } + rr.fileCacheMu.Lock() + defer rr.fileCacheMu.Unlock() if rr.fileCacheHandle != nil { logger.Tracef("Closing cacheHandle:%p for object: %s:/%s", rr.fileCacheHandle, rr.bucket.Name(), rr.object.Name) err := rr.fileCacheHandle.Close() @@ -397,28 +466,30 @@ func (rr *randomReader) Destroy() { func (rr *randomReader) readFull( ctx context.Context, p []byte) (n int, err error) { - // Start a goroutine that will cancel the read operation we block on below if - // the calling context is cancelled, but only if this method has not already - // returned (to avoid souring the reader for the next read if this one is - // successful, since the calling context will eventually be cancelled). - readDone := make(chan struct{}) - defer close(readDone) - - go func() { - select { - case <-readDone: - return - - case <-ctx.Done(): + if rr.config != nil && !rr.config.FileSystem.IgnoreInterrupts { + // Start a goroutine that will cancel the read operation we block on below if + // the calling context is cancelled, but only if this method has not already + // returned (to avoid souring the reader for the next read if this one is + // successful, since the calling context will eventually be cancelled). + readDone := make(chan struct{}) + defer close(readDone) + + go func() { select { case <-readDone: return - default: - rr.cancel() + case <-ctx.Done(): + select { + case <-readDone: + return + + default: + rr.cancel() + } } - } - }() + }() + } // Call through. n, err = io.ReadFull(rr.reader, p) @@ -429,41 +500,137 @@ func (rr *randomReader) readFull( // Ensure that rr.reader is set up for a range for which [start, start+size) is // a prefix. Irrespective of the size requested, we try to fetch more data // from GCS defined by sequentialReadSizeMb flag to serve future read requests. -func (rr *randomReader) startRead( - ctx context.Context, - start int64, - size int64) (err error) { - // Make sure start and size are legal. - if start < 0 || uint64(start) > rr.object.Size || size < 0 { - err = fmt.Errorf( - "range [%d, %d) is illegal for %d-byte object", - start, - start+size, - rr.object.Size) +func (rr *randomReader) startRead(ctx context.Context, start int64, end int64, readType int64) (err error) { + // Begin the read. + ctx, cancel := context.WithCancel(rr.traceHandle.PropagateTraceContext(context.Background(), ctx)) + + if rr.config != nil && rr.config.Read.InactiveStreamTimeout > 0 { + rr.reader, err = NewInactiveTimeoutReader( + ctx, + rr.bucket, + rr.object, + rr.readHandle, + gcs.ByteRange{ + Start: uint64(start), + Limit: uint64(end), + }, + rr.config.Read.InactiveStreamTimeout) + } else { + rr.reader, err = rr.bucket.NewReaderWithReadHandle( + ctx, + &gcs.ReadObjectRequest{ + Name: rr.object.Name, + Generation: rr.object.Generation, + Range: &gcs.ByteRange{ + Start: uint64(start), + Limit: uint64(end), + }, + ReadCompressed: rr.object.HasContentEncodingGzip(), + ReadHandle: rr.readHandle, + }) + } + + // If a file handle is open locally, but the corresponding object doesn't exist + // in GCS, it indicates a file clobbering scenario. This likely occurred because: + // - The file was deleted in GCS while a local handle was still open. + // - The file content was modified leading to different generation number. + var notFoundError *gcs.NotFoundError + if errors.As(err, ¬FoundError) { + err = &gcsfuse_errors.FileClobberedError{ + Err: fmt.Errorf("NewReader: %w", err), + ObjectName: rr.object.Name, + } + return + } + + if err != nil { + err = fmt.Errorf("NewReaderWithReadHandle: %w", err) return } + rr.cancel = cancel + rr.start = start + rr.limit = end + + requestedDataSize := end - start + metrics.CaptureGCSReadMetrics(rr.metricHandle, metrics.ReadTypeNames[readType], requestedDataSize) + + return +} + +// isSeekNeeded determines if the current read at `offset` should be considered a +// seek, given the previous read pattern & the expected offset. +func isSeekNeeded(readType, offset, expectedOffset int64) bool { + if expectedOffset == 0 { + return false + } + + if readType == metrics.ReadTypeRandom { + return offset != expectedOffset + } + + if readType == metrics.ReadTypeSequential { + return offset < expectedOffset || offset > expectedOffset+maxReadSize + } + + return false +} + +// getReadInfo determines the read strategy (sequential or random) for a read +// request at a given offset and returns read metadata. It also updates the +// reader's internal state based on the read pattern. +func (rr *randomReader) getReadInfo(offset int64, seekRecorded bool) readInfo { + readType := rr.readType.Load() + expOffset := rr.expectedOffset.Load() + numSeeks := rr.seeks.Load() + + if !seekRecorded && isSeekNeeded(readType, offset, expOffset) { + numSeeks = rr.seeks.Add(1) + seekRecorded = true + } + + if numSeeks >= minSeeksForRandom { + readType = metrics.ReadTypeRandom + } + + averageReadBytes := rr.totalReadBytes.Load() + if numSeeks > 0 { + averageReadBytes /= numSeeks + } + + if averageReadBytes >= maxReadSize { + readType = metrics.ReadTypeSequential + } + + rr.readType.Store(readType) + return readInfo{ + readType: readType, + expectedOffset: expOffset, + seekRecorded: seekRecorded, + } +} + +// getEndOffset returns the end offset for the range to query GCS. +// Range here is [start, end]. End is computed for sequential reads using +// start offset and size of the data the callers needs. +func (rr *randomReader) getEndOffset( + start int64) (end int64) { // GCS requests are expensive. Prefer to issue read requests defined by // sequentialReadSizeMb flag. Sequential reads will simply sip from the fire house // with each call to ReadAt. In practice, GCS will fill the TCP buffers - // with about 6 MB of data. Requests from outside GCP will be charged + // with about 6 MiB of data. Requests from outside GCP will be charged // about 6MB of egress data, even if less data is read. Inside GCP // regions, GCS egress is free. This logic should limit the number of // GCS read requests, which are not free. // But if we notice random read patterns after a minimum number of seeks, // optimise for random reads. Random reads will read data in chunks of - // (average read size in bytes rounded up to the next MB). - end := int64(rr.object.Size) - readType := util.Sequential - if rr.seeks >= minSeeksForRandom { - readType = util.Random - averageReadBytes := rr.totalReadBytes / rr.seeks + // (average read size in bytes rounded up to the next MiB). + end = int64(rr.object.Size) + if seeks := rr.seeks.Load(); seeks >= minSeeksForRandom { + averageReadBytes := rr.totalReadBytes.Load() / seeks if averageReadBytes < maxReadSize { - randomReadSize := int64(((averageReadBytes / MB) + 1) * MB) - if randomReadSize < minReadSize { - randomReadSize = minReadSize - } + randomReadSize := max(int64(((averageReadBytes/MiB)+1)*MiB), minReadSize) if randomReadSize > maxReadSize { randomReadSize = maxReadSize } @@ -476,37 +643,161 @@ func (rr *randomReader) startRead( // To avoid overloading GCS and to have reasonable latencies, we will only // fetch data of max size defined by sequentialReadSizeMb. - maxSizeToReadFromGCS := int64(rr.sequentialReadSizeMb * MB) + maxSizeToReadFromGCS := int64(rr.sequentialReadSizeMb * MiB) if end-start > maxSizeToReadFromGCS { end = start + maxSizeToReadFromGCS } - // Begin the read. - ctx, cancel := context.WithCancel(context.Background()) - rc, err := rr.bucket.NewReader( - ctx, - &gcs.ReadObjectRequest{ - Name: rr.object.Name, - Generation: rr.object.Generation, - Range: &gcs.ByteRange{ - Start: uint64(start), - Limit: uint64(end), - }, - ReadCompressed: rr.object.HasContentEncodingGzip(), - }) + return +} + +// readerType specifies the go-sdk interface to use for reads. +func readerType(readType int64, bucketType gcs.BucketType) ReaderType { + if readType == metrics.ReadTypeRandom && bucketType.Zonal { + return MultiRangeReader + } + return RangeReader +} + +// skipBytes attempts to advance the reader position to the given offset without +// discarding the existing reader. +// LOCKS_REQUIRED (rr.mu) +func (rr *randomReader) skipBytes(offset int64) { + // When the offset is AFTER the reader position, try to seek forward, within reason. + // This happens when the kernel page cache serves some data. It's very common for + // concurrent reads, often by only a few 128kB fuse read requests. The aim is to + // re-use GCS connection and avoid throwing away already read data. + // For parallel sequential reads to a single file, not throwing away the connections + // is a 15-20x improvement in throughput: 150-200 MiB/s instead of 10 MiB/s. + if rr.reader != nil && rr.start < offset && offset-rr.start < maxReadSize { + bytesToSkip := offset - rr.start + discardedBytes, copyError := io.CopyN(io.Discard, rr.reader, bytesToSkip) + // io.EOF is expected if the reader is shorter than the requested offset to read. + if copyError != nil && !errors.Is(copyError, io.EOF) { + logger.Warnf("Error while skipping reader bytes: %v", copyError) + } + rr.start += discardedBytes + } +} + +// invalidateReaderIfMisalignedOrTooSmall ensures that the existing reader is valid +// for the requested offset and length. If the reader is misaligned (not at the requested +// offset) or cannot serve the full request within its limit, it is closed and discarded. +// LOCKS_REQUIRED (rr.mu) +func (rr *randomReader) invalidateReaderIfMisalignedOrTooSmall(startOffset, endOffset int64) { + // If we have an existing reader, but it's positioned at the wrong place, + // clean it up and throw it away. + // We will also clean up the existing reader if it can't serve the entire request. + dataToRead := math.Min(float64(endOffset), float64(rr.object.Size)) + if rr.reader != nil && (rr.start != startOffset || int64(dataToRead) > rr.limit) { + rr.closeReader() + rr.reader = nil + rr.cancel = nil + } +} + +// readFromExistingRangeReader attempts to read data from an existing reader if one is available. +// If a reader exists and the read is successful, the data is returned. +// Otherwise, it returns an error indicating that a new reader is needed. +// LOCKS_REQUIRED (rr.mu) +func (rr *randomReader) readFromExistingRangeReader(ctx context.Context, p []byte, offset int64) (n int, err error) { + rr.skipBytes(offset) + rr.invalidateReaderIfMisalignedOrTooSmall(offset, offset+int64(len(p))) + if rr.reader != nil { + return rr.readFromRangeReader(ctx, p, offset, offset+int64(len(p)), rr.readType.Load()) + } + return 0, FallbackToNewRangeReader +} + +// readFromRangeReader reads using the NewReader interface of go-sdk. Its uses +// the existing reader if available, otherwise makes a call to GCS. +// LOCKS_REQUIRED (rr.mu) +func (rr *randomReader) readFromRangeReader(ctx context.Context, p []byte, offset int64, end int64, readType int64) (n int, err error) { + // If we don't have a reader, start a read operation. + if rr.reader == nil { + err = rr.startRead(ctx, offset, end, readType) + if err != nil { + err = fmt.Errorf("startRead: %w", err) + return + } + } + + // Now we have a reader positioned at the correct place. Consume as much from + // it as possible. + n, err = rr.readFull(ctx, p) + rr.start += int64(n) + rr.totalReadBytes.Add(uint64(n)) + + // Sanity check. + if rr.start > rr.limit { + err = fmt.Errorf("Reader returned extra bytes: %d", rr.start-rr.limit) + + // Don't attempt to reuse the reader when it's behaving wackily. + rr.closeReader() + rr.reader = nil + rr.cancel = nil + rr.start = -1 + rr.limit = -1 - if err != nil { - err = fmt.Errorf("NewReader: %w", err) return } - rr.reader = rc - rr.cancel = cancel - rr.start = start - rr.limit = end + // Are we finished with this reader now? + if rr.start == rr.limit { + rr.closeReader() + rr.reader = nil + rr.cancel = nil + } - requestedDataSize := end - start - monitor.CaptureGCSReadMetrics(ctx, readType, requestedDataSize) + // Handle errors. + switch { + case err == io.EOF || err == io.ErrUnexpectedEOF: + // For a non-empty buffer, ReadFull returns EOF or ErrUnexpectedEOF only + // if the reader peters out early. That's fine, but it means we should + // have hit the limit above. + if rr.reader != nil { + err = fmt.Errorf("random reader returned early by skipping %d bytes", rr.limit-rr.start) + return + } + + err = nil + + case err != nil: + // Propagate other errors. + err = fmt.Errorf("readFull: %w", err) + return + } + + rr.updateExpectedOffset(offset + int64(n)) + + return +} + +func (rr *randomReader) readFromMultiRangeReader(ctx context.Context, p []byte, offset, end int64, timeout time.Duration) (bytesRead int, err error) { + if rr.mrdWrapper == nil { + return 0, fmt.Errorf("readFromMultiRangeReader: Invalid MultiRangeDownloaderWrapper") + } + if rr.isMRDInUse.CompareAndSwap(false, true) { + rr.mrdWrapper.IncrementRefCount() + } + + bytesRead, err = rr.mrdWrapper.Read(ctx, p, offset, end, rr.metricHandle, rr.traceHandle, false) + rr.totalReadBytes.Add(uint64(bytesRead)) + rr.updateExpectedOffset(offset + int64(bytesRead)) return } + +// closeReader fetches the readHandle before closing the reader instance. +// LOCKS_REQUIRED (rr.mu) +func (rr *randomReader) closeReader() { + rr.readHandle = rr.reader.ReadHandle() + err := rr.reader.Close() + if err != nil { + logger.Warnf("error while closing reader: %v", err) + } +} + +func (rr *randomReader) updateExpectedOffset(offset int64) { + rr.expectedOffset.Store(offset) +} diff --git a/internal/gcsx/random_reader_stretchr_test.go b/internal/gcsx/random_reader_stretchr_test.go new file mode 100644 index 0000000000..eef28c6149 --- /dev/null +++ b/internal/gcsx/random_reader_stretchr_test.go @@ -0,0 +1,1252 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx + +import ( + "context" + "errors" + "io" + "os" + "path" + "strings" + "sync" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/util/diskutil" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/file" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/file/downloader" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + testutil "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +const TestTimeoutForMultiRangeRead = time.Second + +type RandomReaderStretchrTest struct { + suite.Suite + object *gcs.MinObject + mockBucket *storage.TestifyMockBucket + rr checkingRandomReader + cacheDir string + jobManager *downloader.JobManager + cacheHandler *file.CacheHandler +} + +func TestRandomReaderStretchrTestSuite(t *testing.T) { + suite.Run(t, new(RandomReaderStretchrTest)) +} + +func (t *RandomReaderStretchrTest) SetupTest() { + t.rr.ctx = context.Background() + + // Manufacture an object record. + t.object = &gcs.MinObject{ + Name: "foo", + Size: 17, + Generation: 1234, + } + + // Create the bucket. + t.mockBucket = new(storage.TestifyMockBucket) + + t.cacheDir = path.Join(os.Getenv("HOME"), "cache/dir") + lruCache := lru.NewCache(cacheMaxSize) + fileCacheConfig := &cfg.FileCacheConfig{ + EnableCrc: false, + } + cacheDirVolumeBlockSize := diskutil.GetVolumeBlockSize(t.cacheDir) + t.jobManager = downloader.NewJobManager(lruCache, util.DefaultFilePerm, util.DefaultDirPerm, t.cacheDir, sequentialReadSizeInMb, fileCacheConfig, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), cacheDirVolumeBlockSize) + t.cacheHandler = file.NewCacheHandler(lruCache, t.jobManager, t.cacheDir, util.DefaultFilePerm, util.DefaultDirPerm, "", "", false, cacheDirVolumeBlockSize) + + // Set up the reader. + rr := NewRandomReader(t.object, t.mockBucket, sequentialReadSizeInMb, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), nil, nil, 0) + t.rr.wrapped = rr.(*randomReader) +} + +func (t *RandomReaderStretchrTest) TearDownTest() { + t.rr.Destroy() +} + +func (t *RandomReaderStretchrTest) Test_GetReadInfo() { + testCases := []struct { + name string + offset int64 + seekRecorded bool + initialReadType int64 + initialExpOffset int64 + initialNumSeeks uint64 + initialTotalReadBytes uint64 + expectedReadType int64 + expectedNumSeeks uint64 + }{ + { + name: "First Read", + offset: 0, + seekRecorded: false, + initialReadType: metrics.ReadTypeSequential, + initialExpOffset: 0, + initialNumSeeks: 0, + initialTotalReadBytes: 0, + expectedReadType: metrics.ReadTypeSequential, + expectedNumSeeks: 0, + }, + { + name: "Sequential Read", + offset: 10, + seekRecorded: false, + initialReadType: metrics.ReadTypeSequential, + initialExpOffset: 10, + initialNumSeeks: 0, + initialTotalReadBytes: 100, + expectedReadType: metrics.ReadTypeSequential, + expectedNumSeeks: 0, + }, + { + name: "Sequential read with small forward jump and high average read bytes is still sequential", + offset: 100, + seekRecorded: false, + initialReadType: metrics.ReadTypeSequential, + initialExpOffset: 10, + initialNumSeeks: 0, + initialTotalReadBytes: 10000000, + expectedReadType: metrics.ReadTypeSequential, + expectedNumSeeks: 0, + }, + { + name: "Sequential read with large forward jump is a seek", + offset: 50 + maxReadSize + 1, + seekRecorded: false, + initialReadType: metrics.ReadTypeSequential, + initialExpOffset: 50, + initialNumSeeks: 0, + initialTotalReadBytes: 50 * 1024, + expectedReadType: metrics.ReadTypeSequential, + expectedNumSeeks: 1, + }, + { + name: "Sequential read with backward jump is a seek", + offset: 49, + seekRecorded: false, + initialReadType: metrics.ReadTypeSequential, + initialExpOffset: 50, + initialNumSeeks: 0, + initialTotalReadBytes: 50 * 1024, + expectedReadType: metrics.ReadTypeSequential, + expectedNumSeeks: 1, + }, + { + name: "Contiguous random read is not a seek", + offset: 50, + seekRecorded: false, + initialReadType: metrics.ReadTypeRandom, + initialExpOffset: 50, + initialNumSeeks: minSeeksForRandom, + initialTotalReadBytes: 50 * 1024, + expectedReadType: metrics.ReadTypeRandom, + expectedNumSeeks: minSeeksForRandom, + }, + { + name: "Non-contiguous random read is a seek", + offset: 100, + seekRecorded: false, + initialReadType: metrics.ReadTypeRandom, + initialExpOffset: 50, + initialNumSeeks: minSeeksForRandom, + initialTotalReadBytes: 50 * 1024, + expectedReadType: metrics.ReadTypeRandom, + expectedNumSeeks: minSeeksForRandom + 1, + }, + { + name: "Switches to random read after enough seeks", + offset: 50 + maxReadSize + 1, + seekRecorded: false, + initialReadType: metrics.ReadTypeSequential, + initialExpOffset: 50, + initialNumSeeks: minSeeksForRandom - 1, + initialTotalReadBytes: 1000, + expectedReadType: metrics.ReadTypeRandom, + expectedNumSeeks: minSeeksForRandom, + }, + { + name: "Switches back to sequential with high average read bytes", + offset: 100, + seekRecorded: false, + initialReadType: metrics.ReadTypeRandom, + initialExpOffset: 50, + initialNumSeeks: minSeeksForRandom, + initialTotalReadBytes: maxReadSize * (minSeeksForRandom + 1), + expectedReadType: metrics.ReadTypeSequential, + expectedNumSeeks: minSeeksForRandom + 1, + }, + { + name: "Seek recorded: sequential large forward jump", + offset: 50 + maxReadSize + 1, + seekRecorded: true, + initialReadType: metrics.ReadTypeSequential, + initialExpOffset: 50, + initialNumSeeks: 0, + initialTotalReadBytes: 50 * 1024, + expectedReadType: metrics.ReadTypeSequential, + expectedNumSeeks: 0, // Not incremented + }, + { + name: "Seek recorded: sequential backward jump", + offset: 49, + seekRecorded: true, + initialReadType: metrics.ReadTypeSequential, + initialExpOffset: 50, + initialNumSeeks: 1, + initialTotalReadBytes: 50 * 1024, + expectedReadType: metrics.ReadTypeSequential, + expectedNumSeeks: 1, // Not incremented + }, + { + name: "Seek recorded: non-contiguous random read", + offset: 100, + seekRecorded: true, + initialReadType: metrics.ReadTypeRandom, + initialExpOffset: 50, + initialNumSeeks: minSeeksForRandom, + initialTotalReadBytes: 50 * 1024, + expectedReadType: metrics.ReadTypeRandom, + expectedNumSeeks: minSeeksForRandom, // Not incremented + }, + { + name: "Seek recorded: does not switch to random", + offset: 50 + maxReadSize + 1, + seekRecorded: true, + initialReadType: metrics.ReadTypeSequential, + initialExpOffset: 50, + initialNumSeeks: minSeeksForRandom - 1, + initialTotalReadBytes: 1000, + expectedReadType: metrics.ReadTypeSequential, // Does not switch + expectedNumSeeks: minSeeksForRandom - 1, // Not incremented + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + rr := &randomReader{} + rr.readType.Store(tc.initialReadType) + rr.expectedOffset.Store(tc.initialExpOffset) + rr.seeks.Store(tc.initialNumSeeks) + rr.totalReadBytes.Store(tc.initialTotalReadBytes) + + readInfo := rr.getReadInfo(tc.offset, tc.seekRecorded) + assert.Equal(t.T(), tc.expectedReadType, readInfo.readType, "Read type mismatch") + assert.Equal(t.T(), tc.expectedNumSeeks, rr.seeks.Load(), "Number of seeks mismatch") + }) + } +} + +func (t *RandomReaderStretchrTest) Test_ReadAt_ParallelMRDReads() { + // Setup + t.rr.wrapped.reader = nil + t.rr.wrapped.seeks.Store(minSeeksForRandom) + t.rr.wrapped.readType.Store(metrics.ReadTypeRandom) + t.object.Size = 20 * MiB + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + + // Mock bucket and MRD + t.mockBucket.On("BucketType", mock.Anything).Return(gcs.BucketType{Zonal: true}) + fakeMRDWrapper, err := NewMultiRangeDownloaderWrapper(t.mockBucket, t.object, &cfg.Config{}, nil) + require.NoError(t.T(), err) + t.rr.wrapped.mrdWrapper = fakeMRDWrapper + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloader(t.object, testContent), nil) + + // Parallel reads + tasks := []struct { + offset int64 + size int + }{ + {0, 1 * MiB}, + {2 * MiB, 2 * MiB}, + {5 * MiB, 1 * MiB}, + {10 * MiB, 5 * MiB}, + } + + var wg sync.WaitGroup + var totalBytesReadFromTasks uint64 + + for _, task := range tasks { + wg.Add(1) + totalBytesReadFromTasks += uint64(task.size) + go func(offset int64, size int) { + defer wg.Done() + buf := make([]byte, size) + // Each goroutine gets its own context. + ctx := context.Background() + objData, err := t.rr.wrapped.ReadAt(ctx, buf, offset) + + require.NoError(t.T(), err) + require.Equal(t.T(), size, objData.Size) + require.Equal(t.T(), testContent[offset:offset+int64(size)], buf) + }(task.offset, task.size) + } + + wg.Wait() + + // Validation + assert.Equal(t.T(), totalBytesReadFromTasks, t.rr.wrapped.totalReadBytes.Load()) + assert.Equal(t.T(), 1, t.rr.wrapped.mrdWrapper.GetRefCount()) + assert.True(t.T(), t.rr.wrapped.isMRDInUse.Load()) +} + +func (t *RandomReaderStretchrTest) Test_ReaderType() { + testCases := []struct { + name string + readType int64 + start int64 + end int64 + bucketType gcs.BucketType + readerType ReaderType + }{ + { + name: "ZonalBucketRandomRead", + readType: metrics.ReadTypeRandom, + start: 50, + end: 68, + bucketType: gcs.BucketType{Zonal: true}, + readerType: MultiRangeReader, + }, + { + name: "ZonalBucketSequentialRead", + readType: metrics.ReadTypeSequential, + start: 50, + end: 68, + bucketType: gcs.BucketType{Zonal: true}, + readerType: RangeReader, + }, + { + name: "RegularBucketRandomRead", + readType: metrics.ReadTypeRandom, + start: 50, + end: 68, + bucketType: gcs.BucketType{Zonal: false}, + readerType: RangeReader, + }, + { + name: "RegularBucketSequentialRead", + readType: metrics.ReadTypeSequential, + start: 50, + end: 68, + bucketType: gcs.BucketType{Zonal: false}, + readerType: RangeReader, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + readerType := readerType(tc.readType, tc.bucketType) + assert.Equal(t.T(), readerType, tc.readerType) + }) + } +} + +func (t *RandomReaderStretchrTest) Test_GetEndOffset() { + testCases := []struct { + name string + start int64 + objectSize int64 + initialReadType int64 + initialNumSeeks uint64 + initialTotalReadBytes uint64 + sequentialReadSizeMb int32 + expectedEnd int64 + }{ + { + name: "Sequential Read, Fits in sequentialReadSizeMb", + start: 0, + objectSize: 10 * MiB, + initialReadType: metrics.ReadTypeSequential, + initialNumSeeks: 0, + initialTotalReadBytes: 0, + sequentialReadSizeMb: 22, + expectedEnd: 10 * MiB, + }, + { + name: "Sequential Read, Object Larger than sequentialReadSizeMb", + start: 0, + objectSize: 50 * MiB, + initialReadType: metrics.ReadTypeSequential, + initialNumSeeks: 0, + initialTotalReadBytes: 0, + sequentialReadSizeMb: 22, + expectedEnd: 22 * MiB, + }, + { + name: "Sequential Read, Respects object size", + start: 5 * MiB, + objectSize: 7 * MiB, + initialReadType: metrics.ReadTypeSequential, + initialNumSeeks: 0, + initialTotalReadBytes: 0, + sequentialReadSizeMb: 22, + expectedEnd: 7 * MiB, + }, + { + name: "Random Read, Min read size", + start: 0, + objectSize: 5 * MiB, + initialReadType: metrics.ReadTypeRandom, + initialNumSeeks: minSeeksForRandom, + initialTotalReadBytes: 1000, + sequentialReadSizeMb: 22, + expectedEnd: minReadSize, + }, + { + name: "Random Read, Averages less than minReadSize", + start: 0, + objectSize: 50 * MiB, + initialReadType: metrics.ReadTypeRandom, + initialNumSeeks: minSeeksForRandom, + initialTotalReadBytes: 100 * 1024, // 100KiB + sequentialReadSizeMb: 22, + expectedEnd: minReadSize, // Should be atleast minReadSize + }, + { + name: "Random Read, Start Offset Non-Zero", + start: 5 * MiB, + objectSize: 50 * MiB, + initialReadType: metrics.ReadTypeRandom, + initialNumSeeks: minSeeksForRandom, + initialTotalReadBytes: 2 * MiB, // avg read bytes = 1MiB + sequentialReadSizeMb: 22, + expectedEnd: 5*MiB + 2*MiB, // avg read bytes + 1MiB + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + rr := &randomReader{ + object: &gcs.MinObject{Size: uint64(tc.objectSize)}, + sequentialReadSizeMb: tc.sequentialReadSizeMb, + } + rr.readType.Store(tc.initialReadType) + rr.seeks.Store(tc.initialNumSeeks) + rr.totalReadBytes.Store(tc.initialTotalReadBytes) + + end := rr.getEndOffset(tc.start) + + assert.Equal(t.T(), tc.expectedEnd, end, "End offset mismatch") + }) + } +} + +func (t *RandomReaderStretchrTest) Test_IsSeekNeeded() { + testCases := []struct { + name string + readType int64 + offset int64 + expectedOffset int64 + want bool + }{ + { + name: "First read, expectedOffset is 0", + readType: metrics.ReadTypeSequential, + offset: 100, + expectedOffset: 0, + want: false, + }, + { + name: "Random read, same offset", + readType: metrics.ReadTypeRandom, + offset: 100, + expectedOffset: 100, + want: false, + }, + { + name: "Random read, different offset", + readType: metrics.ReadTypeRandom, + offset: 200, + expectedOffset: 100, + want: true, + }, + { + name: "Sequential read, same offset", + readType: metrics.ReadTypeSequential, + offset: 100, + expectedOffset: 100, + want: false, + }, + { + name: "Sequential read, small forward jump within maxReadSize", + readType: metrics.ReadTypeSequential, + offset: 100 + maxReadSize/2, + expectedOffset: 100, + want: false, + }, + { + name: "Sequential read, forward jump to boundary of maxReadSize", + readType: metrics.ReadTypeSequential, + offset: 100 + maxReadSize, + expectedOffset: 100, + want: false, + }, + { + name: "Sequential read, large forward jump beyond maxReadSize", + readType: metrics.ReadTypeSequential, + offset: 100 + maxReadSize + 1, + expectedOffset: 100, + want: true, + }, + { + name: "Sequential read, backward jump", + readType: metrics.ReadTypeSequential, + offset: 99, + expectedOffset: 100, + want: true, + }, + { + name: "Unknown read type", + readType: -1, // An invalid read type + offset: 200, + expectedOffset: 100, + want: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + got := isSeekNeeded(tc.readType, tc.offset, tc.expectedOffset) + assert.Equal(t.T(), tc.want, got) + }) + } +} + +func (t *RandomReaderStretchrTest) Test_ReadFromRangeReader_WhenExistingReaderIsNil() { + testCases := []struct { + name string + inputReadHandle []byte + outputReadHandle []byte + }{ + { + name: "ReadHandlePresent", + inputReadHandle: []byte("fake-handle"), + }, + { + name: "ReadHandleAbsent", + inputReadHandle: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.rr.wrapped.readHandle = tc.inputReadHandle + t.rr.wrapped.reader = nil + t.rr.wrapped.start = 0 + t.object.Size = 5 + dataSize := 5 + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + rc := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + readObjectRequest := &gcs.ReadObjectRequest{ + Name: t.rr.wrapped.object.Name, + Generation: t.rr.wrapped.object.Generation, + Range: &gcs.ByteRange{ + Start: uint64(0), + Limit: t.object.Size, + }, + ReadCompressed: t.rr.wrapped.object.HasContentEncodingGzip(), + ReadHandle: t.rr.wrapped.readHandle, + } + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, readObjectRequest).Return(rc, nil).Times(1) + buf := make([]byte, dataSize) + + n, err := t.rr.wrapped.readFromRangeReader(t.rr.ctx, buf, 0, int64(t.object.Size), metrics.ReadTypeUnknown) + + t.mockBucket.AssertExpectations(t.T()) + assert.NoError(t.T(), err) + assert.Equal(t.T(), dataSize, n) + assert.Equal(t.T(), testContent[:dataSize], buf) + // Verify the reader state. + assert.Nil(t.T(), t.rr.wrapped.reader) + assert.Nil(t.T(), t.rr.wrapped.cancel) + assert.Equal(t.T(), int64(5), t.rr.wrapped.start) + assert.Equal(t.T(), int64(5), t.rr.wrapped.limit) + assert.Equal(t.T(), int64(t.object.Size), t.rr.wrapped.expectedOffset.Load()) + }) + } +} + +func (t *RandomReaderStretchrTest) Test_ReadFromRangeReader_WhenExistingReaderIsNotNil() { + t.rr.wrapped.start = 4 + t.rr.wrapped.limit = 10 + t.rr.wrapped.totalReadBytes.Store(4) + t.object.Size = 10 + dataSize := 4 + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + rc := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.rr.wrapped.reader = rc + t.rr.wrapped.cancel = func() {} + buf := make([]byte, dataSize) + + n, err := t.rr.wrapped.readFromRangeReader(t.rr.ctx, buf, 4, 8, metrics.ReadTypeUnknown) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), dataSize, n) + // Verify the reader state. + assert.Equal(t.T(), rc, t.rr.wrapped.reader) + assert.NotNil(t.T(), t.rr.wrapped.cancel) + assert.Equal(t.T(), int64(8), t.rr.wrapped.start) + assert.Equal(t.T(), int64(10), t.rr.wrapped.limit) + assert.Equal(t.T(), uint64(8), t.rr.wrapped.totalReadBytes.Load()) + assert.Equal(t.T(), int64(8), t.rr.wrapped.expectedOffset.Load()) +} + +func (t *RandomReaderStretchrTest) Test_ReadFromRangeReader_WhenAllDataFromReaderIsRead() { + testCases := []struct { + name string + readHandle []byte + }{ + { + name: "GCSReturnedReadHandle", + readHandle: []byte("fake-handle"), + }, + { + name: "GCSReturnedNoReadHandle", + readHandle: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.rr.wrapped.start = 4 + t.rr.wrapped.limit = 10 + t.rr.wrapped.totalReadBytes.Store(4) + t.object.Size = 10 + dataSize := 6 + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + rc := &fake.FakeReader{ + ReadCloser: getReadCloser(testContent), + Handle: tc.readHandle, + } + t.rr.wrapped.reader = rc + t.rr.wrapped.cancel = func() {} + buf := make([]byte, dataSize) + + n, err := t.rr.wrapped.readFromRangeReader(t.rr.ctx, buf, 4, 10, metrics.ReadTypeUnknown) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), dataSize, n) + // Verify the reader state. + assert.Nil(t.T(), t.rr.wrapped.reader) + assert.Nil(t.T(), t.rr.wrapped.cancel) + assert.Equal(t.T(), int64(10), t.rr.wrapped.start) + assert.Equal(t.T(), int64(10), t.rr.wrapped.limit) + assert.Equal(t.T(), uint64(10), t.rr.wrapped.totalReadBytes.Load()) + assert.Equal(t.T(), int64(10), t.rr.wrapped.expectedOffset.Load()) + expectedReadHandle := tc.readHandle + assert.Equal(t.T(), expectedReadHandle, t.rr.wrapped.readHandle) + }) + } +} + +func (t *RandomReaderStretchrTest) Test_ReadFromRangeReader_WhenReaderHasLessDataThanRequested() { + testCases := []struct { + name string + readHandle []byte + }{ + { + name: "GCSReturnedReadHandle", + readHandle: []byte("fake-handle"), + }, + { + name: "GCSReturnedNoReadHandle", + readHandle: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.rr.wrapped.start = 0 + t.rr.wrapped.limit = 6 + t.rr.wrapped.totalReadBytes.Store(0) + dataSize := 6 + testContent := testutil.GenerateRandomBytes(dataSize) + rc := &fake.FakeReader{ + ReadCloser: getReadCloser(testContent), + Handle: tc.readHandle, + } + t.rr.wrapped.reader = rc + t.rr.wrapped.cancel = func() {} + buf := make([]byte, 10) + + n, err := t.rr.wrapped.readFromRangeReader(t.rr.ctx, buf, 0, 10, metrics.ReadTypeUnknown) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), dataSize, n) + // Verify the reader state. + assert.Nil(t.T(), t.rr.wrapped.reader) + assert.Nil(t.T(), t.rr.wrapped.cancel) + assert.Equal(t.T(), int64(dataSize), t.rr.wrapped.start) + assert.Equal(t.T(), int64(dataSize), t.rr.wrapped.limit) + assert.Equal(t.T(), uint64(dataSize), t.rr.wrapped.totalReadBytes.Load()) + assert.Equal(t.T(), int64(dataSize), t.rr.wrapped.expectedOffset.Load()) + expectedReadHandle := tc.readHandle + assert.Equal(t.T(), expectedReadHandle, t.rr.wrapped.readHandle) + }) + } +} + +func (t *RandomReaderStretchrTest) Test_ReadFromRangeReader_WhenReaderReturnedMoreData() { + testCases := []struct { + name string + readHandle []byte + }{ + { + name: "GCSReturnedReadHandle", + readHandle: []byte("fake-handle"), + }, + { + name: "GCSReturnedNoReadHandle", + readHandle: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.rr.wrapped.start = 0 + t.rr.wrapped.limit = 6 + t.rr.wrapped.totalReadBytes.Store(0) + dataSize := 8 + testContent := testutil.GenerateRandomBytes(dataSize) + rc := &fake.FakeReader{ + ReadCloser: getReadCloser(testContent), + Handle: tc.readHandle, + } + t.rr.wrapped.reader = rc + t.rr.wrapped.cancel = func() {} + buf := make([]byte, 10) + + _, err := t.rr.wrapped.readFromRangeReader(t.rr.ctx, buf, 0, 10, metrics.ReadTypeUnknown) + + assert.True(t.T(), strings.Contains(err.Error(), "extra bytes: 2")) + assert.Nil(t.T(), t.rr.wrapped.reader) + assert.Nil(t.T(), t.rr.wrapped.cancel) + assert.Equal(t.T(), int64(-1), t.rr.wrapped.start) + assert.Equal(t.T(), int64(-1), t.rr.wrapped.limit) + assert.Equal(t.T(), uint64(8), t.rr.wrapped.totalReadBytes.Load()) + assert.Equal(t.T(), int64(0), t.rr.wrapped.expectedOffset.Load()) + expectedReadHandle := tc.readHandle + assert.Equal(t.T(), expectedReadHandle, t.rr.wrapped.readHandle) + }) + } +} + +func (t *RandomReaderStretchrTest) Test_ReadFromRangeReader_WhenReaderReturnedEOF() { + t.rr.wrapped.start = 0 + t.rr.wrapped.limit = 10 + dataSize := 6 + testContent := testutil.GenerateRandomBytes(dataSize) + rc := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.rr.wrapped.reader = rc + t.rr.wrapped.cancel = func() {} + buf := make([]byte, 10) + + _, err := t.rr.wrapped.readFromRangeReader(t.rr.ctx, buf, 0, 10, metrics.ReadTypeUnknown) + + assert.True(t.T(), strings.Contains(err.Error(), "skipping 4 bytes")) + assert.Equal(t.T(), int64(0), t.rr.wrapped.expectedOffset.Load()) +} + +func (t *RandomReaderStretchrTest) Test_ExistingReader_WrongOffset() { + testCases := []struct { + name string + readHandle []byte + }{ + { + name: "ReaderHasReadHandle", + readHandle: []byte("fake-handle"), + }, + { + name: "ReaderHasNoReadHandle", + readHandle: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + // Simulate an existing reader. + t.rr.wrapped.readHandle = tc.readHandle + t.rr.wrapped.reader = &fake.FakeReader{ + ReadCloser: io.NopCloser(strings.NewReader("xxx")), + Handle: tc.readHandle, + } + t.rr.wrapped.cancel = func() {} + t.rr.wrapped.start = 2 + t.rr.wrapped.limit = 5 + readObjectRequest := &gcs.ReadObjectRequest{ + Name: t.rr.wrapped.object.Name, + Generation: t.rr.wrapped.object.Generation, + Range: &gcs.ByteRange{ + Start: uint64(0), + Limit: t.object.Size, + }, + ReadCompressed: t.rr.wrapped.object.HasContentEncodingGzip(), + ReadHandle: t.rr.wrapped.readHandle, + } + t.mockBucket. + On("NewReaderWithReadHandle", mock.Anything, readObjectRequest). + Return(nil, errors.New(string(tc.readHandle))). + Times(1) + t.mockBucket.On("BucketType", mock.Anything).Return(gcs.BucketType{}).Times(2) + + buf := make([]byte, 1) + + _, err := t.rr.ReadAt(buf, 0) + + t.mockBucket.AssertExpectations(t.T()) + assert.NotNil(t.T(), err) + }) + } +} + +func (t *RandomReaderStretchrTest) Test_ReadAt_ExistingReaderLimitIsLessThanRequestedDataSize() { + t.object.Size = 10 + // Simulate an existing reader. + t.rr.wrapped.reader = &fake.FakeReader{ReadCloser: getReadCloser([]byte("xxx")), Handle: []byte("fake")} + t.rr.wrapped.cancel = func() {} + t.rr.wrapped.start = 2 + t.rr.wrapped.limit = 5 + rc := &fake.FakeReader{ReadCloser: getReadCloser([]byte("abcdefgh"))} + expectedHandleInRequest := []byte(t.rr.wrapped.reader.ReadHandle()) + readObjectRequest := &gcs.ReadObjectRequest{ + Name: t.rr.wrapped.object.Name, + Generation: t.rr.wrapped.object.Generation, + Range: &gcs.ByteRange{ + Start: uint64(2), + Limit: t.object.Size, + }, + ReadCompressed: t.rr.wrapped.object.HasContentEncodingGzip(), + ReadHandle: expectedHandleInRequest, + } + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, readObjectRequest).Return(rc, nil) + t.mockBucket.On("BucketType", mock.Anything).Return(gcs.BucketType{}).Times(2) + requestSize := 6 + buf := make([]byte, requestSize) + + data, err := t.rr.ReadAt(buf, 2) + + require.Nil(t.T(), err) + require.Equal(t.T(), rc, t.rr.wrapped.reader) + require.Equal(t.T(), requestSize, data.Size) + require.Equal(t.T(), "abcdef", string(buf[:data.Size])) + assert.Equal(t.T(), uint64(requestSize), t.rr.wrapped.totalReadBytes.Load()) + assert.Equal(t.T(), int64(2+requestSize), t.rr.wrapped.expectedOffset.Load()) + assert.Equal(t.T(), expectedHandleInRequest, t.rr.wrapped.readHandle) +} + +func (t *RandomReaderStretchrTest) Test_ReadAt_ExistingReaderLimitIsLessThanRequestedObjectSize() { + t.object.Size = 5 + // Simulate an existing reader + t.rr.wrapped.reader = &fake.FakeReader{ReadCloser: getReadCloser([]byte("xxx")), Handle: []byte("fake")} + t.rr.wrapped.cancel = func() {} + t.rr.wrapped.start = 0 + t.rr.wrapped.limit = 3 + rc := &fake.FakeReader{ReadCloser: getReadCloser([]byte("abcde"))} + expectedHandleInRequest := t.rr.wrapped.reader.ReadHandle() + readObjectRequest := &gcs.ReadObjectRequest{ + Name: t.rr.wrapped.object.Name, + Generation: t.rr.wrapped.object.Generation, + Range: &gcs.ByteRange{ + Start: uint64(0), + Limit: t.object.Size, + }, + ReadCompressed: t.rr.wrapped.object.HasContentEncodingGzip(), + ReadHandle: expectedHandleInRequest, + } + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, readObjectRequest).Return(rc, nil) + t.mockBucket.On("BucketType", mock.Anything).Return(gcs.BucketType{}).Times(2) + requestSize := 6 + buf := make([]byte, requestSize) + + data, err := t.rr.ReadAt(buf, 0) + + require.Nil(t.T(), err) + require.Nil(t.T(), t.rr.wrapped.reader) + require.Equal(t.T(), int(t.object.Size), data.Size) + require.Equal(t.T(), "abcde", string(buf[:data.Size])) + assert.Equal(t.T(), t.object.Size, t.rr.wrapped.totalReadBytes.Load()) + assert.Equal(t.T(), int64(t.object.Size), t.rr.wrapped.expectedOffset.Load()) + assert.Equal(t.T(), []byte(nil), t.rr.wrapped.readHandle) +} + +func (t *RandomReaderStretchrTest) Test_ReadAt_InvalidOffset() { + testCases := []struct { + name string + dataSize int + start int + }{ + { + name: "InvalidOffset", + dataSize: 50, + start: 68, + }, + { + name: "NegativeOffset", + dataSize: 100, + start: -50, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.rr.wrapped.reader = nil + t.object.Size = uint64(tc.dataSize) + buf := make([]byte, tc.dataSize) + + _, err := t.rr.wrapped.ReadAt(t.rr.ctx, buf, int64(tc.start)) + + assert.Error(t.T(), err) + }) + } +} + +func (t *RandomReaderStretchrTest) Test_ReadAt_ValidateReadType() { + testCases := []struct { + name string + dataSize int + bucketType gcs.BucketType + readRanges [][]int + expectedReadTypes []int64 + expectedSeeks []int + }{ + { + name: "SequentialReadFlat", + dataSize: 100, + bucketType: gcs.BucketType{Zonal: false}, + readRanges: [][]int{{0, 10}, {10, 20}, {20, 35}, {35, 50}}, + expectedReadTypes: []int64{metrics.ReadTypeSequential, metrics.ReadTypeSequential, metrics.ReadTypeSequential, metrics.ReadTypeSequential}, + expectedSeeks: []int{0, 0, 0, 0, 0}, + }, + { + name: "SequentialReadZonal", + dataSize: 100, + bucketType: gcs.BucketType{Zonal: true}, + readRanges: [][]int{{0, 10}, {10, 20}, {20, 35}, {35, 50}}, + expectedReadTypes: []int64{metrics.ReadTypeSequential, metrics.ReadTypeSequential, metrics.ReadTypeSequential, metrics.ReadTypeSequential}, + expectedSeeks: []int{0, 0, 0, 0, 0}, + }, + { + name: "RandomReadFlat", + dataSize: 100, + bucketType: gcs.BucketType{Zonal: false}, + readRanges: [][]int{{0, 50}, {30, 40}, {10, 20}, {20, 30}, {30, 40}}, + expectedReadTypes: []int64{metrics.ReadTypeSequential, metrics.ReadTypeSequential, metrics.ReadTypeRandom, metrics.ReadTypeRandom, metrics.ReadTypeRandom}, + expectedSeeks: []int{0, 1, 2, 2, 2}, + }, + { + name: "RandomReadZonal", + dataSize: 100, + bucketType: gcs.BucketType{Zonal: true}, + readRanges: [][]int{{0, 50}, {30, 40}, {10, 20}, {20, 30}, {30, 40}}, + expectedReadTypes: []int64{metrics.ReadTypeSequential, metrics.ReadTypeSequential, metrics.ReadTypeRandom, metrics.ReadTypeRandom, metrics.ReadTypeRandom}, + expectedSeeks: []int{0, 1, 2, 2, 2}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + assert.Equal(t.T(), len(tc.readRanges), len(tc.expectedReadTypes), "Test Parameter Error: readRanges and expectedReadTypes should have same length") + t.rr.wrapped.reader = nil + t.rr.wrapped.isMRDInUse.Store(false) + t.rr.wrapped.seeks.Store(0) + t.rr.wrapped.readType.Store(metrics.ReadTypeSequential) + t.rr.wrapped.expectedOffset.Store(0) + t.object.Size = uint64(tc.dataSize) + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + fakeMRDWrapper, err := NewMultiRangeDownloaderWrapper(t.mockBucket, t.object, &cfg.Config{}, nil) + assert.Nil(t.T(), err, "Error in creating MRDWrapper") + t.rr.wrapped.mrdWrapper = fakeMRDWrapper + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloaderWithSleep(t.object, testContent, time.Microsecond)) + t.mockBucket.On("BucketType", mock.Anything).Return(tc.bucketType).Times(len(tc.readRanges) * 2) + + for i, readRange := range tc.readRanges { + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, mock.Anything).Return(&fake.FakeReader{ReadCloser: getReadCloser(testContent)}, nil).Once() + buf := make([]byte, readRange[1]-readRange[0]) + + _, err := t.rr.wrapped.ReadAt(t.rr.ctx, buf, int64(readRange[0])) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), tc.expectedReadTypes[i], t.rr.wrapped.readType.Load()) + assert.Equal(t.T(), int64(readRange[1]), t.rr.wrapped.expectedOffset.Load()) + assert.Equal(t.T(), uint64(tc.expectedSeeks[i]), t.rr.wrapped.seeks.Load()) + } + }) + } +} + +// This test validates the bug fix where seeks are not updated correctly in case of zonal bucket random reads (b/410904634). +func (t *RandomReaderStretchrTest) Test_ReadAt_ValidateZonalRandomReads() { + t.rr.wrapped.reader = nil + t.rr.wrapped.isMRDInUse.Store(false) + t.rr.wrapped.seeks.Store(0) + t.rr.wrapped.readType.Store(metrics.ReadTypeSequential) + t.rr.wrapped.expectedOffset.Store(0) + t.rr.wrapped.totalReadBytes.Store(0) + t.object.Size = 20 * MiB + t.mockBucket.On("BucketType", mock.Anything).Return(gcs.BucketType{Zonal: true}) + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + fakeMRDWrapper, err := NewMultiRangeDownloaderWrapper(t.mockBucket, t.object, &cfg.Config{}, nil) + assert.Nil(t.T(), err, "Error in creating MRDWrapper") + t.rr.wrapped.mrdWrapper = fakeMRDWrapper + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, mock.Anything).Return(&fake.FakeReader{ReadCloser: getReadCloser(testContent)}, nil).Twice() + buf := make([]byte, 3*MiB) + + // Sequential read #1 + _, err = t.rr.wrapped.ReadAt(t.rr.ctx, buf, 13*MiB) + assert.NoError(t.T(), err) + // Random read #1 + seeks := 1 + _, err = t.rr.wrapped.ReadAt(t.rr.ctx, buf, 12*MiB) + assert.NoError(t.T(), err) + assert.Equal(t.T(), uint64(seeks), t.rr.wrapped.seeks.Load()) + + readRanges := [][]int{{11 * MiB, 15 * MiB}, {12 * MiB, 14 * MiB}, {10 * MiB, 12 * MiB}, {9 * MiB, 11 * MiB}, {8 * MiB, 10 * MiB}} + // Series of random reads to check if seeks are updated correctly and MRD is invoked always + for _, readRange := range readRanges { + seeks++ + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloaderWithSleep(t.object, testContent, time.Microsecond)) + buf := make([]byte, readRange[1]-readRange[0]) + + _, err := t.rr.wrapped.ReadAt(t.rr.ctx, buf, int64(readRange[0])) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), metrics.ReadTypeRandom, t.rr.wrapped.readType.Load()) + assert.Equal(t.T(), int64(readRange[1]), t.rr.wrapped.expectedOffset.Load()) + assert.Equal(t.T(), uint64(seeks), t.rr.wrapped.seeks.Load()) + } +} + +func (t *RandomReaderStretchrTest) Test_ReadAt_MRDRead() { + testCases := []struct { + name string + dataSize int + offset int + bytesToRead int + }{ + { + name: "ReadChunk", + dataSize: 100, + offset: 37, + bytesToRead: 43, + }, + { + name: "ReadZeroByte", + dataSize: 100, + offset: 37, + bytesToRead: 0, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.rr.wrapped.reader = nil + t.rr.wrapped.isMRDInUse.Store(false) + t.rr.wrapped.expectedOffset.Store(10) + t.rr.wrapped.seeks.Store(minSeeksForRandom + 1) + t.object.Size = uint64(tc.dataSize) + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + fakeMRDWrapper, err := NewMultiRangeDownloaderWrapper(t.mockBucket, t.object, &cfg.Config{}, nil) + assert.Nil(t.T(), err, "Error in creating MRDWrapper") + t.rr.wrapped.mrdWrapper = fakeMRDWrapper + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloaderWithSleep(t.object, testContent, time.Microsecond)).Times(1) + t.mockBucket.On("BucketType", mock.Anything).Return(gcs.BucketType{Zonal: true}).Times(1) + buf := make([]byte, tc.bytesToRead) + + objData, err := t.rr.wrapped.ReadAt(t.rr.ctx, buf, int64(tc.offset)) + + t.mockBucket.AssertNotCalled(t.T(), "NewReaderWithReadHandle", mock.Anything) + assert.NoError(t.T(), err) + assert.Nil(t.T(), t.rr.wrapped.reader) + assert.Equal(t.T(), tc.bytesToRead, objData.Size) + assert.Equal(t.T(), testContent[tc.offset:tc.offset+tc.bytesToRead], buf[:objData.Size]) + if tc.bytesToRead != 0 { + assert.Equal(t.T(), int64(tc.offset+tc.bytesToRead), t.rr.wrapped.expectedOffset.Load()) + } + }) + } +} + +func (t *RandomReaderStretchrTest) Test_ReadFromMultiRangeReader_ReadFull() { + testCases := []struct { + name string + dataSize int + extraSize int + }{ + { + name: "ReadFull", + dataSize: 100, + extraSize: 0, + }, + { + name: "ReadWithLargerBuffer", + dataSize: 100, + extraSize: 10, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.rr.wrapped.reader = nil + t.rr.wrapped.isMRDInUse.Store(false) + t.object.Size = uint64(tc.dataSize) + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + fakeMRDWrapper, err := NewMultiRangeDownloaderWrapper(t.mockBucket, t.object, &cfg.Config{}, nil) + assert.Nil(t.T(), err, "Error in creating MRDWrapper") + t.rr.wrapped.mrdWrapper = fakeMRDWrapper + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloaderWithSleep(t.object, testContent, time.Microsecond)).Times(1) + t.mockBucket.On("BucketType", mock.Anything).Return(gcs.BucketType{Zonal: true}).Times(1) + buf := make([]byte, tc.dataSize+tc.extraSize) + + bytesRead, err := t.rr.wrapped.readFromMultiRangeReader(t.rr.ctx, buf, 0, int64(t.object.Size), TestTimeoutForMultiRangeRead) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), tc.dataSize, bytesRead) + assert.Equal(t.T(), testContent[:tc.dataSize], buf[:bytesRead]) + assert.Equal(t.T(), int64(t.object.Size), t.rr.wrapped.expectedOffset.Load()) + }) + } +} + +func (t *RandomReaderStretchrTest) Test_ReadFromMultiRangeReader_ReadChunk() { + testCases := []struct { + name string + dataSize int + start int + end int + }{ + { + name: "ReadChunk", + dataSize: 100, + start: 37, + end: 93, + }, + } + + for _, tc := range testCases { + t.rr.wrapped.reader = nil + t.object.Size = uint64(tc.dataSize) + testContent := testutil.GenerateRandomBytes(int(t.object.Size)) + fakeMRDWrapper, err := NewMultiRangeDownloaderWrapper(t.mockBucket, t.object, &cfg.Config{}, nil) + assert.Nil(t.T(), err, "Error in creating MRDWrapper") + t.rr.wrapped.mrdWrapper = fakeMRDWrapper + t.mockBucket.On("NewMultiRangeDownloader", mock.Anything, mock.Anything).Return(fake.NewFakeMultiRangeDownloaderWithSleep(t.object, testContent, time.Microsecond)).Times(1) + t.mockBucket.On("BucketType", mock.Anything).Return(gcs.BucketType{Zonal: true}).Times(1) + buf := make([]byte, tc.end-tc.start) + + bytesRead, err := t.rr.wrapped.readFromMultiRangeReader(t.rr.ctx, buf, int64(tc.start), int64(tc.end), TestTimeoutForMultiRangeRead) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), tc.end-tc.start, bytesRead) + assert.Equal(t.T(), testContent[tc.start:tc.end], buf[:bytesRead]) + assert.Equal(t.T(), int64(tc.end), t.rr.wrapped.expectedOffset.Load()) + } +} + +func (t *RandomReaderStretchrTest) Test_ReadFromMultiRangeReader_NilMRDWrapper() { + t.rr.wrapped.mrdWrapper = nil + + bytesRead, err := t.rr.wrapped.readFromMultiRangeReader(t.rr.ctx, make([]byte, t.object.Size), 0, int64(t.object.Size), TestTimeoutForMultiRangeRead) + + assert.ErrorContains(t.T(), err, "readFromMultiRangeReader: Invalid MultiRangeDownloaderWrapper") + assert.Equal(t.T(), 0, bytesRead) +} + +// Validates: +// 1. No change in ReadAt behavior based inactiveStreamTimeout config. +// 2. Valid timeout config creates InactiveTimeoutReader instance of storage.Reader. +func (t *RandomReaderStretchrTest) Test_ReadAt_WithAndWithoutReadConfig() { + testCases := []struct { + name string + config *cfg.Config + expectInactiveTimeoutReader bool + }{ + { + name: "WithoutReadConfig", + config: nil, + expectInactiveTimeoutReader: false, + }, + { + name: "WithReadConfigAndZeroTimeout", + config: &cfg.Config{Read: cfg.ReadConfig{InactiveStreamTimeout: 0}}, + expectInactiveTimeoutReader: false, + }, + { + name: "WithReadConfigAndPositiveTimeout", + config: &cfg.Config{Read: cfg.ReadConfig{InactiveStreamTimeout: 10 * time.Millisecond}}, + expectInactiveTimeoutReader: true, + }, + } + + objectSize := uint64(20) + readOffset := int64(0) + readLength := 10 // Reading only 10 bytes from the complete object reader. + + for _, tc := range testCases { + t.Run(tc.name, func() { + t.SetupTest() // Resets mockBucket, rr, etc. for each sub-test + defer t.TearDownTest() + + t.rr.wrapped.config = tc.config + t.rr.wrapped.reader = nil // Ensure startRead path is taken in ReadAt + t.object.Size = objectSize + // Prepare fake content for the GCS object. + // startRead will attempt to read the entire object [0, objectSize) + // because objectSize is small compared to typical sequentialReadSizeMb. + fakeReaderContent := testutil.GenerateRandomBytes(int(t.object.Size)) + rc := &fake.FakeReader{ReadCloser: getReadCloser(fakeReaderContent)} + expectedReadObjectRequest := &gcs.ReadObjectRequest{ + Name: t.rr.wrapped.object.Name, + Generation: t.rr.wrapped.object.Generation, + Range: &gcs.ByteRange{ + Start: uint64(readOffset), // Read from the beginning + Limit: uint64(t.object.Size), // getReadInfo will determine this limit + }, + ReadCompressed: t.rr.wrapped.object.HasContentEncodingGzip(), + ReadHandle: nil, // No existing read handle + } + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, expectedReadObjectRequest).Return(rc, nil).Once() + // BucketType is called by ReadAt -> getReadInfo -> readerType to determine reader strategy. + t.mockBucket.On("BucketType", mock.Anything).Return(gcs.BucketType{Zonal: false}).Twice() + buf := make([]byte, readLength) + + objectData, err := t.rr.ReadAt(buf, readOffset) + + t.mockBucket.AssertExpectations(t.T()) + assert.NoError(t.T(), err) + assert.Equal(t.T(), readLength, objectData.Size) + assert.Equal(t.T(), fakeReaderContent[:readLength], buf[:objectData.Size]) // Ensure buffer is populated correctly + assert.NotNil(t.T(), t.rr.wrapped.reader, "Reader should be active as partial data read from the requested range.") + assert.NotNil(t.T(), t.rr.wrapped.cancel) + assert.Equal(t.T(), int64(readLength), t.rr.wrapped.start) + assert.Equal(t.T(), int64(t.object.Size), t.rr.wrapped.limit) + _, isInactiveTimeoutReader := t.rr.wrapped.reader.(*InactiveTimeoutReader) + assert.Equal(t.T(), tc.expectInactiveTimeoutReader, isInactiveTimeoutReader) + }) + } +} diff --git a/internal/gcsx/random_reader_test.go b/internal/gcsx/random_reader_test.go index b43402a9ea..be659b6b46 100644 --- a/internal/gcsx/random_reader_test.go +++ b/internal/gcsx/random_reader_test.go @@ -15,7 +15,6 @@ package gcsx import ( - "bytes" "errors" "fmt" "io" @@ -27,14 +26,20 @@ import ( "testing/iotest" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/file" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/file/downloader" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/lru" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/util" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - testutil "github.com/googlecloudplatform/gcsfuse/v2/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util/diskutil" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/file" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/file/downloader" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/gcsfuse_errors" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + testutil "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" "github.com/jacobsa/fuse/fuseops" . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/oglemock" @@ -42,6 +47,8 @@ import ( "golang.org/x/net/context" ) +// NOTE: Please add new tests in random_reader_stretchr_test.go file. This file +// is deprecated and these tests will be moved to the random_reader_stretchr_test.go func TestRandomReader(t *testing.T) { RunTests(t) } //////////////////////////////////////////////////////////////////////// @@ -53,7 +60,7 @@ type checkingRandomReader struct { wrapped *randomReader } -func (rr *checkingRandomReader) ReadAt(p []byte, offset int64) (int, bool, error) { +func (rr *checkingRandomReader) ReadAt(p []byte, offset int64) (ObjectData, error) { rr.wrapped.CheckInvariants() defer rr.wrapped.CheckInvariants() return rr.wrapped.ReadAt(rr.ctx, p, offset) @@ -98,7 +105,7 @@ func (br *blockingReader) Read(p []byte) (n int, err error) { //////////////////////////////////////////////////////////////////////// func rangeStartIs(expected uint64) (m Matcher) { - pred := func(c interface{}) (err error) { + pred := func(c any) (err error) { req := c.(*gcs.ReadObjectRequest) if req.Range == nil { err = errors.New("which has a nil range") @@ -118,7 +125,7 @@ func rangeStartIs(expected uint64) (m Matcher) { } func rangeLimitIs(expected uint64) (m Matcher) { - pred := func(c interface{}) (err error) { + pred := func(c any) (err error) { req := c.(*gcs.ReadObjectRequest) if req.Range == nil { err = errors.New("which has a nil range") @@ -141,10 +148,6 @@ func rangeLimitIs(expected uint64) (m Matcher) { // Boilerplate //////////////////////////////////////////////////////////////////////// -const sequentialReadSizeInMb = 22 -const sequentialReadSizeInBytes = sequentialReadSizeInMb * MB -const CacheMaxSize = 2 * sequentialReadSizeInMb * util.MiB - type RandomReaderTest struct { object *gcs.MinObject bucket storage.MockBucket @@ -152,9 +155,13 @@ type RandomReaderTest struct { cacheDir string jobManager *downloader.JobManager cacheHandler *file.CacheHandler + bucketType gcs.BucketType } -func init() { RegisterTestSuite(&RandomReaderTest{}) } +func init() { + RegisterTestSuite(&RandomReaderTest{bucketType: gcs.BucketType{}}) + RegisterTestSuite(&RandomReaderTest{bucketType: gcs.BucketType{Zonal: true, Hierarchical: true}}) +} var _ SetUpInterface = &RandomReaderTest{} var _ TearDownInterface = &RandomReaderTest{} @@ -174,14 +181,16 @@ func (t *RandomReaderTest) SetUp(ti *TestInfo) { t.bucket = storage.NewMockBucket(ti.MockController, "bucket") t.cacheDir = path.Join(os.Getenv("HOME"), "cache/dir") - lruCache := lru.NewCache(CacheMaxSize) - t.jobManager = downloader.NewJobManager(lruCache, util.DefaultFilePerm, util.DefaultDirPerm, t.cacheDir, sequentialReadSizeInMb, &cfg.FileCacheConfig{ + lruCache := lru.NewCache(cacheMaxSize) + fileCacheConfig := &cfg.FileCacheConfig{ EnableCrc: false, - }) - t.cacheHandler = file.NewCacheHandler(lruCache, t.jobManager, t.cacheDir, util.DefaultFilePerm, util.DefaultDirPerm) + } + cacheDirVolumeBlockSize := diskutil.GetVolumeBlockSize(t.cacheDir) + t.jobManager = downloader.NewJobManager(lruCache, util.DefaultFilePerm, util.DefaultDirPerm, t.cacheDir, sequentialReadSizeInMb, fileCacheConfig, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), cacheDirVolumeBlockSize) + t.cacheHandler = file.NewCacheHandler(lruCache, t.jobManager, t.cacheDir, util.DefaultFilePerm, util.DefaultDirPerm, "", "", false, cacheDirVolumeBlockSize) // Set up the reader. - rr := NewRandomReader(t.object, t.bucket, sequentialReadSizeInMb, nil, false) + rr := NewRandomReader(t.object, t.bucket, sequentialReadSizeInMb, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), nil, nil, 0) t.rr.wrapped = rr.(*randomReader) } @@ -189,17 +198,10 @@ func (t *RandomReaderTest) TearDown() { t.rr.Destroy() } -func getReadCloser(content []byte) io.ReadCloser { - r := bytes.NewReader(content) - rc := io.NopCloser(r) - return rc -} - -func (t *RandomReaderTest) mockNewReaderCallForTestBucket(start uint64, limit uint64, rc io.ReadCloser) { - ExpectCall(t.bucket, "NewReader")( - Any(), - AllOf(rangeStartIs(start), rangeLimitIs(limit))). - WillRepeatedly(Return(rc, nil)) +func (t *RandomReaderTest) mockNewReaderWithHandleCallForTestBucket(start uint64, limit uint64, rd gcs.StorageReader) { + ExpectCall(t.bucket, "NewReaderWithReadHandle")( + Any(), AllOf(rangeStartIs(start), rangeLimitIs(limit))). + WillRepeatedly(Return(rd, nil)) } //////////////////////////////////////////////////////////////////////// @@ -210,54 +212,39 @@ func (t *RandomReaderTest) EmptyRead() { // Nothing should happen. buf := make([]byte, 0) - n, _, err := t.rr.ReadAt(buf, 0) + objectData, err := t.rr.ReadAt(buf, 0) - ExpectEq(0, n) + ExpectEq(0, objectData.Size) ExpectEq(nil, err) } func (t *RandomReaderTest) ReadAtEndOfObject() { buf := make([]byte, 1) - n, _, err := t.rr.ReadAt(buf, int64(t.object.Size)) + objectData, err := t.rr.ReadAt(buf, int64(t.object.Size)) - ExpectEq(0, n) + ExpectEq(0, objectData.Size) ExpectEq(io.EOF, err) } func (t *RandomReaderTest) ReadPastEndOfObject() { buf := make([]byte, 1) - n, cacheHit, err := t.rr.ReadAt(buf, int64(t.object.Size)+1) + objectData, err := t.rr.ReadAt(buf, int64(t.object.Size)+1) - ExpectFalse(cacheHit) - ExpectEq(0, n) + ExpectFalse(objectData.CacheHit) + ExpectEq(0, objectData.Size) ExpectEq(io.EOF, err) } func (t *RandomReaderTest) NoExistingReader() { // The bucket should be called to set up a new reader. - ExpectCall(t.bucket, "NewReader")(Any(), Any()). + ExpectCall(t.bucket, "NewReaderWithReadHandle")(Any(), Any()). WillOnce(Return(nil, errors.New(""))) + ExpectCall(t.bucket, "BucketType")().Times(2).WillOnce(Return(t.bucketType)) buf := make([]byte, 1) - _, _, err := t.rr.ReadAt(buf, 0) - - AssertNe(nil, err) -} - -func (t *RandomReaderTest) ExistingReader_WrongOffset() { - // Simulate an existing reader. - t.rr.wrapped.reader = io.NopCloser(strings.NewReader("xxx")) - t.rr.wrapped.cancel = func() {} - t.rr.wrapped.start = 2 - t.rr.wrapped.limit = 5 - // The bucket should be called to set up a new reader. - ExpectCall(t.bucket, "NewReader")(Any(), Any()). - WillOnce(Return(nil, errors.New(""))) - buf := make([]byte, 1) - - _, _, err := t.rr.ReadAt(buf, 0) + _, err := t.rr.ReadAt(buf, 0) AssertNe(nil, err) } @@ -269,14 +256,17 @@ func (t *RandomReaderTest) ExistingReader_ReadAtOffsetAfterTheReaderPosition() { var readSize int64 = 1 var expectedStartOffsetAfterRead = readAtOffset + readSize // Simulate an existing reader. - rc := io.NopCloser(strings.NewReader(strings.Repeat("x", int(readerLimit)))) + nopCloser := io.NopCloser(strings.NewReader(strings.Repeat("x", int(readerLimit)))) + rc := &fake.FakeReader{ReadCloser: nopCloser} t.rr.wrapped.reader = rc t.rr.wrapped.cancel = func() {} t.rr.wrapped.start = currentStartOffset t.rr.wrapped.limit = readerLimit + ExpectCall(t.bucket, "BucketType")().Times(2).WillOnce(Return(t.bucketType)) + buf := make([]byte, readSize) - _, _, err := t.rr.ReadAt(buf, readAtOffset) + _, err := t.rr.ReadAt(buf, readAtOffset) AssertEq(nil, err) ExpectThat(rc, DeepEquals(t.rr.wrapped.reader)) @@ -285,52 +275,42 @@ func (t *RandomReaderTest) ExistingReader_ReadAtOffsetAfterTheReaderPosition() { } func (t *RandomReaderTest) NewReaderReturnsError() { - ExpectCall(t.bucket, "NewReader")(Any(), Any()). + ExpectCall(t.bucket, "NewReaderWithReadHandle")(Any(), Any()). WillOnce(Return(nil, errors.New("taco"))) + ExpectCall(t.bucket, "BucketType")().Times(2).WillOnce(Return(t.bucketType)) buf := make([]byte, 1) - _, _, err := t.rr.ReadAt(buf, 0) + _, err := t.rr.ReadAt(buf, 0) - ExpectThat(err, Error(HasSubstr("NewReader"))) + ExpectThat(err, Error(HasSubstr("NewReaderWithReadHandle"))) ExpectThat(err, Error(HasSubstr("taco"))) } func (t *RandomReaderTest) ReaderFails() { // Bucket r := iotest.OneByteReader(iotest.TimeoutReader(strings.NewReader("xxx"))) - rc := io.NopCloser(r) + rc := &fake.FakeReader{ReadCloser: io.NopCloser(r)} - ExpectCall(t.bucket, "NewReader")(Any(), Any()). + ExpectCall(t.bucket, "NewReaderWithReadHandle")(Any(), Any()). WillOnce(Return(rc, nil)) + ExpectCall(t.bucket, "BucketType")().Times(2).WillOnce(Return(t.bucketType)) // Call buf := make([]byte, 3) - _, _, err := t.rr.ReadAt(buf, 0) + _, err := t.rr.ReadAt(buf, 0) ExpectThat(err, Error(HasSubstr("readFull"))) ExpectThat(err, Error(HasSubstr(iotest.ErrTimeout.Error()))) } -func (t *RandomReaderTest) ReaderOvershootsRange() { - // Simulate a reader that is supposed to return two more bytes, but actually - // returns three when asked to. - t.rr.wrapped.reader = io.NopCloser(strings.NewReader("xxx")) - t.rr.wrapped.cancel = func() {} - t.rr.wrapped.start = 0 - t.rr.wrapped.limit = 2 - - // Try to read three bytes. - buf := make([]byte, 3) - _, _, err := t.rr.ReadAt(buf, 0) - - ExpectThat(err, Error(HasSubstr("1 too many bytes"))) -} - func (t *RandomReaderTest) ReaderNotExhausted() { // Set up a reader that has three bytes left to give. - rc := &countingCloser{ + cc := &countingCloser{ Reader: strings.NewReader("abc"), } + rc := &fake.FakeReader{ReadCloser: cc} + + ExpectCall(t.bucket, "BucketType")().Times(2).WillOnce(Return(t.bucketType)) t.rr.wrapped.reader = rc t.rr.wrapped.cancel = func() {} @@ -339,14 +319,14 @@ func (t *RandomReaderTest) ReaderNotExhausted() { // Read two bytes. buf := make([]byte, 2) - n, cacheHit, err := t.rr.ReadAt(buf, 1) + objectData, err := t.rr.ReadAt(buf, 1) - ExpectFalse(cacheHit) - ExpectEq(2, n) + ExpectFalse(objectData.CacheHit) + ExpectEq(2, objectData.Size) ExpectEq(nil, err) - ExpectEq("ab", string(buf[:n])) + ExpectEq("ab", string(buf[:objectData.Size])) - ExpectEq(0, rc.closeCount) + ExpectEq(0, cc.closeCount) ExpectEq(rc, t.rr.wrapped.reader) ExpectEq(3, t.rr.wrapped.start) ExpectEq(4, t.rr.wrapped.limit) @@ -358,48 +338,21 @@ func (t *RandomReaderTest) ReaderExhausted_ReadFinished() { Reader: strings.NewReader("abc"), } - t.rr.wrapped.reader = rc + ExpectCall(t.bucket, "BucketType")().Times(2).WillOnce(Return(t.bucketType)) + + t.rr.wrapped.reader = &fake.FakeReader{ReadCloser: rc} t.rr.wrapped.cancel = func() {} t.rr.wrapped.start = 1 t.rr.wrapped.limit = 4 // Read three bytes. buf := make([]byte, 3) - n, cacheHit, err := t.rr.ReadAt(buf, 1) + objectData, err := t.rr.ReadAt(buf, 1) - ExpectFalse(cacheHit) - ExpectEq(3, n) + ExpectFalse(objectData.CacheHit) + ExpectEq(3, objectData.Size) ExpectEq(nil, err) - ExpectEq("abc", string(buf[:n])) - - ExpectEq(1, rc.closeCount) - ExpectEq(nil, t.rr.wrapped.reader) - ExpectEq(nil, t.rr.wrapped.cancel) - ExpectEq(4, t.rr.wrapped.limit) -} - -func (t *RandomReaderTest) ReaderExhausted_ReadNotFinished() { - // Set up a reader that has three bytes left to give. - rc := &countingCloser{ - Reader: strings.NewReader("abc"), - } - - t.rr.wrapped.reader = rc - t.rr.wrapped.cancel = func() {} - t.rr.wrapped.start = 1 - t.rr.wrapped.limit = 4 - - // The bucket should be called at the previous limit to obtain a new reader. - ExpectCall(t.bucket, "NewReader")(Any(), rangeStartIs(4)). - WillOnce(Return(nil, errors.New(""))) - - // Attempt to read four bytes. - buf := make([]byte, 4) - n, cacheHit, _ := t.rr.ReadAt(buf, 1) - - ExpectFalse(cacheHit) - AssertGe(n, 3) - ExpectEq("abc", string(buf[:3])) + ExpectEq("abc", string(buf[:objectData.Size])) ExpectEq(1, rc.closeCount) ExpectEq(nil, t.rr.wrapped.reader) @@ -412,9 +365,12 @@ func (t *RandomReaderTest) PropagatesCancellation() { finishRead := make(chan struct{}) rc := io.NopCloser(&blockingReader{finishRead}) - t.rr.wrapped.reader = rc + ExpectCall(t.bucket, "BucketType")().Times(2).WillOnce(Return(t.bucketType)) + + t.rr.wrapped.reader = &fake.FakeReader{ReadCloser: rc} t.rr.wrapped.start = 1 t.rr.wrapped.limit = 4 + t.rr.wrapped.config = &cfg.Config{FileSystem: cfg.FileSystemConfig{IgnoreInterrupts: false}} // Snoop on when cancel is called. cancelCalled := make(chan struct{}) @@ -450,10 +406,12 @@ func (t *RandomReaderTest) PropagatesCancellation() { func (t *RandomReaderTest) DoesntPropagateCancellationAfterReturning() { // Set up a reader that will return three bytes. - t.rr.wrapped.reader = io.NopCloser(strings.NewReader("xxx")) + t.rr.wrapped.reader = &fake.FakeReader{ReadCloser: getReadCloser([]byte("xxx"))} t.rr.wrapped.start = 1 t.rr.wrapped.limit = 4 + ExpectCall(t.bucket, "BucketType")().Times(2).WillOnce(Return(t.bucketType)) + // Snoop on when cancel is called. cancelCalled := make(chan struct{}) t.rr.wrapped.cancel = func() { close(cancelCalled) } @@ -461,11 +419,11 @@ func (t *RandomReaderTest) DoesntPropagateCancellationAfterReturning() { // Successfully read two bytes using a context whose cancellation we control. ctx, cancel := context.WithCancel(context.Background()) buf := make([]byte, 2) - n, cacheHit, err := t.rr.wrapped.ReadAt(ctx, buf, 1) + objectData, err := t.rr.wrapped.ReadAt(ctx, buf, 1) - ExpectFalse(cacheHit) AssertEq(nil, err) - AssertEq(2, n) + ExpectFalse(objectData.CacheHit) + AssertEq(2, objectData.Size) // If we cancel the calling context now, it should not cause the underlying // read context to be cancelled. @@ -479,14 +437,14 @@ func (t *RandomReaderTest) DoesntPropagateCancellationAfterReturning() { } func (t *RandomReaderTest) UpgradesReadsToObjectSize() { - const objectSize = 2 * MB + const objectSize = 2 * MiB t.object.Size = objectSize const readSize = 10 AssertLt(readSize, objectSize) // Simulate an existing reader at a mismatched offset. - t.rr.wrapped.reader = io.NopCloser(strings.NewReader("xxx")) + t.rr.wrapped.reader = &fake.FakeReader{ReadCloser: getReadCloser([]byte("xxx"))} t.rr.wrapped.cancel = func() {} t.rr.wrapped.start = 2 t.rr.wrapped.limit = 5 @@ -494,19 +452,20 @@ func (t *RandomReaderTest) UpgradesReadsToObjectSize() { // The bucket should be asked to read the entire object, even though we only // ask for readSize bytes below, to minimize the cost for GCS requests. r := strings.NewReader(strings.Repeat("x", objectSize)) - rc := io.NopCloser(r) + rc := &fake.FakeReader{ReadCloser: io.NopCloser(r)} - ExpectCall(t.bucket, "NewReader")( + ExpectCall(t.bucket, "NewReaderWithReadHandle")( Any(), AllOf(rangeStartIs(1), rangeLimitIs(objectSize))). WillOnce(Return(rc, nil)) + ExpectCall(t.bucket, "BucketType")().Times(2).WillOnce(Return(t.bucketType)) // Call through. buf := make([]byte, readSize) - _, cacheHit, err := t.rr.ReadAt(buf, 1) + objectData, err := t.rr.ReadAt(buf, 1) // Check the state now. - ExpectFalse(cacheHit) + ExpectFalse(objectData.CacheHit) ExpectEq(nil, err) ExpectEq(1+readSize, t.rr.wrapped.start) ExpectEq(objectSize, t.rr.wrapped.limit) @@ -514,7 +473,7 @@ func (t *RandomReaderTest) UpgradesReadsToObjectSize() { func (t *RandomReaderTest) UpgradeReadsToAverageSize() { t.object.Size = 1 << 40 - const totalReadBytes = 6 * MB + const totalReadBytes = 6 * MiB const numReads = 2 const avgReadBytes = totalReadBytes / numReads @@ -523,30 +482,32 @@ func (t *RandomReaderTest) UpgradeReadsToAverageSize() { const readSize = 2 * minReadSize // Simulate an existing reader at a mismatched offset. - t.rr.wrapped.seeks = numReads - t.rr.wrapped.totalReadBytes = totalReadBytes - t.rr.wrapped.reader = io.NopCloser(strings.NewReader("xxx")) + t.rr.wrapped.seeks.Store(numReads) + t.rr.wrapped.totalReadBytes.Store(totalReadBytes) + t.rr.wrapped.reader = &fake.FakeReader{ReadCloser: getReadCloser([]byte("xxx"))} t.rr.wrapped.cancel = func() {} t.rr.wrapped.start = 2 t.rr.wrapped.limit = 5 + t.rr.wrapped.expectedOffset.Store(2) // The bucket should be asked to read expectedBytesToRead bytes. r := strings.NewReader(strings.Repeat("x", expectedBytesToRead)) - rc := io.NopCloser(r) + rc := &fake.FakeReader{ReadCloser: io.NopCloser(r)} - ExpectCall(t.bucket, "NewReader")( + ExpectCall(t.bucket, "NewReaderWithReadHandle")( Any(), AllOf( rangeStartIs(start), rangeLimitIs(start+expectedBytesToRead), )).WillOnce(Return(rc, nil)) + ExpectCall(t.bucket, "BucketType")().Times(2).WillOnce(Return(t.bucketType)) // Call through. buf := make([]byte, readSize) - _, cacheHit, err := t.rr.ReadAt(buf, start) + objectData, err := t.rr.ReadAt(buf, start) // Check the state now. - ExpectFalse(cacheHit) + ExpectFalse(objectData.CacheHit) AssertEq(nil, err) ExpectEq(start+expectedBytesToRead, t.rr.wrapped.limit) } @@ -560,37 +521,38 @@ func (t *RandomReaderTest) UpgradesSequentialReads_ExistingReader() { const existingSize = 3 r := strings.NewReader(strings.Repeat("x", existingSize)) - t.rr.wrapped.reader = io.NopCloser(r) + t.rr.wrapped.reader = &fake.FakeReader{ReadCloser: io.NopCloser(r)} t.rr.wrapped.cancel = func() {} t.rr.wrapped.start = 1 t.rr.wrapped.limit = 1 + existingSize // The bucket should be asked to read up to the end of the object. - r = strings.NewReader(strings.Repeat("x", readSize-existingSize)) - rc := io.NopCloser(r) + r = strings.NewReader(strings.Repeat("y", readSize)) + rc := &fake.FakeReader{ReadCloser: io.NopCloser(r)} - ExpectCall(t.bucket, "NewReader")( + ExpectCall(t.bucket, "NewReaderWithReadHandle")( Any(), - AllOf(rangeStartIs(1+existingSize), rangeLimitIs(1+existingSize+sequentialReadSizeInBytes))). + AllOf(rangeStartIs(1), rangeLimitIs(1+sequentialReadSizeInBytes))). WillOnce(Return(rc, nil)) + ExpectCall(t.bucket, "BucketType")().Times(2).WillOnce(Return(t.bucketType)) // Call through. buf := make([]byte, readSize) - _, cacheHit, err := t.rr.ReadAt(buf, 1) + objectData, err := t.rr.ReadAt(buf, 1) // Check the state now. - ExpectFalse(cacheHit) + ExpectFalse(objectData.CacheHit) AssertEq(nil, err) ExpectEq(1+readSize, t.rr.wrapped.start) // Limit is same as the byteRange of last GCS call made. - ExpectEq(1+existingSize+sequentialReadSizeInBytes, t.rr.wrapped.limit) + ExpectEq(1+sequentialReadSizeInBytes, t.rr.wrapped.limit) } func (t *RandomReaderTest) UpgradesSequentialReads_NoExistingReader() { t.object.Size = 1 << 40 - const readSize = 1 * MB + const readSize = 1 * MiB // Set up the custom randomReader. - rr := NewRandomReader(t.object, t.bucket, readSize/MB, nil, false) + rr := NewRandomReader(t.object, t.bucket, readSize/MiB, nil, false, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), nil, nil, 0) t.rr.wrapped = rr.(*randomReader) // Simulate a previous exhausted reader that ended at the offset from which @@ -599,146 +561,47 @@ func (t *RandomReaderTest) UpgradesSequentialReads_NoExistingReader() { t.rr.wrapped.limit = 1 // The bucket should be asked to read up to the end of the object. - r := strings.NewReader(strings.Repeat("x", readSize)) - rc := io.NopCloser(r) + data := strings.Repeat("x", readSize) + r := strings.NewReader(data) + rc := &fake.FakeReader{ReadCloser: io.NopCloser(r)} - ExpectCall(t.bucket, "NewReader")( + ExpectCall(t.bucket, "NewReaderWithReadHandle")( Any(), AllOf(rangeStartIs(1), rangeLimitIs(1+readSize))). WillOnce(Return(rc, nil)) + ExpectCall(t.bucket, "BucketType")().Times(2).WillOnce(Return(t.bucketType)) // Call through. buf := make([]byte, readSize) - _, cacheHit, err := t.rr.ReadAt(buf, 1) + objectData, err := t.rr.ReadAt(buf, 1) // Check the state now. - ExpectFalse(cacheHit) - ExpectEq(nil, err) + ExpectFalse(objectData.CacheHit) + AssertEq(nil, err) + ExpectEq(data, string(buf)) ExpectEq(1+readSize, t.rr.wrapped.start) ExpectEq(1+readSize, t.rr.wrapped.limit) } -func (t *RandomReaderTest) SequentialReads_NoExistingReader_requestedSizeGreaterThanChunkSize() { - t.object.Size = 1 << 40 - const chunkSize = 1 * MB - const readSize = 3 * MB - // Set up the custom randomReader. - rr := NewRandomReader(t.object, t.bucket, chunkSize/MB, nil, false) - t.rr.wrapped = rr.(*randomReader) - // Create readers for each chunk. - chunk1Reader := strings.NewReader(strings.Repeat("x", chunkSize)) - chunk1RC := io.NopCloser(chunk1Reader) - chunk2Reader := strings.NewReader(strings.Repeat("x", chunkSize)) - chunk2RC := io.NopCloser(chunk2Reader) - chunk3Reader := strings.NewReader(strings.Repeat("x", chunkSize)) - chunk3RC := io.NopCloser(chunk3Reader) - // Mock the NewReader calls to return chunkReaders created above. - // We will make 3 GCS calls to satisfy the requested read size. But since we - // already have a reader with 'existingSize' data, we will first read that data - // and then make GCS calls. So call sequence is - // [0, chunkSize) -> newReader - // [hunkSize, chunkSize*2) -> newReader - // [chunkSize*2, chunkSize*3) -> newReader - ExpectCall(t.bucket, "NewReader")( - Any(), - AllOf(rangeStartIs(0), rangeLimitIs(chunkSize))). - WillOnce(Return(chunk1RC, nil)) - ExpectCall(t.bucket, "NewReader")( - Any(), - AllOf(rangeStartIs(chunkSize), rangeLimitIs(chunkSize*2))). - WillOnce(Return(chunk2RC, nil)) - ExpectCall(t.bucket, "NewReader")( - Any(), - AllOf(rangeStartIs(chunkSize*2), rangeLimitIs(chunkSize*3))). - WillOnce(Return(chunk3RC, nil)) - - // Call through. - buf := make([]byte, readSize) - _, cacheHit, err := t.rr.ReadAt(buf, 0) - - // Check the state now. - ExpectFalse(cacheHit) - ExpectEq(nil, err) - // Start is the total data read. - ExpectEq(readSize, t.rr.wrapped.start) - // Limit is same as the byteRange of last GCS call made. - ExpectEq(readSize, t.rr.wrapped.limit) -} - -func (t *RandomReaderTest) SequentialReads_existingReader_requestedSizeGreaterThanChunkSize() { - t.object.Size = 1 << 40 - const chunkSize = 1 * MB - const readSize = 3 * MB - // Set up the custom randomReader. - rr := NewRandomReader(t.object, t.bucket, chunkSize/MB, nil, false) - t.rr.wrapped = rr.(*randomReader) - // Simulate an existing reader at the correct offset, which will be exhausted - // by the read below. - const existingSize = 3 - r := strings.NewReader(strings.Repeat("x", existingSize)) - t.rr.wrapped.reader = io.NopCloser(r) - t.rr.wrapped.cancel = func() {} - t.rr.wrapped.start = 0 - t.rr.wrapped.limit = existingSize - // Create readers for each chunk. - chunk1Reader := strings.NewReader(strings.Repeat("x", chunkSize)) - chunk1RC := io.NopCloser(chunk1Reader) - chunk2Reader := strings.NewReader(strings.Repeat("x", chunkSize)) - chunk2RC := io.NopCloser(chunk2Reader) - chunk3Reader := strings.NewReader(strings.Repeat("x", chunkSize)) - chunk3RC := io.NopCloser(chunk3Reader) - // Mock the NewReader calls to return chunkReaders created above. - // We will make 3 GCS calls to satisfy the requested read size. But since we - // already have a reader with 'existingSize' data, we will first read that data - // and then make GCS calls. So call sequence is - // [0, existingSize) -> existing reader - // [existingSize, existingSize+chunkSize) -> newReader - // [existingSize+chunkSize, existingSize+chunkSize*2) -> newReader - // [existingSize+chunkSize*2, existingSize+chunkSize*3) -> newReader - ExpectCall(t.bucket, "NewReader")( - Any(), - AllOf(rangeStartIs(existingSize), rangeLimitIs(existingSize+chunkSize))). - WillOnce(Return(chunk1RC, nil)) - ExpectCall(t.bucket, "NewReader")( - Any(), - AllOf(rangeStartIs(existingSize+chunkSize), rangeLimitIs(existingSize+chunkSize*2))). - WillOnce(Return(chunk2RC, nil)) - ExpectCall(t.bucket, "NewReader")( - Any(), - AllOf(rangeStartIs(existingSize+chunkSize*2), rangeLimitIs(existingSize+chunkSize*3))). - WillOnce(Return(chunk3RC, nil)) - - // Call through. - buf := make([]byte, readSize) - _, cacheHit, err := t.rr.ReadAt(buf, 0) - - // Check the state now. - ExpectFalse(cacheHit) - ExpectEq(nil, err) - // Start is the total data read. - ExpectEq(readSize, t.rr.wrapped.start) - // Limit is same as the byteRange of last GCS call made. - ExpectEq(existingSize+readSize, t.rr.wrapped.limit) -} - /******************* File cache specific tests ***********************/ func (t *RandomReaderTest) Test_ReadAt_SequentialFullObject() { t.rr.wrapped.fileCacheHandler = t.cacheHandler objectSize := t.object.Size testContent := testutil.GenerateRandomBytes(int(objectSize)) - rc := getReadCloser(testContent) - t.mockNewReaderCallForTestBucket(0, objectSize, rc) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(0, objectSize, rd) ExpectCall(t.bucket, "Name")().WillRepeatedly(Return("test")) + ExpectCall(t.bucket, "BucketType")().WillRepeatedly(Return(t.bucketType)) buf := make([]byte, objectSize) - _, cacheHit, err := t.rr.ReadAt(buf, 0) - ExpectFalse(cacheHit) + objectData, err := t.rr.ReadAt(buf, 0) + ExpectFalse(objectData.CacheHit) ExpectEq(nil, err) ExpectTrue(reflect.DeepEqual(testContent, buf)) - _, cacheHit, err = t.rr.ReadAt(buf, 0) + objectData, err = t.rr.ReadAt(buf, 0) - ExpectTrue(cacheHit) + ExpectTrue(objectData.CacheHit) ExpectEq(nil, err) ExpectTrue(reflect.DeepEqual(testContent, buf)) } @@ -747,17 +610,18 @@ func (t *RandomReaderTest) Test_ReadAt_SequentialRangeRead() { t.rr.wrapped.fileCacheHandler = t.cacheHandler objectSize := t.object.Size testContent := testutil.GenerateRandomBytes(int(objectSize)) - rc := getReadCloser(testContent) - t.mockNewReaderCallForTestBucket(0, objectSize, rc) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(0, objectSize, rd) ExpectCall(t.bucket, "Name")().WillRepeatedly(Return("test")) + ExpectCall(t.bucket, "BucketType")().WillOnce(Return(t.bucketType)) start := 0 end := 10 // not included AssertLt(end, objectSize) buf := make([]byte, end-start) - _, cacheHit, err := t.rr.ReadAt(buf, int64(start)) + objectData, err := t.rr.ReadAt(buf, int64(start)) - ExpectFalse(cacheHit) + ExpectFalse(objectData.CacheHit) ExpectEq(nil, err) ExpectTrue(reflect.DeepEqual(testContent[start:end], buf)) } @@ -767,16 +631,17 @@ func (t *RandomReaderTest) Test_ReadAt_SequentialSubsequentReadOffsetLessThanRea t.object.Size = 20 * util.MiB objectSize := t.object.Size testContent := testutil.GenerateRandomBytes(int(objectSize)) - rc := getReadCloser(testContent) - t.mockNewReaderCallForTestBucket(0, objectSize, rc) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(0, objectSize, rd) ExpectCall(t.bucket, "Name")().WillRepeatedly(Return("test")) + ExpectCall(t.bucket, "BucketType")().WillRepeatedly(Return(t.bucketType)) start1 := 0 end1 := util.MiB // not included AssertLt(end1, objectSize) // First call from offset 0 - sequential read buf := make([]byte, end1-start1) - _, cacheHit, err := t.rr.ReadAt(buf, int64(start1)) - ExpectFalse(cacheHit) + objectData, err := t.rr.ReadAt(buf, int64(start1)) + ExpectFalse(objectData.CacheHit) ExpectEq(nil, err) ExpectTrue(reflect.DeepEqual(testContent[start1:end1], buf)) start2 := 3*util.MiB + 4 @@ -784,9 +649,9 @@ func (t *RandomReaderTest) Test_ReadAt_SequentialSubsequentReadOffsetLessThanRea buf2 := make([]byte, end2-start2) // Assuming start2 offset download in progress - _, cacheHit, err = t.rr.ReadAt(buf2, int64(start2)) + objectData, err = t.rr.ReadAt(buf2, int64(start2)) - ExpectTrue(cacheHit) + ExpectTrue(objectData.CacheHit) ExpectEq(nil, err) ExpectTrue(reflect.DeepEqual(testContent[start2:end2], buf2)) } @@ -798,12 +663,13 @@ func (t *RandomReaderTest) Test_ReadAt_RandomReadNotStartWithZeroOffsetWhenCache testContent := testutil.GenerateRandomBytes(int(objectSize)) start := 5 end := 10 // not included - rc := getReadCloser(testContent[start:]) - t.mockNewReaderCallForTestBucket(uint64(start), objectSize, rc) + rc := &fake.FakeReader{ReadCloser: getReadCloser(testContent[start:])} + t.mockNewReaderWithHandleCallForTestBucket(uint64(start), objectSize, rc) ExpectCall(t.bucket, "Name")().WillRepeatedly(Return("test")) + ExpectCall(t.bucket, "BucketType")().WillRepeatedly(Return(t.bucketType)) buf := make([]byte, end-start) - _, cacheHit, err := t.rr.ReadAt(buf, int64(start)) - ExpectFalse(cacheHit) + objectData, err := t.rr.ReadAt(buf, int64(start)) + ExpectFalse(objectData.CacheHit) ExpectEq(nil, err) ExpectTrue(reflect.DeepEqual(testContent[start:end], buf)) job := t.jobManager.CreateJobIfNotExists(t.object, t.bucket) @@ -811,10 +677,10 @@ func (t *RandomReaderTest) Test_ReadAt_RandomReadNotStartWithZeroOffsetWhenCache ExpectTrue(jobStatus.Name == downloader.NotStarted) // Second read call should be a cache miss - _, cacheHit, err = t.rr.ReadAt(buf, int64(start)) + objectData, err = t.rr.ReadAt(buf, int64(start)) ExpectEq(nil, err) - ExpectFalse(cacheHit) + ExpectFalse(objectData.CacheHit) } func (t *RandomReaderTest) Test_ReadAt_RandomReadNotStartWithZeroOffsetWhenCacheForRangeReadIsTrue() { @@ -824,16 +690,19 @@ func (t *RandomReaderTest) Test_ReadAt_RandomReadNotStartWithZeroOffsetWhenCache testContent := testutil.GenerateRandomBytes(int(objectSize)) start := 5 end := 10 // not included - rc := getReadCloser(testContent[start:]) - t.mockNewReaderCallForTestBucket(uint64(start), objectSize, rc) // Mock for random-reader's NewReader call - rc2 := getReadCloser(testContent) - t.mockNewReaderCallForTestBucket(0, objectSize, rc2) // Mock for download job's NewReader call + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent[start:])} + // Mock for random-reader's NewReader call + t.mockNewReaderWithHandleCallForTestBucket(uint64(start), objectSize, rd) + rd1 := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + // Mock for download job's NewReader call + t.mockNewReaderWithHandleCallForTestBucket(0, objectSize, rd1) ExpectCall(t.bucket, "Name")().WillRepeatedly(Return("test")) + ExpectCall(t.bucket, "BucketType")().WillRepeatedly(Return(t.bucketType)) buf := make([]byte, end-start) - _, cacheHit, err := t.rr.ReadAt(buf, int64(start)) + objectData, err := t.rr.ReadAt(buf, int64(start)) - ExpectFalse(cacheHit) + ExpectFalse(objectData.CacheHit) ExpectEq(nil, err) ExpectTrue(reflect.DeepEqual(testContent[start:end], buf)) job := t.jobManager.GetJob(t.object.Name, t.bucket.Name()) @@ -845,28 +714,31 @@ func (t *RandomReaderTest) Test_ReadAt_SequentialToRandomSubsequentReadOffsetMor t.object.Size = 20 * util.MiB objectSize := t.object.Size testContent := testutil.GenerateRandomBytes(int(objectSize)) - rc := getReadCloser(testContent) - t.mockNewReaderCallForTestBucket(0, objectSize, rc) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + // Mock for download job's NewReader call + t.mockNewReaderWithHandleCallForTestBucket(0, objectSize, rd) ExpectCall(t.bucket, "Name")().WillRepeatedly(Return("test")) + ExpectCall(t.bucket, "BucketType")().WillRepeatedly(Return(t.bucketType)) start1 := 0 end1 := util.MiB // not included AssertLt(end1, objectSize) // First call from offset 0 - sequential read buf := make([]byte, end1-start1) - _, cacheHit, err := t.rr.ReadAt(buf, int64(start1)) - ExpectFalse(cacheHit) + objectData, err := t.rr.ReadAt(buf, int64(start1)) + ExpectFalse(objectData.CacheHit) ExpectEq(nil, err) ExpectTrue(reflect.DeepEqual(testContent[start1:end1], buf)) start2 := 16*util.MiB + 4 end2 := start2 + util.MiB - rc2 := getReadCloser(testContent[start2:]) - t.mockNewReaderCallForTestBucket(uint64(start2), objectSize, rc2) + rd2 := &fake.FakeReader{ReadCloser: getReadCloser(testContent[start2:])} + // Mock for random-reader's NewReader call + t.mockNewReaderWithHandleCallForTestBucket(uint64(start2), objectSize, rd2) buf2 := make([]byte, end2-start2) // Assuming start2 offset download in progress - _, cacheHit, err = t.rr.ReadAt(buf2, int64(start2)) + objectData, err = t.rr.ReadAt(buf2, int64(start2)) - ExpectFalse(cacheHit) + ExpectFalse(objectData.CacheHit) ExpectEq(nil, err) ExpectTrue(reflect.DeepEqual(testContent[start2:end2], buf2)) } @@ -876,36 +748,37 @@ func (t *RandomReaderTest) Test_ReadAt_SequentialToRandomSubsequentReadOffsetLes t.object.Size = 20 * util.MiB objectSize := t.object.Size testContent := testutil.GenerateRandomBytes(int(objectSize)) - rc := getReadCloser(testContent) - t.mockNewReaderCallForTestBucket(0, objectSize, rc) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(0, objectSize, rd) ExpectCall(t.bucket, "Name")().WillRepeatedly(Return("test")) + ExpectCall(t.bucket, "BucketType")().WillRepeatedly(Return(t.bucketType)) start1 := 0 end1 := util.MiB // not included AssertLt(end1, objectSize) // First call from offset 0 - sequential read buf := make([]byte, end1-start1) - _, cacheHit, err := t.rr.ReadAt(buf, int64(start1)) - ExpectFalse(cacheHit) + objectData, err := t.rr.ReadAt(buf, int64(start1)) + ExpectFalse(objectData.CacheHit) ExpectEq(nil, err) ExpectTrue(reflect.DeepEqual(testContent[start1:end1], buf)) start2 := 16*util.MiB + 4 end2 := start2 + util.MiB - rc2 := getReadCloser(testContent[start2:]) - t.mockNewReaderCallForTestBucket(uint64(start2), objectSize, rc2) + rc2 := &fake.FakeReader{ReadCloser: getReadCloser(testContent[start2:])} + t.mockNewReaderWithHandleCallForTestBucket(uint64(start2), objectSize, rc2) buf2 := make([]byte, end2-start2) // Assuming start2 offset download in progress - _, cacheHit, err = t.rr.ReadAt(buf2, int64(start2)) - ExpectFalse(cacheHit) + objectData, err = t.rr.ReadAt(buf2, int64(start2)) + ExpectFalse(objectData.CacheHit) ExpectEq(nil, err) ExpectTrue(reflect.DeepEqual(testContent[start2:end2], buf2)) start3 := util.MiB end3 := start3 + util.MiB buf3 := make([]byte, end3-start3) - _, cacheHit, err = t.rr.ReadAt(buf3, int64(start3)) + objectData, err = t.rr.ReadAt(buf3, int64(start3)) ExpectEq(nil, err) - ExpectTrue(cacheHit) + ExpectTrue(objectData.CacheHit) ExpectTrue(reflect.DeepEqual(testContent[start3:end3], buf3)) } @@ -913,30 +786,32 @@ func (t *RandomReaderTest) Test_ReadAt_CacheMissDueToInvalidJob() { t.rr.wrapped.fileCacheHandler = t.cacheHandler objectSize := t.object.Size testContent := testutil.GenerateRandomBytes(int(objectSize)) - rc1 := getReadCloser(testContent) - t.mockNewReaderCallForTestBucket(0, objectSize, rc1) + rc1 := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(0, objectSize, rc1) ExpectCall(t.bucket, "Name")().WillRepeatedly(Return("test")) + ExpectCall(t.bucket, "BucketType")().WillRepeatedly(Return(t.bucketType)) buf := make([]byte, objectSize) - _, cacheHit, err := t.rr.ReadAt(buf, 0) + objectData, err := t.rr.ReadAt(buf, 0) AssertEq(nil, err) - ExpectFalse(cacheHit) + ExpectFalse(objectData.CacheHit) AssertTrue(reflect.DeepEqual(testContent, buf)) job := t.jobManager.GetJob(t.object.Name, t.bucket.Name()) if job != nil { jobStatus := job.GetStatus().Name AssertTrue(jobStatus == downloader.Downloading || jobStatus == downloader.Completed, fmt.Sprintf("the actual status is %v", jobStatus)) } + err = t.rr.wrapped.fileCacheHandler.InvalidateCache(t.object.Name, t.bucket.Name()) AssertEq(nil, err) // Second reader (rc2) is required, since first reader (rc) is completely read. // Reading again will return EOF. - rc2 := getReadCloser(testContent) - t.mockNewReaderCallForTestBucket(0, objectSize, rc2) + rc2 := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(0, objectSize, rc2) - _, cacheHit, err = t.rr.ReadAt(buf, 0) + objectData, err = t.rr.ReadAt(buf, 0) ExpectEq(nil, err) - ExpectFalse(cacheHit) + ExpectFalse(objectData.CacheHit) ExpectTrue(reflect.DeepEqual(testContent, buf)) ExpectEq(nil, t.rr.wrapped.fileCacheHandle) } @@ -945,13 +820,14 @@ func (t *RandomReaderTest) Test_ReadAt_CachePopulatedAndThenCacheMissDueToInvali t.rr.wrapped.fileCacheHandler = t.cacheHandler objectSize := t.object.Size testContent := testutil.GenerateRandomBytes(int(objectSize)) - rc1 := getReadCloser(testContent) - t.mockNewReaderCallForTestBucket(0, objectSize, rc1) + rd1 := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(0, objectSize, rd1) ExpectCall(t.bucket, "Name")().WillRepeatedly(Return("test")) + ExpectCall(t.bucket, "BucketType")().WillRepeatedly(Return(t.bucketType)) buf := make([]byte, objectSize) - _, cacheHit, err := t.rr.ReadAt(buf, 0) + objectData, err := t.rr.ReadAt(buf, 0) AssertEq(nil, err) - AssertFalse(cacheHit) + AssertFalse(objectData.CacheHit) AssertTrue(reflect.DeepEqual(testContent, buf)) job := t.jobManager.GetJob(t.object.Name, t.bucket.Name()) if job != nil { @@ -962,20 +838,20 @@ func (t *RandomReaderTest) Test_ReadAt_CachePopulatedAndThenCacheMissDueToInvali AssertEq(nil, err) // Second reader (rc2) is required, since first reader (rc) is completely read. // Reading again will return EOF. - rc2 := getReadCloser(testContent) - t.mockNewReaderCallForTestBucket(0, objectSize, rc2) - _, cacheHit, err = t.rr.ReadAt(buf, 0) + rc2 := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(0, objectSize, rc2) + objectData, err = t.rr.ReadAt(buf, 0) ExpectEq(nil, err) - ExpectFalse(cacheHit) + ExpectFalse(objectData.CacheHit) ExpectTrue(reflect.DeepEqual(testContent, buf)) ExpectEq(nil, t.rr.wrapped.fileCacheHandle) - rc3 := getReadCloser(testContent) - t.mockNewReaderCallForTestBucket(0, objectSize, rc3) + rd3 := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(0, objectSize, rd3) - _, cacheHit, err = t.rr.ReadAt(buf, 0) + objectData, err = t.rr.ReadAt(buf, 0) ExpectEq(nil, err) - ExpectFalse(cacheHit) + ExpectFalse(objectData.CacheHit) ExpectTrue(reflect.DeepEqual(testContent, buf)) ExpectNe(nil, t.rr.wrapped.fileCacheHandle) } @@ -984,33 +860,34 @@ func (t *RandomReaderTest) Test_ReadAt_CachePopulatedAndThenCacheMissDueToInvali t.rr.wrapped.fileCacheHandler = t.cacheHandler objectSize := t.object.Size testContent := testutil.GenerateRandomBytes(int(objectSize)) - rc1 := getReadCloser(testContent) - t.mockNewReaderCallForTestBucket(0, objectSize, rc1) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(0, objectSize, rd) ExpectCall(t.bucket, "Name")().WillRepeatedly(Return("test")) + ExpectCall(t.bucket, "BucketType")().WillRepeatedly(Return(t.bucketType)) buf := make([]byte, objectSize) - _, cacheHit, err := t.rr.ReadAt(buf, 0) + objectData, err := t.rr.ReadAt(buf, 0) AssertEq(nil, err) - AssertFalse(cacheHit) + AssertFalse(objectData.CacheHit) AssertTrue(reflect.DeepEqual(testContent, buf)) AssertNe(nil, t.rr.wrapped.fileCacheHandle) err = t.rr.wrapped.fileCacheHandle.Close() AssertEq(nil, err) // Second reader (rc2) is required, since first reader (rc) is completely read. // Reading again will return EOF. - rc2 := getReadCloser(testContent) - t.mockNewReaderCallForTestBucket(0, objectSize, rc2) - _, cacheHit, err = t.rr.ReadAt(buf, 0) + rc2 := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(0, objectSize, rc2) + objectData, err = t.rr.ReadAt(buf, 0) ExpectEq(nil, err) - ExpectFalse(cacheHit) + ExpectFalse(objectData.CacheHit) ExpectTrue(reflect.DeepEqual(testContent, buf)) ExpectEq(nil, t.rr.wrapped.fileCacheHandle) - rc3 := getReadCloser(testContent) - t.mockNewReaderCallForTestBucket(0, objectSize, rc3) + rc3 := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(0, objectSize, rc3) - _, cacheHit, err = t.rr.ReadAt(buf, 0) + objectData, err = t.rr.ReadAt(buf, 0) ExpectEq(nil, err) - ExpectTrue(cacheHit) + ExpectTrue(objectData.CacheHit) ExpectTrue(reflect.DeepEqual(testContent, buf)) ExpectNe(nil, t.rr.wrapped.fileCacheHandle) } @@ -1019,13 +896,14 @@ func (t *RandomReaderTest) Test_ReadAt_IfCacheFileGetsDeleted() { t.rr.wrapped.fileCacheHandler = t.cacheHandler objectSize := t.object.Size testContent := testutil.GenerateRandomBytes(int(objectSize)) - rc1 := getReadCloser(testContent) - t.mockNewReaderCallForTestBucket(0, objectSize, rc1) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(0, objectSize, rd) ExpectCall(t.bucket, "Name")().WillRepeatedly(Return("test")) + ExpectCall(t.bucket, "BucketType")().WillOnce(Return(t.bucketType)) buf := make([]byte, objectSize) - _, cacheHit, err := t.rr.ReadAt(buf, 0) + objectData, err := t.rr.ReadAt(buf, 0) AssertEq(nil, err) - AssertFalse(cacheHit) + AssertFalse(objectData.CacheHit) AssertTrue(reflect.DeepEqual(testContent, buf)) AssertNe(nil, t.rr.wrapped.fileCacheHandle) err = t.rr.wrapped.fileCacheHandle.Close() @@ -1035,28 +913,29 @@ func (t *RandomReaderTest) Test_ReadAt_IfCacheFileGetsDeleted() { filePath := util.GetDownloadPath(t.cacheDir, util.GetObjectPath(t.bucket.Name(), t.object.Name)) err = os.Remove(filePath) AssertEq(nil, err) - // Second reader (rc2) is required, since first reader (rc) is completely read. + // Second reader (rd2) is required, since first reader (rd) is completely read. // Reading again will return EOF. - rc2 := getReadCloser(testContent) - t.mockNewReaderCallForTestBucket(0, objectSize, rc2) + rd2 := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(0, objectSize, rd2) - _, _, err = t.rr.ReadAt(buf, 0) + _, err = t.rr.ReadAt(buf, 0) AssertNe(nil, err) - ExpectTrue(strings.Contains(err.Error(), util.FileNotPresentInCacheErrMsg)) + AssertTrue(errors.Is(err, util.ErrFileNotPresentInCache)) } func (t *RandomReaderTest) Test_ReadAt_IfCacheFileGetsDeletedWithCacheHandleOpen() { t.rr.wrapped.fileCacheHandler = t.cacheHandler objectSize := t.object.Size testContent := testutil.GenerateRandomBytes(int(objectSize)) - rc1 := getReadCloser(testContent) - t.mockNewReaderCallForTestBucket(0, objectSize, rc1) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(0, objectSize, rd) ExpectCall(t.bucket, "Name")().WillRepeatedly(Return("test")) + ExpectCall(t.bucket, "BucketType")().WillRepeatedly(Return(t.bucketType)) buf := make([]byte, objectSize) - _, cacheHit, err := t.rr.ReadAt(buf, 0) + objectData, err := t.rr.ReadAt(buf, 0) AssertEq(nil, err) - AssertFalse(cacheHit) + AssertFalse(objectData.CacheHit) AssertTrue(reflect.DeepEqual(testContent, buf)) AssertNe(nil, t.rr.wrapped.fileCacheHandle) // Delete the local cache file. @@ -1066,10 +945,10 @@ func (t *RandomReaderTest) Test_ReadAt_IfCacheFileGetsDeletedWithCacheHandleOpen // Read via cache only, as we have old fileHandle open and linux // doesn't delete the file until the fileHandle count for the file is zero. - _, cacheHit, err = t.rr.ReadAt(buf, 0) + objectData, err = t.rr.ReadAt(buf, 0) AssertEq(nil, err) - ExpectTrue(cacheHit) + ExpectTrue(objectData.CacheHit) ExpectTrue(reflect.DeepEqual(testContent, buf)) ExpectNe(nil, t.rr.wrapped.fileCacheHandle) } @@ -1078,35 +957,37 @@ func (t *RandomReaderTest) Test_ReadAt_FailedJobRestartAndCacheHit() { t.rr.wrapped.fileCacheHandler = t.cacheHandler objectSize := t.object.Size testContent := testutil.GenerateRandomBytes(int(objectSize)) - rc1 := getReadCloser(testContent) - // First NewReader-call throws error, hence async job fails. + rc := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + // First call goes to file cache succeeded by next call to random reader. + // First NewReaderWithReadHandle-call throws error, hence async job fails. // Later NewReader-call returns a valid readCloser object hence fallback to // GCS read will succeed. - ExpectCall(t.bucket, "NewReader")(Any(), Any()). - WillOnce(Return(nil, errors.New(""))).WillRepeatedly(Return(rc1, nil)) + ExpectCall(t.bucket, "NewReaderWithReadHandle")(Any(), Any()). + WillOnce(Return(nil, errors.New(""))).WillRepeatedly(Return(rc, nil)) ExpectCall(t.bucket, "Name")().WillRepeatedly(Return("test")) + ExpectCall(t.bucket, "BucketType")().WillRepeatedly(Return(t.bucketType)) buf := make([]byte, objectSize) - _, cacheHit, err := t.rr.ReadAt(buf, 0) + objectData, err := t.rr.ReadAt(buf, 0) AssertEq(nil, err) - ExpectFalse(cacheHit) + ExpectFalse(objectData.CacheHit) AssertTrue(reflect.DeepEqual(testContent, buf)) job := t.jobManager.GetJob(t.object.Name, t.bucket.Name()) AssertTrue(job == nil || job.GetStatus().Name == downloader.Failed) // Second reader (rc2) is required, since first reader (rc) is completely read. // Reading again will return EOF. - rc2 := getReadCloser(testContent) - t.mockNewReaderCallForTestBucket(0, objectSize, rc2) + rd2 := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(0, objectSize, rd2) // This call will populate the cache again. - _, cacheHit, err = t.rr.ReadAt(buf, 0) + objectData, err = t.rr.ReadAt(buf, 0) ExpectEq(nil, err) - ExpectFalse(cacheHit) + ExpectFalse(objectData.CacheHit) ExpectTrue(reflect.DeepEqual(testContent, buf)) ExpectNe(nil, t.rr.wrapped.fileCacheHandle) - _, cacheHit, err = t.rr.ReadAt(buf, 0) + objectData, err = t.rr.ReadAt(buf, 0) ExpectEq(nil, err) - ExpectTrue(cacheHit) + ExpectTrue(objectData.CacheHit) ExpectTrue(reflect.DeepEqual(testContent, buf)) ExpectNe(nil, t.rr.wrapped.fileCacheHandle) } @@ -1117,9 +998,10 @@ func (t *RandomReaderTest) Test_tryReadingFromFileCache_CacheHit() { t.rr.wrapped.fileCacheHandler = t.cacheHandler objectSize := t.object.Size testContent := testutil.GenerateRandomBytes(int(objectSize)) - rc := getReadCloser(testContent) - t.mockNewReaderCallForTestBucket(0, objectSize, rc) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(0, objectSize, rd) ExpectCall(t.bucket, "Name")().WillRepeatedly(Return("test")) + ExpectCall(t.bucket, "BucketType")().WillRepeatedly(Return(t.bucketType)) buf := make([]byte, objectSize) // First read will be a cache miss. _, cacheHit, err := t.rr.wrapped.tryReadingFromFileCache(t.rr.ctx, buf, 0) @@ -1153,15 +1035,16 @@ func (t *RandomReaderTest) Test_ReadAt_OffsetEqualToObjectSize() { t.object.Size = util.MiB objectSize := t.object.Size testContent := testutil.GenerateRandomBytes(int(objectSize)) - rc := getReadCloser(testContent) - t.mockNewReaderCallForTestBucket(0, objectSize, rc) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(0, objectSize, rd) ExpectCall(t.bucket, "Name")().WillRepeatedly(Return("test")) + ExpectCall(t.bucket, "BucketType")().WillOnce(Return(t.bucketType)) start1 := 0 end1 := util.MiB // equal to objectSize // First call from offset 0 - objectSize buf := make([]byte, end1-start1) - _, cacheHit, err := t.rr.ReadAt(buf, int64(start1)) - ExpectFalse(cacheHit) + objectData, err := t.rr.ReadAt(buf, int64(start1)) + ExpectFalse(objectData.CacheHit) ExpectEq(nil, err) ExpectTrue(reflect.DeepEqual(testContent[start1:end1], buf)) start2 := util.MiB // offset equal to objectSize @@ -1169,12 +1052,12 @@ func (t *RandomReaderTest) Test_ReadAt_OffsetEqualToObjectSize() { buf2 := make([]byte, end2-start2) // read for offset equal to objectSize - n, cacheHit, err := t.rr.ReadAt(buf2, int64(start2)) + objectData, err = t.rr.ReadAt(buf2, int64(start2)) // nothing should be read - ExpectFalse(cacheHit) + ExpectFalse(objectData.CacheHit) ExpectEq(io.EOF, err) - ExpectEq(0, n) + ExpectEq(0, objectData.Size) } func (t *RandomReaderTest) Test_Destroy_NilCacheHandle() { @@ -1189,9 +1072,10 @@ func (t *RandomReaderTest) Test_Destroy_NonNilCacheHandle() { t.rr.wrapped.fileCacheHandler = t.cacheHandler objectSize := t.object.Size testContent := testutil.GenerateRandomBytes(int(objectSize)) - rc := getReadCloser(testContent) - t.mockNewReaderCallForTestBucket(0, objectSize, rc) + rd := &fake.FakeReader{ReadCloser: getReadCloser(testContent)} + t.mockNewReaderWithHandleCallForTestBucket(0, objectSize, rd) ExpectCall(t.bucket, "Name")().WillRepeatedly(Return("test")) + ExpectCall(t.bucket, "BucketType")().WillOnce(Return(t.bucketType)) buf := make([]byte, objectSize) _, cacheHit, err := t.rr.wrapped.tryReadingFromFileCache(t.rr.ctx, buf, 0) AssertFalse(cacheHit) @@ -1204,6 +1088,19 @@ func (t *RandomReaderTest) Test_Destroy_NonNilCacheHandle() { ExpectEq(nil, t.rr.wrapped.fileCacheHandle) } +func (t *RandomReaderTest) TestNewReader_FileClobbered() { + var notFoundError *gcs.NotFoundError + + ExpectCall(t.bucket, "NewReaderWithReadHandle")(Any(), Any()). + WillOnce(Return(nil, notFoundError)) + + err := t.rr.wrapped.startRead(context.Background(), 0, 1, 0) + + AssertNe(nil, err) + var clobberedErr *gcsfuse_errors.FileClobberedError + AssertTrue(errors.As(err, &clobberedErr)) +} + // TODO (raj-prince) - to add unit tests for failed scenario while reading via cache. // This requires mocking CacheHandle object, whose read method will return some unexpected // error. diff --git a/internal/gcsx/read_manager/mock_read_manager.go b/internal/gcsx/read_manager/mock_read_manager.go new file mode 100644 index 0000000000..6696e6a589 --- /dev/null +++ b/internal/gcsx/read_manager/mock_read_manager.go @@ -0,0 +1,50 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package read_manager + +import ( + "context" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/stretchr/testify/mock" +) + +type MockReadManager struct { + gcsx.ReadManager + mock.Mock +} + +func (m *MockReadManager) ReaderName() string { + return "mock_read_manager" +} + +func (m *MockReadManager) ReadAt(ctx context.Context, req *gcsx.ReadRequest) (gcsx.ReadResponse, error) { + args := m.Called(ctx, req) + return args.Get(0).(gcsx.ReadResponse), args.Error(1) +} + +func (m *MockReadManager) Object() *gcs.MinObject { + args := m.Called() + return args.Get(0).(*gcs.MinObject) +} + +func (m *MockReadManager) Destroy() { + m.Called() +} + +func (m *MockReadManager) CheckInvariants() { + m.Called() +} diff --git a/internal/gcsx/read_manager/read_manager.go b/internal/gcsx/read_manager/read_manager.go new file mode 100644 index 0000000000..323ee5d8fd --- /dev/null +++ b/internal/gcsx/read_manager/read_manager.go @@ -0,0 +1,219 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package read_manager + +import ( + "context" + "errors" + "io" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/jacobsa/fuse/fuseops" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/bufferedread" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/file" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + clientReaders "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx/client_readers" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/workerpool" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "golang.org/x/sync/semaphore" +) + +type ReadManager struct { + gcsx.ReadManager + object *gcs.MinObject + + // readers holds a list of data readers, prioritized for reading. + // e.g., File cache reader, GCS reader. + readers []gcsx.Reader + + // readTypeClassifier tracks the read access pattern (e.g., sequential, random) + // across all readers for a file handle to optimize read strategies. + readTypeClassifier *gcsx.ReadTypeClassifier + + traceHandle tracing.TraceHandle +} + +// ReadManagerConfig holds the configuration parameters for creating a new ReadManager. +type ReadManagerConfig struct { + SequentialReadSizeMB int32 + // Exactly one of these should be set: + FileCacheHandler *file.CacheHandler + SharedChunkCacheManager *file.SharedChunkCacheManager + CacheFileForRangeRead bool + MetricHandle metrics.MetricHandle + TraceHandle tracing.TraceHandle + MrdWrapper *gcsx.MultiRangeDownloaderWrapper + Config *cfg.Config + GlobalMaxBlocksSem *semaphore.Weighted + WorkerPool workerpool.WorkerPool + HandleID fuseops.HandleID + InitialOffset int64 +} + +// NewReadManager creates a new ReadManager for the given GCS object, +// using the provided configuration. It initializes the manager with a +// file cache reader and a GCS reader, prioritizing the file cache reader if available. +func NewReadManager(object *gcs.MinObject, bucket gcs.Bucket, config *ReadManagerConfig) *ReadManager { + // Create a slice to hold all readers. The file cache reader will be added first if it exists. + var readers []gcsx.Reader + + if config.TraceHandle == nil { + config.TraceHandle = tracing.NewNoopTracer() + } + + // If a shared chunk cache handler is provided, use it + if config.SharedChunkCacheManager != nil { + // For SharedChunkCacheManager, create ShareChunkCacheReader directly + reader := gcsx.NewSharedChunkCacheReader( + config.SharedChunkCacheManager, + bucket, + object, + config.MetricHandle, + config.TraceHandle, + config.HandleID, + ) + readers = append(readers, reader) + } else if config.FileCacheHandler != nil { + // For traditional cache handler, use FileCacheReader + fileCacheReader := gcsx.NewFileCacheReader( + object, + bucket, + config.FileCacheHandler, + config.CacheFileForRangeRead, + config.MetricHandle, + config.TraceHandle, + config.HandleID, + ) + readers = append(readers, fileCacheReader) + } + + readClassifier := gcsx.NewReadTypeClassifier(int64(config.SequentialReadSizeMB), config.InitialOffset) + + // If buffered read is enabled, initialize the buffered reader and add it to the readers. + if config.Config.Read.EnableBufferedRead { + readConfig := config.Config.Read + bufferedReadConfig := &bufferedread.BufferedReadConfig{ + MaxPrefetchBlockCnt: readConfig.MaxBlocksPerHandle, + PrefetchBlockSizeBytes: readConfig.BlockSizeMb * util.MiB, + InitialPrefetchBlockCnt: readConfig.StartBlocksPerHandle, + MinBlocksPerHandle: readConfig.MinBlocksPerHandle, + RandomSeekThreshold: readConfig.RandomSeekThreshold, + } + opts := &bufferedread.BufferedReaderOptions{ + Object: object, + Bucket: bucket, + Config: bufferedReadConfig, + GlobalMaxBlocksSem: config.GlobalMaxBlocksSem, + WorkerPool: config.WorkerPool, + MetricHandle: config.MetricHandle, + TraceHandle: config.TraceHandle, + ReadTypeClassifier: readClassifier, + HandleID: config.HandleID, + } + bufferedReader, err := bufferedread.NewBufferedReader(opts) + if err != nil { + logger.Tracef("Failed to create bufferedReader: %v. Buffered reading will be disabled for this file handle.", err) + } else { + readers = append(readers, bufferedReader) + } + } + + // Initialize the GCS reader, which is always present. + gcsReader := clientReaders.NewGCSReader( + object, + bucket, + &clientReaders.GCSReaderConfig{ + MetricHandle: config.MetricHandle, + TraceHandle: config.TraceHandle, + MrdWrapper: config.MrdWrapper, + Config: config.Config, + ReadTypeClassifier: readClassifier, + }, + ) + // Add the GCS reader as a fallback. + readers = append(readers, gcsReader) + + return &ReadManager{ + object: object, + readers: readers, // Readers are prioritized: file cache first, then GCS. + readTypeClassifier: readClassifier, + traceHandle: config.TraceHandle, + } +} + +func (rr *ReadManager) ReaderName() string { + return "read_manager" +} + +func (rr *ReadManager) Object() *gcs.MinObject { + return rr.object +} + +func (rr *ReadManager) CheckInvariants() { + for _, r := range rr.readers { + r.CheckInvariants() + } +} + +// ReadAt attempts to read data from the provided offset, using the configured readers. +// It prioritizes readers in the order they are defined (file cache first, then GCS). +// If a reader returns a FallbackToAnotherReader error, it tries the next reader. +func (rr *ReadManager) ReadAt(ctx context.Context, req *gcsx.ReadRequest) (gcsx.ReadResponse, error) { + var readResponse gcsx.ReadResponse + if req.Offset >= int64(rr.object.Size) && !req.SkipSizeChecks { + return readResponse, io.EOF + } + + // empty read + if len(req.Buffer) == 0 { + return readResponse, nil + } + + // Get read-related information (e.g., read type) and add it to the read request. + // This information is used by underlying readers to optimize read strategies + // based on the access pattern. + req.ReadInfo = rr.readTypeClassifier.GetReadInfo(req.Offset, false) + + var err error + for _, r := range rr.readers { + ctx, span := rr.traceHandle.StartSpan(ctx, r.ReaderName()) + readResponse, err = r.ReadAt(ctx, req) + rr.traceHandle.EndSpan(span) + if err == nil { + rr.readTypeClassifier.RecordRead(req.Offset, int64(readResponse.Size)) + return readResponse, nil + } + if !errors.Is(err, gcsx.FallbackToAnotherReader) { + // Non-fallback error, return it. + return readResponse, err + } + // Fallback to the next reader. + } + + // If all readers failed with FallbackToAnotherReader, return the last response and error. + // This case should not happen as the last reader should always succeed. + return readResponse, err +} + +func (rr *ReadManager) Destroy() { + for _, r := range rr.readers { + r.Destroy() + } +} diff --git a/internal/gcsx/read_manager/read_manager_test.go b/internal/gcsx/read_manager/read_manager_test.go new file mode 100644 index 0000000000..c585480930 --- /dev/null +++ b/internal/gcsx/read_manager/read_manager_test.go @@ -0,0 +1,390 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package read_manager + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "path" + "strings" + "testing" + "testing/iotest" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/bufferedread" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/file" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/file/downloader" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/gcsfuse_errors" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + clientReaders "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx/client_readers" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + testUtil "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util/diskutil" + "github.com/googlecloudplatform/gcsfuse/v3/internal/workerpool" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + "golang.org/x/sync/semaphore" +) + +const ( + MiB = 1024 * 1024 + sequentialReadSizeInMb = 22 + cacheMaxSize = 2 * sequentialReadSizeInMb * MiB +) + +//////////////////////////////////////////////////////////////////////// +// Helpers +//////////////////////////////////////////////////////////////////////// + +func (t *readManagerTest) readManagerConfig(fileCacheEnable bool, bufferedReadEnable bool) *ReadManagerConfig { + config := &ReadManagerConfig{ + SequentialReadSizeMB: sequentialReadSizeInMb, + CacheFileForRangeRead: false, + MetricHandle: metrics.NewNoopMetrics(), + TraceHandle: tracing.NewNoopTracer(), + MrdWrapper: nil, + Config: &cfg.Config{ + Read: cfg.ReadConfig{ + EnableBufferedRead: bufferedReadEnable, + MaxBlocksPerHandle: 10, + BlockSizeMb: 1, + StartBlocksPerHandle: 2, + MinBlocksPerHandle: 2, + }, + }, + GlobalMaxBlocksSem: semaphore.NewWeighted(20), + InitialOffset: 0, + } + if bufferedReadEnable { + t.workerPool, _ = workerpool.NewStaticWorkerPool(5, 20, 25) + t.workerPool.Start() + config.WorkerPool = t.workerPool + } + + if fileCacheEnable { + cacheDir := path.Join(os.Getenv("HOME"), "test_cache_dir") + lruCache := lru.NewCache(cacheMaxSize) + fileCacheConfig := &cfg.FileCacheConfig{EnableCrc: false, ExperimentalDisableSizeCalculationFix: true} + cacheDirVolumeBlockSize := diskutil.GetVolumeBlockSize(cacheDir) + jobManager := downloader.NewJobManager(lruCache, util.DefaultFilePerm, util.DefaultDirPerm, cacheDir, sequentialReadSizeInMb, fileCacheConfig, metrics.NewNoopMetrics(), tracing.NewNoopTracer(), cacheDirVolumeBlockSize) + config.FileCacheHandler = file.NewCacheHandler(lruCache, jobManager, cacheDir, util.DefaultFilePerm, util.DefaultDirPerm, "", "", false, cacheDirVolumeBlockSize) + } else { + config.FileCacheHandler = nil + } + return config +} + +func (t *readManagerTest) mockNewReaderWithHandleCallForTestBucket(start uint64, limit uint64, rd gcs.StorageReader) { + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, mock.MatchedBy(func(rg *gcs.ReadObjectRequest) bool { + return rg != nil && (*rg.Range).Start == start && (*rg.Range).Limit == limit + })).Return(rd, nil).Once() +} + +func getReadCloser(content []byte) io.ReadCloser { + r := bytes.NewReader(content) + rc := io.NopCloser(r) + return rc +} + +func (t *readManagerTest) readAt(dst []byte, offset int64) (gcsx.ReadResponse, error) { + t.readManager.CheckInvariants() + defer t.readManager.CheckInvariants() + return t.readManager.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: dst, + Offset: offset, + }) +} + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +type readManagerTest struct { + suite.Suite + object *gcs.MinObject + mockBucket *storage.TestifyMockBucket + readManager *ReadManager + ctx context.Context + bucketType gcs.BucketType + workerPool workerpool.WorkerPool +} + +func TestNonZonalBucketReadManagerTestSuite(t *testing.T) { + suite.Run(t, &readManagerTest{bucketType: gcs.BucketType{}}) +} + +func TestZonalBucketReadManagerTestSuite(t *testing.T) { + suite.Run(t, &readManagerTest{bucketType: gcs.BucketType{Zonal: true, Hierarchical: true}}) +} + +func (t *readManagerTest) SetupTest() { + t.object = &gcs.MinObject{ + Name: "testObject", + Size: 17, + Generation: 1234, + } + t.mockBucket = new(storage.TestifyMockBucket) + t.ctx = context.Background() + t.readManager = NewReadManager(t.object, t.mockBucket, t.readManagerConfig(true, false)) +} + +func (t *readManagerTest) TearDownTest() { + t.readManager.Destroy() + if t.workerPool != nil { + t.workerPool.Stop() + t.workerPool = nil + } +} + +// ////////////////////////////////////////////////////////////////////// +// Tests +// ////////////////////////////////////////////////////////////////////// +func (t *readManagerTest) Test_NewReadManager_WithFileCacheHandlerOnly() { + config := t.readManagerConfig(true, false) + + rm := NewReadManager(t.object, t.mockBucket, config) + + assert.Equal(t.T(), t.object, rm.Object()) + assert.Len(t.T(), rm.readers, 2) + _, ok1 := rm.readers[0].(*gcsx.FileCacheReader) + _, ok2 := rm.readers[1].(*clientReaders.GCSReader) + assert.True(t.T(), ok1, "First reader should be FileCacheReader") + assert.True(t.T(), ok2, "Second reader should be GCSReader") +} + +func (t *readManagerTest) Test_NewReadManager_WithoutFileCacheAndBufferedRead() { + config := t.readManagerConfig(false, false) + + rm := NewReadManager(t.object, t.mockBucket, config) + + assert.Equal(t.T(), t.object, rm.Object()) + assert.Len(t.T(), rm.readers, 1) + _, ok := rm.readers[0].(*clientReaders.GCSReader) + assert.True(t.T(), ok, "Only reader should be GCSReader") +} + +func (t *readManagerTest) Test_NewReadManager_WithBufferedRead() { + config := t.readManagerConfig(false, true) + + rm := NewReadManager(t.object, t.mockBucket, config) + + assert.Equal(t.T(), t.object, rm.Object()) + assert.Len(t.T(), rm.readers, 2) // BufferedReader and GCSReader + _, ok1 := rm.readers[0].(*bufferedread.BufferedReader) + _, ok2 := rm.readers[1].(*clientReaders.GCSReader) + assert.True(t.T(), ok1, "First reader should be BufferedReader") + assert.True(t.T(), ok2, "Second reader should be GCSReader") +} + +func (t *readManagerTest) Test_NewReadManager_WithFileCacheAndBufferedRead() { + config := t.readManagerConfig(true, true) + defer os.RemoveAll(path.Join(os.Getenv("HOME"), "test_cache_dir")) + + rm := NewReadManager(t.object, t.mockBucket, config) + + assert.Equal(t.T(), t.object, rm.Object()) + assert.Len(t.T(), rm.readers, 3) // FileCacheReader, BufferedReader, GCSReader + _, ok1 := rm.readers[0].(*gcsx.FileCacheReader) + _, ok2 := rm.readers[1].(*bufferedread.BufferedReader) + _, ok3 := rm.readers[2].(*clientReaders.GCSReader) + assert.True(t.T(), ok1, "First reader should be FileCacheReader") + assert.True(t.T(), ok2, "Second reader should be BufferedReader") + assert.True(t.T(), ok3, "Third reader should be GCSReader") +} + +func (t *readManagerTest) Test_NewReadManager_BufferedReaderCreationFails() { + config := t.readManagerConfig(false, true) + // Exhaust the semaphore + config.GlobalMaxBlocksSem = semaphore.NewWeighted(0) + + rm := NewReadManager(t.object, t.mockBucket, config) + + assert.Equal(t.T(), t.object, rm.Object()) + assert.Len(t.T(), rm.readers, 1) // Only GCSReader + _, ok := rm.readers[0].(*clientReaders.GCSReader) + assert.True(t.T(), ok, "Only reader should be GCSReader") +} + +func (t *readManagerTest) Test_ReadAt_EmptyRead() { + // Nothing should happen. + readResponse, err := t.readAt(make([]byte, 0), 0) + + assert.NoError(t.T(), err) + assert.Zero(t.T(), readResponse.Size) +} + +func (t *readManagerTest) Test_ReadAt_InvalidOffset() { + tests := []struct { + name string + offset int64 + }{ + { + name: "ReadAtEndOfObject", + offset: int64(t.object.Size), + }, + { + name: "ReadPastEndOfObject", + offset: int64(t.object.Size) + 1, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func() { + readResponse, err := t.readAt(make([]byte, 1), tc.offset) + + assert.Zero(t.T(), readResponse.Size) + assert.True(t.T(), errors.Is(err, io.EOF), "expected %v error got %v", io.EOF, err) + }) + } +} + +func (t *readManagerTest) Test_ReadAt_NoExistingReader() { + // The bucket should be called to set up a new reader. + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, mock.Anything).Return(nil, errors.New("network error")) + t.mockBucket.On("BucketType", mock.Anything).Return(t.bucketType) + t.mockBucket.On("Name").Return("test-bucket") + + _, err := t.readAt(make([]byte, 1), 0) + + assert.Error(t.T(), err) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *readManagerTest) Test_ReadAt_ReaderFailsWithTimeout() { + t.readManager = NewReadManager(t.object, t.mockBucket, t.readManagerConfig(false, false)) + r := iotest.OneByteReader(iotest.TimeoutReader(strings.NewReader("xxx"))) + rc := &fake.FakeReader{ReadCloser: io.NopCloser(r)} + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, mock.Anything).Return(rc, nil).Once() + t.mockBucket.On("BucketType", mock.Anything).Return(t.bucketType).Times(2) + + _, err := t.readAt(make([]byte, 3), 0) + + assert.Error(t.T(), err) + assert.Contains(t.T(), err.Error(), "timeout") + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *readManagerTest) Test_ReadAt_FileClobbered() { + t.mockBucket.On("NewReaderWithReadHandle", mock.Anything, mock.Anything).Return(nil, &gcs.NotFoundError{}) + t.mockBucket.On("BucketType", mock.Anything).Return(t.bucketType).Times(3) + t.mockBucket.On("Name").Return("test-bucket") + + _, err := t.readAt(make([]byte, 3), 0) + + assert.Error(t.T(), err) + var clobberedErr *gcsfuse_errors.FileClobberedError + assert.True(t.T(), errors.As(err, &clobberedErr)) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *readManagerTest) Test_ReadAt_FullObjectFromCache() { + objectSize := int(t.object.Size) + expectedData := testUtil.GenerateRandomBytes(objectSize) + fakeReader := &fake.FakeReader{ + ReadCloser: getReadCloser(expectedData), + } + // Mock the reader that returns full object data + t.mockNewReaderWithHandleCallForTestBucket(0, t.object.Size, fakeReader) + t.mockBucket.On("Name").Return("test-bucket").Maybe() + t.mockBucket.On("BucketType").Return(t.bucketType) + buf := make([]byte, objectSize) + + // Act: First read (expected to be served via GCS, populating the cache) + firstResp, err := t.readAt(buf, 0) + + // Assert: First read succeeds and returns expected data + assert.NoError(t.T(), err, "First read should not return an error") + assert.Equal(t.T(), objectSize, firstResp.Size) + assert.Equal(t.T(), expectedData, buf, "First read should return expected data") + + clear(buf) + + // Act: Second read (should be served from cache) + secondResp, err := t.readAt(buf, 0) + + // Assert: Second read also succeeds and returns the same cached data + assert.NoError(t.T(), err, "Second read (from cache) should not return an error") + assert.Equal(t.T(), objectSize, secondResp.Size) + assert.Equal(t.T(), expectedData, buf, "Second read should return cached data") + // Verify that bucket mock expectations are met + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *readManagerTest) Test_ReadAt_R1FailsR2Succeeds() { + offset := int64(0) + buf := make([]byte, 10) + expectedResp := gcsx.ReadResponse{Size: 10} + mockReader1 := new(gcsx.MockReader) + mockReader2 := new(gcsx.MockReader) + rm := &ReadManager{ + object: t.object, + readers: []gcsx.Reader{mockReader1, mockReader2}, + readTypeClassifier: gcsx.NewReadTypeClassifier(sequentialReadSizeInMb, 0), + traceHandle: tracing.NewNoopTracer(), + } + mockReader1.On("ReadAt", t.ctx, mock.AnythingOfType("*gcsx.ReadRequest")).Return(gcsx.ReadResponse{}, gcsx.FallbackToAnotherReader).Once() + mockReader1.On("Destroy").Once() + mockReader2.On("ReadAt", t.ctx, mock.AnythingOfType("*gcsx.ReadRequest")).Return(expectedResp, nil).Once() + mockReader2.On("Destroy").Once() + + resp, err := rm.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: offset, + }) + rm.Destroy() + + assert.NoError(t.T(), err, "expected no error when second reader succeeds") + assert.Equal(t.T(), expectedResp, resp, "expected response from second reader") + mockReader1.AssertExpectations(t.T()) + mockReader2.AssertExpectations(t.T()) +} + +func (t *readManagerTest) Test_ReadAt_BufferedReaderFallsBack() { + offset := int64(0) + buf := make([]byte, 10) + mockBufferedReader := new(gcsx.MockReader) + mockGCSReader := new(gcsx.MockReader) + rm := &ReadManager{ + object: t.object, + readers: []gcsx.Reader{mockBufferedReader, mockGCSReader}, + readTypeClassifier: gcsx.NewReadTypeClassifier(sequentialReadSizeInMb, 0), + traceHandle: tracing.NewNoopTracer(), + } + mockBufferedReader.On("ReadAt", t.ctx, mock.AnythingOfType("*gcsx.ReadRequest")).Return(gcsx.ReadResponse{}, gcsx.FallbackToAnotherReader).Once() + mockBufferedReader.On("Destroy").Once() + mockGCSReader.On("ReadAt", t.ctx, mock.AnythingOfType("*gcsx.ReadRequest")).Return(gcsx.ReadResponse{Size: 10}, nil).Once() + mockGCSReader.On("Destroy").Once() + + resp, err := rm.ReadAt(t.ctx, &gcsx.ReadRequest{ + Buffer: buf, + Offset: offset, + }) + rm.Destroy() + + assert.NoError(t.T(), err) + assert.Equal(t.T(), gcsx.ReadResponse{Size: 10}, resp) + mockBufferedReader.AssertExpectations(t.T()) + mockGCSReader.AssertExpectations(t.T()) +} diff --git a/internal/gcsx/read_manager/visual_read_manager.go b/internal/gcsx/read_manager/visual_read_manager.go new file mode 100644 index 0000000000..ff205da78f --- /dev/null +++ b/internal/gcsx/read_manager/visual_read_manager.go @@ -0,0 +1,172 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package read_manager + +import ( + "context" + "errors" + "fmt" + "os" + "sync" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/workloadinsight" +) + +type VisualReadManager struct { + gcsx.ReadManager + wrapped gcsx.ReadManager + + // Renderer for visualizing read I/O patterns. + ioRenderer *workloadinsight.Renderer + + // List of recorded read I/O ranges. + readIOs []workloadinsight.Range + + // Guards access to readIOs slice. + mu sync.Mutex + + // Configuration for workload insight visualization. + cfg cfg.WorkloadInsightConfig + + // forwardMergeThreshold is the threshold in bytes for merging adjacent readIOs. + forwardMergeThreshold uint64 +} + +// NewVisualReadManager creates a new VisualReadManager that wraps +// an existing ReadManager and uses the provided IORenderer to visualize +// read I/O patterns. +// The visualization is output to outputFilePath when Destroy() is called. +// In case outputFilePath is empty, output is printed to stdout. +func NewVisualReadManager(wrapped gcsx.ReadManager, ioRenderer *workloadinsight.Renderer, cfg cfg.WorkloadInsightConfig) *VisualReadManager { + return &VisualReadManager{ + wrapped: wrapped, + ioRenderer: ioRenderer, + readIOs: []workloadinsight.Range{}, + mu: sync.Mutex{}, + cfg: cfg, + forwardMergeThreshold: uint64(cfg.ForwardMergeThresholdMb * gcsx.MiB), + } +} + +func (vrm *VisualReadManager) ReaderName() string { + return "visual_read_manager" +} + +// ReadAt records the read I/O range and delegates the read to the wrapped ReadManager. +func (vrm *VisualReadManager) ReadAt(ctx context.Context, req *gcsx.ReadRequest) (gcsx.ReadResponse, error) { + // Capture the range in the visualizer + if len(req.Buffer) > 0 { + vrm.acceptRange(uint64(req.Offset), uint64(req.Offset)+uint64(len(req.Buffer))) + } + // Delegate to the wrapped ReadManager + return vrm.wrapped.ReadAt(ctx, req) +} + +// Destroy outputs the read I/O visualization and destroys the wrapped ReadManager. +func (vrm *VisualReadManager) Destroy() { + defer vrm.wrapped.Destroy() + + output, err := vrm.ioRenderer.Render(vrm.Object().Name, vrm.Object().Size, vrm.readIOs) + if err != nil { + logger.Warnf("Failed to render read pattern: %v", err) + return + } + if vrm.cfg.OutputFile == "" { + fmt.Println(output) + return + } + + if err := appendToFile(vrm.cfg.OutputFile, output); err != nil { + fmt.Println(output) + logger.Warnf("Failed to append to output file: %v", err) + return + } +} + +func (vrm *VisualReadManager) Object() *gcs.MinObject { + return vrm.wrapped.Object() +} + +func (vrm *VisualReadManager) CheckInvariants() { + vrm.wrapped.CheckInvariants() +} + +// acceptRange records a read I/O range and merges it with existing ranges if possible. +func (vrm *VisualReadManager) acceptRange(start, end uint64) { + if end <= start { + return // Invalid range, ignore + } + + // Clamp end to object size. + end = min(end, vrm.Object().Size) + + newRange := workloadinsight.Range{ + Start: start, + End: end, + } + + vrm.mu.Lock() + defer vrm.mu.Unlock() + + // Try to merge with the last range if possible + if len(vrm.readIOs) > 0 { + lastRange := &vrm.readIOs[len(vrm.readIOs)-1] + + if mergedRange, ok := vrm.mergeRanges(*lastRange, newRange); ok { + // Merge the readIOs by extending the last range + vrm.readIOs[len(vrm.readIOs)-1] = mergedRange + return + } + } + + // No merge possible, add as new range + vrm.readIOs = append(vrm.readIOs, newRange) +} + +// mergeRanges combines two readIOs into a single range. +func (vrm *VisualReadManager) mergeRanges(first, second workloadinsight.Range) (workloadinsight.Range, bool) { + if first.End+vrm.forwardMergeThreshold < second.Start { + return workloadinsight.Range{}, false + } + + if first.End > second.Start { + return workloadinsight.Range{}, false + } + + return workloadinsight.Range{Start: first.Start, End: second.End}, true +} + +// appendToFile appends the given text to the specified output file. +// If the file does not exist, it is created. +func appendToFile(outputFilePath, text string) error { + if outputFilePath == "" { + return errors.New("output file path is empty") + } + + f, err := os.OpenFile(outputFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + return fmt.Errorf("failed to open output file: %w", err) + } + defer f.Close() + + if _, err := f.Write([]byte(text)); err != nil { + return fmt.Errorf("failed to write to output file: %w", err) + } + return nil +} diff --git a/internal/gcsx/read_manager/visual_read_manager_test.go b/internal/gcsx/read_manager/visual_read_manager_test.go new file mode 100644 index 0000000000..24e8579258 --- /dev/null +++ b/internal/gcsx/read_manager/visual_read_manager_test.go @@ -0,0 +1,274 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package read_manager + +import ( + "context" + "os" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/workloadinsight" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestNewVisualReadManager(t *testing.T) { + mockReadManager := &MockReadManager{} + ioRenderer, err := workloadinsight.NewRenderer() + require.NoError(t, err, "Failed to create IORenderer") + + vrm := NewVisualReadManager(mockReadManager, ioRenderer, cfg.WorkloadInsightConfig{}) + + assert.NotNil(t, vrm, "VisualReadManager should not be nil") + assert.Equal(t, mockReadManager, vrm.wrapped, "Wrapped ReadManager should match the input") + assert.Equal(t, ioRenderer, vrm.ioRenderer, "IORenderer should match the input") + assert.Empty(t, vrm.readIOs, "Initial readIOs slice should be empty") +} + +func TestVisualReadManager_AcceptRange(t *testing.T) { + testCase := []struct { + name string + inputRanges [][2]uint64 + expectedRanges []workloadinsight.Range + }{ + { + name: "Non-overlapping ranges", + inputRanges: [][2]uint64{ + {0, 10}, + {20, 30}, + {40, 50}, + }, + expectedRanges: []workloadinsight.Range{ + {Start: 0, End: 10}, + {Start: 20, End: 30}, + {Start: 40, End: 50}, + }, + }, + { + name: "Overlapping ranges", + inputRanges: [][2]uint64{ + {0, 15}, + {10, 25}, + {20, 30}, + }, + expectedRanges: []workloadinsight.Range{ + {Start: 0, End: 15}, + {Start: 10, End: 25}, + {Start: 20, End: 30}, + }, + }, + { + name: "Adjacent ranges", + inputRanges: [][2]uint64{ + {0, 10}, + {10, 20}, + {20, 30}, + }, + expectedRanges: []workloadinsight.Range{ + {Start: 0, End: 30}, + }, + }, + { + name: "Mixed ranges", + inputRanges: [][2]uint64{ + {0, 10}, + {5, 15}, + {20, 25}, + {25, 30}, + {40, 50}, + }, + expectedRanges: []workloadinsight.Range{ + {Start: 0, End: 10}, + {Start: 5, End: 15}, + {Start: 20, End: 30}, + {Start: 40, End: 50}, + }, + }, + } + + for _, tc := range testCase { + t.Run(tc.name, func(t *testing.T) { + mockReadManager := &MockReadManager{} + mockReadManager.On("Object").Return(&gcs.MinObject{Name: "test-object", Size: 100}).Maybe() + ioRenderer, err := workloadinsight.NewRenderer() + require.NoError(t, err, "Failed to create IORenderer") + vrm := NewVisualReadManager(mockReadManager, ioRenderer, cfg.WorkloadInsightConfig{}) + + for _, r := range tc.inputRanges { + vrm.acceptRange(r[0], r[1]) + } + + assert.Equal(t, tc.expectedRanges, vrm.readIOs, "Recorded readIOs should match expected merged ranges") + }) + } +} + +func TestVisualReadManager_MergeRanges(t *testing.T) { + testCases := []struct { + name string + forwardMergeThresholdMb uint64 + first workloadinsight.Range + second workloadinsight.Range + expected workloadinsight.Range + merge bool + }{ + { + name: "Overlapping ranges", + forwardMergeThresholdMb: 0, + first: workloadinsight.Range{Start: 0, End: 10}, + second: workloadinsight.Range{Start: 5, End: 15}, + expected: workloadinsight.Range{}, + merge: false, + }, + { + name: "Adjacent ranges", + forwardMergeThresholdMb: 0, + first: workloadinsight.Range{Start: 10, End: 20}, + second: workloadinsight.Range{Start: 20, End: 30}, + expected: workloadinsight.Range{Start: 10, End: 30}, + merge: true, + }, + { + name: "Non-overlapping ranges", + forwardMergeThresholdMb: 0, + first: workloadinsight.Range{Start: 0, End: 10}, + second: workloadinsight.Range{Start: 15, End: 25}, + expected: workloadinsight.Range{}, + merge: false, + }, + { + name: "Within forward merge threshold", + forwardMergeThresholdMb: 1, // 1 MB + first: workloadinsight.Range{Start: 0, End: 10}, + second: workloadinsight.Range{Start: 1 * MiB, End: 2 * MiB}, + expected: workloadinsight.Range{Start: 0, End: 2 * MiB}, + merge: true, + }, + { + name: "Exceeding forward merge threshold", + forwardMergeThresholdMb: 1, // 1 MB + first: workloadinsight.Range{Start: 0, End: 10}, + second: workloadinsight.Range{Start: 2 * MiB, End: 3 * MiB}, + expected: workloadinsight.Range{}, + merge: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + mockReadManager := &MockReadManager{} + ioRenderer, err := workloadinsight.NewRenderer() + require.NoError(t, err, "Failed to create IORenderer") + vrm := NewVisualReadManager(mockReadManager, ioRenderer, cfg.WorkloadInsightConfig{ForwardMergeThresholdMb: int64(tc.forwardMergeThresholdMb)}) + + mergedRange, ok := vrm.mergeRanges(tc.first, tc.second) + + assert.Equal(t, tc.merge, ok, "Merge result should match expected") + assert.Equal(t, tc.expected, mergedRange, "Merged range should match expected") + }) + } +} + +func TestVisualReadManager_ReadAt(t *testing.T) { + mockReadManager := &MockReadManager{} + mockReadManager.On("Object").Return(&gcs.MinObject{Name: "test-object", Size: 100}).Maybe() + mockReadManager.On("ReadAt", mock.Anything, mock.AnythingOfType("*gcsx.ReadRequest")).Return(gcsx.ReadResponse{}, nil).Once() + ioRenderer, err := workloadinsight.NewRenderer() + require.NoError(t, err, "Failed to create IORenderer") + vrm := NewVisualReadManager(mockReadManager, ioRenderer, cfg.WorkloadInsightConfig{}) + + _, err = vrm.ReadAt(context.Background(), &gcsx.ReadRequest{ + Buffer: make([]byte, 20), + Offset: 10, + }) + require.NoError(t, err, "ReadAt should not return an error") + + expectedRange := workloadinsight.Range{Start: 10, End: 30} + assert.Len(t, vrm.readIOs, 1, "There should be one recorded range") + assert.Equal(t, expectedRange, vrm.readIOs[0], "Recorded range should match expected") + mockReadManager.AssertExpectations(t) +} + +func TestVisualReadManager_Destroy(t *testing.T) { + mockReadManager := &MockReadManager{} + mockReadManager.On("Object").Return(&gcs.MinObject{Name: "test-object", Size: 100}).Maybe() + mockReadManager.On("Destroy").Return().Once() + ioRenderer, err := workloadinsight.NewRenderer() + require.NoError(t, err, "Failed to create IORenderer") + vrm := NewVisualReadManager(mockReadManager, ioRenderer, cfg.WorkloadInsightConfig{}) + vrm.acceptRange(0, 10) + vrm.acceptRange(20, 30) + + vrm.Destroy() + + mockReadManager.AssertExpectations(t) +} + +func TestVisualReadManager_Destroy_WithOutputFile(t *testing.T) { + mockReadManager := &MockReadManager{} + mockReadManager.On("Object").Return(&gcs.MinObject{Name: "test-object", Size: 100}).Maybe() + mockReadManager.On("Destroy").Return().Once() + ioRenderer, err := workloadinsight.NewRenderer() + require.NoError(t, err, "Failed to create IORenderer") + outputFilePath := "test_output.txt" + vrm := NewVisualReadManager(mockReadManager, ioRenderer, cfg.WorkloadInsightConfig{OutputFile: outputFilePath}) + vrm.acceptRange(0, 10) + vrm.acceptRange(20, 40) + + vrm.Destroy() + + // Verify that the output file was created and contains data. + data, err := os.ReadFile(outputFilePath) + assert.NoError(t, err, "Should be able to read the output file") + assert.NotEmpty(t, data, "Output file should not be empty") + err = os.Remove(outputFilePath) + assert.NoError(t, err, "Should be able to delete the output file") + mockReadManager.AssertExpectations(t) +} + +func TestAppendToFile_EmptyFile(t *testing.T) { + outputFilePath := "test_append_output.txt" + text1 := "First line of text.\n" + + err := appendToFile(outputFilePath, text1) + + assert.NoError(t, err, "First appendToFile should not return an error") + data, err := os.ReadFile(outputFilePath) + assert.NoError(t, err, "Should be able to read the output file") + assert.Equal(t, text1, string(data), "Output file content should match expected") + err = os.Remove(outputFilePath) + assert.NoError(t, err, "Should be able to delete the output file") +} + +func TestAppendToFile_NonEmptyFile(t *testing.T) { + outputFilePath := "test_append_output.txt" + text1 := "First line of text.\n" + err := appendToFile(outputFilePath, text1) + require.NoError(t, err, "First appendToFile should not return an error") + + text2 := "Second line of text.\n" + err = appendToFile(outputFilePath, text2) + require.NoError(t, err, "Second appendToFile should not return an error") + + data, err := os.ReadFile(outputFilePath) + assert.NoError(t, err, "Should be able to read the output file") + assert.Equal(t, text1+text2, string(data), "Output file content should match expected") + err = os.Remove(outputFilePath) + assert.NoError(t, err, "Should be able to delete the output file") +} diff --git a/internal/gcsx/read_type_classifier.go b/internal/gcsx/read_type_classifier.go new file mode 100644 index 0000000000..9a114a0dea --- /dev/null +++ b/internal/gcsx/read_type_classifier.go @@ -0,0 +1,208 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx + +import ( + "sync/atomic" + + "github.com/googlecloudplatform/gcsfuse/v3/metrics" +) + +// ReaderType enum values. +const ( + MB = 1 << 20 +) + +// ReadInfo Stores information for this read request. +type ReadInfo struct { + // ReadType stores the read type evaluated for this request. + ReadType int64 + + // ExpectedOffset stores the expected offset for this request. Will be + // used to determine if re-evaluation of ReadType is required or not with range reader. + ExpectedOffset int64 + + // SeekRecorded tells whether a seek has been performed for this read request. + SeekRecorded bool +} + +// ReadTypeClassifier tracks the read access pattern (sequential vs random) across multiple readers. +// It uses heuristics based on the number of seeks and average read size to classify the read pattern. +// It is safe for concurrent use by multiple goroutines. +type ReadTypeClassifier struct { + // ReadType of the reader. Will be sequential by default. + readType atomic.Int64 + + // Specifies the next expected offset for the reads. Used to distinguish between + // sequential and random reads. + expectedOffset atomic.Int64 + + // seeks represents the number of random reads performed by the reader. + seeks atomic.Uint64 + + // totalReadBytes is the total number of bytes read by the reader. + totalReadBytes atomic.Uint64 + + // sequentialReadSizeMb is the configured sequential read size in MB. + sequentialReadSizeMb int64 + + // initialOffset stores the first read offset, helps in determining the initial prefetch size. + initialOffset int64 +} + +func NewReadTypeClassifier(sequentialReadSizeMb int64, initialOffset int64) *ReadTypeClassifier { + state := &ReadTypeClassifier{ + readType: atomic.Int64{}, + expectedOffset: atomic.Int64{}, + seeks: atomic.Uint64{}, + totalReadBytes: atomic.Uint64{}, + sequentialReadSizeMb: sequentialReadSizeMb, + initialOffset: initialOffset, + } + + // Start as sequential read type, keep the existing GCSFuse read behavior. + state.readType.Store(metrics.ReadTypeSequential) + return state +} + +// RecordSeek checks if the read at the given offset is a seek and updates the internal state accordingly. +// Call it before starting the read operation. +func (rtc *ReadTypeClassifier) RecordSeek(offset int64) { + rtc.GetReadInfo(offset, false) +} + +// RecordRead records a read operation of the given size at the given offset. +// This must be called after the read operation. +func (rtc *ReadTypeClassifier) RecordRead(offset int64, sizeRead int64) { + rtc.totalReadBytes.Add(uint64(sizeRead)) + rtc.expectedOffset.Store(offset + sizeRead) +} + +// isSeekNeeded determines if the current read at `offset` should be considered a +// seek, given the previous read pattern & the expected offset. +func (rtc *ReadTypeClassifier) isSeekNeeded(offset int64) bool { + expectedOffset := rtc.expectedOffset.Load() + readType := rtc.readType.Load() + + if expectedOffset == 0 { + return false + } + + // Read from unexpected offset in random read is considered a seek. + if readType == metrics.ReadTypeRandom { + return expectedOffset != offset + } + + // In sequential read, read backward or too far (> maxReadSize) forward is considered a seek. + // This allows for some level of kernel readahead in sequential reads. + if readType == metrics.ReadTypeSequential { + return offset < expectedOffset || offset > expectedOffset+maxReadSize + } + + return false +} + +// GetReadInfo determines the read strategy (sequential or random) for a read +// request at a given offset and returns read metadata. It also updates the +// internal state `readType` based on the read pattern. +// seekRecorded parameter describes whether a seek has already been recorded for this request. +func (rtc *ReadTypeClassifier) GetReadInfo(offset int64, seekRecorded bool) ReadInfo { + previousReadType := rtc.readType.Load() + expOffset := rtc.expectedOffset.Load() + numSeeks := rtc.seeks.Load() + currentTotalReadBytes := rtc.totalReadBytes.Load() + + if !seekRecorded && rtc.isSeekNeeded(offset) { + numSeeks = rtc.seeks.Add(1) + seekRecorded = true + } + + readType := metrics.ReadTypeRandom + averageReadBytes := avgReadBytes(currentTotalReadBytes, numSeeks) + + // Classify as Sequential if: + // 1. The average read size is large enough. + // 2. OR we haven't performed any seeks yet AND the first read was at offset 0. + if averageReadBytes >= maxReadSize || (numSeeks == 0 && rtc.initialOffset == 0) { + readType = metrics.ReadTypeSequential + } + + if readType != previousReadType { + rtc.readType.Store(readType) + } + + return ReadInfo{ + ReadType: readType, + ExpectedOffset: expOffset, + SeekRecorded: seekRecorded, + } +} + +// ComputeSeqPrefetchWindowAndAdjustType computes the sequential IO size heuristically based on +// the current read pattern. It also updates the readType if needed. +// If the read pattern is classified as random, it calculates an appropriate +// read size based on the average read size per seek, bounded by min and max read sizes. +// If the read pattern is sequential, it returns the configured sequential read size. +// Note: The returned prefetch window size is not limited by the object size, caller should +// handle that separately. +func (rtc *ReadTypeClassifier) ComputeSeqPrefetchWindowAndAdjustType() int64 { + currentReadType := rtc.readType.Load() + seeks := rtc.seeks.Load() + + // Evaluate for Random read type if seeks > 0 or the first read was non-zero. + // A non-zero initial offset implies random access even with zero seeks, so we check average read bytes. + if seeks > 0 || rtc.initialOffset > 0 { + averageReadBytes := avgReadBytes(rtc.totalReadBytes.Load(), seeks) + + if averageReadBytes < maxReadSize { + randomReadSize := ((averageReadBytes + MB - 1) / MB) * MB + // Clamp to [minReadSize, maxReadSize] + randomReadSize = min(max(randomReadSize, minReadSize), maxReadSize) + if currentReadType != metrics.ReadTypeRandom { + rtc.readType.Store(metrics.ReadTypeRandom) + } + return int64(randomReadSize) + } + } + if currentReadType != metrics.ReadTypeSequential { + rtc.readType.Store(metrics.ReadTypeSequential) + } + return rtc.sequentialReadSizeMb * MB +} + +// IsReadSequential returns true if the current read pattern is sequential +func (rtc *ReadTypeClassifier) IsReadSequential() bool { + return rtc.readType.Load() == metrics.ReadTypeSequential +} + +// NextExpectedOffset returns the next expected offset for a sequential read. +func (rtc *ReadTypeClassifier) NextExpectedOffset() int64 { + return rtc.expectedOffset.Load() +} + +// avgReadBytes calculates the average read bytes per seek. +// If no seeks have been recorded, it returns the total read bytes. +func avgReadBytes(totalReadBytes uint64, numSeeks uint64) uint64 { + if numSeeks > 0 { + return totalReadBytes / numSeeks + } + return totalReadBytes +} + +// GetSeeks returns the current number of seeks recorded. +// This method is intended for testing purposes. +func (rtc *ReadTypeClassifier) GetSeeks() uint64 { + return rtc.seeks.Load() +} diff --git a/internal/gcsx/read_type_classifier_test.go b/internal/gcsx/read_type_classifier_test.go new file mode 100644 index 0000000000..6524f47fbd --- /dev/null +++ b/internal/gcsx/read_type_classifier_test.go @@ -0,0 +1,651 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx + +import ( + "sync" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/stretchr/testify/assert" +) + +func TestReadTypeClassifier_InitialState(t *testing.T) { + readTypeClassifier := NewReadTypeClassifier(sequentialReadSizeInMb, 0) + + assert.Equal(t, metrics.ReadTypeSequential, readTypeClassifier.readType.Load()) + assert.Equal(t, int64(0), readTypeClassifier.expectedOffset.Load()) + assert.Equal(t, uint64(0), readTypeClassifier.seeks.Load()) + assert.Equal(t, uint64(0), readTypeClassifier.totalReadBytes.Load()) + assert.Equal(t, int64(0), readTypeClassifier.initialOffset) +} + +func TestReadTypeClassifier_IsSeekNeeded(t *testing.T) { + testCases := []struct { + name string + readType int64 + offset int64 + expectedOffset int64 + want bool + }{ + { + name: "First read, expectedOffset is 0", + readType: metrics.ReadTypeSequential, + offset: 100, + expectedOffset: 0, + want: false, + }, + { + name: "Random read, same offset", + readType: metrics.ReadTypeRandom, + offset: 100, + expectedOffset: 100, + want: false, + }, + { + name: "Random read, different offset", + readType: metrics.ReadTypeRandom, + offset: 200, + expectedOffset: 100, + want: true, + }, + { + name: "Sequential read, same offset", + readType: metrics.ReadTypeSequential, + offset: 100, + expectedOffset: 100, + want: false, + }, + { + name: "Sequential read, small forward jump within maxReadSize", + readType: metrics.ReadTypeSequential, + offset: 100 + maxReadSize/2, + expectedOffset: 100, + want: false, + }, + { + name: "Sequential read, forward jump to boundary of maxReadSize", + readType: metrics.ReadTypeSequential, + offset: 100 + maxReadSize, + expectedOffset: 100, + want: false, + }, + { + name: "Sequential read, large forward jump beyond maxReadSize", + readType: metrics.ReadTypeSequential, + offset: 100 + maxReadSize + 1, + expectedOffset: 100, + want: true, + }, + { + name: "Sequential read, backward jump", + readType: metrics.ReadTypeSequential, + offset: 99, + expectedOffset: 100, + want: true, + }, + { + name: "Unknown read type", + readType: -1, // An invalid read type + offset: 200, + expectedOffset: 100, + want: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + classifier := NewReadTypeClassifier(sequentialReadSizeInMb, 0) + classifier.readType.Store(tc.readType) + classifier.expectedOffset.Store(tc.expectedOffset) + + got := classifier.isSeekNeeded(tc.offset) + + assert.Equal(t, tc.want, got) + }) + } +} + +// This test also covers RecordSeek functionality. +func TestReadTypeClassifier_GetReadInfo(t *testing.T) { + testCases := []struct { + name string + offset int64 + seekRecorded bool + initialReadType int64 + initialExpOffset int64 + initialNumSeeks uint64 + initialTotalReadBytes uint64 + initialOffset int64 + expectedReadType int64 + expectedNumSeeks uint64 + }{ + { + name: "First Read", + offset: 0, + seekRecorded: false, + initialReadType: metrics.ReadTypeSequential, + initialExpOffset: 0, + initialNumSeeks: 0, + initialTotalReadBytes: 0, + initialOffset: 0, + expectedReadType: metrics.ReadTypeSequential, + expectedNumSeeks: 0, + }, + { + name: "First Read at non-zero offset", + offset: 100, + seekRecorded: false, + initialReadType: metrics.ReadTypeSequential, + initialExpOffset: 0, + initialNumSeeks: 0, + initialTotalReadBytes: 0, + initialOffset: 100, + expectedReadType: metrics.ReadTypeRandom, + expectedNumSeeks: 0, + }, + { + name: "Sequential Read", + offset: 10, + seekRecorded: false, + initialReadType: metrics.ReadTypeSequential, + initialExpOffset: 10, + initialNumSeeks: 0, + initialTotalReadBytes: 100, + initialOffset: 0, + expectedReadType: metrics.ReadTypeSequential, + expectedNumSeeks: 0, + }, + { + name: "Sequential read with small forward jump and high average read bytes is still sequential", + offset: 100, + seekRecorded: false, + initialReadType: metrics.ReadTypeSequential, + initialExpOffset: 10, + initialNumSeeks: 0, + initialTotalReadBytes: 10000000, + initialOffset: 0, + expectedReadType: metrics.ReadTypeSequential, + expectedNumSeeks: 0, + }, + { + name: "Sequential read with large forward jump is a seek and switches to random", + offset: 50 + maxReadSize + 1, + seekRecorded: false, + initialReadType: metrics.ReadTypeSequential, + initialExpOffset: 50, + initialNumSeeks: 0, + initialTotalReadBytes: 50 * 1024, + initialOffset: 0, + expectedReadType: metrics.ReadTypeRandom, + expectedNumSeeks: 1, + }, + { + name: "Sequential read with backward jump is a seek and switches to random", + offset: 49, + seekRecorded: false, + initialReadType: metrics.ReadTypeSequential, + initialExpOffset: 50, + initialNumSeeks: 0, + initialTotalReadBytes: 50 * 1024, + initialOffset: 0, + expectedReadType: metrics.ReadTypeRandom, + expectedNumSeeks: 1, + }, + { + name: "Contiguous random read is not a seek", + offset: 50, + seekRecorded: false, + initialReadType: metrics.ReadTypeRandom, + initialExpOffset: 50, + initialNumSeeks: 1, + initialTotalReadBytes: 50 * 1024, + initialOffset: 0, + expectedReadType: metrics.ReadTypeRandom, + expectedNumSeeks: 1, + }, + { + name: "Non-contiguous random read is a seek", + offset: 100, + seekRecorded: false, + initialReadType: metrics.ReadTypeRandom, + initialExpOffset: 50, + initialNumSeeks: 1, + initialTotalReadBytes: 50 * 1024, + initialOffset: 0, + expectedReadType: metrics.ReadTypeRandom, + expectedNumSeeks: 2, + }, + { + name: "Switches to random read on seek", + offset: 50 + maxReadSize + 1, + seekRecorded: false, + initialReadType: metrics.ReadTypeSequential, + initialExpOffset: 50, + initialNumSeeks: 0, + initialTotalReadBytes: 1000, + initialOffset: 0, + expectedReadType: metrics.ReadTypeRandom, + expectedNumSeeks: 1, + }, + { + name: "Switches back to sequential with high average read bytes", + offset: 100, + seekRecorded: false, + initialReadType: metrics.ReadTypeRandom, + initialExpOffset: 50, + initialNumSeeks: 1, + initialTotalReadBytes: maxReadSize * 2, + initialOffset: 0, + expectedReadType: metrics.ReadTypeSequential, + expectedNumSeeks: 2, + }, + { + name: "Seek recorded: sequential large forward jump", + offset: 50 + maxReadSize + 1, + seekRecorded: true, + initialReadType: metrics.ReadTypeSequential, + initialExpOffset: 50, + initialNumSeeks: 0, + initialTotalReadBytes: 50 * 1024, + initialOffset: 0, + expectedReadType: metrics.ReadTypeSequential, + expectedNumSeeks: 0, // Not incremented + }, + { + name: "Seek recorded: sequential backward jump switches to random", + offset: 49, + seekRecorded: true, + initialReadType: metrics.ReadTypeSequential, + initialExpOffset: 50, + initialNumSeeks: 1, + initialTotalReadBytes: 50 * 1024, + initialOffset: 0, + expectedReadType: metrics.ReadTypeRandom, + expectedNumSeeks: 1, // Not incremented + }, + { + name: "Seek recorded: non-contiguous random read", + offset: 100, + seekRecorded: true, + initialReadType: metrics.ReadTypeRandom, + initialExpOffset: 50, + initialNumSeeks: 1, + initialTotalReadBytes: 50 * 1024, + initialOffset: 0, + expectedReadType: metrics.ReadTypeRandom, + expectedNumSeeks: 1, // Not incremented + }, + { + name: "Seek recorded: switches to random", + offset: 50 + maxReadSize + 1, + seekRecorded: true, + initialReadType: metrics.ReadTypeSequential, + initialExpOffset: 50, + initialNumSeeks: 1, + initialTotalReadBytes: 1000, + initialOffset: 0, + expectedReadType: metrics.ReadTypeRandom, + expectedNumSeeks: 1, // Not incremented + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + readTypeClassifier := NewReadTypeClassifier(sequentialReadSizeInMb, 0) + readTypeClassifier.readType.Store(tc.initialReadType) + readTypeClassifier.expectedOffset.Store(tc.initialExpOffset) + readTypeClassifier.seeks.Store(tc.initialNumSeeks) + readTypeClassifier.totalReadBytes.Store(tc.initialTotalReadBytes) + readTypeClassifier.initialOffset = tc.initialOffset + + readInfo := readTypeClassifier.GetReadInfo(tc.offset, tc.seekRecorded) + + assert.Equal(t, tc.expectedReadType, readInfo.ReadType, "Read type mismatch") + assert.Equal(t, tc.expectedNumSeeks, readTypeClassifier.seeks.Load(), "Number of seeks mismatch") + }) + } +} + +func TestReadTypeClassifier_RecordRead(t *testing.T) { + testCases := []struct { + name string + initialExpectedOffset int64 + initialTotalReadBytes uint64 + offset int64 + sizeRead int64 + expectedOffset int64 + expectedTotalBytes uint64 + }{ + { + name: "First read", + initialExpectedOffset: 0, + initialTotalReadBytes: 0, + offset: 0, + sizeRead: 10 * MB, + expectedOffset: 10 * MB, + expectedTotalBytes: 10 * MB, + }, + { + name: "Subsequent read", + initialExpectedOffset: 10 * MB, + initialTotalReadBytes: 10 * MB, + offset: 10 * MB, + sizeRead: 5 * MB, + expectedOffset: 15 * MB, + expectedTotalBytes: 15 * MB, + }, + { + name: "Any random read", + initialExpectedOffset: 15 * MB, + initialTotalReadBytes: 15 * MB, + offset: 15 * MB, + sizeRead: 20 * MB, + expectedOffset: 35 * MB, + expectedTotalBytes: 35 * MB, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + classifier := NewReadTypeClassifier(sequentialReadSizeInMb, 0) + classifier.expectedOffset.Store(tc.initialExpectedOffset) + classifier.totalReadBytes.Store(tc.initialTotalReadBytes) + + classifier.RecordRead(tc.offset, tc.sizeRead) + + assert.Equal(t, tc.expectedOffset, classifier.expectedOffset.Load(), "Expected offset mismatch") + assert.Equal(t, tc.expectedTotalBytes, classifier.totalReadBytes.Load(), "Total read bytes mismatch") + }) + } +} + +func TestReadTypeClassifier_ComputeSeqPrefetchWindowAndAdjustType(t *testing.T) { + testCases := []struct { + name string + initialNumSeeks uint64 + initialTotalReadBytes uint64 + initialOffset int64 + sequentialReadSizeMb int64 + expectedSeqPrefetchWindow int64 + }{ + { + name: "Sequential Read, No seek", + initialNumSeeks: 0, + initialTotalReadBytes: 0, + initialOffset: 0, + sequentialReadSizeMb: 22, + expectedSeqPrefetchWindow: 22 * MB, + }, + { + name: "Sequential Read, 1 seek but high average read size", + initialNumSeeks: 1, + initialTotalReadBytes: 100 * MB, + initialOffset: 0, + sequentialReadSizeMb: 22, + expectedSeqPrefetchWindow: 22 * MB, + }, + { + name: "Sequential Read, multiple seeks but low average read size", + initialNumSeeks: 2, + initialTotalReadBytes: 10 * MB, + initialOffset: 0, + sequentialReadSizeMb: 22, + expectedSeqPrefetchWindow: 5 * MB, // Avg is 5MB. + }, + { + name: "Random Read, multiple seeks and low average read size", + initialNumSeeks: 2, + initialTotalReadBytes: 5 * MB, + initialOffset: 0, + sequentialReadSizeMb: 22, + expectedSeqPrefetchWindow: 3 * MB, // Avg is 2.5MB, rounded up to 3MB. + }, + { + name: "Random Read, multiple seeks and very low average read size", + initialNumSeeks: 2, + initialTotalReadBytes: 500 * 1024, // 500KB + initialOffset: 0, + sequentialReadSizeMb: 22, + expectedSeqPrefetchWindow: minReadSize, + }, + { + name: "Random Read, multiple seeks and moderate average read size", + initialNumSeeks: 2, + initialTotalReadBytes: 3 * MB, + initialOffset: 0, + sequentialReadSizeMb: 22, + expectedSeqPrefetchWindow: 2 * MB, // Avg is 1.5MB, rounded up to 2MB. + }, + { + name: "Random Read, multiple seeks and high average read size", + initialNumSeeks: 2, + initialTotalReadBytes: 100 * MB, + initialOffset: 0, + sequentialReadSizeMb: 22, + expectedSeqPrefetchWindow: 22 * MB, // Avg is ~33MB, more than maxReadSize so capped to 22MB. + }, + { + name: "Sequential read, Different sequential read size configured", + initialNumSeeks: 0, + initialTotalReadBytes: 0, + initialOffset: 0, + sequentialReadSizeMb: 10, + expectedSeqPrefetchWindow: 10 * MB, + }, + { + name: "Random Read, multiple seeks and high average read size, 10MB sequential read size", + initialNumSeeks: 2, + initialTotalReadBytes: 100 * MB, + initialOffset: 0, + sequentialReadSizeMb: 10, + expectedSeqPrefetchWindow: 10 * MB, + }, + { + name: "Random Read, 1 seek and low average read size", + initialNumSeeks: 1, + initialTotalReadBytes: 1 * MB, + initialOffset: 0, + sequentialReadSizeMb: 22, + expectedSeqPrefetchWindow: 1 * MB, + }, + { + name: "First read non-zero offset (Random type set, seeks 0)", + initialNumSeeks: 0, + initialTotalReadBytes: 0, + initialOffset: 100, + sequentialReadSizeMb: 22, + expectedSeqPrefetchWindow: minReadSize, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + classifier := NewReadTypeClassifier(tc.sequentialReadSizeMb, 0) + classifier.seeks.Store(tc.initialNumSeeks) + classifier.totalReadBytes.Store(tc.initialTotalReadBytes) + classifier.initialOffset = tc.initialOffset + + seqReadIO := classifier.ComputeSeqPrefetchWindowAndAdjustType() + + assert.Equal(t, tc.expectedSeqPrefetchWindow, seqReadIO, "SeqIO size mismatch") + }) + } +} + +func TestReadTypeClassifier_IsSequentialRead(t *testing.T) { + testCases := []struct { + name string + readType int64 + SequentialRead bool + }{ + { + name: "ReadTypeSequential", + readType: metrics.ReadTypeSequential, + SequentialRead: true, + }, + { + name: "ReadTypeRandom", + readType: metrics.ReadTypeRandom, + SequentialRead: false, + }, + { + name: "ReadTypeUnknown", + readType: -1, + SequentialRead: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + classifier := NewReadTypeClassifier(sequentialReadSizeInMb, 0) + classifier.readType.Store(tc.readType) + + assert.Equal(t, tc.SequentialRead, classifier.IsReadSequential()) + }) + } +} + +func Test_avgReadBytes(t *testing.T) { + testCases := []struct { + name string + totalReadBytes uint64 + numSeeks uint64 + expectedAvgReadBytes uint64 + }{ + { + name: "No seeks", + totalReadBytes: 100 * MB, + numSeeks: 0, + expectedAvgReadBytes: 100 * MB, + }, + { + name: "One seek", + totalReadBytes: 100 * MB, + numSeeks: 1, + expectedAvgReadBytes: 100 * MB, + }, + { + name: "Multiple seeks", + totalReadBytes: 300 * MB, + numSeeks: 3, + expectedAvgReadBytes: 100 * MB, + }, + { + name: "Multiple seeks with remainder", + totalReadBytes: 350 * MB, + numSeeks: 3, + expectedAvgReadBytes: 122333866, // Integer division + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + avg := avgReadBytes(tc.totalReadBytes, tc.numSeeks) + + assert.Equal(t, tc.expectedAvgReadBytes, avg) + }) + } +} + +func TestReadTypeClassifier_SequentialReads(t *testing.T) { + readTypeClassifier := NewReadTypeClassifier(sequentialReadSizeInMb, 0) + + // Simulate 4 reads of 10MB IO. + readSizes := []int64{10 * MB, 10 * MB, 10 * MB, 10 * MB} + var offset int64 = 0 + for _, size := range readSizes { + readTypeClassifier.RecordSeek(offset) + readTypeClassifier.RecordRead(offset, size) + offset += size + assert.Equal(t, metrics.ReadTypeSequential, readTypeClassifier.readType.Load()) + assert.Equal(t, offset, readTypeClassifier.expectedOffset.Load()) + } + + assert.Equal(t, metrics.ReadTypeSequential, readTypeClassifier.readType.Load()) + assert.Equal(t, uint64(0), readTypeClassifier.seeks.Load()) + assert.Equal(t, uint64(40*MB), readTypeClassifier.totalReadBytes.Load()) +} + +func TestReadTypeClassifier_RandomReads(t *testing.T) { + classifier := NewReadTypeClassifier(sequentialReadSizeInMb, 0) + + // Simulate random reads of 5MB each at different offsets. + readSizes := []int64{5 * MB, 5 * MB, 5 * MB, 5 * MB} + offsets := []int64{0, 20 * MB, 10 * MB, 30 * MB} + for i, size := range readSizes { + classifier.RecordSeek(offsets[i]) + classifier.RecordRead(offsets[i], size) + } + + assert.Equal(t, metrics.ReadTypeRandom, classifier.readType.Load(), "Read type mismatch") + assert.Equal(t, uint64(3), classifier.seeks.Load(), "Seek mismatch") + assert.Equal(t, uint64(20*MB), classifier.totalReadBytes.Load(), "Total read bytes mismatch") +} + +func TestReadTypeClassifier_RandomToSequentialRead(t *testing.T) { + classifier := NewReadTypeClassifier(sequentialReadSizeInMb, 0) + // Start with random reads. + randomReadSizes := []int64{2 * MB, 2 * MB, 2 * MB, 2 * MB, 2 * MB} + randomOffsets := []int64{50 * MB, 20 * MB, 10 * MB, 30 * MB, 40 * MB} + for i, size := range randomReadSizes { + classifier.RecordSeek(randomOffsets[i]) + classifier.RecordRead(randomOffsets[i], size) + } + assert.Equal(t, uint64(4), classifier.seeks.Load(), "Seek mismatch") + assert.Equal(t, uint64(10*MB), classifier.totalReadBytes.Load(), "Total read bytes mismatch") + + // Now do large sequential reads from different seek. + seqReadSizes := []int64{20 * MB, 20 * MB, 20 * MB} + var offset int64 = 100 * MB + for _, size := range seqReadSizes { + classifier.RecordSeek(offset) + classifier.RecordRead(offset, size) + offset += size + } + + assert.Equal(t, metrics.ReadTypeSequential, classifier.readType.Load(), "Read type should switch to sequential") + assert.Equal(t, uint64(5), classifier.seeks.Load(), "Seek count should remain the same") + assert.Equal(t, uint64(70*MB), classifier.totalReadBytes.Load(), "Total read bytes mismatch") +} + +func TestReadTypeClassifier_ConcurrentUpdates(t *testing.T) { + classifier := NewReadTypeClassifier(sequentialReadSizeInMb, 0) + var wg sync.WaitGroup + numGoroutines := 10 + readsPerGoroutine := 100 + + for i := 0; i < numGoroutines; i++ { + wg.Add(1) + go func(id int) { + defer wg.Done() + offset := int64(id * 100 * MB) + for j := 0; j < readsPerGoroutine; j++ { + size := int64(5 * MB) + classifier.RecordSeek(offset) + classifier.RecordRead(offset, size) + offset += size + } + }(i) + } + + wg.Wait() + + // After all concurrent updates, check that internal state is consistent. + totalReads := int64(numGoroutines * readsPerGoroutine * 5 * MB) + assert.Equal(t, uint64(totalReads), classifier.totalReadBytes.Load()) + // Read type could be either sequential or random depending on timing, so just check it's valid. + readType := classifier.readType.Load() + assert.True(t, readType == metrics.ReadTypeSequential || readType == metrics.ReadTypeRandom) +} diff --git a/internal/gcsx/reader.go b/internal/gcsx/reader.go new file mode 100644 index 0000000000..69a990b772 --- /dev/null +++ b/internal/gcsx/reader.go @@ -0,0 +1,120 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx + +import ( + "context" + "errors" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" +) + +// FallbackToAnotherReader is returned when data could not be retrieved +// from the current reader, indicating that the caller should attempt to fall back +// to an alternative reader. +var FallbackToAnotherReader = errors.New("fallback to another reader is required") + +// ReadRequest encapsulates the parameters for a read operation. +type ReadRequest struct { + // Buffer is provided by jacobsa/fuse and should be filled with data from the object. + Buffer []byte + + // Offset specifies the starting position in the object from where data should be read. + // Note: This value should not be modified by any reader. It is used by the + // read manager to fall back to the next reader and to record the read operation + // correctly. + Offset int64 + + // SkipSizeChecks, when true, instructs the reader to bypass validation of the + // read request against the object's cached size. This is necessary for + // use cases like tailing a file, where reads might target offsets beyond + // the current object size if remote appends is in progress. + SkipSizeChecks bool + + // ReadInfo contains metadata about the read pattern. + ReadInfo +} + +// GCSReaderRequest represents the request parameters needed to read a data from a GCS object. +type GCSReaderRequest struct { + // Buffer is provided by jacobsa/fuse and should be filled with data from the object. + Buffer []byte + + // Offset specifies the starting position in the object from where data should be read. + Offset int64 + + // This determines GCS range request. + EndOffset int64 + + // This parameter specifies whether the reader needs to be discarded for a new reader. + ForceCreateReader bool + + // This parameter specifies whether size checks against cached object size must be skipped. + SkipSizeChecks bool + + // ReadInfo contains metadata about the read pattern. + *ReadInfo +} + +// ReadResponse represents the response returned as part of a ReadAt call. +// It includes the actual data read and its size. +type ReadResponse struct { + // Data contains slices of bytes read from the object. This is populated when + // the reader returns data directly from its internal buffers. + Data [][]byte + + // Size indicates how many bytes were read into DataBuf. + Size int + + // Callback is a function to be executed after the read operation is completed. + Callback func() +} + +type Reader interface { + // CheckInvariants performs internal consistency checks on the reader state. + CheckInvariants() + + // ReadAt attempts to read data from the object. Depending on the + // implementation, it may either populate the provided buffer `p` directly + // or return data as a slice of byte slices in the `Data` field of the + // ReadResponse. To indicate that the operation should be handled by an + // alternative reader, return the error FallbackToAnotherReader. + // If an error occurs, the size in ReadResponse will be zero. + ReadAt(ctx context.Context, req *ReadRequest) (ReadResponse, error) + + // Return a constant ReaderName associated with the specific reader implementation + ReaderName() string + + // Destroy is called to release any resources held by the reader. + Destroy() +} + +// ReadManager is generally used in higher-level components that need access to object metadata. +// File handle will contain a ReadManager instance and will handle read operations. +type ReadManager interface { + Reader + + // Object returns the underlying GCS object metadata associated with the reader. + Object() *gcs.MinObject +} + +// GCSReader defines an interface for reading data from a GCS object. +// This interface is intended for lower-level interactions with GCS readers. +type GCSReader interface { + // ReadAt reads data into the `Buffer` field of the provided `GCSReaderRequest`, + // starting from the specified offset and ending at the specified end offset. + // It returns a `ReadResponse` indicating the number of bytes successfully read and any error encountered. + ReadAt(ctx context.Context, req *GCSReaderRequest) (ReadResponse, error) +} diff --git a/internal/gcsx/shared_chunk_cache_reader.go b/internal/gcsx/shared_chunk_cache_reader.go new file mode 100644 index 0000000000..81eed3eea9 --- /dev/null +++ b/internal/gcsx/shared_chunk_cache_reader.go @@ -0,0 +1,299 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "syscall" + "time" + + "github.com/google/uuid" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/file" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/jacobsa/fuse/fuseops" +) + +// SharedChunkCacheReader implements on-demand chunk-based reading without prefetching for +// shared cache. It downloads only the chunks needed for a read operation, no prefetching. +type SharedChunkCacheReader struct { + manager *file.SharedChunkCacheManager + bucket gcs.Bucket + object *gcs.MinObject + metricHandle metrics.MetricHandle + traceHandle tracing.TraceHandle + handleID fuseops.HandleID +} + +// NewSharedChunkCacheReader creates a new chunk-based reader for shared cache. +func NewSharedChunkCacheReader( + manager *file.SharedChunkCacheManager, + bucket gcs.Bucket, + object *gcs.MinObject, + metricHandle metrics.MetricHandle, + traceHandle tracing.TraceHandle, + handleID fuseops.HandleID, +) *SharedChunkCacheReader { + return &SharedChunkCacheReader{ + manager: manager, + bucket: bucket, + object: object, + metricHandle: metricHandle, + traceHandle: traceHandle, + handleID: handleID, + } +} + +func (r *SharedChunkCacheReader) ReaderName() string { + return "shared_chunk_cache_reader" +} + +// ReadAt reads data at the specified offset, downloading chunks on-demand. +// Implements the Reader interface. +func (r *SharedChunkCacheReader) ReadAt(ctx context.Context, req *ReadRequest) (ReadResponse, error) { + var readResponse ReadResponse + + // Check if file should be excluded from cache based on regex + if r.manager.ShouldExcludeFromCache(r.bucket, r.object) { + return readResponse, FallbackToAnotherReader + } + + offset := req.Offset + p := req.Buffer + + if offset >= int64(r.object.Size) { + return readResponse, io.EOF + } + + if offset < 0 { + return readResponse, fmt.Errorf("negative offset: %d", offset) + } + + requestID := uuid.New() + logger.Tracef("%.13v <- SharedChunkCache(%s:/%s, offset: %d, size: %d handle: %d)", + requestID, r.bucket.Name(), r.object.Name, offset, len(p), r.handleID) + + startTime := time.Now() + var bytesRead int + var cacheHit bool + var err error + + defer func() { + executionTime := time.Since(startTime) + var requestOutput string + if err != nil { + requestOutput = fmt.Sprintf("err: %v (%v)", err, executionTime) + } else { + requestOutput = fmt.Sprintf("OK (cacheHit: %t, bytes: %d) (%v)", cacheHit, bytesRead, executionTime) + } + + logger.Tracef("%.13v -> %s", requestID, requestOutput) + + // Capture metrics + readType := metrics.ReadTypeRandom + if offset == 0 { + readType = metrics.ReadTypeSequential + } + r.metricHandle.FileCacheReadCount(1, cacheHit, metrics.ReadTypeNames[readType]) + r.metricHandle.FileCacheReadBytesCount(int64(bytesRead), metrics.ReadTypeNames[readType]) + r.metricHandle.FileCacheReadLatencies(ctx, executionTime, cacheHit) + }() + + totalRead := 0 + bufferRemaining := len(p) + currentOffset := offset + + // Read across chunk boundaries if necessary + for bufferRemaining > 0 && currentOffset < int64(r.object.Size) { + // Calculate which chunk contains this offset + chunkIndex := r.manager.GetChunkIndex(currentOffset) + chunkStart := chunkIndex * r.manager.GetChunkSize() + chunkEnd := min(chunkStart+r.manager.GetChunkSize(), int64(r.object.Size)) + + // Try to open the chunk file directly (no stat() to avoid stale NFS cache and reduce round trips) + chunkPath := r.manager.GetChunkPath(r.bucket.Name(), r.object.Name, r.object.Generation, chunkIndex) + chunkFile, openErr := os.Open(chunkPath) + if openErr != nil { + if errors.Is(openErr, syscall.ENOENT) { + // Chunk not in cache - download it + logger.Tracef("Chunk %d not cached, downloading for %s/%s (offset %d)", + chunkIndex, r.bucket.Name(), r.object.Name, currentOffset) + + if downloadErr := r.downloadChunk(ctx, chunkIndex, chunkStart, chunkEnd); downloadErr != nil { + bytesRead = totalRead + cacheHit = false + logger.Warnf("DownloadChunk (%d, %d, %d) failed with: %v, read from GCS reader.", chunkIndex, chunkStart, chunkEnd, downloadErr) + return readResponse, FallbackToAnotherReader + } + cacheHit = false // Cache miss - we had to download the chunk + + // Open the newly downloaded chunk, and file should exist. + chunkFile, openErr = os.Open(chunkPath) + if openErr != nil { + bytesRead = totalRead + logger.Warnf("Failed to open chunk %d after download at path %s: %v, falling back to GCS reader", chunkIndex, chunkPath, openErr) + return readResponse, FallbackToAnotherReader + } + } else { + // Any other error - fallback to GCS + bytesRead = totalRead + cacheHit = false + logger.Warnf("Failed to open chunk %d at path %s: %v, falling back to GCS reader", chunkIndex, chunkPath, openErr) + return readResponse, FallbackToAnotherReader + } + } else { + // Cache hit - chunk was already cached + cacheHit = true + } + defer chunkFile.Close() + + // Calculate exact bytes to read for this request within the chunk + bytesAvailableInChunk := chunkEnd - currentOffset + bytesToRead := min(int64(bufferRemaining), bytesAvailableInChunk) + + // Read only the required bytes from the chunk file at the specific offset + offsetInChunk := currentOffset - chunkStart + n, readErr := chunkFile.ReadAt(p[totalRead:totalRead+int(bytesToRead)], offsetInChunk) + if (readErr != nil && !errors.Is(readErr, io.EOF)) || n < int(bytesToRead) { + bytesRead = totalRead + if n < int(bytesToRead) { + logger.Warnf("Read only %d bytes from chunk %d at path %s, expected %d bytes. falling back to GCS reader.", n, chunkIndex, chunkPath, bytesToRead) + } else { + logger.Warnf("Failed to read chunk %d at path %s: %v, falling back to GCS reader", chunkIndex, chunkPath, readErr) + } + return readResponse, FallbackToAnotherReader + } + + totalRead += n + currentOffset += int64(n) + bufferRemaining -= n + } + + bytesRead = totalRead + if totalRead == 0 && currentOffset >= int64(r.object.Size) { + return readResponse, io.EOF + } + + readResponse.Size = totalRead + return readResponse, nil +} + +// downloadChunk downloads a specific chunk from GCS and caches it atomically. +// This method handles concurrent access and LRU cache eviction race conditions. +// If any cache operation fails, we fallback to reading directly from GCS without caching. +func (r *SharedChunkCacheReader) downloadChunk(ctx context.Context, chunkIndex, chunkStart, chunkEnd int64) error { + objDir := r.manager.GetObjectDir(r.bucket.Name(), r.object.Name, r.object.Generation) + tmpPath := r.manager.GenerateTmpPath(r.bucket.Name(), r.object.Name, r.object.Generation, chunkIndex) + + // Step 1: Create object directory + // Protects against concurrent LRU cache eviction that may have deleted the directory. + // - EEXIST: Ignore, directory already exists (expected) + // - Any other error: Fallback to GCS reader + if err := os.MkdirAll(objDir, r.manager.GetDirPerm()); err != nil { + if !errors.Is(err, syscall.EEXIST) { + return fmt.Errorf("MkDirAll failed: %w", err) + } + } + + // Step 2: Create temporary file with O_EXCL (purely for defensive purposes to handle conflicting + // temporary downloads of the same chunk, given odds of collision are essentially zero with 64-bit + // hash-prefix). + // - ENOENT: Directory was deleted (LRU race), retry once by recreating directory + // - Any other error, including EEXIST (chunk path collision): Fallback to GCS reader + tmpFile, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, r.manager.GetFilePerm()) + if err != nil { + if errors.Is(err, syscall.ENOENT) { + // Directory was deleted by LRU, retry once + if mkdirErr := os.MkdirAll(objDir, r.manager.GetDirPerm()); mkdirErr != nil && !errors.Is(mkdirErr, syscall.EEXIST) { + return fmt.Errorf("MkDirAll retry failed: %w", mkdirErr) + } + // Retry creating temp file + tmpFile, err = os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, r.manager.GetFilePerm()) + if err != nil { + return fmt.Errorf("retry to create tmp file failed: %w", err) + } + } else { + return fmt.Errorf("create temp file failed: %w", err) + } + } + + defer func() { + if tmpFile != nil { + tmpFile.Close() + } + }() + + // Step 3: Create GCS reader for the specific byte range + // Here, generation is important given we store the file under a hash that includes generation. + readReq := &gcs.ReadObjectRequest{ + Name: r.object.Name, + Generation: r.object.Generation, + Range: &gcs.ByteRange{ + Start: uint64(chunkStart), + Limit: uint64(chunkEnd), + }, + } + reader, err := r.bucket.NewReaderWithReadHandle(ctx, readReq) + if err != nil { + os.Remove(tmpPath) // Cleanup + return fmt.Errorf("failed to create GCS reader: %w", err) + } + defer reader.Close() + + // Step 4: Copy data from GCS to temp file + bytesWritten, err := io.Copy(tmpFile, reader) + if err != nil || bytesWritten != (chunkEnd-chunkStart) { + os.Remove(tmpPath) // Cleanup + if err != nil { + return fmt.Errorf("failed to copy data to temp file: %w", err) + } + return fmt.Errorf("incomplete copy, expected %d bytes but wrote %d bytes", (chunkEnd - chunkStart), bytesWritten) + } + + // Close implies Sync() on NFS (intended use case). + if err := tmpFile.Close(); err != nil { + os.Remove(tmpPath) // Cleanup + return fmt.Errorf("failed to close tmpFile: %v", err) + } + tmpFile = nil // Avoid deferred close since file is already closed + + // Step 5: Atomically rename temp file to final location + // Protects against concurrent downloads of the same chunk. + chunkPath := r.manager.GetChunkPath(r.bucket.Name(), r.object.Name, r.object.Generation, chunkIndex) + if err := os.Rename(tmpPath, chunkPath); err != nil { + os.Remove(tmpPath) // Cleanup + return fmt.Errorf("failed to rename temp file: %w", err) + } + + logger.Tracef("Downloaded and cached chunk %d (range %d-%d, %d bytes)", + chunkIndex, chunkStart, chunkEnd, bytesWritten) + + return nil +} + +// CheckInvariants implements the Reader interface. +func (r *SharedChunkCacheReader) CheckInvariants() { +} + +// Destroy implements the Reader interface cleanup. +func (r *SharedChunkCacheReader) Destroy() { + // No resources to clean up for chunk-based reader +} diff --git a/internal/gcsx/shared_chunk_cache_reader_test.go b/internal/gcsx/shared_chunk_cache_reader_test.go new file mode 100644 index 0000000000..ccecc95bd8 --- /dev/null +++ b/internal/gcsx/shared_chunk_cache_reader_test.go @@ -0,0 +1,699 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx + +import ( + "bytes" + "context" + "io" + "os" + "sync" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/file" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + "github.com/googlecloudplatform/gcsfuse/v3/tracing" + "github.com/jacobsa/timeutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +const ( + testBucketName = "test-bucket" + testObjectName = "test-object.txt" + testObjectSize = 1024 * 1024 // 1 MB + testChunkSizeMB = 1 // 1 MB chunks (not 256KB - config is in MB) +) + +type sharedChunkCacheReaderTest struct { + suite.Suite + ctx context.Context + cacheDir string + manager *file.SharedChunkCacheManager + bucket gcs.Bucket + object *gcs.MinObject + reader *SharedChunkCacheReader + objectData []byte +} + +func TestSharedChunkCacheReaderTestSuite(t *testing.T) { + suite.Run(t, new(sharedChunkCacheReaderTest)) +} + +func (t *sharedChunkCacheReaderTest) SetupTest() { + // Arrange - Create test fixtures + t.ctx = context.Background() + t.cacheDir = t.T().TempDir() + + // Create manager with 1 MB chunk size for testing + config := &cfg.FileCacheConfig{ + SharedCacheChunkSizeMb: testChunkSizeMB, + } + var err error + t.manager, err = file.NewSharedChunkCacheManager(t.cacheDir, 0644, 0755, config) + require.NoError(t.T(), err) + + // Create fake bucket with test data + t.bucket = fake.NewFakeBucket(timeutil.RealClock(), testBucketName, gcs.BucketType{}) + + // Create test object data + t.objectData = make([]byte, testObjectSize) + for i := range t.objectData { + t.objectData[i] = byte(i % 256) + } + + // Create object in fake bucket + createReq := &gcs.CreateObjectRequest{ + Name: testObjectName, + Contents: io.NopCloser(bytes.NewReader(t.objectData)), + } + createdObj, err := t.bucket.CreateObject(t.ctx, createReq) + require.NoError(t.T(), err) + + t.object = &gcs.MinObject{ + Name: createdObj.Name, + Size: createdObj.Size, + Generation: createdObj.Generation, + } + + // Create reader + t.reader = NewSharedChunkCacheReader( + t.manager, + t.bucket, + t.object, + metrics.NewNoopMetrics(), + tracing.NewNoopTracer(), + 0, + ) +} + +func (t *sharedChunkCacheReaderTest) TearDownTest() { + if t.cacheDir != "" { + os.RemoveAll(t.cacheDir) + } +} + +func (t *sharedChunkCacheReaderTest) TestNewSharedChunkCacheReader() { + // Arrange + expectedBucketName := testBucketName + expectedObjectName := t.object.Name + expectedGeneration := t.object.Generation + expectedSize := t.object.Size + + // Act - Reader is already created in SetupTest + actualReader := t.reader + + // Assert + assert.NotNil(t.T(), actualReader) + assert.Equal(t.T(), expectedBucketName, actualReader.bucket.Name()) + assert.Equal(t.T(), expectedObjectName, actualReader.object.Name) + assert.Equal(t.T(), expectedGeneration, actualReader.object.Generation) + assert.Equal(t.T(), expectedSize, actualReader.object.Size) +} + +func (t *sharedChunkCacheReaderTest) TestReadAt_SingleChunk() { + // Arrange + buffer := make([]byte, 100) + req := &ReadRequest{ + Offset: 0, + Buffer: buffer, + } + + // Act + resp, err := t.reader.ReadAt(t.ctx, req) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), 100, resp.Size) + assert.Equal(t.T(), t.objectData[:100], buffer[:100]) +} + +func (t *sharedChunkCacheReaderTest) TestReadAt_CacheHit() { + // Arrange - First read to populate cache + buffer1 := make([]byte, 100) + req1 := &ReadRequest{Offset: 0, Buffer: buffer1} + _, err := t.reader.ReadAt(t.ctx, req1) + require.NoError(t.T(), err) + chunkPath := t.manager.GetChunkPath(testBucketName, t.object.Name, t.object.Generation, 0) + require.FileExists(t.T(), chunkPath, "Chunk should be cached after first read") + // Capture chunk file info after first read (cache miss) + fileInfoBeforeCacheHit, err := os.Stat(chunkPath) + require.NoError(t.T(), err) + modTimeBeforeCacheHit := fileInfoBeforeCacheHit.ModTime() + // Arrange - Prepare second read from cached chunk + buffer2 := make([]byte, 100) + req2 := &ReadRequest{Offset: 50, Buffer: buffer2} + expectedData := t.objectData[50:150] + + // Act - Second read should be a cache hit + resp, err := t.reader.ReadAt(t.ctx, req2) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), 100, resp.Size) + assert.Equal(t.T(), expectedData, buffer2[:100]) + // Verify cache hit - chunk file should not have been modified + fileInfoAfterCacheHit, err := os.Stat(chunkPath) + require.NoError(t.T(), err) + modTimeAfterCacheHit := fileInfoAfterCacheHit.ModTime() + assert.Equal(t.T(), modTimeBeforeCacheHit, modTimeAfterCacheHit, "Cache hit: chunk file should not be re-downloaded") +} + +func (t *sharedChunkCacheReaderTest) TestReadAt_AcrossChunkBoundary() { + // Arrange + largeObjectData := make([]byte, 3*1024*1024) + for i := range largeObjectData { + largeObjectData[i] = byte(i % 256) + } + createReq := &gcs.CreateObjectRequest{ + Name: "large-object.txt", + Contents: io.NopCloser(bytes.NewReader(largeObjectData)), + } + createdObj, err := t.bucket.CreateObject(t.ctx, createReq) + require.NoError(t.T(), err) + largeObject := &gcs.MinObject{ + Name: createdObj.Name, + Size: createdObj.Size, + Generation: createdObj.Generation, + } + largeReader := NewSharedChunkCacheReader( + t.manager, + t.bucket, + largeObject, + metrics.NewNoopMetrics(), + tracing.NewNoopTracer(), + 0, + ) + chunkSize := int(t.manager.GetChunkSize()) + buffer := make([]byte, 2000) + offset := int64(chunkSize - 1000) + req := &ReadRequest{ + Offset: offset, + Buffer: buffer, + } + expectedData := largeObjectData[offset : offset+2000] + + // Act + resp, err := largeReader.ReadAt(t.ctx, req) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), 2000, resp.Size) + assert.Equal(t.T(), expectedData, buffer[:2000]) + chunk0Path := t.manager.GetChunkPath(testBucketName, largeObject.Name, largeObject.Generation, 0) + chunk1Path := t.manager.GetChunkPath(testBucketName, largeObject.Name, largeObject.Generation, 1) + assert.FileExists(t.T(), chunk0Path, "First chunk should be cached") + assert.FileExists(t.T(), chunk1Path, "Second chunk should be cached") +} + +func (t *sharedChunkCacheReaderTest) TestReadAt_EOF() { + // Arrange + buffer := make([]byte, 100) + offsetAtEOF := int64(t.object.Size) + req := &ReadRequest{ + Offset: offsetAtEOF, + Buffer: buffer, + } + + // Act + _, err := t.reader.ReadAt(t.ctx, req) + + // Assert + assert.Equal(t.T(), io.EOF, err, "Reading at EOF should return io.EOF") +} + +func (t *sharedChunkCacheReaderTest) TestReadAt_NegativeOffset() { + // Arrange + buffer := make([]byte, 100) + req := &ReadRequest{ + Offset: -10, + Buffer: buffer, + } + + // Act + _, err := t.reader.ReadAt(t.ctx, req) + + // Assert + assert.Error(t.T(), err) + assert.Contains(t.T(), err.Error(), "negative offset") +} + +func (t *sharedChunkCacheReaderTest) TestReadAt_PartialRead() { + // Arrange + buffer := make([]byte, 1000) + offset := int64(t.object.Size - 100) + req := &ReadRequest{ + Offset: offset, + Buffer: buffer, + } + expectedSize := 100 + expectedData := t.objectData[offset:] + + // Act + resp, err := t.reader.ReadAt(t.ctx, req) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), expectedSize, resp.Size, "Should read only remaining 100 bytes") + assert.Equal(t.T(), expectedData, buffer[:100]) +} + +func (t *sharedChunkCacheReaderTest) TestReadAt_ExcludedByRegex() { + // Arrange + excludeConfig := &cfg.FileCacheConfig{ + ExcludeRegex: ".*\\.txt$", + SharedCacheChunkSizeMb: testChunkSizeMB, + } + excludemanager, err := file.NewSharedChunkCacheManager(t.cacheDir, 0644, 0755, excludeConfig) + require.NoError(t.T(), err) + excludeReader := NewSharedChunkCacheReader( + excludemanager, + t.bucket, + t.object, + metrics.NewNoopMetrics(), + tracing.NewNoopTracer(), + 0, + ) + buffer := make([]byte, 100) + req := &ReadRequest{Offset: 0, Buffer: buffer} + + // Act + _, err = excludeReader.ReadAt(t.ctx, req) + + // Assert + assert.ErrorIs(t.T(), err, FallbackToAnotherReader, "Files matching exclude regex should fallback to another reader") +} + +func (t *sharedChunkCacheReaderTest) TestReadAt_MultipleChunks() { + // Arrange + largeObjectData := make([]byte, 5*1024*1024) + for i := range largeObjectData { + largeObjectData[i] = byte(i % 256) + } + createReq := &gcs.CreateObjectRequest{ + Name: "multi-chunk-object.txt", + Contents: io.NopCloser(bytes.NewReader(largeObjectData)), + } + createdObj, err := t.bucket.CreateObject(t.ctx, createReq) + require.NoError(t.T(), err) + largeObject := &gcs.MinObject{ + Name: createdObj.Name, + Size: createdObj.Size, + Generation: createdObj.Generation, + } + largeReader := NewSharedChunkCacheReader( + t.manager, + t.bucket, + largeObject, + metrics.NewNoopMetrics(), + tracing.NewNoopTracer(), + 0, + ) + buffer := make([]byte, largeObject.Size) + req := &ReadRequest{ + Offset: 0, + Buffer: buffer, + } + expectedSize := int(largeObject.Size) + expectedData := largeObjectData + chunkSize := int(t.manager.GetChunkSize()) + expectedNumChunks := (int(largeObject.Size) + chunkSize - 1) / chunkSize + + // Act + resp, err := largeReader.ReadAt(t.ctx, req) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), expectedSize, resp.Size) + assert.Equal(t.T(), expectedData, buffer) + for i := range expectedNumChunks { + chunkPath := t.manager.GetChunkPath(testBucketName, largeObject.Name, largeObject.Generation, int64(i)) + assert.FileExists(t.T(), chunkPath, "Chunk %d should be cached", i) + } +} + +func (t *sharedChunkCacheReaderTest) TestReadAt_ZeroLengthRead() { + // Arrange + buffer := make([]byte, 0) + req := &ReadRequest{ + Offset: 0, + Buffer: buffer, + } + + // Act + resp, err := t.reader.ReadAt(t.ctx, req) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), 0, resp.Size) +} + +func TestSharedChunkCacheReader_ConcurrentReads(t *testing.T) { + // Arrange + cacheDir := t.TempDir() + config := &cfg.FileCacheConfig{ + SharedCacheChunkSizeMb: 1, + } + manager, err := file.NewSharedChunkCacheManager(cacheDir, 0644, 0755, config) + require.NoError(t, err) + bucket := fake.NewFakeBucket(timeutil.RealClock(), testBucketName, gcs.BucketType{}) + objectData := make([]byte, 5*1024*1024) + for i := range objectData { + objectData[i] = byte(i % 256) + } + ctx := context.Background() + createReq := &gcs.CreateObjectRequest{ + Name: testObjectName, + Contents: io.NopCloser(bytes.NewReader(objectData)), + } + createdObj, err := bucket.CreateObject(ctx, createReq) + require.NoError(t, err) + object := &gcs.MinObject{ + Name: createdObj.Name, + Size: createdObj.Size, + Generation: createdObj.Generation, + } + reader := NewSharedChunkCacheReader( + manager, + bucket, + object, + metrics.NewNoopMetrics(), + tracing.NewNoopTracer(), + 0, + ) + wg := sync.WaitGroup{} + const numGoroutines = 5 + wg.Add(numGoroutines) + + // Act - Launch concurrent reads + for i := range numGoroutines { + go func(offset int64) { + defer wg.Done() + buffer := make([]byte, 1000) + req := &ReadRequest{Offset: offset, Buffer: buffer} + resp, readErr := reader.ReadAt(ctx, req) + assert.NoError(t, readErr) + assert.Equal(t, 1000, resp.Size) + assert.Equal(t, objectData[offset:offset+1000], buffer) + }(int64(i * 100000)) + } + + // Assert - Wait for all goroutines to complete successfully + wg.Wait() + assert.True(t, true, "All concurrent reads completed successfully") +} + +func TestSharedChunkCacheReader_ChunkRaceCondition(t *testing.T) { + // Arrange + cacheDir := t.TempDir() + config := &cfg.FileCacheConfig{ + SharedCacheChunkSizeMb: 1, + } + manager, err := file.NewSharedChunkCacheManager(cacheDir, 0644, 0755, config) + require.NoError(t, err) + bucket := fake.NewFakeBucket(timeutil.RealClock(), testBucketName, gcs.BucketType{}) + objectData := make([]byte, 2*1024*1024) + for i := range objectData { + objectData[i] = byte(i % 256) + } + ctx := context.Background() + createReq := &gcs.CreateObjectRequest{ + Name: testObjectName, + Contents: io.NopCloser(bytes.NewReader(objectData)), + } + createdObj, err := bucket.CreateObject(ctx, createReq) + require.NoError(t, err) + object := &gcs.MinObject{ + Name: createdObj.Name, + Size: createdObj.Size, + Generation: createdObj.Generation, + } + reader := NewSharedChunkCacheReader( + manager, + bucket, + object, + metrics.NewNoopMetrics(), + tracing.NewNoopTracer(), + 0, + ) + const numGoroutines = 5 + wg := sync.WaitGroup{} + wg.Add(numGoroutines) + expectedChunkPath := manager.GetChunkPath(testBucketName, object.Name, object.Generation, 0) + + // Act - Launch concurrent reads to same chunk + for i := 0; i < numGoroutines; i++ { + go func() { + defer wg.Done() + buffer := make([]byte, 100) + req := &ReadRequest{Offset: 50, Buffer: buffer} + resp, readErr := reader.ReadAt(ctx, req) + assert.NoError(t, readErr) + assert.Equal(t, 100, resp.Size) + assert.Equal(t, objectData[50:150], buffer) + }() + } + + // Assert - All goroutines should complete and chunk should be cached once + wg.Wait() + assert.FileExists(t, expectedChunkPath, "Chunk should be cached exactly once despite race condition") +} + +// TestSharedChunkCacheReader_DownloadChunkWithDeletedDirectory tests that the system +// handles directory deletion gracefully, recovering and completing the download successfully. +// This verifies resilience against concurrent LRU cache eviction scenarios. +func TestSharedChunkCacheReader_DownloadChunkWithDeletedDirectory(t *testing.T) { + // Arrange + cacheDir := t.TempDir() + config := &cfg.FileCacheConfig{ + SharedCacheChunkSizeMb: 1, + } + manager, err := file.NewSharedChunkCacheManager(cacheDir, 0644, 0755, config) + require.NoError(t, err) + bucket := fake.NewFakeBucket(timeutil.RealClock(), testBucketName, gcs.BucketType{}) + objectData := make([]byte, 2*1024*1024) // 2 MB + for i := range objectData { + objectData[i] = byte(i % 256) + } + ctx := context.Background() + createReq := &gcs.CreateObjectRequest{ + Name: testObjectName, + Contents: io.NopCloser(bytes.NewReader(objectData)), + } + createdObj, err := bucket.CreateObject(ctx, createReq) + require.NoError(t, err) + object := &gcs.MinObject{ + Name: createdObj.Name, + Size: createdObj.Size, + Generation: createdObj.Generation, + } + reader := NewSharedChunkCacheReader( + manager, + bucket, + object, + metrics.NewNoopMetrics(), + tracing.NewNoopTracer(), + 0, + ) + // Pre-delete the cache directory structure to simulate LRU eviction + // The download should handle this and recreate necessary directories + objDir := manager.GetObjectDir(testBucketName, object.Name, object.Generation) + os.RemoveAll(objDir) + + // Act - Trigger download with missing directory structure + buffer := make([]byte, 1024) + req := &ReadRequest{ + Offset: 0, + Buffer: buffer, + } + resp, err := reader.ReadAt(ctx, req) + + // Assert - Should succeed by recreating necessary directories + assert.NoError(t, err, "Should handle missing directories gracefully") + assert.Equal(t, 1024, resp.Size) + assert.Equal(t, objectData[:1024], buffer) + // Verify chunk was successfully cached after recreating directories + chunkPath := manager.GetChunkPath(testBucketName, object.Name, object.Generation, 0) + assert.FileExists(t, chunkPath) +} + +// TestSharedChunkCacheReader_ReadAtFallbackOnDownloadError tests that ReadAt falls back +// to GCS reader when downloadChunk fails after retry. +func TestSharedChunkCacheReader_ReadAtFallbackOnDownloadError(t *testing.T) { + // Arrange + cacheDir := t.TempDir() + // Make cache directory read-only to cause download failures + err := os.Chmod(cacheDir, 0444) + require.NoError(t, err) + defer func() { + _ = os.Chmod(cacheDir, 0755) // Restore for cleanup + }() + config := &cfg.FileCacheConfig{ + SharedCacheChunkSizeMb: 1, + } + manager, err := file.NewSharedChunkCacheManager(cacheDir, 0644, 0755, config) + require.NoError(t, err) + bucket := fake.NewFakeBucket(timeutil.RealClock(), testBucketName, gcs.BucketType{}) + objectData := make([]byte, 1024*1024) // 1 MB + for i := range objectData { + objectData[i] = byte(i % 256) + } + ctx := context.Background() + createReq := &gcs.CreateObjectRequest{ + Name: testObjectName, + Contents: io.NopCloser(bytes.NewReader(objectData)), + } + createdObj, err := bucket.CreateObject(ctx, createReq) + require.NoError(t, err) + object := &gcs.MinObject{ + Name: createdObj.Name, + Size: createdObj.Size, + Generation: createdObj.Generation, + } + reader := NewSharedChunkCacheReader( + manager, + bucket, + object, + metrics.NewNoopMetrics(), + tracing.NewNoopTracer(), + 0, + ) + + // Act - ReadAt should fail to cache and return FallbackToAnotherReader error + buffer := make([]byte, 1024) + req := &ReadRequest{ + Offset: 0, + Buffer: buffer, + } + resp, err := reader.ReadAt(ctx, req) + + // Assert - Should return FallbackToAnotherReader due to permission denied on directory + assert.Error(t, err) + assert.ErrorIs(t, err, FallbackToAnotherReader) + assert.Equal(t, 0, resp.Size) +} + +// TestSharedChunkCacheReader_ReadAtWithCorruptedCache tests ReadAt behavior when +// cached chunk is corrupted (partial/incomplete). +func TestSharedChunkCacheReader_ReadAtWithCorruptedCache(t *testing.T) { + // Arrange + cacheDir := t.TempDir() + config := &cfg.FileCacheConfig{ + SharedCacheChunkSizeMb: 1, + } + manager, err := file.NewSharedChunkCacheManager(cacheDir, 0644, 0755, config) + require.NoError(t, err) + + bucket := fake.NewFakeBucket(timeutil.RealClock(), testBucketName, gcs.BucketType{}) + objectData := make([]byte, 2*1024*1024) // 2 MB + for i := range objectData { + objectData[i] = byte(i % 256) + } + ctx := context.Background() + createReq := &gcs.CreateObjectRequest{ + Name: testObjectName, + Contents: io.NopCloser(bytes.NewReader(objectData)), + } + createdObj, err := bucket.CreateObject(ctx, createReq) + require.NoError(t, err) + object := &gcs.MinObject{ + Name: createdObj.Name, + Size: createdObj.Size, + Generation: createdObj.Generation, + } + reader := NewSharedChunkCacheReader( + manager, + bucket, + object, + metrics.NewNoopMetrics(), + tracing.NewNoopTracer(), + 0, + ) + // First, populate the cache normally + buffer := make([]byte, 1024) + req := &ReadRequest{ + Offset: 0, + Buffer: buffer, + } + resp, err := reader.ReadAt(ctx, req) + require.NoError(t, err) + require.Equal(t, 1024, resp.Size) + // Corrupt the cached chunk by truncating it + chunkPath := manager.GetChunkPath(testBucketName, object.Name, object.Generation, 0) + chunkFile, err := os.OpenFile(chunkPath, os.O_WRONLY|os.O_TRUNC, 0644) + require.NoError(t, err) + // Write only 512 bytes instead of full chunk + _, err = chunkFile.Write(objectData[:512]) + require.NoError(t, err) + chunkFile.Close() + + // Act - Try to read from corrupted cache (should fallback to GCS) + buffer2 := make([]byte, 1024) + req2 := &ReadRequest{ + Offset: 512, // Read beyond corrupted data + Buffer: buffer2, + } + resp2, err2 := reader.ReadAt(ctx, req2) + + // Assert - Should detect corruption and fallback + assert.Error(t, err2) + assert.ErrorIs(t, err2, FallbackToAnotherReader) + assert.Equal(t, 0, resp2.Size) +} + +// TestSharedChunkCacheReader_DownloadChunkGCSError tests downloadChunk behavior +// when GCS reader creation fails. +func TestSharedChunkCacheReader_DownloadChunkGCSError(t *testing.T) { + // Arrange + cacheDir := t.TempDir() + config := &cfg.FileCacheConfig{ + SharedCacheChunkSizeMb: 1, + } + manager, err := file.NewSharedChunkCacheManager(cacheDir, 0644, 0755, config) + require.NoError(t, err) + bucket := fake.NewFakeBucket(timeutil.RealClock(), testBucketName, gcs.BucketType{}) + ctx := context.Background() + // Create object that doesn't exist in bucket + object := &gcs.MinObject{ + Name: "nonexistent-object.txt", + Size: 1024 * 1024, + Generation: 12345, + } + reader := NewSharedChunkCacheReader( + manager, + bucket, + object, + metrics.NewNoopMetrics(), + tracing.NewNoopTracer(), + 0, + ) + + // Act - Try to read non-existent object + buffer := make([]byte, 1024) + req := &ReadRequest{ + Offset: 0, + Buffer: buffer, + } + resp, err := reader.ReadAt(ctx, req) + + // Assert - Should fail and return FallbackToAnotherReader + assert.Error(t, err) + assert.ErrorIs(t, err, FallbackToAnotherReader) + assert.Equal(t, 0, resp.Size) +} diff --git a/internal/gcsx/syncer.go b/internal/gcsx/syncer.go index 96804ad21c..c9ea91fb19 100644 --- a/internal/gcsx/syncer.go +++ b/internal/gcsx/syncer.go @@ -19,15 +19,10 @@ import ( "io" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "golang.org/x/net/context" ) -// MtimeMetadataKey objects are created by Syncer.SyncObject and contain a -// metadata field with this key and with a UTC mtime in the format defined -// by time.RFC3339Nano. -const MtimeMetadataKey = "gcsfuse_mtime" - // Syncer is safe for concurrent access. type Syncer interface { // Given an object record and content that was originally derived from that @@ -47,14 +42,16 @@ type Syncer interface { // NewSyncer creates a syncer that syncs into the supplied bucket. // // When the source object has been changed only by appending, and the source -// object's size is at least appendThreshold, we will "append" to it by writing +// object's size is at least composeThreshold, we will "append" to it by writing // out a temporary blob and composing it with the source object. // // Temporary blobs have names beginning with tmpObjectPrefix. We make an effort // to delete them, but if we are interrupted for some reason we may not be able // to do so. Therefore the user should arrange for garbage collection. func NewSyncer( - appendThreshold int64, + composeThreshold int64, + chunkRetryDeadlineSecs int64, + chunkTransferTimeoutSecs int64, tmpObjectPrefix string, bucket gcs.Bucket) (os Syncer) { // Create the object creators. @@ -62,12 +59,17 @@ func NewSyncer( bucket: bucket, } - appendCreator := newAppendObjectCreator( - tmpObjectPrefix, - bucket) + // Rapid buckets do not currently support Compose, + // so we always write objects in their entirety. + var composeCreator objectCreator + if !bucket.BucketType().RapidWritesEnabled() { + composeCreator = newComposeObjectCreator( + tmpObjectPrefix, + bucket) + } // And the syncer. - os = newSyncer(appendThreshold, fullCreator, appendCreator) + os = newSyncer(composeThreshold, chunkRetryDeadlineSecs, chunkTransferTimeoutSecs, fullCreator, composeCreator) return } @@ -85,44 +87,11 @@ func (oc *fullObjectCreator) Create( objectName string, srcObject *gcs.Object, mtime *time.Time, + chunkRetryDeadlineSecs int64, + chunkTransferTimeoutSecs int64, r io.Reader) (o *gcs.Object, err error) { - metadataMap := make(map[string]string) - - var req *gcs.CreateObjectRequest - if srcObject == nil { - var precond int64 - req = &gcs.CreateObjectRequest{ - Name: objectName, - Contents: r, - GenerationPrecondition: &precond, - Metadata: metadataMap, - } - } else { - for key, value := range srcObject.Metadata { - metadataMap[key] = value - } - - req = &gcs.CreateObjectRequest{ - Name: srcObject.Name, - GenerationPrecondition: &srcObject.Generation, - MetaGenerationPrecondition: &srcObject.MetaGeneration, - Contents: r, - Metadata: metadataMap, - CacheControl: srcObject.CacheControl, - ContentDisposition: srcObject.ContentDisposition, - ContentEncoding: srcObject.ContentEncoding, - ContentType: srcObject.ContentType, - CustomTime: srcObject.CustomTime, - EventBasedHold: srcObject.EventBasedHold, - StorageClass: srcObject.StorageClass, - } - } - - // Any existing mtime value will be overwritten with new value. - if mtime != nil { - metadataMap[MtimeMetadataKey] = mtime.UTC().Format(time.RFC3339Nano) - } - + req := gcs.NewCreateObjectRequest(srcObject, objectName, mtime, chunkRetryDeadlineSecs, chunkTransferTimeoutSecs) + req.Contents = r o, err = oc.bucket.CreateObject(ctx, req) if err != nil { err = fmt.Errorf("CreateObject: %w", err) @@ -143,6 +112,8 @@ type objectCreator interface { objectName string, srcObject *gcs.Object, mtime *time.Time, + chunkRetryDeadlineSecs int64, + chunkTransferTimeoutSecs int64, r io.Reader) (o *gcs.Object, err error) } @@ -152,30 +123,36 @@ type objectCreator interface { // - fullCreator accepts the source object and the full contents with which it // should be overwritten. // -// - appendCreator accepts the source object and the contents that should be +// - composeCreator accepts the source object and the contents that should be // "appended" to it. // -// appendThreshold controls the source object length at which we consider it +// composeThreshold controls the source object length at which we consider it // worthwhile to make the append optimization. It should be set to a value on // the order of the bandwidth to GCS times three times the round trip latency // to GCS (for a small create, a compose, and a delete). func newSyncer( - appendThreshold int64, + composeThreshold int64, + chunkRetryDeadlineSecs int64, + chunkTransferTimeoutSecs int64, fullCreator objectCreator, - appendCreator objectCreator) (os Syncer) { + composeCreator objectCreator) (os Syncer) { os = &syncer{ - appendThreshold: appendThreshold, - fullCreator: fullCreator, - appendCreator: appendCreator, + composeThreshold: composeThreshold, + chunkRetryDeadlineSecs: chunkRetryDeadlineSecs, + chunkTransferTimeoutSecs: chunkTransferTimeoutSecs, + fullCreator: fullCreator, + composeCreator: composeCreator, } return } type syncer struct { - appendThreshold int64 - fullCreator objectCreator - appendCreator objectCreator + composeThreshold int64 + chunkRetryDeadlineSecs int64 + chunkTransferTimeoutSecs int64 + fullCreator objectCreator + composeCreator objectCreator } func (os *syncer) SyncObject( @@ -200,7 +177,7 @@ func (os *syncer) SyncObject( err = fmt.Errorf("error in seeking: %w", err) return } - return os.fullCreator.Create(ctx, objectName, srcObject, sr.Mtime, content) + return os.fullCreator.Create(ctx, objectName, srcObject, sr.Mtime, os.chunkRetryDeadlineSecs, os.chunkTransferTimeoutSecs, content) } // Make sure the dirty threshold makes sense. @@ -231,7 +208,7 @@ func (os *syncer) SyncObject( // Otherwise, we need to create a new generation. If the source object is // long enough, hasn't been dirtied, and has a low enough component count, // then we can make the optimization of not rewriting its contents. - if srcSize >= os.appendThreshold && + if os.composeCreator != nil && srcSize >= os.composeThreshold && sr.DirtyThreshold == srcSize && srcObject.ComponentCount < gcs.MaxComponentCount { _, err = content.Seek(srcSize, 0) @@ -240,7 +217,7 @@ func (os *syncer) SyncObject( return } - o, err = os.appendCreator.Create(ctx, objectName, srcObject, sr.Mtime, content) + o, err = os.composeCreator.Create(ctx, objectName, srcObject, sr.Mtime, os.chunkRetryDeadlineSecs, os.chunkTransferTimeoutSecs, content) } else { _, err = content.Seek(0, 0) if err != nil { @@ -248,7 +225,7 @@ func (os *syncer) SyncObject( return } - o, err = os.fullCreator.Create(ctx, objectName, srcObject, sr.Mtime, content) + o, err = os.fullCreator.Create(ctx, objectName, srcObject, sr.Mtime, os.chunkRetryDeadlineSecs, os.chunkTransferTimeoutSecs, content) } // Deal with errors. diff --git a/internal/gcsx/syncer_bucket.go b/internal/gcsx/syncer_bucket.go index ae77832285..8806a216bd 100644 --- a/internal/gcsx/syncer_bucket.go +++ b/internal/gcsx/syncer_bucket.go @@ -15,7 +15,7 @@ package gcsx import ( - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" ) type SyncerBucket struct { @@ -27,9 +27,11 @@ type SyncerBucket struct { // a gcs.Bucket, or as a Syncer. func NewSyncerBucket( appendThreshold int64, + chunkRetryDeadlineSecs int64, + chunkTransferTimeoutSecs int64, tmpObjectPrefix string, bucket gcs.Bucket, ) SyncerBucket { - syncer := NewSyncer(appendThreshold, tmpObjectPrefix, bucket) + syncer := NewSyncer(appendThreshold, chunkRetryDeadlineSecs, chunkTransferTimeoutSecs, tmpObjectPrefix, bucket) return SyncerBucket{bucket, syncer} } diff --git a/internal/gcsx/syncer_test.go b/internal/gcsx/syncer_test.go index b587542d46..ab74e8c202 100644 --- a/internal/gcsx/syncer_test.go +++ b/internal/gcsx/syncer_test.go @@ -21,9 +21,9 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/fake" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/oglemock" . "github.com/jacobsa/ogletest" @@ -67,6 +67,8 @@ func (t *FullObjectCreatorTest) call() (o *gcs.Object, err error) { t.srcObject.Name, &t.srcObject, &t.mtime, + chunkRetryDeadlineSecs, + chunkTransferTimeoutSecs, strings.NewReader(t.srcContents)) return @@ -175,6 +177,8 @@ func (t *FullObjectCreatorTest) CallsCreateObjectWhenSrcObjectIsNil() { t.srcObject.Name, nil, &t.mtime, + chunkRetryDeadlineSecs, + chunkTransferTimeoutSecs, strings.NewReader(t.srcContents)) t.validateEmptyProperties(req) @@ -194,6 +198,8 @@ func (t *FullObjectCreatorTest) CallsCreateObjectWhenSrcObjectAndMtimeAreNil() { t.srcObject.Name, nil, nil, + chunkRetryDeadlineSecs, + chunkTransferTimeoutSecs, strings.NewReader(t.srcContents)) t.validateEmptyProperties(req) @@ -243,6 +249,8 @@ func (oc *fakeObjectCreator) Create( fileName string, srcObject *gcs.Object, mtime *time.Time, + chunkRetryDeadlineSecs int64, + chunkTransferTimeoutSecs int64, r io.Reader) (o *gcs.Object, err error) { // Have we been called more than once? AssertFalse(oc.called) @@ -267,6 +275,8 @@ func (oc *fakeObjectCreator) Create( const srcObjectContents = "taco" const appendThreshold = int64(len(srcObjectContents)) +const chunkRetryDeadlineSecs = 120 +const chunkTransferTimeoutSecs = 10 type SyncerTest struct { ctx context.Context @@ -291,9 +301,11 @@ func (t *SyncerTest) SetUp(ti *TestInfo) { t.ctx = ti.Ctx // Set up dependencies. - t.bucket = fake.NewFakeBucket(&t.clock, "some_bucket", gcs.NonHierarchical) + t.bucket = fake.NewFakeBucket(&t.clock, "some_bucket", gcs.BucketType{}) t.syncer = newSyncer( appendThreshold, + chunkRetryDeadlineSecs, + chunkTransferTimeoutSecs, &t.fullCreator, &t.appendCreator) @@ -413,6 +425,8 @@ func (t *SyncerTest) SourceTooShortForAppend() { // Recreate the syncer with a higher append threshold. t.syncer = newSyncer( int64(len(srcObjectContents)+1), + chunkRetryDeadlineSecs, + chunkTransferTimeoutSecs, &t.fullCreator, &t.appendCreator) diff --git a/internal/gcsx/temp_file.go b/internal/gcsx/temp_file.go index b4e3b15ce1..0bfe5ac1f9 100644 --- a/internal/gcsx/temp_file.go +++ b/internal/gcsx/temp_file.go @@ -326,7 +326,7 @@ func minInt64(a int64, b int64) int64 { } const ( - minCopyLength = 64 * 1024 * 1024 // 64 MB + minCopyLength = 64 * 1024 * 1024 // 64 MiB ) func (tf *tempFile) ensure(limit int64) error { @@ -336,10 +336,7 @@ func (tf *tempFile) ensure(limit int64) error { if size >= limit { return nil } - n := limit - size - if n < minCopyLength { - n = minCopyLength - } + n := max(limit-size, minCopyLength) n, err = io.CopyN(tf.f, tf.source, n) if err == io.EOF { tf.source.Close() diff --git a/internal/gcsx/temp_file_test.go b/internal/gcsx/temp_file_test.go index 461e36f088..aa39bf21ef 100644 --- a/internal/gcsx/temp_file_test.go +++ b/internal/gcsx/temp_file_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/ogletest" "github.com/jacobsa/timeutil" diff --git a/internal/kernelparams/contract.go b/internal/kernelparams/contract.go new file mode 100644 index 0000000000..2bb775b034 --- /dev/null +++ b/internal/kernelparams/contract.go @@ -0,0 +1,77 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kernelparams + +import ( + "time" + + "github.com/google/uuid" +) + +// Package kernelparams defines the strict schema and contract used for inter-process +// communication between the GCSFuse Sidecar and the GKE GCSFuse CSI Driver. +// +// Purpose: +// This package facilitates the "Zero Configuration" feature where GCSFuse automatically +// determines optimal kernel settings (e.g., for Zonal Buckets) and communicates them +// to the CSI Driver for enforcement. +// +// +// CRITICAL: +// This file acts as a shared contract. Any changes here must be compatible with +// both the producer (GCSFuse) and the consumer (CSI Driver). +// +// BREAKING CHANGES: +// 1. Renaming any JSON tag (e.g., changing `json:"request_id"` to `json:"id"`). +// 2. Removing an existing field from a struct. +// 3. Changing the data type of a field (e.g., string to int). +// 4. Changing the string value of an existing ParamName constant. +// +// NON-BREAKING CHANGES: +// 1. Adding a new field with a new JSON tag. +// 2. Adding a new ParamName constant. +// Follow this guide to make any changes to this contract: TODO(mohit) + +// ParamName acts as an Enum for the parameter keys to ensure contract safety from typo errors. +type ParamName string + +const ( + MaxPagesLimit ParamName = "fuse-max-pages-limit" + TransparentHugePages ParamName = "transparent-hugepages" + MaxReadAheadKb ParamName = "max-read-ahead-kb" + MaxBackgroundRequests ParamName = "fuse-max-background-requests" + CongestionWindowThreshold ParamName = "fuse-congestion-window-threshold" +) + +// KernelParam represents an individual parameter setting. +type KernelParam struct { + Name ParamName `json:"name"` + Value string `json:"value"` +} + +// KernelParamsConfig acts as the primary container for kernel settings. +type KernelParamsConfig struct { + RequestID string `json:"request_id"` + Timestamp string `json:"timestamp"` // Format: 2026-01-12T16:23:05.636831Z + Parameters []KernelParam `json:"parameters"` +} + +// newKernelParamsConfig initializes a new configuration container with a request UUID, CurrentContractVersion and Timestamp. +func newKernelParamsConfig() *KernelParamsConfig { + return &KernelParamsConfig{ + RequestID: uuid.NewString(), + Timestamp: time.Now().Format(time.RFC3339Nano), + } +} diff --git a/internal/kernelparams/kernelparams.go b/internal/kernelparams/kernelparams.go new file mode 100644 index 0000000000..674882e2b4 --- /dev/null +++ b/internal/kernelparams/kernelparams.go @@ -0,0 +1,260 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kernelparams + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "syscall" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "golang.org/x/sys/unix" +) + +// KernelParamsManager wraps KernelParamsConfig with a mutex to ensure thread safety. +type KernelParamsManager struct { + *KernelParamsConfig + mu sync.Mutex +} + +// NewKernelParamsManager creates a new thread-safe configuration manager. +func NewKernelParamsManager() *KernelParamsManager { + return &KernelParamsManager{ + KernelParamsConfig: newKernelParamsConfig(), + } +} + +// getDeviceMajorMinor returns the major and minor device numbers +// for the filesystem mounted at the given mountPoint. +func getDeviceMajorMinor(mountPoint string) (major uint32, minor uint32, err error) { + if runtime.GOOS != "linux" { + return 0, 0, fmt.Errorf("unsupported OS: %s, device major/minor lookup is linux-specific", runtime.GOOS) + } + + fileInfo, err := os.Stat(mountPoint) + if err != nil { + err = fmt.Errorf("os.Stat: %w", err) + return + } + + stat, ok := fileInfo.Sys().(*syscall.Stat_t) + if !ok { + err = fmt.Errorf("fileInfo.Sys() is not of type *syscall.Stat_t") + return + } + + devID := stat.Dev + major = unix.Major(uint64(devID)) + minor = unix.Minor(uint64(devID)) + return +} + +// atomicFileWrite performs a safe write by creating a temporary file and +// renaming it to the target destination. This ensures the config file is +// never left in a partially written state. +func atomicFileWrite(kernelParamsFile string, data []byte) error { + dir := filepath.Dir(kernelParamsFile) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("failed to create directory: %w", err) + } + + tempFile, err := os.CreateTemp(dir, filepath.Base(kernelParamsFile)+"-*") + if err != nil { + return fmt.Errorf("failed to create temp file: %w", err) + } + defer os.Remove(tempFile.Name()) + + if _, err := tempFile.Write(data); err != nil { + return fmt.Errorf("failed to write temp file: %w", err) + } + + if err := tempFile.Close(); err != nil { + return fmt.Errorf("failed to close temp file: %w", err) + } + + return os.Rename(tempFile.Name(), kernelParamsFile) +} + +// PathForParam returns the sysfs path for a given parameter. +func PathForParam(name ParamName, major, minor uint32) (string, error) { + switch name { + case MaxReadAheadKb: + return fmt.Sprintf("/sys/class/bdi/%d:%d/read_ahead_kb", major, minor), nil + + case MaxBackgroundRequests: + return fmt.Sprintf("/sys/fs/fuse/connections/%d/max_background", minor), nil + + case CongestionWindowThreshold: + return fmt.Sprintf("/sys/fs/fuse/connections/%d/congestion_threshold", minor), nil + + case MaxPagesLimit: + return "/sys/module/fuse/parameters/max_pages_limit", nil + + case TransparentHugePages: + return "/sys/kernel/mm/transparent_hugepage/enabled", nil + + default: + logger.Warnf("Unknown parameter name %q found in kernel parameters config. Skipping...", name) + return "", fmt.Errorf("unknown parameter name: %q", name) + } +} + +// writeValue attempts to write a value to a sysfs path. It first tries a direct +// filesystem write (effective if running as root) and falls back to 'sudo tee' +// if a permission error occurs. +// Note: Fallback attempt succeeds only if passwordless sudo is available. +func writeValue(path, value string) error { + data := []byte(value + "\n") + + // Attempt a direct write first it succeeds if. + // 1. GCSFuse is running as root + // 2. GCSFuse has required permissions to modify files. + err := os.WriteFile(path, data, 0644) + + // If direct write fails with a Permission Denied, attempt sudo fallback. + if err != nil && os.IsPermission(err) { + logger.Warnf("Direct write to file path %q failed with error: %v, Attempting to write using sudo..", path, err) + // -n: non-interactive mode. + cmd := exec.Command("sudo", "-n", "tee", path) + cmd.Stdin = strings.NewReader(value + "\n") + + // Capture Stderr to see why sudo/tee failed + var stderr bytes.Buffer + cmd.Stderr = &stderr + + if sudoErr := cmd.Run(); sudoErr != nil { + // Combine the system error with the actual message from stderr + return fmt.Errorf("sudo error: %v, stderr: %q", sudoErr, strings.TrimSpace(stderr.String())) + } + return nil + } + return err +} + +// applyDirectly iterates through all parameters in the config, resolves their +// system paths, and attempts to apply them to the current host using writeValue helper. +func (c *KernelParamsConfig) applyDirectly(mountPoint string) { + major, minor, err := getDeviceMajorMinor(mountPoint) + if err != nil { + logger.Warnf("Failed to apply kernel parameters directly on mount point %q due to err %v", mountPoint, err) + return + } + for _, p := range c.Parameters { + path, err := PathForParam(p.Name, major, minor) + if err != nil { + logger.Warnf("Unable to update setting %q to value %q for the mount point %q due to err: %v", p.Name, p.Value, mountPoint, err) + continue + } + + if err := writeValue(path, p.Value); err != nil { + logger.Warnf("Unable to update setting %q to value %q for the mount point %q due to err: %v", p.Name, p.Value, mountPoint, err) + continue + } + logger.Infof("Setting %q updated successfully to value %q for the mount point %q", p.Name, p.Value, mountPoint) + } +} + +// addParam adds a new parameter to the config or updates the value if the +// parameter already exists. +func (m *KernelParamsManager) addParam(name ParamName, value string) { + m.mu.Lock() + defer m.mu.Unlock() + for i, p := range m.Parameters { + if p.Name == name { + m.Parameters[i].Value = value + return + } + } + m.Parameters = append(m.Parameters, KernelParam{ + Name: name, + Value: value, + }) +} + +// SetMaxPagesLimit adds the max_pages_limit parameter to the config. +func (m *KernelParamsManager) SetMaxPagesLimit(limit int) { + if limit > 0 { + m.addParam(MaxPagesLimit, strconv.Itoa(limit)) + } +} + +// SetTransparentHugePages adds the THP enabled mode to the config. +func (m *KernelParamsManager) SetTransparentHugePages(mode string) { + if mode != "" { + m.addParam(TransparentHugePages, mode) + } +} + +// SetReadAheadKb adds the BDI read_ahead_kb parameter to the config. +func (m *KernelParamsManager) SetReadAheadKb(kb int) { + if kb > 0 { + m.addParam(MaxReadAheadKb, strconv.Itoa(kb)) + } +} + +// SetMaxBackgroundRequests adds the FUSE connection max_background parameter to the config. +func (m *KernelParamsManager) SetMaxBackgroundRequests(limit int) { + if limit > 0 { + m.addParam(MaxBackgroundRequests, strconv.Itoa(limit)) + } +} + +// SetCongestionWindowThreshold adds the FUSE connection congestion_threshold parameter to the config. +func (m *KernelParamsManager) SetCongestionWindowThreshold(threshold int) { + if threshold > 0 { + m.addParam(CongestionWindowThreshold, strconv.Itoa(threshold)) + } +} + +// ApplyGKE atomically writes the KernelParamsConfig to a JSON file at the specified path. +// This is used in GKE environments where CSI Driver (privileged) reads the file +// to apply settings. +func (m *KernelParamsManager) ApplyGKE(kernelParamsFile string) { + m.mu.Lock() + defer m.mu.Unlock() + if len(m.Parameters) == 0 { + return + } + kernelConfigJson, err := json.Marshal(m.KernelParamsConfig) + if err != nil { + logger.Warnf("Failed to marshal kernel parameters config: %v", err) + return + } + logger.Info("Writing kernel parameters to file for GKE environment", "file", kernelParamsFile, "kernel config", m.KernelParamsConfig) + if err := atomicFileWrite(kernelParamsFile, kernelConfigJson); err != nil { + logger.Warnf("Failed to write kernel parameters to file %q: %v", kernelParamsFile, err) + return + } + logger.Info("Successfully wrote kernel parameters to file", "file", kernelParamsFile) +} + +// ApplyNonGKE applies the kernel settings directly to the host's sysfs entries. +func (m *KernelParamsManager) ApplyNonGKE(mountPoint string) { + m.mu.Lock() + defer m.mu.Unlock() + if len(m.Parameters) == 0 { + return + } + logger.Info("Applying kernel parameters directly for non-GKE environment", "mountPoint", mountPoint, "kernel config", m.KernelParamsConfig) + m.applyDirectly(mountPoint) +} diff --git a/internal/kernelparams/kernelparams_test.go b/internal/kernelparams/kernelparams_test.go new file mode 100644 index 0000000000..7762b03595 --- /dev/null +++ b/internal/kernelparams/kernelparams_test.go @@ -0,0 +1,269 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kernelparams + +import ( + "encoding/json" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAtomicFileWrite(t *testing.T) { + tempDir := t.TempDir() + filePath := filepath.Join(tempDir, "test.txt") + data := []byte("hello world") + + err := atomicFileWrite(filePath, data) + assert.NoError(t, err) + + content, err := os.ReadFile(filePath) + assert.NoError(t, err) + assert.Equal(t, data, content) +} + +func TestGetDeviceMajorMinor_ValidPath(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("Skipping test on non-linux OS") + } + tempDir := t.TempDir() + + major, minor, err := getDeviceMajorMinor(tempDir) + + assert.NoError(t, err) + t.Logf("Device major: %d, minor: %d for %s", major, minor, tempDir) +} + +func TestGetDeviceMajorMinor_NonExistentPath(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("Skipping test on non-linux OS") + } + path := "/path/that/does/not/exist" + + _, _, err := getDeviceMajorMinor(path) + + assert.Error(t, err) +} + +func TestPathForParam(t *testing.T) { + tests := []struct { + name ParamName + major uint32 + minor uint32 + expected string + expectErr bool + }{ + {MaxReadAheadKb, 1, 2, "/sys/class/bdi/1:2/read_ahead_kb", false}, + {MaxBackgroundRequests, 1, 2, "/sys/fs/fuse/connections/2/max_background", false}, + {CongestionWindowThreshold, 1, 2, "/sys/fs/fuse/connections/2/congestion_threshold", false}, + {MaxPagesLimit, 1, 2, "/sys/module/fuse/parameters/max_pages_limit", false}, + {TransparentHugePages, 1, 2, "/sys/kernel/mm/transparent_hugepage/enabled", false}, + {"unknown", 1, 2, "", true}, + } + + for _, tt := range tests { + t.Run(string(tt.name), func(t *testing.T) { + + path, err := PathForParam(tt.name, tt.major, tt.minor) + + if tt.expectErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expected, path) + } + }) + } +} + +func TestSetMaxPagesLimit_NewValue(t *testing.T) { + cfg := NewKernelParamsManager() + + cfg.SetMaxPagesLimit(123) + + assert.Len(t, cfg.Parameters, 1) + assert.Equal(t, MaxPagesLimit, cfg.Parameters[0].Name) + assert.Equal(t, "123", cfg.Parameters[0].Value) +} + +func TestSetMaxPagesLimit_UpdateValue(t *testing.T) { + cfg := NewKernelParamsManager() + cfg.SetMaxPagesLimit(123) + + cfg.SetMaxPagesLimit(456) + + assert.Len(t, cfg.Parameters, 1) + assert.Equal(t, MaxPagesLimit, cfg.Parameters[0].Name) + assert.Equal(t, "456", cfg.Parameters[0].Value) +} + +func TestSetTransparentHugePages_NewValue(t *testing.T) { + cfg := NewKernelParamsManager() + + cfg.SetTransparentHugePages("always") + + assert.Len(t, cfg.Parameters, 1) + assert.Equal(t, TransparentHugePages, cfg.Parameters[0].Name) + assert.Equal(t, "always", cfg.Parameters[0].Value) +} + +func TestSetTransparentHugePages_UpdateValue(t *testing.T) { + cfg := NewKernelParamsManager() + cfg.SetTransparentHugePages("always") + + cfg.SetTransparentHugePages("never") + + assert.Len(t, cfg.Parameters, 1) + assert.Equal(t, TransparentHugePages, cfg.Parameters[0].Name) + assert.Equal(t, "never", cfg.Parameters[0].Value) +} + +func TestSetReadAheadKb(t *testing.T) { + cfg := NewKernelParamsManager() + + cfg.SetReadAheadKb(1024) + + assert.Len(t, cfg.Parameters, 1) + assert.Equal(t, MaxReadAheadKb, cfg.Parameters[0].Name) + assert.Equal(t, "1024", cfg.Parameters[0].Value) +} + +func TestSetMaxBackgroundRequests(t *testing.T) { + cfg := NewKernelParamsManager() + + cfg.SetMaxBackgroundRequests(12) + + assert.Len(t, cfg.Parameters, 1) + assert.Equal(t, MaxBackgroundRequests, cfg.Parameters[0].Name) + assert.Equal(t, "12", cfg.Parameters[0].Value) +} + +func TestSetCongestionWindowThreshold(t *testing.T) { + cfg := NewKernelParamsManager() + + cfg.SetCongestionWindowThreshold(9) + + assert.Len(t, cfg.Parameters, 1) + assert.Equal(t, CongestionWindowThreshold, cfg.Parameters[0].Name) + assert.Equal(t, "9", cfg.Parameters[0].Value) +} + +func TestSetMultipleKernelParams(t *testing.T) { + cfg := NewKernelParamsManager() + + cfg.SetMaxPagesLimit(123) + cfg.SetTransparentHugePages("always") + cfg.SetReadAheadKb(456) + + assert.Len(t, cfg.Parameters, 3) + // Verify values + expected := map[ParamName]string{ + MaxPagesLimit: "123", + TransparentHugePages: "always", + MaxReadAheadKb: "456", + } + for _, p := range cfg.Parameters { + val, ok := expected[p.Name] + assert.True(t, ok) + assert.Equal(t, val, p.Value) + } +} + +func TestApplyGKE(t *testing.T) { + tempDir := t.TempDir() + filePath := filepath.Join(tempDir, "kernel_params.json") + cfg := NewKernelParamsManager() + cfg.SetReadAheadKb(1024) + + cfg.ApplyGKE(filePath) + + content, err := os.ReadFile(filePath) + assert.NoError(t, err) + var actualCfg KernelParamsConfig + err = json.Unmarshal(content, &actualCfg) + assert.NoError(t, err) + assert.NotEmpty(t, actualCfg.RequestID) + assert.NotEmpty(t, actualCfg.Timestamp) + assert.Len(t, actualCfg.Parameters, 1) + assert.Equal(t, MaxReadAheadKb, actualCfg.Parameters[0].Name) + assert.Equal(t, "1024", actualCfg.Parameters[0].Value) +} + +func TestApplyGKE_EmptyParams(t *testing.T) { + tempDir := t.TempDir() + filePath := filepath.Join(tempDir, "kernel_params.json") + cfg := NewKernelParamsManager() + + cfg.ApplyGKE(filePath) + + _, err := os.Stat(filePath) + assert.True(t, os.IsNotExist(err)) +} + +func TestWriteValue_DirectWriteSuccess(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("Skipping test on non-linux OS") + } + tempDir := t.TempDir() + path := filepath.Join(tempDir, "test_file") + value := "100" + + err := writeValue(path, value) + + assert.NoError(t, err) + content, err := os.ReadFile(path) + assert.NoError(t, err) + assert.Equal(t, "100\n", string(content)) +} + +func TestWriteValue_DirectWriteFailure_NoSudo(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("Skipping test on non-linux OS") + } + // Use a path in a non-existent directory to trigger a non-permission error + path := filepath.Join(t.TempDir(), "missing_dir", "test_file") + value := "100" + + err := writeValue(path, value) + + assert.Error(t, err) + // Should not be a sudo error, but the original fs error + assert.NotContains(t, err.Error(), "sudo error") +} + +func TestWriteValue_PermissionDenied_SudoFallback(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("Skipping test on non-linux OS") + } + tempDir := t.TempDir() + path := filepath.Join(tempDir, "readonly_file") + // Create file and make it read-only to trigger permission error + err := os.WriteFile(path, []byte("initial"), 0444) + assert.NoError(t, err) + + err = writeValue(path, "new_value") + + // This depends on the environment. + // If sudo works, err is nil. If not, err contains "sudo error". + if err == nil { + content, _ := os.ReadFile(path) + assert.Equal(t, "new_value\n", string(content)) + } else { + assert.Contains(t, err.Error(), "sudo error") + } +} diff --git a/internal/locker/locker.go b/internal/locker/locker.go index f2f8c27cc7..b507a615b2 100644 --- a/internal/locker/locker.go +++ b/internal/locker/locker.go @@ -20,7 +20,7 @@ import ( "sync" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" ) var gEnableInvariantsCheck bool diff --git a/internal/locker/rw_locker.go b/internal/locker/rw_locker.go index a2a9dc0633..5b7ff4a8c2 100644 --- a/internal/locker/rw_locker.go +++ b/internal/locker/rw_locker.go @@ -20,7 +20,7 @@ import ( "sync" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" ) type RWLocker interface { diff --git a/internal/logger/legacy_logger.go b/internal/logger/legacy_logger.go index 0843fe575d..a4faed6d98 100644 --- a/internal/logger/legacy_logger.go +++ b/internal/logger/legacy_logger.go @@ -27,9 +27,9 @@ import ( // individual log messages. // This method is created to support jacobsa/fuse loggers and will be removed // after slog support is added. -func NewLegacyLogger(level slog.Level, prefix string) *log.Logger { - var programLevel = new(slog.LevelVar) - logger := slog.NewLogLogger(defaultLoggerFactory.handler(programLevel, prefix), level) - setLoggingLevel(defaultLoggerFactory.level, programLevel) +func NewLegacyLogger(level slog.Level, prefix, fsName string) *log.Logger { + handler := defaultLoggerFactory.handler(programLevel, prefix).WithAttrs(loggerAttr(fsName)) + logger := slog.NewLogLogger(handler, level) + setLoggingLevel(defaultLoggerFactory.level) return logger } diff --git a/internal/logger/logger.go b/internal/logger/logger.go index f84293dcde..06daf00ac1 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -22,34 +22,44 @@ import ( "log/syslog" "os" "runtime/debug" + "strings" + "sync" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" + "github.com/google/uuid" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" "gopkg.in/natefinch/lumberjack.v2" ) -// Syslog file contains logs from all different programmes running on the VM. -// ProgrammeName is prefixed to all the logs written to syslog. This constant is +// Syslog file contains logs from all different programs running on the VM. +// ProgramName is prefixed to all the logs written to syslog. This constant is // used to filter the logs from syslog and write it to respective log files - // gcsfuse.log in case of GCSFuse. const ( - ProgrammeName string = "gcsfuse" - GCSFuseInBackgroundMode string = "GCSFUSE_IN_BACKGROUND_MODE" - textFormat string = "text" + ProgramName = "gcsfuse" + GCSFuseInBackgroundMode = "GCSFUSE_IN_BACKGROUND_MODE" + MountUUIDEnvKey = "GCSFUSE_MOUNT_UUID" + MountIDKey = "mount-id" // Combination of fsName and GCSFUSE_MOUNT_UUID + textFormat = "text" + // Max possible length can be 32 as UUID has 32 characters excluding 4 hyphens. + mountUUIDLength = 8 ) var ( defaultLoggerFactory *loggerFactory defaultLogger *slog.Logger + mountUUID string + setupMountUUIDOnce sync.Once + programLevel = new(slog.LevelVar) ) // InitLogFile initializes the logger factory to create loggers that print to -// a log file. +// a log file, with MountInstanceID set as a custom attribute. // In case of empty file, it starts writing the log to syslog file, which // is eventually filtered and redirected to a fixed location using syslog // config. // Here, background true means, this InitLogFile has been called for the // background daemon. -func InitLogFile(newLogConfig cfg.LoggingConfig) error { +func InitLogFile(newLogConfig cfg.LoggingConfig, fsName string) error { var f *os.File var sysWriter *syslog.Writer var fileWriter *lumberjack.Logger @@ -80,7 +90,7 @@ func InitLogFile(newLogConfig cfg.LoggingConfig) error { // Suppressing the error while creating the syslog, although logger will // be initialised with stdout/err, log will be printed anywhere. Because, // in this case gcsfuse will be running as daemon. - sysWriter, _ = syslog.New(syslog.LOG_LOCAL7|syslog.LOG_DEBUG, ProgrammeName) + sysWriter, _ = syslog.New(syslog.LOG_LOCAL7|syslog.LOG_DEBUG, ProgramName) } } @@ -92,7 +102,7 @@ func InitLogFile(newLogConfig cfg.LoggingConfig) error { level: string(newLogConfig.Severity), logRotate: newLogConfig.LogRotate, } - defaultLogger = defaultLoggerFactory.newLogger(string(newLogConfig.Severity)) + defaultLogger = defaultLoggerFactory.newLoggerWithMountInstanceID(string(newLogConfig.Severity), fsName) return nil } @@ -109,52 +119,143 @@ func init() { defaultLogger = defaultLoggerFactory.newLogger(cfg.INFO) } -// SetLogFormat updates the log format of default logger. -func SetLogFormat(format string) { - if format == defaultLoggerFactory.format { +// generateMountUUID generates a random string of size from UUID. +func generateMountUUID(size int) (string, error) { + if size <= 0 { + return "", fmt.Errorf("requested size for MountUUID must be positive, but got %d", size) + } + uuid := uuid.New() + uuidStr := strings.ReplaceAll(uuid.String(), "-", "") + if size > len(uuidStr) { + return "", fmt.Errorf("UUID is smaller than requested size %d for MountUUID, UUID: %s", size, uuidStr) + } + return uuidStr[:size], nil +} + +// setupMountUUID handles the retrieval of mountUUID if GCSFuse is in +// background mode or generates one if running in foreground mode. +func setupMountUUID() { + if _, ok := os.LookupEnv(GCSFuseInBackgroundMode); ok { + // If GCSFuse is in background mode then look for the GCSFUSE_MOUNT_UUID in env which was set by the caller of demonize run. + if mountUUID, ok = os.LookupEnv(MountUUIDEnvKey); !ok || mountUUID == "" { + Fatal("Could not retrieve %s env variable or it's empty.", MountUUIDEnvKey) + } return } + // If GCSFuse is not running in the background mode then generate a random UUID. + var err error + if mountUUID, err = generateMountUUID(mountUUIDLength); err != nil { + Fatal("Could not generate MountUUID of length %d, err: %v", mountUUIDLength, err) + } +} + +// MountUUID returns a unique ID for the current GCSFuse mount, +// ensuring the ID is initialized only once. On the first call, it either +// generates a random ID (foreground mode) or retrieves it from the +// GCSFUSE_MOUNT_UUID environment variable (background mode). +// Subsequent calls return the same cached ID. +func MountUUID() string { + setupMountUUIDOnce.Do(setupMountUUID) + return mountUUID +} + +// MountInstanceID returns the InstanceID of current gcsfuse mount. +// This is combination of `fsName` + MountUUID. +// Note: fsName is passed here explicitly, as logger package doesn't know about fsName +// when MountInstanceID method is invoked. +func MountInstanceID(fsName string) string { + return fmt.Sprintf("%s-%s", fsName, MountUUID()) +} + +// UpdateDefaultLogger updates the log format and creates a new logger with MountInstanceID set as custom attribute. +func UpdateDefaultLogger(format, fsName string) { defaultLoggerFactory.format = format - defaultLogger = defaultLoggerFactory.newLogger(defaultLoggerFactory.level) + defaultLogger = defaultLoggerFactory.newLoggerWithMountInstanceID(defaultLoggerFactory.level, fsName) } // Tracef prints the message with TRACE severity in the specified format. -func Tracef(format string, v ...interface{}) { - defaultLogger.Log(context.Background(), LevelTrace, fmt.Sprintf(format, v...)) +func Tracef(format string, v ...any) { + if LevelTrace >= programLevel.Level() { + defaultLogger.Log(context.Background(), LevelTrace, fmt.Sprintf(format, v...)) + } } // Debugf prints the message with DEBUG severity in the specified format. -func Debugf(format string, v ...interface{}) { - defaultLogger.Debug(fmt.Sprintf(format, v...)) +func Debugf(format string, v ...any) { + if LevelDebug >= programLevel.Level() { + defaultLogger.Debug(fmt.Sprintf(format, v...)) + } } // Infof prints the message with INFO severity in the specified format. -func Infof(format string, v ...interface{}) { - defaultLogger.Info(fmt.Sprintf(format, v...)) +func Infof(format string, v ...any) { + if LevelInfo >= programLevel.Level() { + defaultLogger.Info(fmt.Sprintf(format, v...)) + } } // Info prints the message with info severity. func Info(message string, args ...any) { - defaultLogger.Info(message, args...) + if LevelInfo >= programLevel.Level() { + defaultLogger.Info(message, args...) + } } // Warnf prints the message with WARNING severity in the specified format. -func Warnf(format string, v ...interface{}) { - defaultLogger.Warn(fmt.Sprintf(format, v...)) +func Warnf(format string, v ...any) { + if LevelWarn >= programLevel.Level() { + defaultLogger.Warn(fmt.Sprintf(format, v...)) + } } // Errorf prints the message with ERROR severity in the specified format. -func Errorf(format string, v ...interface{}) { - defaultLogger.Error(fmt.Sprintf(format, v...)) +func Errorf(format string, v ...any) { + if LevelError >= programLevel.Level() { + defaultLogger.Error(fmt.Sprintf(format, v...)) + } +} + +// Error prints the message with ERROR severity. +func Error(error string) { + if LevelError >= programLevel.Level() { + defaultLogger.Error(error) + } +} + +type logFunc func(string, ...any) + +var logFHandlers = map[slog.Level]logFunc{ + LevelTrace: Tracef, + LevelDebug: Debugf, + LevelInfo: Infof, + LevelWarn: Warnf, + LevelError: Errorf, +} + +func GetLogFHandler(level slog.Level) logFunc { + if fn, ok := logFHandlers[level]; ok { + return fn + } + + Warnf("logger: unsupported log level: %v", level) + + return logFHandlers[LevelTrace] } // Fatal prints an error log and exits with non-zero exit code. -func Fatal(format string, v ...interface{}) { +func Fatal(format string, v ...any) { Errorf(format, v...) - Errorf(string(debug.Stack())) + Error(string(debug.Stack())) os.Exit(1) } +// SetOutput sets the output destination for the default logger. +// This is primarily used for testing. +func SetOutput(w io.Writer) { + defaultLogger = slog.New(defaultLoggerFactory.createJsonOrTextHandler(w, programLevel, "")) + slog.SetDefault(defaultLogger) +} + type loggerFactory struct { // If nil, log to stdout or stderr. Otherwise, log to this file. file *os.File @@ -167,10 +268,21 @@ type loggerFactory struct { func (f *loggerFactory) newLogger(level string) *slog.Logger { // create a new logger - var programLevel = new(slog.LevelVar) logger := slog.New(f.handler(programLevel, "")) slog.SetDefault(logger) - setLoggingLevel(level, programLevel) + setLoggingLevel(level) + return logger +} + +func loggerAttr(fsName string) []slog.Attr { + return []slog.Attr{slog.String(MountIDKey, MountInstanceID(fsName))} +} + +// create a new logger with mountInstanceID set as custom attribute on logger. +func (f *loggerFactory) newLoggerWithMountInstanceID(level, fsName string) *slog.Logger { + logger := slog.New(f.handler(programLevel, "").WithAttrs(loggerAttr(fsName))) + slog.SetDefault(logger) + setLoggingLevel(level) return logger } diff --git a/internal/logger/logger_test.go b/internal/logger/logger_test.go index 3d22d88683..ec55e512f6 100644 --- a/internal/logger/logger_test.go +++ b/internal/logger/logger_test.go @@ -16,66 +16,47 @@ package logger import ( "bytes" + "fmt" "log/slog" - "os" + "path/filepath" "regexp" + "strings" + "sync" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" + "github.com/stretchr/testify/require" ) const ( - textTraceString = "^time=\"[a-zA-Z0-9/:. ]{26}\" severity=TRACE message=\"TestLogs: www.traceExample.com\"" - textDebugString = "^time=\"[a-zA-Z0-9/:. ]{26}\" severity=DEBUG message=\"TestLogs: www.debugExample.com\"" - textInfoString = "^time=\"[a-zA-Z0-9/:. ]{26}\" severity=INFO message=\"TestLogs: www.infoExample.com\"" - textWarningString = "^time=\"[a-zA-Z0-9/:. ]{26}\" severity=WARNING message=\"TestLogs: www.warningExample.com\"" - textErrorString = "^time=\"[a-zA-Z0-9/:. ]{26}\" severity=ERROR message=\"TestLogs: www.errorExample.com\"" - - jsonTraceString = "^{\"timestamp\":{\"seconds\":\\d{10},\"nanos\":\\d{0,9}},\"severity\":\"TRACE\",\"message\":\"TestLogs: www.traceExample.com\"}" - jsonDebugString = "^{\"timestamp\":{\"seconds\":\\d{10},\"nanos\":\\d{0,9}},\"severity\":\"DEBUG\",\"message\":\"TestLogs: www.debugExample.com\"}" - jsonInfoString = "^{\"timestamp\":{\"seconds\":\\d{10},\"nanos\":\\d{0,9}},\"severity\":\"INFO\",\"message\":\"TestLogs: www.infoExample.com\"}" - jsonWarningString = "^{\"timestamp\":{\"seconds\":\\d{10},\"nanos\":\\d{0,9}},\"severity\":\"WARNING\",\"message\":\"TestLogs: www.warningExample.com\"}" - jsonErrorString = "^{\"timestamp\":{\"seconds\":\\d{10},\"nanos\":\\d{0,9}},\"severity\":\"ERROR\",\"message\":\"TestLogs: www.errorExample.com\"}" + testFsName = "testFS" // This is used in redirectLogsToGivenBuffer to construct the mount instance ID. + textLogPattern = `^time="[a-zA-Z0-9/:. ]{26}" severity=%s message="TestLogs: %s" mount-id=testFS-[0-9a-f]{8}\s*$` + jsonLogPattern = `^{"timestamp":{"seconds":\d{10},"nanos":\d{0,9}},"severity":"%s","message":"TestLogs: %s","mount-id":"testFS-[0-9a-f]{8}"}\s*$` ) -type LoggerTest struct { - suite.Suite -} - -func TestLoggerSuite(t *testing.T) { - suite.Run(t, new(LoggerTest)) -} - // ////////////////////////////////////////////////////////////////////// -// Boilerplate +// Helpers // ////////////////////////////////////////////////////////////////////// -func redirectLogsToGivenBuffer(buf *bytes.Buffer, level string) { - var programLevel = new(slog.LevelVar) - defaultLogger = slog.New( - defaultLoggerFactory.createJsonOrTextHandler(buf, programLevel, "TestLogs: "), - ) - setLoggingLevel(level, programLevel) +func expectedLogRegex(t *testing.T, format, severity, message string) string { + t.Helper() + message = regexp.QuoteMeta(message) + switch format { + case "text": + return fmt.Sprintf(textLogPattern, severity, message) + case "json": + return fmt.Sprintf(jsonLogPattern, severity, message) + default: + return "" + } } -// fetchLogOutputForSpecifiedSeverityLevel takes configured severity and -// functions that write logs as parameter and returns string array containing -// output from each function call. -func fetchLogOutputForSpecifiedSeverityLevel(level string, functions []func()) []string { - // create a logger that writes to buffer at configured level. - var buf bytes.Buffer - redirectLogsToGivenBuffer(&buf, level) - - var output []string - // run the functions provided. - for _, f := range functions { - f() - output = append(output, buf.String()) - buf.Reset() - } - return output +func redirectLogsToGivenBuffer(buf *bytes.Buffer, level string) { + handler := defaultLoggerFactory.createJsonOrTextHandler(buf, programLevel, "TestLogs: ") + handler = handler.WithAttrs(loggerAttr(testFsName)) + defaultLogger = slog.New(handler) + setLoggingLevel(level) } func getTestLoggingFunctions() []func() { @@ -98,180 +79,252 @@ func getTestLoggingFunctions() []func() { } } -func validateOutput(t *testing.T, expected []string, output []string) { - for i := range output { - if expected[i] == "" { - assert.Equal(t, expected[i], output[i]) - } else { - expectedRegexp := regexp.MustCompile(expected[i]) - assert.True(t, expectedRegexp.MatchString(output[i])) - } - } -} - -func validateLogOutputAtSpecifiedFormatAndSeverity(t *testing.T, format string, level string, expectedOutput []string) { +// fetchAllLogLevelOutputsForSpecifiedSeverityLevel sets the log format and severity, +// executes standard logging functions, and returns their output. +func fetchAllLogLevelOutputsForSpecifiedSeverityLevel(t *testing.T, format, level string) []string { + t.Helper() // set log format defaultLoggerFactory.format = format - output := fetchLogOutputForSpecifiedSeverityLevel(level, getTestLoggingFunctions()) + // create a logger that writes to buffer at configured level. + var buf bytes.Buffer + redirectLogsToGivenBuffer(&buf, level) + + var actualLogLines []string + // run the functions provided. + for _, f := range getTestLoggingFunctions() { + f() + actualLogLines = append(actualLogLines, buf.String()) + buf.Reset() + } + return actualLogLines +} - validateOutput(t, expectedOutput, output) +// validateLogOutputs compares the captured log line output with the expected log regex patterns. +func validateLogOutputs(t *testing.T, expectedLogLineRegexes, actualLogLines []string) { + t.Helper() + require.Equal(t, len(expectedLogLineRegexes), len(actualLogLines)) + for i := range actualLogLines { + assert.Regexp(t, expectedLogLineRegexes[i], actualLogLines[i]) + } } // ////////////////////////////////////////////////////////////////////// // Tests // ////////////////////////////////////////////////////////////////////// -func (t *LoggerTest) TestTextFormatLogs_LogLevelOFF() { - var expected = []string{ - "", "", "", "", "", +func TestTextFormatLogs_LogLevelOFF(t *testing.T) { + var expectedLogLineRegexes = []string{ + "", // TRACE + "", // DEBUG + "", // INFO + "", // WARNING + "", // ERROR } - // Assert that nothing is logged when log level is OFF. - validateLogOutputAtSpecifiedFormatAndSeverity(t.T(), "json", cfg.OFF, expected) + actualLogLines := fetchAllLogLevelOutputsForSpecifiedSeverityLevel(t, "text", cfg.OFF) + + validateLogOutputs(t, expectedLogLineRegexes, actualLogLines) } -func (t *LoggerTest) TestTextFormatLogs_LogLevelERROR() { - var expected = []string{ - "", "", "", "", textErrorString, +func TestTextFormatLogs_LogLevelERROR(t *testing.T) { + var expectedLogLineRegexes = []string{ + "", // TRACE + "", // DEBUG + "", // INFO + "", // WARNING + expectedLogRegex(t, "text", "ERROR", "www.errorExample.com"), } - // Assert only error logs are logged when log level is ERROR. - validateLogOutputAtSpecifiedFormatAndSeverity(t.T(), "text", cfg.ERROR, expected) + actualLogLines := fetchAllLogLevelOutputsForSpecifiedSeverityLevel(t, "text", cfg.ERROR) + + validateLogOutputs(t, expectedLogLineRegexes, actualLogLines) } -func (t *LoggerTest) TestTextFormatLogs_LogLevelWARNING() { - var expected = []string{ - "", "", "", textWarningString, textErrorString, +func TestTextFormatLogs_LogLevelWARNING(t *testing.T) { + expectedLogLineRegexes := []string{ + "", // TRACE + "", // DEBUG + "", // INFO + expectedLogRegex(t, "text", "WARNING", "www.warningExample.com"), + expectedLogRegex(t, "text", "ERROR", "www.errorExample.com"), } - // Assert warning and error logs are logged when log level is WARNING. - validateLogOutputAtSpecifiedFormatAndSeverity(t.T(), "text", cfg.WARNING, expected) + actualLogLines := fetchAllLogLevelOutputsForSpecifiedSeverityLevel(t, "text", cfg.WARNING) + + validateLogOutputs(t, expectedLogLineRegexes, actualLogLines) } -func (t *LoggerTest) TestTextFormatLogs_LogLevelINFO() { - var expected = []string{ - "", "", textInfoString, textWarningString, textErrorString, +func TestTextFormatLogs_LogLevelINFO(t *testing.T) { + expectedLogLineRegexes := []string{ + "", // TRACE + "", // DEBUG + expectedLogRegex(t, "text", "INFO", "www.infoExample.com"), + expectedLogRegex(t, "text", "WARNING", "www.warningExample.com"), + expectedLogRegex(t, "text", "ERROR", "www.errorExample.com"), } - // Assert info, warning & error logs are logged when log level is INFO. - validateLogOutputAtSpecifiedFormatAndSeverity(t.T(), "text", cfg.INFO, expected) + actualLogLines := fetchAllLogLevelOutputsForSpecifiedSeverityLevel(t, "text", cfg.INFO) + + validateLogOutputs(t, expectedLogLineRegexes, actualLogLines) } -func (t *LoggerTest) TestTextFormatLogs_LogLevelDEBUG() { - var expected = []string{ - "", textDebugString, textInfoString, textWarningString, textErrorString, +func TestTextFormatLogs_LogLevelDEBUG(t *testing.T) { + expectedLogLineRegexes := []string{ + "", // TRACE + expectedLogRegex(t, "text", "DEBUG", "www.debugExample.com"), + expectedLogRegex(t, "text", "INFO", "www.infoExample.com"), + expectedLogRegex(t, "text", "WARNING", "www.warningExample.com"), + expectedLogRegex(t, "text", "ERROR", "www.errorExample.com"), } - // Assert debug, info, warning & error logs are logged when log level is DEBUG. - validateLogOutputAtSpecifiedFormatAndSeverity(t.T(), "text", cfg.DEBUG, expected) + actualLogLines := fetchAllLogLevelOutputsForSpecifiedSeverityLevel(t, "text", cfg.DEBUG) + + validateLogOutputs(t, expectedLogLineRegexes, actualLogLines) } -func (t *LoggerTest) TestTextFormatLogs_LogLevelTRACE() { - var expected = []string{ - textTraceString, textDebugString, textInfoString, textWarningString, textErrorString, +func TestTextFormatLogs_LogLevelTRACE(t *testing.T) { + expectedLogLineRegexes := []string{ + expectedLogRegex(t, "text", "TRACE", "www.traceExample.com"), + expectedLogRegex(t, "text", "DEBUG", "www.debugExample.com"), + expectedLogRegex(t, "text", "INFO", "www.infoExample.com"), + expectedLogRegex(t, "text", "WARNING", "www.warningExample.com"), + expectedLogRegex(t, "text", "ERROR", "www.errorExample.com"), } - // Assert all logs are logged when log level is TRACE. - validateLogOutputAtSpecifiedFormatAndSeverity(t.T(), "text", cfg.TRACE, expected) + actualLogLines := fetchAllLogLevelOutputsForSpecifiedSeverityLevel(t, "text", cfg.TRACE) + + validateLogOutputs(t, expectedLogLineRegexes, actualLogLines) } -func (t *LoggerTest) TestJSONFormatLogs_LogLevelOFF() { - var expected = []string{ - "", "", "", "", "", +func TestJSONFormatLogs_LogLevelOFF(t *testing.T) { + expectedLogLineRegexes := []string{ + "", // TRACE + "", // DEBUG + "", // INFO + "", // WARNING + "", // ERROR } - // Assert that nothing is logged when log level is OFF. - validateLogOutputAtSpecifiedFormatAndSeverity(t.T(), "json", cfg.OFF, expected) + actualLogLines := fetchAllLogLevelOutputsForSpecifiedSeverityLevel(t, "json", cfg.OFF) + + validateLogOutputs(t, expectedLogLineRegexes, actualLogLines) } -func (t *LoggerTest) TestJSONFormatLogs_LogLevelERROR() { - var expected = []string{ - "", "", "", "", jsonErrorString, +func TestJSONFormatLogs_LogLevelERROR(t *testing.T) { + expectedLogLineRegexes := []string{ + "", // TRACE + "", // DEBUG + "", // INFO + "", // WARNING + expectedLogRegex(t, "json", "ERROR", "www.errorExample.com"), } - // Assert only error logs are logged when log level is ERROR. - validateLogOutputAtSpecifiedFormatAndSeverity(t.T(), "json", cfg.ERROR, expected) + actualLogLines := fetchAllLogLevelOutputsForSpecifiedSeverityLevel(t, "json", cfg.ERROR) + + validateLogOutputs(t, expectedLogLineRegexes, actualLogLines) } -func (t *LoggerTest) TestJSONFormatLogs_LogLevelWARNING() { - var expected = []string{ - "", "", "", jsonWarningString, jsonErrorString, +func TestJSONFormatLogs_LogLevelWARNING(t *testing.T) { + expectedLogLineRegexes := []string{ + "", // TRACE + "", // DEBUG + "", // INFO + expectedLogRegex(t, "json", "WARNING", "www.warningExample.com"), + expectedLogRegex(t, "json", "ERROR", "www.errorExample.com"), } - // Assert warning and error logs are logged when log level is WARNING. - validateLogOutputAtSpecifiedFormatAndSeverity(t.T(), "json", cfg.WARNING, expected) + actualLogLines := fetchAllLogLevelOutputsForSpecifiedSeverityLevel(t, "json", cfg.WARNING) + + validateLogOutputs(t, expectedLogLineRegexes, actualLogLines) } -func (t *LoggerTest) TestJSONFormatLogs_LogLevelINFO() { - var expected = []string{ - "", "", jsonInfoString, jsonWarningString, jsonErrorString, +func TestJSONFormatLogs_LogLevelINFO(t *testing.T) { + expectedLogLineRegexes := []string{ + "", // TRACE + "", // DEBUG + expectedLogRegex(t, "json", "INFO", "www.infoExample.com"), + expectedLogRegex(t, "json", "WARNING", "www.warningExample.com"), + expectedLogRegex(t, "json", "ERROR", "www.errorExample.com"), } - // Assert info, warning & error logs are logged when log level is INFO. - validateLogOutputAtSpecifiedFormatAndSeverity(t.T(), "json", cfg.INFO, expected) + actualLogLines := fetchAllLogLevelOutputsForSpecifiedSeverityLevel(t, "json", cfg.INFO) + + validateLogOutputs(t, expectedLogLineRegexes, actualLogLines) } -func (t *LoggerTest) TestJSONFormatLogs_LogLevelDEBUG() { - var expected = []string{ - "", jsonDebugString, jsonInfoString, jsonWarningString, jsonErrorString, +func TestJSONFormatLogs_LogLevelDEBUG(t *testing.T) { + expectedLogLineRegexes := []string{ + "", // TRACE + expectedLogRegex(t, "json", "DEBUG", "www.debugExample.com"), + expectedLogRegex(t, "json", "INFO", "www.infoExample.com"), + expectedLogRegex(t, "json", "WARNING", "www.warningExample.com"), + expectedLogRegex(t, "json", "ERROR", "www.errorExample.com"), } - // Assert debug, info, warning & error logs are logged when log level is DEBUG. - validateLogOutputAtSpecifiedFormatAndSeverity(t.T(), "json", cfg.DEBUG, expected) + actualLogLines := fetchAllLogLevelOutputsForSpecifiedSeverityLevel(t, "json", cfg.DEBUG) + + validateLogOutputs(t, expectedLogLineRegexes, actualLogLines) } -func (t *LoggerTest) TestJSONFormatLogs_LogLevelTRACE() { - var expected = []string{ - jsonTraceString, jsonDebugString, jsonInfoString, jsonWarningString, jsonErrorString, +func TestJSONFormatLogs_LogLevelTRACE(t *testing.T) { + expectedLogLineRegexes := []string{ + expectedLogRegex(t, "json", "TRACE", "www.traceExample.com"), + expectedLogRegex(t, "json", "DEBUG", "www.debugExample.com"), + expectedLogRegex(t, "json", "INFO", "www.infoExample.com"), + expectedLogRegex(t, "json", "WARNING", "www.warningExample.com"), + expectedLogRegex(t, "json", "ERROR", "www.errorExample.com"), } - // Assert all logs are logged when log level is TRACE. - validateLogOutputAtSpecifiedFormatAndSeverity(t.T(), "json", cfg.TRACE, expected) + actualLogLines := fetchAllLogLevelOutputsForSpecifiedSeverityLevel(t, "json", cfg.TRACE) + + validateLogOutputs(t, expectedLogLineRegexes, actualLogLines) } -func (t *LoggerTest) TestSetLoggingLevel() { - testData := []struct { +func TestSetLoggingLevel(t *testing.T) { + testCases := []struct { + name string inputLevel string - programLevel *slog.LevelVar expectedProgramLevel slog.Level }{ { - cfg.TRACE, - new(slog.LevelVar), - LevelTrace, + name: "TRACE", + inputLevel: cfg.TRACE, + expectedProgramLevel: LevelTrace, }, { - cfg.DEBUG, - new(slog.LevelVar), - LevelDebug, + name: "DEBUG", + inputLevel: cfg.DEBUG, + expectedProgramLevel: LevelDebug, }, { - cfg.WARNING, - new(slog.LevelVar), - LevelWarn, + name: "WARNING", + inputLevel: cfg.WARNING, + expectedProgramLevel: LevelWarn, }, { - cfg.ERROR, - new(slog.LevelVar), - LevelError, + name: "ERROR", + inputLevel: cfg.ERROR, + expectedProgramLevel: LevelError, }, { - cfg.OFF, - new(slog.LevelVar), - LevelOff, + name: "OFF", + inputLevel: cfg.OFF, + expectedProgramLevel: LevelOff, }, } - for _, test := range testData { - setLoggingLevel(test.inputLevel, test.programLevel) - assert.Equal(t.T(), test.programLevel.Level(), test.expectedProgramLevel) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + setLoggingLevel(tc.inputLevel) + + assert.Equal(t, tc.expectedProgramLevel, programLevel.Level()) + }) } } -func (t *LoggerTest) TestInitLogFile() { +func TestInitLogFile(t *testing.T) { + filePath := filepath.Join(t.TempDir(), "log.txt") format := "text" - filePath, _ := os.UserHomeDir() - filePath += "/log.txt" fileSize := int64(100) backupFileCount := int64(2) newLogConfig := cfg.LoggingConfig{ @@ -285,57 +338,231 @@ func (t *LoggerTest) TestInitLogFile() { }, } - err := InitLogFile(newLogConfig) + err := InitLogFile(newLogConfig, testFsName) + + require.NoError(t, err) + require.NotNil(t, defaultLoggerFactory.file) + t.Cleanup(func() { + defaultLoggerFactory.file.Close() // Close file handle to release resources. + }) + assert.Equal(t, filePath, defaultLoggerFactory.file.Name()) + assert.Nil(t, defaultLoggerFactory.sysWriter) + assert.Equal(t, format, defaultLoggerFactory.format) + assert.Equal(t, cfg.DEBUG, defaultLoggerFactory.level) + assert.Equal(t, fileSize, defaultLoggerFactory.logRotate.MaxFileSizeMb) + assert.Equal(t, backupFileCount, defaultLoggerFactory.logRotate.BackupFileCount) + assert.True(t, defaultLoggerFactory.logRotate.Compress) +} + +func TestUpdateDefaultLogger(t *testing.T) { + testCases := []struct { + name string + format string + expectedRegex string + }{ + { + name: "TextFormat", + format: "text", + expectedRegex: expectedLogRegex(t, "text", "INFO", "www.infoExample.com"), + }, + { + name: "JsonFormat", + format: "json", + expectedRegex: expectedLogRegex(t, "json", "INFO", "www.infoExample.com"), + }, + { + name: "EmptyFormatDefaultsToJson", + format: "", + expectedRegex: expectedLogRegex(t, "json", "INFO", "www.infoExample.com"), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Initialize logger factory for each subtest to ensure isolation. + logConfig := cfg.DefaultLoggingConfig() + defaultLoggerFactory = &loggerFactory{ + file: nil, + level: string(logConfig.Severity), // setting log level to INFO by default + logRotate: logConfig.LogRotate, + } + + UpdateDefaultLogger(tc.format, testFsName) + var buf bytes.Buffer + redirectLogsToGivenBuffer(&buf, defaultLoggerFactory.level) + Infof("www.infoExample.com") + + assert.Regexp(t, tc.expectedRegex, buf.String()) + }) + } +} + +func TestGenerateMountUUID_Success(t *testing.T) { + testCases := []struct { + name string + size int + expectedMountUUIDRegex string + }{ + { + name: "TenChars", + size: 10, + expectedMountUUIDRegex: "^[0-9a-f]{10}$", + }, + { + name: "ThirtyTwoChars", + size: 32, + expectedMountUUIDRegex: "^[0-9a-f]{32}$", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + mountUUID, err := generateMountUUID(tc.size) + + require.NoError(t, err) + assert.Regexp(t, tc.expectedMountUUIDRegex, mountUUID) + }) + } +} + +func TestGenerateMountUUID_FailureDueToUUIDSize(t *testing.T) { + mountUUID, err := generateMountUUID(999) - assert.NoError(t.T(), err) - assert.Equal(t.T(), filePath, defaultLoggerFactory.file.Name()) - assert.Nil(t.T(), defaultLoggerFactory.sysWriter) - assert.Equal(t.T(), format, defaultLoggerFactory.format) - assert.Equal(t.T(), cfg.DEBUG, defaultLoggerFactory.level) - assert.Equal(t.T(), fileSize, defaultLoggerFactory.logRotate.MaxFileSizeMb) - assert.Equal(t.T(), backupFileCount, defaultLoggerFactory.logRotate.BackupFileCount) - assert.True(t.T(), defaultLoggerFactory.logRotate.Compress) + require.Error(t, err) + assert.Contains(t, err.Error(), "UUID is smaller than requested size") + assert.Equal(t, "", mountUUID) } -func (t *LoggerTest) TestSetLogFormatToText() { - logConfig := cfg.DefaultLoggingConfig() - defaultLoggerFactory = &loggerFactory{ - file: nil, - level: string(logConfig.Severity), // setting log level to INFO by default - logRotate: logConfig.LogRotate, +func TestGenerateMountUUID_FailureDueToNegativeSize(t *testing.T) { + mountUUID, err := generateMountUUID(0) + + require.Error(t, err) + assert.Contains(t, err.Error(), "MountUUID must be positive") + assert.Equal(t, "", mountUUID) +} + +func TestSetupMountUUID_Success(t *testing.T) { + testCases := []struct { + name string + inBackgroundMode bool + mountUUIDEnv string + expectedID string + expectedMountUUIDRegex string + }{ + { + name: "ForegroundMode", + inBackgroundMode: false, + expectedID: "", + expectedMountUUIDRegex: "^[0-9a-f]{8}$", // default size for MountUUID is 8. + }, + { + name: "BackgroundModeWithInstanceID", + inBackgroundMode: true, + mountUUIDEnv: "12345678", + expectedID: "12345678", + }, } - testData := []struct { - format string - expectedOutput string + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Cleanup(func() { + // Initialize package level variables for each subtest to ensure isolation. + mountUUID = "" + setupMountUUIDOnce = sync.Once{} + }) + if tc.inBackgroundMode { + t.Setenv(GCSFuseInBackgroundMode, "true") + t.Setenv(MountUUIDEnvKey, tc.mountUUIDEnv) + } + + setupMountUUID() + + if tc.inBackgroundMode { + assert.Equal(t, tc.expectedID, mountUUID) + } else { + assert.Len(t, mountUUID, mountUUIDLength) + assert.Regexp(t, tc.expectedMountUUIDRegex, mountUUID) + } + }) + } +} + +func TestGetLogFHandler(t *testing.T) { + testCases := []struct { + name string + level slog.Level + levelString string + message string }{ { - "text", - textInfoString, + name: "Trace level", + level: LevelTrace, + levelString: "TRACE", + message: "trace message", + }, + { + name: "Debug level", + level: LevelDebug, + levelString: "DEBUG", + message: "debug message", + }, + { + name: "Info level", + level: LevelInfo, + levelString: "INFO", + message: "info message", }, { - "json", - jsonInfoString, + name: "Warn level", + level: LevelWarn, + levelString: "WARNING", + message: "warn message", }, { - "", - jsonInfoString, + name: "Error level", + level: LevelError, + levelString: "ERROR", + message: "error message", }, } - for _, test := range testData { - SetLogFormat(test.format) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Arrange + defaultLoggerFactory.format = "text" + var buf bytes.Buffer + // redirect logs to buffer and set level to TRACE to see all logs. + redirectLogsToGivenBuffer(&buf, cfg.TRACE) + + // Act + logFn := GetLogFHandler(tc.level) + logFn(tc.message) + + // Assert + expectedRegex := expectedLogRegex(t, "text", tc.levelString, tc.message) + actualLog := buf.String() + assert.Regexp(t, expectedRegex, actualLog) + }) + } - assert.NotNil(t.T(), defaultLoggerFactory) - assert.NotNil(t.T(), defaultLogger) - assert.Equal(t.T(), defaultLoggerFactory.format, test.format) - // Create a logger using defaultLoggerFactory that writes to buffer. + t.Run("Unsupported level returns trace logger", func(t *testing.T) { + // Arrange + defaultLoggerFactory.format = "text" var buf bytes.Buffer - redirectLogsToGivenBuffer(&buf, defaultLoggerFactory.level) - Infof("www.infoExample.com") - output := buf.String() - // Compare expected and actual log. - expectedRegexp := regexp.MustCompile(test.expectedOutput) - assert.True(t.T(), expectedRegexp.MatchString(output)) - } + // redirect logs to buffer and set level to TRACE to see all logs. + redirectLogsToGivenBuffer(&buf, cfg.TRACE) + message := "unsupported level message" + unsupportedLevel := slog.Level(99) + + // Act + logFn := GetLogFHandler(unsupportedLevel) + logFn(message) + + // Assert + logs := strings.Split(strings.TrimSpace(buf.String()), "\n") + require.Len(t, logs, 2) + expectedWarningRegex := expectedLogRegex(t, "text", "WARNING", fmt.Sprintf("logger: unsupported log level: %v", unsupportedLevel)) + assert.Regexp(t, expectedWarningRegex, logs[0]) + expectedTraceRegex := expectedLogRegex(t, "text", "TRACE", message) + assert.Regexp(t, expectedTraceRegex, logs[1]) + }) } diff --git a/internal/logger/slog_helper.go b/internal/logger/slog_helper.go index 48bd98ae78..1f26af52c8 100644 --- a/internal/logger/slog_helper.go +++ b/internal/logger/slog_helper.go @@ -19,7 +19,7 @@ import ( "strings" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" ) const ( @@ -37,7 +37,7 @@ const ( nanosKey = "nanos" ) -func setLoggingLevel(level string, programLevel *slog.LevelVar) { +func setLoggingLevel(level string) { switch level { // logs having severity >= the configured value will be logged. case cfg.TRACE: diff --git a/internal/monitor/bucket.go b/internal/monitor/bucket.go index 8f8568e895..09a8d7f80e 100644 --- a/internal/monitor/bucket.go +++ b/internal/monitor/bucket.go @@ -17,97 +17,35 @@ package monitor import ( "context" "fmt" - "io" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" - "github.com/googlecloudplatform/gcsfuse/v2/internal/monitor/tags" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "go.opencensus.io/plugin/ochttp" - "go.opencensus.io/stats" - "go.opencensus.io/stats/view" - "go.opencensus.io/tag" + storagev2 "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" ) -var ( - // OpenCensus measures - readBytesCountOC = stats.Int64("gcs/read_bytes_count", "The number of bytes read from GCS objects.", stats.UnitBytes) - readerCountOC = stats.Int64("gcs/reader_count", "The number of GCS object readers opened or closed.", stats.UnitDimensionless) - requestCountOC = stats.Int64("gcs/request_count", "The number of GCS requests processed.", stats.UnitDimensionless) - requestLatencyOC = stats.Float64("gcs/request_latency", "The latency of a GCS request.", stats.UnitMilliseconds) -) +// recordRequest records a request and its latency. +func recordRequest(ctx context.Context, metricHandle metrics.MetricHandle, method metrics.GcsMethod, start time.Time) { + metricHandle.GcsRequestCount(1, method) -// Initialize the metrics. -func init() { - // OpenCensus views (aggregated measures) - if err := view.Register( - &view.View{ - Name: "gcs/read_bytes_count", - Measure: readBytesCountOC, - Description: "The cumulative number of bytes read from GCS objects.", - Aggregation: view.Sum(), - }, - &view.View{ - Name: "gcs/reader_count", - Measure: readerCountOC, - Description: "The cumulative number of GCS object readers opened or closed.", - Aggregation: view.Sum(), - TagKeys: []tag.Key{tags.IOMethod}, - }, - &view.View{ - Name: "gcs/request_count", - Measure: requestCountOC, - Description: "The cumulative number of GCS requests processed.", - Aggregation: view.Sum(), - TagKeys: []tag.Key{tags.GCSMethod}, - }, - &view.View{ - Name: "gcs/request_latencies", - Measure: requestLatencyOC, - Description: "The cumulative distribution of the GCS request latencies.", - Aggregation: ochttp.DefaultLatencyDistribution, - TagKeys: []tag.Key{tags.GCSMethod}, - }); err != nil { - fmt.Printf("Failed to register OpenCensus metrics for GCS client library: %v", err) - } + metricHandle.GcsRequestLatencies(ctx, time.Since(start), method) } -// recordRequest records a request and its latency. -func recordRequest(ctx context.Context, method string, start time.Time) { - if err := stats.RecordWithTags( - ctx, - []tag.Mutator{ - tag.Upsert(tags.GCSMethod, method), - }, - requestCountOC.M(1), - ); err != nil { - // The error should be caused by a bad tag - logger.Errorf("Cannot record request count: %v", err) - } - - latencyUs := time.Since(start).Microseconds() - latencyMs := float64(latencyUs) / 1000.0 - if err := stats.RecordWithTags( - ctx, - []tag.Mutator{ - tag.Upsert(tags.GCSMethod, method), - }, - requestLatencyOC.M(latencyMs), - ); err != nil { - // The error should be caused by a bad tag - logger.Errorf("Cannot record request latency: %v", err) - } +func CaptureMultiRangeDownloaderMetrics(ctx context.Context, metricHandle metrics.MetricHandle, method metrics.GcsMethod, start time.Time) { + recordRequest(ctx, metricHandle, method, start) } // NewMonitoringBucket returns a gcs.Bucket that exports metrics for monitoring -func NewMonitoringBucket(b gcs.Bucket) gcs.Bucket { +func NewMonitoringBucket(b gcs.Bucket, m metrics.MetricHandle) gcs.Bucket { return &monitoringBucket{ - wrapped: b, + wrapped: b, + metricHandle: m, } } type monitoringBucket struct { - wrapped gcs.Bucket + wrapped gcs.Bucket + metricHandle metrics.MetricHandle } func (mb *monitoringBucket) Name() string { @@ -118,17 +56,24 @@ func (mb *monitoringBucket) BucketType() gcs.BucketType { return mb.wrapped.BucketType() } -func (mb *monitoringBucket) NewReader( - ctx context.Context, - req *gcs.ReadObjectRequest) (rc io.ReadCloser, err error) { +func setupReader(ctx context.Context, mb *monitoringBucket, req *gcs.ReadObjectRequest, method metrics.GcsMethod) (gcs.StorageReader, error) { startTime := time.Now() - rc, err = mb.wrapped.NewReader(ctx, req) + rc, err := mb.wrapped.NewReaderWithReadHandle(ctx, req) + if err == nil { - rc = newMonitoringReadCloser(ctx, req.Name, rc) + rc = newMonitoringReadCloser(ctx, req.Name, rc, mb.metricHandle) } - recordRequest(ctx, "NewReader", startTime) + recordRequest(ctx, mb.metricHandle, method, startTime) + return rc, err +} + +func (mb *monitoringBucket) NewReaderWithReadHandle( + ctx context.Context, + req *gcs.ReadObjectRequest) (rd gcs.StorageReader, err error) { + // Using NewReader here also as NewReader() method is not used and will be removed. + rd, err = setupReader(ctx, mb, req, metrics.GcsMethodNewReaderAttr) return } @@ -137,21 +82,35 @@ func (mb *monitoringBucket) CreateObject( req *gcs.CreateObjectRequest) (*gcs.Object, error) { startTime := time.Now() o, err := mb.wrapped.CreateObject(ctx, req) - recordRequest(ctx, "CreateObject", startTime) + recordRequest(ctx, mb.metricHandle, metrics.GcsMethodCreateObjectAttr, startTime) return o, err } func (mb *monitoringBucket) CreateObjectChunkWriter(ctx context.Context, req *gcs.CreateObjectRequest, chunkSize int, callBack func(bytesUploadedSoFar int64)) (gcs.Writer, error) { startTime := time.Now() wc, err := mb.wrapped.CreateObjectChunkWriter(ctx, req, chunkSize, callBack) - recordRequest(ctx, "CreateObjectChunkWriter", startTime) + recordRequest(ctx, mb.metricHandle, metrics.GcsMethodCreateObjectChunkWriterAttr, startTime) + return wc, err +} + +func (mb *monitoringBucket) CreateAppendableObjectWriter(ctx context.Context, req *gcs.CreateObjectChunkWriterRequest) (gcs.Writer, error) { + startTime := time.Now() + wc, err := mb.wrapped.CreateAppendableObjectWriter(ctx, req) + recordRequest(ctx, mb.metricHandle, metrics.GcsMethodCreateAppendableObjectWriterAttr, startTime) return wc, err } -func (mb *monitoringBucket) FinalizeUpload(ctx context.Context, w gcs.Writer) (*gcs.Object, error) { +func (mb *monitoringBucket) FinalizeUpload(ctx context.Context, w gcs.Writer) (*gcs.MinObject, error) { startTime := time.Now() o, err := mb.wrapped.FinalizeUpload(ctx, w) - recordRequest(ctx, "FinalizeUpload", startTime) + recordRequest(ctx, mb.metricHandle, metrics.GcsMethodFinalizeUploadAttr, startTime) + return o, err +} + +func (mb *monitoringBucket) FlushPendingWrites(ctx context.Context, w gcs.Writer) (*gcs.MinObject, error) { + startTime := time.Now() + o, err := mb.wrapped.FlushPendingWrites(ctx, w) + recordRequest(ctx, mb.metricHandle, metrics.GcsMethodFlushPendingWritesAttr, startTime) return o, err } @@ -160,7 +119,7 @@ func (mb *monitoringBucket) CopyObject( req *gcs.CopyObjectRequest) (*gcs.Object, error) { startTime := time.Now() o, err := mb.wrapped.CopyObject(ctx, req) - recordRequest(ctx, "CopyObject", startTime) + recordRequest(ctx, mb.metricHandle, metrics.GcsMethodCopyObjectAttr, startTime) return o, err } @@ -169,7 +128,7 @@ func (mb *monitoringBucket) ComposeObjects( req *gcs.ComposeObjectsRequest) (*gcs.Object, error) { startTime := time.Now() o, err := mb.wrapped.ComposeObjects(ctx, req) - recordRequest(ctx, "ComposeObjects", startTime) + recordRequest(ctx, mb.metricHandle, metrics.GcsMethodComposeObjectsAttr, startTime) return o, err } @@ -178,7 +137,7 @@ func (mb *monitoringBucket) StatObject( req *gcs.StatObjectRequest) (*gcs.MinObject, *gcs.ExtendedObjectAttributes, error) { startTime := time.Now() m, e, err := mb.wrapped.StatObject(ctx, req) - recordRequest(ctx, "StatObject", startTime) + recordRequest(ctx, mb.metricHandle, metrics.GcsMethodStatObjectAttr, startTime) return m, e, err } @@ -187,7 +146,7 @@ func (mb *monitoringBucket) ListObjects( req *gcs.ListObjectsRequest) (*gcs.Listing, error) { startTime := time.Now() listing, err := mb.wrapped.ListObjects(ctx, req) - recordRequest(ctx, "ListObjects", startTime) + recordRequest(ctx, mb.metricHandle, metrics.GcsMethodListObjectsAttr, startTime) return listing, err } @@ -196,7 +155,7 @@ func (mb *monitoringBucket) UpdateObject( req *gcs.UpdateObjectRequest) (*gcs.Object, error) { startTime := time.Now() o, err := mb.wrapped.UpdateObject(ctx, req) - recordRequest(ctx, "UpdateObject", startTime) + recordRequest(ctx, mb.metricHandle, metrics.GcsMethodUpdateObjectAttr, startTime) return o, err } @@ -205,72 +164,83 @@ func (mb *monitoringBucket) DeleteObject( req *gcs.DeleteObjectRequest) error { startTime := time.Now() err := mb.wrapped.DeleteObject(ctx, req) - recordRequest(ctx, "DeleteObject", startTime) + recordRequest(ctx, mb.metricHandle, metrics.GcsMethodDeleteObjectAttr, startTime) return err } +func (mb *monitoringBucket) MoveObject(ctx context.Context, req *gcs.MoveObjectRequest) (*gcs.Object, error) { + startTime := time.Now() + o, err := mb.wrapped.MoveObject(ctx, req) + recordRequest(ctx, mb.metricHandle, metrics.GcsMethodMoveObjectAttr, startTime) + return o, err +} + func (mb *monitoringBucket) DeleteFolder(ctx context.Context, folderName string) error { startTime := time.Now() err := mb.wrapped.DeleteFolder(ctx, folderName) - recordRequest(ctx, "DeleteFolder", startTime) + recordRequest(ctx, mb.metricHandle, metrics.GcsMethodDeleteFolderAttr, startTime) return err } -func (mb *monitoringBucket) GetFolder(ctx context.Context, folderName string) (*gcs.Folder, error) { +func (mb *monitoringBucket) GetFolder(ctx context.Context, req *gcs.GetFolderRequest) (*gcs.Folder, error) { startTime := time.Now() - folder, err := mb.wrapped.GetFolder(ctx, folderName) - recordRequest(ctx, "GetFolder", startTime) + folder, err := mb.wrapped.GetFolder(ctx, req) + recordRequest(ctx, mb.metricHandle, metrics.GcsMethodGetFolderAttr, startTime) return folder, err } func (mb *monitoringBucket) CreateFolder(ctx context.Context, folderName string) (*gcs.Folder, error) { startTime := time.Now() folder, err := mb.wrapped.CreateFolder(ctx, folderName) - recordRequest(ctx, "CreateFolder", startTime) + recordRequest(ctx, mb.metricHandle, metrics.GcsMethodCreateFolderAttr, startTime) return folder, err } func (mb *monitoringBucket) RenameFolder(ctx context.Context, folderName string, destinationFolderId string) (o *gcs.Folder, err error) { startTime := time.Now() o, err = mb.wrapped.RenameFolder(ctx, folderName, destinationFolderId) - recordRequest(ctx, "RenameFolder", startTime) + recordRequest(ctx, mb.metricHandle, metrics.GcsMethodRenameFolderAttr, startTime) return } +func (mb *monitoringBucket) NewMultiRangeDownloader( + ctx context.Context, req *gcs.MultiRangeDownloaderRequest) (mrd gcs.MultiRangeDownloader, err error) { + startTime := time.Now() + mrd, err = mb.wrapped.NewMultiRangeDownloader(ctx, req) + recordRequest(ctx, mb.metricHandle, metrics.GcsMethodNewMultiRangeDownloaderAttr, startTime) + return +} + +func (mb *monitoringBucket) GCSName(obj *gcs.MinObject) string { + return mb.wrapped.GCSName(obj) +} + // recordReader increments the reader count when it's opened or closed. -func recordReader(ctx context.Context, ioMethod string) { - if err := stats.RecordWithTags( - ctx, - []tag.Mutator{ - tag.Upsert(tags.IOMethod, ioMethod), - }, - readerCountOC.M(1), - ); err != nil { - logger.Errorf("Cannot record a reader %v: %v", ioMethod, err) - } +func recordReader(metricHandle metrics.MetricHandle, ioMethod metrics.IoMethod) { + metricHandle.GcsReaderCount(1, ioMethod) } // Monitoring on the object reader -func newMonitoringReadCloser(ctx context.Context, object string, rc io.ReadCloser) io.ReadCloser { - recordReader(ctx, "opened") +func newMonitoringReadCloser(ctx context.Context, object string, rc gcs.StorageReader, metricHandle metrics.MetricHandle) gcs.StorageReader { + recordReader(metricHandle, metrics.IoMethodOpenedAttr) return &monitoringReadCloser{ - ctx: ctx, - object: object, - wrapped: rc, + ctx: ctx, + object: object, + wrapped: rc, + metricHandle: metricHandle, } } type monitoringReadCloser struct { - ctx context.Context - object string - wrapped io.ReadCloser + ctx context.Context + object string + wrapped gcs.StorageReader + metricHandle metrics.MetricHandle } func (mrc *monitoringReadCloser) Read(p []byte) (n int, err error) { n, err = mrc.wrapped.Read(p) - if err == nil || err == io.EOF { - stats.Record(mrc.ctx, readBytesCountOC.M(int64(n))) - } + mrc.metricHandle.GcsReadBytesCount(int64(n)) return } @@ -279,6 +249,10 @@ func (mrc *monitoringReadCloser) Close() (err error) { if err != nil { return fmt.Errorf("close reader: %w", err) } - recordReader(mrc.ctx, "closed") + recordReader(mrc.metricHandle, metrics.IoMethodClosedAttr) return } + +func (mrc *monitoringReadCloser) ReadHandle() (rh storagev2.ReadHandle) { + return mrc.wrapped.ReadHandle() +} diff --git a/internal/monitor/exporter.go b/internal/monitor/exporter.go deleted file mode 100644 index 3a47496533..0000000000 --- a/internal/monitor/exporter.go +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitor - -import ( - "context" - "fmt" - "net/http" - "strings" - "time" - - "contrib.go.opencensus.io/exporter/ocagent" - "contrib.go.opencensus.io/exporter/prometheus" - "contrib.go.opencensus.io/exporter/stackdriver" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/common" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" - "go.opencensus.io/stats/view" -) - -func startStackdriverExporter(exporterIntervalSecs int64) common.ShutdownFn { - if exporterIntervalSecs <= 0 { - logger.Info("Not starting the Stackdriver exporter since exporter-interval is not specified") - return nil - } - logger.Info("Starting Stackdriver exporter") - if stackdriverExporter, err := enableStackdriverExporter(time.Duration(exporterIntervalSecs) * time.Second); err != nil { - logger.Errorf("Unable to start stackdriver exporter: %v", err) - return nil - } else { - logger.Info("Stackdriver exporter started") - return func(_ context.Context) error { - closeStackdriverExporter(stackdriverExporter) - return nil - } - } -} - -func startPrometheusCollectorExporter(port int64) common.ShutdownFn { - if port <= 0 { - logger.Info("Not starting the Prometheus exporter since port is not specified") - return nil - } - if exporter, server, err := enablePrometheusCollectorExporter(port); err != nil { - logger.Errorf("Unable to start Prometheus exporter: %v", err) - return nil - } else { - return func(_ context.Context) error { - closePrometheusCollectorExporter(exporter, server) - return nil - } - } -} - -func startOpenTelemetryCollectorExporter(address string) common.ShutdownFn { - if address == "" { - logger.Info("Not starting the OTel exporter since collector address is not specified") - return nil - } - if ocExporter, err := enableOpenTelemetryCollectorExporter(address); err != nil { - logger.Errorf("Unable to start OC Agent exporter: %v", err) - return nil - } else { - return func(_ context.Context) error { - closeOpenTelemetryCollectorExporter(ocExporter) - return nil - } - } -} - -// SetupOpenCensusExporters starts the relevant OpenCensus exporters. -func SetupOpenCensusExporters(c *cfg.Config) common.ShutdownFn { - stackdriverShutdownFn := startStackdriverExporter(c.Metrics.CloudMetricsExportIntervalSecs) - prometheusShutdownFn := startPrometheusCollectorExporter(c.Metrics.PrometheusPort) - oTelShutdownFn := startOpenTelemetryCollectorExporter(c.Monitoring.ExperimentalOpentelemetryCollectorAddress) - return common.JoinShutdownFunc(stackdriverShutdownFn, prometheusShutdownFn, oTelShutdownFn) -} - -// enableStackdriverExporter starts to collect monitoring metrics and exports -// them to Stackdriver iff the given interval is positive. -func enableStackdriverExporter(interval time.Duration) (*stackdriver.Exporter, error) { - var err error - var stackdriverExporter *stackdriver.Exporter - if stackdriverExporter, err = stackdriver.NewExporter(stackdriver.Options{ - ReportingInterval: interval, - OnError: func(err error) { - logger.Errorf("Fail to send metric: %v", err) - }, - - // For a local metric "http_sent_bytes", the Stackdriver metric type - // would be "custom.googleapis.com/gcsfuse/http_sent_bytes", display - // name would be "Http sent bytes". - MetricPrefix: "custom.googleapis.com/gcsfuse/", - GetMetricDisplayName: func(view *view.View) string { - name := strings.ReplaceAll(view.Name, "_", " ") - if len(name) > 0 { - name = strings.ToUpper(name[:1]) + name[1:] - } - return name - }, - }); err != nil { - return nil, fmt.Errorf("create stackdriver exporter: %w", err) - } - if err = stackdriverExporter.StartMetricsExporter(); err != nil { - return nil, fmt.Errorf("start stackdriver exporter: %w", err) - } - return stackdriverExporter, nil -} - -// closeStackdriverExporter ensures all collected metrics are sent to -// Stackdriver and closes the stackdriverExporter. -func closeStackdriverExporter(stackdriverExporter *stackdriver.Exporter) { - logger.Info("Stopping Stackdriver exporter") - stackdriverExporter.StopMetricsExporter() - logger.Info("Stackdriver exporter stopped") -} - -// enableOpenTelemetryCollectorExporter starts exporting monitoring metrics to -// the OpenTelemetry Collector at the given address. -// Details: https://opentelemetry.io/docs/collector/ -func enableOpenTelemetryCollectorExporter(address string) (*ocagent.Exporter, error) { - logger.Info("Starting OpenTelemetry collector exporter") - ocExporter, err := ocagent.NewExporter( - ocagent.WithAddress(address), - ocagent.WithServiceName("gcsfuse"), - ocagent.WithReconnectionPeriod(5*time.Second), - ) - if err != nil { - return nil, fmt.Errorf("create opentelementry collector exporter: %w", err) - } - - view.RegisterExporter(ocExporter) - logger.Info("OpenTelemetry collector exporter started") - return ocExporter, nil -} - -// closeOpenTelemetryCollectorExporter ensures all collected metrics are sent to -// the OpenTelemetry Collect and closes the exporter. -func closeOpenTelemetryCollectorExporter(ocExporter *ocagent.Exporter) { - logger.Info("Stopping OpenTelemetry collector exporter") - if err := ocExporter.Stop(); err != nil { - logger.Errorf("Error while stopping OpenTelemetry collector exporter") - return - } - logger.Info("OpenTelemetry collector exporter stopped") -} - -// enablePrometheusCollectorExporter starts exporting monitoring metrics for -// the Prometheus to scrape on the given port. -func enablePrometheusCollectorExporter(port int64) (*prometheus.Exporter, *http.Server, error) { - prometheusExporter, err := prometheus.NewExporter( - prometheus.Options{ - OnError: func(err error) { - logger.Errorf("Fail to collect metric: %v", err) - }, - }, - ) - if err != nil { - return nil, nil, fmt.Errorf("create Prometheus collector exporter: %w", err) - } - - view.RegisterExporter(prometheusExporter) - - mux := http.NewServeMux() - mux.HandleFunc("/metrics", prometheusExporter.ServeHTTP) - prometheusServer := &http.Server{ - Addr: fmt.Sprintf(":%d", port), - Handler: mux, - ReadTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, - MaxHeaderBytes: 1 << 20, - } - - go func() { - if err := prometheusServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { - logger.Errorf("Failed to start Prometheus server: %v", err) - } - }() - - logger.Info("Prometheus collector exporter started") - return prometheusExporter, prometheusServer, nil -} - -// closePrometheusCollectorExporter closes the Prometheus exporter. -func closePrometheusCollectorExporter(prometheusExporter *prometheus.Exporter, prometheusServer *http.Server) { - logger.Info("Stopping Prometheus exporter") - if prometheusServer != nil { - if err := prometheusServer.Shutdown(context.Background()); err != nil { - logger.Errorf("Failed to shutdown Prometheus server: %v", err) - } - } - - if prometheusExporter != nil { - view.UnregisterExporter(prometheusExporter) - } -} diff --git a/internal/monitor/otelexporters.go b/internal/monitor/otelexporters.go index eea69fb8ed..0311fdf0a3 100644 --- a/internal/monitor/otelexporters.go +++ b/internal/monitor/otelexporters.go @@ -19,74 +19,125 @@ import ( "fmt" "net/http" "strings" + "sync/atomic" "time" cloudmetric "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/common" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/common" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" "github.com/prometheus/client_golang/prometheus/promhttp" "go.opentelemetry.io/contrib/detectors/gcp" "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/exporters/prometheus" "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/exemplar" "go.opentelemetry.io/otel/sdk/metric/metricdata" "go.opentelemetry.io/otel/sdk/resource" semconv "go.opentelemetry.io/otel/semconv/v1.26.0" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const serviceName = "gcsfuse" const cloudMonitoringMetricPrefix = "custom.googleapis.com/gcsfuse/" +var allowedMetricPrefixes = []string{"fs/", "gcs/", "file_cache/", "buffered_read/", "grpc.", "read/"} + // SetupOTelMetricExporters sets up the metrics exporters -func SetupOTelMetricExporters(ctx context.Context, c *cfg.Config) (shutdownFn common.ShutdownFn) { - shutdownFns := make([]common.ShutdownFn, 0) +func SetupOTelMetricExporters(ctx context.Context, c *cfg.Config, mountID string) (shutdownFn common.ShutdownFn) { + var shutdownFns []common.ShutdownFn options := make([]metric.Option, 0) - opts, shutdownFn := setupPrometheus(c.Metrics.PrometheusPort) + opts, promShutdownFn := setupPrometheus(c.Metrics.PrometheusPort) options = append(options, opts...) - shutdownFns = append(shutdownFns, shutdownFn) + shutdownFns = append(shutdownFns, promShutdownFn) - opts, shutdownFn = setupCloudMonitoring(c.Metrics.CloudMetricsExportIntervalSecs) + opts = setupCloudMonitoring(c.Metrics.CloudMetricsExportIntervalSecs) options = append(options, opts...) - shutdownFns = append(shutdownFns, shutdownFn) - res, err := getResource(ctx) + res, err := getResource(ctx, mountID) if err != nil { logger.Errorf("Error while fetching resource: %v", err) } else { options = append(options, metric.WithResource(res)) } + options = append(options, metric.WithView(dropDisallowedMetricsView), metric.WithExemplarFilter(exemplar.AlwaysOffFilter)) + meterProvider := metric.NewMeterProvider(options...) - shutdownFns = append(shutdownFns, meterProvider.Shutdown) otel.SetMeterProvider(meterProvider) + shutdownFns = append(shutdownFns, meterProvider.Shutdown) + return common.JoinShutdownFunc(shutdownFns...) } -func setupCloudMonitoring(secs int64) ([]metric.Option, common.ShutdownFn) { +// dropUnwantedMetricsView is an OTel View that drops the metrics that don't match the allowed prefixes. +func dropDisallowedMetricsView(i metric.Instrument) (metric.Stream, bool) { + s := metric.Stream{Name: i.Name, Description: i.Description, Unit: i.Unit} + for _, prefix := range allowedMetricPrefixes { + if strings.HasPrefix(i.Name, prefix) { + return s, true + } + } + s.Aggregation = metric.AggregationDrop{} + return s, true +} + +func setupCloudMonitoring(secs int64) []metric.Option { if secs <= 0 { - return nil, nil + return nil } options := []cloudmetric.Option{ cloudmetric.WithMetricDescriptorTypeFormatter(metricFormatter), - cloudmetric.WithFilteredResourceAttributes(func(kv attribute.KeyValue) bool { - // Ensure that PID is available as a metric label on metrics explorer as it'll help distinguish between different mounts on the same node. - return cloudmetric.DefaultResourceAttributesFilter(kv) || - kv.Key == semconv.ProcessPIDKey - }), } exporter, err := cloudmetric.New(options...) if err != nil { logger.Errorf("Error while creating Google Cloud exporter:%v", err) - return nil, nil + return nil + } + + // Wrap the exporter to handle permission denied errors + wrappedExporter := &permissionAwareExporter{ + Exporter: exporter, + } + + reader := metric.NewPeriodicReader(wrappedExporter, metric.WithInterval(time.Duration(secs)*time.Second)) + return []metric.Option{metric.WithReader(reader)} +} + +// permissionAwareExporter wraps a metric.Exporter and disables itself if it encounters +// a PermissionDenied error. This prevents log spam when the environment lacks +// necessary permissions. +type permissionAwareExporter struct { + metric.Exporter + // disabled indicates whether the exporter has been permanently disabled. + disabled atomic.Bool +} + +func (e *permissionAwareExporter) Export(ctx context.Context, rm *metricdata.ResourceMetrics) error { + // Check if disabled before attempting export to save resources and avoid noise. + if e.disabled.Load() { + return nil } - r := metric.NewPeriodicReader(exporter, metric.WithInterval(time.Duration(secs)*time.Second)) - return []metric.Option{metric.WithReader(r)}, r.Shutdown + err := e.Exporter.Export(ctx, rm) + // If we get a PermissionDenied error, disable the exporter to prevent future attempts. + if err != nil && status.Code(err) == codes.PermissionDenied { + if e.disabled.CompareAndSwap(false, true) { + logger.Errorf("Disabling Cloud Monitoring exporter due to permission denied error: %v", err) + } + } + return err +} + +func (e *permissionAwareExporter) ForceFlush(ctx context.Context) error { + if e.disabled.Load() { + return nil + } + return e.Exporter.ForceFlush(ctx) } func metricFormatter(m metricdata.Metrics) string { @@ -102,7 +153,7 @@ func setupPrometheus(port int64) ([]metric.Option, common.ShutdownFn) { return nil, nil } shutdownCh := make(chan context.Context) - done := make(chan interface{}) + done := make(chan any) go serveMetrics(port, shutdownCh, done) return []metric.Option{metric.WithReader(exporter)}, func(ctx context.Context) error { shutdownCh <- ctx @@ -113,7 +164,7 @@ func setupPrometheus(port int64) ([]metric.Option, common.ShutdownFn) { } } -func serveMetrics(port int64, shutdownCh <-chan context.Context, done chan<- interface{}) { +func serveMetrics(port int64, shutdownCh <-chan context.Context, done chan<- any) { logger.Infof("Serving metrics at localhost:%d/metrics", port) mux := http.NewServeMux() mux.Handle("/metrics", promhttp.Handler()) @@ -142,7 +193,7 @@ func serveMetrics(port int64, shutdownCh <-chan context.Context, done chan<- int logger.Info("Prometheus collector exporter started") } -func getResource(ctx context.Context) (*resource.Resource, error) { +func getResource(ctx context.Context, mountID string) (*resource.Resource, error) { return resource.New(ctx, // Use the GCP resource detector to detect information about the GCP platform resource.WithDetectors(gcp.NewDetector()), @@ -150,6 +201,7 @@ func getResource(ctx context.Context) (*resource.Resource, error) { resource.WithAttributes( semconv.ServiceName(serviceName), semconv.ServiceVersion(common.GetVersion()), + semconv.ServiceInstanceID(mountID), ), ) } diff --git a/internal/monitor/otelexporters_test.go b/internal/monitor/otelexporters_test.go new file mode 100644 index 0000000000..5b0931c364 --- /dev/null +++ b/internal/monitor/otelexporters_test.go @@ -0,0 +1,91 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package monitor + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type mockExporter struct { + metric.Exporter + exportFunc func(context.Context, *metricdata.ResourceMetrics) error +} + +func (m *mockExporter) Export(ctx context.Context, rm *metricdata.ResourceMetrics) error { + if m.exportFunc != nil { + return m.exportFunc(ctx, rm) + } + return nil +} + +func (m *mockExporter) ForceFlush(ctx context.Context) error { + return nil +} + +func (m *mockExporter) Shutdown(ctx context.Context) error { + return nil +} + +func TestPermissionAwareExporter_ExportSuccess(t *testing.T) { + mock := &mockExporter{} + exporter := &permissionAwareExporter{Exporter: mock} + + err := exporter.Export(context.Background(), &metricdata.ResourceMetrics{}) + + assert.NoError(t, err) + assert.False(t, exporter.disabled.Load()) +} + +func TestPermissionAwareExporter_ExportPermissionDenied(t *testing.T) { + mock := &mockExporter{ + exportFunc: func(ctx context.Context, rm *metricdata.ResourceMetrics) error { + return status.Error(codes.PermissionDenied, "permission denied") + }, + } + exporter := &permissionAwareExporter{Exporter: mock} + // First call fails and disables + err := exporter.Export(context.Background(), &metricdata.ResourceMetrics{}) + require.Error(t, err) + require.Equal(t, codes.PermissionDenied, status.Code(err)) + require.True(t, exporter.disabled.Load()) + + // Second call should be skipped (return nil) + err = exporter.Export(context.Background(), &metricdata.ResourceMetrics{}) + + assert.NoError(t, err) +} + +func TestPermissionAwareExporter_ExportOtherError(t *testing.T) { + mock := &mockExporter{ + exportFunc: func(ctx context.Context, rm *metricdata.ResourceMetrics) error { + return errors.New("some other error") + }, + } + exporter := &permissionAwareExporter{Exporter: mock} + + err := exporter.Export(context.Background(), &metricdata.ResourceMetrics{}) + + assert.Error(t, err) + assert.False(t, exporter.disabled.Load()) +} diff --git a/internal/monitor/reader.go b/internal/monitor/reader.go deleted file mode 100644 index 5d3ef2f30a..0000000000 --- a/internal/monitor/reader.go +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitor - -import ( - "log" - "strconv" - "time" - - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" - "github.com/googlecloudplatform/gcsfuse/v2/internal/monitor/tags" - "go.opencensus.io/plugin/ochttp" - "go.opencensus.io/stats" - "go.opencensus.io/stats/view" - "go.opencensus.io/tag" - "golang.org/x/net/context" -) - -var ( - // When a first read call is made by the user, we either fetch entire file or x number of bytes from GCS based on the request. - // Now depending on the pagesize multiple read calls will be issued by user to read the entire file. These - // requests will be served from the downloaded data. - // This metric captures only the requests made to GCS, not the subsequent page calls. - gcsReadCountOC = stats.Int64("gcs/read_count", - "Specifies the number of gcs reads made along with type - Sequential/Random", - UnitDimensionless) - downloadBytesCountOC = stats.Int64("gcs/download_bytes_count", - "The cumulative number of bytes downloaded from GCS along with type - Sequential/Random", - UnitBytes) - fileCacheReadCountOC = stats.Int64("file_cache/read_count", - "Specifies the number of read requests made via file cache along with type - Sequential/Random and cache hit - true/false", - UnitDimensionless) - fileCacheReadBytesCountOC = stats.Int64("file_cache/read_bytes_count", - "The cumulative number of bytes read from file cache along with read type - Sequential/Random", - UnitBytes) - fileCacheReadLatencyOC = stats.Int64("file_cache/read_latency", - "Latency of read from file cache along with cache hit - true/false", - UnitMicroseconds) -) - -const NanosecondsInOneMillisecond = 1000000 - -// Initialize the metrics. -func init() { - // GCS related metrics - if err := view.Register( - &view.View{ - Name: "gcs/read_count", - Measure: gcsReadCountOC, - Description: "Specifies the number of gcs reads made along with type - Sequential/Random", - Aggregation: view.Sum(), - TagKeys: []tag.Key{tags.ReadType}, - }, - &view.View{ - Name: "gcs/download_bytes_count", - Measure: downloadBytesCountOC, - Description: "The cumulative number of bytes downloaded from GCS along with type - Sequential/Random", - Aggregation: view.Sum(), - TagKeys: []tag.Key{tags.ReadType}, - }, - // File cache related metrics - &view.View{ - Name: "file_cache/read_count", - Measure: fileCacheReadCountOC, - Description: "Specifies the number of read requests made via file cache along with type - Sequential/Random and cache hit - true/false", - Aggregation: view.Sum(), - TagKeys: []tag.Key{tags.ReadType, tags.CacheHit}, - }, - &view.View{ - Name: "file_cache/read_bytes_count", - Measure: fileCacheReadBytesCountOC, - Description: "The cumulative number of bytes read from file cache along with read type - Sequential/Random", - Aggregation: view.Sum(), - TagKeys: []tag.Key{tags.ReadType}, - }, - &view.View{ - Name: "file_cache/read_latencies", - Measure: fileCacheReadLatencyOC, - Description: "The cumulative distribution of the file cache read latencies along with cache hit - true/false", - Aggregation: ochttp.DefaultLatencyDistribution, - TagKeys: []tag.Key{tags.CacheHit}, - }, - ); err != nil { - log.Fatalf("Failed to register the reader view: %v", err) - } -} - -func CaptureGCSReadMetrics(ctx context.Context, readType string, requestedDataSize int64) { - if err := stats.RecordWithTags( - ctx, - []tag.Mutator{ - tag.Upsert(tags.ReadType, readType), - }, - gcsReadCountOC.M(1), - ); err != nil { - // Error in recording gcsReadCountOC. - logger.Errorf("Cannot record gcsReadCountOC %v", err) - } - - if err := stats.RecordWithTags( - ctx, - []tag.Mutator{ - tag.Upsert(tags.ReadType, readType), - }, - downloadBytesCountOC.M(requestedDataSize), - ); err != nil { - // Error in recording downloadBytesCountOC. - logger.Errorf("Cannot record downloadBytesCountOC %v", err) - } -} - -func CaptureFileCacheMetrics(ctx context.Context, readType string, readDataSize int, cacheHit bool, readLatency time.Duration) { - if err := stats.RecordWithTags( - ctx, - []tag.Mutator{ - tag.Upsert(tags.ReadType, readType), - tag.Upsert(tags.CacheHit, strconv.FormatBool(cacheHit)), - }, - fileCacheReadCountOC.M(1), - ); err != nil { - // Error in recording fileCacheReadCountOC. - logger.Errorf("Cannot record fileCacheReadCountOC %v", err) - } - - if err := stats.RecordWithTags( - ctx, - []tag.Mutator{ - tag.Upsert(tags.ReadType, readType), - }, - fileCacheReadBytesCountOC.M(int64(readDataSize)), - ); err != nil { - // Error in recording fileCacheReadBytesCountOC. - logger.Errorf("Cannot record fileCacheReadBytesCountOC %v", err) - } - - if err := stats.RecordWithTags( - ctx, - []tag.Mutator{ - tag.Upsert(tags.CacheHit, strconv.FormatBool(cacheHit)), - }, - fileCacheReadLatencyOC.M(readLatency.Microseconds()), - ); err != nil { - // Error in recording fileCacheReadLatencyOC. - logger.Errorf("Cannot record fileCacheReadLatencyOC %v", err) - } -} diff --git a/internal/monitor/tags/tags.go b/internal/monitor/tags/tags.go deleted file mode 100644 index 77592afa6b..0000000000 --- a/internal/monitor/tags/tags.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package tags provides the tags to annotate the monitoring metrics. -package tags - -import ( - "go.opencensus.io/tag" -) - -var ( - // IOMethod annotates the event that opens or closes a connection or file. - IOMethod = tag.MustNewKey("io_method") - - // GCSMethod annotates the method called in the GCS client library. - GCSMethod = tag.MustNewKey("gcs_method") - - // FSOp annotates the file system op processed. - FSOp = tag.MustNewKey("fs_op") - - // FSErrCategory reduces the cardinality of FSError by grouping errors together. - FSErrCategory = tag.MustNewKey("fs_error_category") - - // ReadType annotates the read operation with the type - Sequential/Random - ReadType = tag.MustNewKey("read_type") - - // CacheHit annotates the read operation from file cache with true or false. - CacheHit = tag.MustNewKey("cache_hit") -) diff --git a/internal/monitor/traceexporter.go b/internal/monitor/traceexporter.go index 84e4d89b7e..2cb9ec89ab 100644 --- a/internal/monitor/traceexporter.go +++ b/internal/monitor/traceexporter.go @@ -18,22 +18,18 @@ import ( "context" cloudtrace "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/common" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/common" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" - "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/trace" - - "go.opentelemetry.io/contrib/detectors/gcp" - semconv "go.opentelemetry.io/otel/semconv/v1.26.0" ) // SetupTracing bootstraps the OpenTelemetry tracing pipeline. -func SetupTracing(ctx context.Context, c *cfg.Config) common.ShutdownFn { - tp, shutdown, err := newTraceProvider(ctx, c) +func SetupTracing(ctx context.Context, c *cfg.Config, mountID string) common.ShutdownFn { + tp, shutdown, err := newTraceProvider(ctx, c, mountID) if err != nil { logger.Errorf("error occurred while setting up tracing: %v", err) return nil @@ -46,46 +42,69 @@ func SetupTracing(ctx context.Context, c *cfg.Config) common.ShutdownFn { return nil } -func newTraceProvider(ctx context.Context, c *cfg.Config) (trace.TracerProvider, common.ShutdownFn, error) { - switch c.Monitoring.ExperimentalTracingMode { - case "stdout": - return newStdoutTraceProvider() - case "gcptrace": - return newGCPCloudTraceExporter(ctx, c) - default: - return nil, nil, nil +type exporterFactory func() (sdktrace.SpanExporter, error) + +func newTraceProvider(ctx context.Context, c *cfg.Config, mountID string) (trace.TracerProvider, common.ShutdownFn, error) { + var opts []sdktrace.TracerProviderOption + exporterNames := c.Trace.Exporters + + exporterRegistry := map[string]exporterFactory{ + "stdout": func() (sdktrace.SpanExporter, error) { + return newStdoutTraceExporter() + }, + "gcpexporter": func() (sdktrace.SpanExporter, error) { + return newGCPCloudTraceExporter(c) + }, } -} -func newStdoutTraceProvider() (trace.TracerProvider, common.ShutdownFn, error) { - exporter, err := stdouttrace.New( - stdouttrace.WithPrettyPrint()) + + for _, name := range exporterNames { + if expFactory, ok := exporterRegistry[name]; ok { + exporter, err := expFactory() + + if err != nil { + logger.Errorf("failed to initialize %s exporter: %s", name, err) + return nil, nil, err + } + + opts = append(opts, sdktrace.WithBatcher(exporter)) + } + } + + res, err := getResource(ctx, mountID) if err != nil { return nil, nil, err } - tp := sdktrace.NewTracerProvider(sdktrace.WithBatcher(exporter)) + opts = append(opts, sdktrace.WithResource(res), sdktrace.WithSampler(sdktrace.TraceIDRatioBased(c.Trace.SamplingRatio))) + + tp := sdktrace.NewTracerProvider(opts...) + return tp, tp.Shutdown, nil } -func newGCPCloudTraceExporter(ctx context.Context, c *cfg.Config) (*sdktrace.TracerProvider, common.ShutdownFn, error) { - exporter, err := cloudtrace.New() +func newStdoutTraceExporter() (sdktrace.SpanExporter, error) { + exporter, err := stdouttrace.New( + stdouttrace.WithPrettyPrint()) + if err != nil { - return nil, nil, err + return nil, err } - res, err := resource.New(ctx, - // Use the GCP resource detector to detect information about the GCP platform - resource.WithDetectors(gcp.NewDetector()), - resource.WithTelemetrySDK(), - resource.WithAttributes( - semconv.ServiceName(serviceName), - semconv.ServiceVersion(common.GetVersion()), - ), - ) - if err != nil { - return nil, nil, err + + return exporter, nil +} + +func newGCPCloudTraceExporter(c *cfg.Config) (sdktrace.SpanExporter, error) { + var traceOptions []cloudtrace.Option + + if c.Trace.ProjectId != "" { + traceOptions = append(traceOptions, cloudtrace.WithProjectID(c.Trace.ProjectId)) } - tp := sdktrace.NewTracerProvider(sdktrace.WithBatcher(exporter), sdktrace.WithResource(res), sdktrace.WithSampler(sdktrace.TraceIDRatioBased(c.Monitoring.ExperimentalTracingSamplingRatio))) + exporter, err := cloudtrace.New(traceOptions...) - return tp, tp.Shutdown, nil + if err != nil { + return nil, err + } + + return exporter, nil } diff --git a/internal/mount/flag.go b/internal/mount/flag.go index 1e15170d6a..28e969a4f6 100644 --- a/internal/mount/flag.go +++ b/internal/mount/flag.go @@ -36,6 +36,10 @@ const ( // DefaultStatCacheCapacity is the default value for stat-cache-capacity. // This is equivalent of setting metadata-cache: stat-cache-max-size-mb. DefaultStatCacheCapacity = 20460 + + // DefaultTypeCacheSizeMB is the default value for type-cache-max-size-mb. + // This is equivalent of setting metadata-cache: type-cache-max-size-mb. + DefaultTypeCacheSizeMB = 4 ) func (cp ClientProtocol) IsValid() bool { @@ -67,7 +71,7 @@ func ParseOptions(m map[string]string, s string) { // as I can tell there is no way to properly escape or quote a comma in the // options list for an fstab entry. So put our fingers in our ears and hope // that nobody needs a comma. - for _, p := range strings.Split(s, ",") { + for p := range strings.SplitSeq(s, ",") { var name string var value string diff --git a/internal/perf/cpu.go b/internal/perf/cpu.go index 2551458664..0eccc376ab 100644 --- a/internal/perf/cpu.go +++ b/internal/perf/cpu.go @@ -22,7 +22,7 @@ import ( "syscall" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" ) func HandleCPUProfileSignals() { diff --git a/internal/perf/memory.go b/internal/perf/memory.go index bac85e1d65..5818129652 100644 --- a/internal/perf/memory.go +++ b/internal/perf/memory.go @@ -23,7 +23,7 @@ import ( "syscall" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" ) const ( diff --git a/internal/perms/perms_test.go b/internal/perms/perms_test.go index 9cf3d8f837..9a85cd4ad5 100644 --- a/internal/perms/perms_test.go +++ b/internal/perms/perms_test.go @@ -18,7 +18,7 @@ package perms_test import ( "testing" - "github.com/googlecloudplatform/gcsfuse/v2/internal/perms" + "github.com/googlecloudplatform/gcsfuse/v3/internal/perms" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/internal/profiler/cloud_profiler.go b/internal/profiler/cloud_profiler.go new file mode 100644 index 0000000000..4a14d4cfc0 --- /dev/null +++ b/internal/profiler/cloud_profiler.go @@ -0,0 +1,55 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package profiler + +import ( + cloudprofiler "cloud.google.com/go/profiler" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "google.golang.org/api/option" +) + +type startFunctionType func(cloudprofiler.Config, ...option.ClientOption) error + +// SetupCloudProfiler initializes and starts the Cloud Profiler based on the application configuration. +func SetupCloudProfiler(mpc *cfg.CloudProfilerConfig) error { + return setupCloudProfiler(mpc, cloudprofiler.Start) +} + +// setupCloudProfiler is an internal helper function with a configurable start function +// for testing purposes. +func setupCloudProfiler(mpc *cfg.CloudProfilerConfig, startFunc startFunctionType) error { + if !mpc.Enabled { + return nil + } + + pConfig := cloudprofiler.Config{ + Service: mpc.ServiceName, + ServiceVersion: mpc.Label, + MutexProfiling: mpc.Mutex, + NoCPUProfiling: !mpc.Cpu, + NoAllocProfiling: !mpc.AllocatedHeap, + NoHeapProfiling: !mpc.Heap, + NoGoroutineProfiling: !mpc.Goroutines, + AllocForceGC: true, + } + + if err := startFunc(pConfig); err != nil { + return err + } + + logger.Infof("Cloud Profiler started successfully with Service Name: %s for version: %s", mpc.ServiceName, mpc.Label) + return nil +} diff --git a/internal/profiler/cloud_profiler_test.go b/internal/profiler/cloud_profiler_test.go new file mode 100644 index 0000000000..b75b5484d2 --- /dev/null +++ b/internal/profiler/cloud_profiler_test.go @@ -0,0 +1,87 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package profiler + +import ( + "errors" + "testing" + + cloudprofiler "cloud.google.com/go/profiler" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/api/option" +) + +func TestSetupCloudProfiler_Disabled(t *testing.T) { + mockProfilerStartCalled := false + profilerStart := func(_ cloudprofiler.Config, _ ...option.ClientOption) error { + mockProfilerStartCalled = true + return nil + } + cloudProfilerConfig := &cfg.CloudProfilerConfig{ + Enabled: false, + } + + err := setupCloudProfiler(cloudProfilerConfig, profilerStart) + + require.NoError(t, err, "SetupCloudProfiler should not return an error") + assert.False(t, mockProfilerStartCalled, "profilerStart should not be called when profiler is disabled") +} + +func TestSetupCloudProfiler_EnabledSuccess(t *testing.T) { + var capturedProfilerConfig cloudprofiler.Config + mockProfilerStartCalled := false + profilerStart := func(pcfg cloudprofiler.Config, _ ...option.ClientOption) error { + mockProfilerStartCalled = true + capturedProfilerConfig = pcfg + return nil + } + cloudProfilerConfig := &cfg.CloudProfilerConfig{ + Enabled: true, + Label: "v1.2.3", + Mutex: true, + Cpu: true, + AllocatedHeap: false, + Heap: true, + Goroutines: false, + ServiceName: "gcsfuse", + } + + err := setupCloudProfiler(cloudProfilerConfig, profilerStart) + + require.NoError(t, err, "SetupCloudProfiler should not return an error") + require.True(t, mockProfilerStartCalled, "profilerStart should be called") + assert.Equal(t, "gcsfuse", capturedProfilerConfig.Service) + assert.Equal(t, "v1.2.3", capturedProfilerConfig.ServiceVersion) + assert.Equal(t, true, capturedProfilerConfig.MutexProfiling) + assert.Equal(t, false, capturedProfilerConfig.NoCPUProfiling) + assert.Equal(t, true, capturedProfilerConfig.NoAllocProfiling) + assert.Equal(t, false, capturedProfilerConfig.NoHeapProfiling) + assert.Equal(t, true, capturedProfilerConfig.NoGoroutineProfiling) + assert.True(t, capturedProfilerConfig.AllocForceGC) +} + +func TestSetupCloudProfiler_EnabledStartFails(t *testing.T) { + expectedErr := errors.New("profiler failed to start") + profilerStart := func(_ cloudprofiler.Config, _ ...option.ClientOption) error { return expectedErr } + cloudProfilerConfig := &cfg.CloudProfilerConfig{ + Enabled: true, + } + + err := setupCloudProfiler(cloudProfilerConfig, profilerStart) + + assert.EqualError(t, err, expectedErr.Error()) +} diff --git a/internal/ratelimit/throttle_test.go b/internal/ratelimit/throttle_test.go index 03fe8979df..e7680de59f 100644 --- a/internal/ratelimit/throttle_test.go +++ b/internal/ratelimit/throttle_test.go @@ -30,7 +30,7 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/ratelimit" + "github.com/googlecloudplatform/gcsfuse/v3/internal/ratelimit" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "golang.org/x/net/context" diff --git a/internal/ratelimit/throttled_bucket.go b/internal/ratelimit/throttled_bucket.go index 886944f7b2..13cb7e345c 100644 --- a/internal/ratelimit/throttled_bucket.go +++ b/internal/ratelimit/throttled_bucket.go @@ -17,7 +17,8 @@ package ratelimit import ( "io" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + storagev2 "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "golang.org/x/net/context" ) @@ -53,10 +54,9 @@ func (b *throttledBucket) Name() string { func (b *throttledBucket) BucketType() gcs.BucketType { return b.wrapped.BucketType() } - -func (b *throttledBucket) NewReader( +func (b *throttledBucket) NewReaderWithReadHandle( ctx context.Context, - req *gcs.ReadObjectRequest) (rc io.ReadCloser, err error) { + req *gcs.ReadObjectRequest) (rd gcs.StorageReader, err error) { // Wait for permission to call through. err = b.opThrottle.Wait(ctx, 1) @@ -65,15 +65,15 @@ func (b *throttledBucket) NewReader( } // Call through. - rc, err = b.wrapped.NewReader(ctx, req) + rd, err = b.wrapped.NewReaderWithReadHandle(ctx, req) if err != nil { return } // Wrap the result in a throttled layer. - rc = &readerCloser{ - Reader: ThrottledReader(ctx, rc, b.egressThrottle), - Closer: rc, + rd = &throttledGCSReader{ + Reader: ThrottledReader(ctx, rd, b.egressThrottle), + Closer: rd, } return @@ -107,7 +107,20 @@ func (b *throttledBucket) CreateObjectChunkWriter(ctx context.Context, req *gcs. return } -func (b *throttledBucket) FinalizeUpload(ctx context.Context, w gcs.Writer) (*gcs.Object, error) { +func (b *throttledBucket) CreateAppendableObjectWriter(ctx context.Context, req *gcs.CreateObjectChunkWriterRequest) (wc gcs.Writer, err error) { + // Wait for permission to call through. + err = b.opThrottle.Wait(ctx, 1) + if err != nil { + return + } + + // Call through. + wc, err = b.wrapped.CreateAppendableObjectWriter(ctx, req) + + return +} + +func (b *throttledBucket) FinalizeUpload(ctx context.Context, w gcs.Writer) (*gcs.MinObject, error) { // FinalizeUpload is not throttled to prevent permanent data loss in case the // limiter's burst size is exceeded. // Note: CreateObjectChunkWriter, a prerequisite for FinalizeUpload, @@ -115,6 +128,14 @@ func (b *throttledBucket) FinalizeUpload(ctx context.Context, w gcs.Writer) (*gc return b.wrapped.FinalizeUpload(ctx, w) } +func (b *throttledBucket) FlushPendingWrites(ctx context.Context, w gcs.Writer) (*gcs.MinObject, error) { + // FlushPendingWrites is not throttled to prevent permanent data loss in case the + // limiter's burst size is exceeded. + // Note: CreateObjectChunkWriter, a prerequisite for FlushPendingWrites, + // is throttled. + return b.wrapped.FlushPendingWrites(ctx, w) +} + func (b *throttledBucket) CopyObject( ctx context.Context, req *gcs.CopyObjectRequest) (o *gcs.Object, err error) { @@ -205,6 +226,18 @@ func (b *throttledBucket) DeleteObject( return } +func (b *throttledBucket) MoveObject(ctx context.Context, req *gcs.MoveObjectRequest) (*gcs.Object, error) { + // Wait for permission to call through. + err := b.opThrottle.Wait(ctx, 1) + if err != nil { + return nil, err + } + + // Call through. + o, err := b.wrapped.MoveObject(ctx, req) + + return o, err +} func (b *throttledBucket) DeleteFolder(ctx context.Context, folderName string) (err error) { // Wait for permission to call through. err = b.opThrottle.Wait(ctx, 1) @@ -231,7 +264,7 @@ func (b *throttledBucket) RenameFolder(ctx context.Context, folderName string, d return } -func (b *throttledBucket) GetFolder(ctx context.Context, folderName string) (folder *gcs.Folder, err error) { +func (b *throttledBucket) GetFolder(ctx context.Context, req *gcs.GetFolderRequest) (folder *gcs.Folder, err error) { // Wait for permission to call through. err = b.opThrottle.Wait(ctx, 1) if err != nil { @@ -239,7 +272,7 @@ func (b *throttledBucket) GetFolder(ctx context.Context, folderName string) (fol } // Call through. - folder, err = b.wrapped.GetFolder(ctx, folderName) + folder, err = b.wrapped.GetFolder(ctx, req) return folder, err } @@ -257,23 +290,39 @@ func (b *throttledBucket) CreateFolder(ctx context.Context, folderName string) ( return folder, err } +func (b *throttledBucket) NewMultiRangeDownloader( + ctx context.Context, req *gcs.MultiRangeDownloaderRequest) (mrd gcs.MultiRangeDownloader, err error) { + // Call through. + mrd, err = b.wrapped.NewMultiRangeDownloader(ctx, req) + return +} + +func (b *throttledBucket) GCSName(obj *gcs.MinObject) string { + return b.wrapped.GCSName(obj) +} + //////////////////////////////////////////////////////////////////////// // readerCloser //////////////////////////////////////////////////////////////////////// // An io.ReadCloser that forwards read requests to an io.Reader and close -// requests to an io.Closer. -type readerCloser struct { +// , readHandle requests to gcs.StorageReader. +type throttledGCSReader struct { Reader io.Reader - Closer io.Closer + Closer gcs.StorageReader } -func (rc *readerCloser) Read(p []byte) (n int, err error) { +func (rc *throttledGCSReader) Read(p []byte) (n int, err error) { n, err = rc.Reader.Read(p) return } -func (rc *readerCloser) Close() (err error) { +func (rc *throttledGCSReader) Close() (err error) { err = rc.Closer.Close() return } + +func (rc *throttledGCSReader) ReadHandle() (rh storagev2.ReadHandle) { + rh = rc.Closer.ReadHandle() + return +} diff --git a/internal/storage/bucket_handle.go b/internal/storage/bucket_handle.go index e75aa468be..83839f5d8f 100644 --- a/internal/storage/bucket_handle.go +++ b/internal/storage/bucket_handle.go @@ -21,22 +21,18 @@ package storage import ( "context" - "errors" "fmt" "io" - "net/http" "time" "cloud.google.com/go/storage" - control "cloud.google.com/go/storage/control/apiv2" "cloud.google.com/go/storage/control/apiv2/controlpb" "github.com/googleapis/gax-go/v2" - "github.com/googleapis/gax-go/v2/apierror" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" - "google.golang.org/api/googleapi" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" "google.golang.org/api/iterator" + "google.golang.org/grpc/metadata" ) const FullFolderPathHNS = "projects/_/buckets/%s/folders/%s" @@ -44,10 +40,12 @@ const FullBucketPathHNS = "projects/_/buckets/%s" type bucketHandle struct { gcs.Bucket - bucket *storage.BucketHandle - bucketName string - bucketType gcs.BucketType - controlClient StorageControlClient + bucket *storage.BucketHandle + bucketName string + bucketType *gcs.BucketType + controlClient StorageControlClient + billingProject string + writeConfig *cfg.WriteConfig } func (bh *bucketHandle) Name() string { @@ -55,48 +53,25 @@ func (bh *bucketHandle) Name() string { } func (bh *bucketHandle) BucketType() gcs.BucketType { - var nilControlClient *control.StorageControlClient = nil - // Note: The first invocation of this method will be slower due to a required Google Cloud Storage (GCS) fetch. - // Subsequent calls will be significantly faster as the results are cached in memory. - // While this operation is thread-safe, parallel calls during the initial fetch can result in redundant GCS requests. - // To avoid this, it's advisable to call this initially while mounting. - if bh.bucketType == gcs.Nil { - if bh.controlClient == nilControlClient { - bh.bucketType = gcs.NonHierarchical - return bh.bucketType - } - startTime := time.Now() - logger.Infof("GetStorageLayout <- (%s)", bh.bucketName) - storageLayout, err := bh.getStorageLayout() - duration := time.Since(startTime) - // In case bucket does not exist, set type unknown instead of panic. - if err != nil { - bh.bucketType = gcs.Unknown - logger.Errorf("Error returned from GetStorageLayout: %v", err) - return bh.bucketType - } - logger.Infof("GetStorageLayout -> (%s) %v msec", bh.bucketName, duration.Milliseconds()) - - hierarchicalNamespace := storageLayout.GetHierarchicalNamespace() - if hierarchicalNamespace != nil && hierarchicalNamespace.Enabled { - bh.bucketType = gcs.Hierarchical - return bh.bucketType - } - - bh.bucketType = gcs.NonHierarchical - } - - return bh.bucketType + return *bh.bucketType } -func (bh *bucketHandle) NewReader( +func (bh *bucketHandle) NewReaderWithReadHandle( ctx context.Context, - req *gcs.ReadObjectRequest) (io.ReadCloser, error) { + req *gcs.ReadObjectRequest) (reader gcs.StorageReader, err error) { + + defer func() { + err = gcs.GetGCSError(err) + }() + // Initialising the starting offset and the length to be read by the reader. start := int64(0) length := int64(-1) - // Following the semantics of NewReader method. Passing start, length as 0,-1 reads the entire file. - // https://github.com/GoogleCloudPlatform/gcsfuse/blob/34211af652dbaeb012b381a3daf3c94b95f65e00/vendor/cloud.google.com/go/storage/reader.go#L75 + // Following the semantics of NewRangeReader method. + // If length is negative, the object is read until the end. + // If offset is negative, the object is read abs(offset) bytes from the end, + // and length must also be negative to indicate all remaining bytes will be read. + // Ref: https://github.com/GoogleCloudPlatform/gcsfuse/blob/34211af652dbaeb012b381a3daf3c94b95f65e00/vendor/cloud.google.com/go/storage/reader.go#L80 if req.Range != nil { start = int64((*req.Range).Start) end := int64((*req.Range).Limit) @@ -114,10 +89,27 @@ func (bh *bucketHandle) NewReader( obj = obj.ReadCompressed(true) } + // Insert ReadHandle into objectHandle if present. + // Objects that have been opened can be opened again using readHandle at lower latency. + // This produces the exact same object and generation and does not check if + // the generation is still the newest one. + if req.ReadHandle != nil { + obj = obj.ReadHandle(req.ReadHandle) + } + // NewRangeReader creates a "storage.Reader" object which is also io.ReadCloser since it contains both Read() and Close() methods present in io.ReadCloser interface. - return obj.NewRangeReader(ctx, start, length) + storageReader, err := obj.NewRangeReader(ctx, start, length) + if err == nil { + reader = newGCSFullReadCloser(storageReader) + } + return } -func (bh *bucketHandle) DeleteObject(ctx context.Context, req *gcs.DeleteObjectRequest) error { + +func (bh *bucketHandle) DeleteObject(ctx context.Context, req *gcs.DeleteObjectRequest) (err error) { + defer func() { + err = gcs.GetGCSError(err) + }() + obj := bh.bucket.Object(req.Name) // Switching to the requested generation of the object. By default, generation @@ -131,38 +123,23 @@ func (bh *bucketHandle) DeleteObject(ctx context.Context, req *gcs.DeleteObjectR obj = obj.If(storage.Conditions{MetagenerationMatch: *req.MetaGenerationPrecondition}) } - err := obj.Delete(ctx) - // If storage object does not exist, httpclient is returning ErrObjectNotExist error instead of googleapi error - // https://github.com/GoogleCloudPlatform/gcsfuse/blob/7ad451c6f2ead7992e030503e5b66c555b2ebf71/vendor/cloud.google.com/go/storage/http_client.go#L399 + err = obj.Delete(ctx) if err != nil { - switch ee := err.(type) { - case *googleapi.Error: - if ee.Code == http.StatusPreconditionFailed { - err = &gcs.PreconditionError{Err: ee} - } - default: - if err == storage.ErrObjectNotExist { - err = &gcs.NotFoundError{Err: storage.ErrObjectNotExist} - } else { - err = fmt.Errorf("error in deleting object: %w", err) - } - } + err = fmt.Errorf("error in deleting object: %w", err) } - return err - + return } func (bh *bucketHandle) StatObject(ctx context.Context, req *gcs.StatObjectRequest) (m *gcs.MinObject, e *gcs.ExtendedObjectAttributes, err error) { + + defer func() { + err = gcs.GetGCSError(err) + }() + var attrs *storage.ObjectAttrs // Retrieving object attrs through Go Storage Client. attrs, err = bh.bucket.Object(req.Name).Attrs(ctx) - - // If error is of type storage.ErrObjectNotExist - if err == storage.ErrObjectNotExist { - err = &gcs.NotFoundError{Err: err} // Special case error that object not found in the bucket. - return - } if err != nil { err = fmt.Errorf("error in fetching object attributes: %w", err) return @@ -210,33 +187,42 @@ func (bh *bucketHandle) getObjectHandleWithPreconditionsSet(req *gcs.CreateObjec } func (bh *bucketHandle) CreateObject(ctx context.Context, req *gcs.CreateObjectRequest) (o *gcs.Object, err error) { + defer func() { + err = gcs.GetGCSError(err) + }() + obj := bh.getObjectHandleWithPreconditionsSet(req) // Creating a NewWriter with requested attributes, using Go Storage Client. // Chuck size for resumable upload is default i.e. 16MB. wc := obj.NewWriter(ctx) + wc.ChunkRetryDeadline = time.Duration(req.ChunkRetryDeadlineSecs) * time.Second + wc.ChunkTransferTimeout = time.Duration(req.ChunkTransferTimeoutSecs) * time.Second wc = storageutil.SetAttrsInWriter(wc, req) - wc.ProgressFunc = func(bytesUploadedSoFar int64) { - logger.Tracef("gcs: Req %#16x: -- CreateObject(%q): %20v bytes uploaded so far", ctx.Value(gcs.ReqIdField), req.Name, bytesUploadedSoFar) - } + wc.ProgressFunc = req.CallBack + // Zonal buckets strictly require the appendable API. For Pirlo buckets, the + // appendable API provides file-like semantics (immediate data visibility) + // but defers regional durability until the object is finalized. Users can + // choose to keep objects unfinalized by setting the FinalizeFileOnClose flag + // to false, which allows further appends, but if they keep it unfinalized + // it never becomes regionally durable. + wc.Append = bh.BucketType().RapidWritesEnabled() + // By default, objects in zonal buckets are not finalized on close, whereas objects in + // pirlo buckets are. This behavior is controlled by the finalizeFileOnClose flag. + // When writer.Append is false, then this parameter is anyways ignored. + // Refer: https://github.com/googleapis/google-cloud-go/blob/bf56afb2a15301500b9981ee76ccc5f449e3f545/storage/writer.go#L160 + wc.FinalizeOnClose = bh.writeConfig.FinalizeFileOnClose // Copy the contents to the writer. if _, err = io.Copy(wc, req.Contents); err != nil { - err = fmt.Errorf("error in io.Copy: %w", err) + err = fmt.Errorf("failed io.Copy for %q: %w", req.Name, err) return } // We can't use defer to close the writer, because we need to close the // writer successfully before calling Attrs() method of writer. if err = wc.Close(); err != nil { - var gErr *googleapi.Error - if errors.As(err, &gErr) { - if gErr.Code == http.StatusPreconditionFailed { - err = &gcs.PreconditionError{Err: err} - return - } - } - err = fmt.Errorf("error in closing writer : %w", err) + err = fmt.Errorf("failed closing writer for %q: %w", req.Name, err) return } @@ -245,42 +231,100 @@ func (bh *bucketHandle) CreateObject(ctx context.Context, req *gcs.CreateObjectR o = storageutil.ObjectAttrsToBucketObject(attrs) return } + func (bh *bucketHandle) CreateObjectChunkWriter(ctx context.Context, req *gcs.CreateObjectRequest, chunkSize int, callBack func(bytesUploadedSoFar int64)) (gcs.Writer, error) { obj := bh.getObjectHandleWithPreconditionsSet(req) wc := &ObjectWriter{obj.NewWriter(ctx)} wc.ChunkSize = chunkSize wc.Writer = storageutil.SetAttrsInWriter(wc.Writer, req) - if callBack == nil { - callBack = func(bytesUploadedSoFar int64) { - logger.Tracef("gcs: Req %#16x: -- UploadBlock(%q): %20v bytes uploaded so far", ctx.Value(gcs.ReqIdField), req.Name, bytesUploadedSoFar) - } - } + wc.ChunkRetryDeadline = time.Duration(req.ChunkRetryDeadlineSecs) * time.Second + wc.ChunkTransferTimeout = time.Duration(req.ChunkTransferTimeoutSecs) * time.Second wc.ProgressFunc = callBack - + // Zonal buckets strictly require the appendable API. For Pirlo buckets, the + // appendable API provides file-like semantics (immediate data visibility) + // but defers regional durability until the object is finalized. Users can + // choose to keep objects unfinalized by setting the FinalizeFileOnClose flag + // to false, which allows further appends, but if they keep it unfinalized + // it never becomes regionally durable. + wc.Append = bh.BucketType().RapidWritesEnabled() + // By default, objects in zonal buckets are not finalized on close, whereas objects in + // pirlo buckets are. This behavior is controlled by the finalizeFileOnClose flag. + // When writer.Append is false, then this parameter is anyways ignored. + // Refer: https://github.com/googleapis/google-cloud-go/blob/bf56afb2a15301500b9981ee76ccc5f449e3f545/storage/writer.go#L160 + wc.FinalizeOnClose = bh.writeConfig.FinalizeFileOnClose return wc, nil } -func (bh *bucketHandle) FinalizeUpload(ctx context.Context, w gcs.Writer) (o *gcs.Object, err error) { +func (bh *bucketHandle) CreateAppendableObjectWriter(ctx context.Context, + req *gcs.CreateObjectChunkWriterRequest) (gcs.Writer, error) { + obj := bh.getObjectHandleWithPreconditionsSet(&req.CreateObjectRequest) + // To create the takeover writer, the objectHandle.Generation must be set. + obj = obj.Generation(*req.CreateObjectRequest.GenerationPrecondition) + + opts := storage.AppendableWriterOpts{ + ChunkSize: req.ChunkSize, + ProgressFunc: req.CallBack, + FinalizeOnClose: bh.writeConfig.FinalizeFileOnClose, + } + + tw, off, err := obj.NewWriterFromAppendableObject(ctx, &opts) // Takeover writer tw created from offset off. + + if err != nil { + err = fmt.Errorf("error while creating appendable object writer : %w", err) + return nil, err + } + + if off != req.Offset { + err = fmt.Errorf("takeover offset %d for the created appendable object writer does not match the requested offset %d", off, req.Offset) + // Offset mismatch implies a stale object view. Return PreconditionError to trigger metadata cache eviction. + return nil, &gcs.PreconditionError{Err: err} + } + w := &ObjectWriter{tw} + return w, err +} + +func (bh *bucketHandle) FinalizeUpload(ctx context.Context, w gcs.Writer) (o *gcs.MinObject, err error) { + defer func() { + err = gcs.GetGCSError(err) + }() + if err = w.Close(); err != nil { - var gErr *googleapi.Error - if errors.As(err, &gErr) { - if gErr.Code == http.StatusPreconditionFailed { - err = &gcs.PreconditionError{Err: err} - return - } - } err = fmt.Errorf("error in closing writer : %w", err) return } attrs := w.Attrs() // Retrieving the attributes of the created object. - // Converting attrs to type *Object. - o = storageutil.ObjectAttrsToBucketObject(attrs) + // Converting attrs to type *MinObject. + o = storageutil.ObjectAttrsToMinObject(attrs) + return +} + +func (bh *bucketHandle) FlushPendingWrites(ctx context.Context, w gcs.Writer) (o *gcs.MinObject, err error) { + defer func() { + err = gcs.GetGCSError(err) + }() + + _, err = w.Flush() + if err != nil { + err = fmt.Errorf("error in FlushPendingWrites : %w", err) + return + } + + attrs := w.Attrs() // Retrieving the attributes of the created object. + // Converting attrs to type *MinObject. + o = storageutil.ObjectAttrsToMinObject(attrs) + if o == nil { + return nil, fmt.Errorf("FlushPendingWrites: nil object returned after w.Flush()") + } return } func (bh *bucketHandle) CopyObject(ctx context.Context, req *gcs.CopyObjectRequest) (o *gcs.Object, err error) { + defer func() { + err = gcs.GetGCSError(err) + }() + srcObj := bh.bucket.Object(req.SrcName) dstObj := bh.bucket.Object(req.DstName) @@ -297,17 +341,7 @@ func (bh *bucketHandle) CopyObject(ctx context.Context, req *gcs.CopyObjectReque objAttrs, err := dstObj.CopierFrom(srcObj).Run(ctx) if err != nil { - switch ee := err.(type) { - case *googleapi.Error: - if ee.Code == http.StatusPreconditionFailed { - err = &gcs.PreconditionError{Err: ee} - } - if ee.Code == http.StatusNotFound { - err = &gcs.NotFoundError{Err: storage.ErrObjectNotExist} - } - default: - err = fmt.Errorf("error in copying object: %w", err) - } + err = fmt.Errorf("error in copying object: %w", err) return } // Converting objAttrs to type *Object @@ -316,23 +350,24 @@ func (bh *bucketHandle) CopyObject(ctx context.Context, req *gcs.CopyObjectReque } func getProjectionValue(req gcs.Projection) storage.Projection { - // Explicitly converting Projection Value because the ProjectionVal interface of jacobsa/gcloud and Go Client API are not coupled correctly. - var convertedProjection storage.Projection // Stores the Projection Value according to the Go Client API Interface. - switch int(req) { - // Projection Value 0 in jacobsa/gcloud maps to Projection Value 1 in Go Client API, that is for "full". - case 0: - convertedProjection = storage.Projection(1) - // Projection Value 1 in jacobsa/gcloud maps to Projection Value 2 in Go Client API, that is for "noAcl". - case 1: - convertedProjection = storage.Projection(2) - // Default Projection value in jacobsa/gcloud library is 0 that maps to 1 in Go Client API interface, and that is for "full". + // Map gcs.Projection enum to storage.Projection enum. + // The two libraries use different enum values for the same concepts. + switch req { + case gcs.Full: + return storage.ProjectionFull + case gcs.NoAcl: + return storage.ProjectionNoACL default: - convertedProjection = storage.Projection(1) + // Default to Full projection for any unknown values + return storage.ProjectionFull } - return convertedProjection } func (bh *bucketHandle) ListObjects(ctx context.Context, req *gcs.ListObjectsRequest) (listing *gcs.Listing, err error) { + defer func() { + err = gcs.GetGCSError(err) + }() + // Converting *ListObjectsRequest to type *storage.Query as expected by the Go Storage Client. query := &storage.Query{ Delimiter: req.Delimiter, @@ -340,14 +375,26 @@ func (bh *bucketHandle) ListObjects(ctx context.Context, req *gcs.ListObjectsReq Projection: getProjectionValue(req.ProjectionVal), IncludeTrailingDelimiter: req.IncludeTrailingDelimiter, IncludeFoldersAsPrefixes: req.IncludeFoldersAsPrefixes, + StartOffset: req.StartOffset, //MaxResults: , (Field not present in storage.Query of Go Storage Library but present in ListObjectsQuery in Jacobsa code.) } - err = query.SetAttrSelection([]string{"Name", "Size", "Generation", "Metageneration", "Updated", "Metadata", "ContentEncoding", "CRC32C"}) + minObjAttrs := []string{"Name", "Size", "Generation", "Metageneration", "Updated", "Metadata", "ContentEncoding", "CRC32C"} + if bh.BucketType().Zonal { + // For regional buckets, partial response API fails to populate the Finalized field.(b/398916957) + // For objects in regional buckets, this field will be *unset*. + minObjAttrs = append(minObjAttrs, "Finalized") + } + err = query.SetAttrSelection(minObjAttrs) + if err != nil { err = fmt.Errorf("error while setting attribute selection for List Object query :%w", err) return } + if bh.billingProject != "" { + ctx = metadata.AppendToOutgoingContext(ctx, "x-goog-user-project", bh.billingProject) + } + itr := bh.bucket.Objects(ctx, query) // Returning iterator to the list of objects. pi := itr.PageInfo() pi.MaxSize = req.MaxResults @@ -397,6 +444,10 @@ func (bh *bucketHandle) ListObjects(ctx context.Context, req *gcs.ListObjectsReq } func (bh *bucketHandle) UpdateObject(ctx context.Context, req *gcs.UpdateObjectRequest) (o *gcs.Object, err error) { + defer func() { + err = gcs.GetGCSError(err) + }() + obj := bh.bucket.Object(req.Name) if req.Generation != 0 { @@ -436,31 +487,21 @@ func (bh *bucketHandle) UpdateObject(ctx context.Context, req *gcs.UpdateObjectR attrs, err := obj.Update(ctx, updateQuery) - if err == nil { - // Converting objAttrs to type *Object - o = storageutil.ObjectAttrsToBucketObject(attrs) + if err != nil { + err = fmt.Errorf("error in updating object: %w", err) return } - // If storage object does not exist, httpclient is returning ErrObjectNotExist error instead of googleapi error - // https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/vendor/cloud.google.com/go/storage/http_client.go#L516 - switch ee := err.(type) { - case *googleapi.Error: - if ee.Code == http.StatusPreconditionFailed { - err = &gcs.PreconditionError{Err: ee} - } - default: - if err == storage.ErrObjectNotExist { - err = &gcs.NotFoundError{Err: storage.ErrObjectNotExist} - } else { - err = fmt.Errorf("error in updating object: %w", err) - } - } - + // Converting objAttrs to type *Object + o = storageutil.ObjectAttrsToBucketObject(attrs) return } func (bh *bucketHandle) ComposeObjects(ctx context.Context, req *gcs.ComposeObjectsRequest) (o *gcs.Object, err error) { + defer func() { + err = gcs.GetGCSError(err) + }() + dstObj := bh.bucket.Object(req.DstName) dstObjConds := storage.Conditions{} @@ -499,37 +540,66 @@ func (bh *bucketHandle) ComposeObjects(ctx context.Context, req *gcs.ComposeObje // Composing Source Objects to Destination Object using Composer created through Go Storage Client. attrs, err := dstObj.ComposerFrom(srcObjList...).Run(ctx) if err != nil { - switch ee := err.(type) { - case *googleapi.Error: - if ee.Code == http.StatusPreconditionFailed { - err = &gcs.PreconditionError{Err: ee} - } - if ee.Code == http.StatusNotFound { - err = &gcs.NotFoundError{Err: storage.ErrObjectNotExist} - } - default: - err = fmt.Errorf("error in composing object: %w", err) - } + err = fmt.Errorf("error in composing object: %w", err) return } // Converting attrs to type *Object. o = storageutil.ObjectAttrsToBucketObject(attrs) - return } func (bh *bucketHandle) DeleteFolder(ctx context.Context, folderName string) (err error) { + defer func() { + err = gcs.GetGCSError(err) + }() + var callOptions []gax.CallOption err = bh.controlClient.DeleteFolder(ctx, &controlpb.DeleteFolderRequest{ Name: fmt.Sprintf(FullFolderPathHNS, bh.bucketName, folderName), }, callOptions...) + return +} + +func (bh *bucketHandle) MoveObject(ctx context.Context, req *gcs.MoveObjectRequest) (o *gcs.Object, err error) { + defer func() { + err = gcs.GetGCSError(err) + }() + + obj := bh.bucket.Object(req.SrcName) + + // Switching to the requested generation of source object. + if req.SrcGeneration != 0 { + obj = obj.Generation(req.SrcGeneration) + } + + // Putting a condition that the metaGeneration of source should match *req.SrcMetaGenerationPrecondition for move operation to occur. + if req.SrcMetaGenerationPrecondition != nil { + obj = obj.If(storage.Conditions{MetagenerationMatch: *req.SrcMetaGenerationPrecondition}) + } - return err + dstMoveObject := storage.MoveObjectDestination{ + Object: req.DstName, + Conditions: nil, + } + + attrs, err := obj.Move(ctx, dstMoveObject) + if err != nil { + err = fmt.Errorf("error in moving object: %w", err) + return + } + + // Converting objAttrs to type *Object + o = storageutil.ObjectAttrsToBucketObject(attrs) + return } func (bh *bucketHandle) RenameFolder(ctx context.Context, folderName string, destinationFolderId string) (folder *gcs.Folder, err error) { + defer func() { + err = gcs.GetGCSError(err) + }() + var controlFolder *controlpb.Folder req := &controlpb.RenameFolderRequest{ Name: fmt.Sprintf(FullFolderPathHNS, bh.bucketName, folderName), @@ -537,53 +607,47 @@ func (bh *bucketHandle) RenameFolder(ctx context.Context, folderName string, des } resp, err := bh.controlClient.RenameFolder(ctx, req) if err != nil { - return nil, err + err = fmt.Errorf("error in renaming folder: %w", err) + return } // Wait blocks until the long-running operation is completed, // returning the response and any errors encountered. controlFolder, err = resp.Wait(ctx) - folder = gcs.GCSFolder(bh.bucketName, controlFolder) + if err != nil { + err = fmt.Errorf("error in getting result from renaming folder response: %w", err) + return + } - return folder, err + folder = gcs.GCSFolder(bh.bucketName, controlFolder) + return } -// TODO: Consider adding this method to the bucket interface if additional -// layout options are needed in the future. -func (bh *bucketHandle) getStorageLayout() (*controlpb.StorageLayout, error) { - var callOptions []gax.CallOption - stoargeLayout, err := bh.controlClient.GetStorageLayout(context.Background(), &controlpb.GetStorageLayoutRequest{ - Name: fmt.Sprintf("projects/_/buckets/%s/storageLayout", bh.bucketName), - Prefix: "", - RequestId: "", - }, callOptions...) - - return stoargeLayout, err -} +func (bh *bucketHandle) GetFolder(ctx context.Context, req *gcs.GetFolderRequest) (folder *gcs.Folder, err error) { + defer func() { + err = gcs.GetGCSError(err) + }() -func (bh *bucketHandle) GetFolder(ctx context.Context, folderName string) (*gcs.Folder, error) { var callOptions []gax.CallOption - - clientFolder, err := bh.controlClient.GetFolder(ctx, &controlpb.GetFolderRequest{ - Name: fmt.Sprintf(FullFolderPathHNS, bh.bucketName, folderName), + var clientFolder *controlpb.Folder + clientFolder, err = bh.controlClient.GetFolder(ctx, &controlpb.GetFolderRequest{ + Name: fmt.Sprintf(FullFolderPathHNS, bh.bucketName, req.Name), }, callOptions...) if err != nil { - err = fmt.Errorf("error getting metadata for folder: %s, %w", folderName, err) - var gcsAPIErr *apierror.APIError - if errors.As(err, &gcsAPIErr) { - if "NotFound" == gcsAPIErr.GRPCStatus().Code().String() { - return nil, &gcs.NotFoundError{Err: err} - } - } - return nil, err + err = fmt.Errorf("error getting metadata for folder: %s, %w", req.Name, err) + return } - folderResponse := gcs.GCSFolder(bh.bucketName, clientFolder) - return folderResponse, err + folder = gcs.GCSFolder(bh.bucketName, clientFolder) + return } -func (bh *bucketHandle) CreateFolder(ctx context.Context, folderName string) (*gcs.Folder, error) { +func (bh *bucketHandle) CreateFolder(ctx context.Context, folderName string) (folder *gcs.Folder, err error) { + defer func() { + err = gcs.GetGCSError(err) + }() + req := &controlpb.CreateFolderRequest{ Parent: fmt.Sprintf(FullBucketPathHNS, bh.bucketName), FolderId: folderName, @@ -592,12 +656,41 @@ func (bh *bucketHandle) CreateFolder(ctx context.Context, folderName string) (*g clientFolder, err := bh.controlClient.CreateFolder(ctx, req) if err != nil { - return nil, err + err = fmt.Errorf("error in creating folder: %w", err) + return } - folder := gcs.GCSFolder(bh.bucketName, clientFolder) + folder = gcs.GCSFolder(bh.bucketName, clientFolder) + return +} + +func (bh *bucketHandle) NewMultiRangeDownloader( + ctx context.Context, req *gcs.MultiRangeDownloaderRequest) (mrd gcs.MultiRangeDownloader, err error) { + defer func() { + err = gcs.GetGCSError(err) + }() + + obj := bh.bucket.Object(req.Name) + + // Switching to the requested generation of object. + if req.Generation != 0 { + obj = obj.Generation(req.Generation) + } + + if req.ReadCompressed { + obj = obj.ReadCompressed(true) + } + + if req.ReadHandle != nil { + obj = obj.ReadHandle(req.ReadHandle) + } + + mrd, err = obj.NewMultiRangeDownloader(ctx) + return +} - return folder, nil +func (bh *bucketHandle) GCSName(obj *gcs.MinObject) string { + return obj.Name } func isStorageConditionsNotEmpty(conditions storage.Conditions) bool { diff --git a/internal/storage/bucket_handle_test.go b/internal/storage/bucket_handle_test.go index d40fc663b1..df3c1db428 100644 --- a/internal/storage/bucket_handle_test.go +++ b/internal/storage/bucket_handle_test.go @@ -26,7 +26,8 @@ import ( "cloud.google.com/go/storage" control "cloud.google.com/go/storage/control/apiv2" "cloud.google.com/go/storage/control/apiv2/controlpb" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -51,16 +52,6 @@ var ContentDisposition string = "ContentDisposition" // Hence, we are not writing tests for these flows. // https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/vendor/github.com/fsouza/fake-gcs-server/fakestorage/object.go#L515 -func objectsToObjectNames(objects []*gcs.Object) (objectNames []string) { - objectNames = make([]string, len(objects)) - for i, object := range objects { - if object != nil { - objectNames[i] = object.Name - } - } - return -} - func minObjectsToMinObjectNames(minObjects []*gcs.MinObject) (objectNames []string) { objectNames = make([]string, len(minObjects)) for i, object := range minObjects { @@ -71,6 +62,22 @@ func minObjectsToMinObjectNames(minObjects []*gcs.MinObject) (objectNames []stri return } +func createBucketHandle(testSuite *BucketHandleTest, resp *controlpb.StorageLayout) { + var err error + + testSuite.mockClient = new(MockStorageControlClient) + testSuite.fakeStorage = NewFakeStorageWithMockClient(testSuite.mockClient, cfg.HTTP2) + testSuite.storageHandle = testSuite.fakeStorage.CreateStorageHandle() + + testSuite.mockClient.On("GetStorageLayout", mock.Anything, mock.Anything, mock.Anything). + Return(resp, nil) + testSuite.bucketHandle, err = testSuite.storageHandle.BucketHandle(context.Background(), TestBucketName, "") + testSuite.bucketHandle.controlClient = testSuite.mockClient + + assert.NotNil(testSuite.T(), testSuite.bucketHandle) + assert.Nil(testSuite.T(), err) +} + type BucketHandleTest struct { suite.Suite bucketHandle *bucketHandle @@ -84,22 +91,21 @@ func TestBucketHandleTestSuite(testSuite *testing.T) { } func (testSuite *BucketHandleTest) SetupTest() { - testSuite.fakeStorage = NewFakeStorage() - testSuite.storageHandle = testSuite.fakeStorage.CreateStorageHandle() - ctx := context.Background() - testSuite.bucketHandle = testSuite.storageHandle.BucketHandle(ctx, TestBucketName, "") - testSuite.mockClient = new(MockStorageControlClient) - testSuite.bucketHandle.controlClient = testSuite.mockClient - - assert.NotNil(testSuite.T(), testSuite.bucketHandle) + testSuite.mockClient = nil + testSuite.fakeStorage = nil + testSuite.storageHandle = nil + testSuite.bucketHandle = nil } func (testSuite *BucketHandleTest) TearDownTest() { - testSuite.fakeStorage.ShutDown() + if testSuite.fakeStorage != nil { + testSuite.fakeStorage.ShutDown() + } } -func (testSuite *BucketHandleTest) TestNewReaderMethodWithCompleteRead() { - rc, err := testSuite.bucketHandle.NewReader(context.Background(), +func (testSuite *BucketHandleTest) TestNewReaderWithReadHandleMethodWithCompleteRead() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) + rc, err := testSuite.bucketHandle.NewReaderWithReadHandle(context.Background(), &gcs.ReadObjectRequest{ Name: TestObjectName, Range: &gcs.ByteRange{ @@ -116,11 +122,12 @@ func (testSuite *BucketHandleTest) TestNewReaderMethodWithCompleteRead() { assert.Equal(testSuite.T(), ContentInTestObject, string(buf[:])) } -func (testSuite *BucketHandleTest) TestNewReaderMethodWithRangeRead() { +func (testSuite *BucketHandleTest) TestNewReaderWithReadHandleMethodWithRangeRead() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) start := uint64(2) limit := uint64(8) - rc, err := testSuite.bucketHandle.NewReader(context.Background(), + rc, err := testSuite.bucketHandle.NewReaderWithReadHandle(context.Background(), &gcs.ReadObjectRequest{ Name: TestObjectName, Range: &gcs.ByteRange{ @@ -137,8 +144,9 @@ func (testSuite *BucketHandleTest) TestNewReaderMethodWithRangeRead() { assert.Equal(testSuite.T(), ContentInTestObject[start:limit], string(buf[:])) } -func (testSuite *BucketHandleTest) TestNewReaderMethodWithNilRange() { - rc, err := testSuite.bucketHandle.NewReader(context.Background(), +func (testSuite *BucketHandleTest) TestNewReaderWithReadHandleMethodWithNilRange() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) + rc, err := testSuite.bucketHandle.NewReaderWithReadHandle(context.Background(), &gcs.ReadObjectRequest{ Name: TestObjectName, Range: nil, @@ -152,8 +160,11 @@ func (testSuite *BucketHandleTest) TestNewReaderMethodWithNilRange() { assert.Equal(testSuite.T(), ContentInTestObject, string(buf[:])) } -func (testSuite *BucketHandleTest) TestNewReaderMethodWithInValidObject() { - rc, err := testSuite.bucketHandle.NewReader(context.Background(), +func (testSuite *BucketHandleTest) TestNewReaderWithReadHandleMethodWithInValidObject() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) + var notFoundErr *gcs.NotFoundError + + rc, err := testSuite.bucketHandle.NewReaderWithReadHandle(context.Background(), &gcs.ReadObjectRequest{ Name: missingObjectName, Range: &gcs.ByteRange{ @@ -163,11 +174,13 @@ func (testSuite *BucketHandleTest) TestNewReaderMethodWithInValidObject() { }) assert.NotNil(testSuite.T(), err) + assert.True(testSuite.T(), errors.As(err, ¬FoundErr)) assert.Nil(testSuite.T(), rc) } -func (testSuite *BucketHandleTest) TestNewReaderMethodWithValidGeneration() { - rc, err := testSuite.bucketHandle.NewReader(context.Background(), +func (testSuite *BucketHandleTest) TestNewReaderWithReadHandleMethodWithValidGeneration() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) + rc, err := testSuite.bucketHandle.NewReaderWithReadHandle(context.Background(), &gcs.ReadObjectRequest{ Name: TestObjectName, Range: &gcs.ByteRange{ @@ -185,8 +198,11 @@ func (testSuite *BucketHandleTest) TestNewReaderMethodWithValidGeneration() { assert.Equal(testSuite.T(), ContentInTestObject, string(buf[:])) } -func (testSuite *BucketHandleTest) TestNewReaderMethodWithInvalidGeneration() { - rc, err := testSuite.bucketHandle.NewReader(context.Background(), +func (testSuite *BucketHandleTest) TestNewReaderWithReadHandleMethodWithInvalidGeneration() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) + var notFoundErr *gcs.NotFoundError + + rc, err := testSuite.bucketHandle.NewReaderWithReadHandle(context.Background(), &gcs.ReadObjectRequest{ Name: TestObjectName, Range: &gcs.ByteRange{ @@ -197,11 +213,13 @@ func (testSuite *BucketHandleTest) TestNewReaderMethodWithInvalidGeneration() { }) assert.NotNil(testSuite.T(), err) + assert.True(testSuite.T(), errors.As(err, ¬FoundErr)) assert.Nil(testSuite.T(), rc) } -func (testSuite *BucketHandleTest) TestNewReaderMethodWithCompressionEnabled() { - rc, err := testSuite.bucketHandle.NewReader(context.Background(), +func (testSuite *BucketHandleTest) TestNewReaderWithReadHandleMethodWithCompressionEnabled() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) + rc, err := testSuite.bucketHandle.NewReaderWithReadHandle(context.Background(), &gcs.ReadObjectRequest{ Name: TestGzipObjectName, Range: &gcs.ByteRange{ @@ -219,8 +237,9 @@ func (testSuite *BucketHandleTest) TestNewReaderMethodWithCompressionEnabled() { assert.Equal(testSuite.T(), ContentInTestGzipObjectCompressed, string(buf)) } -func (testSuite *BucketHandleTest) TestNewReaderMethodWithCompressionDisabled() { - rc, err := testSuite.bucketHandle.NewReader(context.Background(), +func (testSuite *BucketHandleTest) TestNewReaderWithReadHandleMethodWithCompressionDisabled() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) + rc, err := testSuite.bucketHandle.NewReaderWithReadHandle(context.Background(), &gcs.ReadObjectRequest{ Name: TestGzipObjectName, Range: &gcs.ByteRange{ @@ -238,7 +257,52 @@ func (testSuite *BucketHandleTest) TestNewReaderMethodWithCompressionDisabled() assert.Equal(testSuite.T(), ContentInTestGzipObjectDecompressed, string(buf)) } +// Fakestorage doesn't support readHandle concept +func (testSuite *BucketHandleTest) TestNewReaderWithReadHandleMethodWithoutReadHandle() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) + rd, err := testSuite.bucketHandle.NewReaderWithReadHandle(context.Background(), + &gcs.ReadObjectRequest{ + Name: TestObjectName, + Range: &gcs.ByteRange{ + Start: uint64(0), + Limit: uint64(len(ContentInTestObject)), + }, + ReadHandle: nil, + }) + + assert.Nil(testSuite.T(), err) + defer rd.Close() + buf := make([]byte, len(ContentInTestObject)) + _, err = rd.Read(buf) + assert.Nil(testSuite.T(), err) + //assert.Equal(testSuite.T(), len(rd.ReadHandle()), 0) + assert.Equal(testSuite.T(), ContentInTestObject, string(buf[:])) +} + +// Fakestorage doesn't support readHandle concept +func (testSuite *BucketHandleTest) TestNewReaderWithReadHandleMethodWithReadHandle() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) + rd, err := testSuite.bucketHandle.NewReaderWithReadHandle(context.Background(), + &gcs.ReadObjectRequest{ + Name: TestObjectName, + Range: &gcs.ByteRange{ + Start: uint64(0), + Limit: uint64(len(ContentInTestObject)), + }, + ReadHandle: []byte("opaque-handle"), + }) + + assert.Nil(testSuite.T(), err) + defer rd.Close() + buf := make([]byte, len(ContentInTestObject)) + _, err = rd.Read(buf) + assert.Nil(testSuite.T(), err) + //assert.Equal(testSuite.T(), len(rd.ReadHandle()), 0) + assert.Equal(testSuite.T(), ContentInTestObject, string(buf[:])) +} + func (testSuite *BucketHandleTest) TestDeleteObjectMethodWithValidObject() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) err := testSuite.bucketHandle.DeleteObject(context.Background(), &gcs.DeleteObjectRequest{ Name: TestObjectName, @@ -250,6 +314,7 @@ func (testSuite *BucketHandleTest) TestDeleteObjectMethodWithValidObject() { } func (testSuite *BucketHandleTest) TestDeleteObjectMethodWithMissingObject() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) err := testSuite.bucketHandle.DeleteObject(context.Background(), &gcs.DeleteObjectRequest{ Name: missingObjectName, @@ -257,10 +322,11 @@ func (testSuite *BucketHandleTest) TestDeleteObjectMethodWithMissingObject() { MetaGenerationPrecondition: nil, }) - assert.Equal(testSuite.T(), "gcs.NotFoundError: storage: object doesn't exist", err.Error()) + assert.NotNil(testSuite.T(), err) } func (testSuite *BucketHandleTest) TestDeleteObjectMethodWithMissingGeneration() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) err := testSuite.bucketHandle.DeleteObject(context.Background(), &gcs.DeleteObjectRequest{ Name: TestObjectName, @@ -271,7 +337,8 @@ func (testSuite *BucketHandleTest) TestDeleteObjectMethodWithMissingGeneration() } func (testSuite *BucketHandleTest) TestDeleteObjectMethodWithZeroGeneration() { - // Note: fake-gcs-server doesn'testSuite respect Generation or other conditions in + createBucketHandle(testSuite, &controlpb.StorageLayout{}) + // Note: fake-gcs-server doesn't respect Generation or other conditions in // delete operations. This unit test will be helpful when fake-gcs-server // start respecting these conditions, or we move to other testing framework. err := testSuite.bucketHandle.DeleteObject(context.Background(), @@ -285,6 +352,7 @@ func (testSuite *BucketHandleTest) TestDeleteObjectMethodWithZeroGeneration() { } func (testSuite *BucketHandleTest) TestStatObjectMethodWithValidObject() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) _, _, err := testSuite.bucketHandle.StatObject(context.Background(), &gcs.StatObjectRequest{ Name: TestObjectName, @@ -294,6 +362,7 @@ func (testSuite *BucketHandleTest) TestStatObjectMethodWithValidObject() { } func (testSuite *BucketHandleTest) TestStatObjectMethodWithReturnExtendedObjectAttributesTrue() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) m, e, err := testSuite.bucketHandle.StatObject(context.Background(), &gcs.StatObjectRequest{ Name: TestObjectName, @@ -306,6 +375,7 @@ func (testSuite *BucketHandleTest) TestStatObjectMethodWithReturnExtendedObjectA } func (testSuite *BucketHandleTest) TestStatObjectMethodWithReturnExtendedObjectAttributesFalse() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) m, e, err := testSuite.bucketHandle.StatObject(context.Background(), &gcs.StatObjectRequest{ Name: TestObjectName, @@ -318,6 +388,7 @@ func (testSuite *BucketHandleTest) TestStatObjectMethodWithReturnExtendedObjectA } func (testSuite *BucketHandleTest) TestStatObjectMethodWithMissingObject() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) var notfound *gcs.NotFoundError _, _, err := testSuite.bucketHandle.StatObject(context.Background(), @@ -329,6 +400,7 @@ func (testSuite *BucketHandleTest) TestStatObjectMethodWithMissingObject() { } func (testSuite *BucketHandleTest) TestCopyObjectMethodWithValidObject() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) _, err := testSuite.bucketHandle.CopyObject(context.Background(), &gcs.CopyObjectRequest{ SrcName: TestObjectName, @@ -341,6 +413,7 @@ func (testSuite *BucketHandleTest) TestCopyObjectMethodWithValidObject() { } func (testSuite *BucketHandleTest) TestCopyObjectMethodWithMissingObject() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) var notfound *gcs.NotFoundError _, err := testSuite.bucketHandle.CopyObject(context.Background(), @@ -355,6 +428,7 @@ func (testSuite *BucketHandleTest) TestCopyObjectMethodWithMissingObject() { } func (testSuite *BucketHandleTest) TestCopyObjectMethodWithInvalidGeneration() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) var notfound *gcs.NotFoundError _, err := testSuite.bucketHandle.CopyObject(context.Background(), @@ -369,6 +443,7 @@ func (testSuite *BucketHandleTest) TestCopyObjectMethodWithInvalidGeneration() { } func (testSuite *BucketHandleTest) TestCreateObjectMethodWithValidObject() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) content := "Creating a new object" obj, err := testSuite.bucketHandle.CreateObject(context.Background(), &gcs.CreateObjectRequest{ @@ -382,6 +457,7 @@ func (testSuite *BucketHandleTest) TestCreateObjectMethodWithValidObject() { } func (testSuite *BucketHandleTest) TestCreateObjectMethodWithGenerationAsZero() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) content := "Creating a new object" var generation int64 = 0 obj, err := testSuite.bucketHandle.CreateObject(context.Background(), @@ -397,6 +473,7 @@ func (testSuite *BucketHandleTest) TestCreateObjectMethodWithGenerationAsZero() } func (testSuite *BucketHandleTest) TestCreateObjectMethodWithGenerationAsZeroWhenObjectAlreadyExists() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) content := "Creating a new object" var generation int64 = 0 var precondition *gcs.PreconditionError @@ -423,6 +500,7 @@ func (testSuite *BucketHandleTest) TestCreateObjectMethodWithGenerationAsZeroWhe } func (testSuite *BucketHandleTest) TestCreateObjectMethodWhenGivenGenerationObjectNotExist() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) var precondition *gcs.PreconditionError content := "Creating a new object" var crc32 uint32 = 45 @@ -441,6 +519,7 @@ func (testSuite *BucketHandleTest) TestCreateObjectMethodWhenGivenGenerationObje } func (testSuite *BucketHandleTest) TestBucketHandle_CreateObjectChunkWriter() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) var generation0 int64 = 0 var generationNon0 int64 = 786 var metaGeneration0 int64 = 0 @@ -515,29 +594,58 @@ func (testSuite *BucketHandleTest) TestBucketHandle_CreateObjectChunkWriter() { } } -func (testSuite *BucketHandleTest) TestBucketHandle_CreateObjectChunkWriterWithNilCallback() { - var metaGeneration0 int64 = 0 - objectName := "test_object_1" - chunkSize := 1024 * 1024 - - w, err := testSuite.bucketHandle.CreateObjectChunkWriter(context.Background(), - &gcs.CreateObjectRequest{ - Name: objectName, - GenerationPrecondition: nil, - MetaGenerationPrecondition: &metaGeneration0, +func (testSuite *BucketHandleTest) TestBucketHandle_WriterAttributes() { + tests := []struct { + name string + bucketType gcs.BucketType + finalizeFileOnClose bool + expectedAppend bool + }{ + { + name: "StandardBucket", + bucketType: gcs.BucketType{}, + finalizeFileOnClose: true, + expectedAppend: false, }, - chunkSize, - nil, - ) + { + name: "ZonalBucket", + bucketType: gcs.BucketType{Zonal: true}, + finalizeFileOnClose: false, + expectedAppend: true, + }, + { + name: "PirloBucket_RapidEnabled", + bucketType: gcs.BucketType{Pirlo: gcs.PirloStateRapidWritesEnabled}, + finalizeFileOnClose: true, + expectedAppend: true, + }, + { + name: "PirloBucket_RapidDisabled", + bucketType: gcs.BucketType{Pirlo: gcs.PirloStateRapidWritesDisabled}, + finalizeFileOnClose: false, + expectedAppend: false, + }, + } - require.NoError(testSuite.T(), err) - objWr, ok := (w).(*ObjectWriter) - require.True(testSuite.T(), ok) - require.NotNil(testSuite.T(), objWr) - assert.NotNil(testSuite.T(), objWr.ProgressFunc) + for _, tt := range tests { + testSuite.T().Run(tt.name, func(t *testing.T) { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) + testSuite.bucketHandle.bucketType = &tt.bucketType + testSuite.bucketHandle.writeConfig = &cfg.WriteConfig{FinalizeFileOnClose: tt.finalizeFileOnClose} + + w, err := testSuite.bucketHandle.CreateObjectChunkWriter(context.Background(), &gcs.CreateObjectRequest{Name: "test_object"}, 1024, nil) + + require.NoError(t, err) + objWr, ok := w.(*ObjectWriter) + require.True(t, ok) + assert.Equal(t, tt.expectedAppend, objWr.Append) + assert.Equal(t, tt.finalizeFileOnClose, objWr.FinalizeOnClose) + }) + } } func (testSuite *BucketHandleTest) TestBucketHandle_FinalizeUploadSuccess() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) var generation0 int64 = 0 tests := []struct { @@ -562,22 +670,7 @@ func (testSuite *BucketHandleTest) TestBucketHandle_FinalizeUploadSuccess() { for _, tt := range tests { testSuite.T().Run(tt.name, func(t *testing.T) { - progressFunc := func(_ int64) {} - wr, err := testSuite.bucketHandle.CreateObjectChunkWriter(context.Background(), - &gcs.CreateObjectRequest{ - Name: tt.objectName, - GenerationPrecondition: tt.generation, - }, - tt.chunkSize, - progressFunc, - ) - require.NoError(t, err) - objWr, ok := (wr).(*ObjectWriter) - require.True(t, ok) - require.NotNil(t, objWr) - assert.Equal(t, tt.objectName, objWr.ObjectName()) - assert.Equal(t, tt.chunkSize, objWr.ChunkSize) - assert.Equal(t, reflect.ValueOf(progressFunc).Pointer(), reflect.ValueOf(objWr.ProgressFunc).Pointer()) + wr := testSuite.createObjectChunkWriter(t, tt.objectName, tt.generation, tt.chunkSize) o, err := testSuite.bucketHandle.FinalizeUpload(context.Background(), wr) @@ -587,45 +680,78 @@ func (testSuite *BucketHandleTest) TestBucketHandle_FinalizeUploadSuccess() { } } -func (testSuite *BucketHandleTest) createObject(objName string) { - testSuite.T().Helper() +func (testSuite *BucketHandleTest) TestFinalizeUploadWithGenerationAsZeroWhenObjectAlreadyExists() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) + // Pre-create the object (creating writer and finalizing upload). + objName := "pre_created_test_object" var generation int64 = 0 - wr, err := testSuite.bucketHandle.CreateObjectChunkWriter(context.Background(), - &gcs.CreateObjectRequest{ - Name: objName, - GenerationPrecondition: &generation, - }, - 100, - func(_ int64) {}) - require.NoError(testSuite.T(), err) - assert.Equal(testSuite.T(), objName, wr.ObjectName()) - + wr := testSuite.createObjectChunkWriter(testSuite.T(), objName, &generation, 100) o, err := testSuite.bucketHandle.FinalizeUpload(context.Background(), wr) - require.NoError(testSuite.T(), err) assert.NotNil(testSuite.T(), o) + // Create Object Writer again when object already exists. + wr = testSuite.createObjectChunkWriter(testSuite.T(), objName, &generation, 100) + + o, err = testSuite.bucketHandle.FinalizeUpload(context.Background(), wr) + + assert.Error(testSuite.T(), err) + assert.IsType(testSuite.T(), &gcs.PreconditionError{}, err) + assert.Nil(testSuite.T(), o) } -func (testSuite *BucketHandleTest) TestFinalizeUploadWithGenerationAsZeroWhenObjectAlreadyExists() { - objName := "pre_created_test_object" - testSuite.createObject(objName) - // Create Object Writer again when object already exists. - var generation int64 = 0 +func (testSuite *BucketHandleTest) createObjectChunkWriter(t *testing.T, objectName string, generation *int64, chunkSize int) gcs.Writer { + t.Helper() + progressFunc := func(_ int64) {} wr, err := testSuite.bucketHandle.CreateObjectChunkWriter(context.Background(), &gcs.CreateObjectRequest{ - Name: objName, - GenerationPrecondition: &generation, + Name: objectName, + GenerationPrecondition: generation, }, - 100, - func(_ int64) {}) - require.NoError(testSuite.T(), err) - assert.Equal(testSuite.T(), objName, wr.ObjectName()) + chunkSize, + progressFunc, + ) + require.NoError(t, err) + objWr, ok := (wr).(*ObjectWriter) + require.True(t, ok) + require.NotNil(t, objWr) + assert.Equal(t, objectName, objWr.ObjectName()) + assert.Equal(t, chunkSize, objWr.ChunkSize) + assert.Equal(t, reflect.ValueOf(progressFunc).Pointer(), reflect.ValueOf(objWr.ProgressFunc).Pointer()) - o, err := testSuite.bucketHandle.FinalizeUpload(context.Background(), wr) + return wr +} - assert.Error(testSuite.T(), err) - assert.IsType(testSuite.T(), &gcs.PreconditionError{}, err) - assert.Nil(testSuite.T(), o) +func (testSuite *BucketHandleTest) TestFlushPendingWritesFails() { + // These tests only run with HTTP client because fake storage server is not + // integrated with GRPC. + var generation0 int64 = 0 + tests := []struct { + bucketType string + expectedErr string + }{ + { + bucketType: "multiregion", + expectedErr: "Flush not supported unless client uses gRPC and Append is set to true", + }, + { + bucketType: "zone", + expectedErr: "append not supported on HTTP Client", + }, + } + + for _, tt := range tests { + testSuite.T().Run(tt.bucketType, func(t *testing.T) { + createBucketHandle(testSuite, &controlpb.StorageLayout{ + LocationType: tt.bucketType, + }) + wr := testSuite.createObjectChunkWriter(t, TestObjectName, &generation0, 100) + + _, err := testSuite.bucketHandle.FlushPendingWrites(context.Background(), wr) + + require.Error(t, err) + assert.ErrorContains(t, err, tt.expectedErr) + }) + } } func (testSuite *BucketHandleTest) TestGetProjectValueWhenGcloudProjectionIsNoAcl() { @@ -647,6 +773,7 @@ func (testSuite *BucketHandleTest) TestGetProjectValueWhenGcloudProjectionIsDefa } func (testSuite *BucketHandleTest) TestListObjectMethodWithPrefixObjectExist() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) obj, err := testSuite.bucketHandle.ListObjects(context.Background(), &gcs.ListObjectsRequest{ Prefix: "gcsfuse/", @@ -664,6 +791,7 @@ func (testSuite *BucketHandleTest) TestListObjectMethodWithPrefixObjectExist() { } func (testSuite *BucketHandleTest) TestListObjectMethodWithPrefixObjectDoesNotExist() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) obj, err := testSuite.bucketHandle.ListObjects(context.Background(), &gcs.ListObjectsRequest{ Prefix: "PrefixObjectDoesNotExist", @@ -680,6 +808,7 @@ func (testSuite *BucketHandleTest) TestListObjectMethodWithPrefixObjectDoesNotEx } func (testSuite *BucketHandleTest) TestListObjectMethodWithIncludeTrailingDelimiterFalse() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) obj, err := testSuite.bucketHandle.ListObjects(context.Background(), &gcs.ListObjectsRequest{ Prefix: "gcsfuse/", @@ -697,6 +826,7 @@ func (testSuite *BucketHandleTest) TestListObjectMethodWithIncludeTrailingDelimi // If Delimiter is empty, all the objects will appear with same prefix. func (testSuite *BucketHandleTest) TestListObjectMethodWithEmptyDelimiter() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) obj, err := testSuite.bucketHandle.ListObjects(context.Background(), &gcs.ListObjectsRequest{ Prefix: "gcsfuse/", @@ -715,6 +845,7 @@ func (testSuite *BucketHandleTest) TestListObjectMethodWithEmptyDelimiter() { // We have 5 objects in fakeserver. func (testSuite *BucketHandleTest) TestListObjectMethodForMaxResult() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) fiveObj, err := testSuite.bucketHandle.ListObjects(context.Background(), &gcs.ListObjectsRequest{ Prefix: "", @@ -752,6 +883,7 @@ func (testSuite *BucketHandleTest) TestListObjectMethodForMaxResult() { } func (testSuite *BucketHandleTest) TestListObjectMethodWithMissingMaxResult() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) // Validate that ee have 5 objects in fakeserver fiveObjWithMaxResults, err := testSuite.bucketHandle.ListObjects(context.Background(), &gcs.ListObjectsRequest{ @@ -781,6 +913,7 @@ func (testSuite *BucketHandleTest) TestListObjectMethodWithMissingMaxResult() { } func (testSuite *BucketHandleTest) TestListObjectMethodWithZeroMaxResult() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) // Validate that we have 5 objects in fakeserver fiveObj, err := testSuite.bucketHandle.ListObjects(context.Background(), &gcs.ListObjectsRequest{ @@ -815,6 +948,7 @@ func (testSuite *BucketHandleTest) TestListObjectMethodWithZeroMaxResult() { // Hence, we are not writing tests for these parameters // https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/vendor/github.com/fsouza/fake-gcs-server/fakestorage/object.go#L795 func (testSuite *BucketHandleTest) TestUpdateObjectMethodWithValidObject() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) // Metadata value before updating object minObj, _, err := testSuite.bucketHandle.StatObject(context.Background(), &gcs.StatObjectRequest{ @@ -851,6 +985,7 @@ func (testSuite *BucketHandleTest) TestUpdateObjectMethodWithValidObject() { } func (testSuite *BucketHandleTest) TestUpdateObjectMethodWithMissingObject() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) var notfound *gcs.NotFoundError _, err := testSuite.bucketHandle.UpdateObject(context.Background(), @@ -870,19 +1005,17 @@ func (testSuite *BucketHandleTest) TestUpdateObjectMethodWithMissingObject() { // Read content of an object and return func (testSuite *BucketHandleTest) readObjectContent(ctx context.Context, req *gcs.ReadObjectRequest) (buffer string) { - rc, err := testSuite.bucketHandle.NewReader(ctx, &gcs.ReadObjectRequest{ - Name: req.Name, - Range: req.Range}) - + rc, err := testSuite.bucketHandle.NewReaderWithReadHandle(ctx, req) assert.Nil(testSuite.T(), err) defer rc.Close() - buf := make([]byte, req.Range.Limit) + buf := make([]byte, req.Range.Limit-req.Range.Start) _, err = rc.Read(buf) assert.Nil(testSuite.T(), err) return string(buf[:]) } func (testSuite *BucketHandleTest) TestComposeObjectMethodWithDstObjectExist() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) // Reading content before composing it buffer := testSuite.readObjectContent(context.Background(), &gcs.ReadObjectRequest{ @@ -953,6 +1086,7 @@ func (testSuite *BucketHandleTest) TestComposeObjectMethodWithDstObjectExist() { } func (testSuite *BucketHandleTest) TestComposeObjectMethodWithOneSrcObject() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) var notfound *gcs.NotFoundError // Checking that dstObject does not exist _, _, err := testSuite.bucketHandle.StatObject(context.Background(), @@ -1016,6 +1150,7 @@ func (testSuite *BucketHandleTest) TestComposeObjectMethodWithOneSrcObject() { } func (testSuite *BucketHandleTest) TestComposeObjectMethodWithTwoSrcObjects() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) var notfound *gcs.NotFoundError _, _, err := testSuite.bucketHandle.StatObject(context.Background(), &gcs.StatObjectRequest{ @@ -1097,6 +1232,7 @@ func (testSuite *BucketHandleTest) TestComposeObjectMethodWithTwoSrcObjects() { } func (testSuite *BucketHandleTest) TestComposeObjectMethodWhenSrcObjectDoesNotExist() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) var notfound *gcs.NotFoundError _, _, err := testSuite.bucketHandle.StatObject(context.Background(), &gcs.StatObjectRequest{ @@ -1134,6 +1270,7 @@ func (testSuite *BucketHandleTest) TestComposeObjectMethodWhenSrcObjectDoesNotEx } func (testSuite *BucketHandleTest) TestComposeObjectMethodWhenSourceIsNil() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) _, err := testSuite.bucketHandle.ComposeObjects(context.Background(), &gcs.ComposeObjectsRequest{ DstName: TestObjectName, @@ -1159,6 +1296,7 @@ func (testSuite *BucketHandleTest) TestComposeObjectMethodWhenSourceIsNil() { } func (testSuite *BucketHandleTest) TestNameMethod() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) name := testSuite.bucketHandle.Name() assert.Equal(testSuite.T(), TestBucketName, name) @@ -1186,6 +1324,7 @@ func (testSuite *BucketHandleTest) TestIsStorageConditionsNotEmptyWithNonEmptyCo } func (testSuite *BucketHandleTest) TestComposeObjectMethodWhenDstObjectDoesNotExist() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) var notfound *gcs.NotFoundError _, _, err := testSuite.bucketHandle.StatObject(context.Background(), &gcs.StatObjectRequest{ @@ -1276,6 +1415,7 @@ func (testSuite *BucketHandleTest) TestComposeObjectMethodWhenDstObjectDoesNotEx } func (testSuite *BucketHandleTest) TestComposeObjectMethodWithOneSrcObjectIsDstObject() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) // Checking source object 1 exists. This will also be the destination object. srcMinObj1, _, err := testSuite.bucketHandle.StatObject(context.Background(), &gcs.StatObjectRequest{ @@ -1373,59 +1513,104 @@ func (testSuite *BucketHandleTest) TestComposeObjectMethodWithOneSrcObjectIsDstO } func (testSuite *BucketHandleTest) TestBucketTypeForHierarchicalNameSpaceTrue() { - testSuite.mockClient.On("GetStorageLayout", mock.Anything, mock.Anything, mock.Anything). - Return(&controlpb.StorageLayout{ - HierarchicalNamespace: &controlpb.StorageLayout_HierarchicalNamespace{Enabled: true}, - }, nil) + createBucketHandle(testSuite, &controlpb.StorageLayout{ + HierarchicalNamespace: &controlpb.StorageLayout_HierarchicalNamespace{Enabled: true}, + }) + + testSuite.bucketHandle.BucketType() + + assert.Equal(testSuite.T(), &gcs.BucketType{Hierarchical: true}, testSuite.bucketHandle.bucketType, "Expected Hierarchical bucket type") +} + +func (testSuite *BucketHandleTest) TestBucketTypeForZonalLocationType() { + createBucketHandle(testSuite, &controlpb.StorageLayout{ + LocationType: "zone", + }) + + testSuite.bucketHandle.BucketType() + + assert.Equal(testSuite.T(), &gcs.BucketType{Zonal: true}, testSuite.bucketHandle.bucketType, "Expected Zonal bucket type") +} + +func (testSuite *BucketHandleTest) TestBucketTypeForZonalLocationTypeAndHierarchicalNameSpaceTrue() { + createBucketHandle(testSuite, &controlpb.StorageLayout{ + HierarchicalNamespace: &controlpb.StorageLayout_HierarchicalNamespace{Enabled: true}, + LocationType: "zone", + }) testSuite.bucketHandle.BucketType() - assert.Equal(testSuite.T(), gcs.Hierarchical, testSuite.bucketHandle.bucketType, "Expected Hierarchical bucket type") + assert.Equal(testSuite.T(), &gcs.BucketType{Hierarchical: true, Zonal: true}, testSuite.bucketHandle.bucketType, "Expected Zonal bucket type") } func (testSuite *BucketHandleTest) TestBucketTypeForHierarchicalNameSpaceFalse() { - testSuite.mockClient.On("GetStorageLayout", mock.Anything, mock.Anything, mock.Anything). - Return(&controlpb.StorageLayout{ - HierarchicalNamespace: &controlpb.StorageLayout_HierarchicalNamespace{Enabled: false}, - }, nil) + createBucketHandle(testSuite, &controlpb.StorageLayout{ + HierarchicalNamespace: &controlpb.StorageLayout_HierarchicalNamespace{Enabled: false}, + }) testSuite.bucketHandle.BucketType() - assert.Equal(testSuite.T(), gcs.NonHierarchical, testSuite.bucketHandle.bucketType, "Expected NonHierarchical bucket type") + assert.Equal(testSuite.T(), &gcs.BucketType{}, testSuite.bucketHandle.bucketType, "Expected default bucket type") } -func (testSuite *BucketHandleTest) TestBucketTypeWithError() { +func (testSuite *BucketHandleTest) TestBucketHandleWithError() { var x *controlpb.StorageLayout + var err error + + testSuite.mockClient = new(MockStorageControlClient) + testSuite.fakeStorage = NewFakeStorageWithMockClient(testSuite.mockClient, cfg.HTTP2) + testSuite.storageHandle = testSuite.fakeStorage.CreateStorageHandle() + // Test when the client returns an error. testSuite.mockClient.On("GetStorageLayout", mock.Anything, mock.Anything, mock.Anything).Return(x, errors.New("mocked error")) - testSuite.bucketHandle.BucketType() + testSuite.bucketHandle, err = testSuite.storageHandle.BucketHandle(context.Background(), TestBucketName, "") + + assert.Nil(testSuite.T(), testSuite.bucketHandle) + assert.Contains(testSuite.T(), err.Error(), "mocked error") +} + +func (testSuite *BucketHandleTest) TestBucketHandleWithRapidAppendsEnabled() { + var err error + + testSuite.mockClient = new(MockStorageControlClient) + testSuite.fakeStorage = NewFakeStorageWithMockClient(testSuite.mockClient, cfg.HTTP2) + testSuite.storageHandle = testSuite.fakeStorage.CreateStorageHandle() - assert.Equal(testSuite.T(), gcs.Unknown, testSuite.bucketHandle.bucketType, "Expected Unknown when there's an error") + testSuite.mockClient.On("GetStorageLayout", mock.Anything, mock.Anything, mock.Anything).Return(&controlpb.StorageLayout{}, nil) + testSuite.mockClient.On("getClient", mock.Anything, mock.Anything).Return(&storage.Client{}, nil) + + testSuite.bucketHandle, err = testSuite.storageHandle.BucketHandle(context.Background(), TestBucketName, "") + + assert.NotNil(testSuite.T(), testSuite.bucketHandle) + assert.Nil(testSuite.T(), err) } func (testSuite *BucketHandleTest) TestBucketTypeWithHierarchicalNamespaceIsNil() { - testSuite.mockClient.On("GetStorageLayout", mock.Anything, mock.Anything, mock.Anything). - Return(&controlpb.StorageLayout{}, nil) + createBucketHandle(testSuite, &controlpb.StorageLayout{}) testSuite.bucketHandle.BucketType() - assert.Equal(testSuite.T(), gcs.NonHierarchical, testSuite.bucketHandle.bucketType, "Expected NonHierarchical bucket type") + assert.Equal(testSuite.T(), &gcs.BucketType{}, testSuite.bucketHandle.bucketType, "Expected default bucket type") } func (testSuite *BucketHandleTest) TestDefaultBucketTypeWithControlClientNil() { + createBucketHandle(testSuite, &controlpb.StorageLayout{}) var nilControlClient *control.StorageControlClient = nil testSuite.bucketHandle.controlClient = nilControlClient testSuite.bucketHandle.BucketType() - assert.Equal(testSuite.T(), gcs.NonHierarchical, testSuite.bucketHandle.bucketType, "Expected NonHierarchical bucket type") + assert.Equal(testSuite.T(), &gcs.BucketType{}, testSuite.bucketHandle.bucketType, "Expected default bucket type") } func (testSuite *BucketHandleTest) TestDeleteFolderWhenFolderExitForHierarchicalBucket() { + createBucketHandle(testSuite, &controlpb.StorageLayout{ + HierarchicalNamespace: &controlpb.StorageLayout_HierarchicalNamespace{Enabled: true}, + }) ctx := context.Background() deleteFolderReq := controlpb.DeleteFolderRequest{Name: fmt.Sprintf(FullFolderPathHNS, TestBucketName, TestFolderName)} testSuite.mockClient.On("DeleteFolder", ctx, &deleteFolderReq, mock.Anything).Return(nil) - testSuite.bucketHandle.bucketType = gcs.Hierarchical + testSuite.bucketHandle.bucketType = &gcs.BucketType{Hierarchical: true} err := testSuite.bucketHandle.DeleteFolder(ctx, TestFolderName) @@ -1435,9 +1620,12 @@ func (testSuite *BucketHandleTest) TestDeleteFolderWhenFolderExitForHierarchical func (testSuite *BucketHandleTest) TestDeleteFolderWhenFolderNotExistForHierarchicalBucket() { ctx := context.Background() + createBucketHandle(testSuite, &controlpb.StorageLayout{ + HierarchicalNamespace: &controlpb.StorageLayout_HierarchicalNamespace{Enabled: true}, + }) deleteFolderReq := controlpb.DeleteFolderRequest{Name: fmt.Sprintf(FullFolderPathHNS, TestBucketName, missingFolderName)} testSuite.mockClient.On("DeleteFolder", mock.Anything, &deleteFolderReq, mock.Anything).Return(errors.New("mock error")) - testSuite.bucketHandle.bucketType = gcs.Hierarchical + testSuite.bucketHandle.bucketType = &gcs.BucketType{Hierarchical: true} err := testSuite.bucketHandle.DeleteFolder(ctx, missingFolderName) @@ -1447,15 +1635,18 @@ func (testSuite *BucketHandleTest) TestDeleteFolderWhenFolderNotExistForHierarch func (testSuite *BucketHandleTest) TestGetFolderWhenFolderExistsForHierarchicalBucket() { ctx := context.Background() + createBucketHandle(testSuite, &controlpb.StorageLayout{ + HierarchicalNamespace: &controlpb.StorageLayout_HierarchicalNamespace{Enabled: true}, + }) folderPath := fmt.Sprintf(FullFolderPathHNS, TestBucketName, TestFolderName) getFolderReq := controlpb.GetFolderRequest{Name: folderPath} mockFolder := controlpb.Folder{ Name: folderPath, } testSuite.mockClient.On("GetFolder", ctx, &getFolderReq, mock.Anything).Return(&mockFolder, nil) - testSuite.bucketHandle.bucketType = gcs.Hierarchical + testSuite.bucketHandle.bucketType = &gcs.BucketType{Hierarchical: true} - result, err := testSuite.bucketHandle.GetFolder(ctx, TestFolderName) + result, err := testSuite.bucketHandle.GetFolder(ctx, &gcs.GetFolderRequest{Name: TestFolderName}) testSuite.mockClient.AssertExpectations(testSuite.T()) assert.Nil(testSuite.T(), err) @@ -1464,12 +1655,15 @@ func (testSuite *BucketHandleTest) TestGetFolderWhenFolderExistsForHierarchicalB func (testSuite *BucketHandleTest) TestGetFolderWhenFolderDoesNotExistsForHierarchicalBucket() { ctx := context.Background() + createBucketHandle(testSuite, &controlpb.StorageLayout{ + HierarchicalNamespace: &controlpb.StorageLayout_HierarchicalNamespace{Enabled: true}, + }) folderPath := fmt.Sprintf(FullFolderPathHNS, TestBucketName, missingFolderName) getFolderReq := controlpb.GetFolderRequest{Name: folderPath} testSuite.mockClient.On("GetFolder", ctx, &getFolderReq, mock.Anything).Return(nil, status.Error(codes.NotFound, "folder not found")) - testSuite.bucketHandle.bucketType = gcs.Hierarchical + testSuite.bucketHandle.bucketType = &gcs.BucketType{Hierarchical: true} - result, err := testSuite.bucketHandle.GetFolder(ctx, missingFolderName) + result, err := testSuite.bucketHandle.GetFolder(ctx, &gcs.GetFolderRequest{Name: missingFolderName}) testSuite.mockClient.AssertExpectations(testSuite.T()) assert.Nil(testSuite.T(), result) @@ -1478,9 +1672,12 @@ func (testSuite *BucketHandleTest) TestGetFolderWhenFolderDoesNotExistsForHierar func (testSuite *BucketHandleTest) TestRenameFolderWithError() { ctx := context.Background() + createBucketHandle(testSuite, &controlpb.StorageLayout{ + HierarchicalNamespace: &controlpb.StorageLayout_HierarchicalNamespace{Enabled: true}, + }) renameFolderReq := controlpb.RenameFolderRequest{Name: fmt.Sprintf(FullFolderPathHNS, TestBucketName, TestFolderName), DestinationFolderId: TestRenameFolder} testSuite.mockClient.On("RenameFolder", mock.Anything, &renameFolderReq, mock.Anything).Return(nil, errors.New("mock error")) - testSuite.bucketHandle.bucketType = gcs.Hierarchical + testSuite.bucketHandle.bucketType = &gcs.BucketType{Hierarchical: true} _, err := testSuite.bucketHandle.RenameFolder(ctx, TestFolderName, TestRenameFolder) @@ -1489,9 +1686,12 @@ func (testSuite *BucketHandleTest) TestRenameFolderWithError() { } func (testSuite *BucketHandleTest) TestCreateFolderWithError() { + createBucketHandle(testSuite, &controlpb.StorageLayout{ + HierarchicalNamespace: &controlpb.StorageLayout_HierarchicalNamespace{Enabled: true}, + }) createFolderReq := controlpb.CreateFolderRequest{Parent: fmt.Sprintf(FullBucketPathHNS, TestBucketName), FolderId: TestFolderName, Recursive: true} testSuite.mockClient.On("CreateFolder", context.Background(), &createFolderReq, mock.Anything).Return(nil, errors.New("mock error")) - testSuite.bucketHandle.bucketType = gcs.Hierarchical + testSuite.bucketHandle.bucketType = &gcs.BucketType{Hierarchical: true} folder, err := testSuite.bucketHandle.CreateFolder(context.Background(), TestFolderName) @@ -1501,12 +1701,15 @@ func (testSuite *BucketHandleTest) TestCreateFolderWithError() { } func (testSuite *BucketHandleTest) TestCreateFolderWithGivenName() { + createBucketHandle(testSuite, &controlpb.StorageLayout{ + HierarchicalNamespace: &controlpb.StorageLayout_HierarchicalNamespace{Enabled: true}, + }) mockFolder := controlpb.Folder{ Name: fmt.Sprintf(FullFolderPathHNS, TestBucketName, TestFolderName), } createFolderReq := controlpb.CreateFolderRequest{Parent: fmt.Sprintf(FullBucketPathHNS, TestBucketName), FolderId: TestFolderName, Recursive: true} testSuite.mockClient.On("CreateFolder", context.Background(), &createFolderReq, mock.Anything).Return(&mockFolder, nil) - testSuite.bucketHandle.bucketType = gcs.Hierarchical + testSuite.bucketHandle.bucketType = &gcs.BucketType{Hierarchical: true} folder, err := testSuite.bucketHandle.CreateFolder(context.Background(), TestFolderName) diff --git a/internal/storage/caching/fast_stat_bucket.go b/internal/storage/caching/fast_stat_bucket.go index 58a1cf56b5..8f89d26c06 100644 --- a/internal/storage/caching/fast_stat_bucket.go +++ b/internal/storage/caching/fast_stat_bucket.go @@ -15,34 +15,51 @@ package caching import ( + "errors" "fmt" - "io" "strings" "sync" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/metadata" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" "golang.org/x/net/context" "github.com/jacobsa/timeutil" ) +// A *CacheMissError value is an error that indicates an object name or a +// particular generation for that name were not found from cache. +type CacheMissError struct { + Err error +} + +func (cme *CacheMissError) Error() string { + return fmt.Sprintf("CacheMissError: %v", cme.Err) +} + // Create a bucket that caches object records returned by the supplied wrapped // bucket. Records are invalidated when modifications are made through this // bucket, and after the supplied TTL. func NewFastStatBucket( - ttl time.Duration, + primaryCacheTTL time.Duration, cache metadata.StatCache, clock timeutil.Clock, - wrapped gcs.Bucket) (b gcs.Bucket) { + wrapped gcs.Bucket, + negativeCacheTTL time.Duration, + isTypeCacheDeprecated bool, + implicitDir bool, +) (b gcs.Bucket) { fsb := &fastStatBucket{ - cache: cache, - clock: clock, - wrapped: wrapped, - ttl: ttl, + cache: cache, + clock: clock, + wrapped: wrapped, + primaryCacheTTL: primaryCacheTTL, + negativeCacheTTL: negativeCacheTTL, + isTypeCacheDeprecated: isTypeCacheDeprecated, + implicitDir: implicitDir, } b = fsb @@ -66,7 +83,15 @@ type fastStatBucket struct { // Constant data ///////////////////////// - ttl time.Duration + // TTL for entries for existing files and folders in the cache. + primaryCacheTTL time.Duration + // TTL for entries for non-existing files and folders in the cache. + negativeCacheTTL time.Duration + + // Flag to enable deprecation logic of Type cache. + isTypeCacheDeprecated bool + + implicitDir bool } //////////////////////////////////////////////////////////////////////// @@ -78,7 +103,7 @@ func (b *fastStatBucket) insertMultiple(objs []*gcs.Object) { b.mu.Lock() defer b.mu.Unlock() - expiration := b.clock.Now().Add(b.ttl) + expiration := b.clock.Now().Add(b.primaryCacheTTL) for _, o := range objs { m := storageutil.ConvertObjToMinObject(o) b.cache.Insert(m, expiration) @@ -86,11 +111,69 @@ func (b *fastStatBucket) insertMultiple(objs []*gcs.Object) { } // LOCKS_EXCLUDED(b.mu) -func (b *fastStatBucket) insertMultipleMinObjects(minObjs []*gcs.MinObject) { +// insertListing caches all objects and sub-directories discovered during a GCS listing. +// It explicitly handles the "implicit directory" edge case where a directory exists +// only as a prefix to other objects. +func (b *fastStatBucket) insertListing(ctx context.Context, listing *gcs.Listing, dirName string) { b.mu.Lock() defer b.mu.Unlock() - expiration := b.clock.Now().Add(b.ttl) + // Critical check after acquiring lock: If the operation context was cancelled, + // we must not update the cache with this stale data. + if ctx != nil && ctx.Err() != nil { + return + } + + expiration := b.clock.Now().Add(b.primaryCacheTTL) + + // 1. Parent Directory Inference (Implicit Check) + // If the listing contains objects or sub-directories but the directory itself + // is not returned as an explicit object, we infer and cache it as an + // implicit directory. + dirHasContents := len(listing.MinObjects) > 0 || len(listing.CollapsedRuns) > 0 + isDirInListing := len(listing.MinObjects) > 0 && listing.MinObjects[0].Name == dirName + if b.implicitDir && dirHasContents && !isDirInListing { + b.cache.InsertImplicitDir(dirName, expiration) + } + + // 2. Cache Explicit Objects + for _, o := range listing.MinObjects { + b.cache.Insert(o, expiration) + } + + // Do not cache implicit directories if the flag is not passed. + if !b.implicitDir { + return + } + + // 3. Cache Sub-directories (Collapsed Runs) + // These represent folders discovered via prefixes in the ListObjects response. + for _, p := range listing.CollapsedRuns { + // Ensure the prefix follows directory naming conventions (trailing slash). + // Although 'collapsedRuns' is expected to contain only directories, we perform + // this defensive check to prevent processing malformed prefixes. + if !strings.HasSuffix(p, "/") { + logger.Errorf("fastStatBucket: ignoring malformed prefix name: %s", p) + continue + } + + // Cache the prefix as a minimal object (implicit directory marker). + b.cache.InsertImplicitDir(p, expiration) + } +} + +// LOCKS_EXCLUDED(b.mu) +func (b *fastStatBucket) insertMultipleMinObjects(ctx context.Context, minObjs []*gcs.MinObject) { + b.mu.Lock() + defer b.mu.Unlock() + + // Critical check after acquiring lock: If the operation context was cancelled, + // we must not update the cache with this stale data. + if ctx != nil && ctx.Err() != nil { + return + } + + expiration := b.clock.Now().Add(b.primaryCacheTTL) for _, o := range minObjs { b.cache.Insert(o, expiration) } @@ -107,11 +190,17 @@ func (b *fastStatBucket) eraseEntriesWithGivenPrefix(folderName string) { // insertHierarchicalListing saves the objects in cache excluding zero byte objects corresponding to folders // by iterating objects present in listing and saves prefixes as folders (all prefixes are folders in hns) by // iterating collapsedRuns of listing. -func (b *fastStatBucket) insertHierarchicalListing(listing *gcs.Listing) { +func (b *fastStatBucket) insertHierarchicalListing(ctx context.Context, listing *gcs.Listing) { b.mu.Lock() defer b.mu.Unlock() - expiration := b.clock.Now().Add(b.ttl) + // Critical check after acquiring lock: If the operation context was cancelled, + // we must not update the cache with this stale data. + if ctx != nil && ctx.Err() != nil { + return + } + + expiration := b.clock.Now().Add(b.primaryCacheTTL) for _, o := range listing.MinObjects { if !strings.HasSuffix(o.Name, "/") { @@ -138,12 +227,16 @@ func (b *fastStatBucket) insert(o *gcs.Object) { b.insertMultiple([]*gcs.Object{o}) } +func (b *fastStatBucket) insertMinObject(ctx context.Context, o *gcs.MinObject) { + b.insertMultipleMinObjects(ctx, []*gcs.MinObject{o}) +} + // LOCKS_EXCLUDED(b.mu) func (b *fastStatBucket) insertFolder(f *gcs.Folder) { b.mu.Lock() defer b.mu.Unlock() - b.cache.InsertFolder(f, b.clock.Now().Add(b.ttl)) + b.cache.InsertFolder(f, b.clock.Now().Add(b.primaryCacheTTL)) } // LOCKS_EXCLUDED(b.mu) @@ -151,7 +244,7 @@ func (b *fastStatBucket) addNegativeEntry(name string) { b.mu.Lock() defer b.mu.Unlock() - expiration := b.clock.Now().Add(b.ttl) + expiration := b.clock.Now().Add(b.negativeCacheTTL) b.cache.AddNegativeEntry(name, expiration) } @@ -160,7 +253,7 @@ func (b *fastStatBucket) addNegativeEntryForFolder(name string) { b.mu.Lock() defer b.mu.Unlock() - expiration := b.clock.Now().Add(b.ttl) + expiration := b.clock.Now().Add(b.negativeCacheTTL) b.cache.AddNegativeEntryForFolder(name, expiration) } @@ -201,10 +294,15 @@ func (b *fastStatBucket) BucketType() gcs.BucketType { return b.wrapped.BucketType() } -func (b *fastStatBucket) NewReader( +func (b *fastStatBucket) NewReaderWithReadHandle( ctx context.Context, - req *gcs.ReadObjectRequest) (rc io.ReadCloser, err error) { - rc, err = b.wrapped.NewReader(ctx, req) + req *gcs.ReadObjectRequest) (rd gcs.StorageReader, err error) { + rd, err = b.wrapped.NewReaderWithReadHandle(ctx, req) + + var notFoundError *gcs.NotFoundError + if errors.As(err, ¬FoundError) { + b.invalidate(req.Name) + } return } @@ -212,11 +310,10 @@ func (b *fastStatBucket) NewReader( func (b *fastStatBucket) CreateObject( ctx context.Context, req *gcs.CreateObjectRequest) (o *gcs.Object, err error) { - // Throw away any existing record for this object. - b.invalidate(req.Name) - // TODO: create object to be replaced with create folder api once integrated o, err = b.wrapped.CreateObject(ctx, req) + // Throw away any existing record for this object even if there was an error but do it after the API call. + b.invalidate(req.Name) if err != nil { return } @@ -231,18 +328,42 @@ func (b *fastStatBucket) CreateObjectChunkWriter(ctx context.Context, req *gcs.C return b.wrapped.CreateObjectChunkWriter(ctx, req, chunkSize, callBack) } -func (b *fastStatBucket) FinalizeUpload(ctx context.Context, writer gcs.Writer) (*gcs.Object, error) { +func (b *fastStatBucket) CreateAppendableObjectWriter(ctx context.Context, req *gcs.CreateObjectChunkWriterRequest) (gcs.Writer, error) { + w, err := b.wrapped.CreateAppendableObjectWriter(ctx, req) + var precondErr *gcs.PreconditionError + if errors.As(err, &precondErr) { + // If creating a takeover writer fails with a precondition error (for e.g. offset mismatch), + // it indicates our local stat cache is out of sync with the remote object state. + // Throw away the existing record in such cases. + b.invalidate(req.Name) + } + return w, err +} + +func (b *fastStatBucket) FinalizeUpload(ctx context.Context, writer gcs.Writer) (*gcs.MinObject, error) { + o, err := b.wrapped.FinalizeUpload(ctx, writer) + // Throw away any existing record for this object even if there was an error but do it after the API call. name := writer.ObjectName() - // Throw away any existing record for this object. b.invalidate(name) + // Record the new object only if err is nil. + if err == nil { + b.insertMinObject(ctx, o) + } - o, err := b.wrapped.FinalizeUpload(ctx, writer) + return o, err +} + +func (b *fastStatBucket) FlushPendingWrites(ctx context.Context, writer gcs.Writer) (*gcs.MinObject, error) { + o, err := b.wrapped.FlushPendingWrites(ctx, writer) + + // Throw away any existing record for this object even if there was an error but do it after the API call. + name := writer.ObjectName() + b.invalidate(name) // Record the new object if err is nil. if err == nil { - b.insert(o) + b.insertMinObject(ctx, o) } - return o, err } @@ -250,11 +371,9 @@ func (b *fastStatBucket) FinalizeUpload(ctx context.Context, writer gcs.Writer) func (b *fastStatBucket) CopyObject( ctx context.Context, req *gcs.CopyObjectRequest) (o *gcs.Object, err error) { - // Throw away any existing record for the destination name. - b.invalidate(req.DstName) - - // Copy the object. o, err = b.wrapped.CopyObject(ctx, req) + // Throw away any existing record for the destination name even if there was an error but do it after the API call. + b.invalidate(req.DstName) if err != nil { return } @@ -269,11 +388,9 @@ func (b *fastStatBucket) CopyObject( func (b *fastStatBucket) ComposeObjects( ctx context.Context, req *gcs.ComposeObjectsRequest) (o *gcs.Object, err error) { - // Throw away any existing record for the destination name. - b.invalidate(req.DstName) - - // Copy the object. o, err = b.wrapped.ComposeObjects(ctx, req) + // Throw away any existing record for the destination name even if there was an error but do it after the API call. + b.invalidate(req.DstName) if err != nil { return } @@ -317,6 +434,14 @@ func (b *fastStatBucket) StatObject( return } + // Cache Miss Handling + if req.FetchOnlyFromCache { + return nil, nil, &CacheMissError{ + Err: fmt.Errorf("cache miss for %q", req.Name), + } + } + + // Standard fallback to GCS. return b.StatObjectFromGcs(ctx, req) } @@ -330,13 +455,17 @@ func (b *fastStatBucket) ListObjects( return } - if b.BucketType() == gcs.Hierarchical { - b.insertHierarchicalListing(listing) + if b.BucketType().Hierarchical { + b.insertHierarchicalListing(ctx, listing) return } - // note anything we found. - b.insertMultipleMinObjects(listing.MinObjects) + if b.isTypeCacheDeprecated { + b.insertListing(ctx, listing, req.Prefix) + } else { + // note anything we found. + b.insertMultipleMinObjects(ctx, listing.MinObjects) + } return } @@ -344,11 +473,9 @@ func (b *fastStatBucket) ListObjects( func (b *fastStatBucket) UpdateObject( ctx context.Context, req *gcs.UpdateObjectRequest) (o *gcs.Object, err error) { - // Throw away any existing record for this object. - b.invalidate(req.Name) - - // Update the object. o, err = b.wrapped.UpdateObject(ctx, req) + // Throw away any existing record for this object even if there was an error but do it after the API call. + b.invalidate(req.Name) if err != nil { return } @@ -363,18 +490,53 @@ func (b *fastStatBucket) UpdateObject( func (b *fastStatBucket) DeleteObject( ctx context.Context, req *gcs.DeleteObjectRequest) (err error) { - b.invalidate(req.Name) + if req.OnlyDeleteFromCache { + b.addNegativeEntry(req.Name) + return nil + } err = b.wrapped.DeleteObject(ctx, req) + // In case of successful delete, add a negative entry to the cache. + if err == nil { + b.addNegativeEntry(req.Name) + return + } + // If the delete failed due to a precondition error or not found error, + // invalidate the cache entry as the object's state is uncertain. + // For other errors, we don't touch the cache because the object likely + // still exists. + var preconditionErr *gcs.PreconditionError + var notFoundErr *gcs.NotFoundError + if errors.As(err, &preconditionErr) || errors.As(err, ¬FoundErr) { + b.invalidate(req.Name) + } return } +func (b *fastStatBucket) MoveObject(ctx context.Context, req *gcs.MoveObjectRequest) (*gcs.Object, error) { + o, err := b.wrapped.MoveObject(ctx, req) + // Throw away any existing record for the source and destination name even if there was an error but do it after the API call. + b.invalidate(req.SrcName) + b.invalidate(req.DstName) + if err != nil { + return nil, err + } + + // Record the new version. + b.insert(o) + + return o, nil +} + func (b *fastStatBucket) DeleteFolder(ctx context.Context, folderName string) error { err := b.wrapped.DeleteFolder(ctx, folderName) + // In case of an error; invalidate the cached entry. This will make sure that + // gcsfuse is not caching possibly erroneous status of the folder and next + // call will hit GCS backend to probe the latest status. if err != nil { - return err + b.invalidate(folderName) + } else { + b.addNegativeEntryForFolder(folderName) } - // TODO: Caching negative entries for both objects and folders will be implemented together due to test failures. - b.invalidate(folderName) return err } @@ -391,18 +553,18 @@ func (b *fastStatBucket) StatObjectFromGcs(ctx context.Context, } // Put the object in cache. - o := storageutil.ConvertMinObjectToObject(m) - b.insert(o) + b.insertMinObject(ctx, m) return } -func (b *fastStatBucket) GetFolder(ctx context.Context, prefix string) (*gcs.Folder, error) { - if hit, entry := b.lookUpFolder(prefix); hit { +func (b *fastStatBucket) GetFolder(ctx context.Context, req *gcs.GetFolderRequest) (*gcs.Folder, error) { + // Cache Lookup + if hit, entry := b.lookUpFolder(req.Name); hit { // Negative entries result in NotFoundError. if entry == nil { err := &gcs.NotFoundError{ - Err: fmt.Errorf("negative cache entry for folder %v", prefix), + Err: fmt.Errorf("negative cache entry for folder %q", req.Name), } return nil, err @@ -411,12 +573,18 @@ func (b *fastStatBucket) GetFolder(ctx context.Context, prefix string) (*gcs.Fol return entry, nil } + if req.FetchOnlyFromCache { + return nil, &CacheMissError{ + Err: fmt.Errorf("cache miss for %q", req.Name), + } + } + // Fetch the Folder from GCS - return b.getFolderFromGCS(ctx, prefix) + return b.getFolderFromGCS(ctx, req) } -func (b *fastStatBucket) getFolderFromGCS(ctx context.Context, prefix string) (*gcs.Folder, error) { - f, err := b.wrapped.GetFolder(ctx, prefix) +func (b *fastStatBucket) getFolderFromGCS(ctx context.Context, req *gcs.GetFolderRequest) (*gcs.Folder, error) { + f, err := b.wrapped.GetFolder(ctx, req) if err == nil { b.insertFolder(f) @@ -425,16 +593,15 @@ func (b *fastStatBucket) getFolderFromGCS(ctx context.Context, prefix string) (* // Special case: NotFoundError -> negative entry. if _, ok := err.(*gcs.NotFoundError); ok { - b.addNegativeEntryForFolder(prefix) + b.addNegativeEntryForFolder(req.Name) } return nil, err } func (b *fastStatBucket) CreateFolder(ctx context.Context, folderName string) (f *gcs.Folder, err error) { - // Throw away any existing record for this folder. - b.invalidate(folderName) - f, err = b.wrapped.CreateFolder(ctx, folderName) + // Throw away any existing record for this folder even if there was an error but do it after the API call. + b.invalidate(folderName) if err != nil { return } @@ -458,3 +625,18 @@ func (b *fastStatBucket) RenameFolder(ctx context.Context, folderName string, de return f, err } + +func (b *fastStatBucket) NewMultiRangeDownloader( + ctx context.Context, req *gcs.MultiRangeDownloaderRequest) (mrd gcs.MultiRangeDownloader, err error) { + mrd, err = b.wrapped.NewMultiRangeDownloader(ctx, req) + + var notFoundError *gcs.NotFoundError + if errors.As(err, ¬FoundError) { + b.invalidate(req.Name) + } + return +} + +func (b *fastStatBucket) GCSName(obj *gcs.MinObject) string { + return b.wrapped.GCSName(obj) +} diff --git a/internal/storage/caching/fast_stat_bucket_test.go b/internal/storage/caching/fast_stat_bucket_test.go index 4e670d813c..53e5845608 100644 --- a/internal/storage/caching/fast_stat_bucket_test.go +++ b/internal/storage/caching/fast_stat_bucket_test.go @@ -17,14 +17,17 @@ package caching_test import ( "errors" "fmt" + "io" + "strings" "testing" "time" gostorage "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/caching" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/caching/mock_gcscaching" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/caching" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/caching/mock_gcscaching" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/oglemock" . "github.com/jacobsa/ogletest" @@ -38,7 +41,10 @@ func TestFastStatBucket(t *testing.T) { RunTests(t) } // Boilerplate //////////////////////////////////////////////////////////////////////// -const ttl = time.Second +const primaryCacheTTL = time.Second +const negativeCacheTTL = time.Second * 5 +const isTypeCacheDeprecated = true +const isImplicitDir = true type fastStatBucketTest struct { cache mock_gcscaching.MockStatCache @@ -57,10 +63,13 @@ func (t *fastStatBucketTest) SetUp(ti *TestInfo) { t.wrapped = storage.NewMockBucket(ti.MockController, "wrapped") t.bucket = caching.NewFastStatBucket( - ttl, + primaryCacheTTL, t.cache, &t.clock, - t.wrapped) + t.wrapped, + negativeCacheTTL, + isTypeCacheDeprecated, + isImplicitDir) } //////////////////////////////////////////////////////////////////////// @@ -128,7 +137,7 @@ func (t *CreateObjectTest) WrappedSucceeds() { WillOnce(Return(obj, nil)) // Insert - ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(ttl))) + ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(primaryCacheTTL))) // Call o, err := t.bucket.CreateObject(context.TODO(), &gcs.CreateObjectRequest{}) @@ -208,6 +217,92 @@ func (t *CreateObjectChunkWriterTest) WrappedSucceeds() { ExpectEq(wr, gotWr) } +//////////////////////////////////////////////////////////////////////// +// CreateAppendableObjectWriterTest +//////////////////////////////////////////////////////////////////////// + +type CreateAppendableObjectWriterTest struct { + fastStatBucketTest +} + +func init() { RegisterTestSuite(&CreateAppendableObjectWriterTest{}) } + +func (t *CreateAppendableObjectWriterTest) CallsWrappedWithExpectedParameters() { + const name = "taco" + const offset int64 = 10 + const chunkSize = 1024 + ctx := context.TODO() + // Wrapped + var wrappedReq *gcs.CreateObjectChunkWriterRequest + ExpectCall(t.wrapped, "CreateAppendableObjectWriter")(Any(), Any()). + WillOnce(DoAll(SaveArg(1, &wrappedReq), Return(nil, errors.New("")))) + // Call + req := &gcs.CreateObjectChunkWriterRequest{ + CreateObjectRequest: gcs.CreateObjectRequest{ + Name: name, + }, + Offset: offset, + ChunkSize: chunkSize, + } + + _, _ = t.bucket.CreateAppendableObjectWriter(ctx, req) + + AssertNe(nil, wrappedReq) + ExpectEq(req, wrappedReq) +} + +func (t *CreateAppendableObjectWriterTest) WrappedFails() { + ctx := context.TODO() + req := &gcs.CreateObjectChunkWriterRequest{} + var err error + // Wrapped + ExpectCall(t.wrapped, "CreateAppendableObjectWriter")(Any(), Any()). + WillOnce(Return(nil, errors.New("taco"))) + + // Call + _, err = t.bucket.CreateAppendableObjectWriter(ctx, req) + + ExpectThat(err, Error(HasSubstr("taco"))) +} + +func (t *CreateAppendableObjectWriterTest) WrappedSucceeds() { + ctx := context.TODO() + req := &gcs.CreateObjectChunkWriterRequest{} + var err error + // Wrapped + wr := &storage.ObjectWriter{ + Writer: &gostorage.Writer{}, + } + ExpectCall(t.wrapped, "CreateAppendableObjectWriter")(Any(), Any()). + WillOnce(Return(wr, nil)) + + // Call + gotWr, err := t.bucket.CreateAppendableObjectWriter(ctx, req) + + AssertEq(nil, err) + ExpectEq(wr, gotWr) +} + +func (t *CreateAppendableObjectWriterTest) WrappedReturnsPreconditionError() { + const name = "taco" + ctx := context.TODO() + req := &gcs.CreateObjectChunkWriterRequest{ + CreateObjectRequest: gcs.CreateObjectRequest{ + Name: name, + }, + } + // Erase + ExpectCall(t.cache, "Erase")(name) + // Wrapped + ExpectCall(t.wrapped, "CreateAppendableObjectWriter")(Any(), Any()). + WillOnce(Return(nil, &gcs.PreconditionError{Err: errors.New("precondition failed")})) + + // Call + _, err := t.bucket.CreateAppendableObjectWriter(ctx, req) + + ExpectThat(err, Error(HasSubstr("precondition failed"))) +} + //////////////////////////////////////////////////////////////////////// // FinalizeUpload //////////////////////////////////////////////////////////////////////// @@ -228,7 +323,7 @@ func (t *FinalizeUploadTest) CallsEraseAndWrappedWithExpectedParameter() { // Wrapped var wrappedWriter gcs.Writer ExpectCall(t.wrapped, "FinalizeUpload")(Any(), Any()). - WillOnce(DoAll(SaveArg(1, &wrappedWriter), Return(&gcs.Object{}, errors.New("")))) + WillOnce(DoAll(SaveArg(1, &wrappedWriter), Return(&gcs.MinObject{}, errors.New("")))) // Call _, _ = t.bucket.FinalizeUpload(context.TODO(), writer) @@ -246,7 +341,7 @@ func (t *FinalizeUploadTest) WrappedFails() { ExpectCall(t.cache, "Erase")(Any()) // Wrapped ExpectCall(t.wrapped, "FinalizeUpload")(Any(), Any()). - WillOnce(Return(&gcs.Object{}, errors.New("taco"))) + WillOnce(Return(&gcs.MinObject{}, errors.New("taco"))) // Call o, err := t.bucket.FinalizeUpload(context.TODO(), writer) @@ -265,9 +360,9 @@ func (t *FinalizeUploadTest) WrappedSucceeds() { ExpectCall(t.cache, "Erase")(Any()) // Wrapped ExpectCall(t.wrapped, "FinalizeUpload")(Any(), Any()). - WillOnce(Return(&gcs.Object{}, nil)) + WillOnce(Return(&gcs.MinObject{}, nil)) // Insert - ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(ttl))) + ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(primaryCacheTTL))) // Call o, err := t.bucket.FinalizeUpload(context.TODO(), writer) @@ -276,6 +371,62 @@ func (t *FinalizeUploadTest) WrappedSucceeds() { ExpectNe(nil, o) } +//////////////////////////////////////////////////////////////////////// +// FinalizeUpload +//////////////////////////////////////////////////////////////////////// + +type FlushPendingWritesTest struct { + fastStatBucketTest +} + +func init() { RegisterTestSuite(&FlushPendingWritesTest{}) } + +func (t *FlushPendingWritesTest) WrappedFails() { + const name = "taco" + writer := &storage.ObjectWriter{ + Writer: &gostorage.Writer{ObjectAttrs: gostorage.ObjectAttrs{Name: name}}, + } + // Expect cache Erase. + ExpectCall(t.cache, "Erase")(name) + // Expect call to Wrapped method. + var wrappedWriter gcs.Writer + mockObject := &gcs.MinObject{Size: 10} + ExpectCall(t.wrapped, "FlushPendingWrites")(Any(), Any()). + WillOnce(DoAll(SaveArg(1, &wrappedWriter), Return(mockObject, errors.New("taco")))) + + // Call. + gotObject, err := t.bucket.FlushPendingWrites(context.TODO(), writer) + + ExpectEq(writer, wrappedWriter) + ExpectEq(mockObject, gotObject) + ExpectThat(err, Error(HasSubstr("taco"))) +} + +func (t *FlushPendingWritesTest) WrappedSucceeds() { + const name = "taco" + writer := &storage.ObjectWriter{ + Writer: &gostorage.Writer{ObjectAttrs: gostorage.ObjectAttrs{Name: name}}, + } + var err error + // Expect cache Erase. + ExpectCall(t.cache, "Erase")(name) + // Wrapped. + mockObject := &gcs.MinObject{Size: 10} + ExpectCall(t.wrapped, "FlushPendingWrites")(Any(), Any()). + WillOnce(Return(mockObject, nil)) + // Insert. + var cachedMinObject *gcs.MinObject + ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(primaryCacheTTL))). + WillOnce(DoAll(SaveArg(0, &cachedMinObject))) + + // Call + gotObject, err := t.bucket.FlushPendingWrites(context.TODO(), writer) + + AssertEq(nil, err) + ExpectEq(mockObject, gotObject) + ExpectEq(mockObject.Size, cachedMinObject.Size) +} + //////////////////////////////////////////////////////////////////////// // CopyObject //////////////////////////////////////////////////////////////////////// @@ -343,7 +494,7 @@ func (t *CopyObjectTest) WrappedSucceeds() { WillOnce(Return(obj, nil)) // Insert - ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(ttl))) + ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(primaryCacheTTL))) // Call o, err := t.bucket.CopyObject(context.TODO(), &gcs.CopyObjectRequest{}) @@ -421,7 +572,7 @@ func (t *ComposeObjectsTest) WrappedSucceeds() { WillOnce(Return(obj, nil)) // Insert - ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(ttl))) + ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(primaryCacheTTL))) // Call o, err := t.bucket.ComposeObjects(context.TODO(), &gcs.ComposeObjectsRequest{}) @@ -519,7 +670,7 @@ func (t *StatObjectTest) IgnoresCacheEntryWhenForceFetchFromGcsIsTrue() { WillOnce(Return(minObjFromGcs, extObjAttrFromGcs, nil)) // Insert - ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(ttl))) + ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(primaryCacheTTL))) m, e, err := t.bucket.StatObject(context.TODO(), req) AssertEq(nil, err) @@ -551,7 +702,7 @@ func (t *StatObjectTest) TestStatObject_ForceFetchFromGcsTrueAndReturnExtendedOb WillOnce(Return(minObjFromGcs, &gcs.ExtendedObjectAttributes{}, nil)) // Insert - ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(ttl))) + ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(primaryCacheTTL))) m, e, err := t.bucket.StatObject(context.TODO(), req) AssertEq(nil, err) @@ -631,7 +782,7 @@ func (t *StatObjectTest) WrappedSaysNotFound() { // AddNegativeEntry ExpectCall(t.cache, "AddNegativeEntry")( name, - timeutil.TimeEq(t.clock.Now().Add(ttl))) + timeutil.TimeEq(t.clock.Now().Add(negativeCacheTTL))) // Call req := &gcs.StatObjectRequest{ @@ -659,7 +810,7 @@ func (t *StatObjectTest) WrappedSucceeds() { WillOnce(Return(minObj, nil, nil)) // Insert - ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(ttl))) + ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(primaryCacheTTL))) // Call req := &gcs.StatObjectRequest{ @@ -696,7 +847,7 @@ func (t *ListObjectsTest) EmptyListing() { expected := &gcs.Listing{} ExpectCall(t.wrapped, "BucketType")(). - WillOnce(Return(gcs.NonHierarchical)) + WillOnce(Return(gcs.BucketType{})) ExpectCall(t.wrapped, "ListObjects")(Any(), Any()). WillOnce(Return(expected, nil)) @@ -713,7 +864,7 @@ func (t *ListObjectsTest) EmptyListingForHNS() { expected := &gcs.Listing{} ExpectCall(t.wrapped, "BucketType")(). - WillOnce(Return(gcs.Hierarchical)) + WillOnce(Return(gcs.BucketType{Hierarchical: true})) ExpectCall(t.wrapped, "ListObjects")(Any(), Any()). WillOnce(Return(expected, nil)) @@ -735,13 +886,14 @@ func (t *ListObjectsTest) NonEmptyListing() { } ExpectCall(t.wrapped, "BucketType")(). - WillOnce(Return(gcs.NonHierarchical)) + WillOnce(Return(gcs.BucketType{})) ExpectCall(t.wrapped, "ListObjects")(Any(), Any()). WillOnce(Return(expected, nil)) // Insert - ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(ttl))).Times(2) + ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(primaryCacheTTL))).Times(2) + ExpectCall(t.cache, "InsertImplicitDir")(Any(), timeutil.TimeEq(t.clock.Now().Add(primaryCacheTTL))).Times(1) // Call listing, err := t.bucket.ListObjects(context.TODO(), &gcs.ListObjectsRequest{}) @@ -761,15 +913,15 @@ func (t *ListObjectsTest) NonEmptyListingForHNS() { } ExpectCall(t.wrapped, "BucketType")(). - WillOnce(Return(gcs.Hierarchical)) + WillOnce(Return(gcs.BucketType{Hierarchical: true})) ExpectCall(t.wrapped, "ListObjects")(Any(), Any()). WillOnce(Return(expected, nil)) // insert - ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(ttl))).Times(2) + ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(primaryCacheTTL))).Times(2) - ExpectCall(t.cache, "InsertFolder")(Any(), timeutil.TimeEq(t.clock.Now().Add(ttl))).Times(1) + ExpectCall(t.cache, "InsertFolder")(Any(), timeutil.TimeEq(t.clock.Now().Add(primaryCacheTTL))).Times(1) // call listing, err := t.bucket.ListObjects(context.TODO(), &gcs.ListObjectsRequest{}) @@ -778,6 +930,227 @@ func (t *ListObjectsTest) NonEmptyListingForHNS() { ExpectEq(expected, listing) } +func (t *ListObjectsTest) NonEmptyListingWithCancelledContext() { + // Wrapped + o0 := &gcs.MinObject{Name: "taco"} + o1 := &gcs.MinObject{Name: "burrito"} + expected := &gcs.Listing{ + MinObjects: []*gcs.MinObject{o0, o1}, + } + ExpectCall(t.wrapped, "BucketType")(). + WillOnce(Return(gcs.BucketType{})) + // Create a cancellable context. + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Cancel immediately. + ExpectCall(t.wrapped, "ListObjects")(ctx, Any()). + WillOnce(Return(expected, nil)) + // Insert not called. + ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(primaryCacheTTL))).Times(0) + + // Call + listing, err := t.bucket.ListObjects(ctx, &gcs.ListObjectsRequest{}) + + AssertEq(nil, err) + ExpectEq(expected, listing) +} + +func (t *ListObjectsTest) NonEmptyListingWithCancelledContextForHNS() { + // wrapped + o0 := &gcs.MinObject{Name: "taco"} + o1 := &gcs.MinObject{Name: "burrito"} + expected := &gcs.Listing{ + MinObjects: []*gcs.MinObject{o0, o1}, + CollapsedRuns: []string{"p0", "p1/"}, + } + ExpectCall(t.wrapped, "BucketType")(). + WillOnce(Return(gcs.BucketType{Hierarchical: true})) + // Create a cancellable context. + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Cancel immediately. + ExpectCall(t.wrapped, "ListObjects")(ctx, Any()). + WillOnce(Return(expected, nil)) + // insert not called. + ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(primaryCacheTTL))).Times(0) + ExpectCall(t.cache, "InsertFolder")(Any(), timeutil.TimeEq(t.clock.Now().Add(primaryCacheTTL))).Times(0) + + // call + listing, err := t.bucket.ListObjects(ctx, &gcs.ListObjectsRequest{}) + + AssertEq(nil, err) + ExpectEq(expected, listing) +} + +//////////////////////////////////////////////////////////////////////// +// ListObjectsTest_InsertListing +//////////////////////////////////////////////////////////////////////// + +type ListObjectsTest_InsertListing struct { + fastStatBucketTest +} + +func init() { RegisterTestSuite(&ListObjectsTest_InsertListing{}) } + +func (t *ListObjectsTest_InsertListing) SetUp(ti *TestInfo) { + t.fastStatBucketTest.SetUp(ti) + t.bucket = caching.NewFastStatBucket( + primaryCacheTTL, + t.cache, + &t.clock, + t.wrapped, + negativeCacheTTL, + true, + true) +} + +func (t *ListObjectsTest_InsertListing) callAndVerify(ctx context.Context, isHNS bool, listing *gcs.Listing, prefix string, expectedInserts []*gcs.MinObject, expectedImplicitDirs []string) { + // Wrapped + ExpectCall(t.wrapped, "BucketType")(). + WillOnce(Return(gcs.BucketType{Hierarchical: isHNS})) + ExpectCall(t.wrapped, "ListObjects")(Any(), Any()). + WillOnce(Return(listing, nil)) + // Register expectations. + for _, obj := range expectedInserts { + ExpectCall(t.cache, "Insert")(Pointee(DeepEquals(*obj)), Any()) + } + for _, dir := range expectedImplicitDirs { + ExpectCall(t.cache, "InsertImplicitDir")(dir, Any()) + } + + // Call + gotListing, err := t.bucket.ListObjects(ctx, &gcs.ListObjectsRequest{Prefix: prefix}) + + AssertEq(nil, err) + AssertEq(listing, gotListing) +} + +func (t *ListObjectsTest_InsertListing) EmptyListing() { + listing := &gcs.Listing{} + expectedInserts := []*gcs.MinObject{} + expectedImplicitDirs := []string{} + + t.callAndVerify(context.TODO(), false, listing, "dir/", expectedInserts, expectedImplicitDirs) +} + +func (t *ListObjectsTest_InsertListing) ObjectsOnly() { + listing := &gcs.Listing{ + MinObjects: []*gcs.MinObject{ + {Name: "dir/a", Size: 1}, + {Name: "dir/b", Size: 2}, + }, + } + expectedInserts := []*gcs.MinObject{ + {Name: "dir/a", Size: 1}, + {Name: "dir/b", Size: 2}, + } + expectedImplicitDirs := []string{"dir/"} + + t.callAndVerify(context.TODO(), false, listing, "dir/", expectedInserts, expectedImplicitDirs) +} + +func (t *ListObjectsTest_InsertListing) CollapsedRunsOnly() { + listing := &gcs.Listing{ + CollapsedRuns: []string{"dir/a/", "dir/b/"}, + } + expectedImplicitDirs := []string{"dir/", "dir/a/", "dir/b/"} + expectedInserts := []*gcs.MinObject{} + + t.callAndVerify(context.TODO(), false, listing, "dir/", expectedInserts, expectedImplicitDirs) +} + +func (t *ListObjectsTest_InsertListing) ObjectsAndCollapsedRuns() { + listing := &gcs.Listing{ + MinObjects: []*gcs.MinObject{ + {Name: "dir/a", Size: 1}, + }, + CollapsedRuns: []string{"dir/b/"}, + } + expectedInserts := []*gcs.MinObject{ + {Name: "dir/a", Size: 1}, + } + expectedImplicitDirs := []string{"dir/", "dir/b/"} + + t.callAndVerify(context.TODO(), false, listing, "dir/", expectedInserts, expectedImplicitDirs) +} + +func (t *ListObjectsTest_InsertListing) ImplicitDir() { + listing := &gcs.Listing{ + MinObjects: []*gcs.MinObject{ + {Name: "dir/a", Size: 1}, + }, + } + expectedInserts := []*gcs.MinObject{ + {Name: "dir/a", Size: 1}, + } + expectedImplicitDirs := []string{"dir/"} + + t.callAndVerify(context.TODO(), false, listing, "dir/", expectedInserts, expectedImplicitDirs) +} + +func (t *ListObjectsTest_InsertListing) ObjectSameAsCollapsedRun() { + listing := &gcs.Listing{ + MinObjects: []*gcs.MinObject{ + {Name: "dir/a/", Size: 0}, + }, + CollapsedRuns: []string{"dir/a/"}, + } + expectedInserts := []*gcs.MinObject{ + {Name: "dir/a/", Size: 0}, + } + expectedImplicitDirs := []string{"dir/", "dir/a/"} + + t.callAndVerify(context.TODO(), false, listing, "dir/", expectedInserts, expectedImplicitDirs) +} + +func (t *ListObjectsTest_InsertListing) cancelledContextDoesNotUpdatesCache(isHNS bool) { + // Helper function to test for context cancelled scenarios. + // 1. Setup + listing := &gcs.Listing{ + CollapsedRuns: []string{"dir/a/", "dir/b/"}, + MinObjects: []*gcs.MinObject{ + {Name: "dir/file.txt", Size: 123}, + }, + } + // Create a cancellable context. + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Cancel immediately. + expectedInserts := []*gcs.MinObject{} + expectedImplicitDirs := []string{} + + t.callAndVerify(ctx, isHNS, listing, "dir/", expectedInserts, expectedImplicitDirs) +} + +func (t *ListObjectsTest_InsertListing) TestInsertListing_ContextCancelledDoesNotUpdatesCache_HNSBucket() { + t.cancelledContextDoesNotUpdatesCache(true) +} + +func (t *ListObjectsTest_InsertListing) TestInsertListing_ContextCancelledDoesNotUpdatesCache_FlatBucket() { + t.cancelledContextDoesNotUpdatesCache(false) +} + +func (t *ListObjectsTest_InsertListing) ImplicitDirFalse_CollapsedRunsNotCached() { + // Re-initialize bucket with implicitDir = false + t.bucket = caching.NewFastStatBucket( + primaryCacheTTL, + t.cache, + &t.clock, + t.wrapped, + negativeCacheTTL, + true, + false) + listing := &gcs.Listing{ + MinObjects: []*gcs.MinObject{ + {Name: "dir/a", Size: 1}, + }, + CollapsedRuns: []string{"dir/b/"}, + } + expectedInserts := []*gcs.MinObject{ + {Name: "dir/a", Size: 1}, + } + expectedImplicitDirs := []string{} + + t.callAndVerify(context.TODO(), false, listing, "dir/", expectedInserts, expectedImplicitDirs) +} + //////////////////////////////////////////////////////////////////////// // UpdateObject //////////////////////////////////////////////////////////////////////// @@ -843,7 +1216,7 @@ func (t *UpdateObjectTest) WrappedSucceeds() { WillOnce(Return(obj, nil)) // Insert - ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(ttl))) + ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(primaryCacheTTL))) // Call o, err := t.bucket.UpdateObject(context.TODO(), &gcs.UpdateObjectRequest{}) @@ -867,12 +1240,9 @@ func (t *DeleteObjectTest) deleteObject(name string) (err error) { return } -func (t *DeleteObjectTest) CallsEraseAndWrapped() { +func (t *DeleteObjectTest) CallsWrapped() { const name = "taco" - // Erase - ExpectCall(t.cache, "Erase")(name) - // Wrapped var wrappedReq *gcs.DeleteObjectRequest ExpectCall(t.wrapped, "DeleteObject")(Any(), Any()). @@ -885,13 +1255,10 @@ func (t *DeleteObjectTest) CallsEraseAndWrapped() { ExpectEq(name, wrappedReq.Name) } -func (t *DeleteObjectTest) WrappedFails() { +func (t *DeleteObjectTest) WrappedFails_GenericError() { const name = "" var err error - // Erase - ExpectCall(t.cache, "Erase")(Any()) - // Wrapped ExpectCall(t.wrapped, "DeleteObject")(Any(), Any()). WillOnce(Return(errors.New("taco"))) @@ -902,12 +1269,40 @@ func (t *DeleteObjectTest) WrappedFails() { ExpectThat(err, Error(HasSubstr("taco"))) } -func (t *DeleteObjectTest) WrappedSucceeds() { +func (t *DeleteObjectTest) WrappedReturnsPreconditionError() { + const name = "taco" + // Erase + ExpectCall(t.cache, "Erase")(name) + // Wrapped + ExpectCall(t.wrapped, "DeleteObject")(Any(), Any()). + WillOnce(Return(&gcs.PreconditionError{Err: errors.New("precondition failed")})) + + // Call. + err := t.deleteObject(name) + + ExpectThat(err, Error(HasSubstr("precondition failed"))) +} + +func (t *DeleteObjectTest) WrappedReturnsNotFoundError() { + const name = "taco" + // Erase + ExpectCall(t.cache, "Erase")(name) + // Wrapped + ExpectCall(t.wrapped, "DeleteObject")(Any(), Any()). + WillOnce(Return(&gcs.NotFoundError{Err: errors.New("object not found")})) + + // Call. + err := t.deleteObject(name) + + ExpectThat(err, Error(HasSubstr("object not found"))) +} + +func (t *DeleteObjectTest) WrappedSucceeds_AddsNegativeEntry() { const name = "" var err error - // Erase - ExpectCall(t.cache, "Erase")(Any()) + // AddNegativeEntry + ExpectCall(t.cache, "AddNegativeEntry")(Any(), Any()) // Wrapped ExpectCall(t.wrapped, "DeleteObject")(Any(), Any()). @@ -918,6 +1313,22 @@ func (t *DeleteObjectTest) WrappedSucceeds() { AssertEq(nil, err) } +func (t *DeleteObjectTest) OnlyDeleteFromCache() { + const name = "taco" + req := &gcs.DeleteObjectRequest{ + Name: name, + OnlyDeleteFromCache: true, + } + // Expect AddNegativeEntry call. + ExpectCall(t.cache, "AddNegativeEntry")( + name, + timeutil.TimeEq(t.clock.Now().Add(negativeCacheTTL))) + + err := t.bucket.DeleteObject(context.TODO(), req) + + AssertEq(nil, err) +} + func (t *StatObjectTest) TestShouldReturnFromCacheWhenEntryIsPresent() { const name = "some-name" folder := &gcs.Folder{ @@ -926,7 +1337,7 @@ func (t *StatObjectTest) TestShouldReturnFromCacheWhenEntryIsPresent() { ExpectCall(t.cache, "LookUpFolder")(name, Any()). WillOnce(Return(true, folder)) - result, err := t.bucket.GetFolder(context.TODO(), name) + result, err := t.bucket.GetFolder(context.TODO(), &gcs.GetFolderRequest{Name: name}) AssertEq(nil, err) ExpectThat(result, Pointee(DeepEquals(*folder))) @@ -938,7 +1349,7 @@ func (t *StatObjectTest) TestShouldReturnNotFoundErrorWhenNilEntryIsReturned() { ExpectCall(t.cache, "LookUpFolder")(name, Any()). WillOnce(Return(true, nil)) - result, err := t.bucket.GetFolder(context.TODO(), name) + result, err := t.bucket.GetFolder(context.TODO(), &gcs.GetFolderRequest{Name: name}) ExpectThat(err, HasSameTypeAs(&gcs.NotFoundError{})) AssertEq(nil, result) @@ -949,15 +1360,16 @@ func (t *StatObjectTest) TestShouldCallGetFolderWhenEntryIsNotPresent() { folder := &gcs.Folder{ Name: name, } + getFolderReq := &gcs.GetFolderRequest{Name: name} ExpectCall(t.cache, "LookUpFolder")(name, Any()). WillOnce(Return(false, nil)) ExpectCall(t.cache, "InsertFolder")(folder, Any()). WillOnce(Return()) - ExpectCall(t.wrapped, "GetFolder")(Any(), name). + ExpectCall(t.wrapped, "GetFolder")(Any(), getFolderReq). WillOnce(Return(folder, nil)) - result, err := t.bucket.GetFolder(context.TODO(), name) + result, err := t.bucket.GetFolder(context.TODO(), getFolderReq) AssertEq(nil, err) ExpectThat(result, Pointee(DeepEquals(*folder))) @@ -966,13 +1378,14 @@ func (t *StatObjectTest) TestShouldCallGetFolderWhenEntryIsNotPresent() { func (t *StatObjectTest) TestShouldReturnNilWhenErrorIsReturnedFromGetFolder() { const name = "some-name" error := errors.New("connection error") + getFolderReq := &gcs.GetFolderRequest{Name: name} ExpectCall(t.cache, "LookUpFolder")(name, Any()). WillOnce(Return(false, nil)) - ExpectCall(t.wrapped, "GetFolder")(Any(), name). + ExpectCall(t.wrapped, "GetFolder")(Any(), getFolderReq). WillOnce(Return(nil, error)) - folder, result := t.bucket.GetFolder(context.TODO(), name) + folder, result := t.bucket.GetFolder(context.TODO(), getFolderReq) AssertEq(nil, folder) AssertEq(error, result) @@ -995,6 +1408,71 @@ func (t *StatObjectTest) TestRenameFolder() { ExpectEq(result, folder) } +func (t *StatObjectTest) FetchOnlyFromCacheFalse() { + const name = "taco" + req := &gcs.StatObjectRequest{ + Name: name, + FetchOnlyFromCache: false, + } + // We expect a call to GCS, so we mock the wrapped bucket. + ExpectCall(t.cache, "LookUp")(name, Any()). + WillOnce(Return(false, nil)) + + minObj := &gcs.MinObject{Name: name} + ExpectCall(t.wrapped, "StatObject")(Any(), Any()). + WillOnce(Return(minObj, nil, nil)) + ExpectCall(t.cache, "Insert")(Any(), Any()) + + m, _, err := t.bucket.StatObject(context.TODO(), req) + + AssertEq(nil, err) + ExpectEq(minObj, m) +} + +func (t *StatObjectTest) FetchOnlyFromCacheTrue_CacheHitPositive() { + const name = "taco" + req := &gcs.StatObjectRequest{ + Name: name, + FetchOnlyFromCache: true, + } + minObj := &gcs.MinObject{Name: name} + ExpectCall(t.cache, "LookUp")(name, Any()). + WillOnce(Return(true, minObj)) + + m, _, err := t.bucket.StatObject(context.TODO(), req) + + AssertEq(nil, err) + ExpectEq(minObj, m) +} + +func (t *StatObjectTest) FetchOnlyFromCacheTrue_CacheHitNegative() { + const name = "taco" + req := &gcs.StatObjectRequest{ + Name: name, + FetchOnlyFromCache: true, + } + ExpectCall(t.cache, "LookUp")(name, Any()). + WillOnce(Return(true, nil)) + + _, _, err := t.bucket.StatObject(context.TODO(), req) + + ExpectThat(err, HasSameTypeAs(&gcs.NotFoundError{})) +} + +func (t *StatObjectTest) FetchOnlyFromCacheTrue_CacheMiss() { + const name = "taco" + req := &gcs.StatObjectRequest{ + Name: name, + FetchOnlyFromCache: true, + } + ExpectCall(t.cache, "LookUp")(name, Any()). + WillOnce(Return(false, nil)) + + _, _, err := t.bucket.StatObject(context.TODO(), req) + + ExpectThat(err, HasSameTypeAs(&caching.CacheMissError{})) +} + type DeleteFolderTest struct { fastStatBucketTest } @@ -1003,9 +1481,10 @@ func init() { RegisterTestSuite(&DeleteFolderTest{}) } func (t *DeleteFolderTest) Test_DeleteFolder_Success() { const name = "some-name" + ExpectCall(t.cache, "AddNegativeEntryForFolder")(name, Any()). + WillOnce(Return()) ExpectCall(t.wrapped, "DeleteFolder")(Any(), name). WillOnce(Return(nil)) - ExpectCall(t.cache, "Erase")(name).WillOnce(Return()) err := t.bucket.DeleteFolder(context.TODO(), name) @@ -1014,6 +1493,8 @@ func (t *DeleteFolderTest) Test_DeleteFolder_Success() { func (t *DeleteFolderTest) Test_DeleteFolder_Failure() { const name = "some-name" + // Erase + ExpectCall(t.cache, "Erase")(Any()) ExpectCall(t.wrapped, "DeleteFolder")(Any(), name). WillOnce(Return(fmt.Errorf("mock error"))) @@ -1058,3 +1539,207 @@ func (t *CreateFolderTest) TestCreateFolderWhenGCSCallFails() { AssertNe(nil, err) AssertEq(nil, result) } + +type MoveObjectTest struct { + fastStatBucketTest +} + +func init() { RegisterTestSuite(&MoveObjectTest{}) } + +func (t *MoveObjectTest) MoveObjectFails() { + const srcName = "taco" + const dstName = "burrito" + + // Erase + ExpectCall(t.cache, "Erase")(dstName) + ExpectCall(t.cache, "Erase")(srcName) + + // Wrapped + ExpectCall(t.wrapped, "MoveObject")(Any(), Any()).WillOnce(Return(nil, errors.New("taco"))) + + // Call + _, err := t.bucket.MoveObject(context.TODO(), &gcs.MoveObjectRequest{SrcName: srcName, DstName: dstName}) + + ExpectThat(err, Error(HasSubstr("taco"))) +} + +func (t *MoveObjectTest) MoveObjectSucceeds() { + const dstName = "burrito" + // Erase + ExpectCall(t.cache, "Erase")(Any()).Times(2) + + // Wrap object + obj := &gcs.Object{ + Name: dstName, + Generation: 1234, + } + ExpectCall(t.wrapped, "MoveObject")(Any(), Any()).WillOnce(Return(obj, nil)) + + // Insert in cache + ExpectCall(t.cache, "Insert")(Any(), timeutil.TimeEq(t.clock.Now().Add(primaryCacheTTL))) + + // Call + o, err := t.bucket.MoveObject(context.TODO(), &gcs.MoveObjectRequest{}) + + AssertEq(nil, err) + ExpectEq(obj, o) +} + +//////////////////////////////////////////////////////////////////////// +// NewReaderWithReadHandleTest +//////////////////////////////////////////////////////////////////////// + +type NewReaderWithReadHandleTest struct { + fastStatBucketTest +} + +func init() { RegisterTestSuite(&NewReaderWithReadHandleTest{}) } + +func (t *NewReaderWithReadHandleTest) CallsWrappedAndInvalidatesOnNotFound() { + const name = "some-name" + // Expect: wrapped bucket returns NotFoundError + var wrappedReq *gcs.ReadObjectRequest + ExpectCall(t.wrapped, "NewReaderWithReadHandle")(Any(), Any()). + WillOnce(DoAll(SaveArg(1, &wrappedReq), Return(nil, &gcs.NotFoundError{Err: errors.New("not found")}))) + // Expect: cache invalidate is called + ExpectCall(t.cache, "Erase")(name) + + // Call + req := &gcs.ReadObjectRequest{Name: name} + rd, err := t.bucket.NewReaderWithReadHandle(context.TODO(), req) + + AssertEq(nil, rd) + ExpectThat(err, HasSameTypeAs(&gcs.NotFoundError{})) + AssertEq(name, wrappedReq.Name) +} + +func (t *NewReaderWithReadHandleTest) CallsWrappedAndDoesNotInvalidateOnSuccess() { + const name = "some-name" + expectedReader := &fake.FakeReader{ReadCloser: io.NopCloser(strings.NewReader("abc")), Handle: []byte("fake")} + // Expect: wrapped returns reader, no error + ExpectCall(t.wrapped, "NewReaderWithReadHandle")(Any(), Any()). + WillOnce(Return(expectedReader, nil)) + + // Call + req := &gcs.ReadObjectRequest{Name: name} + rd, err := t.bucket.NewReaderWithReadHandle(context.TODO(), req) + + AssertEq(nil, err) + ExpectEq(expectedReader, rd) +} + +//////////////////////////////////////////////////////////////////////// +// NewMultiRangeDownloader +//////////////////////////////////////////////////////////////////////// + +type NewMultiRangeDownloaderTest struct { + fastStatBucketTest +} + +func init() { RegisterTestSuite(&NewMultiRangeDownloaderTest{}) } + +func (t *NewMultiRangeDownloaderTest) CallsWrappedAndInvalidatesOnNotFound() { + const name = "some-name" + // Expect: wrapped bucket returns NotFoundError + var wrappedReq *gcs.MultiRangeDownloaderRequest + ExpectCall(t.wrapped, "NewMultiRangeDownloader")(Any(), Any()). + WillOnce(DoAll(SaveArg(1, &wrappedReq), Return(nil, &gcs.NotFoundError{Err: errors.New("not found")}))) + // Expect: cache invalidate is called + ExpectCall(t.cache, "Erase")(name) + + // Call + req := &gcs.MultiRangeDownloaderRequest{Name: name} + mrd, err := t.bucket.NewMultiRangeDownloader(context.TODO(), req) + + AssertEq(nil, mrd) + ExpectThat(err, HasSameTypeAs(&gcs.NotFoundError{})) + AssertEq(name, wrappedReq.Name) +} + +func (t *NewMultiRangeDownloaderTest) CallsWrappedAndDoesNotInvalidateOnSuccess() { + const name = "some-name" + expectedMrd := fake.NewFakeMultiRangeDownloader(&gcs.MinObject{Name: name}, nil) + // Expect: wrapped returns mrd, no error + ExpectCall(t.wrapped, "NewMultiRangeDownloader")(Any(), Any()). + WillOnce(Return(expectedMrd, nil)) + + // Call + req := &gcs.MultiRangeDownloaderRequest{Name: name} + mrd, err := t.bucket.NewMultiRangeDownloader(context.TODO(), req) + + AssertEq(nil, err) + ExpectEq(expectedMrd, mrd) +} + +//////////////////////////////////////////////////////////////////////// +// GetFolder +//////////////////////////////////////////////////////////////////////// + +type GetFolderTest struct { + fastStatBucketTest +} + +func init() { RegisterTestSuite(&GetFolderTest{}) } + +func (t *GetFolderTest) FetchOnlyFromCacheFalse() { + const name = "taco/" + req := &gcs.GetFolderRequest{ + Name: name, + FetchOnlyFromCache: false, + } + folder := &gcs.Folder{Name: name} + ExpectCall(t.cache, "LookUpFolder")(name, Any()). + WillOnce(Return(false, nil)) + ExpectCall(t.wrapped, "GetFolder")(Any(), Any()). + WillOnce(Return(folder, nil)) + ExpectCall(t.cache, "InsertFolder")(Any(), Any()) + + f, err := t.bucket.GetFolder(context.TODO(), req) + + AssertEq(nil, err) + ExpectEq(folder, f) +} + +func (t *GetFolderTest) FetchOnlyFromCacheTrue_CacheHitPositive() { + const name = "taco/" + req := &gcs.GetFolderRequest{ + Name: name, + FetchOnlyFromCache: true, + } + folder := &gcs.Folder{Name: name} + ExpectCall(t.cache, "LookUpFolder")(name, Any()). + WillOnce(Return(true, folder)) + + f, err := t.bucket.GetFolder(context.TODO(), req) + + AssertEq(nil, err) + ExpectEq(folder, f) +} + +func (t *GetFolderTest) FetchOnlyFromCacheTrue_CacheHitNegative() { + const name = "taco/" + req := &gcs.GetFolderRequest{ + Name: name, + FetchOnlyFromCache: true, + } + ExpectCall(t.cache, "LookUpFolder")(name, Any()). + WillOnce(Return(true, nil)) + + _, err := t.bucket.GetFolder(context.TODO(), req) + + ExpectThat(err, HasSameTypeAs(&gcs.NotFoundError{})) +} + +func (t *GetFolderTest) FetchOnlyFromCacheTrue_CacheMiss() { + const name = "taco/" + req := &gcs.GetFolderRequest{ + Name: name, + FetchOnlyFromCache: true, + } + ExpectCall(t.cache, "LookUpFolder")(name, Any()). + WillOnce(Return(false, nil)) + + _, err := t.bucket.GetFolder(context.TODO(), req) + + ExpectThat(err, HasSameTypeAs(&caching.CacheMissError{})) +} diff --git a/internal/storage/caching/integration_test.go b/internal/storage/caching/integration_test.go index c2a070e29e..0d6288e496 100644 --- a/internal/storage/caching/integration_test.go +++ b/internal/storage/caching/integration_test.go @@ -18,13 +18,13 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/lru" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/metadata" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/caching" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/fake" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/caching" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/ogletest" "github.com/jacobsa/timeutil" @@ -59,13 +59,17 @@ func (t *IntegrationTest) SetUp(ti *TestInfo) { const cacheCapacity = 100 lruCache := lru.NewCache(cfg.AverageSizeOfPositiveStatCacheEntry * cacheCapacity) cache := metadata.NewStatCacheBucketView(lruCache, "") - t.wrapped = fake.NewFakeBucket(&t.clock, bucketName, gcs.NonHierarchical) + t.wrapped = fake.NewFakeBucket(&t.clock, bucketName, gcs.BucketType{}) t.bucket = caching.NewFastStatBucket( - ttl, + primaryCacheTTL, cache, &t.clock, - t.wrapped) + t.wrapped, + negativeCacheTTL, + isTypeCacheDeprecated, + isImplicitDir, + ) } func (t *IntegrationTest) stat(name string) (o *gcs.Object, err error) { @@ -101,7 +105,7 @@ func (t *IntegrationTest) CreateInsertsIntoCache() { } func (t *IntegrationTest) StatInsertsIntoCache() { - const name = "taco" + const name = "foo" var err error // Create an object through the back door. @@ -183,7 +187,7 @@ func (t *IntegrationTest) PositiveCacheExpiration() { AssertEq(nil, err) // Advance time. - t.clock.AdvanceTime(ttl + time.Millisecond) + t.clock.AdvanceTime(primaryCacheTTL + time.Millisecond) // StatObject should no longer see it. _, err = t.stat(name) @@ -286,7 +290,7 @@ func (t *IntegrationTest) NegativeCacheExpiration() { AssertEq(nil, err) // Advance time. - t.clock.AdvanceTime(ttl + time.Millisecond) + t.clock.AdvanceTime(negativeCacheTTL + time.Millisecond) // Now StatObject should see it. o, err := t.stat(name) diff --git a/internal/storage/caching/mock_gcscaching/mock_stat_cache.go b/internal/storage/caching/mock_gcscaching/mock_stat_cache.go index b4691755d9..139992cb1c 100644 --- a/internal/storage/caching/mock_gcscaching/mock_stat_cache.go +++ b/internal/storage/caching/mock_gcscaching/mock_stat_cache.go @@ -12,8 +12,8 @@ import ( time "time" unsafe "unsafe" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/metadata" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" oglemock "github.com/jacobsa/oglemock" ) @@ -37,7 +37,7 @@ func (m *mockStatCache) AddNegativeEntryForFolder(p0 string, p1 time.Time) { "AddNegativeEntryForFolder", name, line, - []interface{}{p0, p1}) + []any{p0, p1}) if len(retVals) != 0 { panic(fmt.Sprintf("mockStatCache.AddNegativeEntryforFolder: invalid return values: %v", retVals)) @@ -71,7 +71,7 @@ func (m *mockStatCache) AddNegativeEntry(p0 string, p1 time.Time) { "AddNegativeEntry", file, line, - []interface{}{p0, p1}) + []any{p0, p1}) if len(retVals) != 0 { panic(fmt.Sprintf("mockStatCache.AddNegativeEntry: invalid return values: %v", retVals)) @@ -88,7 +88,7 @@ func (m *mockStatCache) Erase(p0 string) { "Erase", file, line, - []interface{}{p0}) + []any{p0}) if len(retVals) != 0 { panic(fmt.Sprintf("mockStatCache.Erase: invalid return values: %v", retVals)) @@ -105,7 +105,7 @@ func (m *mockStatCache) Insert(p0 *gcs.MinObject, p1 time.Time) { "Insert", file, line, - []interface{}{p0, p1}) + []any{p0, p1}) if len(retVals) != 0 { panic(fmt.Sprintf("mockStatCache.Insert: invalid return values: %v", retVals)) @@ -122,7 +122,7 @@ func (m *mockStatCache) LookUp(p0 string, p1 time.Time) (o0 bool, o1 *gcs.MinObj "LookUp", file, line, - []interface{}{p0, p1}) + []any{p0, p1}) if len(retVals) != 2 { panic(fmt.Sprintf("mockStatCache.LookUp: invalid return values: %v", retVals)) @@ -151,7 +151,7 @@ func (m *mockStatCache) InsertFolder(p0 *gcs.Folder, p1 time.Time) { "InsertFolder", file, line, - []interface{}{p0, p1}) + []any{p0, p1}) if len(retVals) != 0 { panic(fmt.Sprintf("mockStatCache.InsertFolder: invalid return values: %v", retVals)) @@ -168,7 +168,7 @@ func (m *mockStatCache) LookUpFolder(p0 string, p1 time.Time) (o0 bool, o1 *gcs. "LookUpFolder", file, line, - []interface{}{p0, p1}) + []any{p0, p1}) if len(retVals) != 2 { panic(fmt.Sprintf("mockStatCache.LookUpFolder: invalid return values: %v", retVals)) @@ -197,9 +197,26 @@ func (m *mockStatCache) EraseEntriesWithGivenPrefix(p0 string) { "EraseEntriesWithGivenPrefix", file, line, - []interface{}{p0}) + []any{p0}) if len(retVals) != 0 { panic(fmt.Sprintf("mockStatCache.LookUpFolder: invalid return values: %v", retVals)) } } + +func (m *mockStatCache) InsertImplicitDir(p0 string, p1 time.Time) { + // Get a file name and line number for the caller. + _, file, line, _ := runtime.Caller(1) + + // Hand the call off to the controller, which does most of the work. + retVals := m.controller.HandleMethodCall( + m, + "InsertImplicitDir", + file, + line, + []any{p0, p1}) + + if len(retVals) != 0 { + panic(fmt.Sprintf("mockStatCache.InsertImplicitDir: invalid return values: %v", retVals)) + } +} diff --git a/internal/storage/control_client_wrapper.go b/internal/storage/control_client_wrapper.go index 4fc87da24a..daf6c71e27 100644 --- a/internal/storage/control_client_wrapper.go +++ b/internal/storage/control_client_wrapper.go @@ -16,10 +16,16 @@ package storage import ( "context" + "fmt" + "time" control "cloud.google.com/go/storage/control/apiv2" "cloud.google.com/go/storage/control/apiv2/controlpb" "github.com/googleapis/gax-go/v2" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" ) type StorageControlClient interface { @@ -37,3 +43,206 @@ type StorageControlClient interface { CreateFolder(ctx context.Context, req *controlpb.CreateFolderRequest, opts ...gax.CallOption) (*controlpb.Folder, error) } + +// storageControlClientWithBillingProject is a wrapper for an existing +// StorageControlClient object in that it passes +// the billing project in every call made through the base StorageControlClient. +type storageControlClientWithBillingProject struct { + raw StorageControlClient + billingProject string +} + +func (sccwbp *storageControlClientWithBillingProject) contextWithBillingProject(ctx context.Context) context.Context { + return metadata.AppendToOutgoingContext(ctx, "x-goog-user-project", sccwbp.billingProject) +} + +func (sccwbp *storageControlClientWithBillingProject) GetStorageLayout(ctx context.Context, + req *controlpb.GetStorageLayoutRequest, + opts ...gax.CallOption) (*controlpb.StorageLayout, error) { + return sccwbp.raw.GetStorageLayout(sccwbp.contextWithBillingProject(ctx), req, opts...) +} + +func (sccwbp *storageControlClientWithBillingProject) DeleteFolder(ctx context.Context, + req *controlpb.DeleteFolderRequest, + opts ...gax.CallOption) error { + return sccwbp.raw.DeleteFolder(sccwbp.contextWithBillingProject(ctx), req, opts...) +} + +func (sccwbp *storageControlClientWithBillingProject) GetFolder(ctx context.Context, req *controlpb.GetFolderRequest, opts ...gax.CallOption) (*controlpb.Folder, error) { + return sccwbp.raw.GetFolder(sccwbp.contextWithBillingProject(ctx), req, opts...) +} + +func (sccwbp *storageControlClientWithBillingProject) RenameFolder(ctx context.Context, req *controlpb.RenameFolderRequest, opts ...gax.CallOption) (*control.RenameFolderOperation, error) { + // Don't pass billing-project for LROs as it's not supported. + return sccwbp.raw.RenameFolder(ctx, req, opts...) +} + +func (sccwbp *storageControlClientWithBillingProject) CreateFolder(ctx context.Context, req *controlpb.CreateFolderRequest, opts ...gax.CallOption) (*controlpb.Folder, error) { + return sccwbp.raw.CreateFolder(sccwbp.contextWithBillingProject(ctx), req, opts...) +} + +func withBillingProject(controlClient StorageControlClient, billingProject string) StorageControlClient { + if billingProject != "" { + controlClient = &storageControlClientWithBillingProject{raw: controlClient, billingProject: billingProject} + } + return controlClient +} + +// storageControlClientWithRetry is a wrapper for an existing StorageControlClient object +// which implements gcsfuse-level retry logic if any of the control-client call gets stalled or returns a retryable error. +// It makes time-bound attempts to call the underlying StorageControlClient methods, +// retrying on errors that should be retried according to gcsfuse's retry logic. +type storageControlClientWithRetry struct { + raw StorageControlClient + retryConfig *storageutil.RetryConfig + + // Whether or not to enable retries for GetStorageLayout call. + enableRetriesOnStorageLayoutAPI bool + // Whether or not to enable retries for folder APIs. + enableRetriesOnFolderAPIs bool +} + +func (sccwros *storageControlClientWithRetry) GetStorageLayout(ctx context.Context, + req *controlpb.GetStorageLayoutRequest, + opts ...gax.CallOption) (*controlpb.StorageLayout, error) { + if !sccwros.enableRetriesOnStorageLayoutAPI { + return sccwros.raw.GetStorageLayout(ctx, req, opts...) + } + + apiCall := func(attemptCtx context.Context) (*controlpb.StorageLayout, error) { + return sccwros.raw.GetStorageLayout(attemptCtx, req, opts...) + } + + return storageutil.ExecuteWithRetryAtLogLevel(ctx, sccwros.retryConfig, "GetStorageLayout", req.Name, apiCall, logger.LevelInfo) +} + +func (sccwros *storageControlClientWithRetry) DeleteFolder(ctx context.Context, + req *controlpb.DeleteFolderRequest, + opts ...gax.CallOption) error { + if !sccwros.enableRetriesOnFolderAPIs { + return sccwros.raw.DeleteFolder(ctx, req, opts...) + } + + apiCall := func(attemptCtx context.Context) (any, error) { + err := sccwros.raw.DeleteFolder(attemptCtx, req, opts...) + return struct{}{}, err + } + + _, err := storageutil.ExecuteWithRetry(ctx, sccwros.retryConfig, "DeleteFolder", req.Name, apiCall) + return err +} + +func (sccwros *storageControlClientWithRetry) GetFolder(ctx context.Context, + req *controlpb.GetFolderRequest, + opts ...gax.CallOption) (*controlpb.Folder, error) { + if !sccwros.enableRetriesOnFolderAPIs { + return sccwros.raw.GetFolder(ctx, req, opts...) + } + + apiCall := func(attemptCtx context.Context) (*controlpb.Folder, error) { + return sccwros.raw.GetFolder(attemptCtx, req, opts...) + } + + return storageutil.ExecuteWithRetry(ctx, sccwros.retryConfig, "GetFolder", req.Name, apiCall) +} + +func (sccwros *storageControlClientWithRetry) RenameFolder(ctx context.Context, + req *controlpb.RenameFolderRequest, + opts ...gax.CallOption) (*control.RenameFolderOperation, error) { + if !sccwros.enableRetriesOnFolderAPIs { + return sccwros.raw.RenameFolder(ctx, req, opts...) + } + + apiCall := func(attemptCtx context.Context) (*control.RenameFolderOperation, error) { + return sccwros.raw.RenameFolder(attemptCtx, req, opts...) + } + + reqDescription := fmt.Sprintf("%q to %q", req.Name, req.DestinationFolderId) + return storageutil.ExecuteWithRetry(ctx, sccwros.retryConfig, "RenameFolder", reqDescription, apiCall) +} + +func (sccwros *storageControlClientWithRetry) CreateFolder(ctx context.Context, + req *controlpb.CreateFolderRequest, + opts ...gax.CallOption) (*controlpb.Folder, error) { + if !sccwros.enableRetriesOnFolderAPIs { + return sccwros.raw.CreateFolder(ctx, req, opts...) + } + + apiCall := func(attemptCtx context.Context) (*controlpb.Folder, error) { + return sccwros.raw.CreateFolder(attemptCtx, req, opts...) + } + + reqDescription := fmt.Sprintf("%q in %q", req.FolderId, req.Parent) + return storageutil.ExecuteWithRetry(ctx, sccwros.retryConfig, "CreateFolder", reqDescription, apiCall) +} + +// newRetryWrapper creates a new StorageControlClient with retry capabilities. +// It accepts various parameters to configure the retry behavior. +// The returned control client retries storage-layout. +// It also retries folder-related APIs if `retryFolderAPIs` is true. +func newRetryWrapper(controlClient StorageControlClient, clientConfig *storageutil.StorageClientConfig, retryDeadline, totalRetryBudget, initialBackoff time.Duration, retryFolderAPIs bool) StorageControlClient { + // Avoid creating a nested wrapper. + raw := controlClient + if sccwros, ok := controlClient.(*storageControlClientWithRetry); ok { + raw = sccwros.raw + } + + retryConfig := storageutil.NewRetryConfig(clientConfig, retryDeadline, totalRetryBudget, initialBackoff) + return &storageControlClientWithRetry{ + raw: raw, + retryConfig: retryConfig, + enableRetriesOnStorageLayoutAPI: true, + enableRetriesOnFolderAPIs: retryFolderAPIs, + } +} + +// withRetryOnAllAPIs wraps a StorageControlClient to do a time-bound retry approach for retryable errors for all API calls through it. +func withRetryOnAllAPIs(controlClient StorageControlClient, + clientConfig *storageutil.StorageClientConfig) StorageControlClient { + return newRetryWrapper(controlClient, clientConfig, storageutil.DefaultRetryDeadline, storageutil.DefaultTotalRetryBudget, storageutil.DefaultInitialBackoff, true) +} + +// withRetryOnStorageLayout wraps a StorageControlClient to do a time-bound retry approach for retryable errors for the GetStorageLayout call through it. +func withRetryOnStorageLayout(controlClient StorageControlClient, + clientConfig *storageutil.StorageClientConfig) StorageControlClient { + return newRetryWrapper(controlClient, clientConfig, storageutil.DefaultRetryDeadline, storageutil.DefaultTotalRetryBudget, storageutil.DefaultInitialBackoff, false) + +} + +func storageControlClientGaxRetryOptions(clientConfig *storageutil.StorageClientConfig) []gax.CallOption { + return []gax.CallOption{ + gax.WithTimeout(storageutil.DefaultTotalRetryBudget), + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.ResourceExhausted, + codes.Unavailable, + codes.DeadlineExceeded, + codes.Internal, + codes.Unknown, + }, gax.Backoff{ + Max: clientConfig.MaxRetrySleep, + Multiplier: clientConfig.RetryMultiplier, + }) + }), + } +} + +// addGaxRetriesForFolderAPIs updates the passed raw control client +// to add gax retries according to the given config in-place. +func addGaxRetriesForFolderAPIs(rawControlClient *control.StorageControlClient, + clientConfig *storageutil.StorageClientConfig) error { + if rawControlClient == nil || clientConfig == nil { + return fmt.Errorf("invalid input: %v, %v", rawControlClient, clientConfig) + } + if rawControlClient.CallOptions == nil { + return fmt.Errorf("cannot apply gax retries for folder APIs to raw control client: CallOptions is nil") + } + + *rawControlClient.CallOptions = control.StorageControlCallOptions{} + gaxRetryOptions := storageControlClientGaxRetryOptions(clientConfig) + rawControlClient.CallOptions.RenameFolder = gaxRetryOptions + rawControlClient.CallOptions.GetFolder = gaxRetryOptions + rawControlClient.CallOptions.CreateFolder = gaxRetryOptions + rawControlClient.CallOptions.DeleteFolder = gaxRetryOptions + return nil +} diff --git a/internal/storage/control_client_wrapper_test.go b/internal/storage/control_client_wrapper_test.go new file mode 100644 index 0000000000..c618bff109 --- /dev/null +++ b/internal/storage/control_client_wrapper_test.go @@ -0,0 +1,774 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + "testing" + "time" + + "cloud.google.com/go/storage/control/apiv2/controlpb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + control "cloud.google.com/go/storage/control/apiv2" + "github.com/googleapis/gax-go/v2" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" +) + +// stallingStorageControlClient is a wrapper that introduces a controllable delay +// to every call, to simulate network latency for testing timeout-based retries. +type stallingStorageControlClient struct { + wrapped StorageControlClient + stallDurationForGetStorageLayout *time.Duration + stallDurationForFolderAPIs *time.Duration +} + +func (s *stallingStorageControlClient) GetStorageLayout(ctx context.Context, req *controlpb.GetStorageLayoutRequest, opts ...gax.CallOption) (*controlpb.StorageLayout, error) { + if s.stallDurationForGetStorageLayout != nil { + select { + case <-time.After(*s.stallDurationForGetStorageLayout): + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return s.wrapped.GetStorageLayout(ctx, req, opts...) +} + +func (s *stallingStorageControlClient) DeleteFolder(ctx context.Context, req *controlpb.DeleteFolderRequest, opts ...gax.CallOption) error { + if s.stallDurationForFolderAPIs != nil { + select { + case <-time.After(*s.stallDurationForFolderAPIs): + case <-ctx.Done(): + return ctx.Err() + } + } + return s.wrapped.DeleteFolder(ctx, req, opts...) +} + +func (s *stallingStorageControlClient) GetFolder(ctx context.Context, req *controlpb.GetFolderRequest, opts ...gax.CallOption) (*controlpb.Folder, error) { + if s.stallDurationForFolderAPIs != nil { + select { + case <-time.After(*s.stallDurationForFolderAPIs): + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return s.wrapped.GetFolder(ctx, req, opts...) +} + +func (s *stallingStorageControlClient) RenameFolder(ctx context.Context, req *controlpb.RenameFolderRequest, opts ...gax.CallOption) (*control.RenameFolderOperation, error) { + if s.stallDurationForFolderAPIs != nil { + select { + case <-time.After(*s.stallDurationForFolderAPIs): + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return s.wrapped.RenameFolder(ctx, req, opts...) +} + +func (s *stallingStorageControlClient) CreateFolder(ctx context.Context, req *controlpb.CreateFolderRequest, opts ...gax.CallOption) (*controlpb.Folder, error) { + if s.stallDurationForFolderAPIs != nil { + select { + case <-time.After(*s.stallDurationForFolderAPIs): + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return s.wrapped.CreateFolder(ctx, req, opts...) +} + +type ControlClientRetryWrapperTest struct { + suite.Suite + // The raw mock client for setting expectations on return values. + mockRawClient *MockStorageControlClient + ctx context.Context + stallingClient *stallingStorageControlClient + // The simulated execution time for each GetStorageLayout call made through stallingClient. + stallDurationForGetStorageLayout time.Duration +} + +type StorageLayoutRetryWrapperTest struct { + ControlClientRetryWrapperTest +} + +type AllApiRetryWrapperTest struct { + ControlClientRetryWrapperTest + // The execution time for each folder API call made through stallingClient. Can be adjusted + // per test. + stallDurationForFolderAPIs time.Duration +} + +func TestControlClientWrapperTestSuite(t *testing.T) { + t.Run("StorageLayoutRetryWrapperTest", func(t *testing.T) { + suite.Run(t, new(StorageLayoutRetryWrapperTest)) + }) + t.Run("AllApiRetryWrapperTest", func(t *testing.T) { + suite.Run(t, new(AllApiRetryWrapperTest)) + }) +} + +func (t *ControlClientRetryWrapperTest) SetupTest() { + t.mockRawClient = new(MockStorageControlClient) + t.ctx = context.Background() + t.stallDurationForGetStorageLayout = 0 +} + +func (t *StorageLayoutRetryWrapperTest) SetupTest() { + t.ControlClientRetryWrapperTest.SetupTest() + t.stallingClient = &stallingStorageControlClient{ + wrapped: t.mockRawClient, + stallDurationForGetStorageLayout: &t.stallDurationForGetStorageLayout, + } +} + +func (t *AllApiRetryWrapperTest) SetupTest() { + t.ControlClientRetryWrapperTest.SetupTest() + t.stallDurationForFolderAPIs = 0 + t.stallingClient = &stallingStorageControlClient{ + wrapped: t.mockRawClient, + stallDurationForGetStorageLayout: &t.stallDurationForGetStorageLayout, + stallDurationForFolderAPIs: &t.stallDurationForFolderAPIs, + } +} + +func (t *StorageLayoutRetryWrapperTest) TestGetStorageLayout_SuccessOnFirstAttempt() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, false) + req := &controlpb.GetStorageLayoutRequest{Name: "some/bucket"} + expectedLayout := &controlpb.StorageLayout{Location: "some-location"} + t.mockRawClient.On("GetStorageLayout", mock.Anything, req, mock.Anything).Return(expectedLayout, nil).Once() + + // Act + layout, err := client.GetStorageLayout(t.ctx, req) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), expectedLayout, layout) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *StorageLayoutRetryWrapperTest) TestGetStorageLayout_RetryableErrorThenSuccess() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, false) + req := &controlpb.GetStorageLayoutRequest{Name: "some/bucket"} + expectedLayout := &controlpb.StorageLayout{Location: "some-location"} + retryableErr := status.Error(codes.Unavailable, "try again") + // First call fails, second succeeds. + t.mockRawClient.On("GetStorageLayout", mock.Anything, req, mock.Anything).Return(nil, retryableErr).Once() + t.mockRawClient.On("GetStorageLayout", mock.Anything, req, mock.Anything).Return(expectedLayout, nil).Once() + + // Act + layout, err := client.GetStorageLayout(t.ctx, req) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), expectedLayout, layout) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *StorageLayoutRetryWrapperTest) TestGetStorageLayout_NonRetryableError() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, false) + req := &controlpb.GetStorageLayoutRequest{Name: "some/bucket"} + nonRetryableErr := status.Error(codes.NotFound, "does not exist") + t.mockRawClient.On("GetStorageLayout", mock.Anything, req, mock.Anything).Return(nil, nonRetryableErr).Once() + + // Act + layout, err := client.GetStorageLayout(t.ctx, req) + + // Assert + assert.Error(t.T(), err) + assert.Nil(t.T(), layout) + assert.Contains(t.T(), err.Error(), "failed with a non-retryable error") + assert.Contains(t.T(), err.Error(), nonRetryableErr.Error()) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *StorageLayoutRetryWrapperTest) TestGetStorageLayout_AllAttemptsTimeOut() { + // Arrange + // This test requires different retry parameters, so we create a new client. + client := t.newHelperRetryWrapper(t.stallingClient, 1000*time.Microsecond, 10000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, false) + req := &controlpb.GetStorageLayoutRequest{Name: "some/bucket"} + // Set stall time to be longer than the attempt timeout. + t.stallDurationForGetStorageLayout = 6000 * time.Microsecond + + // Act + _, err := client.GetStorageLayout(t.ctx, req) + + // The mock should never be called because every attempt will time out. + assert.ErrorIs(t.T(), err, context.DeadlineExceeded) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *StorageLayoutRetryWrapperTest) TestGetFolder_IsNotRetried() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, false) + req := &controlpb.GetFolderRequest{Name: "some/folder"} + retryableErr := status.Error(codes.Unavailable, "try again") + // Mock the raw client to return a retryable error once. + t.mockRawClient.On("GetFolder", mock.Anything, req, mock.Anything).Return(nil, retryableErr).Once() + + // Act + folder, err := client.GetFolder(t.ctx, req) + + // Assert + assert.Error(t.T(), err) + assert.Nil(t.T(), folder) + assert.Equal(t.T(), retryableErr, err) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *StorageLayoutRetryWrapperTest) TestDeleteFolder_IsNotRetried() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, false) + req := &controlpb.DeleteFolderRequest{Name: "some/folder"} + retryableErr := status.Error(codes.Unavailable, "try again") + // Mock the raw client to return a retryable error once. + t.mockRawClient.On("DeleteFolder", mock.Anything, req, mock.Anything).Return(retryableErr).Once() + + // Act + err := client.DeleteFolder(t.ctx, req) + + // Assert + assert.Error(t.T(), err) + assert.Equal(t.T(), retryableErr, err) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *StorageLayoutRetryWrapperTest) TestCreateFolder_IsNotRetried() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, false) + req := &controlpb.CreateFolderRequest{Parent: "some/", FolderId: "folder"} + retryableErr := status.Error(codes.Unavailable, "try again") + // Mock the raw client to return a retryable error once. + t.mockRawClient.On("CreateFolder", mock.Anything, req, mock.Anything).Return(nil, retryableErr).Once() + + // Act + folder, err := client.CreateFolder(t.ctx, req) + + // Assert + assert.Error(t.T(), err) + assert.Nil(t.T(), folder) + assert.Equal(t.T(), retryableErr, err) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *StorageLayoutRetryWrapperTest) TestRenameFolder_IsNotRetried() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, false) + req := &controlpb.RenameFolderRequest{Name: "some/folder", DestinationFolderId: "new/folder"} + retryableErr := status.Error(codes.Unavailable, "try again") + // Mock the raw client to return a retryable error once. + t.mockRawClient.On("RenameFolder", mock.Anything, req, mock.Anything).Return(nil, retryableErr).Once() + + // Act + op, err := client.RenameFolder(t.ctx, req) + + // Assert + assert.Error(t.T(), err) + assert.Nil(t.T(), op) + assert.Equal(t.T(), retryableErr, err) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *ControlClientRetryWrapperTest) newHelperRetryWrapper(controlClient StorageControlClient, retryDeadline, totalRetryBudget, initialBackoff, maxRetrySleep time.Duration, backoffMultiplier float64, retryFolderAPIs bool) StorageControlClient { + t.T().Helper() + clientConfig := &storageutil.StorageClientConfig{ + MaxRetrySleep: maxRetrySleep, + RetryMultiplier: backoffMultiplier, + } + return newRetryWrapper(t.stallingClient, clientConfig, retryDeadline, totalRetryBudget, initialBackoff, retryFolderAPIs) +} + +func (t *AllApiRetryWrapperTest) TestGetStorageLayout_SuccessOnFirstAttempt() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, true) + req := &controlpb.GetStorageLayoutRequest{Name: "some/bucket"} + expectedLayout := &controlpb.StorageLayout{Location: "some-location"} + t.mockRawClient.On("GetStorageLayout", mock.Anything, req, mock.Anything).Return(expectedLayout, nil).Once() + + // Act + layout, err := client.GetStorageLayout(t.ctx, req) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), expectedLayout, layout) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *AllApiRetryWrapperTest) TestGetStorageLayout_RetryableErrorThenSuccess() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, true) + req := &controlpb.GetStorageLayoutRequest{Name: "some/bucket"} + expectedLayout := &controlpb.StorageLayout{Location: "some-location"} + retryableErr := status.Error(codes.Unavailable, "try again") + // First call fails, second succeeds. + t.mockRawClient.On("GetStorageLayout", mock.Anything, req, mock.Anything).Return(nil, retryableErr).Once() + t.mockRawClient.On("GetStorageLayout", mock.Anything, req, mock.Anything).Return(expectedLayout, nil).Once() + + // Act + layout, err := client.GetStorageLayout(t.ctx, req) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), expectedLayout, layout) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *AllApiRetryWrapperTest) TestGetStorageLayout_NonRetryableError() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, true) + req := &controlpb.GetStorageLayoutRequest{Name: "some/bucket"} + nonRetryableErr := status.Error(codes.NotFound, "does not exist") + t.mockRawClient.On("GetStorageLayout", mock.Anything, req, mock.Anything).Return(nil, nonRetryableErr).Once() + + // Act + layout, err := client.GetStorageLayout(t.ctx, req) + + // Assert + assert.Error(t.T(), err) + assert.Nil(t.T(), layout) + assert.Contains(t.T(), err.Error(), "failed with a non-retryable error") + assert.Contains(t.T(), err.Error(), nonRetryableErr.Error()) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *AllApiRetryWrapperTest) TestGetStorageLayout_AllAttemptsTimeOut() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 1000*time.Microsecond, 10000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, true) + req := &controlpb.GetStorageLayoutRequest{Name: "some/bucket"} + // Set stall time to be longer than the attempt timeout. + t.stallDurationForGetStorageLayout = 6000 * time.Microsecond + + // Act + _, err := client.GetStorageLayout(t.ctx, req) + + // The mock should never be called because every attempt will time out. + assert.ErrorIs(t.T(), err, context.DeadlineExceeded) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *AllApiRetryWrapperTest) TestDeleteFolder_SuccessOnFirstAttempt() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, true) + req := &controlpb.DeleteFolderRequest{Name: "some/folder"} + t.mockRawClient.On("DeleteFolder", mock.Anything, req, mock.Anything).Return(nil).Once() + + // Act + err := client.DeleteFolder(t.ctx, req) + + // Assert + assert.NoError(t.T(), err) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *AllApiRetryWrapperTest) TestDeleteFolder_RetryableErrorThenSuccess() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, true) + req := &controlpb.DeleteFolderRequest{Name: "some/folder"} + retryableErr := status.Error(codes.Unavailable, "try again") + // First call fails, second succeeds. + t.mockRawClient.On("DeleteFolder", mock.Anything, req, mock.Anything).Return(retryableErr).Once() + t.mockRawClient.On("DeleteFolder", mock.Anything, req, mock.Anything).Return(nil).Once() + + // Act + err := client.DeleteFolder(t.ctx, req) + + // Assert + assert.NoError(t.T(), err) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *AllApiRetryWrapperTest) TestDeleteFolder_NonRetryableError() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, true) + req := &controlpb.DeleteFolderRequest{Name: "some/folder"} + nonRetryableErr := status.Error(codes.NotFound, "does not exist") + t.mockRawClient.On("DeleteFolder", mock.Anything, req, mock.Anything).Return(nonRetryableErr).Once() + + // Act + err := client.DeleteFolder(t.ctx, req) + + // Assert + assert.Error(t.T(), err) + assert.Contains(t.T(), err.Error(), "failed with a non-retryable error") + assert.Contains(t.T(), err.Error(), nonRetryableErr.Error()) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *AllApiRetryWrapperTest) TestDeleteFolder_AllAttemptsTimeOut() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 1000*time.Microsecond, 10000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, true) + req := &controlpb.DeleteFolderRequest{Name: "some/folder"} + // Set stall time to be longer than the attempt timeout. + t.stallDurationForFolderAPIs = 6000 * time.Microsecond + + // Act + err := client.DeleteFolder(t.ctx, req) + + // The mock should never be called because every attempt will time out. + assert.Error(t.T(), err) + assert.ErrorIs(t.T(), err, context.DeadlineExceeded) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *AllApiRetryWrapperTest) TestGetFolder_SuccessOnFirstAttempt() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, true) + req := &controlpb.GetFolderRequest{Name: "some/folder"} + expectedFolder := &controlpb.Folder{Name: "some/folder"} + t.mockRawClient.On("GetFolder", mock.Anything, req, mock.Anything).Return(expectedFolder, nil).Once() + + // Act + folder, err := client.GetFolder(t.ctx, req) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), expectedFolder, folder) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *AllApiRetryWrapperTest) TestGetFolder_RetryableErrorThenSuccess() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, true) + req := &controlpb.GetFolderRequest{Name: "some/folder"} + expectedFolder := &controlpb.Folder{Name: "some/folder"} + retryableErr := status.Error(codes.Unavailable, "try again") + // First call fails, second succeeds. + t.mockRawClient.On("GetFolder", mock.Anything, req, mock.Anything).Return(nil, retryableErr).Once() + t.mockRawClient.On("GetFolder", mock.Anything, req, mock.Anything).Return(expectedFolder, nil).Once() + + // Act + folder, err := client.GetFolder(t.ctx, req) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), expectedFolder, folder) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *AllApiRetryWrapperTest) TestGetFolder_NonRetryableError() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, true) + req := &controlpb.GetFolderRequest{Name: "some/folder"} + nonRetryableErr := status.Error(codes.NotFound, "does not exist") + t.mockRawClient.On("GetFolder", mock.Anything, req, mock.Anything).Return(nil, nonRetryableErr).Once() + + // Act + folder, err := client.GetFolder(t.ctx, req) + + // Assert + assert.Error(t.T(), err) + assert.Nil(t.T(), folder) + assert.Contains(t.T(), err.Error(), "failed with a non-retryable error") + assert.Contains(t.T(), err.Error(), nonRetryableErr.Error()) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *AllApiRetryWrapperTest) TestGetFolder_AllAttemptsTimeOut() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 1000*time.Microsecond, 10000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, true) + req := &controlpb.GetFolderRequest{Name: "some/folder"} + // Set execution time to be longer than the attempt timeout. + t.stallDurationForFolderAPIs = 6000 * time.Microsecond + + // Act + _, err := client.GetFolder(t.ctx, req) + + // Assert: The mock should never be called because every attempt will time out. + assert.Error(t.T(), err) + assert.ErrorIs(t.T(), err, context.DeadlineExceeded) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *AllApiRetryWrapperTest) TestRenameFolder_SuccessOnFirstAttempt() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, true) + req := &controlpb.RenameFolderRequest{Name: "some/folder", DestinationFolderId: "new/folder"} + expectedOp := &control.RenameFolderOperation{} + t.mockRawClient.On("RenameFolder", mock.Anything, req, mock.Anything).Return(expectedOp, nil).Once() + + // Act + op, err := client.RenameFolder(t.ctx, req) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), expectedOp, op) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *AllApiRetryWrapperTest) TestRenameFolder_RetryableErrorThenSuccess() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, true) + req := &controlpb.RenameFolderRequest{Name: "some/folder", DestinationFolderId: "new/folder"} + expectedOp := &control.RenameFolderOperation{} + retryableErr := status.Error(codes.Unavailable, "try again") + // First call fails, second succeeds. + t.mockRawClient.On("RenameFolder", mock.Anything, req, mock.Anything).Return(nil, retryableErr).Once() + t.mockRawClient.On("RenameFolder", mock.Anything, req, mock.Anything).Return(expectedOp, nil).Once() + + // Act + op, err := client.RenameFolder(t.ctx, req) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), expectedOp, op) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *AllApiRetryWrapperTest) TestRenameFolder_NonRetryableError() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, true) + req := &controlpb.RenameFolderRequest{Name: "some/folder", DestinationFolderId: "new/folder"} + nonRetryableErr := status.Error(codes.NotFound, "does not exist") + t.mockRawClient.On("RenameFolder", mock.Anything, req, mock.Anything).Return(nil, nonRetryableErr).Once() + + // Act + op, err := client.RenameFolder(t.ctx, req) + + // Assert + assert.Error(t.T(), err) + assert.Nil(t.T(), op) + assert.Contains(t.T(), err.Error(), "failed with a non-retryable error") + assert.Contains(t.T(), err.Error(), nonRetryableErr.Error()) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *AllApiRetryWrapperTest) TestRenameFolder_AllAttemptsTimeOut() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 1000*time.Microsecond, 10000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, true) + req := &controlpb.RenameFolderRequest{Name: "some/folder", DestinationFolderId: "new/folder"} + // Set execution time to be longer than the attempt timeout. + t.stallDurationForFolderAPIs = 6000 * time.Microsecond + + // Act + _, err := client.RenameFolder(t.ctx, req) + + // Assert: The mock should never be called because every attempt will time out. + assert.Error(t.T(), err) + assert.ErrorIs(t.T(), err, context.DeadlineExceeded) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *AllApiRetryWrapperTest) TestCreateFolder_SuccessOnFirstAttempt() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, true) + req := &controlpb.CreateFolderRequest{Parent: "some/", FolderId: "folder"} + expectedFolder := &controlpb.Folder{Name: "some/folder"} + t.mockRawClient.On("CreateFolder", mock.Anything, req, mock.Anything).Return(expectedFolder, nil).Once() + + // Act + folder, err := client.CreateFolder(t.ctx, req) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), expectedFolder, folder) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *AllApiRetryWrapperTest) TestCreateFolder_RetryableErrorThenSuccess() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, true) + req := &controlpb.CreateFolderRequest{Parent: "some/", FolderId: "folder"} + expectedFolder := &controlpb.Folder{Name: "some/folder"} + retryableErr := status.Error(codes.Unavailable, "try again") + // First call fails, second succeeds. + t.mockRawClient.On("CreateFolder", mock.Anything, req, mock.Anything).Return(nil, retryableErr).Once() + t.mockRawClient.On("CreateFolder", mock.Anything, req, mock.Anything).Return(expectedFolder, nil).Once() + + // Act + folder, err := client.CreateFolder(t.ctx, req) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), expectedFolder, folder) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *AllApiRetryWrapperTest) TestCreateFolder_NonRetryableError() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 100*time.Microsecond, 1000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, true) + req := &controlpb.CreateFolderRequest{Parent: "some/", FolderId: "folder"} + nonRetryableErr := status.Error(codes.NotFound, "does not exist") + t.mockRawClient.On("CreateFolder", mock.Anything, req, mock.Anything).Return(nil, nonRetryableErr).Once() + + // Act + folder, err := client.CreateFolder(t.ctx, req) + + // Assert + assert.Error(t.T(), err) + assert.Nil(t.T(), folder) + assert.Contains(t.T(), err.Error(), "failed with a non-retryable error") + assert.Contains(t.T(), err.Error(), nonRetryableErr.Error()) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (t *AllApiRetryWrapperTest) TestCreateFolder_AllAttemptsTimeOut() { + // Arrange + client := t.newHelperRetryWrapper(t.stallingClient, 1000*time.Microsecond, 10000*time.Microsecond, time.Microsecond, 10*time.Microsecond, 2, true) + req := &controlpb.CreateFolderRequest{Parent: "some/", FolderId: "folder"} + // Set execution time to be longer than the attempt timeout. + t.stallDurationForFolderAPIs = 6000 * time.Microsecond + + // Act + _, err := client.CreateFolder(t.ctx, req) + + // Assert: The mock should never be called because every attempt will time out. + assert.Error(t.T(), err) + assert.ErrorIs(t.T(), err, context.DeadlineExceeded) + t.mockRawClient.AssertExpectations(t.T()) +} + +func (testSuite *StorageLayoutRetryWrapperTest) TestWithRetryOnStorageLayout_WrapsClient() { + // Arrange + mockClient := new(MockStorageControlClient) + clientConfig := storageutil.GetDefaultStorageClientConfig(keyFile) + + // Act + wrappedClient := withRetryOnStorageLayout(mockClient, &clientConfig) + + // Assert + require.NotNil(testSuite.T(), wrappedClient) + retryWrapper, ok := wrappedClient.(*storageControlClientWithRetry) + require.True(testSuite.T(), ok, "The returned client should be of type *storageControlClientWithRetry") + assert.Same(testSuite.T(), mockClient, retryWrapper.raw) + assert.True(testSuite.T(), retryWrapper.enableRetriesOnStorageLayoutAPI, "Retries should be enabled for storage layout APIs") + assert.False(testSuite.T(), retryWrapper.enableRetriesOnFolderAPIs, "Retries should not be enabled for folder APIs") +} + +func (testSuite *StorageLayoutRetryWrapperTest) TestWithRetryOnStorageLayout_UnwrapsNestedRetryClient() { + // Arrange + mockClient := new(MockStorageControlClient) + clientConfig := storageutil.GetDefaultStorageClientConfig(keyFile) + // Create a client that is already wrapped. + alreadyWrappedClient := withRetryOnStorageLayout(mockClient, &clientConfig) + + // Act + // Wrap it again. + doubleWrappedClient := withRetryOnStorageLayout(alreadyWrappedClient, &clientConfig) + + // Assert + require.NotNil(testSuite.T(), doubleWrappedClient) + retryWrapper, ok := doubleWrappedClient.(*storageControlClientWithRetry) + require.True(testSuite.T(), ok, "The returned client should be of type *storageControlClientWithRetry") + assert.Same(testSuite.T(), mockClient, retryWrapper.raw, "Should unwrap the nested retry client") + assert.NotSame(testSuite.T(), alreadyWrappedClient, retryWrapper.raw) + assert.True(testSuite.T(), retryWrapper.enableRetriesOnStorageLayoutAPI, "Retries should be enabled for storage layout APIs") + assert.False(testSuite.T(), retryWrapper.enableRetriesOnFolderAPIs, "Retries should not be enabled for folder APIs") +} + +func (testSuite *AllApiRetryWrapperTest) TestWithRetryOnAllAPIs_WrapsClient() { + // Arrange + mockClient := new(MockStorageControlClient) + clientConfig := storageutil.GetDefaultStorageClientConfig(keyFile) + + // Act + wrappedClient := withRetryOnAllAPIs(mockClient, &clientConfig) + + // Assert + require.NotNil(testSuite.T(), wrappedClient) + retryWrapper, ok := wrappedClient.(*storageControlClientWithRetry) + require.True(testSuite.T(), ok, "The returned client should be of type *storageControlClientWithRetry") + assert.Same(testSuite.T(), mockClient, retryWrapper.raw) + assert.True(testSuite.T(), retryWrapper.enableRetriesOnStorageLayoutAPI, "Retries should be enabled for storage layout APIs") + assert.True(testSuite.T(), retryWrapper.enableRetriesOnFolderAPIs, "Retries should be enabled for folder APIs") +} + +func (testSuite *AllApiRetryWrapperTest) TestWithRetryOnAllAPIs_UnwrapsNestedRetryClient() { + // Arrange + mockClient := new(MockStorageControlClient) + clientConfig := storageutil.GetDefaultStorageClientConfig(keyFile) + // Create a client that is already wrapped. + alreadyWrappedClient := withRetryOnAllAPIs(mockClient, &clientConfig) + + // Act + // Wrap it again. + doubleWrappedClient := withRetryOnAllAPIs(alreadyWrappedClient, &clientConfig) + + // Assert + require.NotNil(testSuite.T(), doubleWrappedClient) + retryWrapper, ok := doubleWrappedClient.(*storageControlClientWithRetry) + require.True(testSuite.T(), ok, "The returned client should be of type *storageControlClientWithRetry") + assert.Same(testSuite.T(), mockClient, retryWrapper.raw, "Should unwrap the nested retry client") + assert.NotSame(testSuite.T(), alreadyWrappedClient, retryWrapper.raw) + assert.True(testSuite.T(), retryWrapper.enableRetriesOnStorageLayoutAPI, "Retries should be enabled for storage layout APIs") + assert.True(testSuite.T(), retryWrapper.enableRetriesOnFolderAPIs, "Retries should be enabled for folder APIs") +} + +type ControlClientGaxRetryWrapperTest struct { + suite.Suite +} + +func TestControlClientGaxRetryWrapperTestSuite(t *testing.T) { + suite.Run(t, new(ControlClientGaxRetryWrapperTest)) +} + +func (testSuite *ControlClientGaxRetryWrapperTest) TestStorageControlClientGaxRetryOptions() { + // Arrange + clientConfig := storageutil.GetDefaultStorageClientConfig(keyFile) + + // Act + gaxOpts := storageControlClientGaxRetryOptions(&clientConfig) + + // Assert + require.NotEmpty(testSuite.T(), gaxOpts) + require.Len(testSuite.T(), gaxOpts, 2) +} + +func (testSuite *ControlClientGaxRetryWrapperTest) TestAddGaxRetriesForFolderAPIs_NilRawControlClient() { + // Arrange + clientConfig := storageutil.GetDefaultStorageClientConfig(keyFile) + + // Act + err := addGaxRetriesForFolderAPIs(nil, &clientConfig) + + // Assert + require.Error(testSuite.T(), err) +} + +func (testSuite *ControlClientGaxRetryWrapperTest) TestAddGaxRetriesForFolderAPIs_NilClientConfig() { + // Arrange + rawClient := &control.StorageControlClient{} + + // Act + err := addGaxRetriesForFolderAPIs(rawClient, nil) + + // Assert + require.Error(testSuite.T(), err) +} + +func (testSuite *ControlClientGaxRetryWrapperTest) TestAddGaxRetriesForFolderAPIs_AppliesGaxOptions() { + // Arrange + rawControlClient := &control.StorageControlClient{CallOptions: &control.StorageControlCallOptions{}} + clientConfig := storageutil.GetDefaultStorageClientConfig(keyFile) + + // Act + err := addGaxRetriesForFolderAPIs(rawControlClient, &clientConfig) + + // Assert + require.NoError(testSuite.T(), err) + require.NotNil(testSuite.T(), rawControlClient.CallOptions) + assert.Empty(testSuite.T(), rawControlClient.CallOptions.GetStorageLayout) // GetStorageLayout should not have GAX retries applied + assert.Len(testSuite.T(), rawControlClient.CallOptions.DeleteFolder, 2) // DeleteFolder should have GAX retries applied + assert.Len(testSuite.T(), rawControlClient.CallOptions.GetFolder, 2) // GetFolder should have GAX retries applied + assert.Len(testSuite.T(), rawControlClient.CallOptions.CreateFolder, 2) // CreateFolder should have GAX retries applied + assert.Len(testSuite.T(), rawControlClient.CallOptions.RenameFolder, 2) // RenameFolder should have GAX retries applied +} diff --git a/internal/storage/debug_bucket.go b/internal/storage/debug_bucket.go index a3404e8767..2bba7ea0b7 100644 --- a/internal/storage/debug_bucket.go +++ b/internal/storage/debug_bucket.go @@ -20,8 +20,9 @@ import ( "sync/atomic" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + storagev2 "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "golang.org/x/net/context" ) @@ -53,13 +54,13 @@ func (b *debugBucket) mintRequestID() (id uint64) { func (b *debugBucket) requestLogf( id uint64, format string, - v ...interface{}) { + v ...any) { logger.Tracef("gcs: Req %#16x: %s", id, fmt.Sprintf(format, v...)) } func (b *debugBucket) startRequest( format string, - v ...interface{}) (id uint64, desc string, start time.Time) { + v ...any) (id uint64, desc string, start time.Time) { start = time.Now() id = b.mintRequestID() desc = fmt.Sprintf(format, v...) @@ -92,7 +93,7 @@ type debugReader struct { requestID uint64 desc string startTime time.Time - wrapped io.ReadCloser + wrapped gcs.StorageReader } func (dr *debugReader) Read(p []byte) (n int, err error) { @@ -117,6 +118,10 @@ func (dr *debugReader) Close() (err error) { return } +func (dr *debugReader) ReadHandle() storagev2.ReadHandle { + return dr.wrapped.ReadHandle() +} + //////////////////////////////////////////////////////////////////////// // Bucket interface //////////////////////////////////////////////////////////////////////// @@ -129,16 +134,14 @@ func (b *debugBucket) BucketType() gcs.BucketType { return b.wrapped.BucketType() } -func (b *debugBucket) NewReader( - ctx context.Context, - req *gcs.ReadObjectRequest) (rc io.ReadCloser, err error) { - id, desc, start := b.startRequest("Read(%q, %v)", req.Name, req.Range) +func setupReader(ctx context.Context, b *debugBucket, req *gcs.ReadObjectRequest, method string) (gcs.StorageReader, error) { + id, desc, start := b.startRequest("%s(%q, %v)", method, req.Name, req.Range) // Call through. - rc, err = b.wrapped.NewReader(ctx, req) + rc, err := b.wrapped.NewReaderWithReadHandle(ctx, req) if err != nil { b.finishRequest(id, desc, start, &err) - return + return rc, err } // Return a special reader that prings debug info. @@ -149,7 +152,13 @@ func (b *debugBucket) NewReader( startTime: start, wrapped: rc, } + return rc, err +} +func (b *debugBucket) NewReaderWithReadHandle( + ctx context.Context, + req *gcs.ReadObjectRequest) (rd gcs.StorageReader, err error) { + rd, err = setupReader(ctx, b, req, "ReadWithReadHandle") return } @@ -158,20 +167,41 @@ func (b *debugBucket) CreateObject( req *gcs.CreateObjectRequest) (o *gcs.Object, err error) { id, desc, start := b.startRequest("CreateObject(%q)", req.Name) defer b.finishRequest(id, desc, start, &err) - - o, err = b.wrapped.CreateObject(context.WithValue(ctx, gcs.ReqIdField, id), req) + if req.CallBack == nil { + req.CallBack = func(bytesUploadedSoFar int64) { + logger.Tracef("gcs: Req %#16x: -- UploadBlock(%q): %20v bytes uploaded so far", id, req.Name, bytesUploadedSoFar) + } + } + o, err = b.wrapped.CreateObject(ctx, req) return } func (b *debugBucket) CreateObjectChunkWriter(ctx context.Context, req *gcs.CreateObjectRequest, chunkSize int, callBack func(bytesUploadedSoFar int64)) (wc gcs.Writer, err error) { id, desc, start := b.startRequest("CreateObjectChunkWriter(%q)", req.Name) defer b.finishRequest(id, desc, start, &err) + if callBack == nil { + callBack = func(bytesUploadedSoFar int64) { + logger.Tracef("gcs: Req %#16x: -- UploadBlock(%q): %20v bytes uploaded so far", id, req.Name, bytesUploadedSoFar) + } + } + wc, err = b.wrapped.CreateObjectChunkWriter(ctx, req, chunkSize, callBack) + return +} - wc, err = b.wrapped.CreateObjectChunkWriter(context.WithValue(ctx, gcs.ReqIdField, id), req, chunkSize, callBack) +func (b *debugBucket) CreateAppendableObjectWriter(ctx context.Context, + req *gcs.CreateObjectChunkWriterRequest) (wc gcs.Writer, err error) { + id, desc, start := b.startRequest("CreateAppendableObjectWriter(%q, %d)", req.Name, req.Offset) + defer b.finishRequest(id, desc, start, &err) + if req.CallBack == nil { + req.CallBack = func(bytesUploadedSoFar int64) { + logger.Tracef("gcs: Req %#16x: -- UploadBlock(%q): %20v bytes uploaded so far", id, req.Name, bytesUploadedSoFar) + } + } + wc, err = b.wrapped.CreateAppendableObjectWriter(ctx, req) return } -func (b *debugBucket) FinalizeUpload(ctx context.Context, w gcs.Writer) (o *gcs.Object, err error) { +func (b *debugBucket) FinalizeUpload(ctx context.Context, w gcs.Writer) (o *gcs.MinObject, err error) { id, desc, start := b.startRequest("FinalizeUpload(%q)", w.ObjectName()) defer b.finishRequest(id, desc, start, &err) @@ -179,6 +209,14 @@ func (b *debugBucket) FinalizeUpload(ctx context.Context, w gcs.Writer) (o *gcs. return } +func (b *debugBucket) FlushPendingWrites(ctx context.Context, w gcs.Writer) (o *gcs.MinObject, err error) { + id, desc, start := b.startRequest("FlushPendingWrites(%q)", w.ObjectName()) + defer b.finishRequest(id, desc, start, &err) + + o, err = b.wrapped.FlushPendingWrites(ctx, w) + return +} + func (b *debugBucket) CopyObject( ctx context.Context, req *gcs.CopyObjectRequest) (o *gcs.Object, err error) { @@ -219,7 +257,7 @@ func (b *debugBucket) StatObject( func (b *debugBucket) ListObjects( ctx context.Context, req *gcs.ListObjectsRequest) (listing *gcs.Listing, err error) { - id, desc, start := b.startRequest("ListObjects(%q)", req.Prefix) + id, desc, start := b.startRequest("ListObjects(Prefix=%q, StartOffset=%q, MaxResults=%d)", req.Prefix, req.StartOffset, req.MaxResults) defer b.finishRequest(id, desc, start, &err) listing, err = b.wrapped.ListObjects(ctx, req) @@ -246,6 +284,17 @@ func (b *debugBucket) DeleteObject( return } +func (b *debugBucket) MoveObject(ctx context.Context, req *gcs.MoveObjectRequest) (*gcs.Object, error) { + var err error + var o *gcs.Object + id, desc, start := b.startRequest("MoveObject(%q, %q)", req.SrcName, req.DstName) + + defer b.finishRequest(id, desc, start, &err) + + o, err = b.wrapped.MoveObject(ctx, req) + return o, err +} + func (b *debugBucket) DeleteFolder(ctx context.Context, folderName string) (err error) { id, desc, start := b.startRequest("DeleteFolder(%q)", folderName) defer b.finishRequest(id, desc, start, &err) @@ -254,11 +303,11 @@ func (b *debugBucket) DeleteFolder(ctx context.Context, folderName string) (err return err } -func (b *debugBucket) GetFolder(ctx context.Context, folderName string) (folder *gcs.Folder, err error) { - id, desc, start := b.startRequest("GetFolder(%q)", folderName) +func (b *debugBucket) GetFolder(ctx context.Context, req *gcs.GetFolderRequest) (folder *gcs.Folder, err error) { + id, desc, start := b.startRequest("GetFolder(%q)", req.Name) defer b.finishRequest(id, desc, start, &err) - folder, err = b.wrapped.GetFolder(ctx, folderName) + folder, err = b.wrapped.GetFolder(ctx, req) return } @@ -271,9 +320,82 @@ func (b *debugBucket) CreateFolder(ctx context.Context, folderName string) (fold } func (b *debugBucket) RenameFolder(ctx context.Context, folderName string, destinationFolderId string) (o *gcs.Folder, err error) { - id, desc, start := b.startRequest("RenameFolder(%q)", folderName) + id, desc, start := b.startRequest("RenameFolder(%q, %q)", folderName, destinationFolderId) defer b.finishRequest(id, desc, start, &err) o, err = b.wrapped.RenameFolder(ctx, folderName, destinationFolderId) return o, err } + +type debugMultiRangeDownloader struct { + object string + bucket *debugBucket + requestID uint64 + desc string + startTime time.Time + wrapped gcs.MultiRangeDownloader +} + +func (dmrd *debugMultiRangeDownloader) Add(output io.Writer, offset, length int64, callback func(int64, int64, error)) { + id, desc, start := dmrd.bucket.startRequest("MultiRangeDownloader.Add(%s, [%v,%v))", dmrd.object, offset, offset+length) + wrapperCallback := func(offset int64, length int64, err error) { + defer dmrd.bucket.finishRequest(id, desc, start, &err) + if callback != nil { + callback(offset, length, err) + } + } + dmrd.wrapped.Add(output, offset, length, wrapperCallback) +} + +func (dmrd *debugMultiRangeDownloader) Close() (err error) { + id, desc, start := dmrd.bucket.startRequest("MultiRangeDownloader.Close()") + defer dmrd.bucket.finishRequest(id, desc, start, &err) + err = dmrd.wrapped.Close() + return +} + +func (dmrd *debugMultiRangeDownloader) Wait() { + id, desc, start := dmrd.bucket.startRequest("MultiRangeDownloader.Wait()") + var err error + defer dmrd.bucket.finishRequest(id, desc, start, &err) + dmrd.wrapped.Wait() +} + +func (dmrd *debugMultiRangeDownloader) Error() (err error) { + err = dmrd.wrapped.Error() + return +} + +func (dmrd *debugMultiRangeDownloader) GetHandle() []byte { + id, desc, start := dmrd.bucket.startRequest("MultiRangeDownloader.GetHandle()") + var err error + defer dmrd.bucket.finishRequest(id, desc, start, &err) + return dmrd.wrapped.GetHandle() +} + +func (b *debugBucket) NewMultiRangeDownloader( + ctx context.Context, req *gcs.MultiRangeDownloaderRequest) (mrd gcs.MultiRangeDownloader, err error) { + id, desc, start := b.startRequest("NewMultiRangeDownloader(%q)", req.Name) + defer b.finishRequest(id, desc, start, &err) + + // Call through. + mrd, err = b.wrapped.NewMultiRangeDownloader(ctx, req) + if err != nil { + return + } + + // Return a special reader that prints debug info. + mrd = &debugMultiRangeDownloader{ + object: req.Name, + bucket: b, + requestID: id, + desc: desc, + startTime: start, + wrapped: mrd, + } + return +} + +func (b *debugBucket) GCSName(obj *gcs.MinObject) string { + return obj.Name +} diff --git a/internal/storage/dummy_io_bucket.go b/internal/storage/dummy_io_bucket.go new file mode 100644 index 0000000000..035118747e --- /dev/null +++ b/internal/storage/dummy_io_bucket.go @@ -0,0 +1,350 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + "errors" + "io" + "sync" + "time" + + storagev2 "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" +) + +const MB = 1024 * 1024 + +type DummyIOBucketParams struct { + ReaderLatency time.Duration + PerMBLatency time.Duration +} + +// dummyIOBucket is a wrapper over gcs.Bucket that implements gcs.Bucket interface. +// It directly delegates all calls to the wrapped bucket, and performs dummy IO for +// read and write operations. +type dummyIOBucket struct { + wrapped gcs.Bucket + readerLatency time.Duration + perMBLatency time.Duration +} + +// NewDummyIOBucket creates a new dummyIOBucket wrapping the given gcs.Bucket. +// If the wrapped bucket is nil, it returns nil. +func NewDummyIOBucket(wrapped gcs.Bucket, params DummyIOBucketParams) gcs.Bucket { + if wrapped == nil { + return nil + } + + return &dummyIOBucket{ + wrapped: wrapped, + readerLatency: params.ReaderLatency, + perMBLatency: params.PerMBLatency, + } +} + +// Name returns the name of the bucket. +func (d *dummyIOBucket) Name() string { + return d.wrapped.Name() +} + +// BucketType returns the type of the bucket. +func (d *dummyIOBucket) BucketType() gcs.BucketType { + return d.wrapped.BucketType() +} + +// NewReaderWithReadHandle creates a reader for reading object contents. +// Returns a dummy reader that serves zeros efficiently instead of reading from GCS. +func (d *dummyIOBucket) NewReaderWithReadHandle( + ctx context.Context, + req *gcs.ReadObjectRequest) (gcs.StorageReader, error) { + + if req.Range == nil { + return nil, errors.New("range must be specified for dummy IO bucket") + } + + rangeLen := int64(req.Range.Limit) - int64(req.Range.Start) + if rangeLen <= 0 { + return nil, errors.New("invalid range: limit is less than start") + } + + // Simulate network latency if specified. + if d.readerLatency > 0 { + time.Sleep(d.readerLatency) + } + + return newDummyReader(uint64(rangeLen), d.perMBLatency), nil +} + +// NewMultiRangeDownloader creates a multi-range downloader for object contents. +func (d *dummyIOBucket) NewMultiRangeDownloader( + ctx context.Context, + req *gcs.MultiRangeDownloaderRequest) (gcs.MultiRangeDownloader, error) { + return &dummyMultiRangeDownloader{perMBLatency: d.perMBLatency}, nil +} + +// CreateObject creates or overwrites an object. +// TODO: Add custom logic for Write path if needed +func (d *dummyIOBucket) CreateObject( + ctx context.Context, + req *gcs.CreateObjectRequest) (*gcs.Object, error) { + return d.wrapped.CreateObject(ctx, req) +} + +// CreateObjectChunkWriter creates a writer for resumable uploads. +// TODO: Add custom logic for Write path if needed +func (d *dummyIOBucket) CreateObjectChunkWriter( + ctx context.Context, + req *gcs.CreateObjectRequest, + chunkSize int, + callBack func(bytesUploadedSoFar int64)) (gcs.Writer, error) { + return d.wrapped.CreateObjectChunkWriter(ctx, req, chunkSize, callBack) +} + +// CreateAppendableObjectWriter creates a writer to append to an existing object. +// TODO: Add custom logic for Write path if needed +func (d *dummyIOBucket) CreateAppendableObjectWriter( + ctx context.Context, + req *gcs.CreateObjectChunkWriterRequest) (gcs.Writer, error) { + return d.wrapped.CreateAppendableObjectWriter(ctx, req) +} + +// FinalizeUpload completes the write operation and creates the object on GCS. +// TODO: Add custom logic for Write path if needed +func (d *dummyIOBucket) FinalizeUpload( + ctx context.Context, + writer gcs.Writer) (*gcs.MinObject, error) { + return d.wrapped.FinalizeUpload(ctx, writer) +} + +// FlushPendingWrites flushes pending data in the writer buffer for zonal buckets. +// TODO: Add custom logic for Write path if needed +func (d *dummyIOBucket) FlushPendingWrites( + ctx context.Context, + writer gcs.Writer) (*gcs.MinObject, error) { + return d.wrapped.FlushPendingWrites(ctx, writer) +} + +// CopyObject copies an object to a new name. +// Directly delegates to wrapped bucket. +func (d *dummyIOBucket) CopyObject( + ctx context.Context, + req *gcs.CopyObjectRequest) (*gcs.Object, error) { + return d.wrapped.CopyObject(ctx, req) +} + +// ComposeObjects composes one or more source objects into a single destination object. +// Directly delegates to wrapped bucket. +func (d *dummyIOBucket) ComposeObjects( + ctx context.Context, + req *gcs.ComposeObjectsRequest) (*gcs.Object, error) { + return d.wrapped.ComposeObjects(ctx, req) +} + +// StatObject returns current information about the object. +// Directly delegates to wrapped bucket. +func (d *dummyIOBucket) StatObject( + ctx context.Context, + req *gcs.StatObjectRequest) (*gcs.MinObject, *gcs.ExtendedObjectAttributes, error) { + return d.wrapped.StatObject(ctx, req) +} + +// ListObjects lists the objects in the bucket that meet the criteria. +// Directly delegates to wrapped bucket. +func (d *dummyIOBucket) ListObjects( + ctx context.Context, + req *gcs.ListObjectsRequest) (*gcs.Listing, error) { + return d.wrapped.ListObjects(ctx, req) +} + +// UpdateObject updates the object specified by request. +// Directly delegates to wrapped bucket. +func (d *dummyIOBucket) UpdateObject( + ctx context.Context, + req *gcs.UpdateObjectRequest) (*gcs.Object, error) { + return d.wrapped.UpdateObject(ctx, req) +} + +// DeleteObject deletes an object. +// Directly delegates to wrapped bucket. +func (d *dummyIOBucket) DeleteObject( + ctx context.Context, + req *gcs.DeleteObjectRequest) error { + return d.wrapped.DeleteObject(ctx, req) +} + +// MoveObject moves an object to a new name. +// Directly delegates to wrapped bucket. +func (d *dummyIOBucket) MoveObject( + ctx context.Context, + req *gcs.MoveObjectRequest) (*gcs.Object, error) { + return d.wrapped.MoveObject(ctx, req) +} + +// DeleteFolder deletes a folder. +// Directly delegates to wrapped bucket. +func (d *dummyIOBucket) DeleteFolder(ctx context.Context, folderName string) error { + return d.wrapped.DeleteFolder(ctx, folderName) +} + +// GetFolder retrieves folder information. +// Directly delegates to wrapped bucket. +func (d *dummyIOBucket) GetFolder(ctx context.Context, req *gcs.GetFolderRequest) (*gcs.Folder, error) { + return d.wrapped.GetFolder(ctx, req) +} + +// RenameFolder atomically renames a folder for Hierarchical bucket. +// Directly delegates to wrapped bucket. +func (d *dummyIOBucket) RenameFolder( + ctx context.Context, + folderName string, + destinationFolderId string) (*gcs.Folder, error) { + return d.wrapped.RenameFolder(ctx, folderName, destinationFolderId) +} + +// CreateFolder creates a new folder. +// Directly delegates to wrapped bucket. +func (d *dummyIOBucket) CreateFolder(ctx context.Context, folderName string) (*gcs.Folder, error) { + return d.wrapped.CreateFolder(ctx, folderName) +} + +// GCSName returns the original GCS name for the object. +// Directly delegates to wrapped bucket. +func (d *dummyIOBucket) GCSName(object *gcs.MinObject) string { + return d.wrapped.GCSName(object) +} + +//////////////////////////////////////////////////////////////////////// +// dummyReader +//////////////////////////////////////////////////////////////////////// + +// dummyReader is an efficient reader that serves dummy data. +// Reading beyond the specified length returns io.EOF. +// Also, it always returns a non-nil read handle. +type dummyReader struct { + totalLen uint64 // Total length of data to serve + bytesRead uint64 // Number of bytes already read + readHandle storagev2.ReadHandle + perMBLatency time.Duration +} + +func calculateLatency(bytes int64, perMBLatency time.Duration) time.Duration { + if perMBLatency <= 0 { + return 0 + } + return time.Duration(float64(bytes) * float64(perMBLatency.Nanoseconds()) / float64(MB)) +} + +// newDummyReader creates a new dummyReader with the specified total length. +func newDummyReader(totalLen uint64, perMBLatency time.Duration) *dummyReader { + return &dummyReader{ + totalLen: totalLen, + bytesRead: 0, + readHandle: []byte{}, // Always return a non-nil read handle + perMBLatency: perMBLatency, + } +} + +// Read reads up to len(p) bytes into p, filling it with zeros. +// Returns io.EOF when the total length has been reached. +func (dr *dummyReader) Read(p []byte) (n int, err error) { + // If we've already read all the data, return EOF + if dr.bytesRead >= dr.totalLen { + return 0, io.EOF + } + + // Calculate how many bytes we can still read + remaining := dr.totalLen - dr.bytesRead + + // Determine how many bytes to read in this call + toRead := uint64(len(p)) + if toRead > remaining { + toRead = remaining + } + + // Simulate per-MB latency if specified + time.Sleep(calculateLatency(int64(toRead), dr.perMBLatency)) + + dr.bytesRead += toRead + + // If we've read all the data, return EOF along with the last bytes + if dr.bytesRead >= dr.totalLen { + return int(toRead), io.EOF + } + + return int(toRead), nil +} + +// Close closes the reader. For dummy reader, this is a no-op. +func (dr *dummyReader) Close() error { + return nil +} + +// ReadHandle returns the read handle. For dummy reader, this returns a nil handle. +func (dr *dummyReader) ReadHandle() storagev2.ReadHandle { + return dr.readHandle +} + +//////////////////////////////////////////////////////////////////////// +// dummyMultiRangeDownloader +//////////////////////////////////////////////////////////////////////// + +type dummyMultiRangeDownloader struct { + perMBLatency time.Duration + wg sync.WaitGroup +} + +// zeroReader is an io.Reader that always reads zeros. +type zeroReader struct{} + +func (zeroReader) Read(p []byte) (int, error) { + clear(p) + return len(p), nil +} + +func (d *dummyMultiRangeDownloader) Add(output io.Writer, offset, length int64, callback func(int64, int64, error)) { + d.wg.Add(1) + go func() { + defer d.wg.Done() + + // Simulate latency + time.Sleep(calculateLatency(length, d.perMBLatency)) + + // Write zeros + // output writer is bytes.Buffer which implements io.ReaderFrom interface + bytesWritten, err := io.Copy(output, io.LimitReader(zeroReader{}, length)) + + if callback != nil { + callback(offset, bytesWritten, err) + } + }() +} + +func (d *dummyMultiRangeDownloader) Close() error { + d.Wait() + return nil +} + +func (d *dummyMultiRangeDownloader) Wait() { + d.wg.Wait() +} + +func (d *dummyMultiRangeDownloader) Error() error { + return nil +} + +func (d *dummyMultiRangeDownloader) GetHandle() []byte { + return []byte("dummy-handle") +} diff --git a/internal/storage/dummy_io_bucket_test.go b/internal/storage/dummy_io_bucket_test.go new file mode 100644 index 0000000000..388ce18483 --- /dev/null +++ b/internal/storage/dummy_io_bucket_test.go @@ -0,0 +1,654 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "sync" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewDummyIOBucket(t *testing.T) { + testCases := []struct { + name string + wrapped gcs.Bucket + expected gcs.Bucket + }{ + { + name: "nil_wrapped", + wrapped: nil, + expected: nil, + }, + { + name: "non_nil_wrapped", + wrapped: &TestifyMockBucket{}, + expected: &dummyIOBucket{wrapped: &TestifyMockBucket{}}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := NewDummyIOBucket(tc.wrapped, DummyIOBucketParams{}) + assert.Equal(t, tc.expected, result) + }) + } +} + +func TestDummyIOBucket_Name(t *testing.T) { + mockBucket := &TestifyMockBucket{} + mockBucket.On("Name").Return("test-bucket") + dummyBucket := NewDummyIOBucket(mockBucket, DummyIOBucketParams{}) + + result := dummyBucket.Name() + + assert.Equal(t, "test-bucket", result) + mockBucket.AssertExpectations(t) +} + +func TestDummyIOBucket_BucketType(t *testing.T) { + mockBucket := &TestifyMockBucket{} + expectedType := gcs.BucketType{Hierarchical: false, Zonal: false} + mockBucket.On("BucketType").Return(expectedType) + dummyBucket := NewDummyIOBucket(mockBucket, DummyIOBucketParams{}) + require.NotNil(t, dummyBucket) + + result := dummyBucket.BucketType() + + assert.Equal(t, expectedType, result) + mockBucket.AssertExpectations(t) +} + +func TestDummyIOBucket_DeleteObject(t *testing.T) { + mockBucket := &TestifyMockBucket{} + ctx := context.Background() + req := &gcs.DeleteObjectRequest{Name: "test-object"} + mockBucket.On("DeleteObject", ctx, req).Return(nil) + dummyBucket := NewDummyIOBucket(mockBucket, DummyIOBucketParams{}) + require.NotNil(t, dummyBucket) + + err := dummyBucket.DeleteObject(ctx, req) + + assert.NoError(t, err) + mockBucket.AssertExpectations(t) +} + +func TestDummyIOBucket_DeleteObject_Error(t *testing.T) { + mockBucket := &TestifyMockBucket{} + ctx := context.Background() + req := &gcs.DeleteObjectRequest{Name: "test-object"} + expectedErr := errors.New("delete failed") + mockBucket.On("DeleteObject", ctx, req).Return(expectedErr) + dummyBucket := NewDummyIOBucket(mockBucket, DummyIOBucketParams{}) + require.NotNil(t, dummyBucket) + + err := dummyBucket.DeleteObject(ctx, req) + + assert.Error(t, err) + assert.Equal(t, expectedErr, err) + mockBucket.AssertExpectations(t) +} + +func TestDummyIOBucket_StatObject(t *testing.T) { + mockBucket := &TestifyMockBucket{} + ctx := context.Background() + req := &gcs.StatObjectRequest{Name: "test-object"} + expectedMinObj := &gcs.MinObject{Name: "test-object"} + expectedExtAttrs := &gcs.ExtendedObjectAttributes{} + mockBucket.On("StatObject", ctx, req).Return(expectedMinObj, expectedExtAttrs, nil) + dummyBucket := NewDummyIOBucket(mockBucket, DummyIOBucketParams{}) + require.NotNil(t, dummyBucket) + + minObj, extAttrs, err := dummyBucket.StatObject(ctx, req) + + assert.NoError(t, err) + assert.Equal(t, expectedMinObj, minObj) + assert.Equal(t, expectedExtAttrs, extAttrs) + mockBucket.AssertExpectations(t) +} + +func TestDummyIOBucket_ListObjects(t *testing.T) { + mockBucket := &TestifyMockBucket{} + ctx := context.Background() + req := &gcs.ListObjectsRequest{Prefix: "test-"} + expectedListing := &gcs.Listing{} + mockBucket.On("ListObjects", ctx, req).Return(expectedListing, nil) + dummyBucket := NewDummyIOBucket(mockBucket, DummyIOBucketParams{}) + require.NotNil(t, dummyBucket) + + listing, err := dummyBucket.ListObjects(ctx, req) + + assert.NoError(t, err) + assert.Equal(t, expectedListing, listing) + mockBucket.AssertExpectations(t) +} + +func TestDummyIOBucket_CopyObject(t *testing.T) { + mockBucket := &TestifyMockBucket{} + ctx := context.Background() + req := &gcs.CopyObjectRequest{ + SrcName: "source-object", + DstName: "dest-object", + } + expectedObj := &gcs.Object{Name: "dest-object"} + mockBucket.On("CopyObject", ctx, req).Return(expectedObj, nil) + dummyBucket := NewDummyIOBucket(mockBucket, DummyIOBucketParams{}) + require.NotNil(t, dummyBucket) + + obj, err := dummyBucket.CopyObject(ctx, req) + + assert.NoError(t, err) + assert.Equal(t, expectedObj, obj) + mockBucket.AssertExpectations(t) +} + +func TestDummyIOBucket_DeleteFolder(t *testing.T) { + mockBucket := &TestifyMockBucket{} + ctx := context.Background() + folderName := "test-folder" + mockBucket.On("DeleteFolder", ctx, folderName).Return(nil) + dummyBucket := NewDummyIOBucket(mockBucket, DummyIOBucketParams{}) + require.NotNil(t, dummyBucket) + + err := dummyBucket.DeleteFolder(ctx, folderName) + + assert.NoError(t, err) + mockBucket.AssertExpectations(t) +} + +func TestDummyIOBucket_GetFolder(t *testing.T) { + mockBucket := &TestifyMockBucket{} + ctx := context.Background() + folderName := "test-folder" + expectedFolder := &gcs.Folder{Name: folderName} + mockBucket.On("GetFolder", ctx, &gcs.GetFolderRequest{Name: folderName}).Return(expectedFolder, nil) + dummyBucket := NewDummyIOBucket(mockBucket, DummyIOBucketParams{}) + require.NotNil(t, dummyBucket) + + folder, err := dummyBucket.GetFolder(ctx, &gcs.GetFolderRequest{Name: folderName}) + + assert.NoError(t, err) + assert.Equal(t, expectedFolder, folder) + mockBucket.AssertExpectations(t) +} + +func TestDummyIOBucket_CreateFolder(t *testing.T) { + mockBucket := &TestifyMockBucket{} + ctx := context.Background() + folderName := "new-folder" + expectedFolder := &gcs.Folder{Name: folderName} + mockBucket.On("CreateFolder", ctx, folderName).Return(expectedFolder, nil) + dummyBucket := NewDummyIOBucket(mockBucket, DummyIOBucketParams{}) + require.NotNil(t, dummyBucket) + + folder, err := dummyBucket.CreateFolder(ctx, folderName) + + assert.NoError(t, err) + assert.Equal(t, expectedFolder, folder) + mockBucket.AssertExpectations(t) +} + +func TestDummyIOBucket_GCSName(t *testing.T) { + mockBucket := &TestifyMockBucket{} + obj := &gcs.MinObject{Name: "test-object"} + expectedName := "gcs-name" + mockBucket.On("GCSName", obj).Return(expectedName) + dummyBucket := NewDummyIOBucket(mockBucket, DummyIOBucketParams{}) + require.NotNil(t, dummyBucket) + + name := dummyBucket.GCSName(obj) + + assert.Equal(t, expectedName, name) + mockBucket.AssertExpectations(t) +} + +func TestDummyIOBucket_MoveObject(t *testing.T) { + mockBucket := &TestifyMockBucket{} + ctx := context.Background() + req := &gcs.MoveObjectRequest{ + SrcName: "source-object", + DstName: "dest-object", + } + expectedObj := &gcs.Object{Name: "dest-object"} + mockBucket.On("MoveObject", ctx, req).Return(expectedObj, nil) + dummyBucket := NewDummyIOBucket(mockBucket, DummyIOBucketParams{}) + require.NotNil(t, dummyBucket) + + obj, err := dummyBucket.MoveObject(ctx, req) + + assert.NoError(t, err) + assert.Equal(t, expectedObj, obj) + mockBucket.AssertExpectations(t) +} + +func TestDummyIOBucket_UpdateObject(t *testing.T) { + mockBucket := &TestifyMockBucket{} + ctx := context.Background() + req := &gcs.UpdateObjectRequest{ + Name: "test-object", + } + expectedObj := &gcs.Object{Name: "test-object"} + mockBucket.On("UpdateObject", ctx, req).Return(expectedObj, nil) + dummyBucket := NewDummyIOBucket(mockBucket, DummyIOBucketParams{}) + require.NotNil(t, dummyBucket) + + obj, err := dummyBucket.UpdateObject(ctx, req) + + assert.NoError(t, err) + assert.Equal(t, expectedObj, obj) + mockBucket.AssertExpectations(t) +} + +func TestDummyIOBucket_NewReaderWithReadHandle(t *testing.T) { + req := &gcs.ReadObjectRequest{ + Name: "test-object", + Range: &gcs.ByteRange{ + Start: 0, + Limit: 100, + }, + } + dummyBucket := NewDummyIOBucket(&TestifyMockBucket{}, DummyIOBucketParams{ReaderLatency: 0}) + + reader, err := dummyBucket.NewReaderWithReadHandle(context.Background(), req) + + assert.NoError(t, err) + assert.NotNil(t, reader) + assert.IsType(t, &dummyReader{}, reader) + assert.Equal(t, uint64(100), reader.(*dummyReader).totalLen) + assert.Equal(t, uint64(0), reader.(*dummyReader).bytesRead) +} + +func TestDummyIOBucket_NewReaderWithReadHandle_NoRange(t *testing.T) { + req := &gcs.ReadObjectRequest{ + Name: "test-object", + // No Range specified + } + dummyBucket := NewDummyIOBucket(&TestifyMockBucket{}, DummyIOBucketParams{ReaderLatency: 0}) + + reader, err := dummyBucket.NewReaderWithReadHandle(context.Background(), req) + + assert.Error(t, err) + assert.Nil(t, reader) +} + +func TestDummyIOBucket_NewReaderWithReadHandle_InvalidRange(t *testing.T) { + req := &gcs.ReadObjectRequest{ + Name: "test-object", + Range: &gcs.ByteRange{ + Start: 100, + Limit: 50, // Invalid range: Limit < Start + }, + } + dummyBucket := NewDummyIOBucket(&TestifyMockBucket{}, DummyIOBucketParams{ReaderLatency: 0}) + + reader, err := dummyBucket.NewReaderWithReadHandle(context.Background(), req) + + assert.Error(t, err) + assert.Nil(t, reader) +} + +func TestDummyIOBucket_NewReaderWithReadHandle_WithLatency(t *testing.T) { + req := &gcs.ReadObjectRequest{ + Name: "test-object", + Range: &gcs.ByteRange{ + Start: 0, + Limit: 100, + }, + } + dummyBucket := NewDummyIOBucket(&TestifyMockBucket{}, DummyIOBucketParams{ReaderLatency: 5 * time.Millisecond}) + + start := time.Now() + reader, err := dummyBucket.NewReaderWithReadHandle(context.Background(), req) + elapsed := time.Since(start) + + assert.GreaterOrEqual(t, elapsed.Milliseconds(), int64(5)) + assert.NoError(t, err) + assert.NotNil(t, reader) + assert.IsType(t, &dummyReader{}, reader) + assert.Equal(t, uint64(100), reader.(*dummyReader).totalLen) + assert.Equal(t, uint64(0), reader.(*dummyReader).bytesRead) +} + +//////////////////////////////////////////////////////////////////////// +// Test for calculateLatency +//////////////////////////////////////////////////////////////////////// + +func TestCalculateLatency(t *testing.T) { + const MB = 1024 * 1024 + testCases := []struct { + name string + bytes int64 + perMBLatency time.Duration + expected time.Duration + }{ + { + name: "ZeroLatency", + bytes: MB, + perMBLatency: 0, + expected: 0, + }, + { + name: "NegativeLatency", + bytes: MB, + perMBLatency: -10 * time.Millisecond, + expected: 0, + }, + { + name: "ZeroBytes", + bytes: 0, + perMBLatency: 100 * time.Millisecond, + expected: 0, + }, + { + name: "OneMB", + bytes: MB, + perMBLatency: 100 * time.Millisecond, + expected: 100 * time.Millisecond, + }, + { + name: "MultipleMBs", + bytes: 5 * MB, + perMBLatency: 100 * time.Millisecond, + expected: 500 * time.Millisecond, + }, + { + name: "FractionOfMB", + bytes: MB / 2, + perMBLatency: 100 * time.Millisecond, + expected: 50 * time.Millisecond, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := calculateLatency(tc.bytes, tc.perMBLatency) + assert.Equal(t, tc.expected, result) + }) + } +} + +// ////////////////////////////////////////////////////////////////////// +// Test for dummyReader +// ////////////////////////////////////////////////////////////////////// +func TestDummyReader_NewDummyReader(t *testing.T) { + dummyReader := newDummyReader(10, 0) + + assert.Equal(t, uint64(10), dummyReader.totalLen) + assert.Equal(t, uint64(0), dummyReader.bytesRead) + assert.NotNil(t, dummyReader.readHandle) +} + +func TestDummyReader_ReadFull(t *testing.T) { + dummyReader := newDummyReader(10, 0) + + buffer := make([]byte, 10) + n, err := dummyReader.Read(buffer) + + assert.Error(t, err) + assert.Equal(t, io.EOF, err) + assert.Equal(t, 10, n) + assert.Equal(t, []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, buffer) +} + +func TestDummyReader_ReadPartial(t *testing.T) { + dummyReader := newDummyReader(10, 0) + + buffer := make([]byte, 5) + n, err := dummyReader.Read(buffer) + + assert.NoError(t, err) + assert.Equal(t, 5, n) + assert.Equal(t, []byte{0, 0, 0, 0, 0}, buffer) +} + +func TestDummyReader_ReadBeyondEOF(t *testing.T) { + dummyReader := newDummyReader(10, 0) + // First read 8 bytes + buffer1 := make([]byte, 8) + n1, err1 := dummyReader.Read(buffer1) + require.NoError(t, err1) + require.Equal(t, 8, n1) + require.Equal(t, []byte{0, 0, 0, 0, 0, 0, 0, 0}, buffer1) + + // Then read 5 bytes, which goes beyond EOF + buffer2 := make([]byte, 5) + n2, err2 := dummyReader.Read(buffer2) + + assert.Error(t, err2) + assert.Equal(t, io.EOF, err2) + assert.Equal(t, 2, n2) + assert.Equal(t, []byte{0, 0}, buffer2[:n2]) +} + +func TestDummyReader_Close(t *testing.T) { + dummyReader := newDummyReader(10, 0) + + err := dummyReader.Close() + + assert.NoError(t, err) +} + +func TestDummyReader_ReadHandle(t *testing.T) { + dummyReader := newDummyReader(10, 0) + + handle := dummyReader.ReadHandle() + + assert.NotNil(t, handle) +} + +func TestDummyReader_ReadWithLatency(t *testing.T) { + perMBLatency := 10 * time.Millisecond + dummyReader := newDummyReader(1024*1024, perMBLatency) // 1 MB total length + + buffer := make([]byte, 512*1024) // Read 512 KB + start := time.Now() + n, err := dummyReader.Read(buffer) + elapsed := time.Since(start) + + assert.NoError(t, err) + assert.Equal(t, 512*1024, n) + assert.GreaterOrEqual(t, elapsed, 5*time.Millisecond) +} + +//////////////////////////////////////////////////////////////////////// +// Test for dummyMultiRangeDownloader +//////////////////////////////////////////////////////////////////////// + +func TestDummyIOBucket_NewMultiRangeDownloader(t *testing.T) { + latency := 5 * time.Millisecond + params := DummyIOBucketParams{ + PerMBLatency: latency, + } + dummyBucket := NewDummyIOBucket(&TestifyMockBucket{}, params) + + mrd, err := dummyBucket.NewMultiRangeDownloader(context.Background(), &gcs.MultiRangeDownloaderRequest{}) + dmrd, ok := mrd.(*dummyMultiRangeDownloader) + + assert.NoError(t, err) + assert.NotNil(t, mrd) + assert.True(t, ok) + assert.Equal(t, latency, dmrd.perMBLatency) +} + +func TestDummyMultiRangeDownloader_Add_Single(t *testing.T) { + mrd := &dummyMultiRangeDownloader{} + var output bytes.Buffer + length := int64(100) + offset := int64(50) + var cbOffset, cbBytesWritten int64 + var cbErr error + var wg sync.WaitGroup + wg.Add(1) + + mrd.Add(&output, offset, length, func(o, bw int64, e error) { + cbOffset = o + cbBytesWritten = bw + cbErr = e + wg.Done() + }) + wg.Wait() // Wait for callback to be called + + assert.NoError(t, cbErr) + assert.Equal(t, offset, cbOffset) + assert.Equal(t, length, cbBytesWritten) + assert.Equal(t, int(length), output.Len()) + assert.Equal(t, make([]byte, length), output.Bytes()) +} + +func TestDummyMultiRangeDownloader_Add_ZeroLength(t *testing.T) { + mrd := &dummyMultiRangeDownloader{} + var output bytes.Buffer + length := int64(0) + offset := int64(50) + var cbOffset, cbBytesWritten int64 + var cbErr error + var wg sync.WaitGroup + wg.Add(1) + + mrd.Add(&output, offset, length, func(o, bw int64, e error) { + cbOffset = o + cbBytesWritten = bw + cbErr = e + wg.Done() + }) + wg.Wait() // Wait for callback to be called + + assert.NoError(t, cbErr) + assert.Equal(t, offset, cbOffset) + assert.Equal(t, length, cbBytesWritten) + assert.Equal(t, int(length), output.Len()) +} + +func TestDummyMultiRangeDownloader_Add_MultipleConcurrent(t *testing.T) { + mrd := &dummyMultiRangeDownloader{} + numAdds := 5 + errChan := make(chan error, numAdds) + + for i := 0; i < numAdds; i++ { + go func(i int) { + var output bytes.Buffer + length := int64(100 + i*10) + offset := int64(50 + i*100) + mrd.Add(&output, offset, length, func(o, bw int64, e error) { + if e != nil { + errChan <- fmt.Errorf("callback error: %w", e) + return + } + if o != offset { + errChan <- fmt.Errorf("offset mismatch: got %d, want %d", o, offset) + return + } + if bw != length { + errChan <- fmt.Errorf("bytesWritten mismatch: got %d, want %d", bw, length) + return + } + if int64(output.Len()) != length { + errChan <- fmt.Errorf("output length mismatch: got %d, want %d", output.Len(), length) + return + } + if !bytes.Equal(make([]byte, length), output.Bytes()) { + errChan <- fmt.Errorf("output content mismatch") + return + } + errChan <- nil + }) + }(i) + } + mrd.Wait() // Wait for all Add goroutines to finish writing + + for i := 0; i < numAdds; i++ { + err := <-errChan + assert.NoError(t, err) + } +} + +func TestDummyMultiRangeDownloader_Add_ValidateContent(t *testing.T) { + mrd := &dummyMultiRangeDownloader{} + var output bytes.Buffer + length := int64(5) + offset := int64(0) + + mrd.Add(&output, offset, length, nil) + mrd.Wait() + + // Check content + result := output.Bytes() + assert.Equal(t, int(length), len(result)) + // Verify the returned data is zeros + assert.Equal(t, make([]byte, length), result[:]) +} + +func TestDummyMultiRangeDownloader_Close(t *testing.T) { + mrd := &dummyMultiRangeDownloader{} + var output bytes.Buffer + length := int64(100) + callbackDone := make(chan bool, 1) + + mrd.Add(&output, 0, length, func(o, bw int64, e error) { + time.Sleep(10 * time.Millisecond) // Simulate some work in callback + callbackDone <- true + }) + err := mrd.Close() // Close should wait for the Add to complete. + + assert.NoError(t, err) + select { + case <-callbackDone: + // success + default: + t.Fatal("callback was not called after Close") + } +} + +func TestDummyMultiRangeDownloader_Latency(t *testing.T) { + perMBLatency := 100 * time.Millisecond + mrd := &dummyMultiRangeDownloader{perMBLatency: perMBLatency} + var output bytes.Buffer + length := int64(MB / 2) // 0.5 MB + expectedLatencyNs := float64(length) * float64(perMBLatency.Nanoseconds()) / float64(MB) + expectedLatency := time.Duration(expectedLatencyNs) + + start := time.Now() + mrd.Add(&output, 0, length, nil) + mrd.Wait() + elapsed := time.Since(start) + + // Allow some tolerance for scheduling delays + assert.GreaterOrEqual(t, elapsed, expectedLatency) + assert.Less(t, elapsed, expectedLatency*2, "Latency was too high") +} + +func TestDummyMultiRangeDownloader_GetHandle(t *testing.T) { + mrd := &dummyMultiRangeDownloader{} + + handle := mrd.GetHandle() + + assert.Equal(t, []byte("dummy-handle"), handle) +} + +func TestDummyMultiRangeDownloader_Error(t *testing.T) { + mrd := &dummyMultiRangeDownloader{} + + err := mrd.Error() + + assert.NoError(t, err) +} diff --git a/internal/storage/fake/bucket.go b/internal/storage/fake/bucket.go index 424f8b9ee0..f22e821dd7 100644 --- a/internal/storage/fake/bucket.go +++ b/internal/storage/fake/bucket.go @@ -23,15 +23,15 @@ import ( "fmt" "hash/crc32" "io" + "maps" "path/filepath" "sort" "strings" "time" "unicode/utf8" - "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" "github.com/jacobsa/syncutil" "github.com/jacobsa/timeutil" ) @@ -392,21 +392,35 @@ func preconditionChecks(b *bucket, req *gcs.CreateObjectRequest, contents []byte return } -func createOrUpdateFakeObject(b *bucket, req *gcs.CreateObjectRequest, contents []byte) (o *gcs.Object, err error) { - // Create an object record from the given attributes. - var fo fakeObject = b.mintObject(req, contents) - o = copyObject(&fo.metadata) - +func createOrUpdateFakeObject(b *bucket, req *gcs.CreateObjectRequest, contents []byte, isAppend bool) (o *gcs.Object, err error) { + var fo fakeObject // Replace an entry in or add an entry to our list of objects. existingIndex := b.objects.find(req.Name) if existingIndex < len(b.objects) { + var content []byte + if isAppend { + // If this is an append operation, then we will update the fake object with the appended content. + b.mu.Unlock() + existingContent, err := storageutil.ReadObject(context.Background(), b, req.Name) + b.mu.Lock() + if err != nil { + return nil, fmt.Errorf("error while reading existing content in fake object : %v", err) + } + content = append(existingContent, contents...) + } else { + // If the bucket type is not zonal, then we will overwrite. + content = contents + } + fo = b.mintObject(req, content) b.objects[existingIndex] = fo } else { + fo = b.mintObject(req, contents) b.objects = append(b.objects, fo) sort.Sort(b.objects) } + o = copyObject(&fo.metadata) - if b.BucketType() == gcs.Hierarchical { + if b.BucketType().Hierarchical { b.addFolderEntry(req.Name) } return @@ -430,7 +444,7 @@ func (b *bucket) createObjectLocked( if err != nil { return nil, err } - return createOrUpdateFakeObject(b, req, contents) + return createOrUpdateFakeObject(b, req, contents, false) } // Create a reader based on the supplied request, also returning the index @@ -453,7 +467,11 @@ func (b *bucket) newReaderLocked( // Does the generation match? if req.Generation != 0 && req.Generation != o.metadata.Generation { - err = storage.ErrObjectNotExist + err = &gcs.NotFoundError{ + Err: fmt.Errorf( + "object %s generation %v not found", req.Name, req.Generation), + } + return } @@ -501,9 +519,7 @@ func copyMetadata(in map[string]string) (out map[string]string) { } out = make(map[string]string) - for k, v := range in { - out[k] = v - } + maps.Copy(out, in) return } @@ -521,6 +537,7 @@ func copyMinObject(o *gcs.Object) *gcs.MinObject { copy.Generation = o.Generation copy.MetaGeneration = o.MetaGeneration copy.Updated = o.Updated + copy.Finalized = o.Finalized copy.Metadata = copyMetadata(o.Metadata) copy.ContentEncoding = o.ContentEncoding copy.CRC32C = o.CRC32C @@ -563,6 +580,9 @@ func (b *bucket) ListObjects( // Find the range of indexes within the array to scan. indexStart := b.objects.lowerBound(nameStart) + if req.StartOffset != "" { + indexStart = max(indexStart, b.objects.lowerBound(req.StartOffset)) + } prefixLimit := b.objects.prefixUpperBound(req.Prefix) indexLimit := minInt(indexStart+maxResults, prefixLimit) @@ -656,9 +676,7 @@ func (b *bucket) ListObjects( } // LOCKS_EXCLUDED(b.mu) -func (b *bucket) NewReader( - ctx context.Context, - req *gcs.ReadObjectRequest) (rc io.ReadCloser, err error) { +func (b *bucket) NewReaderWithReadHandle(ctx context.Context, req *gcs.ReadObjectRequest) (rd gcs.StorageReader, err error) { b.mu.Lock() defer b.mu.Unlock() @@ -667,7 +685,11 @@ func (b *bucket) NewReader( return } - rc = io.NopCloser(r) + rc := io.NopCloser(r) + rd = &FakeReader{ + ReadCloser: rc, + Handle: []byte("opaque-handle"), + } return } @@ -683,10 +705,37 @@ func (b *bucket) CreateObject( } func (b *bucket) CreateObjectChunkWriter(ctx context.Context, req *gcs.CreateObjectRequest, _ int, _ func(bytesUploadedSoFar int64)) (gcs.Writer, error) { - return NewFakeObjectWriter(b, req) + return NewFakeObjectWriter(b, req, false) +} + +func (b *bucket) CreateAppendableObjectWriter(ctx context.Context, req *gcs.CreateObjectChunkWriterRequest) (gcs.Writer, error) { + index := b.objects.find(req.Name) + if index != len(b.objects) { + obj := b.objects[index] + if obj.metadata.Generation == 0 { + return nil, fmt.Errorf("storage: ObjectHandle.Generation must be set to use NewWriterFromAppendableObject") + } + } + return NewFakeObjectWriter(b, &req.CreateObjectRequest, true) +} + +func (b *bucket) FlushPendingWrites(ctx context.Context, w gcs.Writer) (*gcs.MinObject, error) { + b.mu.Lock() + defer b.mu.Unlock() + + _, err := w.Flush() + if err != nil { + return nil, err + } + + fakeObjectWriter, ok := w.(*FakeObjectWriter) + if !ok { + return nil, fmt.Errorf("could not type assert gcs.Writer to FakeObjectWriter") + } + return fakeObjectWriter.Object, nil } -func (b *bucket) FinalizeUpload(ctx context.Context, w gcs.Writer) (*gcs.Object, error) { +func (b *bucket) FinalizeUpload(ctx context.Context, w gcs.Writer) (*gcs.MinObject, error) { b.mu.Lock() defer b.mu.Unlock() @@ -1004,6 +1053,66 @@ func (b *bucket) DeleteObject( return } +func (b *bucket) MoveObject(ctx context.Context, req *gcs.MoveObjectRequest) (*gcs.Object, error) { + // Check that the destination name is legal. + err := checkName(req.DstName) + if err != nil { + return nil, err + } + + // Does the source object exist? + srcIndex := b.objects.find(req.SrcName) + if srcIndex == len(b.objects) { + err = &gcs.NotFoundError{ + Err: fmt.Errorf("object %q not found", req.SrcName), + } + return nil, err + } + + // Does it have the correct generation? + if req.SrcGeneration != 0 && + b.objects[srcIndex].metadata.Generation != req.SrcGeneration { + err = &gcs.NotFoundError{ + Err: fmt.Errorf("object %s generation %d not found", req.SrcName, req.SrcGeneration), + } + return nil, err + } + + // Does it have the correct meta-generation? + if req.SrcMetaGenerationPrecondition != nil { + p := *req.SrcMetaGenerationPrecondition + if b.objects[srcIndex].metadata.MetaGeneration != p { + err = &gcs.PreconditionError{ + Err: fmt.Errorf("object %q has meta-generation %d", req.SrcName, b.objects[srcIndex].metadata.MetaGeneration), + } + return nil, err + } + } + + // Move it and assign a new generation number, to ensure that the generation + // number for the destination name is strictly increasing. + dst := b.objects[srcIndex] + dst.metadata.Name = req.DstName + dst.metadata.MediaLink = "http://localhost/download/storage/fake/" + req.DstName + + b.prevGeneration++ + dst.metadata.Generation = b.prevGeneration + + // Remove the source object. + b.objects = append(b.objects[:srcIndex], b.objects[srcIndex+1:]...) + // Insert dest object into our array. + existingIndex := b.objects.find(req.DstName) + if existingIndex < len(b.objects) { + b.objects[existingIndex] = dst + } else { + b.objects = append(b.objects, dst) + sort.Sort(b.objects) + } + + o := copyObject(&dst.metadata) + return o, err +} + func (b *bucket) DeleteFolder(ctx context.Context, folderName string) (err error) { b.mu.Lock() defer b.mu.Unlock() @@ -1033,20 +1142,20 @@ func (b *bucket) DeleteFolder(ctx context.Context, folderName string) (err error return } -func (b *bucket) GetFolder(ctx context.Context, foldername string) (*gcs.Folder, error) { +func (b *bucket) GetFolder(ctx context.Context, req *gcs.GetFolderRequest) (*gcs.Folder, error) { b.mu.Lock() defer b.mu.Unlock() // Does the folder exist? - index := b.folders.find(foldername) + index := b.folders.find(req.Name) if index == len(b.folders) { err := &gcs.NotFoundError{ - Err: fmt.Errorf("object %s not found", foldername), + Err: fmt.Errorf("object %s not found", req.Name), } return nil, err } - return &gcs.Folder{Name: foldername}, nil + return &gcs.Folder{Name: req.Name}, nil } func (b *bucket) CreateFolder(ctx context.Context, folderName string) (*gcs.Folder, error) { @@ -1141,3 +1250,24 @@ func (b *bucket) RenameFolder(ctx context.Context, folderName string, destinatio return folder, nil } + +func (b *bucket) NewMultiRangeDownloader( + ctx context.Context, req *gcs.MultiRangeDownloaderRequest) (gcs.MultiRangeDownloader, error) { + index := b.objects.find(req.Name) + if index >= len(b.objects) || index < 0 { + return nil, &gcs.NotFoundError{Err: fmt.Errorf("not found object %s in fake-bucket", req.Name)} + } + obj := b.objects[index] + if req.Generation != 0 && obj.metadata.Generation != req.Generation { + return nil, &gcs.NotFoundError{Err: fmt.Errorf("not found object %s in fake-bucket with generation %v", req.Name, req.Generation)} + } + if obj.data == nil { + return nil, fmt.Errorf("found no content for object %s in fake-bucket", req.Name) + } + + return &fakeMultiRangeDownloader{obj: &obj}, nil +} + +func (b *bucket) GCSName(obj *gcs.MinObject) string { + return obj.Name +} diff --git a/internal/storage/fake/bucket_test.go b/internal/storage/fake/bucket_test.go index 50c2d762f5..898a2f8368 100644 --- a/internal/storage/fake/bucket_test.go +++ b/internal/storage/fake/bucket_test.go @@ -18,8 +18,8 @@ import ( "testing" "time" - gcstesting "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/fake/testing" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + gcstesting "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake/testing" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "github.com/jacobsa/ogletest" "github.com/jacobsa/timeutil" "golang.org/x/net/context" @@ -35,7 +35,7 @@ func init() { deps.Clock = clock // Set up the bucket. - deps.Bucket = NewFakeBucket(clock, "some_bucket", gcs.NonHierarchical) + deps.Bucket = NewFakeBucket(clock, "some_bucket", gcs.BucketType{}) return } diff --git a/internal/storage/fake/fake_multi_range_downloader.go b/internal/storage/fake/fake_multi_range_downloader.go new file mode 100644 index 0000000000..453f5d0085 --- /dev/null +++ b/internal/storage/fake/fake_multi_range_downloader.go @@ -0,0 +1,167 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fake + +import ( + "fmt" + "io" + "sync" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" +) + +// This struct is an implementation of the gcs.MultiRangeDownloader interface. +type fakeMultiRangeDownloader struct { + gcs.MultiRangeDownloader + obj *fakeObject + wg sync.WaitGroup + err error + defaultErr error + statusErr error + sleepTime time.Duration // Sleep time to simulate real-world. + shortRead bool + handle []byte +} + +func createFakeObject(obj *gcs.MinObject, data []byte) fakeObject { + fullObj := storageutil.ConvertMinObjectToObject(obj) + return fakeObject{ + metadata: *fullObj, + data: data, + } +} + +func NewFakeMultiRangeDownloader(obj *gcs.MinObject, data []byte) gcs.MultiRangeDownloader { + return NewFakeMultiRangeDownloaderWithSleepAndDefaultError(obj, data, time.Millisecond, nil) +} + +func NewFakeMultiRangeDownloaderWithHandle(obj *gcs.MinObject, data []byte, handle []byte) gcs.MultiRangeDownloader { + mrd := NewFakeMultiRangeDownloader(obj, data) + if fmrd, ok := mrd.(*fakeMultiRangeDownloader); ok { + fmrd.handle = handle + } + return mrd +} + +func NewFakeMultiRangeDownloaderWithShortRead(obj *gcs.MinObject, data []byte) gcs.MultiRangeDownloader { + fakeObject := createFakeObject(obj, data) + return &fakeMultiRangeDownloader{ + obj: &fakeObject, + shortRead: true, + } +} + +func NewFakeMultiRangeDownloaderWithSleep(obj *gcs.MinObject, data []byte, sleepTime time.Duration) gcs.MultiRangeDownloader { + return NewFakeMultiRangeDownloaderWithSleepAndDefaultError(obj, data, sleepTime, nil) +} + +func NewFakeMultiRangeDownloaderWithSleepAndDefaultError(obj *gcs.MinObject, data []byte, sleepTime time.Duration, err error) gcs.MultiRangeDownloader { + fakeObj := createFakeObject(obj, data) + return &fakeMultiRangeDownloader{ + obj: &fakeObj, + sleepTime: sleepTime, + defaultErr: err, + } +} + +func NewFakeMultiRangeDownloaderWithStatusError(obj *gcs.MinObject, data []byte, err error) gcs.MultiRangeDownloader { + fakeObj := createFakeObject(obj, data) + return &fakeMultiRangeDownloader{ + obj: &fakeObj, + sleepTime: 0, + defaultErr: nil, + statusErr: err, + } +} + +func (fmrd *fakeMultiRangeDownloader) Add(output io.Writer, offset, length int64, callback func(int64, int64, error)) { + if fmrd.defaultErr != nil { + if callback != nil { + callback(offset, 0, fmrd.defaultErr) + } + return + } + obj := fmrd.obj + size := int64(len(obj.data)) + var err error + // Apply input checks as defined at https://github.com/googleapis/go-storage-prelaunch/blob/a5db2abd53775941df67b3337eabaf8d00ef0762/storage/reader.go#L373 . + if length < 0 { + err = fmt.Errorf("length < 0") + } else if offset > size { + err = fmt.Errorf("out of range. offset (%v) > size of content (%v) of %s", offset, size, obj.metadata.Name) + } else if offset <= -size { + offset = 0 + length = size + } else if offset < 0 { + offset = size + offset + length = min(length, size-offset) + } else { + length = min(length, size-offset) + } + if err != nil { + // If inputs aren't correct, fail immediately and return callback. + fmrd.err = err + if callback != nil { + callback(offset, 0, err) + } + return + } + + // Record this additional goroutine. + fmrd.wg.Add(1) + + go func() { + // clear this goroutine from waitgroup. + defer fmrd.wg.Done() + + if fmrd.shortRead { + length /= 2 + } + + time.Sleep(fmrd.sleepTime) + var n int + n, err = output.Write(obj.data[offset : offset+length]) + if err != nil || int64(n) != length { + err = fmt.Errorf("failed to write %v bytes to writer through multi-range-downloader, bytes written = %v, error = %v", length, n, err) + } + + if callback != nil { + callback(offset, int64(n), err) + } + // Don't clear pre-existing error in downloader. + if fmrd.err != nil { + fmrd.err = err + } + }() +} + +func (fmrd *fakeMultiRangeDownloader) Close() error { + fmrd.Wait() + return fmrd.err +} + +func (fmrd *fakeMultiRangeDownloader) Wait() { + fmrd.wg.Wait() +} + +func (fmrd *fakeMultiRangeDownloader) Error() error { + return fmrd.statusErr +} + +func (fmrd *fakeMultiRangeDownloader) GetHandle() []byte { + return fmrd.handle +} diff --git a/internal/storage/fake/fake_object_writer.go b/internal/storage/fake/fake_object_writer.go index efd94e7f38..6b7fd2946f 100644 --- a/internal/storage/fake/fake_object_writer.go +++ b/internal/storage/fake/fake_object_writer.go @@ -24,7 +24,8 @@ import ( "io" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" ) // FakeObjectWriter is a mock implementation of storage.Writer used by FakeBucket. @@ -34,10 +35,16 @@ type FakeObjectWriter struct { storage.ObjectAttrs bkt *bucket req *gcs.CreateObjectRequest - Object *gcs.Object // Object created by writer + Object *gcs.MinObject // Object created by writer + append bool } func (w *FakeObjectWriter) Write(p []byte) (n int, err error) { + contents := w.buf.Bytes() + // Validate for preconditions. + if err := preconditionChecks(w.bkt, w.req, contents); err != nil { + return 0, err + } return w.buf.Write(p) } @@ -49,14 +56,22 @@ func (w *FakeObjectWriter) Close() error { return err } - o, err := createOrUpdateFakeObject(w.bkt, w.req, contents) + o, err := createOrUpdateFakeObject(w.bkt, w.req, contents, w.append) if err == nil { - w.Object = o + w.Object = storageutil.ConvertObjToMinObject(o) } return err } +func (w *FakeObjectWriter) Flush() (int64, error) { + err := w.Close() + if err != nil { + return 0, err + } + return int64(w.buf.Len()), nil +} + func (w *FakeObjectWriter) ObjectName() string { return w.Name } @@ -64,7 +79,7 @@ func (w *FakeObjectWriter) Attrs() *storage.ObjectAttrs { return &w.ObjectAttrs } -func NewFakeObjectWriter(b *bucket, req *gcs.CreateObjectRequest) (w gcs.Writer, err error) { +func NewFakeObjectWriter(b *bucket, req *gcs.CreateObjectRequest, append bool) (w gcs.Writer, err error) { // Check that the name is legal. err = checkName(req.Name) if err != nil { @@ -78,6 +93,7 @@ func NewFakeObjectWriter(b *bucket, req *gcs.CreateObjectRequest) (w gcs.Writer, ObjectAttrs: storage.ObjectAttrs{ Name: req.Name, }, + append: append, } wr.ContentType = req.ContentType diff --git a/internal/storage/fake/fake_reader.go b/internal/storage/fake/fake_reader.go new file mode 100644 index 0000000000..7b960b9204 --- /dev/null +++ b/internal/storage/fake/fake_reader.go @@ -0,0 +1,31 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fake + +import ( + "io" + + storagev2 "cloud.google.com/go/storage" +) + +// FakeReader implements gcs.StorageReader interface +type FakeReader struct { + io.ReadCloser + Handle []byte +} + +func (fr *FakeReader) ReadHandle() storagev2.ReadHandle { + return fr.Handle +} diff --git a/internal/storage/fake/testing/bucket_tests.go b/internal/storage/fake/testing/bucket_tests.go index 54bccf970d..9dd9847075 100644 --- a/internal/storage/fake/testing/bucket_tests.go +++ b/internal/storage/fake/testing/bucket_tests.go @@ -17,6 +17,7 @@ package testing import ( + "bytes" "context" "crypto/md5" "crypto/rand" @@ -32,9 +33,8 @@ import ( "time" "unicode" - "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/ogletest" "github.com/jacobsa/timeutil" @@ -177,7 +177,7 @@ func interestingNames() (names []string) { // All codepoints in Unicode general categories C* (control and special) and // Z* (space), except for: // - // * Cn (non-character and reserved), which is not included in unicode.C. + // * Cn (non-character and reserved), which is large. // * Co (private usage), which is large. // * Cs (surrages), which is large. // * U+000A and U+000D, which are forbidden by the docs. @@ -187,6 +187,10 @@ func interestingNames() (names []string) { continue } + if unicode.In(r, unicode.Cn) { + continue + } + if unicode.In(r, unicode.Co) { continue } @@ -271,7 +275,7 @@ func readMultiple( }() // Open a reader. - rc, err := bucket.NewReader(ctx, reqs[i]) + rc, err := bucket.NewReaderWithReadHandle(ctx, reqs[i]) if err != nil { err = fmt.Errorf("NewReader: %v", err) return @@ -293,8 +297,8 @@ func readMultiple( } // Run several workers. - const parallelsim = 32 - for i := 0; i < parallelsim; i++ { + const parallelism = 32 + for range parallelism { group.Go(func() (err error) { for i := range indices { handleRequest(ctx, i) @@ -308,6 +312,76 @@ func readMultiple( return } +// Issue all of the supplied read requests +// using multi-range-read approach, +// with some degree of parallelism. +func readMultipleUsingMultiRangeDownloader( + ctx context.Context, + bucket gcs.Bucket, + req *gcs.MultiRangeDownloaderRequest, + ranges []gcs.ByteRange) (contents []*bytes.Buffer, errs []error) { + group, ctx := errgroup.WithContext(ctx) + if len(ranges) == 0 { + return // nothing to do + } + + // Open a reader. + mrd, err := bucket.NewMultiRangeDownloader(ctx, req) + if err != nil { + AddFailure("failed to get multi-range-downloader for object %s: %v", req.Name, err) + return + } + // Not checking error on mrd.Close() here as it is expected to fail for + // negative test cases. These negative cases are tested through errs[i]. + defer mrd.Close() + + // Feed indices into a channel. + indices := make(chan int, len(ranges)) + for i := range ranges { + indices <- i + } + close(indices) + + // Set up a function that deals with one request. + contents = make([]*bytes.Buffer, len(ranges)) + errs = make([]error, len(ranges)) + + handleRequest := func(i int) { + if ranges[i].Limit > ranges[i].Start { + size := ranges[i].Limit - ranges[i].Start + contents[i] = bytes.NewBuffer(make([]byte, 0, size)) + // Add a new range to read. + mrd.Add(contents[i], int64(ranges[i].Start), int64(size), func(offset, length int64, err error) { + // Mark error for this range if the callback returned error for it. + errs[i] = err + }) + } else { + // Create a dummy buffer to avoid unnecessary initialization failures. + contents[i] = bytes.NewBuffer(make([]byte, 0, 1)) + // Add a new range to read. + mrd.Add(contents[i], int64(ranges[i].Start), 0, func(offset, length int64, err error) { + // Mark error for this range if the callback returned error for it. + errs[i] = err + }) + } + } + + // Run several workers. + const parallelism = 32 + for range parallelism { + group.Go(func() (err error) { + for i := range indices { + handleRequest(i) + } + + return + }) + } + + AssertEq(nil, group.Wait()) + return +} + // Invoke the supplied function for each string, with some degree of // parallelism. func forEachString( @@ -325,7 +399,7 @@ func forEachString( // Consume the strings. const parallelism = 128 - for i := 0; i < parallelism; i++ { + for range parallelism { group.Go(func() (err error) { for s := range c { err = f(ctx, s) @@ -379,7 +453,7 @@ func (t *bucketTest) readObject(objectName string) (contents string, err error) Name: objectName, } - reader, err := t.bucket.NewReader(t.ctx, req) + reader, err := t.bucket.NewReaderWithReadHandle(t.ctx, req) if err != nil { return } @@ -433,6 +507,7 @@ func (t *bucketTest) assertOnObjectAttributes(expectedMinObj *gcs.MinObject, exp ExpectThat(expectedMinObj.Generation, Equals(o.Generation)) ExpectThat(expectedMinObj.MetaGeneration, Equals(o.MetaGeneration)) ExpectThat(expectedMinObj.Updated, DeepEquals(o.Updated)) + ExpectThat(expectedMinObj.Finalized, DeepEquals(o.Finalized)) ExpectThat(expectedMinObj.Metadata, DeepEquals(o.Metadata)) ExpectThat(expectedMinObj.ContentEncoding, Equals(o.ContentEncoding)) ExpectThat(expectedMinObj.CRC32C, Equals(o.CRC32C)) @@ -568,6 +643,7 @@ func (t *createTest) ObjectAttributes_Default() { ExpectEq("STANDARD", o.StorageClass) ExpectThat(o.Deleted, timeutil.TimeEq(time.Time{})) ExpectThat(o.Updated, t.matchesStartTime(createTime)) + ExpectThat(o.Finalized, timeutil.TimeEq(time.Time{})) // Make sure it matches when we stat object. minObj, extendedAttr, err := t.bucket.StatObject( @@ -622,6 +698,7 @@ func (t *createTest) ObjectAttributes_Explicit() { ExpectThat(o.Deleted, DeepEquals(time.Time{})) ExpectThat(o.Deleted, timeutil.TimeEq(time.Time{})) ExpectThat(o.Updated, t.matchesStartTime(createTime)) + ExpectThat(o.Finalized, timeutil.TimeEq(time.Time{})) // Make sure it matches when we stat object. minObj, extendedAttr, err := t.bucket.StatObject( @@ -1280,6 +1357,7 @@ func (t *copyTest) DestinationDoesntExist() { ExpectThat(dst.Deleted, DeepEquals(time.Time{})) ExpectThat(dst.Deleted, timeutil.TimeEq(time.Time{})) ExpectThat(dst.Updated, t.matchesStartTime(createTime)) + ExpectThat(dst.Finalized, timeutil.TimeEq(time.Time{})) // The object should be readable. contents, err := storageutil.ReadObject(t.ctx, t.bucket, "bar") @@ -1370,6 +1448,7 @@ func (t *copyTest) DestinationExists() { ExpectThat(dst.Deleted, DeepEquals(time.Time{})) ExpectThat(dst.Deleted, timeutil.TimeEq(time.Time{})) ExpectThat(dst.Updated, t.matchesStartTime(createTime)) + ExpectThat(dst.Finalized, timeutil.TimeEq(time.Time{})) // The object should be readable. contents, err := storageutil.ReadObject(t.ctx, t.bucket, "bar") @@ -1443,6 +1522,7 @@ func (t *copyTest) DestinationIsSameName() { ExpectThat(dst.Deleted, DeepEquals(time.Time{})) ExpectThat(dst.Deleted, timeutil.TimeEq(time.Time{})) ExpectThat(dst.Updated, t.matchesStartTime(createTime)) + ExpectThat(dst.Finalized, timeutil.TimeEq(time.Time{})) // The object should be readable. contents, err := storageutil.ReadObject(t.ctx, t.bucket, "foo") @@ -1688,7 +1768,7 @@ func (t *composeTest) createSources( group, ctx := errgroup.WithContext(t.ctx) const parallelism = 128 - for i := 0; i < parallelism; i++ { + for range parallelism { group.Go(func() (err error) { for i := range indices { // Create an object. Include some metadata; it should be ignored by @@ -1765,6 +1845,7 @@ func (t *composeTest) OneSimpleSource() { ExpectEq("STANDARD", o.StorageClass) ExpectThat(o.Deleted, timeutil.TimeEq(time.Time{})) ExpectThat(o.Updated, t.matchesStartTime(composeTime)) + ExpectThat(o.Finalized, timeutil.TimeEq(time.Time{})) // Check contents. contents, err := storageutil.ReadObject(t.ctx, t.bucket, "foo") @@ -1824,6 +1905,7 @@ func (t *composeTest) TwoSimpleSources() { ExpectEq("STANDARD", o.StorageClass) ExpectThat(o.Deleted, timeutil.TimeEq(time.Time{})) ExpectThat(o.Updated, t.matchesStartTime(composeTime)) + ExpectThat(o.Finalized, timeutil.TimeEq(time.Time{})) // Check contents. contents, err := storageutil.ReadObject(t.ctx, t.bucket, "foo") @@ -1880,6 +1962,7 @@ func (t *composeTest) ManySimpleSources() { ExpectEq("STANDARD", o.StorageClass) ExpectThat(o.Deleted, timeutil.TimeEq(time.Time{})) ExpectThat(o.Updated, t.matchesStartTime(composeTime)) + ExpectThat(o.Finalized, timeutil.TimeEq(time.Time{})) for _, src := range sources { ExpectLt(src.Generation, o.Generation) @@ -1951,6 +2034,7 @@ func (t *composeTest) RepeatedSources() { ExpectEq("STANDARD", o.StorageClass) ExpectThat(o.Deleted, timeutil.TimeEq(time.Time{})) ExpectThat(o.Updated, t.matchesStartTime(composeTime)) + ExpectThat(o.Finalized, timeutil.TimeEq(time.Time{})) // Check contents. contents, err := storageutil.ReadObject(t.ctx, t.bucket, "foo") @@ -2033,6 +2117,7 @@ func (t *composeTest) CompositeSources() { ExpectEq("STANDARD", o.StorageClass) ExpectThat(o.Deleted, timeutil.TimeEq(time.Time{})) ExpectThat(o.Updated, t.matchesStartTime(composeTime)) + ExpectThat(o.Finalized, timeutil.TimeEq(time.Time{})) // Check contents. contents, err := storageutil.ReadObject(t.ctx, t.bucket, "foo") @@ -2231,7 +2316,7 @@ func (t *composeTest) ExplicitGenerations_OneDoesntExist() { }, }) - ExpectThat(err, HasSameTypeAs(storage.ErrObjectNotExist)) + ExpectThat(err, HasSameTypeAs(&gcs.NotFoundError{})) // Make sure the destination object doesn't exist. _, _, err = t.bucket.StatObject( @@ -2515,7 +2600,7 @@ func (t *composeTest) TooManySources() { DstName: "foo", } - for i := 0; i < gcs.MaxSourcesPerComposeRequest+1; i++ { + for range gcs.MaxSourcesPerComposeRequest + 1 { req.Sources = append(req.Sources, gcs.ComposeSource{Name: src.Name}) } @@ -2546,7 +2631,7 @@ func (t *composeTest) ComponentCountLimits() { DstName: "medium", } - for i := 0; i < gcs.MaxSourcesPerComposeRequest; i++ { + for range gcs.MaxSourcesPerComposeRequest { req.Sources = append(req.Sources, gcs.ComposeSource{Name: small.Name}) } @@ -2562,7 +2647,7 @@ func (t *composeTest) ComponentCountLimits() { DstName: "large", } - for i := 0; i < gcs.MaxSourcesPerComposeRequest; i++ { + for range gcs.MaxSourcesPerComposeRequest { req.Sources = append(req.Sources, gcs.ComposeSource{Name: medium.Name}) } @@ -2681,7 +2766,7 @@ func (t *readTest) ObjectNameDoesntExist() { Name: "foobar", } - rc, err := t.bucket.NewReader(t.ctx, req) + rc, err := t.bucket.NewReaderWithReadHandle(t.ctx, req) if err == nil { defer rc.Close() _, err = rc.Read(make([]byte, 1)) @@ -2700,7 +2785,7 @@ func (t *readTest) EmptyObject() { Name: "foo", } - r, err := t.bucket.NewReader(t.ctx, req) + r, err := t.bucket.NewReaderWithReadHandle(t.ctx, req) AssertEq(nil, err) contents, err := io.ReadAll(r) @@ -2720,7 +2805,7 @@ func (t *readTest) NonEmptyObject() { Name: "foo", } - r, err := t.bucket.NewReader(t.ctx, req) + r, err := t.bucket.NewReaderWithReadHandle(t.ctx, req) AssertEq(nil, err) contents, err := io.ReadAll(r) @@ -2748,14 +2833,14 @@ func (t *readTest) ParticularGeneration_NeverExisted() { Generation: o.Generation + 1, } - rc, err := t.bucket.NewReader(t.ctx, req) + rc, err := t.bucket.NewReaderWithReadHandle(t.ctx, req) if err == nil { defer rc.Close() _, err = rc.Read(make([]byte, 1)) } - AssertThat(err, HasSameTypeAs(storage.ErrObjectNotExist)) - ExpectThat(err, Error(MatchesRegexp("(?i)object doesn't exist"))) + AssertThat(err, HasSameTypeAs(&gcs.NotFoundError{})) + ExpectThat(err, Error(MatchesRegexp("(?i)not found|404"))) } func (t *readTest) ParticularGeneration_HasBeenDeleted() { @@ -2784,7 +2869,7 @@ func (t *readTest) ParticularGeneration_HasBeenDeleted() { Generation: o.Generation, } - rc, err := t.bucket.NewReader(t.ctx, req) + rc, err := t.bucket.NewReaderWithReadHandle(t.ctx, req) if err == nil { defer rc.Close() _, err = rc.Read(make([]byte, 1)) @@ -2811,7 +2896,7 @@ func (t *readTest) ParticularGeneration_Exists() { Generation: o.Generation, } - r, err := t.bucket.NewReader(t.ctx, req) + r, err := t.bucket.NewReaderWithReadHandle(t.ctx, req) AssertEq(nil, err) contents, err := io.ReadAll(r) @@ -2850,19 +2935,19 @@ func (t *readTest) ParticularGeneration_ObjectHasBeenOverwritten() { Generation: o.Generation, } - rc, err := t.bucket.NewReader(t.ctx, req) + rc, err := t.bucket.NewReaderWithReadHandle(t.ctx, req) if err == nil { defer rc.Close() _, err = rc.Read(make([]byte, 1)) } - AssertThat(err, HasSameTypeAs(storage.ErrObjectNotExist)) - ExpectThat(err, Error(MatchesRegexp("(?i)object doesn't exist"))) + AssertThat(err, HasSameTypeAs(&gcs.NotFoundError{})) + ExpectThat(err, Error(MatchesRegexp("(?i)not found|404"))) // Reading by the new generation should work. req.Generation = o2.Generation - rc, err = t.bucket.NewReader(t.ctx, req) + rc, err = t.bucket.NewReaderWithReadHandle(t.ctx, req) AssertEq(nil, err) contents, err := io.ReadAll(rc) @@ -3029,6 +3114,337 @@ func (t *readTest) Ranges_NonEmptyObject() { } } +//////////////////////////////////////////////////////////////////////// +// Read - multirange +//////////////////////////////////////////////////////////////////////// + +type readMultiRangeTest struct { + bucketTest +} + +func (t *readMultiRangeTest) ObjectNameDoesntExist() { + req := &gcs.MultiRangeDownloaderRequest{ + Name: "foobar", + } + + _, err := t.bucket.NewMultiRangeDownloader(t.ctx, req) + + AssertThat(err, HasSameTypeAs(&gcs.NotFoundError{})) + ExpectThat(err, Error(MatchesRegexp("(?i)not found|404"))) +} + +func (t *readMultiRangeTest) EmptyObject() { + // Create + AssertEq(nil, t.createObject("foo", "")) + + // Read + req := &gcs.MultiRangeDownloaderRequest{ + Name: "foo", + } + + mrd, err := t.bucket.NewMultiRangeDownloader(t.ctx, req) + AssertEq(nil, err) + + // Close + AssertEq(nil, mrd.Close()) +} + +func (t *readMultiRangeTest) NonEmptyObject() { + // Create + AssertEq(nil, t.createObject("foo", "taco")) + + // Read + req := &gcs.MultiRangeDownloaderRequest{ + Name: "foo", + } + + mrd, err := t.bucket.NewMultiRangeDownloader(t.ctx, req) + AssertEq(nil, err) + + outBuffer := bytes.NewBufferString("") + callbackCalled := false + mrd.Add(outBuffer, 0, 4, func(offset int64, length int64, err error) { + AssertEq(nil, err) + AssertEq("taco", outBuffer.String()) + callbackCalled = true + }) + + // Close + mrd.Wait() + AssertTrue(callbackCalled) + AssertEq(nil, mrd.Close()) +} + +func (t *readMultiRangeTest) ParticularGeneration_NeverExisted() { + // Create an object. + o, err := storageutil.CreateObject( + t.ctx, + t.bucket, + "foo", + []byte{}) + + AssertEq(nil, err) + AssertGt(o.Generation, 0) + + // Attempt to read a different generation. + req := &gcs.MultiRangeDownloaderRequest{ + Name: "foo", + Generation: o.Generation + 1, + } + _, err = t.bucket.NewMultiRangeDownloader(t.ctx, req) + + AssertThat(err, HasSameTypeAs(&gcs.NotFoundError{})) + ExpectThat(err, Error(MatchesRegexp("(?i)not found|404"))) +} + +func (t *readMultiRangeTest) ParticularGeneration_HasBeenDeleted() { + // Create an object. + o, err := storageutil.CreateObject( + t.ctx, + t.bucket, + "foo", + []byte{}) + + AssertEq(nil, err) + AssertGt(o.Generation, 0) + + // Delete it. + err = t.bucket.DeleteObject( + t.ctx, + &gcs.DeleteObjectRequest{ + Name: "foo", + }) + + AssertEq(nil, err) + + // Attempt to read by generation. + req := &gcs.MultiRangeDownloaderRequest{ + Name: "foo", + Generation: o.Generation, + } + _, err = t.bucket.NewMultiRangeDownloader(t.ctx, req) + + AssertThat(err, HasSameTypeAs(&gcs.NotFoundError{})) + ExpectThat(err, Error(MatchesRegexp("(?i)not found|404"))) +} + +func (t *readMultiRangeTest) ParticularGeneration_Exists() { + // Create an object. + o, err := storageutil.CreateObject( + t.ctx, + t.bucket, + "foo", + []byte("taco")) + + AssertEq(nil, err) + AssertGt(o.Generation, 0) + + // Attempt to read the correct generation. + req := &gcs.MultiRangeDownloaderRequest{ + Name: "foo", + Generation: o.Generation, + } + + mrd, err := t.bucket.NewMultiRangeDownloader(t.ctx, req) + AssertEq(nil, err) + + outBuffer := bytes.NewBufferString("") + callbackCalled := false + mrd.Add(outBuffer, 0, 4, func(offset int64, length int64, err error) { + AssertEq(nil, err) + AssertEq("taco", outBuffer.String()) + callbackCalled = true + }) + + // Close + AssertEq(nil, mrd.Close()) + AssertTrue(callbackCalled) +} + +func (t *readMultiRangeTest) ParticularGeneration_ObjectHasBeenOverwritten() { + // Create an object. + o, err := storageutil.CreateObject( + t.ctx, + t.bucket, + "foo", + []byte("taco")) + + AssertEq(nil, err) + AssertGt(o.Generation, 0) + + // Overwrite with a new generation. + o2, err := storageutil.CreateObject( + t.ctx, + t.bucket, + "foo", + []byte("burrito")) + + AssertEq(nil, err) + AssertGt(o2.Generation, 0) + AssertNe(o.Generation, o2.Generation) + + // Reading by the old generation should fail. + req := &gcs.MultiRangeDownloaderRequest{ + Name: "foo", + Generation: o.Generation, + } + + _, err = t.bucket.NewMultiRangeDownloader(t.ctx, req) + AssertThat(err, HasSameTypeAs(&gcs.NotFoundError{})) + ExpectThat(err, Error(MatchesRegexp("(?i)not found|404"))) + + // Reading by the new generation should work. + req.Generation = o2.Generation + mrd, err := t.bucket.NewMultiRangeDownloader(t.ctx, req) + + AssertEq(nil, err) + AssertNe(nil, mrd) + + size := len("burrito") + callbackCalled := false + writer := bytes.NewBufferString("") + mrd.Add(writer, int64(0), int64(size), func(offset, length int64, err error) { + AssertEq(nil, err) + AssertEq("burrito", writer.String(), + "Failed to match buf=%v, content=%v", writer.Bytes(), []byte("burrito")) + callbackCalled = true + }) + + // Close + AssertEq(nil, mrd.Close()) + AssertTrue(callbackCalled) +} + +func (t *readMultiRangeTest) Ranges_EmptyObject() { + // Create an empty object. + AssertEq(nil, t.createObject("foo", "")) + + // Test cases. + testCases := []struct { + br gcs.ByteRange + expectError bool + }{ + // Empty without knowing object length + {gcs.ByteRange{Start: 0, Limit: 0}, false}, + + {gcs.ByteRange{Start: 1, Limit: 2}, true}, + {gcs.ByteRange{Start: 1, Limit: 1}, true}, + {gcs.ByteRange{Start: 1, Limit: 0}, true}, + + // Not empty without knowing object length + {gcs.ByteRange{Start: 0, Limit: 1}, false}, + {gcs.ByteRange{Start: 0, Limit: 17}, false}, + } + + // Turn test cases into read requests. + req := &gcs.MultiRangeDownloaderRequest{ + Name: "foo", + } + + var ranges []gcs.ByteRange + for _, tc := range testCases { + ranges = append(ranges, tc.br) + } + + // Make each request. + contents, errs := readMultipleUsingMultiRangeDownloader( + t.ctx, + t.bucket, + req, + ranges) + + AssertEq(len(testCases), len(contents)) + AssertEq(len(testCases), len(errs)) + for i, tc := range testCases { + desc := fmt.Sprintf("Test case %d, range %v", i, tc.br) + if tc.expectError { + ExpectNe(nil, errs[i], desc) + } else { + ExpectEq(nil, errs[i], "%s", desc) + ExpectEq("", contents[i].String(), "%s", desc) + } + } +} + +func (t *readMultiRangeTest) Ranges_NonEmptyObject() { + // Create an object of length four. + AssertEq(nil, t.createObject("foo", "taco")) + + // Test cases. + testCases := []struct { + br gcs.ByteRange + expectedContents string + expectError bool + }{ + // Left anchored + {gcs.ByteRange{Start: 0, Limit: 5}, "taco", false}, + {gcs.ByteRange{Start: 0, Limit: 4}, "taco", false}, + {gcs.ByteRange{Start: 0, Limit: 3}, "tac", false}, + {gcs.ByteRange{Start: 0, Limit: 2}, "ta", false}, + {gcs.ByteRange{Start: 0, Limit: 1}, "t", false}, + {gcs.ByteRange{Start: 0, Limit: 0}, "", false}, + + // Floating left edge + {gcs.ByteRange{Start: 1, Limit: 5}, "aco", false}, + {gcs.ByteRange{Start: 1, Limit: 4}, "aco", false}, + {gcs.ByteRange{Start: 1, Limit: 3}, "ac", false}, + {gcs.ByteRange{Start: 1, Limit: 2}, "a", false}, + {gcs.ByteRange{Start: 1, Limit: 1}, "", false}, + {gcs.ByteRange{Start: 1, Limit: 0}, "", false}, + + // Left edge at right edge of object + {gcs.ByteRange{Start: 4, Limit: 17}, "", false}, + {gcs.ByteRange{Start: 4, Limit: 5}, "", false}, + {gcs.ByteRange{Start: 4, Limit: 4}, "", false}, + {gcs.ByteRange{Start: 4, Limit: 1}, "", false}, + {gcs.ByteRange{Start: 4, Limit: 0}, "", false}, + + // Left edge past right edge of object + {gcs.ByteRange{Start: 5, Limit: 17}, "", true}, + {gcs.ByteRange{Start: 5, Limit: 5}, "", true}, + {gcs.ByteRange{Start: 5, Limit: 4}, "", true}, + {gcs.ByteRange{Start: 5, Limit: 1}, "", true}, + {gcs.ByteRange{Start: 5, Limit: 0}, "", true}, + + // Left edge is 2^63 - 1 + {gcs.ByteRange{Start: math.MaxInt64, Limit: 5}, "", true}, + {gcs.ByteRange{Start: math.MaxInt64, Limit: 4}, "", true}, + {gcs.ByteRange{Start: math.MaxInt64, Limit: 1}, "", true}, + {gcs.ByteRange{Start: math.MaxInt64, Limit: 0}, "", true}, + } + + // Turn test cases into read requests. + req := &gcs.MultiRangeDownloaderRequest{ + Name: "foo", + } + + var ranges []gcs.ByteRange + for _, tc := range testCases { + ranges = append(ranges, tc.br) + } + + // Make each request. + contents, errs := readMultipleUsingMultiRangeDownloader( + t.ctx, + t.bucket, + req, + ranges) + + AssertEq(len(testCases), len(contents)) + AssertEq(len(testCases), len(errs)) + for i, tc := range testCases { + desc := fmt.Sprintf("Test case %d, range %v", i, tc.br) + if tc.expectError { + ExpectNe(nil, errs[i], "%q", desc) + } else { + ExpectEq(nil, errs[i], "%q", desc) + if tc.br.Limit > tc.br.Start { + ExpectEq(tc.expectedContents, contents[i].String(), "%s", desc) + } + } + } +} + //////////////////////////////////////////////////////////////////////// // Stat //////////////////////////////////////////////////////////////////////// @@ -3075,6 +3491,7 @@ func (t *statTest) StatAfterCreating() { ExpectEq(len("taco"), m.Size) ExpectThat(e.Deleted, timeutil.TimeEq(time.Time{})) ExpectThat(m.Updated, timeutil.TimeEq(orig.Updated)) + ExpectThat(m.Finalized, timeutil.TimeEq(time.Time{})) } func (t *statTest) StatAfterOverwriting() { @@ -3111,6 +3528,7 @@ func (t *statTest) StatAfterOverwriting() { ExpectEq(len("burrito"), m.Size) ExpectThat(e.Deleted, timeutil.TimeEq(time.Time{})) ExpectThat(m.Updated, timeutil.TimeEq(o2.Updated)) + ExpectThat(m.Finalized, timeutil.TimeEq(time.Time{})) } func (t *statTest) StatAfterUpdating() { @@ -3164,6 +3582,7 @@ func (t *statTest) StatAfterUpdating() { ExpectEq(len("taco"), m.Size) ExpectThat(e.Deleted, timeutil.TimeEq(time.Time{})) ExpectThat(m.Updated, timeutil.TimeEq(o2.Updated)) + ExpectThat(m.Finalized, timeutil.TimeEq(time.Time{})) } //////////////////////////////////////////////////////////////////////// @@ -3668,7 +4087,7 @@ func (t *deleteTest) NoParticularGeneration_Successful() { Name: "a", } - rc, err := t.bucket.NewReader(t.ctx, req) + rc, err := t.bucket.NewReaderWithReadHandle(t.ctx, req) if err == nil { defer rc.Close() _, err = rc.Read(make([]byte, 1)) @@ -4453,7 +4872,7 @@ func (t *cancellationTest) ReadObject() { // Create a reader for the object using a cancellable context. ctx, cancel := context.WithCancel(t.ctx) - rc, err := t.bucket.NewReader( + rc, err := t.bucket.NewReaderWithReadHandle( ctx, &gcs.ReadObjectRequest{ Name: name, diff --git a/internal/storage/fake/testing/register_bucket_tests.go b/internal/storage/fake/testing/register_bucket_tests.go index cdcd96be66..9019cd2001 100644 --- a/internal/storage/fake/testing/register_bucket_tests.go +++ b/internal/storage/fake/testing/register_bucket_tests.go @@ -18,7 +18,7 @@ import ( "errors" "reflect" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "golang.org/x/net/context" "golang.org/x/text/cases" "golang.org/x/text/language" @@ -138,6 +138,7 @@ func RegisterBucketTests(makeDeps func(context.Context) BucketTestDeps) { ©Test{}, &composeTest{}, &readTest{}, + &readMultiRangeTest{}, &statTest{}, &updateTest{}, &deleteTest{}, diff --git a/internal/storage/fake_storage_util.go b/internal/storage/fake_storage_util.go index 4d30d02a16..2c764a1462 100644 --- a/internal/storage/fake_storage_util.go +++ b/internal/storage/fake_storage_util.go @@ -16,7 +16,9 @@ package storage import ( "github.com/fsouza/fake-gcs-server/fakestorage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" ) const TestBucketName string = "gcsfuse-default-bucket" @@ -36,7 +38,7 @@ const MetaDataKey string = "key" const TestGzipObjectName string = "gcsfuse/test_gzip.txt" // ContentInTestGzipObjectCompressed is a gzip-compressed content for gzip tests. -// It was created by uploading a small file to GCS using `gsutil cp -Z` and then +// It was created by uploading a small file to GCS using `gcloud storage cp --gzip-local-all` and then // downloading it as it is (compressed as present on GCS) using go storage client // library. To view/change it, open it in a gzip.newReader() ur using a gzip plugin // in the IDE. If you do change it, remember to update ContentInTestGzipObjectDecompressed @@ -53,10 +55,21 @@ type FakeStorage interface { type fakeStorage struct { fakeStorageServer *fakestorage.Server + mockClient *MockStorageControlClient + protocol cfg.Protocol } func (f *fakeStorage) CreateStorageHandle() (sh StorageHandle) { - sh = &storageClient{client: f.fakeStorageServer.Client()} + if f.mockClient == nil { + f.mockClient = new(MockStorageControlClient) + } + sh = &storageClient{ + httpClient: f.fakeStorageServer.Client(), + grpcClient: f.fakeStorageServer.Client(), + grpcClientWithBidiConfig: f.fakeStorageServer.Client(), + storageControlClient: f.mockClient, + clientConfig: storageutil.StorageClientConfig{ClientProtocol: f.protocol, WriteConfig: &cfg.WriteConfig{}}, + } return } @@ -75,6 +88,19 @@ func NewFakeStorage() FakeStorage { return fakeStorage } +func NewFakeStorageWithMockClient(mc *MockStorageControlClient, protocol cfg.Protocol) FakeStorage { + f, err := createFakeStorageServer(getTestFakeStorageObject()) + if err != nil { + panic(err) + } + fakeStorage := &fakeStorage{ + fakeStorageServer: f, + mockClient: mc, + protocol: protocol, + } + return fakeStorage +} + func getTestFakeStorageObject() []fakestorage.Object { var fakeObjects []fakestorage.Object testObjectRootFolder := fakestorage.Object{ diff --git a/internal/storage/full_read_closer.go b/internal/storage/full_read_closer.go new file mode 100644 index 0000000000..c112ad9f54 --- /dev/null +++ b/internal/storage/full_read_closer.go @@ -0,0 +1,53 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "io" + + storagev2 "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" +) + +// gcsFullReadCloser wraps a gcs.StorageReader and ensures that the Read call reads the entire response up to the buffer size even if the wrapped read returns data in smaller chunks. +type gcsFullReadCloser struct { + wrapped gcs.StorageReader +} + +func newGCSFullReadCloser(reader gcs.StorageReader) gcs.StorageReader { + return gcsFullReadCloser{wrapped: reader} +} + +// Read reads exactly len(buf) bytes from the wrapped StorageReader into buf. +// 1. the number of bytes copied and an EOF if response size < buffer size +// 2. n == len(buf) if and only if err == nil. +func (frc gcsFullReadCloser) Read(buf []byte) (n int, err error) { + n, err = io.ReadFull(frc.wrapped, buf) + if err == io.ErrUnexpectedEOF { + // if an EOF is encountered before reading the full length of the buffer, + // ReadFull returns an ErrUnexpectedEOF error. This needs to be convered + // to EOF in order to have a consistent behavior (error) with and without gcsFullReadCloser. + err = io.EOF + } + return n, err +} + +func (frc gcsFullReadCloser) ReadHandle() (rh storagev2.ReadHandle) { + return frc.wrapped.ReadHandle() +} + +func (frc gcsFullReadCloser) Close() (err error) { + return frc.wrapped.Close() +} diff --git a/internal/storage/full_read_closer_test.go b/internal/storage/full_read_closer_test.go new file mode 100644 index 0000000000..bfcbe09f35 --- /dev/null +++ b/internal/storage/full_read_closer_test.go @@ -0,0 +1,96 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "bytes" + "io" + "testing" + + storagev2 "cloud.google.com/go/storage" + "github.com/stretchr/testify/assert" +) + +// twoBytesStorageReader reads at most 2 bytes from the buffer in one go. +type twoBytesStorageReader struct { + buf *bytes.Buffer +} + +func (psr twoBytesStorageReader) ReadHandle() (rh storagev2.ReadHandle) { + return nil +} + +func (psr twoBytesStorageReader) Close() (err error) { + return nil +} + +func (psr twoBytesStorageReader) Read(b []byte) (n int, err error) { + maxBytes := 2 + bufLen := min(len(b), maxBytes) + temp := make([]byte, bufLen) + n, err = psr.buf.Read(temp) + copy(b, temp) + return n, err +} + +func TestFullReaderCloser(t *testing.T) { + t.Parallel() + tests := []struct { + name string + bufSize int + data []byte + expectedData []byte + expectedErr error + }{ + { + name: "large_buffer", + data: []byte("0123"), + bufSize: 5, + expectedData: []byte("0123"), + expectedErr: io.EOF, + }, + { + name: "small_buffer", + data: []byte("0123"), + bufSize: 2, + expectedData: []byte("01"), + expectedErr: nil, + }, + { + name: "equal_buffer", + data: []byte("0123"), + bufSize: 4, + expectedData: []byte("0123"), + expectedErr: nil, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + storageReader := twoBytesStorageReader{ + buf: new(bytes.Buffer), + } + storageReader.buf.Write(tc.data) + fullReadCloser := newGCSFullReadCloser(storageReader) + buffer := make([]byte, tc.bufSize) + + n, err := fullReadCloser.Read(buffer) + + assert.Equal(t, len(tc.expectedData), n) + assert.Equal(t, tc.expectedErr, err) + assert.Equal(t, tc.expectedData[:n], buffer[:n]) + }) + } +} diff --git a/internal/storage/gcs/bucket.go b/internal/storage/gcs/bucket.go index 746c0050c9..e01e6591f2 100644 --- a/internal/storage/gcs/bucket.go +++ b/internal/storage/gcs/bucket.go @@ -21,20 +21,34 @@ import ( "golang.org/x/net/context" ) -// BucketType represents different types of buckets like -// Hierarchical or NonHierarchical as constants. -type BucketType int +type PirloState int -// BucketType enum values. const ( - // A default value of "nil" indicates that the bucket - // type has not been specified. - Nil BucketType = iota - NonHierarchical - Hierarchical - Unknown + // PirloStateNone indicates the bucket is not a Pirlo bucket. + PirloStateNone PirloState = iota + // PirloStateRapidWritesEnabled indicates it is a Pirlo bucket with rapid writes enabled. + PirloStateRapidWritesEnabled + // PirloStateRapidWritesDisabled indicates it is a Pirlo bucket with rapid writes disabled. + PirloStateRapidWritesDisabled ) +// BucketType represents bucket features. +type BucketType struct { + Hierarchical bool + Zonal bool + Pirlo PirloState +} + +func (bt BucketType) IsRapid() bool { + return bt.Zonal || bt.Pirlo != PirloStateNone +} + +// RapidWritesEnabled returns true if the bucket supports rapid writes +// and they are currently active. +func (bt BucketType) RapidWritesEnabled() bool { + return bt.Zonal || bt.Pirlo == PirloStateRapidWritesEnabled +} + const ( // ReqIdField is the key for the value of // GCS req-id in context. @@ -49,6 +63,7 @@ const ( // purposes, such as the fake implementation in fake/bucket.go. type Writer interface { io.WriteCloser + Flush() (int64, error) ObjectName() string Attrs() *storage.ObjectAttrs } @@ -64,7 +79,7 @@ type Writer interface { type Bucket interface { Name() string - // Return Type of bucket e.g. Hierarchical or NonHierarchical + // Return Type of bucket. BucketType() BucketType // Create a reader for the contents of a particular generation of an object. @@ -76,9 +91,21 @@ type Bucket interface { // // Official documentation: // https://cloud.google.com/storage/docs/json_api/v1/objects/get - NewReader( + // Connection is established using the readHandle if not nil. + // ReadHandle helps in reducing the latency by eleminating auth/metadata checks when a valid readHandle is passed. + // ReadHandle is valid when its not nil, not expired and belongs to the same client. + NewReaderWithReadHandle( ctx context.Context, - req *ReadObjectRequest) (io.ReadCloser, error) + req *ReadObjectRequest) (StorageReader, error) + + // Create a new multi-range downloader for the contents of a particular generation of an object. + // On a nil error, the caller must arrange for the reader to be closed when + // it is no longer needed. + // + // Non-existent objects cause either this method or the first read from the + // resulting reader to return an error of type *NotFoundError. + NewMultiRangeDownloader( + ctx context.Context, req *MultiRangeDownloaderRequest) (MultiRangeDownloader, error) // Create or overwrite an object according to the supplied request. The new // object is guaranteed to exist immediately for the purposes of reading (and @@ -93,13 +120,28 @@ type Bucket interface { req *CreateObjectRequest) (*Object, error) // CreateObjectChunkWriter creates a *storage.Writer that can be used for - // resumable uploads. The new object will be available for reading after the - // writer is closed (object is finalised). + // chunked uploads. Depending on the underlying bucket's capabilities, it is + // used for either resumable uploads or appendable object uploads. For standard + // resumable uploads, the object becomes available for reading only after the + // writer is closed (finalized). For appendable uploads, the unfinalized + // object is available for reading immediately. CreateObjectChunkWriter(ctx context.Context, req *CreateObjectRequest, chunkSize int, callBack func(bytesUploadedSoFar int64)) (Writer, error) + // CreateAppendableObjectWriter creates a Writer for "Takeover" + // operations. It allows appending to an existing, unfinalized object by + // leveraging its specific generation number. Unlike a standard new object upload, + // this attaches to an existing unfinalized stream. All bytes written will be + // appended continuing from the offset passed via the CreateObjectChunkWriterRequest. + CreateAppendableObjectWriter(ctx context.Context, + req *CreateObjectChunkWriterRequest) (Writer, error) + // FinalizeUpload closes the storage.Writer which completes the write // operation and creates an object on GCS. - FinalizeUpload(ctx context.Context, writer Writer) (*Object, error) + FinalizeUpload(ctx context.Context, writer Writer) (*MinObject, error) + + // FlushPendingWrites is used for zonal buckets to flush any pending data in + // the writer buffer. The object is not finalized and can be appended further. + FlushPendingWrites(ctx context.Context, writer Writer) (*MinObject, error) // Copy an object to a new name, preserving all metadata. Any existing // generation of the destination name will be overwritten. @@ -160,12 +202,27 @@ type Bucket interface { ctx context.Context, req *DeleteObjectRequest) error + // MoveObject moves an object to a new name, preserving all metadata. + // + // This function overwrites any existing object at the destination name. + // + // Returns a record for the newly created object. + // TODO: Add official documentation link whenever it's available. + MoveObject(ctx context.Context, req *MoveObjectRequest) (*Object, error) + DeleteFolder(ctx context.Context, folderName string) error - GetFolder(ctx context.Context, folderName string) (*Folder, error) + GetFolder(ctx context.Context, req *GetFolderRequest) (*Folder, error) // Atomically rename folder for Hierarchical bucket. RenameFolder(ctx context.Context, folderName string, destinationFolderId string) (*Folder, error) CreateFolder(ctx context.Context, folderName string) (*Folder, error) + + // GCSName returns the original GCS name for the object. + // + // Some Bucket implementations modify the Name field of the MinObject before + // returning it, in which case, users must use this function to get the + // original name. + GCSName(object *MinObject) string } diff --git a/internal/storage/gcs/bucket_test.go b/internal/storage/gcs/bucket_test.go new file mode 100644 index 0000000000..1c3ea9e092 --- /dev/null +++ b/internal/storage/gcs/bucket_test.go @@ -0,0 +1,109 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcs + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBucketType_IsRapid(t *testing.T) { + testCases := []struct { + name string + zonal bool + pirlo PirloState + expected bool + }{ + { + name: "Neither Zonal nor Pirlo", + zonal: false, + pirlo: PirloStateNone, + expected: false, + }, + { + name: "Only Zonal is true", + zonal: true, + pirlo: PirloStateNone, + expected: true, + }, + { + name: "Pirlo Rapid Enabled", + zonal: false, + pirlo: PirloStateRapidWritesEnabled, + expected: true, + }, + { + name: "Pirlo Rapid Disabled", + zonal: false, + pirlo: PirloStateRapidWritesDisabled, + expected: true, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + bt := BucketType{ + Zonal: tc.zonal, + Pirlo: tc.pirlo, + } + + assert.Equal(t, tc.expected, bt.IsRapid()) + }) + } +} + +func TestBucketType_RapidWritesEnabled(t *testing.T) { + testCases := []struct { + name string + zonal bool + pirlo PirloState + expected bool + }{ + { + name: "Neither Zonal nor Pirlo", + zonal: false, + pirlo: PirloStateNone, + expected: false, + }, + { + name: "Only Zonal is true", + zonal: true, + pirlo: PirloStateNone, + expected: true, + }, + { + name: "Pirlo Rapid Enabled", + zonal: false, + pirlo: PirloStateRapidWritesEnabled, + expected: true, + }, + { + name: "Pirlo Rapid Disabled", + zonal: false, + pirlo: PirloStateRapidWritesDisabled, + expected: false, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + bt := BucketType{ + Zonal: tc.zonal, + Pirlo: tc.pirlo, + } + + assert.Equal(t, tc.expected, bt.RapidWritesEnabled()) + }) + } +} diff --git a/internal/storage/gcs/errors.go b/internal/storage/gcs/errors.go index ccc4941581..9457e074ff 100644 --- a/internal/storage/gcs/errors.go +++ b/internal/storage/gcs/errors.go @@ -14,7 +14,16 @@ package gcs -import "fmt" +import ( + "errors" + "fmt" + "net/http" + + "cloud.google.com/go/storage" + "google.golang.org/api/googleapi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) // A *NotFoundError value is an error that indicates an object name or a // particular generation for that name were not found. @@ -35,3 +44,40 @@ type PreconditionError struct { func (pe *PreconditionError) Error() string { return fmt.Sprintf("gcs.PreconditionError: %v", pe.Err) } + +// GetGCSError converts an error returned by go-sdk into gcsfuse specific common gcs error. +func GetGCSError(err error) error { + if err == nil { + return nil + } + + // Http client error. + var gErr *googleapi.Error + if errors.As(err, &gErr) { + switch gErr.Code { + case http.StatusNotFound: + return &NotFoundError{Err: err} + case http.StatusPreconditionFailed: + return &PreconditionError{Err: err} + } + } + + // RPC error (all gRPC client including control client). + if rpcErr, ok := status.FromError(err); ok { + switch rpcErr.Code() { + case codes.NotFound: + return &NotFoundError{Err: err} + case codes.FailedPrecondition: + return &PreconditionError{Err: err} + } + } + + // If storage object doesn't exist, go-sdk returns as ErrObjectNotExist. + // Important to note: currently go-sdk doesn't format/convert error coming from the control-client. + // Ref: http://shortn/_CY9Jyqf2wF + if errors.Is(err, storage.ErrObjectNotExist) { + return &NotFoundError{Err: err} + } + + return err +} diff --git a/internal/storage/gcs/errors_test.go b/internal/storage/gcs/errors_test.go new file mode 100644 index 0000000000..4ed1e8d022 --- /dev/null +++ b/internal/storage/gcs/errors_test.go @@ -0,0 +1,150 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcs + +import ( + "fmt" + "net/http" + "testing" + + "cloud.google.com/go/storage" + "github.com/googleapis/gax-go/v2/apierror" + "github.com/stretchr/testify/assert" + "google.golang.org/api/googleapi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "errors" +) + +func TestGetGCSError(t *testing.T) { + preconditionAPIErr, ok := apierror.FromError(status.Error(codes.FailedPrecondition, codes.FailedPrecondition.String())) + assert.True(t, ok) + + notFoundAPIErr, ok := apierror.FromError(status.Error(codes.NotFound, codes.NotFound.String())) + assert.True(t, ok) + + otherAPIErr, ok := apierror.FromError(status.Error(codes.Internal, codes.Internal.String())) + assert.True(t, ok) + + // TODO: to directly create the error for "wrapped_grpc_status_NotFound" sub-test, when the following issue is resolved. + // Ref: https://github.com/grpc/grpc-go/issues/8102 + grpcNotFoundErr := status.Error(codes.NotFound, "not found") + assert.True(t, ok) + + testCases := []struct { + name string + inputErr error + expectedErr error + }{ + { + name: "nil_error", + inputErr: nil, + expectedErr: nil, + }, + { + name: "googleapi.Error_NotFound", + inputErr: &googleapi.Error{Code: http.StatusNotFound}, + expectedErr: &NotFoundError{Err: &googleapi.Error{Code: http.StatusNotFound}}, + }, + { + name: "googleapi.Error_PreconditionFailed", + inputErr: &googleapi.Error{Code: http.StatusPreconditionFailed}, + expectedErr: &PreconditionError{Err: &googleapi.Error{Code: http.StatusPreconditionFailed}}, + }, + { + name: "googleapi.Error_other_code", + inputErr: &googleapi.Error{Code: http.StatusBadRequest}, + expectedErr: &googleapi.Error{Code: http.StatusBadRequest}, + }, + { + name: "wrapped_googleapi.Error_NotFound", + inputErr: fmt.Errorf("wrapped: %w", &googleapi.Error{Code: http.StatusNotFound}), + expectedErr: &NotFoundError{Err: fmt.Errorf("wrapped: %w", &googleapi.Error{Code: http.StatusNotFound})}, + }, + { + name: "grpc_status_NotFound", + inputErr: status.Error(codes.NotFound, "not found"), + expectedErr: &NotFoundError{Err: status.Error(codes.NotFound, "not found")}, + }, + { + name: "grpc_status_FailedPrecondition", + inputErr: status.Error(codes.FailedPrecondition, "failed precondition"), + expectedErr: &PreconditionError{Err: status.Error(codes.FailedPrecondition, "failed precondition")}, + }, + { + name: "grpc_status_other_code", + inputErr: status.Error(codes.Internal, "internal error"), + expectedErr: status.Error(codes.Internal, "internal error"), + }, + { + name: "other_error", + inputErr: errors.New("some error"), + expectedErr: errors.New("some error"), + }, + { + name: "wrapped_grpc_status_NotFound", + inputErr: fmt.Errorf("wrapped: %w", grpcNotFoundErr), + expectedErr: &NotFoundError{Err: fmt.Errorf("wrapped: %w", grpcNotFoundErr)}, + }, + { + name: "GCS_Precondition_error", + inputErr: &PreconditionError{Err: errors.New("precondition error")}, + expectedErr: &PreconditionError{Err: errors.New("precondition error")}, + }, + { + name: "GCS_NotFound_error", + inputErr: &NotFoundError{Err: errors.New("not found error")}, + expectedErr: &NotFoundError{Err: errors.New("not found error")}, + }, + { + name: "wrapped_GCS_Precondition_error", + inputErr: fmt.Errorf("wrapped: %w", &PreconditionError{Err: errors.New("precondition error")}), + expectedErr: fmt.Errorf("wrapped: %w", &PreconditionError{Err: errors.New("precondition error")}), + }, + { + name: "storage_object_not_exist", + inputErr: storage.ErrObjectNotExist, + expectedErr: &NotFoundError{Err: storage.ErrObjectNotExist}, + }, + { + name: "precondition_apierror", + inputErr: preconditionAPIErr, + expectedErr: &PreconditionError{Err: preconditionAPIErr}, + }, + { + name: "notfound_apierror", + inputErr: notFoundAPIErr, + expectedErr: &NotFoundError{Err: notFoundAPIErr}, + }, + { + name: "other_apierror", + inputErr: otherAPIErr, + expectedErr: otherAPIErr, + }, + { + name: "wrapped_precondition_apierror", + inputErr: fmt.Errorf("wrapped: %w", preconditionAPIErr), + expectedErr: &PreconditionError{Err: fmt.Errorf("wrapped: %w", preconditionAPIErr)}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got := GetGCSError(tc.inputErr) + assert.Equal(t, tc.expectedErr, got) + }) + } +} diff --git a/internal/storage/gcs/multi_range_downloader.go b/internal/storage/gcs/multi_range_downloader.go new file mode 100644 index 0000000000..fcd473b93e --- /dev/null +++ b/internal/storage/gcs/multi_range_downloader.go @@ -0,0 +1,27 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcs + +import "io" + +// An interface to generalize the MultiRangeDownloader +// structure in go-storage module to ease our testing. +type MultiRangeDownloader interface { + Add(output io.Writer, offset, length int64, callback func(int64, int64, error)) + Close() error + Wait() + Error() error + GetHandle() []byte +} diff --git a/internal/storage/gcs/object.go b/internal/storage/gcs/object.go index baa6220617..095b88734b 100644 --- a/internal/storage/gcs/object.go +++ b/internal/storage/gcs/object.go @@ -46,6 +46,7 @@ type Object struct { StorageClass string Deleted time.Time Updated time.Time + Finalized time.Time // As of 2015-06-03, the official GCS documentation for this property // (https://tinyurl.com/2zjza2cu) says this: @@ -86,6 +87,7 @@ type MinObject struct { Generation int64 MetaGeneration int64 Updated time.Time + Finalized time.Time Metadata map[string]string ContentEncoding string CRC32C *uint32 // Missing for CMEK buckets @@ -111,3 +113,7 @@ type ExtendedObjectAttributes struct { func (mo MinObject) HasContentEncodingGzip() bool { return mo.ContentEncoding == ContentEncodingGzip } + +func (mo MinObject) IsUnfinalized() bool { + return mo.Finalized.IsZero() +} diff --git a/internal/storage/gcs/object_test.go b/internal/storage/gcs/object_test.go index a5ed7ab87f..8e7f32995b 100644 --- a/internal/storage/gcs/object_test.go +++ b/internal/storage/gcs/object_test.go @@ -16,6 +16,7 @@ package gcs import ( "testing" + "time" . "github.com/jacobsa/ogletest" ) @@ -52,3 +53,15 @@ func (t *ObjectTest) HasContentEncodingGzipNegative() { AssertFalse(mo.HasContentEncodingGzip()) } } + +func (t *ObjectTest) IsFinalized() { + mo := MinObject{Finalized: time.Date(2025, time.June, 19, 18, 23, 30, 0, time.UTC)} + + AssertFalse(mo.IsUnfinalized()) +} + +func (t *ObjectTest) IsNotFinalized() { + mo := MinObject{} + + AssertTrue(mo.IsUnfinalized()) +} diff --git a/internal/storage/gcs/request.go b/internal/storage/gcs/request.go index 30eef0bd2c..a54db4cb24 100644 --- a/internal/storage/gcs/request.go +++ b/internal/storage/gcs/request.go @@ -19,6 +19,7 @@ import ( "fmt" "io" + storagev2 "cloud.google.com/go/storage" storagev1 "google.golang.org/api/storage/v1" ) @@ -54,6 +55,28 @@ type CreateObjectRequest struct { StorageClass string Acl []*storagev1.ObjectAccessControl + // ChunkRetryDeadlineSecs sets the total deadline for retrying a chunk upload + // during resumable uploads. + // + // For resumable uploads, if a chunk upload fails or stalls, retries will be + // attempted until this deadline is reached. + // + // The default value is 120 seconds. + // + // NOTE: This is not supported in gRPC (e.g. CreateAppendableObjectWriter). + ChunkRetryDeadlineSecs int64 + + // ChunkTransferTimeoutSecs sets a per-chunk request timeout for resumable uploads. + // + // For resumable uploads, the Writer will terminate the request and attempt a retry + // if the request to upload a particular chunk stalls for longer than this duration. + // Retries will continue until ChunkRetryDeadlineSecs is reached. + // + // The default value is 10 seconds. + // + // NOTE: This is not supported in gRPC (e.g. CreateAppendableObjectWriter). + ChunkTransferTimeoutSecs int64 + // A reader from which to obtain the contents of the object. Must be non-nil. Contents io.Reader @@ -74,6 +97,9 @@ type CreateObjectRequest struct { // meta-generation for the object name is equal to the given value. This is // only meaningful in conjunction with GenerationPrecondition. MetaGenerationPrecondition *int64 + + // CallBack function is called after upload of each chunk. + CallBack func(bytesUploadedSoFar int64) } // A request to copy an object to a new name, preserving all metadata. @@ -158,6 +184,12 @@ type ComposeSource struct { Generation int64 } +// StorageReader implements the storage.Reader. +type StorageReader interface { + ReadHandle() storagev2.ReadHandle + io.ReadCloser +} + // ByteRange is a [start, limit) range of bytes within an object. // // Its semantics are as follows: @@ -192,6 +224,25 @@ type ReadObjectRequest struct { // If present, read the contents of the GCS object as it is on GCS. // This might not be honoured by all the implementations. ReadCompressed bool + + // ReadHandle associated with the object. This would be periodically refreshed. + ReadHandle []byte +} + +// A request to read the contents of an object at a particular generation. +type MultiRangeDownloaderRequest struct { + // The name of the object to read. + Name string + + // The generation of the object to read. Zero means the latest generation. + Generation int64 + + // If present, read the contents of the GCS object as it is on GCS. + // This might not be honoured by all the implementations. + ReadCompressed bool + + // ReadHandle associated with the object. This would be periodically refreshed. + ReadHandle []byte } type StatObjectRequest struct { @@ -204,6 +255,26 @@ type StatObjectRequest struct { // Controls whether StatObject response includes GCS ExtendedObjectAttributes. ReturnExtendedObjectAttributes bool + + // FetchOnlyFromCache determines if the request should be served exclusively from the stat cache. + // + // If true, the request performs a cache lookup. On a cache miss, it returns a CacheMissError + // and does not fall back to GCS. + // + // If false, the request falls back to GCS on a cache miss. + FetchOnlyFromCache bool +} + +type GetFolderRequest struct { + Name string + + // FetchOnlyFromCache determines if the request should be served exclusively from the stat cache. + // + // If true, the request performs a cache lookup. On a cache miss, it returns a CacheMissError + // and does not fall back to GCS. + // + // If false, the request falls back to GCS on a cache miss. + FetchOnlyFromCache bool } type Projection int64 @@ -282,6 +353,20 @@ type ListObjectsRequest struct { // the current flow, default value will be full and callers can override it // using this param. ProjectionVal Projection + + // FetchOnlyFromCache determines if the request should be served exclusively from the stat cache. + // + // If true, the request performs a cache lookup. On a cache miss, it returns a CacheMissError + // and does not fall back to GCS. + // + // If false, the cache is bypassed entirely and the request is served directly from GCS. + // + // Note: This flag is currently only respected when IsTypeCacheDeprecated is true. + FetchOnlyFromCache bool + + // StartOffset is used to filter results to objects whose names are + // lexicographically equal to or after startOffset. + StartOffset string } // Listing contains a set of objects and delimter-based collapsed runs returned @@ -375,4 +460,38 @@ type DeleteObjectRequest struct { // with the given name (and optionally generation), and its meta-generation // is not equal to this value. MetaGenerationPrecondition *int64 + + // OnlyDeleteFromCache controls whether the deletion is restricted to the local cache. + // + // If true, it updates the cache with a negative entry and skips the GCS call. + // If false, it proceeds with the standard GCS deletion and updates the cache on success. + OnlyDeleteFromCache bool +} + +// MoveObjectRequest represents a request to move or rename an object. +type MoveObjectRequest struct { + SrcName string // Source object name + DstName string // Destination object name + + // The generation of the source object to move, or zero for the latest + // generation. + SrcGeneration int64 + + // If non-nil, the destination object will be created/overwritten only if the + // current meta-generation for the source object is equal to the given value. + SrcMetaGenerationPrecondition *int64 +} + +// CreateObjectChunkWriterRequest represents a request to create a storage.Writer +// which can be used for appendable object writes via the CreateAppendableObjectWriter +// method. +type CreateObjectChunkWriterRequest struct { + CreateObjectRequest + + // Size of each chunk to be uploaded to GCS + ChunkSize int + + // Offset from where write has to start. Used only in case of appends flows. + // Default value is zero which means it's a new object write. + Offset int64 } diff --git a/internal/storage/gcs/request_helper.go b/internal/storage/gcs/request_helper.go new file mode 100644 index 0000000000..ecc183b548 --- /dev/null +++ b/internal/storage/gcs/request_helper.go @@ -0,0 +1,65 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcs + +import ( + "maps" + "time" +) + +// MtimeMetadataKey objects are created by Syncer.SyncObject and contain a +// metadata field with this key and with a UTC mtime in the format defined +// by time.RFC3339Nano. +const MtimeMetadataKey = "gcsfuse_mtime" + +func NewCreateObjectRequest(srcObject *Object, objectName string, mtime *time.Time, chunkRetryDeadlineSecs, chunkTransferTimeoutSecs int64) *CreateObjectRequest { + metadataMap := make(map[string]string) + var req *CreateObjectRequest + if srcObject == nil { + var preCond int64 + req = &CreateObjectRequest{ + Name: objectName, + GenerationPrecondition: &preCond, + Metadata: metadataMap, + ChunkTransferTimeoutSecs: chunkTransferTimeoutSecs, + ChunkRetryDeadlineSecs: chunkRetryDeadlineSecs, + } + } else { + maps.Copy(metadataMap, srcObject.Metadata) + + req = &CreateObjectRequest{ + Name: srcObject.Name, + GenerationPrecondition: &srcObject.Generation, + MetaGenerationPrecondition: &srcObject.MetaGeneration, + Metadata: metadataMap, + CacheControl: srcObject.CacheControl, + ContentDisposition: srcObject.ContentDisposition, + ContentEncoding: srcObject.ContentEncoding, + ContentType: srcObject.ContentType, + CustomTime: srcObject.CustomTime, + EventBasedHold: srcObject.EventBasedHold, + StorageClass: srcObject.StorageClass, + ChunkRetryDeadlineSecs: chunkRetryDeadlineSecs, + ChunkTransferTimeoutSecs: chunkTransferTimeoutSecs, + } + } + + // Any existing mtime value will be overwritten with new value. + if mtime != nil { + metadataMap[MtimeMetadataKey] = mtime.UTC().Format(time.RFC3339Nano) + } + + return req +} diff --git a/internal/storage/gcs/request_helper_test.go b/internal/storage/gcs/request_helper_test.go new file mode 100644 index 0000000000..8f92016728 --- /dev/null +++ b/internal/storage/gcs/request_helper_test.go @@ -0,0 +1,112 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcs + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestCreateObjectRequest(t *testing.T) { + now := time.Now() + + tests := []struct { + name string + srcObject *Object + objectName string + mtime *time.Time + chunkRetryDeadlineSecs int64 + chunkTransferTimeoutSecs int64 + expectedRequest *CreateObjectRequest + }{ + { + name: "nil_srcObject", + objectName: "new-object.txt", + mtime: &now, + chunkRetryDeadlineSecs: 60, + chunkTransferTimeoutSecs: 30, + expectedRequest: &CreateObjectRequest{ + Name: "new-object.txt", + GenerationPrecondition: &[]int64{0}[0], // Default precondition + Metadata: map[string]string{ + MtimeMetadataKey: now.UTC().Format(time.RFC3339Nano), + }, + ChunkRetryDeadlineSecs: 60, + ChunkTransferTimeoutSecs: 30, + }, + }, + { + name: "existing_srcObject", + srcObject: &Object{ + Name: "existing-object.txt", + Generation: 12345, + MetaGeneration: 67890, + Metadata: map[string]string{"key1": "value1", "key2": "value2"}, + CacheControl: "public, max-age=3600", + ContentDisposition: "attachment; filename=\"myfile.txt\"", + ContentEncoding: "gzip", + ContentType: "text/plain", + CustomTime: now.Add(-24 * time.Hour).String(), + EventBasedHold: true, + StorageClass: "STANDARD", + }, + mtime: &now, + chunkRetryDeadlineSecs: 120, + chunkTransferTimeoutSecs: 60, + expectedRequest: &CreateObjectRequest{ + Name: "existing-object.txt", + GenerationPrecondition: &[]int64{12345}[0], + MetaGenerationPrecondition: &[]int64{67890}[0], + Metadata: map[string]string{ + "key1": "value1", + "key2": "value2", + MtimeMetadataKey: now.UTC().Format(time.RFC3339Nano), + }, + CacheControl: "public, max-age=3600", + ContentDisposition: "attachment; filename=\"myfile.txt\"", + ContentEncoding: "gzip", + ContentType: "text/plain", + CustomTime: now.Add(-24 * time.Hour).String(), + EventBasedHold: true, + StorageClass: "STANDARD", + ChunkRetryDeadlineSecs: 120, + ChunkTransferTimeoutSecs: 60, + }, + }, + { + name: "nil_mtime_nil_srcObject", + objectName: "no-mtime.txt", + chunkRetryDeadlineSecs: 60, + chunkTransferTimeoutSecs: 30, + expectedRequest: &CreateObjectRequest{ + Name: "no-mtime.txt", + GenerationPrecondition: &[]int64{0}[0], + Metadata: map[string]string{}, + ChunkRetryDeadlineSecs: 60, + ChunkTransferTimeoutSecs: 30, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := NewCreateObjectRequest(tt.srcObject, tt.objectName, tt.mtime, tt.chunkRetryDeadlineSecs, tt.chunkTransferTimeoutSecs) + + assert.Equal(t, tt.expectedRequest, req) + }) + } +} diff --git a/internal/storage/mock/mock_writer.go b/internal/storage/mock/mock_writer.go index 77c891c015..087ecdf346 100644 --- a/internal/storage/mock/mock_writer.go +++ b/internal/storage/mock/mock_writer.go @@ -15,56 +15,43 @@ package mock import ( - "bytes" - "fmt" "io" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/stretchr/testify/mock" ) -// MockWriter implements io.WriteCloser and is used in unit tests to mock +// Writer implements io.WriteCloser and is used in unit tests to mock // the behavior of a GCS object writer. This is particular used with // storage.TestifyMockBucket implementation and allows for controlled testing of // interactions with the writer without relying on actual GCS operations. -type MockWriter struct { +type Writer struct { io.WriteCloser - buf bytes.Buffer storage.ObjectAttrs - errorOnClose bool - errorOnWrite bool + mock.Mock } -func (w *MockWriter) Write(p []byte) (n int, err error) { - if w.errorOnWrite { - return 0, fmt.Errorf("error while writing") - } - return w.buf.Write(p) +func (mw *Writer) Write(p []byte) (n int, err error) { + args := mw.Called(p) + return args.Int(0), args.Error(1) } -func (w *MockWriter) Close() error { - if w.errorOnClose { - return fmt.Errorf("error while closing writer") - } - return nil +func (mw *Writer) Attrs() *storage.ObjectAttrs { + args := mw.Called() + return args.Get(0).(*storage.ObjectAttrs) } -func (w *MockWriter) ObjectName() string { - return w.Name -} -func (w *MockWriter) Attrs() *storage.ObjectAttrs { - return &w.ObjectAttrs +func (mw *Writer) Close() error { + args := mw.Called() + return args.Error(0) } -func NewMockWriter(objName string, errorOnWrite, errorOnClose bool) gcs.Writer { - wr := &MockWriter{ - buf: bytes.Buffer{}, - errorOnWrite: errorOnWrite, - errorOnClose: errorOnClose, - ObjectAttrs: storage.ObjectAttrs{ - Name: objName, - }, - } +func (mw *Writer) Flush() (int64, error) { + args := mw.Called() + return args.Get(0).(int64), args.Error(1) +} - return wr +func (mw *Writer) ObjectName() string { + args := mw.Called() + return args.String(0) } diff --git a/internal/storage/mock/testify_mock_bucket.go b/internal/storage/mock/testify_mock_bucket.go new file mode 100644 index 0000000000..c6db55df23 --- /dev/null +++ b/internal/storage/mock/testify_mock_bucket.go @@ -0,0 +1,163 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mock + +import ( + "context" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/stretchr/testify/mock" +) + +// TODO: Rename to mock bucket once deprecated ogle mock bucket is removed from all usages in unit tests +type TestifyMockBucket struct { + mock.Mock +} + +func (m *TestifyMockBucket) Name() string { + args := m.Called() + return args.String(0) +} + +func (m *TestifyMockBucket) BucketType() gcs.BucketType { + args := m.Called() + return args.Get(0).(gcs.BucketType) +} + +func (m *TestifyMockBucket) NewReaderWithReadHandle(ctx context.Context, req *gcs.ReadObjectRequest) (gcs.StorageReader, error) { + args := m.Called(ctx, req) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(gcs.StorageReader), args.Error(1) +} + +func (m *TestifyMockBucket) CreateObject(ctx context.Context, req *gcs.CreateObjectRequest) (*gcs.Object, error) { + args := m.Called(ctx, req) + if args.Get(1) != nil { + return nil, args.Error(1) + } + return args.Get(0).(*gcs.Object), nil +} + +func (m *TestifyMockBucket) CreateObjectChunkWriter(ctx context.Context, req *gcs.CreateObjectRequest, chunkSize int, callBack func(bytesUploadedSoFar int64)) (wc gcs.Writer, err error) { + args := m.Called(ctx, req) + if args.Get(1) != nil { + return nil, args.Error(1) + } + return args.Get(0).(gcs.Writer), nil +} + +func (m *TestifyMockBucket) CreateAppendableObjectWriter(ctx context.Context, req *gcs.CreateObjectChunkWriterRequest) (wc gcs.Writer, err error) { + args := m.Called(ctx, req) + if args.Get(1) != nil { + return nil, args.Error(1) + } + return args.Get(0).(gcs.Writer), nil +} + +func (m *TestifyMockBucket) FinalizeUpload(ctx context.Context, w gcs.Writer) (*gcs.MinObject, error) { + args := m.Called(ctx, w) + return args.Get(0).(*gcs.MinObject), args.Error(1) +} + +func (m *TestifyMockBucket) FlushPendingWrites(ctx context.Context, w gcs.Writer) (*gcs.MinObject, error) { + args := m.Called(ctx, w) + return args.Get(0).(*gcs.MinObject), args.Error(1) +} + +func (m *TestifyMockBucket) CopyObject(ctx context.Context, req *gcs.CopyObjectRequest) (*gcs.Object, error) { + args := m.Called(ctx, req) + return args.Get(0).(*gcs.Object), args.Error(1) +} + +func (m *TestifyMockBucket) ComposeObjects(ctx context.Context, req *gcs.ComposeObjectsRequest) (*gcs.Object, error) { + args := m.Called(ctx, req) + return args.Get(0).(*gcs.Object), args.Error(1) +} + +func (m *TestifyMockBucket) StatObject(ctx context.Context, req *gcs.StatObjectRequest) (*gcs.MinObject, *gcs.ExtendedObjectAttributes, error) { + args := m.Called(ctx, req) + if args.Get(2) != nil { + return nil, nil, args.Error(2) + } + return args.Get(0).(*gcs.MinObject), args.Get(1).(*gcs.ExtendedObjectAttributes), nil +} + +func (m *TestifyMockBucket) ListObjects(ctx context.Context, req *gcs.ListObjectsRequest) (*gcs.Listing, error) { + args := m.Called(ctx, req) + return args.Get(0).(*gcs.Listing), args.Error(1) +} + +func (m *TestifyMockBucket) UpdateObject(ctx context.Context, req *gcs.UpdateObjectRequest) (*gcs.Object, error) { + args := m.Called(ctx, req) + return args.Get(0).(*gcs.Object), args.Error(1) +} + +func (m *TestifyMockBucket) DeleteObject(ctx context.Context, req *gcs.DeleteObjectRequest) error { + args := m.Called(ctx, req) + return args.Error(0) +} + +func (m *TestifyMockBucket) MoveObject(ctx context.Context, req *gcs.MoveObjectRequest) (*gcs.Object, error) { + args := m.Called(ctx, req) + if args.Get(0) != nil { + return args.Get(0).(*gcs.Object), nil + } + return nil, args.Error(1) +} + +func (m *TestifyMockBucket) DeleteFolder(ctx context.Context, folderName string) error { + args := m.Called(ctx, folderName) + return args.Error(0) +} + +func (m *TestifyMockBucket) GetFolder(ctx context.Context, req *gcs.GetFolderRequest) (*gcs.Folder, error) { + args := m.Called(ctx, req) + if args.Get(0) != nil { + return args.Get(0).(*gcs.Folder), nil + } + return nil, args.Error(1) +} + +func (m *TestifyMockBucket) RenameFolder(ctx context.Context, folderName string, destinationFolderId string) (*gcs.Folder, error) { + args := m.Called(ctx, folderName, destinationFolderId) + if args.Get(0) != nil { + return args.Get(0).(*gcs.Folder), nil + } + return nil, args.Error(1) +} + +func (m *TestifyMockBucket) CreateFolder(ctx context.Context, folderName string) (*gcs.Folder, error) { + args := m.Called(ctx, folderName) + if args.Get(0) != nil { + return args.Get(0).(*gcs.Folder), nil + } + return nil, args.Error(1) +} + +func (m *TestifyMockBucket) NewMultiRangeDownloader( + ctx context.Context, req *gcs.MultiRangeDownloaderRequest) (gcs.MultiRangeDownloader, error) { + args := m.Called(ctx, req) + if args.Get(0) != nil { + return args.Get(0).(gcs.MultiRangeDownloader), nil + } + return nil, args.Error(1) +} + +func (m *TestifyMockBucket) GCSName(obj *gcs.MinObject) string { + args := m.Called(obj) + return args.Get(0).(string) +} diff --git a/internal/storage/mock_bucket.go b/internal/storage/mock_bucket.go index 46309c1636..049d8b101b 100644 --- a/internal/storage/mock_bucket.go +++ b/internal/storage/mock_bucket.go @@ -8,11 +8,10 @@ package storage import ( fmt "fmt" - io "io" runtime "runtime" unsafe "unsafe" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" oglemock "github.com/jacobsa/oglemock" context "golang.org/x/net/context" ) @@ -55,7 +54,7 @@ func (m *mockBucket) ComposeObjects(p0 context.Context, p1 *gcs.ComposeObjectsRe "ComposeObjects", file, line, - []interface{}{p0, p1}) + []any{p0, p1}) if len(retVals) != 2 { panic(fmt.Sprintf("mockBucket.ComposeObjects: invalid return values: %v", retVals)) @@ -84,7 +83,7 @@ func (m *mockBucket) CopyObject(p0 context.Context, p1 *gcs.CopyObjectRequest) ( "CopyObject", file, line, - []interface{}{p0, p1}) + []any{p0, p1}) if len(retVals) != 2 { panic(fmt.Sprintf("mockBucket.CopyObject: invalid return values: %v", retVals)) @@ -113,7 +112,7 @@ func (m *mockBucket) CreateObject(p0 context.Context, p1 *gcs.CreateObjectReques "CreateObject", file, line, - []interface{}{p0, p1}) + []any{p0, p1}) if len(retVals) != 2 { panic(fmt.Sprintf("mockBucket.CreateObject: invalid return values: %v", retVals)) @@ -142,7 +141,7 @@ func (m *mockBucket) CreateObjectChunkWriter(p0 context.Context, p1 *gcs.CreateO "CreateObjectChunkWriter", file, line, - []interface{}{p0, p1, p2, p3}) + []any{p0, p1, p2, p3}) if len(retVals) != 2 { panic(fmt.Sprintf("mockBucket.CreateObjectChunkWriter: invalid return values: %v", retVals)) @@ -161,7 +160,36 @@ func (m *mockBucket) CreateObjectChunkWriter(p0 context.Context, p1 *gcs.CreateO return } -func (m *mockBucket) FinalizeUpload(p0 context.Context, p1 gcs.Writer) (o0 *gcs.Object, o1 error) { +func (m *mockBucket) CreateAppendableObjectWriter(p0 context.Context, p1 *gcs.CreateObjectChunkWriterRequest) (o0 gcs.Writer, o1 error) { + // Get a file name and line number for the caller. + _, file, line, _ := runtime.Caller(1) + + // Hand the call off to the controller, which does most of the work. + retVals := m.controller.HandleMethodCall( + m, + "CreateAppendableObjectWriter", + file, + line, + []any{p0, p1}) + + if len(retVals) != 2 { + panic(fmt.Sprintf("mockBucket.CreateAppendableObjectWriter: invalid return values: %v", retVals)) + } + + // o0 storageWriter + if retVals[0] != nil { + o0 = retVals[0].(gcs.Writer) + } + + // o1 error + if retVals[1] != nil { + o1 = retVals[1].(error) + } + + return +} + +func (m *mockBucket) FinalizeUpload(p0 context.Context, p1 gcs.Writer) (o0 *gcs.MinObject, o1 error) { // Get a file name and line number for the caller. _, file, line, _ := runtime.Caller(1) @@ -171,7 +199,7 @@ func (m *mockBucket) FinalizeUpload(p0 context.Context, p1 gcs.Writer) (o0 *gcs. "FinalizeUpload", file, line, - []interface{}{p0, p1}) + []any{p0, p1}) if len(retVals) != 2 { panic(fmt.Sprintf("mockBucket.FinalizeUpload: invalid return values: %v", retVals)) @@ -179,7 +207,35 @@ func (m *mockBucket) FinalizeUpload(p0 context.Context, p1 gcs.Writer) (o0 *gcs. // o0 *gcs.Object if retVals[0] != nil { - o0 = retVals[0].(*gcs.Object) + o0 = retVals[0].(*gcs.MinObject) + } + // o1 error + if retVals[1] != nil { + o1 = retVals[1].(error) + } + + return +} + +func (m *mockBucket) FlushPendingWrites(p0 context.Context, p1 gcs.Writer) (o0 *gcs.MinObject, o1 error) { + // Get a file name and line number for the caller. + _, file, line, _ := runtime.Caller(1) + + // Hand the call off to the controller, which does most of the work. + retVals := m.controller.HandleMethodCall( + m, + "FlushPendingWrites", + file, + line, + []any{p0, p1}) + + if len(retVals) != 2 { + panic(fmt.Sprintf("mockBucket.FlushPendingWrites: invalid return values: %v", retVals)) + } + + // o0 *gcs.MinObject + if retVals[0] != nil { + o0 = retVals[0].(*gcs.MinObject) } // o1 error if retVals[1] != nil { @@ -199,7 +255,7 @@ func (m *mockBucket) DeleteObject(p0 context.Context, p1 *gcs.DeleteObjectReques "DeleteObject", file, line, - []interface{}{p0, p1}) + []any{p0, p1}) if len(retVals) != 1 { panic(fmt.Sprintf("mockBucket.DeleteObject: invalid return values: %v", retVals)) @@ -213,6 +269,37 @@ func (m *mockBucket) DeleteObject(p0 context.Context, p1 *gcs.DeleteObjectReques return } +func (m *mockBucket) MoveObject(p0 context.Context, p1 *gcs.MoveObjectRequest) (*gcs.Object, error) { + var o0 *gcs.Object + var o1 error + // Get a file name and line number for the caller. + _, file, line, _ := runtime.Caller(1) + + // Hand the call off to the controller, which does most of the work. + retVals := m.controller.HandleMethodCall( + m, + "MoveObject", + file, + line, + []any{p0, p1}) + + if len(retVals) != 2 { + panic(fmt.Sprintf("mockBucket.MoveObject: invalid return values: %v", retVals)) + } + + // o0 *Object + if retVals[0] != nil { + o0 = retVals[0].(*gcs.Object) + } + + // o1 error + if retVals[1] != nil { + o1 = retVals[1].(error) + } + + return o0, o1 +} + func (m *mockBucket) DeleteFolder(ctx context.Context, folderName string) (o0 error) { // Get a file name and line number for the caller. _, file, line, _ := runtime.Caller(1) @@ -223,7 +310,7 @@ func (m *mockBucket) DeleteFolder(ctx context.Context, folderName string) (o0 er "DeleteFolder", file, line, - []interface{}{ctx, folderName}) + []any{ctx, folderName}) if len(retVals) != 1 { panic(fmt.Sprintf("mockBucket.DeleteFolder: invalid return values: %v", retVals)) } @@ -244,7 +331,7 @@ func (m *mockBucket) ListObjects(p0 context.Context, p1 *gcs.ListObjectsRequest) "ListObjects", file, line, - []interface{}{p0, p1}) + []any{p0, p1}) if len(retVals) != 2 { panic(fmt.Sprintf("mockBucket.ListObjects: invalid return values: %v", retVals)) @@ -273,7 +360,7 @@ func (m *mockBucket) Name() (o0 string) { "Name", file, line, - []interface{}{}) + []any{}) if len(retVals) != 1 { panic(fmt.Sprintf("mockBucket.Name: invalid return values: %v", retVals)) @@ -297,7 +384,7 @@ func (m *mockBucket) BucketType() (o0 gcs.BucketType) { "BucketType", file, line, - []interface{}{}) + []any{}) if len(retVals) != 1 { panic(fmt.Sprintf("mockBucket.BucketType: invalid return values: %v", retVals)) @@ -311,25 +398,25 @@ func (m *mockBucket) BucketType() (o0 gcs.BucketType) { return } -func (m *mockBucket) NewReader(p0 context.Context, p1 *gcs.ReadObjectRequest) (o0 io.ReadCloser, o1 error) { +func (m *mockBucket) NewReaderWithReadHandle(p0 context.Context, p1 *gcs.ReadObjectRequest) (o0 gcs.StorageReader, o1 error) { // Get a file name and line number for the caller. _, file, line, _ := runtime.Caller(1) // Hand the call off to the controller, which does most of the work. retVals := m.controller.HandleMethodCall( m, - "NewReader", + "NewReaderWithReadHandle", file, line, - []interface{}{p0, p1}) + []any{p0, p1}) if len(retVals) != 2 { - panic(fmt.Sprintf("mockBucket.NewReader: invalid return values: %v", retVals)) + panic(fmt.Sprintf("mockBucket.NewReaderWithReadHandle: invalid return values: %v", retVals)) } - // o0 io.ReadCloser + // o0 gcs.StorageReader if retVals[0] != nil { - o0 = retVals[0].(io.ReadCloser) + o0 = retVals[0].(gcs.StorageReader) } // o1 error @@ -351,7 +438,7 @@ func (m *mockBucket) StatObject(p0 context.Context, "StatObject", file, line, - []interface{}{p0, p1}) + []any{p0, p1}) if len(retVals) != 3 { panic(fmt.Sprintf("mockBucket.StatObject: invalid return values: %v", retVals)) @@ -385,7 +472,7 @@ func (m *mockBucket) UpdateObject(p0 context.Context, p1 *gcs.UpdateObjectReques "UpdateObject", file, line, - []interface{}{p0, p1}) + []any{p0, p1}) if len(retVals) != 2 { panic(fmt.Sprintf("mockBucket.UpdateObject: invalid return values: %v", retVals)) @@ -406,7 +493,7 @@ func (m *mockBucket) UpdateObject(p0 context.Context, p1 *gcs.UpdateObjectReques func (m *mockBucket) GetFolder( ctx context.Context, - prefix string) (o0 *gcs.Folder, o1 error) { + req *gcs.GetFolderRequest) (o0 *gcs.Folder, o1 error) { // Get a file name and line number for the caller. _, file, line, _ := runtime.Caller(1) @@ -416,7 +503,7 @@ func (m *mockBucket) GetFolder( "GetFolder", file, line, - []interface{}{ctx, prefix}) + []any{ctx, req}) if len(retVals) != 2 { panic(fmt.Sprintf("mockBucket.GetFolder: invalid return values: %v", retVals)) @@ -443,7 +530,7 @@ func (m *mockBucket) CreateFolder(ctx context.Context, prefix string) (o0 *gcs.F "CreateFolder", file, line, - []interface{}{ctx, prefix}) + []any{ctx, prefix}) if len(retVals) != 2 { panic(fmt.Sprintf("mockBucket.GetFolder: invalid return values: %v", retVals)) @@ -470,7 +557,7 @@ func (m *mockBucket) RenameFolder(ctx context.Context, folderName string, destin "RenameFolder", file, line, - []interface{}{ctx, folderName, destinationFolderId}) + []any{ctx, folderName, destinationFolderId}) if len(retVals) != 2 { panic(fmt.Sprintf("mockBucket.RenameFolder: invalid return values: %v", retVals)) @@ -486,3 +573,37 @@ func (m *mockBucket) RenameFolder(ctx context.Context, folderName string, destin } return } + +func (m *mockBucket) GCSName(obj *gcs.MinObject) string { + return obj.Name +} + +func (m *mockBucket) NewMultiRangeDownloader( + p0 context.Context, p1 *gcs.MultiRangeDownloaderRequest) (o0 gcs.MultiRangeDownloader, o1 error) { + // Get a file name and line number for the caller. + _, file, line, _ := runtime.Caller(1) + + // Hand the call off to the controller, which does most of the work. + retVals := m.controller.HandleMethodCall( + m, + "NewMultiRangeDownloader", + file, + line, + []any{p0, p1}) + + if len(retVals) != 2 { + panic(fmt.Sprintf("mockBucket.NewMultiRangeDownloader: invalid return values: %v", retVals)) + } + + // o0 io.ReadCloser + if retVals[0] != nil { + o0 = retVals[0].(gcs.MultiRangeDownloader) + } + + // o1 error + if retVals[1] != nil { + o1 = retVals[1].(error) + } + + return +} diff --git a/internal/storage/mock_control_client.go b/internal/storage/mock_control_client.go index d76446de5d..3c935454ce 100644 --- a/internal/storage/mock_control_client.go +++ b/internal/storage/mock_control_client.go @@ -34,7 +34,11 @@ func (m *MockStorageControlClient) GetStorageLayout(ctx context.Context, req *controlpb.GetStorageLayoutRequest, opts ...gax.CallOption) (*controlpb.StorageLayout, error) { args := m.Called(ctx, req, opts) - return args.Get(0).(*controlpb.StorageLayout), args.Error(1) + + if args[1] != nil { + return nil, args.Error(1) + } + return args.Get(0).(*controlpb.StorageLayout), nil } // Implement the DeleteFolder method for the mock. diff --git a/internal/storage/storage_handle.go b/internal/storage/storage_handle.go index 3a38fe884b..7353f93d01 100644 --- a/internal/storage/storage_handle.go +++ b/internal/storage/storage_handle.go @@ -15,19 +15,29 @@ package storage import ( + "context" + "errors" "fmt" + "net" "net/http" "os" "strconv" + "strings" + "time" "cloud.google.com/go/storage" control "cloud.google.com/go/storage/control/apiv2" + "cloud.google.com/go/storage/control/apiv2/controlpb" "cloud.google.com/go/storage/experimental" "github.com/googleapis/gax-go/v2" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" - "golang.org/x/net/context" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "go.opentelemetry.io/otel" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "golang.org/x/oauth2" option "google.golang.org/api/option" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -42,6 +52,14 @@ const ( // Ref: https://github.com/googleapis/google-cloud-go/blob/main/storage/option.go#L30 dynamicReadReqIncreaseRateEnv = "DYNAMIC_READ_REQ_INCREASE_RATE" dynamicReadReqInitialTimeoutEnv = "DYNAMIC_READ_REQ_INITIAL_TIMEOUT" + + zonalLocationType = "zone" + + // DirectPath detection parameters - used for fast-fail detection during client creation + directPathDetectionMaxAttempts = 5 + directPathDetectionTimeout = 15 * time.Second + directPathDetectionMaxBackoff = 5 * time.Second + directPathDetectionMaxRetryDuration = 1 * time.Minute ) type StorageHandle interface { @@ -50,111 +68,228 @@ type StorageHandle interface { // to that project rather than to the bucket's owning project. // // A user-project is required for all operations on Requester Pays buckets. - BucketHandle(ctx context.Context, bucketName string, billingProject string) (bh *bucketHandle) + BucketHandle(ctx context.Context, bucketName string, billingProject string) (bh *bucketHandle, err error) } type storageClient struct { - client *storage.Client - storageControlClient *control.StorageControlClient - directPathDetector *gRPCDirectPathDetector -} - -type gRPCDirectPathDetector struct { - clientOptions []option.ClientOption -} - -// isDirectPathPossible checks if gRPC direct connectivity is available for a specific bucket -// from the environment where the client is running. A `nil` error represents Direct Connectivity was -// detected. -func (pd *gRPCDirectPathDetector) isDirectPathPossible(ctx context.Context, bucketName string) error { - return storage.CheckDirectConnectivitySupported(ctx, bucketName, pd.clientOptions...) + httpClient *storage.Client + grpcClient *storage.Client + grpcClientWithBidiConfig *storage.Client + clientConfig storageutil.StorageClientConfig + // rawStorageControlClientWithoutGaxRetries is without any retries. + rawStorageControlClientWithoutGaxRetries *control.StorageControlClient + // rawStorageControlClientWithGaxRetries is with retry for Folder APIs. + rawStorageControlClientWithGaxRetries *control.StorageControlClient + // storageControlClient is with retry for GetStorageLayout and with handling for billing project. + storageControlClient StorageControlClient } // Return clientOpts for both gRPC client and control client. -func createClientOptionForGRPCClient(clientConfig *storageutil.StorageClientConfig) (clientOpts []option.ClientOption, err error) { - // Add Custom endpoint option. +func createClientOptionForGRPCClient(ctx context.Context, clientConfig *storageutil.StorageClientConfig, enableBidiConfig bool) (clientOpts []option.ClientOption, err error) { + // Add custom endpoint if provided. if clientConfig.CustomEndpoint != "" { + clientOpts = append(clientOpts, option.WithEndpoint(storageutil.StripScheme(clientConfig.CustomEndpoint))) + + // TODO(b/390799251): Check if this line can be merged with below anonymousAccess check. if clientConfig.AnonymousAccess { - clientOpts = append(clientOpts, option.WithEndpoint(storageutil.StripScheme(clientConfig.CustomEndpoint))) - // Explicitly disable auth in case of custom-endpoint, aligned with the http-client. - // TODO: to revisit here when supporting TPC for grpc client. - clientOpts = append(clientOpts, option.WithoutAuthentication()) clientOpts = append(clientOpts, option.WithGRPCDialOption(grpc.WithTransportCredentials(insecure.NewCredentials()))) - } else { - err = fmt.Errorf("GRPC client doesn't support auth for custom-endpoint. Please set anonymous-access: true via config-file.") - return } + } + + // Configure authentication. + if clientConfig.AnonymousAccess { + clientOpts = append(clientOpts, option.WithoutAuthentication()) + } else if clientConfig.EnableGoogleLibAuth { + var authOpts []option.ClientOption + authOpts, _, err = storageutil.GetClientAuthOptionsAndToken(ctx, clientConfig) + if err != nil { + return nil, fmt.Errorf("failed to get client auth options and token: %w", err) + } + clientOpts = append(clientOpts, authOpts...) } else { - if clientConfig.AnonymousAccess { - clientOpts = append(clientOpts, option.WithoutAuthentication()) - } else { - tokenSrc, tokenCreationErr := storageutil.CreateTokenSource(clientConfig) - if tokenCreationErr != nil { - err = fmt.Errorf("while fetching tokenSource: %w", tokenCreationErr) - return - } - clientOpts = append(clientOpts, option.WithTokenSource(tokenSrc)) + var tokenSrc oauth2.TokenSource + tokenSrc, err = storageutil.CreateTokenSource(clientConfig) + if err != nil { + return nil, fmt.Errorf("while fetching token source: %w", err) + } + clientOpts = append(clientOpts, option.WithTokenSource(tokenSrc)) + } + + // Additional client options. + if enableBidiConfig { + clientOpts = append(clientOpts, experimental.WithGRPCBidiReads()) + } + + if clientConfig.LocalSocketAddress != "" { + dialer := &net.Dialer{} + // The port can be 0, in which case the OS will choose a local port. + // The format of SocketAddress is expected to be IP address. + // TODO: check if this approach works for CTK or whether interface name needs to be passed. + if err := storageutil.ConfigureDialerWithLocalAddr(dialer, clientConfig.LocalSocketAddress); err != nil { + return nil, fmt.Errorf("failed to configure dialer with local-socket-address %q: %w", clientConfig.LocalSocketAddress, err) } + clientOpts = append(clientOpts, option.WithGRPCDialOption(grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { + return dialer.DialContext(ctx, "tcp", addr) + }))) + } + + if clientConfig.TracingEnabled { + clientOpts = append(clientOpts, option.WithGRPCDialOption(grpc.WithStatsHandler(otelgrpc.NewClientHandler()))) } clientOpts = append(clientOpts, option.WithGRPCConnectionPool(clientConfig.GrpcConnPoolSize)) clientOpts = append(clientOpts, option.WithUserAgent(clientConfig.UserAgent)) - // Turning off the go-sdk metrics exporter to prevent any problems. - // TODO (kislaykishore) - to revisit here for monitoring support. - clientOpts = append(clientOpts, storage.WithDisabledClientMetrics()) - return + + if clientConfig.EnableGrpcMetrics && clientConfig.IsGKE { + // Pass the OpenTelemetry MeterProvider to the Go storage client, + // using the new WithMeterProvider client option. + mp := otel.GetMeterProvider() + if sdkmp, ok := mp.(*sdkmetric.MeterProvider); ok { + // pass in if sdkmp is of type *sdkmetric.MeterProvider (not a No-op) + clientOpts = append(clientOpts, experimental.WithMeterProvider(sdkmp)) + } + } else if !clientConfig.EnableGrpcMetrics { + clientOpts = append(clientOpts, storage.WithDisabledClientMetrics()) + } + + return clientOpts, nil } -// Followed https://pkg.go.dev/cloud.google.com/go/storage#hdr-Experimental_gRPC_API to create the gRPC client. -func createGRPCClientHandle(ctx context.Context, clientConfig *storageutil.StorageClientConfig) (sc *storage.Client, err error) { - if clientConfig.ClientProtocol != cfg.GRPC { - return nil, fmt.Errorf("client-protocol requested is not GRPC: %s", clientConfig.ClientProtocol) +func setRetryConfig(ctx context.Context, sc *storage.Client, clientConfig *storageutil.StorageClientConfig) { + if sc == nil || clientConfig == nil { + logger.Fatal("setRetryConfig: Empty storage client or clientConfig") + return } + // ShouldRetry function checks if an operation should be retried based on the + // response of operation (error.Code). + // RetryAlways causes all operations to be checked for retries using + // ShouldRetry function. + // Without RetryAlways, only those operations are checked for retries which + // are idempotent. + // https://github.com/googleapis/google-cloud-go/blob/main/storage/storage.go#L1953 + retryOpts := []storage.RetryOption{storage.WithBackoff(gax.Backoff{ + Max: clientConfig.MaxRetrySleep, + Multiplier: clientConfig.RetryMultiplier, + }), + storage.WithPolicy(storage.RetryAlways), + storage.WithMaxAttempts(clientConfig.MaxRetryAttempts), + storage.WithMaxRetryDuration(0), + storage.WithErrorFunc(func(err error) bool { + return storageutil.ShouldRetryWithMonitoring(ctx, err, clientConfig.MetricHandle) + })} + + sc.SetRetry(retryOpts...) +} + +// Followed https://pkg.go.dev/cloud.google.com/go/storage#hdr-Experimental_gRPC_API to create the gRPC client. +func createGRPCClientHandle(ctx context.Context, clientConfig *storageutil.StorageClientConfig, isbucketRapid bool, enableBidiConfig bool, bucketName string, billingProject string) (*storage.Client, error) { if err := os.Setenv("GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS", "true"); err != nil { - logger.Fatal("error setting direct path env var: %v", err) + return nil, fmt.Errorf("error setting direct path env var: %w", err) } + defer unSetDirectPathEnvVariable() var clientOpts []option.ClientOption - clientOpts, err = createClientOptionForGRPCClient(clientConfig) + clientOpts, err := createClientOptionForGRPCClient(ctx, clientConfig, enableBidiConfig) if err != nil { return nil, fmt.Errorf("error in getting clientOpts for gRPC client: %w", err) } - sc, err = storage.NewGRPCClient(ctx, clientOpts...) + // Add DirectPath enforcement - client creation will fail if DirectPath is not available + clientOpts = append(clientOpts, experimental.WithDirectConnectivityEnforced()) + + sc, err := storage.NewGRPCClient(ctx, clientOpts...) if err != nil { - err = fmt.Errorf("NewGRPCClient: %w", err) + return nil, fmt.Errorf("NewGRPCClient: %w", err) + } + + // Set the production level retry config. + defer func() { + logger.Infof("Applying production retry config after DirectPath verification.") + setRetryConfig(ctx, sc, clientConfig) + }() + + // Direct-path verification is fatal for regional. Todo(b/503624405): Make it fatal for all after making the dummy-stat reliable. + if verifyErr := verifyDirectPathConnectivity(ctx, clientConfig, bucketName, sc, billingProject); verifyErr != nil { + logger.Warnf("DirectPath verification failed with error: %v", verifyErr) + if !isbucketRapid { + return nil, verifyErr + } + } else { + logger.Infof("DirectPath verification succeeded, continuing with DirectPath.") + } + + return sc, nil +} + +func verifyDirectPathConnectivity(ctx context.Context, clientConfig *storageutil.StorageClientConfig, bucketName string, sc *storage.Client, billingProject string) error { + // Verify DirectPath connection by performing an stat call on the bucket + logger.Infof("Verifying DirectPath connectivity for bucket %q with stat call", bucketName) + + var notFoundError *gcs.NotFoundError + var testObject = "gcsfuse-dp-object" + bucketHandle := sc.Bucket(bucketName) + if billingProject != "" { + bucketHandle = bucketHandle.UserProject(billingProject) + } + + dpClientConfig := &storageutil.StorageClientConfig{ + MaxRetrySleep: directPathDetectionMaxBackoff, + RetryMultiplier: clientConfig.RetryMultiplier, + MaxRetryAttempts: directPathDetectionMaxAttempts, } + retryConfig := storageutil.NewRetryConfig(dpClientConfig, directPathDetectionTimeout, directPathDetectionMaxRetryDuration, storageutil.DefaultInitialBackoff) + apiCall := func(attemptCtx context.Context) (*storage.ObjectAttrs, error) { + return bucketHandle.Object(testObject).Attrs(attemptCtx) + } + + // Disable Go SDK retries for this call to let ExecuteWithRetry handle it. + sc.SetRetry(storage.WithMaxAttempts(1)) + + _, statErr := storageutil.ExecuteWithRetryAtLogLevel(ctx, retryConfig, "Attrs", testObject, apiCall, logger.LevelInfo) + + // We should get a notFound error and not any error when the object doesn't exist. + // Any error other than notFound is treated as dp connection failure. + if statErr != nil && !errors.As(gcs.GetGCSError(statErr), ¬FoundError) { + return fmt.Errorf("DirectPath verification failed for bucket %q: %w", bucketName, statErr) + } + + return nil +} + +func unSetDirectPathEnvVariable() { // Unset the environment variable, since it's used only while creation of grpc client. if err := os.Unsetenv("GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS"); err != nil { - logger.Fatal("error while unsetting direct path env var: %v", err) + logger.Errorf("error while unsetting direct path env var: %v", err) } - - return } func createHTTPClientHandle(ctx context.Context, clientConfig *storageutil.StorageClientConfig) (sc *storage.Client, err error) { var clientOpts []option.ClientOption + var tokenSrc oauth2.TokenSource = nil - // Add WithHttpClient option. - if clientConfig.ClientProtocol == cfg.HTTP1 || clientConfig.ClientProtocol == cfg.HTTP2 { - var httpClient *http.Client - httpClient, err = storageutil.CreateHttpClient(clientConfig) + if clientConfig.AnonymousAccess { + clientOpts = append(clientOpts, option.WithoutAuthentication()) + } else if clientConfig.EnableGoogleLibAuth { + var authOpts []option.ClientOption + authOpts, tokenSrc, err = storageutil.GetClientAuthOptionsAndToken(ctx, clientConfig) if err != nil { - err = fmt.Errorf("while creating http endpoint: %w", err) - return + return nil, fmt.Errorf("failed to get client auth options and token: %w", err) } - - clientOpts = append(clientOpts, option.WithHTTPClient(httpClient)) - } else { - return nil, fmt.Errorf("client-protocol requested is not HTTP1 or HTTP2: %s", clientConfig.ClientProtocol) + clientOpts = append(clientOpts, authOpts...) } - if clientConfig.AnonymousAccess { - clientOpts = append(clientOpts, option.WithoutAuthentication()) + // Add WithHttpClient option. + var httpClient *http.Client + httpClient, err = storageutil.CreateHttpClient(clientConfig, tokenSrc) + if err != nil { + err = fmt.Errorf("while creating http endpoint: %w", err) + return } + clientOpts = append(clientOpts, option.WithHTTPClient(httpClient)) + // Create client with JSON read flow, if EnableJasonRead flag is set. if clientConfig.ExperimentalEnableJsonRead { clientOpts = append(clientOpts, storage.WithJSONReads()) @@ -188,90 +323,212 @@ func createHTTPClientHandle(ctx context.Context, clientConfig *storageutil.Stora TargetPercentile: clientConfig.ReadStallRetryConfig.ReqTargetPercentile, })) } - return storage.NewClient(ctx, clientOpts...) + sc, err = storage.NewClient(ctx, clientOpts...) + if err != nil { + err = fmt.Errorf("go http storage client creation failed: %w", err) + return + } + setRetryConfig(ctx, sc, clientConfig) + return } -// NewStorageHandle returns the handle of http or grpc Go storage client based on the -// provided StorageClientConfig.ClientProtocol. -// Please check out the StorageClientConfig to know about the parameters used in -// http and gRPC client. -func NewStorageHandle(ctx context.Context, clientConfig storageutil.StorageClientConfig) (sh StorageHandle, err error) { - var sc *storage.Client +func (sh *storageClient) lookupBucketType(bucketName string) (*gcs.BucketType, error) { + if sh.storageControlClient == nil { + return &gcs.BucketType{}, nil // Assume defaults + } + + startTime := time.Now() + logger.Infof("GetStorageLayout <- (%s)", bucketName) + storageLayout, err := sh.getStorageLayout(bucketName) + duration := time.Since(startTime) + + if err != nil { + return nil, err + } + + logger.Infof("GetStorageLayout -> (%s) %v msec", bucketName, duration.Milliseconds()) + + pirloState := gcs.PirloStateNone + if sh.clientConfig.ExperimentalEnablePirlo { + if sh.clientConfig.WriteConfig != nil && sh.clientConfig.WriteConfig.EnableRapidWrites { + pirloState = gcs.PirloStateRapidWritesEnabled + } else { + pirloState = gcs.PirloStateRapidWritesDisabled + } + } + + return &gcs.BucketType{ + Hierarchical: storageLayout.GetHierarchicalNamespace().GetEnabled(), + Zonal: storageLayout.GetLocationType() == zonalLocationType, + Pirlo: pirloState, + }, nil +} + +func (sh *storageClient) getStorageLayout(bucketName string) (*controlpb.StorageLayout, error) { + var callOptions []gax.CallOption + stoargeLayout, err := sh.storageControlClient.GetStorageLayout(context.Background(), &controlpb.GetStorageLayoutRequest{ + Name: fmt.Sprintf("projects/_/buckets/%s/storageLayout", bucketName), + Prefix: "", + RequestId: "", + }, callOptions...) + + return stoargeLayout, err +} + +// NewStorageHandle creates control client and stores client config to allow dynamic +// creation of http or grpc client. +func NewStorageHandle(ctx context.Context, clientConfig storageutil.StorageClientConfig, billingProject string) (sh StorageHandle, err error) { // The default protocol for the Go Storage control client's folders API is gRPC. // gcsfuse will initially mirror this behavior due to the client's lack of HTTP support. - var controlClient *control.StorageControlClient + var controlClient StorageControlClient + var rawStorageControlClientWithoutGaxRetries *control.StorageControlClient + var rawStorageControlClientWithGaxRetries *control.StorageControlClient var clientOpts []option.ClientOption - var directPathDetector *gRPCDirectPathDetector - if clientConfig.ClientProtocol == cfg.GRPC { - sc, err = createGRPCClientHandle(ctx, &clientConfig) - if err == nil { - clientOpts, err = createClientOptionForGRPCClient(&clientConfig) - directPathDetector = &gRPCDirectPathDetector{clientOptions: clientOpts} + + // Control-client is needed for folder APIs and for getting storage-layout of the bucket. + // GetStorageLayout API is not supported for storage-testbench, which are identified by custom-endpoint containing localhost. + if clientConfig.EnableHNS && !strings.Contains(clientConfig.CustomEndpoint, "localhost") { + // For control client, we don't pass billingProject to avoid setting it globally via option.WithQuotaProject. + // The wrapper storageControlClientWithBillingProject will manually add it to the context for supported calls. + clientOpts, err = createClientOptionForGRPCClient(ctx, &clientConfig, false) + if err != nil { + return nil, fmt.Errorf("error in getting clientOpts for gRPC client: %w", err) + } + rawStorageControlClientWithoutGaxRetries, err = storageutil.CreateGRPCControlClient(ctx, clientOpts, true) + if err != nil { + return nil, fmt.Errorf("could not create StorageControl Client without default gax retries: %w", err) } - } else if clientConfig.ClientProtocol == cfg.HTTP1 || clientConfig.ClientProtocol == cfg.HTTP2 { - sc, err = createHTTPClientHandle(ctx, &clientConfig) + // rawStorageControlClientWithGaxRetries cannot be just a wrapper over rawStorageControlClientWithoutGaxRetries, + // as it has its own dedicated array of CallOptions, and we need to keep those independent. + rawStorageControlClientWithGaxRetries, err = storageutil.CreateGRPCControlClient(ctx, clientOpts, false) + if err != nil { + return nil, fmt.Errorf("could not create StorageControl Client with default gax retries: %w", err) + } + err = addGaxRetriesForFolderAPIs(rawStorageControlClientWithGaxRetries, &clientConfig) + if err != nil { + return nil, fmt.Errorf("could not add custom gax retries to StorageControl Client: %w", err) + } + // special handling for mounts created with custom billing projects. + controlClientWithBillingProject := withBillingProject(rawStorageControlClientWithoutGaxRetries, billingProject) + // Wrap the control client with retry-on-stall logic. + // This will retry on only on GetStorageLayout call for all buckets. + controlClient = withRetryOnStorageLayout(controlClientWithBillingProject, &clientConfig) } else { - err = fmt.Errorf("invalid client-protocol requested: %s", clientConfig.ClientProtocol) + logger.Infof("Skipping storage control client creation because custom-endpoint %q was passed, which is assumed to be a storage testbench server because of 'localhost' in it.", clientConfig.CustomEndpoint) } - if err != nil { - err = fmt.Errorf("go storage client creation failed: %w", err) - return + sh = &storageClient{ + rawStorageControlClientWithoutGaxRetries: rawStorageControlClientWithoutGaxRetries, + rawStorageControlClientWithGaxRetries: rawStorageControlClientWithGaxRetries, + storageControlClient: controlClient, + clientConfig: clientConfig, } + return +} - // TODO: We will implement an additional check for the HTTP control client protocol once the Go SDK supports HTTP. - // TODO: Custom endpoints do not currently support gRPC. Remove this additional check once TPC(custom-endpoint) supports gRPC. - if clientConfig.EnableHNS && clientConfig.CustomEndpoint == "" { - clientOpts, err = createClientOptionForGRPCClient(&clientConfig) - if err != nil { - return nil, fmt.Errorf("error in getting clientOpts for gRPC client: %w", err) +func (sh *storageClient) getClient(ctx context.Context, isBucketRapid bool, bucketName string, billingProject string) (*storage.Client, error) { + var err error + if isBucketRapid { + if sh.grpcClientWithBidiConfig == nil { + sh.grpcClientWithBidiConfig, err = createGRPCClientHandle(ctx, &sh.clientConfig, isBucketRapid, true, bucketName, billingProject) } - controlClient, err = storageutil.CreateGRPCControlClient(ctx, clientOpts, &clientConfig) - if err != nil { - return nil, fmt.Errorf("could not create StorageControl Client: %w", err) + return sh.grpcClientWithBidiConfig, err + } + + if sh.clientConfig.ClientProtocol == cfg.GRPC { + return sh.createNonBidiGRPCClientWithHttpFallback(ctx, bucketName, billingProject) + } + + if sh.clientConfig.ClientProtocol == cfg.HTTP1 || sh.clientConfig.ClientProtocol == cfg.HTTP2 { + if sh.httpClient == nil { + sh.httpClient, err = createHTTPClientHandle(ctx, &sh.clientConfig) } + return sh.httpClient, err } - // ShouldRetry function checks if an operation should be retried based on the - // response of operation (error.Code). - // RetryAlways causes all operations to be checked for retries using - // ShouldRetry function. - // Without RetryAlways, only those operations are checked for retries which - // are idempotent. - // https://github.com/googleapis/google-cloud-go/blob/main/storage/storage.go#L1953 - sc.SetRetry( - storage.WithBackoff(gax.Backoff{ - Max: clientConfig.MaxRetrySleep, - Multiplier: clientConfig.RetryMultiplier, - }), - storage.WithPolicy(storage.RetryAlways), - storage.WithErrorFunc(storageutil.ShouldRetry)) + return nil, fmt.Errorf("invalid client-protocol requested: %s", sh.clientConfig.ClientProtocol) +} - // The default MaxRetryAttempts value is 0 indicates no limit. - if clientConfig.MaxRetryAttempts != 0 { - sc.SetRetry(storage.WithMaxAttempts(clientConfig.MaxRetryAttempts)) +func (sh *storageClient) createNonBidiGRPCClientWithHttpFallback(ctx context.Context, bucketName string, billingProject string) (*storage.Client, error) { + if sh.grpcClient != nil { + return sh.grpcClient, nil } - sh = &storageClient{client: sc, storageControlClient: controlClient, directPathDetector: directPathDetector} - return + var err error + sh.grpcClient, err = createGRPCClientHandle(ctx, &sh.clientConfig, false, false, bucketName, billingProject) + // No error means we are able to successfully create a grpc client with direct path. Return it. + if err == nil { + return sh.grpcClient, nil + } + + // We will reach here when we failed to create a grpc client with direct path. + // Decide whether to create a http client based on grpPathStrategy param. + if sh.clientConfig.GrpcPathStrategy == cfg.DirectPathOnly { + logger.Infof("Grpc dp is not available and not falling back to Http as gRPC path strategy is set to DirectPathOnly") + return nil, err + } + + // When grpcPathStrategy=DirectPathWithFallback, create a http client. + logger.Infof("Grpc dp is not available and falling back to Http.") + if sh.httpClient == nil { + sh.httpClient, err = createHTTPClientHandle(ctx, &sh.clientConfig) + } + + return sh.httpClient, err +} + +// controlClientForBucketHandle returns a storage control client for the given bucket handle, +// which takes care of properly adding support for retries and for billing project. +func (sh *storageClient) controlClientForBucketHandle(bucketType *gcs.BucketType, billingProject string) StorageControlClient { + if sh.rawStorageControlClientWithGaxRetries == nil || sh.rawStorageControlClientWithoutGaxRetries == nil { + return nil + } + + var controlClientWithoutBillingProject StorageControlClient + if bucketType.IsRapid() || sh.clientConfig.ExperimentalNonrapidFolderApiStallRetry { + // sh.storageControlClient already contains handling for billing project, + // and enhanced retries for GetStorageLayout API call. Extending it here for + // retries for folder APIs. + // For rapid buckets, wrap the control client with retry-on-all-APIs. + controlClientWithoutBillingProject = withRetryOnAllAPIs(sh.rawStorageControlClientWithoutGaxRetries, &sh.clientConfig) + } else { + // Apply GAX retries to the raw storage control client and returns a copy of it, + // as it is important to avoid overwriting it, + // as it is used with enhanced retries used by zonal buckets. + controlClientWithoutBillingProject = withRetryOnStorageLayout(sh.rawStorageControlClientWithGaxRetries, &sh.clientConfig) + } + + // Special handling for mounts created with custom billing projects. + // Wrap it with billing-project, if there is any. + return withBillingProject(controlClientWithoutBillingProject, billingProject) } -func (sh *storageClient) BucketHandle(ctx context.Context, bucketName string, billingProject string) (bh *bucketHandle) { - storageBucketHandle := sh.client.Bucket(bucketName) +func (sh *storageClient) BucketHandle(ctx context.Context, bucketName string, billingProject string) (bh *bucketHandle, err error) { + var client *storage.Client + bucketType, err := sh.lookupBucketType(bucketName) + if err != nil { + return nil, fmt.Errorf("storageLayout call failed: %s", err) + } + client, err = sh.getClient(ctx, bucketType.IsRapid(), bucketName, billingProject) + if err != nil { + return nil, err + } + + storageBucketHandle := client.Bucket(bucketName) if billingProject != "" { storageBucketHandle = storageBucketHandle.UserProject(billingProject) } + controlClient := sh.controlClientForBucketHandle(bucketType, billingProject) bh = &bucketHandle{ - bucket: storageBucketHandle, - bucketName: bucketName, - controlClient: sh.storageControlClient, - } - if sh.directPathDetector != nil { - if err := sh.directPathDetector.isDirectPathPossible(ctx, bucketName); err != nil { - logger.Warnf("Direct path connectivity unavailable for %s, reason: %v", bucketName, err) - } + bucket: storageBucketHandle, + bucketName: bucketName, + controlClient: controlClient, + bucketType: bucketType, + billingProject: billingProject, + writeConfig: sh.clientConfig.WriteConfig, } + return } diff --git a/internal/storage/storage_handle_test.go b/internal/storage/storage_handle_test.go index a3ded1e24c..49a3bd7fd3 100644 --- a/internal/storage/storage_handle_test.go +++ b/internal/storage/storage_handle_test.go @@ -17,25 +17,54 @@ package storage import ( "context" "fmt" + "net" "net/url" + "os" "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + control "cloud.google.com/go/storage/control/apiv2" + "cloud.google.com/go/storage/control/apiv2/controlpb" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "go.opentelemetry.io/otel" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "google.golang.org/grpc" + "google.golang.org/grpc/peer" ) const invalidBucketName string = "will-not-be-present-in-fake-server" const projectID string = "valid-project-id" +var keyFile = "storageutil/testdata/key.json" + +// A fake implementation of control.StorageControlServer for testing. +type fakeStorageControlServer struct { + controlpb.UnimplementedStorageControlServer + // Last received request's peer address. + remoteAddr net.Addr +} + +func (s *fakeStorageControlServer) CreateFolder(ctx context.Context, in *controlpb.CreateFolderRequest) (*controlpb.Folder, error) { + p, ok := peer.FromContext(ctx) + if !ok { + return nil, fmt.Errorf("peer not found") + } + s.remoteAddr = p.Addr + return &controlpb.Folder{}, nil +} + type StorageHandleTest struct { suite.Suite - fakeStorage FakeStorage - ctx context.Context + fakeStorage FakeStorage + mockClient *MockStorageControlClient + clientConfig *storageutil.StorageClientConfig + ctx context.Context } func TestStorageHandleTestSuite(t *testing.T) { @@ -43,7 +72,10 @@ func TestStorageHandleTestSuite(t *testing.T) { } func (testSuite *StorageHandleTest) SetupTest() { - testSuite.fakeStorage = NewFakeStorage() + testSuite.mockClient = new(MockStorageControlClient) + sc := storageutil.GetDefaultStorageClientConfig("") + testSuite.clientConfig = &sc + testSuite.fakeStorage = NewFakeStorageWithMockClient(testSuite.mockClient, cfg.HTTP2) testSuite.ctx = context.Background() } @@ -51,74 +83,145 @@ func (testSuite *StorageHandleTest) TearDownTest() { testSuite.fakeStorage.ShutDown() } +func (testSuite *StorageHandleTest) mockStorageLayout(bucketType gcs.BucketType) { + storageLayout := &controlpb.StorageLayout{ + HierarchicalNamespace: &controlpb.StorageLayout_HierarchicalNamespace{Enabled: false}, + LocationType: "nil", + } + + if bucketType.Zonal { + storageLayout.HierarchicalNamespace = &controlpb.StorageLayout_HierarchicalNamespace{Enabled: true} + storageLayout.LocationType = "zone" + } + + if bucketType.Hierarchical { + storageLayout.HierarchicalNamespace = &controlpb.StorageLayout_HierarchicalNamespace{Enabled: true} + storageLayout.LocationType = "multiregion" + } + + testSuite.mockClient.On("GetStorageLayout", mock.Anything, mock.Anything, mock.Anything).Return(storageLayout, nil) +} + +// Helpers + +func (testSuite *StorageHandleTest) controlClientCallOptionsWithoutRetry() *control.StorageControlCallOptions { + testSuite.T().Helper() + return &control.StorageControlCallOptions{} +} + +func (testSuite *StorageHandleTest) controlClientCallOptionsWithRetry() *control.StorageControlCallOptions { + testSuite.T().Helper() + clientConfig := &storageutil.StorageClientConfig{MaxRetrySleep: 100 * time.Microsecond, MaxRetryAttempts: 5} + gaxRetryOptions := storageControlClientGaxRetryOptions(clientConfig) + return &control.StorageControlCallOptions{ + CreateFolder: gaxRetryOptions, + GetFolder: gaxRetryOptions, + DeleteFolder: gaxRetryOptions, + RenameFolder: gaxRetryOptions, + } +} + +// Test functions + func (testSuite *StorageHandleTest) TestBucketHandleWhenBucketExistsWithEmptyBillingProject() { storageHandle := testSuite.fakeStorage.CreateStorageHandle() - bucketHandle := storageHandle.BucketHandle(testSuite.ctx, TestBucketName, "") + testSuite.mockStorageLayout(gcs.BucketType{}) + bucketHandle, err := storageHandle.BucketHandle(testSuite.ctx, TestBucketName, "") assert.NotNil(testSuite.T(), bucketHandle) + assert.Nil(testSuite.T(), err) assert.Equal(testSuite.T(), TestBucketName, bucketHandle.bucketName) - assert.Equal(testSuite.T(), gcs.Nil, bucketHandle.bucketType) + assert.False(testSuite.T(), bucketHandle.bucketType.Zonal) + assert.False(testSuite.T(), bucketHandle.bucketType.Hierarchical) } func (testSuite *StorageHandleTest) TestBucketHandleWhenBucketDoesNotExistWithEmptyBillingProject() { storageHandle := testSuite.fakeStorage.CreateStorageHandle() - bucketHandle := storageHandle.BucketHandle(testSuite.ctx, invalidBucketName, "") + testSuite.mockClient.On("GetStorageLayout", mock.Anything, mock.Anything, mock.Anything). + Return(nil, fmt.Errorf("bucket does not exist")) + bucketHandle, err := storageHandle.BucketHandle(testSuite.ctx, invalidBucketName, "") - assert.Nil(testSuite.T(), bucketHandle.Bucket) + assert.NotNil(testSuite.T(), err) + assert.Nil(testSuite.T(), bucketHandle) } func (testSuite *StorageHandleTest) TestBucketHandleWhenBucketExistsWithNonEmptyBillingProject() { storageHandle := testSuite.fakeStorage.CreateStorageHandle() - bucketHandle := storageHandle.BucketHandle(testSuite.ctx, TestBucketName, projectID) + testSuite.mockStorageLayout(gcs.BucketType{Hierarchical: true}) + + bucketHandle, err := storageHandle.BucketHandle(testSuite.ctx, TestBucketName, projectID) assert.NotNil(testSuite.T(), bucketHandle) + assert.Nil(testSuite.T(), err) assert.Equal(testSuite.T(), TestBucketName, bucketHandle.bucketName) - assert.Equal(testSuite.T(), gcs.Nil, bucketHandle.bucketType) + assert.False(testSuite.T(), bucketHandle.bucketType.Zonal) + assert.True(testSuite.T(), bucketHandle.bucketType.Hierarchical) + // verify the billing account set. + testHandle := bucketHandle + assert.Equal(testSuite.T(), bucketHandle.bucket, testHandle.bucket.UserProject(projectID)) } func (testSuite *StorageHandleTest) TestBucketHandleWhenBucketDoesNotExistWithNonEmptyBillingProject() { storageHandle := testSuite.fakeStorage.CreateStorageHandle() - bucketHandle := storageHandle.BucketHandle(testSuite.ctx, invalidBucketName, projectID) + testSuite.mockClient.On("GetStorageLayout", mock.Anything, mock.Anything, mock.Anything). + Return(nil, fmt.Errorf("bucket does not exist")) + bucketHandle, err := storageHandle.BucketHandle(testSuite.ctx, invalidBucketName, projectID) - assert.Nil(testSuite.T(), bucketHandle.Bucket) + assert.Nil(testSuite.T(), bucketHandle) + assert.NotNil(testSuite.T(), err) +} + +func (testSuite *StorageHandleTest) TestLookupBucketType_PirloEnabled() { + sc := storageutil.GetDefaultStorageClientConfig(keyFile) + sc.ExperimentalEnablePirlo = true + sc.WriteConfig = &cfg.WriteConfig{EnableRapidWrites: true} + sh, err := NewStorageHandle(testSuite.ctx, sc, "") + require.NoError(testSuite.T(), err) + client := sh.(*storageClient) + client.storageControlClient = testSuite.mockClient + testSuite.mockStorageLayout(gcs.BucketType{Zonal: true}) + + bt, err := client.lookupBucketType(TestBucketName) + + assert.NoError(testSuite.T(), err) + assert.Equal(testSuite.T(), gcs.PirloStateRapidWritesEnabled, bt.Pirlo) } func (testSuite *StorageHandleTest) TestNewStorageHandleHttp2Disabled() { - sc := storageutil.GetDefaultStorageClientConfig() // by default http1 enabled + sc := storageutil.GetDefaultStorageClientConfig(keyFile) // by default http1 enabled - handleCreated, err := NewStorageHandle(testSuite.ctx, sc) + handleCreated, err := NewStorageHandle(testSuite.ctx, sc, "") assert.Nil(testSuite.T(), err) assert.NotNil(testSuite.T(), handleCreated) } func (testSuite *StorageHandleTest) TestNewStorageHandleHttp2EnabledAndAuthEnabled() { - sc := storageutil.GetDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig(keyFile) sc.ClientProtocol = cfg.HTTP2 sc.AnonymousAccess = false - handleCreated, err := NewStorageHandle(testSuite.ctx, sc) + handleCreated, err := NewStorageHandle(testSuite.ctx, sc, "") - assert.NotNil(testSuite.T(), err) - assert.Contains(testSuite.T(), err.Error(), "no such file or directory") - assert.Nil(testSuite.T(), handleCreated) + assert.NoError(testSuite.T(), err) + assert.NotNil(testSuite.T(), handleCreated) } func (testSuite *StorageHandleTest) TestNewStorageHandleWithZeroMaxConnsPerHost() { - sc := storageutil.GetDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig(keyFile) sc.MaxConnsPerHost = 0 - handleCreated, err := NewStorageHandle(testSuite.ctx, sc) + handleCreated, err := NewStorageHandle(testSuite.ctx, sc, "") assert.Nil(testSuite.T(), err) assert.NotNil(testSuite.T(), handleCreated) } func (testSuite *StorageHandleTest) TestNewStorageHandleWhenUserAgentIsSet() { - sc := storageutil.GetDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig(keyFile) sc.UserAgent = "gcsfuse/unknown (Go version go1.20-pre3 cl/474093167 +a813be86df) appName (GPN:Gcsfuse-DLC)" - handleCreated, err := NewStorageHandle(testSuite.ctx, sc) + handleCreated, err := NewStorageHandle(testSuite.ctx, sc, "") assert.Nil(testSuite.T(), err) assert.NotNil(testSuite.T(), handleCreated) @@ -127,139 +230,147 @@ func (testSuite *StorageHandleTest) TestNewStorageHandleWhenUserAgentIsSet() { func (testSuite *StorageHandleTest) TestNewStorageHandleWithCustomEndpointAndAuthEnabled() { url, err := url.Parse(storageutil.CustomEndpoint) assert.Nil(testSuite.T(), err) - sc := storageutil.GetDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig(keyFile) sc.CustomEndpoint = url.String() sc.AnonymousAccess = false - handleCreated, err := NewStorageHandle(testSuite.ctx, sc) + handleCreated, err := NewStorageHandle(testSuite.ctx, sc, "") - assert.NotNil(testSuite.T(), err) - assert.Contains(testSuite.T(), err.Error(), "no such file or directory") - assert.Nil(testSuite.T(), handleCreated) + assert.NoError(testSuite.T(), err) + assert.NotNil(testSuite.T(), handleCreated) } // This will fail while fetching the token-source, since key-file doesn't exist. func (testSuite *StorageHandleTest) TestNewStorageHandleWhenCustomEndpointIsNilAndAuthEnabled() { - sc := storageutil.GetDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig(keyFile) sc.CustomEndpoint = "" sc.AnonymousAccess = false - handleCreated, err := NewStorageHandle(testSuite.ctx, sc) + handleCreated, err := NewStorageHandle(testSuite.ctx, sc, "") - assert.NotNil(testSuite.T(), err) - assert.Contains(testSuite.T(), err.Error(), "no such file or directory") - assert.Nil(testSuite.T(), handleCreated) + assert.NoError(testSuite.T(), err) + assert.NotNil(testSuite.T(), handleCreated) } -func (testSuite *StorageHandleTest) TestNewStorageHandleWhenKeyFileIsEmpty() { - sc := storageutil.GetDefaultStorageClientConfig() +func (testSuite *StorageHandleTest) TestNewStorageHandleWhenAnonymousAccessTrue() { + sc := storageutil.GetDefaultStorageClientConfig(keyFile) sc.KeyFile = "" + sc.AnonymousAccess = true - handleCreated, err := NewStorageHandle(testSuite.ctx, sc) + handleCreated, err := NewStorageHandle(testSuite.ctx, sc, "") assert.Nil(testSuite.T(), err) assert.NotNil(testSuite.T(), handleCreated) } func (testSuite *StorageHandleTest) TestNewStorageHandleWhenReuseTokenUrlFalse() { - sc := storageutil.GetDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig(keyFile) sc.ReuseTokenFromUrl = false - handleCreated, err := NewStorageHandle(testSuite.ctx, sc) + handleCreated, err := NewStorageHandle(testSuite.ctx, sc, "") assert.Nil(testSuite.T(), err) assert.NotNil(testSuite.T(), handleCreated) } func (testSuite *StorageHandleTest) TestNewStorageHandleWhenTokenUrlIsSet() { - sc := storageutil.GetDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig(keyFile) sc.TokenUrl = storageutil.CustomTokenUrl - handleCreated, err := NewStorageHandle(testSuite.ctx, sc) + handleCreated, err := NewStorageHandle(testSuite.ctx, sc, "") assert.Nil(testSuite.T(), err) assert.NotNil(testSuite.T(), handleCreated) } func (testSuite *StorageHandleTest) TestNewStorageHandleWhenJsonReadEnabled() { - sc := storageutil.GetDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig(keyFile) sc.ExperimentalEnableJsonRead = true - handleCreated, err := NewStorageHandle(testSuite.ctx, sc) + handleCreated, err := NewStorageHandle(testSuite.ctx, sc, "") assert.Nil(testSuite.T(), err) assert.NotNil(testSuite.T(), handleCreated) } -func (testSuite *StorageHandleTest) TestNewStorageHandleWithInvalidClientProtocol() { - sc := storageutil.GetDefaultStorageClientConfig() - sc.ExperimentalEnableJsonRead = true - sc.ClientProtocol = "test-protocol" +func (testSuite *StorageHandleTest) TestNewStorageHandleWithoutBillingProject() { + // Arrange. + sc := storageutil.GetDefaultStorageClientConfig(keyFile) + sc.EnableHNS = true - handleCreated, err := NewStorageHandle(testSuite.ctx, sc) + // Act. + handleCreated, err := NewStorageHandle(testSuite.ctx, sc, "") - assert.NotNil(testSuite.T(), err) - assert.Nil(testSuite.T(), handleCreated) - assert.Contains(testSuite.T(), err.Error(), "invalid client-protocol requested: test-protocol") + // Assert. + assert.Nil(testSuite.T(), err) + assert.NotNil(testSuite.T(), handleCreated) + storageClient, ok := handleCreated.(*storageClient) + assert.NotNil(testSuite.T(), storageClient) + assert.True(testSuite.T(), ok) + // Confirm that the returned storage-handle's control-client is of type storageControlClientWithRetry + retrierControlClient, ok := storageClient.storageControlClient.(*storageControlClientWithRetry) + require.True(testSuite.T(), ok, "retrierControlClient should be of type *storageControlClientWithRetry") + require.NotNil(testSuite.T(), retrierControlClient, "retrierControlClient should not be nil") + assert.True(testSuite.T(), retrierControlClient.enableRetriesOnStorageLayoutAPI, "enableRetriesOnStorageLayoutAPI should be true") + assert.False(testSuite.T(), retrierControlClient.enableRetriesOnFolderAPIs, "enableRetriesOnFolderAPIs should be false") + // Confirm that it has no underlying storageControlClientWithBillingProject in it. + _, ok = retrierControlClient.raw.(*storageControlClientWithBillingProject) + assert.False(testSuite.T(), ok, "raw should be of type *storageControlClientWithBillingProject") } -func (testSuite *StorageHandleTest) TestNewStorageHandleDirectPathDetector() { - testCases := []struct { - name string - clientProtocol cfg.Protocol - expectDirectPathDetector bool - }{ - { - name: "grpcWithNonNilDirectPathDetector", - clientProtocol: cfg.GRPC, - expectDirectPathDetector: true, - }, - { - name: "http1WithNilDirectPathDetector", - clientProtocol: cfg.HTTP1, - expectDirectPathDetector: false, - }, - { - name: "http2WithNilDirectPathDetector", - clientProtocol: cfg.HTTP2, - expectDirectPathDetector: false, - }, - } +func (testSuite *StorageHandleTest) TestNewStorageHandleWithBillingProject() { + // Arrange. + sc := storageutil.GetDefaultStorageClientConfig(keyFile) + sc.EnableHNS = true - for _, tc := range testCases { - testSuite.Run(tc.name, func() { - sc := storageutil.GetDefaultStorageClientConfig() - sc.ExperimentalEnableJsonRead = true - sc.ClientProtocol = tc.clientProtocol + // Act. + handleCreated, err := NewStorageHandle(testSuite.ctx, sc, projectID) - handleCreated, err := NewStorageHandle(testSuite.ctx, sc) - assert.Nil(testSuite.T(), err) - assert.NotNil(testSuite.T(), handleCreated) + // Assert. + assert.Nil(testSuite.T(), err) + assert.NotNil(testSuite.T(), handleCreated) + storageClient, ok := handleCreated.(*storageClient) + assert.NotNil(testSuite.T(), storageClient) + assert.True(testSuite.T(), ok) + retrierControlClient := storageClient.storageControlClient.(*storageControlClientWithRetry) + // Confirm that the returned storage-handle's control-client is of type storageControlClientWithBillingProject + // and its billing-project is same as the one passed while + // creating the storage-handle. + // Check that storageControlClient is wrapped correctly and billing project is set. + billingProjectControlClient, ok := retrierControlClient.raw.(*storageControlClientWithBillingProject) + require.True(testSuite.T(), ok, "raw should be of type *storageControlClientWithBillingProject") + require.NotNil(testSuite.T(), billingProjectControlClient, "storageControlClientWithBillingProject should not be nil") + assert.Equal(testSuite.T(), projectID, billingProjectControlClient.billingProject, "billingProject should match the provided projectID") + assert.NotNil(testSuite.T(), billingProjectControlClient.raw, "raw client inside storageControlClientWithBillingProject should not be nil") +} - storageClient, ok := handleCreated.(*storageClient) - assert.True(testSuite.T(), ok) +func (testSuite *StorageHandleTest) TestNewStorageHandleWithInvalidClientProtocol() { + fakeStorage := NewFakeStorageWithMockClient(testSuite.mockClient, "test-protocol") + testSuite.mockStorageLayout(gcs.BucketType{}) + sh := fakeStorage.CreateStorageHandle() + defer fakeStorage.ShutDown() + assert.NotNil(testSuite.T(), sh) + bh, err := sh.BucketHandle(testSuite.ctx, TestBucketName, projectID) - if tc.expectDirectPathDetector { - assert.NotNil(testSuite.T(), storageClient.directPathDetector) - } else { - assert.Nil(testSuite.T(), storageClient.directPathDetector) - } - }) - } + assert.Nil(testSuite.T(), bh) + assert.NotNil(testSuite.T(), err) + assert.Contains(testSuite.T(), err.Error(), "invalid client-protocol requested: test-protocol") } -func (testSuite *StorageHandleTest) TestCreateGRPCClientHandle() { - sc := storageutil.GetDefaultStorageClientConfig() - sc.ClientProtocol = cfg.GRPC +func (testSuite *StorageHandleTest) TestUnSetDirectPathEnvVariable() { + // Set the environment variable + testSuite.T().Setenv("GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS", "true") - storageClient, err := createGRPCClientHandle(testSuite.ctx, &sc) + // Call the function + unSetDirectPathEnvVariable() - assert.Nil(testSuite.T(), err) - assert.NotNil(testSuite.T(), storageClient) + // Verify the environment variable is unset + _, isSet := os.LookupEnv("GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS") + assert.False(testSuite.T(), isSet) } func (testSuite *StorageHandleTest) TestCreateHTTPClientHandle() { - sc := storageutil.GetDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig(keyFile) storageClient, err := createHTTPClientHandle(testSuite.ctx, &sc) @@ -267,36 +378,79 @@ func (testSuite *StorageHandleTest) TestCreateHTTPClientHandle() { assert.NotNil(testSuite.T(), storageClient) } -func (testSuite *StorageHandleTest) TestNewStorageHandleWithGRPCClientProtocol() { - sc := storageutil.GetDefaultStorageClientConfig() - sc.ClientProtocol = cfg.GRPC +func (testSuite *StorageHandleTest) TestCreateHTTPClientHandleWithAnonymousAccess() { + sc := storageutil.GetDefaultStorageClientConfig("incorrect_path") + sc.AnonymousAccess = true - storageClient, err := NewStorageHandle(testSuite.ctx, sc) + storageClient, err := createHTTPClientHandle(testSuite.ctx, &sc) assert.Nil(testSuite.T(), err) assert.NotNil(testSuite.T(), storageClient) } -func (testSuite *StorageHandleTest) TestCreateGRPCClientHandle_WithHTTPClientProtocol() { - sc := storageutil.GetDefaultStorageClientConfig() - sc.ClientProtocol = cfg.HTTP1 +func (testSuite *StorageHandleTest) TestCreateGRPCClientWithSocketAddress() { + // Start a local server to inspect incoming connections. + server := &fakeStorageControlServer{} + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(testSuite.T(), err) + serveErr := make(chan error, 1) + grpcServer := grpc.NewServer() + controlpb.RegisterStorageControlServer(grpcServer, server) + go func() { + serveErr <- grpcServer.Serve(listener) + }() + + defer grpcServer.Stop() + // Configure the client to use a specific local IP address. + testSuite.clientConfig.CustomEndpoint = listener.Addr().String() + testSuite.clientConfig.LocalSocketAddress = "127.0.0.1" + testSuite.clientConfig.AnonymousAccess = true + ctx := context.Background() + clientOpts, err := createClientOptionForGRPCClient(ctx, testSuite.clientConfig, false) + require.NoError(testSuite.T(), err) + controlClient, err := storageutil.CreateGRPCControlClient(ctx, clientOpts, false) + require.NoError(testSuite.T(), err) + require.NotNil(testSuite.T(), controlClient) + + // Have the client connect to the test server. + // This will not fail, as we have implemented the "CreateFolder" method. + _, err = controlClient.CreateFolder(ctx, &controlpb.CreateFolderRequest{}) + assert.NoError(testSuite.T(), err) + // Stop the server and check for any serving errors. + grpcServer.Stop() + err = <-serveErr + if err != nil && err != grpc.ErrServerStopped { + testSuite.T().Fatalf("grpcServer.Serve failed: %v", err) + } + // The defer call will also try to stop, which is fine. + + // Verify on the server side that the client's connection originates from the specified IP address. + host, _, err := net.SplitHostPort(server.remoteAddr.String()) + require.NoError(testSuite.T(), err) + assert.Equal(testSuite.T(), testSuite.clientConfig.LocalSocketAddress, host) +} + +func (testSuite *StorageHandleTest) TestCreateGRPCClientWithInvalidSocketAddress() { + // Configure the client to use an invalid local IP address. + testSuite.clientConfig.LocalSocketAddress = "invalid-address" + testSuite.clientConfig.AnonymousAccess = true + ctx := context.Background() - storageClient, err := createGRPCClientHandle(testSuite.ctx, &sc) + // Attempt to create client options, which should fail. + clientOpts, err := createClientOptionForGRPCClient(ctx, testSuite.clientConfig, false) - assert.NotNil(testSuite.T(), err) - assert.Nil(testSuite.T(), storageClient) - assert.Contains(testSuite.T(), err.Error(), fmt.Sprintf("client-protocol requested is not GRPC: %s", cfg.HTTP1)) + assert.Error(testSuite.T(), err) + assert.Nil(testSuite.T(), clientOpts) } -func (testSuite *StorageHandleTest) TestCreateHTTPClientHandle_WithGRPCClientProtocol() { - sc := storageutil.GetDefaultStorageClientConfig() +func (testSuite *StorageHandleTest) TestNewStorageHandleWithGRPCClientProtocol() { + sc := storageutil.GetDefaultStorageClientConfig(keyFile) sc.ClientProtocol = cfg.GRPC - storageClient, err := createHTTPClientHandle(testSuite.ctx, &sc) + storageClient, err := NewStorageHandle(testSuite.ctx, sc, "") - assert.NotNil(testSuite.T(), err) - assert.Nil(testSuite.T(), storageClient) - assert.Contains(testSuite.T(), err.Error(), fmt.Sprintf("client-protocol requested is not HTTP1 or HTTP2: %s", cfg.GRPC)) + assert.Nil(testSuite.T(), err) + assert.NotNil(testSuite.T(), storageClient) } func (testSuite *StorageHandleTest) TestCreateHTTPClientHandle_WithReadStallRetry() { @@ -316,7 +470,7 @@ func (testSuite *StorageHandleTest) TestCreateHTTPClientHandle_WithReadStallRetr for _, tc := range testCases { testSuite.Run(tc.name, func() { - sc := storageutil.GetDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig(keyFile) sc.ReadStallRetryConfig.Enable = tc.enableReadStallRetry storageClient, err := createHTTPClientHandle(testSuite.ctx, &sc) @@ -344,7 +498,7 @@ func (testSuite *StorageHandleTest) TestCreateHTTPClientHandle_ReadStallInitialR for _, tc := range testCases { testSuite.Run(tc.name, func() { - sc := storageutil.GetDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig(keyFile) sc.ReadStallRetryConfig.Enable = true sc.ReadStallRetryConfig.InitialReqTimeout = tc.initialReqTimeout @@ -373,7 +527,7 @@ func (testSuite *StorageHandleTest) TestCreateHTTPClientHandle_ReadStallMinReqTi for _, tc := range testCases { testSuite.Run(tc.name, func() { - sc := storageutil.GetDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig(keyFile) sc.ReadStallRetryConfig.Enable = true sc.ReadStallRetryConfig.MinReqTimeout = tc.minReqTimeout @@ -410,7 +564,7 @@ func (testSuite *StorageHandleTest) TestCreateHTTPClientHandle_ReadStallReqIncre for _, tc := range testCases { testSuite.Run(tc.name, func() { - sc := storageutil.GetDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig(keyFile) sc.ReadStallRetryConfig.Enable = true sc.ReadStallRetryConfig.ReqIncreaseRate = tc.reqIncreaseRate @@ -461,7 +615,7 @@ func (testSuite *StorageHandleTest) TestCreateHTTPClientHandle_ReadStallReqTarge for _, tc := range testCases { testSuite.Run(tc.name, func() { - sc := storageutil.GetDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig(keyFile) sc.ReadStallRetryConfig.Enable = true sc.ReadStallRetryConfig.ReqTargetPercentile = tc.reqTargetPercentile @@ -478,38 +632,53 @@ func (testSuite *StorageHandleTest) TestCreateHTTPClientHandle_ReadStallReqTarge } func (testSuite *StorageHandleTest) TestNewStorageHandleWithGRPCClientWithCustomEndpointNilAndAuthEnabled() { - sc := storageutil.GetDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig(keyFile) sc.CustomEndpoint = "" sc.AnonymousAccess = false sc.ClientProtocol = cfg.GRPC - handleCreated, err := NewStorageHandle(testSuite.ctx, sc) + handleCreated, err := NewStorageHandle(testSuite.ctx, sc, "") - assert.NotNil(testSuite.T(), err) - assert.Contains(testSuite.T(), err.Error(), "no such file or directory") - assert.Nil(testSuite.T(), handleCreated) + assert.NoError(testSuite.T(), err) + assert.NotNil(testSuite.T(), handleCreated) } func (testSuite *StorageHandleTest) TestNewStorageHandleWithGRPCClientWithCustomEndpointAndAuthEnabled() { url, err := url.Parse(storageutil.CustomEndpoint) assert.Nil(testSuite.T(), err) - sc := storageutil.GetDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig(keyFile) sc.CustomEndpoint = url.String() sc.AnonymousAccess = false sc.ClientProtocol = cfg.GRPC + sc.TokenUrl = storageutil.CustomTokenUrl + sc.KeyFile = "" - handleCreated, err := NewStorageHandle(testSuite.ctx, sc) + handleCreated, err := NewStorageHandle(testSuite.ctx, sc, "") - assert.NotNil(testSuite.T(), err) - assert.Contains(testSuite.T(), err.Error(), "GRPC client doesn't support auth for custom-endpoint. Please set anonymous-access: true via config-file.") - assert.Nil(testSuite.T(), handleCreated) + assert.Nil(testSuite.T(), err) + assert.NotNil(testSuite.T(), handleCreated) +} + +func (testSuite *StorageHandleTest) TestNewStorageHandleWithGRPCClientWithCustomEndpointAndAuthDisabled() { + url, err := url.Parse(storageutil.CustomEndpoint) + assert.Nil(testSuite.T(), err) + sc := storageutil.GetDefaultStorageClientConfig(keyFile) + sc.CustomEndpoint = url.String() + sc.ClientProtocol = cfg.GRPC + sc.TokenUrl = storageutil.CustomTokenUrl + sc.KeyFile = "" + + handleCreated, err := NewStorageHandle(testSuite.ctx, sc, "") + + assert.Nil(testSuite.T(), err) + assert.NotNil(testSuite.T(), handleCreated) } func (testSuite *StorageHandleTest) TestCreateStorageHandleWithEnableHNSTrue() { - sc := storageutil.GetDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig(keyFile) sc.EnableHNS = true - sh, err := NewStorageHandle(testSuite.ctx, sc) + sh, err := NewStorageHandle(testSuite.ctx, sc, "") assert.Nil(testSuite.T(), err) assert.NotNil(testSuite.T(), sh) @@ -518,32 +687,426 @@ func (testSuite *StorageHandleTest) TestCreateStorageHandleWithEnableHNSTrue() { func (testSuite *StorageHandleTest) TestNewStorageHandleWithCustomEndpointAndEnableHNSTrue() { url, err := url.Parse(storageutil.CustomEndpoint) require.NoError(testSuite.T(), err) - sc := storageutil.GetDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig(keyFile) sc.CustomEndpoint = url.String() sc.EnableHNS = true - sh, err := NewStorageHandle(testSuite.ctx, sc) + sh, err := NewStorageHandle(testSuite.ctx, sc, "") assert.NoError(testSuite.T(), err) assert.NotNil(testSuite.T(), sh) } func (testSuite *StorageHandleTest) TestCreateClientOptionForGRPCClient() { - sc := storageutil.GetDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig(keyFile) + + clientOption, err := createClientOptionForGRPCClient(context.TODO(), &sc, false) + + assert.Nil(testSuite.T(), err) + assert.NotNil(testSuite.T(), clientOption) +} + +func (testSuite *StorageHandleTest) Test_CreateClientOptionForGRPCClient_WithoutGoogleLibAuth() { + sc := storageutil.GetDefaultStorageClientConfig(keyFile) + sc.EnableGoogleLibAuth = false + + clientOption, err := createClientOptionForGRPCClient(context.TODO(), &sc, false) + + assert.Nil(testSuite.T(), err) + assert.NotNil(testSuite.T(), clientOption) +} - clientOption, err := createClientOptionForGRPCClient(&sc) +func (testSuite *StorageHandleTest) Test_CreateHTTPClientHandle_WithoutGoogleLibAuth() { + sc := storageutil.GetDefaultStorageClientConfig(keyFile) + sc.EnableGoogleLibAuth = false + + httpClient, err := createHTTPClientHandle(context.TODO(), &sc) + + assert.Nil(testSuite.T(), err) + assert.NotNil(testSuite.T(), httpClient) +} + +func (testSuite *StorageHandleTest) Test_CreateClientOptionForGRPCClient_WithTracing() { + sc := storageutil.GetDefaultStorageClientConfig(keyFile) + sc.TracingEnabled = true + + clientOption, err := createClientOptionForGRPCClient(context.TODO(), &sc, false) assert.Nil(testSuite.T(), err) assert.NotNil(testSuite.T(), clientOption) } +func (testSuite *StorageHandleTest) Test_CreateClientOptionForGRPCClient_WithTracingAddsOneOption() { + scWithoutTracing := storageutil.GetDefaultStorageClientConfig(keyFile) + scWithoutTracing.TracingEnabled = false + optsWithoutTracing, err := createClientOptionForGRPCClient(context.TODO(), &scWithoutTracing, false) + assert.Nil(testSuite.T(), err) + scWithTracing := storageutil.GetDefaultStorageClientConfig(keyFile) + scWithTracing.TracingEnabled = true + + optsWithTracing, err := createClientOptionForGRPCClient(context.TODO(), &scWithTracing, false) + + assert.Nil(testSuite.T(), err) + assert.NotNil(testSuite.T(), optsWithTracing) + assert.Len(testSuite.T(), optsWithTracing, len(optsWithoutTracing)+1, "Enabling tracing should add exactly one client option.") +} + +func (testSuite *StorageHandleTest) Test_CreateClientOptionForGRPCClient_WithGrpcMetrics() { + oldProvider := otel.GetMeterProvider() + defer otel.SetMeterProvider(oldProvider) + + sdkProvider := sdkmetric.NewMeterProvider() + otel.SetMeterProvider(sdkProvider) + + scWithMetrics := storageutil.GetDefaultStorageClientConfig(keyFile) + scWithMetrics.ClientProtocol = cfg.GRPC + scWithMetrics.EnableGrpcMetrics = true + + optsWithMetrics, err := createClientOptionForGRPCClient(context.TODO(), &scWithMetrics, false) + assert.Nil(testSuite.T(), err) + assert.NotNil(testSuite.T(), optsWithMetrics) + + scWithoutMetrics := storageutil.GetDefaultStorageClientConfig(keyFile) + scWithoutMetrics.ClientProtocol = cfg.GRPC + scWithoutMetrics.EnableGrpcMetrics = false + optsWithoutMetrics, err := createClientOptionForGRPCClient(context.TODO(), &scWithoutMetrics, false) + require.NoError(testSuite.T(), err) + assert.NotNil(testSuite.T(), optsWithoutMetrics) +} + +func (testSuite *StorageHandleTest) Test_CreateClientOptionForGRPCClient_AuthFailures() { + tests := []struct { + name string + modifyConfig func(sc *storageutil.StorageClientConfig) + expectError bool + expectNilOpts bool + }{ + { + name: "Invalid token URL with google lib auth", + modifyConfig: func(sc *storageutil.StorageClientConfig) { + sc.TokenUrl = ":" + sc.KeyFile = "" + sc.EnableGoogleLibAuth = true + }, + }, + { + name: "Invalid token URL without google lib auth", + modifyConfig: func(sc *storageutil.StorageClientConfig) { + sc.TokenUrl = ":" + sc.KeyFile = "" + sc.EnableGoogleLibAuth = false + }, + }, + { + name: "Invalid key file path with google lib auth", + modifyConfig: func(sc *storageutil.StorageClientConfig) { + sc.KeyFile = "incorrect_path" + sc.EnableGoogleLibAuth = true + }, + }, + { + name: "Invalid key file path without google lib auth", + modifyConfig: func(sc *storageutil.StorageClientConfig) { + sc.KeyFile = "incorrect_path" + sc.EnableGoogleLibAuth = false + }, + }, + } + + for _, tt := range tests { + testSuite.T().Run(tt.name, func(t *testing.T) { + sc := storageutil.GetDefaultStorageClientConfig(keyFile) + sc.ClientProtocol = cfg.GRPC + tt.modifyConfig(&sc) + + clientOption, err := createClientOptionForGRPCClient(context.TODO(), &sc, false) + + assert.Error(t, err) + assert.Nil(t, clientOption) + }) + } +} + +func (testSuite *StorageHandleTest) Test_CreateHTTPClientHandle_AuthFailures() { + tests := []struct { + name string + modifyConfig func(sc *storageutil.StorageClientConfig) + expectError bool + expectNilOpts bool + }{ + { + name: "Invalid token URL with google lib auth", + modifyConfig: func(sc *storageutil.StorageClientConfig) { + sc.TokenUrl = ":" + sc.KeyFile = "" + sc.EnableGoogleLibAuth = true + }, + }, + { + name: "Invalid token URL without google lib auth", + modifyConfig: func(sc *storageutil.StorageClientConfig) { + sc.TokenUrl = ":" + sc.KeyFile = "" + sc.EnableGoogleLibAuth = false + }, + }, + { + name: "Invalid key file path with google lib auth", + modifyConfig: func(sc *storageutil.StorageClientConfig) { + sc.KeyFile = "incorrect_path" + sc.EnableGoogleLibAuth = true + }, + }, + { + name: "Invalid Key File Path without google lib auth", + modifyConfig: func(sc *storageutil.StorageClientConfig) { + sc.KeyFile = "incorrect_path" + sc.EnableGoogleLibAuth = false + }, + }, + } + + for _, tt := range tests { + testSuite.T().Run(tt.name, func(t *testing.T) { + sc := storageutil.GetDefaultStorageClientConfig(keyFile) + tt.modifyConfig(&sc) + + httpClient, err := createHTTPClientHandle(context.TODO(), &sc) + + assert.Error(t, err) + assert.Nil(t, httpClient) + }) + } +} + func (testSuite *StorageHandleTest) TestNewStorageHandleWithMaxRetryAttemptsNotZero() { - sc := storageutil.GetDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig(keyFile) sc.MaxRetryAttempts = 100 - handleCreated, err := NewStorageHandle(testSuite.ctx, sc) + handleCreated, err := NewStorageHandle(testSuite.ctx, sc, "") if assert.NoError(testSuite.T(), err) { assert.NotNil(testSuite.T(), handleCreated) } } + +func (testSuite *StorageHandleTest) TestControlClientForBucketHandle_NilControlClient() { + // Arrange + sh := &storageClient{} // storageControlClient is nil by default + + // Act + controlClient := sh.controlClientForBucketHandle(&gcs.BucketType{}, "") + + // Assert + assert.Nil(testSuite.T(), controlClient) +} + +func (testSuite *StorageHandleTest) TestControlClientForBucketHandle() { + tests := []struct { + name string + isZonal bool + pirloState gcs.PirloState + billingProject string + folderAPIStallRetry bool + expectFolderRetries bool + expectGaxRetriesUsed bool + }{ + { + name: "ZonalBucket_NoBillingProject", + isZonal: true, + billingProject: "", + expectFolderRetries: true, + expectGaxRetriesUsed: false, + }, + { + name: "ZonalBucket_WithBillingProject", + isZonal: true, + billingProject: "test-project", + expectFolderRetries: true, + expectGaxRetriesUsed: false, + }, + { + name: "PirloBucket_NoBillingProject", + pirloState: gcs.PirloStateRapidWritesEnabled, + billingProject: "", + expectFolderRetries: true, + expectGaxRetriesUsed: false, + }, + { + name: "PirloBucket_WithBillingProject", + pirloState: gcs.PirloStateRapidWritesEnabled, + billingProject: "test-project", + expectFolderRetries: true, + expectGaxRetriesUsed: false, + }, + { + name: "NonZonalBucket_NoBillingProject_NoFolderApiStallRetryFix", + isZonal: false, + billingProject: "", + folderAPIStallRetry: false, + expectFolderRetries: false, + expectGaxRetriesUsed: true, + }, + { + name: "NonZonalBucket_WithBillingProject_NoFolderApiStallRetryFix", + isZonal: false, + billingProject: "test-project", + folderAPIStallRetry: false, + expectFolderRetries: false, + expectGaxRetriesUsed: true, + }, + { + name: "NonZonalBucket_NoBillingProject_WithFolderApiStallRetryFix", + isZonal: false, + billingProject: "", + folderAPIStallRetry: true, + expectFolderRetries: true, + expectGaxRetriesUsed: false, + }, + { + name: "NonZonalBucket_WithBillingProject_WithFolderApiStallRetryFix", + isZonal: false, + billingProject: "test-project", + folderAPIStallRetry: true, + expectFolderRetries: true, + expectGaxRetriesUsed: false, + }, + } + + for _, tc := range tests { + testSuite.Run(tc.name, func() { + // Arrange + clientConfig := storageutil.GetDefaultStorageClientConfig(keyFile) + clientConfig.ExperimentalNonrapidFolderApiStallRetry = tc.folderAPIStallRetry + mockRawControlClientWithRetries := &control.StorageControlClient{CallOptions: testSuite.controlClientCallOptionsWithRetry()} + mockRawControlClientWithoutRetries := &control.StorageControlClient{CallOptions: testSuite.controlClientCallOptionsWithoutRetry()} + sh := &storageClient{ + rawStorageControlClientWithoutGaxRetries: mockRawControlClientWithoutRetries, + rawStorageControlClientWithGaxRetries: mockRawControlClientWithRetries, + clientConfig: clientConfig, + } + bucketType := &gcs.BucketType{Zonal: tc.isZonal, Pirlo: tc.pirloState} + + // Act + controlClient := sh.controlClientForBucketHandle(bucketType, tc.billingProject) + + // Assert + require.NotNil(testSuite.T(), controlClient) + underlyingControlClient := controlClient + if tc.billingProject != "" { + billingProjectWrapper, ok := controlClient.(*storageControlClientWithBillingProject) + require.True(testSuite.T(), ok, "Expected a billing project wrapper") + assert.Equal(testSuite.T(), tc.billingProject, billingProjectWrapper.billingProject) + underlyingControlClient = billingProjectWrapper.raw + } else { + _, isBillingWrapper := controlClient.(*storageControlClientWithBillingProject) + assert.False(testSuite.T(), isBillingWrapper, "Did not expect a billing project wrapper") + } + retryWrapper, ok := underlyingControlClient.(*storageControlClientWithRetry) + require.True(testSuite.T(), ok, "Expected a retry wrapper") + assert.True(testSuite.T(), retryWrapper.enableRetriesOnStorageLayoutAPI, "Retries should always be enabled for storage layout APIs") + assert.Equal(testSuite.T(), tc.expectFolderRetries, retryWrapper.enableRetriesOnFolderAPIs) + gaxClient, ok := retryWrapper.raw.(*control.StorageControlClient) + require.True(testSuite.T(), ok) + if tc.expectGaxRetriesUsed { + assert.Same(testSuite.T(), mockRawControlClientWithRetries, gaxClient) + } else { + assert.Same(testSuite.T(), mockRawControlClientWithoutRetries, gaxClient) + } + }) + } +} + +func (testSuite *StorageHandleTest) TestControlClientForBucketHandle_NonZonalBucket_ThenZonalBucket_WithoutBillingProject_NoFolderApiStallRetryFix() { + // Arrange + clientConfig := storageutil.GetDefaultStorageClientConfig(keyFile) + mockRawControlClientWithRetries := &control.StorageControlClient{CallOptions: testSuite.controlClientCallOptionsWithRetry()} + mockRawControlClientWithoutRetries := &control.StorageControlClient{CallOptions: testSuite.controlClientCallOptionsWithoutRetry()} + sh := &storageClient{ + rawStorageControlClientWithoutGaxRetries: mockRawControlClientWithoutRetries, + rawStorageControlClientWithGaxRetries: mockRawControlClientWithRetries, + clientConfig: clientConfig, + } + + // Act + // create control-client for non-ZB, which should create a control-client with gax retries. + bucketType := gcs.BucketType{Zonal: false} + controlClientForNonZB := sh.controlClientForBucketHandle(&bucketType, "") + + // Assert + require.NotNil(testSuite.T(), controlClientForNonZB) + // Check that the raw control client is a storageControlClientWithRetry and also uses GAX retries. + controlClientWithAllRetriesNonZB, ok := controlClientForNonZB.(*storageControlClientWithRetry) + require.True(testSuite.T(), ok) + require.NotNil(testSuite.T(), controlClientWithAllRetriesNonZB) + assert.True(testSuite.T(), controlClientWithAllRetriesNonZB.enableRetriesOnStorageLayoutAPI, "Retries should be enabled for storage layout API on non-zonal buckets") + assert.False(testSuite.T(), controlClientWithAllRetriesNonZB.enableRetriesOnFolderAPIs, "Retries should not be enabled for folder APIs on non-zonal buckets") + require.Same(testSuite.T(), mockRawControlClientWithRetries, controlClientWithAllRetriesNonZB.raw) + + // Act + // create control-client for ZB afterwards, which should create a storageControlClientWithRetry a raw control.StorageControlClient without gax retries. + bucketType = gcs.BucketType{Zonal: true} + controlClientForZB := sh.controlClientForBucketHandle(&bucketType, "") + + // Assert + require.NotNil(testSuite.T(), controlClientForZB) + // Check that the control client is a storageControlClientWithRetry with all APIs retried. + controlClientWithRetry, ok := controlClientForZB.(*storageControlClientWithRetry) + require.True(testSuite.T(), ok, "Expected a control client with retry") + assert.Same(testSuite.T(), mockRawControlClientWithoutRetries, controlClientWithRetry.raw) + assert.True(testSuite.T(), controlClientWithRetry.enableRetriesOnFolderAPIs, "Retries should be enabled for folder APIs on zonal buckets") + assert.True(testSuite.T(), controlClientWithRetry.enableRetriesOnStorageLayoutAPI, "Retries should be enabled for storage layout API on zonal buckets") + require.Same(testSuite.T(), mockRawControlClientWithoutRetries, controlClientWithRetry.raw) +} + +func (testSuite *StorageHandleTest) TestControlClientForBucketHandle_NonZonalBucket_ThenZonalBucket_WithBillingProject_NoFolderApiStallRetryFix() { + // Arrange + billingProject := "test-project" + clientConfig := storageutil.GetDefaultStorageClientConfig(keyFile) + mockRawControlClientWithRetries := &control.StorageControlClient{CallOptions: testSuite.controlClientCallOptionsWithRetry()} + mockRawControlClientWithoutRetries := &control.StorageControlClient{CallOptions: testSuite.controlClientCallOptionsWithoutRetry()} + sh := &storageClient{ + rawStorageControlClientWithoutGaxRetries: mockRawControlClientWithoutRetries, + rawStorageControlClientWithGaxRetries: mockRawControlClientWithRetries, + clientConfig: clientConfig, + } + + // Act + // create control-client for non-ZB, which should create a control-client with gax retries. + bucketType := gcs.BucketType{Zonal: false} + controlClientForNonZB := sh.controlClientForBucketHandle(&bucketType, billingProject) + + // Assert + require.NotNil(testSuite.T(), controlClientForNonZB) + // Check that the control client is not a storageControlClientWithRetry and uses GAX retries. + controlClientWithBillingProjectAndAllRetriesForNonZB, ok := controlClientForNonZB.(*storageControlClientWithBillingProject) + require.True(testSuite.T(), ok) + require.NotNil(testSuite.T(), controlClientWithBillingProjectAndAllRetriesForNonZB) + assert.Equal(testSuite.T(), billingProject, controlClientWithBillingProjectAndAllRetriesForNonZB.billingProject) + // Check that the underlying control client is a storageControlClientWithRetry and also uses GAX retries. + controlClientWithAllRetriesNonZB, ok := controlClientWithBillingProjectAndAllRetriesForNonZB.raw.(*storageControlClientWithRetry) + require.True(testSuite.T(), ok) + require.NotNil(testSuite.T(), controlClientWithAllRetriesNonZB) + assert.True(testSuite.T(), controlClientWithAllRetriesNonZB.enableRetriesOnStorageLayoutAPI, "Retries should be enabled for storage layout API on non-zonal buckets") + assert.False(testSuite.T(), controlClientWithAllRetriesNonZB.enableRetriesOnFolderAPIs, "Retries should not be enabled for folder APIs on non-zonal buckets") + require.Same(testSuite.T(), mockRawControlClientWithRetries, controlClientWithAllRetriesNonZB.raw) + + // Act + // create control-client for ZB afterwards, which should create a storageControlClientWithRetry a raw control.StorageControlClient without gax retries. + bucketType = gcs.BucketType{Zonal: true} + controlClientForZB := sh.controlClientForBucketHandle(&bucketType, billingProject) + + // Assert + require.NotNil(testSuite.T(), controlClientForZB) + // Check that the control client contains a storageControlClientWithBillingProject. + controlClientWithBillingProjectForZB, ok := controlClientForZB.(*storageControlClientWithBillingProject) + require.True(testSuite.T(), ok) + require.NotNil(testSuite.T(), controlClientWithBillingProjectForZB) + // Check that the control client is a storageControlClientWithRetry with all APIs retried. + controlClientWithRetry, ok := controlClientWithBillingProjectForZB.raw.(*storageControlClientWithRetry) + require.True(testSuite.T(), ok, "Expected a control client with retry") + require.NotNil(testSuite.T(), controlClientWithRetry) + assert.True(testSuite.T(), controlClientWithRetry.enableRetriesOnFolderAPIs, "Retries should be enabled for folder APIs on zonal buckets") + assert.True(testSuite.T(), controlClientWithRetry.enableRetriesOnStorageLayoutAPI, "Retries should be enabled for storage layout API on zonal buckets") + assert.Same(testSuite.T(), mockRawControlClientWithoutRetries, controlClientWithRetry.raw) +} diff --git a/internal/storage/storageutil/auth_client_option.go b/internal/storage/storageutil/auth_client_option.go new file mode 100644 index 0000000000..9787dc1ce4 --- /dev/null +++ b/internal/storage/storageutil/auth_client_option.go @@ -0,0 +1,75 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storageutil + +import ( + "context" + "fmt" + + "cloud.google.com/go/auth" + "cloud.google.com/go/auth/oauth2adapt" + auth2 "github.com/googlecloudplatform/gcsfuse/v3/internal/auth" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "golang.org/x/oauth2" + "google.golang.org/api/option" +) + +// GetClientAuthOptionsAndToken returns client options and a token source using either a token URL or fallback to key file/ADC. +func GetClientAuthOptionsAndToken(ctx context.Context, config *StorageClientConfig) ([]option.ClientOption, oauth2.TokenSource, error) { + // If Token URL is provided, attempt to fetch token source directly. + if config.TokenUrl != "" { + tokenSrc, err := auth2.NewTokenSourceFromURL(ctx, config.TokenUrl, config.ReuseTokenFromUrl) + if err != nil { + return nil, nil, fmt.Errorf("while fetching token source: %w", err) + } + + clientOpts := []option.ClientOption{option.WithTokenSource(tokenSrc)} + return clientOpts, tokenSrc, nil + } + + // Fallback: Use key file credentials. + cred, err := auth2.GetCredentials(config.KeyFile) + if err != nil { + return nil, nil, fmt.Errorf("while fetching credentials: %w", err) + } + + tokenSrc := oauth2adapt.TokenSourceFromTokenProvider(cred.TokenProvider) + + retryConfig := NewRetryConfig(config, DefaultRetryDeadline, DefaultTotalRetryBudget, DefaultInitialBackoff) + + apiCall := func(attemptCtx context.Context) (string, error) { + d, err := cred.UniverseDomain(attemptCtx) + return d, err + } + + domain, err := ExecuteWithRetryAtLogLevel(ctx, retryConfig, "cred.UniverseDomain", "credentials", apiCall, logger.LevelInfo) + if err != nil { + logger.Errorf("failed to get UniverseDomain: %v, setting default universe domain", err) + // Setting default universe domain to googleapis.com in case we are unable to fetch the domain. + domain = auth2.UniverseDomainDefault + } + logger.Infof("Success in fetching cred.UniverseDomain") + + // Temporary Workaround: We've created a small auth object here that omits the 'quota project ID' + // to bypass a known issue (b/442805436) in the current authentication library. + // TODO: Remove this workaround once issue b/442805436 is resolved in the library. + newCreds := auth.NewCredentials(&auth.CredentialsOptions{ + TokenProvider: cred.TokenProvider, + UniverseDomainProvider: auth.CredentialsPropertyFunc(func(_ context.Context) (string, error) { return domain, nil }), + }) + clientOpts := []option.ClientOption{option.WithUniverseDomain(domain), option.WithAuthCredentials(newCreds)} + + return clientOpts, tokenSrc, nil +} diff --git a/internal/storage/storageutil/auth_client_option_test.go b/internal/storage/storageutil/auth_client_option_test.go new file mode 100644 index 0000000000..9b2c295ecf --- /dev/null +++ b/internal/storage/storageutil/auth_client_option_test.go @@ -0,0 +1,83 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storageutil + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "google.golang.org/api/option" +) + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func Test_GetClientAuthOptionsAndToken_TokenUrlSuccess(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprintln(w, `{"access_token":"dummy-token","token_type":"Bearer"}`) + })) + defer server.Close() + config := &StorageClientConfig{ + TokenUrl: server.URL, + ReuseTokenFromUrl: false, + KeyFile: "testdata/key.json", + } + + clientOpts, tokenSrc, err := GetClientAuthOptionsAndToken(context.TODO(), config) + + assert.NoError(t, err) + assert.NotNil(t, tokenSrc) + assert.Len(t, clientOpts, 1) // Only tokenSource option attached +} + +func Test_GetClientAuthOptionsAndToken_TokenUrlError(t *testing.T) { + config := &StorageClientConfig{TokenUrl: ":"} + + clientOpts, tokenSrc, err := GetClientAuthOptionsAndToken(context.TODO(), config) + + assert.Error(t, err) + assert.Nil(t, tokenSrc) + assert.Empty(t, clientOpts) +} + +func Test_GetClientAuthOptionsAndToken_FallbackToKeyFileSuccess(t *testing.T) { + config := &StorageClientConfig{ + TokenUrl: "", // triggers fallback + KeyFile: "testdata/key.json", + } + + clientOpts, tokenSrc, err := GetClientAuthOptionsAndToken(context.TODO(), config) + + assert.NoError(t, err) + assert.NotNil(t, tokenSrc) + assert.Len(t, clientOpts, 2) // UniverseDomain + AuthCredentials +} + +func Test_GetClientAuthOptionsAndToken_FallbackToKeyFileError(t *testing.T) { + config := &StorageClientConfig{TokenUrl: "", KeyFile: "fake-key"} + var clientOpts []option.ClientOption + + clientOpts, tokenSrc, err := GetClientAuthOptionsAndToken(context.TODO(), config) + + assert.Error(t, err) + assert.Nil(t, tokenSrc) + assert.Empty(t, clientOpts) +} diff --git a/internal/storage/storageutil/client.go b/internal/storage/storageutil/client.go index 48d081eb16..2022e650a7 100644 --- a/internal/storage/storageutil/client.go +++ b/internal/storage/storageutil/client.go @@ -1,4 +1,4 @@ -// Copyright 2023 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,30 +17,51 @@ package storageutil import ( "crypto/tls" "fmt" + "net" "net/http" + "net/http/httptrace" "strings" "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/auth" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/auth" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" + dns "github.com/ncruces/go-dns" + "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel" "golang.org/x/net/context" "golang.org/x/oauth2" ) +// ConfigureDialerWithLocalAddr resolves the provided socket address and returns a net.TCPAddr. +// The port can be 0, in which case the OS will choose a local port. +// The format of SocketAddress is expected to be IP address. +func ConfigureDialerWithLocalAddr(dialer *net.Dialer, socketAddress string) error { + localAddr, err := net.ResolveTCPAddr("tcp", socketAddress+":0") + if err != nil { + return fmt.Errorf("failed to resolve socket address %q: %w", socketAddress, err) + } + dialer.LocalAddr = localAddr + return nil +} + const urlSchemeSeparator = "://" type StorageClientConfig struct { /** Common client parameters. */ // ClientProtocol decides the go-sdk client to create. - ClientProtocol cfg.Protocol - UserAgent string - CustomEndpoint string - KeyFile string - TokenUrl string - ReuseTokenFromUrl bool - MaxRetrySleep time.Duration - RetryMultiplier float64 + ClientProtocol cfg.Protocol + UserAgent string + CustomEndpoint string + KeyFile string + TokenUrl string + ReuseTokenFromUrl bool + ExperimentalNonrapidFolderApiStallRetry bool + MaxRetrySleep time.Duration + RetryMultiplier float64 + LocalSocketAddress string /** HTTP client parameters. */ MaxConnsPerHost int @@ -52,18 +73,47 @@ type StorageClientConfig struct { /** Grpc client parameters. */ GrpcConnPoolSize int + GrpcPathStrategy cfg.DirectPathStrategy // Enabling new API flow for HNS bucket. EnableHNS bool + // EnableGoogleLibAuth indicates whether to use the google library authentication flow + EnableGoogleLibAuth bool + + ExperimentalEnablePirlo bool ReadStallRetryConfig cfg.ReadStallGcsRetriesConfig + + MetricHandle metrics.MetricHandle + + TracingEnabled bool + + EnableHTTPDNSCache bool + + EnableGrpcMetrics bool + + // IsGKE inspects the mountPoint and indicates if running in a GKE environment. + IsGKE bool + + WriteConfig *cfg.WriteConfig } -func CreateHttpClient(storageClientConfig *StorageClientConfig) (httpClient *http.Client, err error) { +func CreateHttpClient(storageClientConfig *StorageClientConfig, tokenSrc oauth2.TokenSource) (httpClient *http.Client, err error) { + dialer := net.Dialer{} + if storageClientConfig.LocalSocketAddress != "" { + if err := ConfigureDialerWithLocalAddr(&dialer, storageClientConfig.LocalSocketAddress); err != nil { + return nil, fmt.Errorf("failed to configure dialer with local-socket-address %q: %w", storageClientConfig.LocalSocketAddress, err) + } + } + if storageClientConfig.EnableHTTPDNSCache { + dialer.Resolver = dns.NewCachingResolver(nil, dns.MinCacheTTL(1*time.Minute)) + } + var transport *http.Transport // Using http1 makes the client more performant. if storageClientConfig.ClientProtocol == cfg.HTTP1 { transport = &http.Transport{ + DialContext: dialer.DialContext, Proxy: http.ProxyFromEnvironment, MaxConnsPerHost: storageClientConfig.MaxConnsPerHost, MaxIdleConnsPerHost: storageClientConfig.MaxIdleConnsPerHost, @@ -75,6 +125,7 @@ func CreateHttpClient(storageClientConfig *StorageClientConfig) (httpClient *htt } else { // For http2, change in MaxConnsPerHost doesn't affect the performance. transport = &http.Transport{ + DialContext: dialer.DialContext, Proxy: http.ProxyFromEnvironment, DisableKeepAlives: true, MaxConnsPerHost: storageClientConfig.MaxConnsPerHost, @@ -94,11 +145,14 @@ func CreateHttpClient(storageClientConfig *StorageClientConfig) (httpClient *htt Timeout: storageClientConfig.HttpClientTimeout, } } else { - var tokenSrc oauth2.TokenSource - tokenSrc, err = CreateTokenSource(storageClientConfig) - if err != nil { - err = fmt.Errorf("while fetching tokenSource: %w", err) - return + if tokenSrc == nil { + // CreateTokenSource only if tokenSrc is nil, which means it wasn't provided externally. + // This indicates the EnableGoogleLibAuth flag is disabled. + tokenSrc, err = CreateTokenSource(storageClientConfig) + if err != nil { + err = fmt.Errorf("while fetching tokenSource: %w", err) + return nil, err + } } // Custom http client for Go Client. @@ -114,6 +168,12 @@ func CreateHttpClient(storageClientConfig *StorageClientConfig) (httpClient *htt wrapped: httpClient.Transport, UserAgent: storageClientConfig.UserAgent, } + + if storageClientConfig.TracingEnabled { + httpClient.Transport = otelhttp.NewTransport(httpClient.Transport, otelhttp.WithClientTrace(func(ctx context.Context) *httptrace.ClientTrace { + return otelhttptrace.NewClientTrace(ctx) + }), otelhttp.WithTracerProvider(otel.GetTracerProvider())) + } } return httpClient, err } @@ -126,6 +186,10 @@ func CreateTokenSource(storageClientConfig *StorageClientConfig) (tokenSrc oauth // StripScheme strips the scheme part of given url. func StripScheme(url string) string { + // Don't strip off the scheme part for google-internal schemes. + if strings.HasPrefix(url, "dns:///") || strings.HasPrefix(url, "google-c2p:///") || strings.HasPrefix(url, "google:///") { + return url + } if strings.Contains(url, urlSchemeSeparator) { url = strings.SplitN(url, urlSchemeSeparator, 2)[1] } diff --git a/internal/storage/storageutil/client_test.go b/internal/storage/storageutil/client_test.go index 3eb8965a11..bfd5ed8d12 100644 --- a/internal/storage/storageutil/client_test.go +++ b/internal/storage/storageutil/client_test.go @@ -4,6 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // +// // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software @@ -15,88 +16,92 @@ package storageutil import ( + "net" "net/http" + "net/http/httptest" + "slices" "testing" - "github.com/jacobsa/oglematchers" - . "github.com/jacobsa/ogletest" + "go.opentelemetry.io/otel" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" "golang.org/x/oauth2" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" ) -func TestClient(t *testing.T) { RunTests(t) } +var keyFile = "testdata/key.json" -type clientTest struct { +func TestClient(t *testing.T) { + suite.Run(t, new(clientTest)) } -func init() { RegisterTestSuite(&clientTest{}) } - -// Helpers +type clientTest struct { + suite.Suite +} -func (t *clientTest) validateProxyInTransport(httpClient *http.Client) { - userAgentRT, ok := httpClient.Transport.(*userAgentRoundTripper) - AssertEq(true, ok) - oauthTransport, ok := userAgentRT.wrapped.(*oauth2.Transport) - AssertEq(true, ok) - transport, ok := oauthTransport.Base.(*http.Transport) - AssertEq(true, ok) - if ok { - ExpectEq(http.ProxyFromEnvironment, transport.Proxy) - } +func newInMemoryExporter(t *testing.T) *tracetest.InMemoryExporter { + t.Helper() + ex := tracetest.NewInMemoryExporter() + t.Cleanup(func() { + ex.Reset() + }) + otel.SetTracerProvider(sdktrace.NewTracerProvider(sdktrace.WithSyncer(ex))) + return ex } // Tests func (t *clientTest) TestCreateHttpClientWithHttp1() { - sc := GetDefaultStorageClientConfig() // By default http1 enabled + sc := GetDefaultStorageClientConfig(keyFile) // By default http1 enabled - httpClient, err := CreateHttpClient(&sc) + httpClient, err := CreateHttpClient(&sc, nil) - ExpectEq(nil, err) - ExpectNe(nil, httpClient) - ExpectEq(sc.HttpClientTimeout, httpClient.Timeout) + assert.NoError(t.T(), err) + assert.NotNil(t.T(), httpClient) + assert.Equal(t.T(), sc.HttpClientTimeout, httpClient.Timeout) } func (t *clientTest) TestCreateHttpClientWithHttp2() { - sc := GetDefaultStorageClientConfig() + sc := GetDefaultStorageClientConfig(keyFile) - httpClient, err := CreateHttpClient(&sc) + httpClient, err := CreateHttpClient(&sc, nil) - ExpectEq(nil, err) - ExpectNe(nil, httpClient) - ExpectEq(sc.HttpClientTimeout, httpClient.Timeout) + assert.NoError(t.T(), err) + assert.NotNil(t.T(), httpClient) + assert.Equal(t.T(), sc.HttpClientTimeout, httpClient.Timeout) } func (t *clientTest) TestCreateHttpClientWithHttp1AndAuthEnabled() { - sc := GetDefaultStorageClientConfig() // By default http1 enabled + sc := GetDefaultStorageClientConfig(keyFile) // By default http1 enabled sc.AnonymousAccess = false // Act: this method add tokenSource and clientOptions. - httpClient, err := CreateHttpClient(&sc) + httpClient, err := CreateHttpClient(&sc, nil) - AssertNe(nil, err) - ExpectThat(err, oglematchers.Error(oglematchers.HasSubstr("no such file or directory"))) - AssertEq(nil, httpClient) + assert.NoError(t.T(), err) + assert.NotNil(t.T(), httpClient) } func (t *clientTest) TestCreateHttpClientWithHttp2AndAuthEnabled() { - sc := GetDefaultStorageClientConfig() + sc := GetDefaultStorageClientConfig(keyFile) sc.AnonymousAccess = false // Act: this method add tokenSource and clientOptions. - httpClient, err := CreateHttpClient(&sc) + httpClient, err := CreateHttpClient(&sc, nil) - AssertNe(nil, err) - ExpectThat(err, oglematchers.Error(oglematchers.HasSubstr("no such file or directory"))) - AssertEq(nil, httpClient) + assert.NoError(t.T(), err) + assert.NotNil(t.T(), httpClient) } func (t *clientTest) TestCreateTokenSrc() { - sc := GetDefaultStorageClientConfig() + sc := GetDefaultStorageClientConfig(keyFile) tokenSrc, err := CreateTokenSource(&sc) - AssertNe(nil, err) - ExpectThat(err, oglematchers.Error(oglematchers.HasSubstr("no such file or directory"))) - ExpectNe(nil, &tokenSrc) + assert.NoError(t.T(), err) + assert.NotNil(t.T(), &tokenSrc) } func (t *clientTest) TestStripScheme() { @@ -124,9 +129,93 @@ func (t *clientTest) TestStripScheme() { input: "bad://http://localhost:888://", expectedOutput: "http://localhost:888://", }, + { + input: "dns:///localhost:888://", + expectedOutput: "dns:///localhost:888://", + }, + { + input: "google-c2p:///localhost:888://", + expectedOutput: "google-c2p:///localhost:888://", + }, + { + input: "google:///localhost:888://", + expectedOutput: "google:///localhost:888://", + }, } { output := StripScheme(tc.input) - AssertEq(tc.expectedOutput, output) + assert.Equal(t.T(), tc.expectedOutput, output) } } + +func (t *clientTest) TestCreateHttpClientWithHttpTracing() { + ex := newInMemoryExporter(t.T()) + sc := GetDefaultStorageClientConfig(keyFile) + sc.TracingEnabled = true + sc.UserAgent = "test-agent" + var tokenSrc = oauth2.StaticTokenSource(&oauth2.Token{AccessToken: "test-token"}) + + var userAgent, authHeader string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + userAgent = r.Header.Get("User-Agent") + authHeader = r.Header.Get("Authorization") + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + httpClient, err := CreateHttpClient(&sc, tokenSrc) + require.NoError(t.T(), err) + require.NotNil(t.T(), httpClient) + + _, err = httpClient.Get(server.URL) + + assert.NoError(t.T(), err) + assert.Equal(t.T(), "test-agent", userAgent) + assert.Equal(t.T(), "Bearer test-token", authHeader) + ss := ex.GetSpans() + assert.Condition(t.T(), func() bool { + return slices.ContainsFunc(ss, func(s tracetest.SpanStub) bool { return s.Name == "http.connect" }) + }) + assert.Condition(t.T(), func() bool { + return slices.ContainsFunc(ss, func(s tracetest.SpanStub) bool { return s.Name == "http.send" }) + }) +} + +func (t *clientTest) TestCreateHttpClientWithSocketAddress() { + // Start a local server to inspect incoming connections. + var remoteAddr string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + remoteAddr = r.RemoteAddr + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + // Configure the client to use a specific local IP address. + sc := GetDefaultStorageClientConfig(keyFile) + sc.LocalSocketAddress = "127.0.0.1" + // Use a static token to avoid network calls for token acquisition. + var tokenSrc = oauth2.StaticTokenSource(&oauth2.Token{AccessToken: "test-token"}) + httpClient, err := CreateHttpClient(&sc, tokenSrc) + require.NoError(t.T(), err) + require.NotNil(t.T(), httpClient) + + // Have the client connect to the test server. + _, err = httpClient.Get(server.URL) + require.NoError(t.T(), err) + + // Verify on the server side that the client's connection originates from the specified IP address. + host, _, err := net.SplitHostPort(remoteAddr) + require.NoError(t.T(), err) + assert.Equal(t.T(), sc.LocalSocketAddress, host) +} + +func (t *clientTest) TestCreateHttpClientWithInvalidSocketAddress() { + // Configure the client to use an invalid local IP address. + sc := GetDefaultStorageClientConfig(keyFile) + sc.LocalSocketAddress = "invalid-address" + // Use a static token to avoid network calls for token acquisition. + var tokenSrc = oauth2.StaticTokenSource(&oauth2.Token{AccessToken: "test-token"}) + + httpClient, err := CreateHttpClient(&sc, tokenSrc) + + assert.Error(t.T(), err) + assert.Nil(t.T(), httpClient) +} diff --git a/internal/storage/storageutil/control_client.go b/internal/storage/storageutil/control_client.go index b4eceb4d62..bb0caf1438 100644 --- a/internal/storage/storageutil/control_client.go +++ b/internal/storage/storageutil/control_client.go @@ -18,42 +18,13 @@ import ( "context" "fmt" "os" - "time" control "cloud.google.com/go/storage/control/apiv2" - "github.com/googleapis/gax-go/v2" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" "google.golang.org/api/option" - "google.golang.org/grpc/codes" ) -func storageControlClientRetryOptions(clientConfig *StorageClientConfig) []gax.CallOption { - return []gax.CallOption{ - gax.WithTimeout(300000 * time.Millisecond), - gax.WithRetry(func() gax.Retryer { - return gax.OnCodes([]codes.Code{ - codes.ResourceExhausted, - codes.Unavailable, - codes.DeadlineExceeded, - codes.Internal, - codes.Unknown, - }, gax.Backoff{ - Max: clientConfig.MaxRetrySleep, - Multiplier: clientConfig.RetryMultiplier, - }) - }), - } -} - -func setRetryConfigForFolderAPIs(sc *control.StorageControlClient, clientConfig *StorageClientConfig) { - sc.CallOptions.RenameFolder = storageControlClientRetryOptions(clientConfig) - sc.CallOptions.GetFolder = storageControlClientRetryOptions(clientConfig) - sc.CallOptions.GetStorageLayout = storageControlClientRetryOptions(clientConfig) - sc.CallOptions.CreateFolder = storageControlClientRetryOptions(clientConfig) - sc.CallOptions.DeleteFolder = storageControlClientRetryOptions(clientConfig) -} - -func CreateGRPCControlClient(ctx context.Context, clientOpts []option.ClientOption, clientConfig *StorageClientConfig) (controlClient *control.StorageControlClient, err error) { +func CreateGRPCControlClient(ctx context.Context, clientOpts []option.ClientOption, disableDefaultGaxRetries bool) (controlClient *control.StorageControlClient, err error) { if err := os.Setenv("GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS", "true"); err != nil { logger.Fatal("error setting direct path env var: %v", err) } @@ -63,8 +34,10 @@ func CreateGRPCControlClient(ctx context.Context, clientOpts []option.ClientOpti return nil, fmt.Errorf("NewStorageControlClient: %w", err) } - // Set retries for control client. - setRetryConfigForFolderAPIs(controlClient, clientConfig) + // Remove default gax retry options if requested. + if disableDefaultGaxRetries { + *controlClient.CallOptions = control.StorageControlCallOptions{} + } // Unset the environment variable, since it's used only while creation of grpc client. if err := os.Unsetenv("GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS"); err != nil { diff --git a/internal/storage/storageutil/control_client_test.go b/internal/storage/storageutil/control_client_test.go index fe5ff47a6a..47e872a470 100644 --- a/internal/storage/storageutil/control_client_test.go +++ b/internal/storage/storageutil/control_client_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "google.golang.org/api/option" ) @@ -37,21 +38,33 @@ func (testSuite *ControlClientTest) SetupTest() { func (testSuite *ControlClientTest) TearDownTest() { } -func (testSuite *ControlClientTest) TestStorageControlClientRetryOptions() { - clientConfig := GetDefaultStorageClientConfig() +func (testSuite *ControlClientTest) TestStorageControlClientWithGaxRetries() { + var clientOpts []option.ClientOption + clientOpts = append(clientOpts, option.WithoutAuthentication()) - gaxOpts := storageControlClientRetryOptions(&clientConfig) + controlClient, err := CreateGRPCControlClient(context.Background(), clientOpts, false) - assert.NotNil(testSuite.T(), gaxOpts) + require.Nil(testSuite.T(), err) + require.NotNil(testSuite.T(), controlClient) + require.NotNil(testSuite.T(), controlClient.CallOptions) + assert.Greater(testSuite.T(), len(controlClient.CallOptions.CreateFolder), 0) + assert.Greater(testSuite.T(), len(controlClient.CallOptions.GetFolder), 0) + assert.Greater(testSuite.T(), len(controlClient.CallOptions.DeleteFolder), 0) + assert.Greater(testSuite.T(), len(controlClient.CallOptions.RenameFolder), 0) } -func (testSuite *ControlClientTest) TestStorageControlClient() { +func (testSuite *ControlClientTest) TestStorageControlClientWithoutGaxRetries() { var clientOpts []option.ClientOption clientOpts = append(clientOpts, option.WithoutAuthentication()) - clientConfig := GetDefaultStorageClientConfig() - controlClient, err := CreateGRPCControlClient(context.Background(), clientOpts, &clientConfig) + controlClient, err := CreateGRPCControlClient(context.Background(), clientOpts, true) - assert.Nil(testSuite.T(), err) - assert.NotNil(testSuite.T(), controlClient) + require.Nil(testSuite.T(), err) + require.NotNil(testSuite.T(), controlClient) + if controlClient.CallOptions != nil { + assert.Empty(testSuite.T(), controlClient.CallOptions.CreateFolder) + assert.Empty(testSuite.T(), controlClient.CallOptions.GetFolder) + assert.Empty(testSuite.T(), controlClient.CallOptions.DeleteFolder) + assert.Empty(testSuite.T(), controlClient.CallOptions.RenameFolder) + } } diff --git a/internal/storage/storageutil/create_empty_objects.go b/internal/storage/storageutil/create_empty_objects.go index 4419b0c35b..9683af442a 100644 --- a/internal/storage/storageutil/create_empty_objects.go +++ b/internal/storage/storageutil/create_empty_objects.go @@ -15,7 +15,7 @@ package storageutil import ( - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "golang.org/x/net/context" ) diff --git a/internal/storage/storageutil/create_object.go b/internal/storage/storageutil/create_object.go index cf506799d8..d262c83db3 100644 --- a/internal/storage/storageutil/create_object.go +++ b/internal/storage/storageutil/create_object.go @@ -17,7 +17,7 @@ package storageutil import ( "bytes" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "golang.org/x/net/context" ) diff --git a/internal/storage/storageutil/create_objects.go b/internal/storage/storageutil/create_objects.go index 1402bef5d9..1536900ab2 100644 --- a/internal/storage/storageutil/create_objects.go +++ b/internal/storage/storageutil/create_objects.go @@ -15,7 +15,7 @@ package storageutil import ( - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "golang.org/x/net/context" "golang.org/x/sync/errgroup" ) @@ -43,7 +43,7 @@ func CreateObjects( // Create the objects in parallel. const parallelism = 64 - for i := 0; i < parallelism; i++ { + for range parallelism { group.Go(func() (err error) { for r := range recordChan { _, err = CreateObject( diff --git a/internal/storage/storageutil/custom_retry.go b/internal/storage/storageutil/custom_retry.go index f0d58628dc..ea6a4cc3d3 100644 --- a/internal/storage/storageutil/custom_retry.go +++ b/internal/storage/storageutil/custom_retry.go @@ -15,16 +15,34 @@ package storageutil import ( + "context" + "errors" + "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" "google.golang.org/api/googleapi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// retryAction defines the classification of a retry decision. +type retryAction int + +const ( + // noRetry indicates the error is not retryable. + noRetry retryAction = iota + // retryTransient indicates the error is transient and retryable as per Go-SDK retry policy. + retryTransient + // retry401 indicates a 401 Unauthorized error which requires a retry due to credentials refresh. + retry401 + // retryUnauthenticated indicates a gRPC Unauthenticated error which requires a retry. + retryUnauthenticated ) -func ShouldRetry(err error) (b bool) { - b = storage.ShouldRetry(err) - if b { - logger.Infof("Retrying for the error: %v", err) - return +func determineRetryAction(err error) retryAction { + if storage.ShouldRetry(err) { + return retryTransient } // HTTP 401 errors - Invalid Credentials @@ -36,10 +54,61 @@ func ShouldRetry(err error) (b bool) { // TODO: Please incorporate the correct fix post resolution of the above issue. if typed, ok := err.(*googleapi.Error); ok { if typed.Code == 401 { - b = true - logger.Infof("Retrying for error-code 401: %v", err) - return + return retry401 } } - return + + // This is the same case as above, but for gRPC UNAUTHENTICATED errors. See + // https://github.com/golang/oauth2/issues/623 + // TODO: Please incorporate the correct fix post resolution of the above issue. + if status, ok := status.FromError(err); ok { + if status.Code() == codes.Unauthenticated { + return retryUnauthenticated + } + } + return noRetry +} + +// ShouldRetryWithoutLogging checks if the error is transient and should be retried. +// This method is same as ShouldRetry except it doesn't add warning logs. +func ShouldRetryWithoutLogging(err error) bool { + return determineRetryAction(err) != noRetry +} + +// ShouldRetry checks if the given error is transient and should be retried. +// It logs a warning message with the error details for any retryable error. +// Returns true if the error is retryable, false otherwise. +func ShouldRetry(err error) bool { + switch determineRetryAction(err) { + case retryTransient: + logger.Warnf("Retrying for the error: %v", err) + return true + case retry401: + logger.Warnf("Retrying for error-code 401: %v", err) + return true + case retryUnauthenticated: + logger.Warnf("Retrying for UNAUTHENTICATED error: %v", err) + return true + default: + return false + } +} + +func ShouldRetryWithMonitoring(ctx context.Context, err error, metricHandle metrics.MetricHandle) bool { + if err == nil { + return false + } + + retry := ShouldRetry(err) + if !retry { + return false + } + // Record metrics + val := metrics.RetryErrorCategoryOTHERERRORSAttr + if errors.Is(err, context.DeadlineExceeded) { + val = metrics.RetryErrorCategorySTALLEDREADREQUESTAttr + } + + metricHandle.GcsRetryCount(1, val) + return retry } diff --git a/internal/storage/storageutil/custom_retry_test.go b/internal/storage/storageutil/custom_retry_test.go index 2e0b9111a9..e65f134297 100644 --- a/internal/storage/storageutil/custom_retry_test.go +++ b/internal/storage/storageutil/custom_retry_test.go @@ -15,14 +15,22 @@ package storageutil import ( + "bytes" + "context" "errors" "io" "net" "net/url" + "os" + "sync" "testing" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/metrics" "github.com/stretchr/testify/assert" "google.golang.org/api/googleapi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func TestShouldRetryReturnsTrueWithGoogleApiError(t *testing.T) { @@ -118,3 +126,244 @@ func TestShouldRetryReturnsTrueForConnectionRefusedAndResetErrors(t *testing.T) }) } } + +func TestShouldRetryReturnsTrueForUnauthenticatedGrpcErrors(t *testing.T) { + testCases := []struct { + name string + err error + expectedResult bool + }{ + { + name: "UNAUTHENTICATED", + err: status.Error(codes.Unauthenticated, "Request had invalid authentication credentials. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project."), + expectedResult: true, + }, + { + name: "PERMISSION_DENIED", + err: status.Error(codes.PermissionDenied, "unauthorized"), + expectedResult: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualResult := ShouldRetry(tc.err) + assert.Equal(t, tc.expectedResult, actualResult) + }) + } +} + +func TestShouldRetryWithoutLogging(t *testing.T) { + testCases := []struct { + name string + err error + expectedResult bool + }{ + { + name: "401 error - retryable", + err: &googleapi.Error{ + Code: 401, + Body: "Invalid Credential", + }, + expectedResult: true, + }, + { + name: "Unauthenticated error - retryable", + err: status.Error(codes.Unauthenticated, "unauthenticated"), + expectedResult: true, + }, + { + name: "400 error - non-retryable", + err: &googleapi.Error{ + Code: 400, + }, + expectedResult: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var buf logBuffer + logger.SetOutput(&buf) + defer logger.SetOutput(os.Stdout) + + // Act + actualResult := ShouldRetryWithoutLogging(tc.err) + + // Assert + assert.Equal(t, tc.expectedResult, actualResult) + assert.Empty(t, buf.String()) + }) + } +} + +func TestDetermineRetryAction(t *testing.T) { + // Arrange + testCases := []struct { + name string + err error + expected retryAction + }{ + { + name: "NilError", + err: nil, + expected: noRetry, + }, + { + name: "GoogleApiError400", + err: &googleapi.Error{Code: 400}, + expected: noRetry, + }, + { + name: "GoogleApiError401", + err: &googleapi.Error{Code: 401}, + expected: retry401, + }, + { + name: "GoogleApiError429", + err: &googleapi.Error{Code: 429}, + expected: retryTransient, + }, + { + name: "UnauthenticatedGrpcError", + err: status.Error(codes.Unauthenticated, "unauthenticated"), + expected: retryUnauthenticated, + }, + { + name: "PermissionDeniedGrpcError", + err: status.Error(codes.PermissionDenied, "permission denied"), + expected: noRetry, + }, + { + name: "UnexpectedEOF", + err: io.ErrUnexpectedEOF, + expected: retryTransient, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Act + actual := determineRetryAction(tc.err) + + // Assert + assert.Equal(t, tc.expected, actual) + }) + } +} + +// logBuffer is a thread-safe buffer for capturing logs in tests. +type logBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *logBuffer) Write(p []byte) (n int, err error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *logBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +func TestShouldRetryLogsWarning(t *testing.T) { + // Arrange + var buf logBuffer + logger.SetOutput(&buf) + defer logger.SetOutput(os.Stdout) + var err401 = &googleapi.Error{ + Code: 401, + Body: "Invalid Credential", + } + + // Act + retry := ShouldRetry(err401) + + // Assert + assert.True(t, retry) + assert.Contains(t, buf.String(), "WARNING") + assert.Contains(t, buf.String(), "Retrying for error-code 401") +} + +type fakeMetricHandle struct { + metrics.MetricHandle + + gcsRetryCountCalled bool + gcsRetryCountInc int64 + gcsRetryErrorCategory string +} + +func (m *fakeMetricHandle) GcsRetryCount(inc int64, val metrics.RetryErrorCategory) { + m.gcsRetryCountCalled = true + m.gcsRetryCountInc = inc + m.gcsRetryErrorCategory = string(val) +} + +func TestShouldRetryWithMonitoringForNonRetryableErrors(t *testing.T) { + testCases := []struct { + name string + err error + }{ + { + name: "nil error", + err: nil, + }, + { + name: "non-retryable error", + err: &googleapi.Error{Code: 400}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeMetrics := &fakeMetricHandle{ + MetricHandle: metrics.NewNoopMetrics(), + } + + shouldRetry := ShouldRetryWithMonitoring(context.Background(), tc.err, fakeMetrics) + + assert.False(t, shouldRetry) + assert.False(t, fakeMetrics.gcsRetryCountCalled) + }) + } +} + +func TestShouldRetryWithMonitoringForRetryableErrors(t *testing.T) { + retryableErr := &googleapi.Error{Code: 429} + + testCases := []struct { + name string + err error + expectedMetricCategory string + }{ + { + name: "retryable error, DeadlineExceeded", + err: context.DeadlineExceeded, + expectedMetricCategory: "STALLED_READ_REQUEST", + }, + { + name: "retryable error, not DeadlineExceeded", + err: retryableErr, + expectedMetricCategory: "OTHER_ERRORS", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeMetrics := &fakeMetricHandle{ + MetricHandle: metrics.NewNoopMetrics(), + } + + shouldRetry := ShouldRetryWithMonitoring(context.Background(), tc.err, fakeMetrics) + + assert.True(t, shouldRetry) + assert.True(t, fakeMetrics.gcsRetryCountCalled) + assert.Equal(t, int64(1), fakeMetrics.gcsRetryCountInc) + assert.Equal(t, tc.expectedMetricCategory, fakeMetrics.gcsRetryErrorCategory) + }) + } +} diff --git a/internal/storage/storageutil/delete_all_objects.go b/internal/storage/storageutil/delete_all_objects.go index 3ec8b77575..018420ef85 100644 --- a/internal/storage/storageutil/delete_all_objects.go +++ b/internal/storage/storageutil/delete_all_objects.go @@ -15,7 +15,7 @@ package storageutil import ( - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "golang.org/x/net/context" "golang.org/x/sync/errgroup" ) @@ -53,7 +53,7 @@ func DeleteAllObjects( // Delete the objects in parallel. const parallelism = 64 - for i := 0; i < parallelism; i++ { + for range parallelism { group.Go(func() error { for objectName := range objectNames { err := bucket.DeleteObject( diff --git a/common/telemetry.go b/internal/storage/storageutil/delete_object.go similarity index 59% rename from common/telemetry.go rename to internal/storage/storageutil/delete_object.go index 572eca36d6..50db4d91fc 100644 --- a/common/telemetry.go +++ b/internal/storage/storageutil/delete_object.go @@ -12,25 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -package common +package storageutil import ( - "context" - "errors" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "golang.org/x/net/context" ) -type ShutdownFn func(ctx context.Context) error - -// JoinShutdownFunc combines the provided shutdown functions into a single function. -func JoinShutdownFunc(shutdownFns ...ShutdownFn) ShutdownFn { - return func(ctx context.Context) error { - var err error - for _, fn := range shutdownFns { - if fn == nil { - continue - } - err = errors.Join(err, fn(ctx)) - } - return err +// DeleteObject deletes an object in the given bucket with the given name. +func DeleteObject( + ctx context.Context, + bucket gcs.Bucket, + name string) error { + req := &gcs.DeleteObjectRequest{ + Name: name, + Generation: 0, } + + return bucket.DeleteObject(ctx, req) } diff --git a/internal/storage/storageutil/list_all.go b/internal/storage/storageutil/list_all.go index 7b72dbf8e6..a2e19a7019 100644 --- a/internal/storage/storageutil/list_all.go +++ b/internal/storage/storageutil/list_all.go @@ -15,7 +15,7 @@ package storageutil import ( - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "golang.org/x/net/context" ) diff --git a/internal/storage/storageutil/list_prefix.go b/internal/storage/storageutil/list_prefix.go index 501fa0618a..d68292e105 100644 --- a/internal/storage/storageutil/list_prefix.go +++ b/internal/storage/storageutil/list_prefix.go @@ -17,7 +17,7 @@ package storageutil import ( "fmt" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "golang.org/x/net/context" ) diff --git a/internal/storage/storageutil/object_attrs.go b/internal/storage/storageutil/object_attrs.go index 3ddc1232a9..66f7189655 100644 --- a/internal/storage/storageutil/object_attrs.go +++ b/internal/storage/storageutil/object_attrs.go @@ -19,7 +19,7 @@ import ( "time" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" storagev1 "google.golang.org/api/storage/v1" ) @@ -96,6 +96,7 @@ func ObjectAttrsToBucketObject(attrs *storage.ObjectAttrs) *gcs.Object { StorageClass: attrs.StorageClass, Deleted: attrs.Deleted, Updated: attrs.Updated, + Finalized: attrs.Finalized, ComponentCount: attrs.ComponentCount, ContentDisposition: attrs.ContentDisposition, CustomTime: string(attrs.CustomTime.Format(time.RFC3339)), @@ -122,6 +123,7 @@ func ObjectAttrsToMinObject(attrs *storage.ObjectAttrs) *gcs.MinObject { Generation: attrs.Generation, MetaGeneration: attrs.Metageneration, Updated: attrs.Updated, + Finalized: attrs.Finalized, } } @@ -172,6 +174,7 @@ func ConvertObjToMinObject(o *gcs.Object) *gcs.MinObject { Metadata: o.Metadata, ContentEncoding: o.ContentEncoding, CRC32C: o.CRC32C, + Finalized: o.Finalized, } } @@ -209,6 +212,7 @@ func ConvertMinObjectAndExtendedObjectAttributesToObject(m *gcs.MinObject, Generation: m.Generation, MetaGeneration: m.MetaGeneration, Updated: m.Updated, + Finalized: m.Finalized, Metadata: m.Metadata, ContentEncoding: m.ContentEncoding, ContentType: e.ContentType, @@ -242,5 +246,6 @@ func ConvertMinObjectToObject(m *gcs.MinObject) *gcs.Object { Metadata: m.Metadata, ContentEncoding: m.ContentEncoding, CRC32C: m.CRC32C, + Finalized: m.Finalized, } } diff --git a/internal/storage/storageutil/object_attrs_test.go b/internal/storage/storageutil/object_attrs_test.go index 23a9c4a70f..769bf0649d 100644 --- a/internal/storage/storageutil/object_attrs_test.go +++ b/internal/storage/storageutil/object_attrs_test.go @@ -21,7 +21,7 @@ import ( "time" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" . "github.com/jacobsa/ogletest" storagev1 "google.golang.org/api/storage/v1" ) @@ -98,6 +98,7 @@ func (t objectAttrsTest) TestObjectAttrsToBucketObjectMethod() { Created: timeAttr, Deleted: timeAttr, Updated: timeAttr, + Finalized: timeAttr, CustomerKeySHA256: "CustomerKeySHA256", KMSKeyName: "KMSKeyName", Prefix: "Prefix", @@ -133,6 +134,7 @@ func (t objectAttrsTest) TestObjectAttrsToBucketObjectMethod() { ExpectEq(object.MetaGeneration, attrs.Metageneration) ExpectEq(object.StorageClass, attrs.StorageClass) ExpectEq(object.Updated.String(), attrs.Updated.String()) + ExpectEq(object.Finalized.String(), attrs.Finalized.String()) ExpectEq(object.Deleted.String(), attrs.Deleted.String()) ExpectEq(object.ContentDisposition, attrs.ContentDisposition) ExpectEq(object.CustomTime, customeTimeExpected) @@ -241,6 +243,7 @@ func (t objectAttrsTest) Test_ConvertObjToMinObject_WithValidObject() { Generation: generation, MetaGeneration: metaGeneration, Updated: currentTime, + Finalized: currentTime, Metadata: metadata, ContentEncoding: contentEncode, CRC32C: &crc32C, @@ -254,6 +257,7 @@ func (t objectAttrsTest) Test_ConvertObjToMinObject_WithValidObject() { ExpectEq(generation, gcsMinObject.Generation) ExpectEq(metaGeneration, gcsMinObject.MetaGeneration) ExpectTrue(currentTime.Equal(gcsMinObject.Updated)) + ExpectTrue(currentTime.Equal(gcsMinObject.Finalized)) ExpectEq(contentEncode, gcsMinObject.ContentEncoding) ExpectEq(metadata, gcsMinObject.Metadata) ExpectEq(crc32C, *gcsMinObject.CRC32C) @@ -345,6 +349,7 @@ func (t objectAttrsTest) Test_ConvertObjToExtendedObjectAttributes_WithNonNilMin Generation: int64(444), MetaGeneration: int64(555), Updated: timeAttr, + Finalized: timeAttr, Metadata: map[string]string{"test_key": "test_value"}, ContentEncoding: "test_encoding", } @@ -372,6 +377,7 @@ func (t objectAttrsTest) Test_ConvertObjToExtendedObjectAttributes_WithNonNilMin ExpectEq(gcsObject.Generation, minObject.Generation) ExpectEq(gcsObject.MetaGeneration, minObject.MetaGeneration) ExpectEq(0, gcsObject.Updated.Compare(minObject.Updated)) + ExpectEq(0, gcsObject.Finalized.Compare(minObject.Finalized)) ExpectEq(gcsObject.Metadata, minObject.Metadata) ExpectEq(gcsObject.ContentEncoding, minObject.ContentEncoding) ExpectEq(gcsObject.ContentType, extendedObjAttr.ContentType) @@ -407,6 +413,7 @@ func (t objectAttrsTest) Test_ConvertMinObjectToObject_WithNonNilMinObject() { Generation: int64(444), MetaGeneration: int64(555), Updated: timeAttr, + Finalized: timeAttr, Metadata: map[string]string{"test_key": "test_value"}, ContentEncoding: "test_encoding", CRC32C: &crc32C, @@ -420,6 +427,7 @@ func (t objectAttrsTest) Test_ConvertMinObjectToObject_WithNonNilMinObject() { ExpectEq(gcsObject.Generation, minObject.Generation) ExpectEq(gcsObject.MetaGeneration, minObject.MetaGeneration) ExpectEq(0, gcsObject.Updated.Compare(minObject.Updated)) + ExpectEq(0, gcsObject.Finalized.Compare(minObject.Finalized)) ExpectEq(gcsObject.Metadata, minObject.Metadata) ExpectEq(gcsObject.ContentEncoding, minObject.ContentEncoding) ExpectEq(gcsObject.ContentType, "") diff --git a/internal/storage/storageutil/read_object.go b/internal/storage/storageutil/read_object.go index 7827df16e7..2e9359058a 100644 --- a/internal/storage/storageutil/read_object.go +++ b/internal/storage/storageutil/read_object.go @@ -18,7 +18,7 @@ import ( "fmt" "io" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "golang.org/x/net/context" ) @@ -33,7 +33,7 @@ func ReadObject( Name: name, } - rc, err := bucket.NewReader(ctx, req) + rc, err := bucket.NewReaderWithReadHandle(ctx, req) if err != nil { return } diff --git a/internal/storage/storageutil/retry.go b/internal/storage/storageutil/retry.go new file mode 100644 index 0000000000..dd8200e27c --- /dev/null +++ b/internal/storage/storageutil/retry.go @@ -0,0 +1,224 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storageutil + +import ( + "context" + "fmt" + "log/slog" + "math/rand" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" +) + +const ( + // Default retry parameters. + DefaultRetryDeadline = 30 * time.Second + DefaultTotalRetryBudget = 5 * time.Minute + DefaultInitialBackoff = 1 * time.Second +) + +// exponentialBackoffConfig is config parameters +// needed to create an exponentialBackoff. +type exponentialBackoffConfig struct { + // initial duration for next backoff. + initial time.Duration + // max duration for next backoff. + max time.Duration + // The rate at which the backoff duration should grow + // over subsequent calls to next(). + multiplier float64 +} + +// exponentialBackoff holds the duration parameters for exponential backoff. +type exponentialBackoff struct { + // config used to create this backoff object. + config exponentialBackoffConfig + // Duration for next backoff. Capped at max. Returned by next(). + next time.Duration +} + +// newExponentialBackoff returns a new exponentialBackoff given +// the config for it. +func newExponentialBackoff(config *exponentialBackoffConfig) *exponentialBackoff { + return &exponentialBackoff{ + config: *config, + next: config.initial, + } +} + +// nextDuration returns the next backoff duration. +func (b *exponentialBackoff) nextDuration() time.Duration { + next := b.next + b.next = min(b.config.max, time.Duration(float64(b.next)*b.config.multiplier)) + return next +} + +// waitWithJitter waits for the next backoff duration with added jitter. +// The jitter adds randomness to the backoff duration to prevent the thundering herd problem. +// This is similar to how gax-retries backoff after each failed retry. +func (b *exponentialBackoff) waitWithJitter(ctx context.Context) error { + // If the context is already cancelled, return immediately. + if err := ctx.Err(); err != nil { + return err + } + + nextDuration := b.nextDuration() + jitteryBackoffDuration := time.Duration(1 + rand.Int63n(max(1, int64(nextDuration)))) + timer := time.NewTimer(jitteryBackoffDuration) + defer timer.Stop() + + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// RetryConfig holds configuration for retrying an operation. +type RetryConfig struct { + // Time-limit on every individual retry attempt. + RetryDeadline time.Duration + // Total duration allowed across all the attempts. + TotalRetryBudget time.Duration + // Max attempts to make for the operation. + MaxAttempts int + // Config for managing backoff durations in-between retry attempts. + BackoffConfig exponentialBackoffConfig +} + +// NewRetryConfig creates a new RetryConfig. +func NewRetryConfig(clientConfig *StorageClientConfig, retryDeadline, totalRetryBudget, initialBackoff time.Duration) *RetryConfig { + return &RetryConfig{ + RetryDeadline: retryDeadline, + TotalRetryBudget: totalRetryBudget, + MaxAttempts: clientConfig.MaxRetryAttempts, + BackoffConfig: exponentialBackoffConfig{ + initial: initialBackoff, + max: clientConfig.MaxRetrySleep, + multiplier: clientConfig.RetryMultiplier, + }, + } +} + +// ExecuteWithCustomShouldRetryAtLogLevel encapsulates the retry logic over a given operation. +// It performs time-bound, exponential backoff retries for a given API call. +// It is expected that the given apiCall returns a structure, and not an HTTP response, +// so that it does not leave behind any trace of a pending operation on server. +// It also has an option to control the log level of the initial attempt log, +// while subsequent retries are always logged at Warning level. +// It also accepts a custom shouldRetry predicate function. +func ExecuteWithCustomShouldRetryAtLogLevel[T any]( + ctx context.Context, + config *RetryConfig, + operationName string, + reqDescription string, + apiCall func(attemptCtx context.Context) (T, error), + shouldRetry func(err error) bool, + logLevel slog.Level, // Used to log the initial attempt at the supplied log level. Subsequent retries are logged at Warning level. +) (T, error) { + var zero T + // If the context is already cancelled, return immediately. + if err := ctx.Err(); err != nil { + return zero, err + } + + parentCtx, cancel := context.WithTimeout(ctx, config.TotalRetryBudget) + defer cancel() + + // Create a new backoff controller specific to this api call. + backoff := newExponentialBackoff(&config.BackoffConfig) + var lastErr error + for i := 0; ; i++ { + attemptCtx, attemptCancel := context.WithTimeout(parentCtx, config.RetryDeadline) + + if i == 0 { + logger.GetLogFHandler(logLevel)("Calling %s request for %q with deadline=%v", operationName, reqDescription, config.RetryDeadline) + } else { + logger.GetLogFHandler(logger.LevelWarn)("Retrying %s for %q with deadline=%v (error: %v) ...", operationName, reqDescription, config.RetryDeadline, lastErr) + } + + result, err := apiCall(attemptCtx) + lastErr = err + // Cancel attemptCtx after it is no longer needed, to free up its resources. + attemptCancel() + + if err == nil { + return result, nil + } + + if config.MaxAttempts > 0 && i+1 >= config.MaxAttempts { + return zero, fmt.Errorf("%s for %q failed after %d attempts (last server/client error = %v)", operationName, reqDescription, config.MaxAttempts, err) + } + + // If the error is not retryable, return it immediately. + if !shouldRetry(err) { + return zero, fmt.Errorf("%s for %q failed with a non-retryable error: %w", operationName, reqDescription, err) + } + + // If the parent context is cancelled/timed-out, we should stop retrying. + if parentCtx.Err() != nil { + return zero, fmt.Errorf("%s for %q failed after multiple retries (last server/client error = %v): %w", operationName, reqDescription, err, parentCtx.Err()) + } + + // Do a jittery backoff after each retry. + parentCtxErr := backoff.waitWithJitter(parentCtx) + if parentCtxErr != nil { + return zero, fmt.Errorf("%s for %q failed after multiple retries (last server/client error = %v): %w", operationName, reqDescription, err, parentCtxErr) + } + } +} + +// ExecuteWithCustomShouldRetry retries a given operation using a custom shouldRetry predicate, logging the initial attempt at trace level. +func ExecuteWithCustomShouldRetry[T any]( + ctx context.Context, + config *RetryConfig, + operationName string, + reqDescription string, + apiCall func(attemptCtx context.Context) (T, error), + shouldRetry func(err error) bool, +) (T, error) { + return ExecuteWithCustomShouldRetryAtLogLevel(ctx, config, operationName, reqDescription, apiCall, shouldRetry, logger.LevelTrace) +} + +// ExecuteWithRetryAtLogLevel encapsulates the retry logic over a given operation. +// It performs time-bound, exponential backoff retries for a given API call. +// It is expected that the given apiCall returns a structure, and not an HTTP response, +// so that it does not leave behind any trace of a pending operation on server. +// It also has an option to control the log level of the initial attempt log, +// while subsequent retries are always logged at Warning level. +func ExecuteWithRetryAtLogLevel[T any]( + ctx context.Context, + config *RetryConfig, + operationName string, + reqDescription string, + apiCall func(attemptCtx context.Context) (T, error), + logLevel slog.Level, // Used to log the initial attempt at the supplied log level. Subsequent retries are logged at Warning level. +) (T, error) { + return ExecuteWithCustomShouldRetryAtLogLevel(ctx, config, operationName, reqDescription, apiCall, ShouldRetryWithoutLogging, logLevel) +} + +// ExecuteWithRetry retries a given operation, logging the initial attempt at trace level. +func ExecuteWithRetry[T any]( + ctx context.Context, + config *RetryConfig, + operationName string, + reqDescription string, + apiCall func(attemptCtx context.Context) (T, error), +) (T, error) { + return ExecuteWithCustomShouldRetry(ctx, config, operationName, reqDescription, apiCall, ShouldRetryWithoutLogging) +} diff --git a/internal/storage/storageutil/retry_test.go b/internal/storage/storageutil/retry_test.go new file mode 100644 index 0000000000..a970eea19f --- /dev/null +++ b/internal/storage/storageutil/retry_test.go @@ -0,0 +1,605 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storageutil + +import ( + "context" + "errors" + "os" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type ExponentialBackoffTestSuite struct { + suite.Suite +} + +func TestExponentialBackoffTestSuite(t *testing.T) { + suite.Run(t, new(ExponentialBackoffTestSuite)) +} + +func TestExecuteWithRetryTestSuite(t *testing.T) { + suite.Run(t, new(ExecuteWithRetryTestSuite)) +} + +func TestRetryConfigTestSuite(t *testing.T) { + suite.Run(t, new(RetryConfigTestSuite)) +} + +func (t *ExponentialBackoffTestSuite) TestNewBackoff() { + initial := 1 * time.Second + maxValue := 10 * time.Second + multiplier := 2.0 + + b := newExponentialBackoff(&exponentialBackoffConfig{ + initial: initial, + max: maxValue, + multiplier: multiplier, + }) + + assert.NotNil(t.T(), b) + assert.Equal(t.T(), initial, b.next) + assert.Equal(t.T(), initial, b.config.initial) + assert.Equal(t.T(), maxValue, b.config.max) + assert.Equal(t.T(), multiplier, b.config.multiplier) +} + +func (t *ExponentialBackoffTestSuite) TestNext() { + initial := 1 * time.Second + maxValue := 3 * time.Second + multiplier := 2.0 + b := newExponentialBackoff(&exponentialBackoffConfig{ + initial: initial, + max: maxValue, + multiplier: multiplier, + }) + + // First call to next() should return initial, and update current. + assert.Equal(t.T(), 1*time.Second, b.nextDuration()) + + // Second call. + assert.Equal(t.T(), 2*time.Second, b.nextDuration()) + + // Third call - capped at max. + assert.Equal(t.T(), 3*time.Second, b.nextDuration()) + + // Should stay capped at max. + assert.Equal(t.T(), 3*time.Second, b.nextDuration()) +} + +func (t *ExponentialBackoffTestSuite) TestWaitWithJitter_ContextCancelled() { + initial := 100 * time.Microsecond // A long duration to ensure cancellation happens first. + maxValue := 5 * initial + b := newExponentialBackoff(&exponentialBackoffConfig{ + initial: initial, + max: maxValue, + multiplier: 2.0, + }) + ctx, cancel := context.WithCancel(context.Background()) + // Cancel the context immediately. + cancel() + + start := time.Now() + err := b.waitWithJitter(ctx) + elapsed := time.Since(start) + + assert.ErrorIs(t.T(), err, context.Canceled) + // The function should return almost immediately without backoff sleep. + assert.Less(t.T(), elapsed, 20*time.Millisecond, "waitWithJitter should return quickly when context is cancelled") +} + +func (t *ExponentialBackoffTestSuite) TestWaitWithJitter_NoContextCancelled() { + initial := 100 * time.Millisecond // Much larger than OS scheduler latency + maxValue := 5 * initial + b := newExponentialBackoff(&exponentialBackoffConfig{ + initial: initial, + max: maxValue, + multiplier: 2.0, + }) + ctx := context.Background() + + start := time.Now() + err := b.waitWithJitter(ctx) + elapsed := time.Since(start) + + assert.NoError(t.T(), err) + // Strict bound: allows at most 120ms for a max 100ms sleep. + assert.LessOrEqual(t.T(), elapsed, initial+20*time.Millisecond, "waitWithJitter should not wait excessively long") +} + +func (t *ExponentialBackoffTestSuite) TestWaitWithJitter_FirstAttemptBackoffGrowth() { + // Arrange + initial := 100 * time.Millisecond // Much larger than OS scheduler latency. + maxValue := 500 * time.Millisecond + multiplier := 2.0 + b := newExponentialBackoff(&exponentialBackoffConfig{ + initial: initial, + max: maxValue, + multiplier: multiplier, + }) + ctx := context.Background() + + expectedNext := initial + + // Act + start := time.Now() + err := b.waitWithJitter(ctx) + elapsed := time.Since(start) + + // Assert + assert.NoError(t.T(), err) + + expectedNext = min(maxValue, time.Duration(float64(expectedNext)*multiplier)) + assert.Equal(t.T(), expectedNext, b.next) + assert.LessOrEqual(t.T(), elapsed, initial+20*time.Millisecond) +} + +func (t *ExponentialBackoffTestSuite) TestWaitWithJitter_ConsecutiveAttemptsBackoffGrowth() { + // Arrange + initial := 100 * time.Millisecond // Much larger than OS scheduler latency. + maxValue := 500 * time.Millisecond + multiplier := 2.0 + b := newExponentialBackoff(&exponentialBackoffConfig{ + initial: initial, + max: maxValue, + multiplier: multiplier, + }) + ctx := context.Background() + + expectedNext := initial + + for range 4 { + // Act + start := time.Now() + err := b.waitWithJitter(ctx) + elapsed := time.Since(start) + + currentBackoff := expectedNext + + // Assert + require.NoError(t.T(), err) + + expectedNext = min(maxValue, time.Duration(float64(expectedNext)*multiplier)) + assert.Equal(t.T(), expectedNext, b.next) + require.LessOrEqual(t.T(), elapsed, currentBackoff+20*time.Millisecond) + } +} + +func (t *ExponentialBackoffTestSuite) TestWaitWithJitter_BoundsRespectMax() { + // Arrange + initial := 5 * time.Millisecond + maxValue := 20 * time.Millisecond + multiplier := 2.0 + b := newExponentialBackoff(&exponentialBackoffConfig{ + initial: initial, + max: maxValue, + multiplier: multiplier, + }) + expectedNext := initial + ctx := context.Background() + + // Call many times to let it saturate the max value and test the ceiling bounds + for range 20 { + // Act + start := time.Now() + err := b.waitWithJitter(ctx) + elapsed := time.Since(start) + + currentBackoff := expectedNext + + // Assert + require.NoError(t.T(), err) + expectedNext = min(maxValue, time.Duration(float64(expectedNext)*multiplier)) + // Measured wait time should be capped by current backoff ceiling plus a scheduler safety margin. + require.LessOrEqual(t.T(), elapsed, currentBackoff+20*time.Millisecond) + } +} + +type RetryConfigTestSuite struct { + suite.Suite +} + +func (t *RetryConfigTestSuite) TestNewRetryConfig() { + // Arrange + clientConfig := &StorageClientConfig{ + MaxRetrySleep: 10 * time.Second, + RetryMultiplier: 2.5, + } + retryDeadline := 5 * time.Second + totalRetryBudget := 30 * time.Second + initialBackoff := 500 * time.Millisecond + + // Act + retryConfig := NewRetryConfig(clientConfig, retryDeadline, totalRetryBudget, initialBackoff) + + // Assert + assert.NotNil(t.T(), retryConfig) + assert.Equal(t.T(), retryDeadline, retryConfig.RetryDeadline) + assert.Equal(t.T(), totalRetryBudget, retryConfig.TotalRetryBudget) + assert.Equal(t.T(), initialBackoff, retryConfig.BackoffConfig.initial) + assert.Equal(t.T(), clientConfig.MaxRetrySleep, retryConfig.BackoffConfig.max) + assert.Equal(t.T(), clientConfig.RetryMultiplier, retryConfig.BackoffConfig.multiplier) +} + +type ExecuteWithRetryTestSuite struct { + suite.Suite + retryConfig *RetryConfig +} + +func (t *ExecuteWithRetryTestSuite) SetupTest() { + t.retryConfig = &RetryConfig{ + RetryDeadline: 50 * time.Millisecond, + TotalRetryBudget: 200 * time.Millisecond, + BackoffConfig: exponentialBackoffConfig{ + initial: 1 * time.Millisecond, + max: 10 * time.Millisecond, + multiplier: 2, + }, + } +} + +func (t *ExecuteWithRetryTestSuite) TestExecuteWithRetry_SuccessOnFirstAttempt() { + // Arrange + var callCount int + apiCall := func(ctx context.Context) (string, error) { + callCount++ + return "success", nil + } + + // Act + result, err := ExecuteWithRetry(context.Background(), t.retryConfig, "testOp", "testReq", apiCall) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), "success", result) + assert.Equal(t.T(), 1, callCount) +} + +func (t *ExecuteWithRetryTestSuite) TestExecuteWithRetry_SuccessAfterRetry() { + // Arrange + var callCount int + retryableErr := status.Error(codes.Unavailable, "server unavailable") + apiCall := func(ctx context.Context) (string, error) { + callCount++ + if callCount == 1 { + return "", retryableErr + } + return "success", nil + } + + // Act + result, err := ExecuteWithRetry(context.Background(), t.retryConfig, "testOp", "testReq", apiCall) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), "success", result) + assert.Equal(t.T(), 2, callCount) +} + +func (t *ExecuteWithRetryTestSuite) TestExecuteWithRetry_FailureOnNonRetryableError() { + // Arrange + var callCount int + nonRetryableErr := errors.New("non-retryable error") + apiCall := func(ctx context.Context) (string, error) { + callCount++ + return "", nonRetryableErr + } + + // Act + result, err := ExecuteWithRetry(context.Background(), t.retryConfig, "testOp", "testReq", apiCall) + + // Assert + assert.ErrorIs(t.T(), err, nonRetryableErr) + assert.Empty(t.T(), result) + assert.Equal(t.T(), 1, callCount) +} + +func (t *ExecuteWithRetryTestSuite) TestExecuteWithRetry_RetryableThenNonRetryableError() { + // Arrange + var callCount int + retryableErr := status.Error(codes.Unavailable, "server unavailable") + nonRetryableErr := errors.New("non-retryable error") + apiCall := func(ctx context.Context) (string, error) { + callCount++ + if callCount == 1 { + return "", retryableErr + } + return "", nonRetryableErr + } + + // Act + result, err := ExecuteWithRetry(context.Background(), t.retryConfig, "testOp", "testReq", apiCall) + + // Assert + assert.ErrorIs(t.T(), err, nonRetryableErr) + assert.Empty(t.T(), result) + assert.Equal(t.T(), 2, callCount) +} + +func (t *ExecuteWithRetryTestSuite) TestExecuteWithRetry_Timeout() { + // Arrange + stallDuration := t.retryConfig.RetryDeadline + 100*time.Millisecond + var callCount int + apiCall := func(ctx context.Context) (string, error) { + callCount++ + // Simulate a call that always takes longer than the per-attempt deadline. + select { + case <-time.After(stallDuration): + // This case should not be hit, as the context deadline + // is shorter than stallDuration. + return "", errors.New("simulated apiCall finished before context timeout") + case <-ctx.Done(): + return "", ctx.Err() + } + } + + // Act + _, err := ExecuteWithRetry(context.Background(), t.retryConfig, "testOp", "testReq", apiCall) + + // Assert + assert.ErrorIs(t.T(), err, context.DeadlineExceeded, "Expected context.DeadlineExceeded because each attempt is designed to "+ + "take longer than the per-attempt deadline.") +} + +func (t *ExecuteWithRetryTestSuite) TestExecuteWithRetry_TotalRetryBudgetExceeded() { + // Arrange + var callCount int + // Set a short total retry budget. + t.retryConfig.TotalRetryBudget = 30 * time.Millisecond + t.retryConfig.RetryDeadline = 10 * time.Millisecond + t.retryConfig.BackoffConfig.initial = 2 * time.Millisecond // Ensure backoff pushes it over the edge. + apiCall := func(ctx context.Context) (string, error) { + callCount++ + return "", status.Error(codes.Unavailable, "server unavailable") + } + + // Act + _, err := ExecuteWithRetry(context.Background(), t.retryConfig, "testOp", "testReq", apiCall) + + // Assert + assert.ErrorIs(t.T(), err, context.DeadlineExceeded, "The error should be from the total retry budget timeout") + assert.Greater(t.T(), callCount, 1) +} + +func (t *ExecuteWithRetryTestSuite) TestExecuteWithRetry_ParentContextTimeoutShorterThanRetryDeadline() { + // Arrange + var callCount int + t.retryConfig.RetryDeadline = 100 * time.Millisecond + t.retryConfig.TotalRetryBudget = 1000 * time.Millisecond + stallDuration := t.retryConfig.RetryDeadline + 100*time.Millisecond + // Set a parent context timeout that is shorter than the total retry budget. + parentCtx, cancel := context.WithTimeout(context.Background(), t.retryConfig.RetryDeadline-50*time.Millisecond) + defer cancel() + apiCall := func(ctx context.Context) (string, error) { + callCount++ + select { + case <-time.After(stallDuration): + case <-ctx.Done(): + return "", ctx.Err() + } + // This will always fail with a retryable error. + return "", status.Error(codes.Unavailable, "server unavailable") + } + + // Act + // The parent context will be checked within ExecuteWithRetry before the first attempt, + // but the attempt will still proceed. The attempt's context will expire + // due to the parent's timeout. + result, err := ExecuteWithRetry(parentCtx, t.retryConfig, "testOp", "testReq", apiCall) + + // Assert + assert.ErrorIs(t.T(), err, context.DeadlineExceeded, "The error should be from the parent context's timeout") + assert.Empty(t.T(), result) + assert.Equal(t.T(), 1, callCount, "apiCall should have been called once") +} + +func (t *ExecuteWithRetryTestSuite) TestExecuteWithRetry_ParentContextTimeoutBetweenDeadlines() { + // Arrange + var callCount int + stallDuration := t.retryConfig.RetryDeadline + 100*time.Millisecond + // Set a parent context timeout that is longer than one attempt but shorter than the total budget. + parentCtx, cancel := context.WithTimeout(context.Background(), t.retryConfig.RetryDeadline+50*time.Millisecond) + defer cancel() + apiCall := func(ctx context.Context) (string, error) { + callCount++ + select { + case <-time.After(stallDuration): + case <-ctx.Done(): + return "", ctx.Err() + } + // This will always fail with a retryable error. + return "", status.Error(codes.Unavailable, "server unavailable") + } + + // Act + result, err := ExecuteWithRetry(parentCtx, t.retryConfig, "testOp", "testReq", apiCall) + + // Assert + assert.ErrorIs(t.T(), err, context.DeadlineExceeded, "The error should be from the parent context's timeout") + assert.Empty(t.T(), result) + assert.Greater(t.T(), callCount, 0, "apiCall should have been called at least once") +} + +func (t *ExecuteWithRetryTestSuite) TestExecuteWithRetry_ParentContextTimeoutLongerThanBudget() { + // Arrange + stallDuration := t.retryConfig.RetryDeadline + 100*time.Millisecond + parentCtx, cancel := context.WithTimeout(context.Background(), t.retryConfig.TotalRetryBudget+100*time.Millisecond) + defer cancel() + apiCall := func(ctx context.Context) (string, error) { + select { + case <-time.After(stallDuration): + case <-ctx.Done(): + return "", ctx.Err() + } + return "", status.Error(codes.Unavailable, "server unavailable") + } + + // Act + result, err := ExecuteWithRetry(parentCtx, t.retryConfig, "testOp", "testReq", apiCall) + + // Assert + assert.ErrorIs(t.T(), err, context.DeadlineExceeded, "The error should be from context created in ExecuteWithRetry") + assert.Empty(t.T(), result) +} + +func (t *ExecuteWithRetryTestSuite) TestExecuteWithRetry_ParentContextAlreadyCancelled() { + // Arrange + var callCount int + parentCtx, cancel := context.WithCancel(context.Background()) + cancel() // Cancel the context immediately. + apiCall := func(ctx context.Context) (string, error) { + callCount++ + return "should not be called", nil + } + + // Act + _, err := ExecuteWithRetry(parentCtx, t.retryConfig, "testOp", "testReq", apiCall) + + // Assert + assert.ErrorIs(t.T(), err, context.Canceled) + assert.Equal(t.T(), 0, callCount, "apiCall should not have been executed") +} + +func (t *ExecuteWithRetryTestSuite) TestExecuteWithRetry_MaxAttemptsReached() { + // Arrange + var callCount int + t.retryConfig.MaxAttempts = 2 + retryableErr := status.Error(codes.Unavailable, "server unavailable") + apiCall := func(ctx context.Context) (string, error) { + callCount++ + return "", retryableErr + } + + // Act + _, err := ExecuteWithRetry(context.Background(), t.retryConfig, "testOp", "testReq", apiCall) + + // Assert + assert.Error(t.T(), err) + assert.Contains(t.T(), err.Error(), "failed after 2 attempts") + assert.Equal(t.T(), 2, callCount, "apiCall should have been called exactly 2 times") +} + +func (t *ExecuteWithRetryTestSuite) TestExecuteWithCustomShouldRetry_SuccessWithCustomPredicate() { + // Arrange + var callCount int + customErr := errors.New("my custom transient error") + apiCall := func(ctx context.Context) (string, error) { + callCount++ + if callCount == 1 { + return "", customErr + } + return "success", nil + } + customShouldRetry := func(err error) bool { + return errors.Is(err, customErr) + } + + // Act + result, err := ExecuteWithCustomShouldRetry(context.Background(), t.retryConfig, "testOp", "testReq", apiCall, customShouldRetry) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), "success", result) + assert.Equal(t.T(), 2, callCount) +} + +func (t *ExecuteWithRetryTestSuite) TestExecuteWithCustomShouldRetry_FailWithCustomPredicate() { + // Arrange + var callCount int + defaultRetryableErr := status.Error(codes.Unavailable, "server unavailable") + apiCall := func(ctx context.Context) (string, error) { + callCount++ + return "", defaultRetryableErr + } + // A custom predicate that rejects everything (returns false) + customShouldRetry := func(err error) bool { + return false + } + + // Act + result, err := ExecuteWithCustomShouldRetry(context.Background(), t.retryConfig, "testOp", "testReq", apiCall, customShouldRetry) + + // Assert + assert.ErrorIs(t.T(), err, defaultRetryableErr) + assert.Empty(t.T(), result) + assert.Equal(t.T(), 1, callCount) +} + +func (t *ExecuteWithRetryTestSuite) TestExecuteWithCustomShouldRetry_CompositionWithDefault() { + // Arrange + var callCount int + customErr := errors.New("my custom transient error") + defaultRetryableErr := status.Error(codes.Unavailable, "server unavailable") + apiCall := func(ctx context.Context) (string, error) { + callCount++ + if callCount == 1 { + return "", defaultRetryableErr + } + if callCount == 2 { + return "", customErr + } + return "success", nil + } + // Wrap default ShouldRetry and also allow customErr + customShouldRetry := func(err error) bool { + return ShouldRetry(err) || errors.Is(err, customErr) + } + + // Act + result, err := ExecuteWithCustomShouldRetry(context.Background(), t.retryConfig, "testOp", "testReq", apiCall, customShouldRetry) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), "success", result) + assert.Equal(t.T(), 3, callCount) +} + +func (t *ExecuteWithRetryTestSuite) TestExecuteWithRetry_LogsErrorContext() { + // Arrange + var buf logBuffer + logger.SetOutput(&buf) + defer logger.SetOutput(os.Stdout) + var callCount int + retryableErr := errors.New("transient failure 429") + apiCall := func(ctx context.Context) (string, error) { + callCount++ + if callCount == 1 { + return "", retryableErr + } + return "success", nil + } + // We need a shouldRetry function that returns true for retryableErr. + customShouldRetry := func(err error) bool { + return err.Error() == "transient failure 429" + } + + // Act + // Call ExecuteWithCustomShouldRetryAtLogLevel with LevelWarn, so we can capture the Retrying log + _, err := ExecuteWithCustomShouldRetryAtLogLevel(context.Background(), t.retryConfig, "testOp", "testReq", apiCall, customShouldRetry, logger.LevelWarn) + + // Assert + assert.NoError(t.T(), err) + assert.Equal(t.T(), 2, callCount) + assert.Contains(t.T(), buf.String(), "Retrying testOp") + assert.Contains(t.T(), buf.String(), "testReq") + assert.Contains(t.T(), buf.String(), "transient failure 429") +} diff --git a/internal/storage/storageutil/test_util.go b/internal/storage/storageutil/test_util.go index c71efd6d0b..ccfc943340 100644 --- a/internal/storage/storageutil/test_util.go +++ b/internal/storage/storageutil/test_util.go @@ -17,15 +17,14 @@ package storageutil import ( "time" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" ) const CustomEndpoint = "https://localhost:9000" -const DummyKeyFile = "test/test_creds.json" const CustomTokenUrl = "http://custom-token-url" // GetDefaultStorageClientConfig is only for test. -func GetDefaultStorageClientConfig() (clientConfig StorageClientConfig) { +func GetDefaultStorageClientConfig(keyFile string) (clientConfig StorageClientConfig) { return StorageClientConfig{ ClientProtocol: cfg.HTTP1, MaxConnsPerHost: 10, @@ -36,12 +35,13 @@ func GetDefaultStorageClientConfig() (clientConfig StorageClientConfig) { RetryMultiplier: 2, UserAgent: "gcsfuse/unknown (Go version go1.20-pre3 cl/474093167 +a813be86df) (GCP:gcsfuse)", CustomEndpoint: "", - KeyFile: DummyKeyFile, + KeyFile: keyFile, TokenUrl: "", ReuseTokenFromUrl: true, ExperimentalEnableJsonRead: false, - AnonymousAccess: true, - EnableHNS: false, + AnonymousAccess: false, + EnableHNS: true, + EnableGoogleLibAuth: true, ReadStallRetryConfig: cfg.ReadStallGcsRetriesConfig{ Enable: false, InitialReqTimeout: 20 * time.Second, @@ -50,5 +50,6 @@ func GetDefaultStorageClientConfig() (clientConfig StorageClientConfig) { ReqIncreaseRate: 15, ReqTargetPercentile: 0.99, }, + WriteConfig: &cfg.WriteConfig{}, } } diff --git a/internal/storage/storageutil/testdata/key.json b/internal/storage/storageutil/testdata/key.json new file mode 100644 index 0000000000..59d8020acb --- /dev/null +++ b/internal/storage/storageutil/testdata/key.json @@ -0,0 +1,13 @@ +{ + "type": "service_account", + "project_id": "project_id", + "private_key_id": "private_key_id", + "private_key": "private_key", + "client_email": "client_email", + "client_id": "client_id", + "auth_uri": "auth_uri", + "token_uri": "token_uri", + "auth_provider_x509_cert_url": "auth_provider_x509_cert_url", + "client_x509_cert_url": "client_x509_cert_url", + "universe_domain": "googleapis.com" +} diff --git a/internal/storage/storageutil/unsupported_object_util.go b/internal/storage/storageutil/unsupported_path_util.go similarity index 60% rename from internal/storage/storageutil/unsupported_object_util.go rename to internal/storage/storageutil/unsupported_path_util.go index 87cc80464b..9bbf76d601 100644 --- a/internal/storage/storageutil/unsupported_object_util.go +++ b/internal/storage/storageutil/unsupported_path_util.go @@ -15,33 +15,35 @@ package storageutil import ( + "slices" "strings" ) var ( - unsupportedObjectNameSubstrings = []string{"//"} - unsupportedObjectNamePrefixes = []string{"/"} - unsupportedObjectNames = []string{""} + unsupportedPathNameSubstrings = []string{"//", "/../", "/./"} + unsupportedPathNamePrefixes = []string{"/"} + unsupportedPathNameSuffix = []string{"/.", "/.."} + unsupportedPathNames = []string{"", ".", ".."} ) -// IsUnsupportedObjectName returns true if the passed +// IsUnsupportedPath returns true if the passed // string is a valid GCS object name or prefix, // which is unsupported in GCSFuse. -func IsUnsupportedObjectName(name string) bool { - for _, substring := range unsupportedObjectNameSubstrings { +func IsUnsupportedPath(name string) bool { + for _, substring := range unsupportedPathNameSubstrings { if strings.Contains(name, substring) { return true } } - for _, prefix := range unsupportedObjectNamePrefixes { + for _, prefix := range unsupportedPathNamePrefixes { if strings.HasPrefix(name, prefix) { return true } } - for _, unsupportedObjectName := range unsupportedObjectNames { - if name == unsupportedObjectName { + for _, suffix := range unsupportedPathNameSuffix { + if strings.HasSuffix(name, suffix) { return true } } - return false + return slices.Contains(unsupportedPathNames, name) } diff --git a/internal/storage/storageutil/unsupported_object_util_test.go b/internal/storage/storageutil/unsupported_path_util_test.go similarity index 76% rename from internal/storage/storageutil/unsupported_object_util_test.go rename to internal/storage/storageutil/unsupported_path_util_test.go index f23344ae49..9337c00eb8 100644 --- a/internal/storage/storageutil/unsupported_object_util_test.go +++ b/internal/storage/storageutil/unsupported_path_util_test.go @@ -18,7 +18,7 @@ import ( "fmt" "testing" - . "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + . "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) @@ -38,7 +38,7 @@ func TestGcsUtil(t *testing.T) { // ////////////////////////////////////////////////////////////////////// // Tests // ////////////////////////////////////////////////////////////////////// -func (ts *GcsUtilTest) TestIsUnsupportedObjectName() { +func (ts *GcsUtilTest) TestIsUnsupportedPathName() { cases := []struct { name string isUnsupported bool @@ -75,11 +75,35 @@ func (ts *GcsUtilTest) TestIsUnsupportedObjectName() { name: "", isUnsupported: true, }, + { + name: "foo/.", + isUnsupported: true, + }, + { + name: "foo/..", + isUnsupported: true, + }, + { + name: "foo/./", + isUnsupported: true, + }, + { + name: "foo/../", + isUnsupported: true, + }, + { + name: "foo/.config", + isUnsupported: false, + }, + { + name: "foo/c..d", + isUnsupported: false, + }, } for _, tc := range cases { ts.Run(fmt.Sprintf("name=%s", tc.name), func() { - assert.Equal(ts.T(), tc.isUnsupported, IsUnsupportedObjectName(tc.name)) + assert.Equal(ts.T(), tc.isUnsupported, IsUnsupportedPath(tc.name)) }) } } diff --git a/internal/storage/testify_mock_bucket.go b/internal/storage/testify_mock_bucket.go index 8a822921c4..e140e4ce41 100644 --- a/internal/storage/testify_mock_bucket.go +++ b/internal/storage/testify_mock_bucket.go @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,9 +16,8 @@ package storage import ( "context" - "io" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "github.com/stretchr/testify/mock" ) @@ -37,9 +36,12 @@ func (m *TestifyMockBucket) BucketType() gcs.BucketType { return args.Get(0).(gcs.BucketType) } -func (m *TestifyMockBucket) NewReader(ctx context.Context, req *gcs.ReadObjectRequest) (io.ReadCloser, error) { +func (m *TestifyMockBucket) NewReaderWithReadHandle(ctx context.Context, req *gcs.ReadObjectRequest) (gcs.StorageReader, error) { args := m.Called(ctx, req) - return args.Get(0).(io.ReadCloser), args.Error(1) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(gcs.StorageReader), args.Error(1) } func (m *TestifyMockBucket) CreateObject(ctx context.Context, req *gcs.CreateObjectRequest) (*gcs.Object, error) { @@ -58,9 +60,22 @@ func (m *TestifyMockBucket) CreateObjectChunkWriter(ctx context.Context, req *gc return args.Get(0).(gcs.Writer), nil } -func (m *TestifyMockBucket) FinalizeUpload(ctx context.Context, w gcs.Writer) (*gcs.Object, error) { +func (m *TestifyMockBucket) CreateAppendableObjectWriter(ctx context.Context, req *gcs.CreateObjectChunkWriterRequest) (wc gcs.Writer, err error) { + args := m.Called(ctx, req) + if args.Get(1) != nil { + return nil, args.Error(1) + } + return args.Get(0).(gcs.Writer), nil +} + +func (m *TestifyMockBucket) FinalizeUpload(ctx context.Context, w gcs.Writer) (*gcs.MinObject, error) { args := m.Called(ctx, w.ObjectName()) - return args.Get(0).(*gcs.Object), args.Error(1) + return args.Get(0).(*gcs.MinObject), args.Error(1) +} + +func (m *TestifyMockBucket) FlushPendingWrites(ctx context.Context, w gcs.Writer) (*gcs.MinObject, error) { + args := m.Called(ctx, w) + return args.Get(0).(*gcs.MinObject), args.Error(1) } func (m *TestifyMockBucket) CopyObject(ctx context.Context, req *gcs.CopyObjectRequest) (*gcs.Object, error) { @@ -96,13 +111,21 @@ func (m *TestifyMockBucket) DeleteObject(ctx context.Context, req *gcs.DeleteObj return args.Error(0) } +func (m *TestifyMockBucket) MoveObject(ctx context.Context, req *gcs.MoveObjectRequest) (*gcs.Object, error) { + args := m.Called(ctx, req) + if args.Get(0) != nil { + return args.Get(0).(*gcs.Object), nil + } + return nil, args.Error(1) +} + func (m *TestifyMockBucket) DeleteFolder(ctx context.Context, folderName string) error { args := m.Called(ctx, folderName) return args.Error(0) } -func (m *TestifyMockBucket) GetFolder(ctx context.Context, folderName string) (*gcs.Folder, error) { - args := m.Called(ctx, folderName) +func (m *TestifyMockBucket) GetFolder(ctx context.Context, req *gcs.GetFolderRequest) (*gcs.Folder, error) { + args := m.Called(ctx, req) if args.Get(0) != nil { return args.Get(0).(*gcs.Folder), nil } @@ -124,3 +147,17 @@ func (m *TestifyMockBucket) CreateFolder(ctx context.Context, folderName string) } return nil, args.Error(1) } + +func (m *TestifyMockBucket) NewMultiRangeDownloader( + ctx context.Context, req *gcs.MultiRangeDownloaderRequest) (gcs.MultiRangeDownloader, error) { + args := m.Called(ctx, req) + if args.Get(0) != nil { + return args.Get(0).(gcs.MultiRangeDownloader), nil + } + return nil, args.Error(1) +} + +func (m *TestifyMockBucket) GCSName(obj *gcs.MinObject) string { + args := m.Called(obj) + return args.Get(0).(string) +} diff --git a/internal/util/diskutil/disk_util.go b/internal/util/diskutil/disk_util.go new file mode 100644 index 0000000000..e1248b8117 --- /dev/null +++ b/internal/util/diskutil/disk_util.go @@ -0,0 +1,68 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package diskutil + +import ( + "syscall" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" +) + +const ( + // defaultVolumeBlockSize is the block-size used if statfs fails. + // 4 KiB + defaultVolumeBlockSize uint64 = 4096 + + // maxVolumeBlockSize is the max block-size supported for sanity. Beyond this, block-size returned by statfs will be considered suspiciously large and will be set to defaultVolumeBlockSize. + // 1 MiB + maxVolumeBlockSize uint64 = 1024 * 1024 +) + +// GetSpeculativeFileSizeOnDisk calculates the theoretical disk space a file will +// consume given its actual content size and the filesystem's block size. It rounds +// up the content size to the nearest block boundary to simulate block allocation. +func GetSpeculativeFileSizeOnDisk(fileContentSize, volumeBlockSize uint64) uint64 { + if volumeBlockSize <= 1 { + return fileContentSize + } + numBlocks := (fileContentSize + volumeBlockSize - 1) / volumeBlockSize + return numBlocks * volumeBlockSize +} + +// GetVolumeBlockSize retrieves the block size of the file system containing the given path +// using statfs sys-call. The block-size can be 0 or 2^n. The most common value is 4096. +func GetVolumeBlockSize(path string) uint64 { + var stat syscall.Statfs_t + if err := syscall.Statfs(path, &stat); err != nil { + logger.Errorf("statsfs failed for %q: %v. Defaulting to block-size %d for this directory.", path, err, defaultVolumeBlockSize) + return defaultVolumeBlockSize + } + // Prefer Frsize (fragment size) over Bsize for actual disk allocation if available. + // Refer https://github.com/cockroachdb/pebble/pull/1072. + blockSize := uint64(stat.Bsize) + if stat.Frsize > 0 { + blockSize = uint64(stat.Frsize) + } + // Sanity check: If the value is 0 or suspiciously large, fallback to default. + if blockSize == 0 { + logger.Errorf("statfs for %q returned Bsize = 0, so defaulting to %d", path, defaultVolumeBlockSize) + return defaultVolumeBlockSize + } + if blockSize > maxVolumeBlockSize { + logger.Errorf("statfs for %q returned Bsize (%d), which is too high, so defaulting to %d", path, blockSize, defaultVolumeBlockSize) + return defaultVolumeBlockSize + } + return blockSize +} diff --git a/internal/util/diskutil/disk_util_test.go b/internal/util/diskutil/disk_util_test.go new file mode 100644 index 0000000000..bd81540ab3 --- /dev/null +++ b/internal/util/diskutil/disk_util_test.go @@ -0,0 +1,101 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package diskutil_test + +import ( + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/util/diskutil" + "github.com/stretchr/testify/assert" +) + +func TestGetSpeculativeFileSizeOnDisk(t *testing.T) { + testcases := []struct { + name string + fileContentSize uint64 + volumeBlockSize uint64 + expectedSize uint64 + }{ + { + name: "Zero_Block_Size", + fileContentSize: 100, + volumeBlockSize: 0, + expectedSize: 100, + }, + { + name: "Block_Size_One", + fileContentSize: 100, + volumeBlockSize: 1, + expectedSize: 100, + }, + { + name: "Zero_File_Size", + fileContentSize: 0, + volumeBlockSize: 4096, + expectedSize: 0, + }, + { + name: "File_Size_Less_Than_Block_Size", + fileContentSize: 1, + volumeBlockSize: 4096, + expectedSize: 4096, + }, + { + name: "File_Size_Equal_To_Block_Size", + fileContentSize: 4096, + volumeBlockSize: 4096, + expectedSize: 4096, + }, + { + name: "File_Size_Greater_Than_Block_Size", + fileContentSize: 4097, + volumeBlockSize: 4096, + expectedSize: 8192, + }, + { + name: "File_Size_Much_Greater_Than_Block_Size", + fileContentSize: 10000, + volumeBlockSize: 4096, + expectedSize: 12288, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + speculativeSize := diskutil.GetSpeculativeFileSizeOnDisk(tc.fileContentSize, tc.volumeBlockSize) + + assert.Equal(t, tc.expectedSize, speculativeSize) + }) + } +} + +func TestGetVolumeBlockSize_ProperDir(t *testing.T) { + dir := t.TempDir() + + blockSize := diskutil.GetVolumeBlockSize(dir) + + // expect actual block-size (which is positive and power of 2) if directory exists and is proper. + assert.True(t, blockSize > 0 && ((blockSize&(blockSize-1)) == 0), "Block-size of a directory should be a power of 2. Got: %d", blockSize) +} + +func TestGetVolumeBlockSize_InvalidDir(t *testing.T) { + dir := "/path/that/does/not/exist" + expectedVolumeBlockSize := uint64(4096) + + blockSize := diskutil.GetVolumeBlockSize(dir) + + // expect default value if directory doesn't exist. + assert.Equal(t, expectedVolumeBlockSize, blockSize) +} diff --git a/internal/util/file_util.go b/internal/util/file_util.go new file mode 100644 index 0000000000..b5ba14fdd2 --- /dev/null +++ b/internal/util/file_util.go @@ -0,0 +1,112 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +// Available file access modes, corresponding to O_RDONLY, O_WRONLY, O_RDWR. +const ( + ReadOnly int = iota + WriteOnly + ReadWrite +) + +// Available file flags. +const ( + O_APPEND int = 1 << iota // O_APPEND + O_DIRECT // O_DIRECT +) + +// OpenMode represents the file open mode. +type OpenMode struct { + // accessMode defines the mutually exclusive access modes for opening a file. + accessMode int + + // fileFlags defines flags that modify the open/read/write behavior. + // These can be combined using bitwise OR. + fileFlags int +} + +// NewOpenMode constructs OpenMode given the accessMode and fileFlags. +func NewOpenMode(accessMode int, fileFlags int) OpenMode { + return OpenMode{ + accessMode: accessMode, + fileFlags: fileFlags, + } +} + +func (om OpenMode) AccessMode() int { + return om.accessMode +} + +func (om OpenMode) FileFlags() int { + return om.fileFlags +} + +// IsAppend checks if the file was opened in append mode. +// A file is considered in append mode (as suited for GCSFuse logic) if it is not +// opened as read-only and the O_APPEND flag is set. +func (om OpenMode) IsAppend() bool { + return om.accessMode != ReadOnly && om.fileFlags&O_APPEND != 0 +} + +// IsDirect checks if the O_DIRECT flag is set, indicating that I/O should +// bypass the kernel's page cache. +func (om OpenMode) IsDirect() bool { + return om.fileFlags&O_DIRECT != 0 +} + +// OpenFlagAttributes provides an abstraction for the open flags received from +// the FUSE kernel. This interface is necessary because the concrete type for open +// flags in `jacobsa/fuse` (e.g., in `fuseops.OpenFileOp` and `fuseops.CreateFileOp`) +// resides in an internal package and cannot be directly referenced. +// +// This abstraction allows a single function, `FileOpenMode`, to process flags +// from different FUSE operations and also simplifies unit testing. +type OpenFlagAttributes interface { + IsReadOnly() bool + IsWriteOnly() bool + IsReadWrite() bool + IsAppend() bool + IsDirect() bool +} + +// Function to obtain the mutually exclusive access mode based on the flags passed. +func getAccessMode(flags OpenFlagAttributes) int { + if flags.IsReadOnly() { + return ReadOnly + } else if flags.IsWriteOnly() { + return WriteOnly + } else { + return ReadWrite + } +} + +// Combine behavior-modifying file flags like O_DIRECT and O_APPEND. +func getFileFlags(flags OpenFlagAttributes) int { + var fileFlags int + if flags.IsAppend() { + fileFlags |= O_APPEND + } + if flags.IsDirect() { + fileFlags |= O_DIRECT + } + return fileFlags +} + +// FileOpenMode analyzes the open flags to determine the file's open mode. +func FileOpenMode(flags OpenFlagAttributes) OpenMode { + accessMode := getAccessMode(flags) + fileFlags := getFileFlags(flags) + return NewOpenMode(accessMode, fileFlags) +} diff --git a/internal/util/file_util_test.go b/internal/util/file_util_test.go new file mode 100644 index 0000000000..6f23c4fe03 --- /dev/null +++ b/internal/util/file_util_test.go @@ -0,0 +1,107 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// mockOpenFlags is a mock implementation of OpenFlagAttributes for testing. +type mockOpenFlags struct { + fReadOnly bool + fWriteOnly bool + fReadWrite bool + fAppend bool + fODirect bool +} + +func (f *mockOpenFlags) IsReadOnly() bool { return f.fReadOnly } +func (f *mockOpenFlags) IsWriteOnly() bool { return f.fWriteOnly } +func (f *mockOpenFlags) IsReadWrite() bool { return f.fReadWrite } +func (f *mockOpenFlags) IsAppend() bool { return f.fAppend } +func (f *mockOpenFlags) IsDirect() bool { return f.fODirect } + +func TestFileOpenMode(t *testing.T) { + testCases := []struct { + name string + flags OpenFlagAttributes + expectedMode OpenMode + }{ + { + name: "ReadOnly", + flags: &mockOpenFlags{ + fReadOnly: true, + }, + expectedMode: NewOpenMode(ReadOnly, 0), + }, + { + name: "WriteOnly", + flags: &mockOpenFlags{ + fWriteOnly: true, + }, + expectedMode: NewOpenMode(WriteOnly, 0), + }, + { + name: "ReadWrite", + flags: &mockOpenFlags{ + fReadWrite: true, + }, + expectedMode: NewOpenMode(ReadWrite, 0), + }, + { + name: "ReadWrite with Append", + flags: &mockOpenFlags{ + fReadWrite: true, + fAppend: true, + }, + expectedMode: NewOpenMode(ReadWrite, O_APPEND), + }, + { + name: "WriteOnly with Append and O_DIRECT", + flags: &mockOpenFlags{ + fWriteOnly: true, + fAppend: true, + fODirect: true, + }, + expectedMode: NewOpenMode(WriteOnly, O_APPEND|O_DIRECT), + }, + { + name: "ReadOnly with O_DIRECT", + flags: &mockOpenFlags{ + fReadOnly: true, + fODirect: true, + }, + expectedMode: NewOpenMode(ReadOnly, O_DIRECT), + }, + { + name: "ReadWrite with all behavioural flags", + flags: &mockOpenFlags{ + fReadWrite: true, + fAppend: true, + fODirect: true, + }, + expectedMode: NewOpenMode(ReadWrite, O_APPEND|O_DIRECT), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + mode := FileOpenMode(tc.flags) + assert.Equal(t, tc.expectedMode, mode) + }) + } +} diff --git a/internal/util/sizeof.go b/internal/util/sizeof.go index 9b4a885478..e30b3849b2 100644 --- a/internal/util/sizeof.go +++ b/internal/util/sizeof.go @@ -16,32 +16,22 @@ package util import ( "reflect" + "unsafe" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "google.golang.org/api/googleapi" - storagev1 "google.golang.org/api/storage/v1" ) var ( - // pointerSize represents the size of the pointer of any type. - pointerSize int - emptyStringSize int - emptyStringArraySize int - emptyObjectAccessControlSize int - emptyObjectAccessControlProjectTeamSize int + emptyStringSize int + emptyStringArraySize int ) func init() { - var i int - pointerSize = int(reflect.TypeOf(&i).Size()) var s string emptyStringSize = int(reflect.TypeOf(s).Size()) var sArray []string emptyStringArraySize = int(reflect.TypeOf(sArray).Size()) - var emptyObjectAccessControl storagev1.ObjectAccessControl - emptyObjectAccessControlSize = int(reflect.TypeOf(emptyObjectAccessControl).Size()) - var emptyObjectAccessControlProjectTeam storagev1.ObjectAccessControlProjectTeam - emptyObjectAccessControlProjectTeamSize = int(reflect.TypeOf(emptyObjectAccessControlProjectTeam).Size()) } // Definitions/conventions (not based on a standard, but just made up for convenience). @@ -104,11 +94,17 @@ func init() { // For e.g. if an int is 8 bytes and an empty string is 16 bytes, // then UnsafeSizeOf(&struct{int, string}) // return 24 (8+16). +// Warning: This function uses generics and unsafe.Sizeof directly +// without reflection for performance reasons. It MUST ONLY be called +// with a pointer to a concrete type (e.g. *gcs.Folder). If it is +// called with a pointer to an interface (e.g. *any), it will return +// the size of the interface header (16 bytes) rather than the size +// of the underlying concrete type. func UnsafeSizeOf[T any](ptr *T) int { if ptr == nil { return 0 } - return int(reflect.TypeOf(*ptr).Size()) + return int(unsafe.Sizeof(*ptr)) } func contentSizeOfString(s *string) int { @@ -167,28 +163,6 @@ func contentSizeOfServerResponse(sr *googleapi.ServerResponse) (size int) { return } -func contentSizeOfObjectAccessControlProjectTeam(oacpt *storagev1.ObjectAccessControlProjectTeam) (size int) { - if oacpt == nil { - return - } - - // Account for string members. - for _, strPtr := range []*string{ - &oacpt.ProjectNumber, &oacpt.Team, - } { - size += contentSizeOfString(strPtr) - } - - // Account for string-array members. - for _, strArrayPtr := range []*[]string{ - &oacpt.ForceSendFields, &oacpt.NullFields, - } { - size += contentSizeOfArrayOfStrings(strArrayPtr) - } - - return -} - // NestedSizeOfGcsMinObject returns the full nested memory size // of the gcs.MinObject pointed by the passed pointer. // Improvement scope: This can be generalized to a general-struct @@ -219,3 +193,19 @@ func NestedSizeOfGcsMinObject(m *gcs.MinObject) (size int) { return } + +// NestedSizeOfGcsFolder returns the full nested memory size +// of the gcs.Folder pointed by the passed pointer. +func NestedSizeOfGcsFolder(f *gcs.Folder) (size int) { + if f == nil { + return + } + + // Get raw size of the structure. + size = UnsafeSizeOf(f) + + // Account for string members. + size += contentSizeOfString(&f.Name) + + return +} diff --git a/internal/util/sizeof_bench_test.go b/internal/util/sizeof_bench_test.go new file mode 100644 index 0000000000..b1c685c8aa --- /dev/null +++ b/internal/util/sizeof_bench_test.go @@ -0,0 +1,46 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package util + +import ( + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" +) + +func BenchmarkUnsafeSizeOf_Int(b *testing.B) { + val := 5 + b.ResetTimer() + for range b.N { + _ = UnsafeSizeOf(&val) + } +} + +func BenchmarkUnsafeSizeOf_String(b *testing.B) { + val := "benchmark_test_string" + b.ResetTimer() + for range b.N { + _ = UnsafeSizeOf(&val) + } +} + +func BenchmarkUnsafeSizeOf_MinObject(b *testing.B) { + val := gcs.MinObject{ + Name: "benchmark_test_min_object", + } + b.ResetTimer() + for range b.N { + _ = UnsafeSizeOf(&val) + } +} diff --git a/internal/util/sizeof_test.go b/internal/util/sizeof_test.go index 3bbd61675e..58ee84b67b 100644 --- a/internal/util/sizeof_test.go +++ b/internal/util/sizeof_test.go @@ -20,23 +20,12 @@ import ( "time" "unsafe" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" "google.golang.org/api/googleapi" ) -//////////////////////////////////////////////////////////////////////// -// Boilerplate -//////////////////////////////////////////////////////////////////////// - -type SizeofTest struct { - suite.Suite -} - -func TestSizeOfSuite(t *testing.T) { - suite.Run(t, new(SizeofTest)) -} +// ////////////////////////////////////////////////////////////////////// var ( i int @@ -45,105 +34,80 @@ var ( b byte stringIntMap map[string]int - sizeOfInt int - sizeOfIntPtr int - sizeOfUInt32 int - sizeOfUInt32Ptr int - sizeOfByte int - sizeOfEmptyIntArray int - sizeOfEmptyStringIntMap int - sizeOfEmptyStruct int - sizeOfEmptyMinObject int -) - -func (t *SizeofTest) SetupTest() { - type emptyStruct struct{} - - sizeOfInt = int(unsafe.Sizeof(i)) - sizeOfIntPtr = int(unsafe.Sizeof(&i)) - sizeOfUInt32 = int(unsafe.Sizeof(ui32)) - sizeOfUInt32Ptr = int(unsafe.Sizeof(&ui32)) - sizeOfByte = int(unsafe.Sizeof(b)) - sizeOfEmptyIntArray = int(unsafe.Sizeof(intArray)) + sizeOfInt = int(unsafe.Sizeof(i)) + sizeOfIntPtr = int(unsafe.Sizeof(&i)) + sizeOfUInt32 = int(unsafe.Sizeof(ui32)) + sizeOfUInt32Ptr = int(unsafe.Sizeof(&ui32)) + sizeOfByte = int(unsafe.Sizeof(b)) + sizeOfEmptyIntArray = int(unsafe.Sizeof(intArray)) sizeOfEmptyStringIntMap = int(unsafe.Sizeof(stringIntMap)) - sizeOfEmptyStruct = int(unsafe.Sizeof(emptyStruct{})) - assert.Equal(t.T(), 0, sizeOfEmptyStruct) - sizeOfEmptyMinObject = int(unsafe.Sizeof(gcs.MinObject{})) -} + sizeOfEmptyStruct = int(unsafe.Sizeof(struct{}{})) +) //////////////////////////////////////////////////////////////////////// // Tests //////////////////////////////////////////////////////////////////////// -func (t *SizeofTest) TestUnsafeSizeOf() { - for _, tc := range []struct { - t any - expected_size int - }{ - { - t: i, - expected_size: sizeOfInt, - }, - { - t: &i, - expected_size: sizeOfIntPtr, - }, - { - t: ui32, - expected_size: sizeOfUInt32, - }, - { - t: &ui32, - expected_size: sizeOfUInt32Ptr, - }, - { - t: struct { - }{}, - expected_size: sizeOfEmptyStruct, - }, - { - t: struct { - x int - }{}, - expected_size: sizeOfEmptyStruct + sizeOfInt, - }, - { - t: struct { - a int - b1, b2, b3 byte - c string - }{}, - expected_size: sizeOfEmptyStruct + sizeOfInt + 3*sizeOfByte + 5 /*for-padding-for-alignment*/ + emptyStringSize, - }, - { - t: "", - expected_size: emptyStringSize, - }, - { - t: "hello", - expected_size: emptyStringSize, - }, - { - t: []int{1, 2, 3}, - expected_size: sizeOfEmptyIntArray, - }, - { - t: []string{"few ", "fewfgwe", "", "fewawef"}, - expected_size: emptyStringArraySize, - }, - { - t: map[string]int{"few ": 432, "fewfgwe": -21, "": 1, "fewawef": 0}, - expected_size: sizeOfEmptyStringIntMap, - }, - } { - calculatedSize := UnsafeSizeOf(&tc.t) - assert.Equal(t.T(), tc.expected_size, calculatedSize) +func TestUnsafeSizeOf(t *testing.T) { + assert.Equal(t, sizeOfInt, UnsafeSizeOf(&i)) + + ptrToI := &i + assert.Equal(t, sizeOfIntPtr, UnsafeSizeOf(&ptrToI)) + + assert.Equal(t, sizeOfUInt32, UnsafeSizeOf(&ui32)) + + ptrToUi32 := &ui32 + assert.Equal(t, sizeOfUInt32Ptr, UnsafeSizeOf(&ptrToUi32)) + + var emptyStructVal struct{} + assert.Equal(t, sizeOfEmptyStruct, UnsafeSizeOf(&emptyStructVal)) + + var structVal1 struct { + x int } + assert.Equal(t, sizeOfEmptyStruct+sizeOfInt, UnsafeSizeOf(&structVal1)) + + var structVal2 struct { + a int + b1, b2, b3 byte + c string + } + assert.Equal(t, sizeOfEmptyStruct+sizeOfInt+3*sizeOfByte+5 /*for-padding-for-alignment*/ +emptyStringSize, UnsafeSizeOf(&structVal2)) + + emptyStr := "" + assert.Equal(t, emptyStringSize, UnsafeSizeOf(&emptyStr)) + + helloStr := "hello" + assert.Equal(t, emptyStringSize, UnsafeSizeOf(&helloStr)) + + intArrayVal := []int{1, 2, 3} + assert.Equal(t, sizeOfEmptyIntArray, UnsafeSizeOf(&intArrayVal)) + + stringArrayVal := []string{"few ", "fewfgwe", "", "fewawef"} + assert.Equal(t, emptyStringArraySize, UnsafeSizeOf(&stringArrayVal)) + + stringIntMapVal := map[string]int{"few ": 432, "fewfgwe": -21, "": 1, "fewawef": 0} + assert.Equal(t, sizeOfEmptyStringIntMap, UnsafeSizeOf(&stringIntMapVal)) + + var emptyMinObj gcs.MinObject + assert.Equal(t, int(unsafe.Sizeof(emptyMinObj)), UnsafeSizeOf(&emptyMinObj)) + + ptrToM := &emptyMinObj + assert.Equal(t, int(unsafe.Sizeof(ptrToM)), UnsafeSizeOf(&ptrToM)) + + var emptyFolder gcs.Folder + assert.Equal(t, int(unsafe.Sizeof(emptyFolder)), UnsafeSizeOf(&emptyFolder)) + + ptrToF := &emptyFolder + assert.Equal(t, int(unsafe.Sizeof(ptrToF)), UnsafeSizeOf(&ptrToF)) + + var nilInt *int = nil + assert.Equal(t, 0, UnsafeSizeOf(nilInt)) } -func (t *SizeofTest) TestContentSizeOfString() { +func TestContentSizeOfString(t *testing.T) { for _, tc := range []struct { str string expected_content_size int @@ -162,11 +126,11 @@ func (t *SizeofTest) TestContentSizeOfString() { expected_content_size: 11, }, } { - assert.Equal(t.T(), tc.expected_content_size, contentSizeOfString(&tc.str)) + assert.Equal(t, tc.expected_content_size, contentSizeOfString(&tc.str)) } } -func (t *SizeofTest) TestContentSizeOfArrayOfStrings() { +func TestContentSizeOfArrayOfStrings(t *testing.T) { for _, tc := range []struct { strs []string expected_content_size int @@ -192,11 +156,11 @@ func (t *SizeofTest) TestContentSizeOfArrayOfStrings() { expected_content_size: 2*emptyStringSize + 5 + 11, }, } { - assert.Equal(t.T(), tc.expected_content_size, contentSizeOfArrayOfStrings(&tc.strs)) + assert.Equal(t, tc.expected_content_size, contentSizeOfArrayOfStrings(&tc.strs)) } } -func (t *SizeofTest) TestContentSizeOfStringToStringMap() { +func TestContentSizeOfStringToStringMap(t *testing.T) { for _, tc := range []struct { m map[string]string expected_content_size int @@ -222,11 +186,11 @@ func (t *SizeofTest) TestContentSizeOfStringToStringMap() { expected_content_size: emptyStringSize + 1 + emptyStringSize + 2 + emptyStringSize + 3 + emptyStringSize + 5, }, } { - assert.Equal(t.T(), tc.expected_content_size, contentSizeOfStringToStringMap(&tc.m)) + assert.Equal(t, tc.expected_content_size, contentSizeOfStringToStringMap(&tc.m)) } } -func (t *SizeofTest) TestContentSizeOfStringToStringArrayMap() { +func TestContentSizeOfStringToStringArrayMap(t *testing.T) { for _, tc := range []struct { m map[string][]string expected_content_size int @@ -252,11 +216,11 @@ func (t *SizeofTest) TestContentSizeOfStringToStringArrayMap() { expected_content_size: emptyStringSize + 1 + emptyStringArraySize + emptyStringSize + 2 + emptyStringSize + 2 + emptyStringSize + 3 + emptyStringArraySize + emptyStringSize + 5 + emptyStringSize + 4, }, } { - assert.Equal(t.T(), tc.expected_content_size, contentSizeOfStringToStringArrayMap(&tc.m)) + assert.Equal(t, tc.expected_content_size, contentSizeOfStringToStringArrayMap(&tc.m)) } } -func (t *SizeofTest) TestContentSizeOfServerResponse() { +func TestContentSizeOfServerResponse(t *testing.T) { for _, tc := range []struct { sr googleapi.ServerResponse expected_content_size int @@ -282,11 +246,11 @@ func (t *SizeofTest) TestContentSizeOfServerResponse() { expected_content_size: emptyStringSize + 1 + emptyStringArraySize + emptyStringSize + 2 + emptyStringSize + 3 + emptyStringArraySize + emptyStringSize + 5, }, } { - assert.Equal(t.T(), tc.expected_content_size, contentSizeOfServerResponse(&tc.sr)) + assert.Equal(t, tc.expected_content_size, contentSizeOfServerResponse(&tc.sr)) } } -func (t *SizeofTest) TestNestedSizeOfGcsMinObject() { +func TestNestedSizeOfGcsMinObject(t *testing.T) { const name string = "my-object" const contentEncoding string = "gzip/none" var generation int64 = 858734898 @@ -315,5 +279,40 @@ func (t *SizeofTest) TestNestedSizeOfGcsMinObject() { expectedSize += len(name) + len(contentEncoding) + sizeOfUInt32 expectedSize += customMetadataFieldsContentSize - assert.Equal(t.T(), expectedSize, NestedSizeOfGcsMinObject(&m)) + assert.Equal(t, expectedSize, NestedSizeOfGcsMinObject(&m)) +} + +func TestNestedSizeOfGcsFolder(t *testing.T) { + f1 := &gcs.Folder{} + f2 := &gcs.Folder{ + Name: "folder/name/", + } + + testCases := []struct { + name string + folder *gcs.Folder + expected int + }{ + { + name: "NilFolder", + folder: nil, + expected: 0, + }, + { + name: "EmptyFolder", + folder: f1, + expected: UnsafeSizeOf(f1), + }, + { + name: "FolderWithName", + folder: f2, + expected: UnsafeSizeOf(f2) + len(f2.Name), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(st *testing.T) { + assert.Equal(st, tc.expected, NestedSizeOfGcsFolder(tc.folder)) + }) + } } diff --git a/internal/util/test_util.go b/internal/util/test_util.go index d48e10079b..e550827798 100644 --- a/internal/util/test_util.go +++ b/internal/util/test_util.go @@ -21,8 +21,18 @@ import "math/rand" func GenerateRandomBytes(length int) []byte { randBytes := make([]byte, length) - for i := 0; i < length; i++ { + for i := range length { randBytes[i] = byte(rand.Intn(26) + 65) } return randBytes } + +// ConvertReadResponseToBytes concatenates the data slices from a read response into a single byte slice. +func ConvertReadResponseToBytes(data [][]byte, size int) []byte { + buf := make([]byte, size) + bytesCopied := 0 + for _, dataSlice := range data { + bytesCopied += copy(buf[bytesCopied:], dataSlice) + } + return buf +} diff --git a/internal/util/util.go b/internal/util/util.go index 757f84cbd7..74164b836e 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -15,7 +15,6 @@ package util import ( - "context" "fmt" "math" "os" @@ -28,12 +27,10 @@ import ( const ( GCSFUSE_PARENT_PROCESS_DIR = "gcsfuse-parent-process-dir" - // Constants for read types - Sequential/Random - Sequential = "Sequential" - Random = "Random" - Parallel = "Parallel" - MaxMiBsInUint64 uint64 = math.MaxUint64 >> 20 + MaxMiBsInInt64 int64 = math.MaxInt64 >> 20 + MiB = 1024 * 1024 + KiB = 1024 // HeapSizeToRssConversionFactor is a constant factor // which we multiply to the calculated heap-size @@ -96,12 +93,6 @@ func BytesToHigherMiBs(bytes uint64) uint64 { return MaxMiBsInUint64 + 1 } const bytesInOneMiB uint64 = 1 << 20 - return uint64(math.Ceil(float64(bytes) / float64(bytesInOneMiB))) -} - -// IsolateContextFromParentContext creates a copy of the parent context which is -// not cancelled when parent context is cancelled. -func IsolateContextFromParentContext(ctx context.Context) (context.Context, context.CancelFunc) { - ctx = context.WithoutCancel(ctx) - return context.WithCancel(ctx) + // Use integer arithmetic and bitwise shift to avoid slow float conversions + return (bytes + bytesInOneMiB - 1) >> 20 } diff --git a/internal/util/util_benchmark_test.go b/internal/util/util_benchmark_test.go new file mode 100644 index 0000000000..82760859dd --- /dev/null +++ b/internal/util/util_benchmark_test.go @@ -0,0 +1,26 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "testing" +) + +func BenchmarkBytesToHigherMiBs(b *testing.B) { + bytes := uint64(1048576) // 1 MiB + for b.Loop() { + _ = BytesToHigherMiBs(bytes) + } +} diff --git a/internal/util/util_test.go b/internal/util/util_test.go index cbd6b1f8c1..c30363e0f4 100644 --- a/internal/util/util_test.go +++ b/internal/util/util_test.go @@ -15,7 +15,6 @@ package util import ( - "context" "math" "os" "path/filepath" @@ -214,17 +213,3 @@ func (ts *UtilTest) TestBytesToHigherMiBs() { assert.Equal(ts.T(), tc.mib, BytesToHigherMiBs(tc.bytes)) } } - -func (ts *UtilTest) TestIsolateContextFromParentContext() { - parentCtx, parentCtxCancel := context.WithCancel(context.Background()) - - // Call the method and cancel the parent context. - newCtx, newCtxCancel := IsolateContextFromParentContext(parentCtx) - parentCtxCancel() - - // Validate new context is not cancelled after parent's cancellation. - assert.NoError(ts.T(), newCtx.Err()) - // Cancel the new context and validate. - newCtxCancel() - assert.ErrorIs(ts.T(), newCtx.Err(), context.Canceled) -} diff --git a/internal/workerpool/static_worker_pool.go b/internal/workerpool/static_worker_pool.go new file mode 100644 index 0000000000..1f2bda7410 --- /dev/null +++ b/internal/workerpool/static_worker_pool.go @@ -0,0 +1,180 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package workerpool + +import ( + "fmt" + "runtime" + "sync" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" +) + +// staticWorkerPool starts all the workers (goroutines) on startup and keeps them running. +// It keep two types of workers - priority and normal. Priority workers will only +// execute tasks that are marked as urgent while scheduling. Normal workers will +// execute both urgent and normal tasks, but gives precedence to urgent task. +type staticWorkerPool struct { + priorityWorker uint32 // Number of priority workers in this pool. + normalWorker uint32 // Number of normal workers in this pool. + + // Stop channel to notify all the workers to stop. + stop chan bool + + // Wait group to wait for all workers to finish. + wg sync.WaitGroup + + // Channels for normal and priority tasks. + priorityCh chan Task + normalCh chan Task +} + +// NewStaticWorkerPool creates a new thread pool +func NewStaticWorkerPool(priorityWorker uint32, normalWorker uint32, readGlobalMaxBlocks int64) (*staticWorkerPool, error) { + totalWorkers := priorityWorker + normalWorker + if totalWorkers == 0 { + return nil, fmt.Errorf("staticWorkerPool: can't create with 0 workers, priority: %d, normal: %d", priorityWorker, normalWorker) + } + + logger.Infof("staticWorkerPool: creating with %d normal, and %d priority workers.", normalWorker, priorityWorker) + + // The channel capacity is set to the minimum of a worker-based buffer size + // and a global cap. This prevents creating overly large channels, which can be + // slow to initialize and consume unnecessary memory. The cap is based on + // `readGlobalMaxBlocks` because we can't schedule more download tasks than this at once. + priorityChSize := min(int(priorityWorker)*200, int(2*readGlobalMaxBlocks)) + normalChSize := min(int(normalWorker)*5000, int(2*readGlobalMaxBlocks)) + return &staticWorkerPool{ + priorityWorker: priorityWorker, + normalWorker: normalWorker, + stop: make(chan bool), + priorityCh: make(chan Task, priorityChSize), + normalCh: make(chan Task, normalChSize), + }, nil +} + +// NewStaticWorkerPoolForCurrentCPU creates and starts a new worker pool. The +// number of workers is determined based on the number of available CPUs and +// the provided readGlobalMaxBlocks. +func NewStaticWorkerPoolForCurrentCPU(readGlobalMaxBlocks int64) (WorkerPool, error) { + return newStaticWorkerPoolForCurrentCPU(readGlobalMaxBlocks, runtime.NumCPU) +} + +// newStaticWorkerPoolForCurrentCPU is an unexported helper for testing. +func newStaticWorkerPoolForCurrentCPU(readGlobalMaxBlocks int64, numCPU func() int) (WorkerPool, error) { + // It's a general heuristic to use 2-3 times the number of CPUs for I/O-bound tasks. + // We use 3x here as a balance between parallelism and resource consumption. + const workersPerCPU = 3 + totalWorkers := workersPerCPU * numCPU() + + // Since the number of concurrent download tasks is limited by readGlobalMaxBlocks, + // creating more workers beyond this limit offers no performance gain and wastes + // resources. Hence, we cap total workers to ceil(1.1 * readGlobalMaxBlocks). + if cappedWorkers := (11*readGlobalMaxBlocks + 9) / 10; int64(totalWorkers) > cappedWorkers { + totalWorkers = int(cappedWorkers) + } + + // 10% of total workers for priority, rounded up. + priorityWorkers := (totalWorkers + 9) / 10 + normalWorkers := totalWorkers - priorityWorkers + + wp, err := NewStaticWorkerPool(uint32(priorityWorkers), uint32(normalWorkers), readGlobalMaxBlocks) + if err != nil { + return nil, err + } + + wp.Start() + return wp, nil +} + +// Start all the workers and wait till they start receiving requests +func (swp *staticWorkerPool) Start() { + for i := uint32(0); i < swp.priorityWorker; i++ { + swp.wg.Add(1) + go swp.do(true) + } + + for i := uint32(0); i < swp.normalWorker; i++ { + swp.wg.Add(1) + go swp.do(false) + } +} + +// Stop all the workers threads and wait for them to finish processing. +func (swp *staticWorkerPool) Stop() { + // Notify all workers to stop. + logger.Infof("staticWorkerPool: stopping all the workers.") + close(swp.stop) + + swp.wg.Wait() + + // Close the channel after all workers are done. + close(swp.priorityCh) + close(swp.normalCh) +} + +// Schedule schedules tasks to the worker pool. +// Pass urgent as true for priority scheduling. +func (swp *staticWorkerPool) Schedule(urgent bool, task Task) { + // urgent specifies the priority of this task. + // true means high priority and false means low priority + if urgent { + swp.priorityCh <- task + } else { + swp.normalCh <- task + } +} + +// do is the core routine that runs in each worker thread. +// It will keep listening to the channel for tasks and execute them. +func (swp *staticWorkerPool) do(priority bool) { + defer swp.wg.Done() + + if priority { + // Worker only listens to the priority channel. + for { + select { + case <-swp.stop: + return + default: + select { + case <-swp.stop: + return + case task := <-swp.priorityCh: + task.Execute() + } + } + } + } else { + // Worker listens to both channels but gives priority to the priority channel. + for { + select { + case <-swp.stop: + return + case task := <-swp.priorityCh: + task.Execute() + default: + select { + case <-swp.stop: + return + case task := <-swp.priorityCh: + task.Execute() + case task := <-swp.normalCh: + task.Execute() + } + } + } + } +} diff --git a/internal/workerpool/static_worker_pool_test.go b/internal/workerpool/static_worker_pool_test.go new file mode 100644 index 0000000000..941d3bda32 --- /dev/null +++ b/internal/workerpool/static_worker_pool_test.go @@ -0,0 +1,257 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package workerpool + +import ( + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type dummyTask struct { + executed bool +} + +func (d *dummyTask) Execute() { + d.executed = true +} + +func TestNewStaticWorkerPool_Success(t *testing.T) { + tests := []struct { + name string + priorityWorker uint32 + normalWorker uint32 + readMaxBlocks int64 + expectedPriorityCh int + expectedNormalCh int + }{ + { + name: "worker-based size is smaller", + priorityWorker: 2, + normalWorker: 1, + readMaxBlocks: 1000, + // priority: min(2*200, 2*1000) = 400 + // normal: min(1*5000, 2*1000) = 2000 + expectedPriorityCh: 400, + expectedNormalCh: 2000, + }, + { + name: "global cap is smaller", + priorityWorker: 50, + normalWorker: 10, + readMaxBlocks: 100, + // priority: min(50*200, 2*100) = 200 + // normal: min(10*5000, 2*100) = 200 + expectedPriorityCh: 200, + expectedNormalCh: 200, + }, + { + name: "zero normal workers", + priorityWorker: 1, + normalWorker: 0, + readMaxBlocks: 100, + // priority: min(1*200, 2*100) = 200 + // normal: min(0*5000, 2*100) = 0 + expectedPriorityCh: 200, + expectedNormalCh: 0, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + pool, err := NewStaticWorkerPool(tc.priorityWorker, tc.normalWorker, tc.readMaxBlocks) + + assert.NoError(t, err) + assert.NotNil(t, pool) + assert.Equal(t, tc.priorityWorker, pool.priorityWorker) + assert.Equal(t, tc.normalWorker, pool.normalWorker) + assert.Equal(t, tc.expectedPriorityCh, cap(pool.priorityCh)) + assert.Equal(t, tc.expectedNormalCh, cap(pool.normalCh)) + pool.Stop() // Clean up + }) + } +} + +func TestNewStaticWorkerPool_Failure(t *testing.T) { + pool, err := NewStaticWorkerPool(0, 0, 0) + + assert.Error(t, err) + assert.Nil(t, pool) + assert.Panics(t, pool.Stop, "Stop should panic if pool is nil") +} + +func TestStaticWorkerPool_Start(t *testing.T) { + pool, err := NewStaticWorkerPool(2, 3, 5) + require.NoError(t, err) + require.NotNil(t, pool) + + pool.Start() + defer pool.Stop() + + // Add a task in the channel and later see, that channel will be empty after execution. + dt := &dummyTask{} + pool.priorityCh <- dt + // Wait for the task to be executed. + assert.Eventually(t, func() bool { + return dt.executed + }, 100*time.Millisecond, time.Millisecond, "Task was not executed in time.") + assert.Equal(t, 0, len(pool.priorityCh), "Priority channel should be empty after task execution.") +} + +func TestStaticWorkerPool_SchedulePriorityTask(t *testing.T) { + pool, err := NewStaticWorkerPool(2, 3, 5) + require.NoError(t, err) + require.NotNil(t, pool) + pool.Start() + defer pool.Stop() + + dt := &dummyTask{} + pool.Schedule(true, dt) + + // Wait for the task to be executed. + assert.Eventually(t, func() bool { + return dt.executed + }, 100*time.Millisecond, time.Millisecond, "Task was not executed in time.") +} + +func TestStaticWorkerPool_ScheduleNormalTask(t *testing.T) { + pool, err := NewStaticWorkerPool(2, 3, 5) + require.NoError(t, err) + require.NotNil(t, pool) + pool.Start() + defer pool.Stop() + + dt := &dummyTask{} + pool.Schedule(false, dt) + + // Wait for the task to be executed. + require.Eventually(t, func() bool { + return dt.executed + }, 100*time.Millisecond, time.Millisecond, "Priority task was not executed in time.") +} + +func TestStaticWorkerPool_HighNumberOfTasks(t *testing.T) { + pool, err := NewStaticWorkerPool(5, 10, 15) + require.NoError(t, err) + require.NotNil(t, pool) + pool.Start() + defer pool.Stop() + + // Schedule a large number of tasks. + for i := range 100 { + dt := &dummyTask{} + pool.Schedule(i%2 == 0, dt) // Alternate between priority and normal tasks + } + + // Wait for all tasks to be executed. + assert.Eventually(t, func() bool { + return len(pool.priorityCh) == 0 && len(pool.normalCh) == 0 + }, 500*time.Millisecond, 10*time.Millisecond, "Not all tasks were executed in time.") +} + +func TestStaticWorkerPool_ScheduleAfterStop(t *testing.T) { + pool, err := NewStaticWorkerPool(2, 3, 5) + require.NoError(t, err) + require.NotNil(t, pool) + pool.Start() + + pool.Stop() + + assert.Panics(t, func() { pool.Schedule(true, &dummyTask{}) }, "Should panic when scheduling after cancel.") +} + +func TestStaticWorkerPool_Stop(t *testing.T) { + pool, err := NewStaticWorkerPool(2, 3, 5) + require.NoError(t, err) + require.NotNil(t, pool) + pool.Start() + + // Stop the pool and check if channels are closed. + pool.Stop() + + assert.Panics(t, func() { pool.stop <- true }, "normalCh channel is not closed.") + assert.Panics(t, func() { pool.normalCh <- &dummyTask{} }, "normalCh channel is not closed.") + assert.Panics(t, func() { pool.priorityCh <- &dummyTask{} }, "priorityCh channel is not closed.") +} + +func TestNewStaticWorkerPoolForCurrentCPU(t *testing.T) { + readGlobalMaxBlocks := int64(100) + + pool, err := NewStaticWorkerPoolForCurrentCPU(readGlobalMaxBlocks) + + require.NoError(t, err) + require.NotNil(t, pool) + defer pool.Stop() + staticPool, ok := pool.(*staticWorkerPool) + require.True(t, ok, "The returned pool should be of type *staticWorkerPool") + // Re-calculate the expected number of workers based on the real CPU count + // to verify the logic. + totalWorkers := 3 * runtime.NumCPU() + if cappedWorkers := (11*readGlobalMaxBlocks + 9) / 10; int64(totalWorkers) > cappedWorkers { + totalWorkers = int(cappedWorkers) + } + expectedPriorityWorkers := (totalWorkers + 9) / 10 + expectedNormalWorkers := totalWorkers - expectedPriorityWorkers + dt := &dummyTask{} + pool.Schedule(true, dt) + assert.Equal(t, uint32(expectedPriorityWorkers), staticPool.priorityWorker) + assert.Equal(t, uint32(expectedNormalWorkers), staticPool.normalWorker) + // Verify that the pool is functional. + assert.Eventually(t, func() bool { return dt.executed }, 100*time.Millisecond, time.Millisecond, "Task was not executed in time.") +} + +func Test_newStaticWorkerPoolForCurrentCPU(t *testing.T) { + testCases := []struct { + name string + readGlobalMaxBlocks int64 + mockNumCPU func() int + expectedPriorityWorkers uint32 + expectedNormalWorkers uint32 + }{ + { + name: "low CPU count, workers not capped", + readGlobalMaxBlocks: 100, + mockNumCPU: func() int { return 2 }, + // totalWorkers = 3*2=6. priority=ceil(0.1*6)=1, normal=5. + expectedPriorityWorkers: 1, + expectedNormalWorkers: 5, + }, + { + name: "high CPU count, workers capped by max blocks", + readGlobalMaxBlocks: 50, + mockNumCPU: func() int { return 100 }, + // totalWorkers = 3*100=300, capped to ceil(1.1*50)=55. priority=ceil(0.1*55)=6, normal=49. + expectedPriorityWorkers: 6, + expectedNormalWorkers: 49, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + pool, err := newStaticWorkerPoolForCurrentCPU(tc.readGlobalMaxBlocks, tc.mockNumCPU) + + require.NoError(t, err) + require.NotNil(t, pool) + defer pool.Stop() + staticPool, ok := pool.(*staticWorkerPool) + require.True(t, ok, "The returned pool should be of type *staticWorkerPool") + assert.Equal(t, tc.expectedPriorityWorkers, staticPool.priorityWorker) + assert.Equal(t, tc.expectedNormalWorkers, staticPool.normalWorker) + }) + } +} diff --git a/internal/workerpool/worker_pool.go b/internal/workerpool/worker_pool.go new file mode 100644 index 0000000000..28d615393c --- /dev/null +++ b/internal/workerpool/worker_pool.go @@ -0,0 +1,31 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package workerpool + +// Task interface defines the contract for a runnable task. +type Task interface { + Execute() +} + +type WorkerPool interface { + // Start initializes the worker pool and prepares it to accept tasks. + Start() + + // Stop gracefully shuts down the worker pool, waiting for all tasks to complete. + Stop() + + // Schedule adds a task to the worker pool for execution. + Schedule(urgent bool, task Task) +} diff --git a/internal/workloadinsight/io_renderer.go b/internal/workloadinsight/io_renderer.go new file mode 100644 index 0000000000..4181e5206c --- /dev/null +++ b/internal/workloadinsight/io_renderer.go @@ -0,0 +1,298 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package workloadinsight + +import ( + "fmt" + "math" + "sort" + "strconv" + "strings" +) + +const ( + blockChar = "█" + emptyChar = " " + labelHeader = "[offset,len)" +) + +// humanReadable formats a byte size into a compact string (KB, MB, GB). +func humanReadable(size uint64) string { + const ( + KB = 1024 + MB = KB * 1024 + GB = MB * 1024 + ) + switch { + case size >= GB: + return fmt.Sprintf("%.1fG", float64(size)/float64(GB)) + case size >= MB: + return fmt.Sprintf("%.1fM", float64(size)/float64(MB)) + case size >= KB: + return fmt.Sprintf("%.1fK", float64(size)/float64(KB)) + default: + return fmt.Sprintf("%dB", size) + } +} + +// Range represents a byte range [Start, End). +type Range struct { + Start uint64 + End uint64 +} + +// Renderer renders I/O byte ranges as ASCII plots to visualize the access patterns. +type Renderer struct { + plotWidth int // number of columns used for plotting area + labelWidth int // width of left label column (0 = auto) + pad int // spaces between label and plot +} + +func NewRenderer() (*Renderer, error) { + return NewRendererWithSettings(100, 20, 2) +} + +// NewRendererWithSettings returns a Renderer with the specified settings. +// Returns an error if any setting is invalid (e.g. negative). +func NewRendererWithSettings(plotWidth, labelWidth, pad int) (*Renderer, error) { + if labelWidth < len(labelHeader) { + return nil, fmt.Errorf("labelWidth must be at least %d", len(labelHeader)) + } + + if pad < 0 { + return nil, fmt.Errorf("plotWidth and pad must be non-negative") + } + + if plotWidth < 1 { + return nil, fmt.Errorf("plotWidth must be positive") + } + + return &Renderer{ + plotWidth: plotWidth, + labelWidth: labelWidth, + pad: pad, + }, nil +} + +// Render renders the given ranges for a single file of the specified size +// and returns the ASCII representation as a string. +func (r *Renderer) Render(name string, size uint64, ranges []Range) (string, error) { + var sb strings.Builder + header, err := r.buildHeader(name, size, ranges) + if err != nil { + return "", err + } + sb.WriteString(header) + + for i := range ranges { + line, err := r.buildRow(size, ranges[i]) + if err != nil { + return "", err + } + sb.WriteString(line) + if i < len(ranges)-1 { + sb.WriteByte('\n') + } + } + sb.WriteByte('\n') + return sb.String(), nil +} + +// buildStats builds statistics about the given ranges for a single file +// and returns them as a string. +func (r *Renderer) buildStats(ranges []Range) string { + length := len(ranges) + if length <= 0 { + return "" + } + + sizes := make([]uint64, length) + sum := uint64(0) + for i, rg := range ranges { + sizes[i] = rg.End - rg.Start + sum += sizes[i] + } + sort.Slice(sizes, func(i, j int) bool { return sizes[i] < sizes[j] }) + + var b []byte + b = append(b, "Total IOs: "...) + b = strconv.AppendInt(b, int64(length), 10) + b = append(b, "\nIO Size Distributions: (Min: "...) + b = append(b, humanReadable(sizes[0])...) + b = append(b, ", Median: "...) + b = append(b, humanReadable(sizes[length/2])...) + b = append(b, ", Max: "...) + b = append(b, humanReadable(sizes[length-1])...) + b = append(b, ", Avg: "...) + b = append(b, humanReadable(sum/uint64(length))...) + b = append(b, ")\n"...) + return string(b) +} + +// buildHeader composes the header (filename, tick marks, numeric labels) +// and returns it as a string to be prepended to the chart rows. +// E.g.: +// +// Name: demo.txt +// +// 0B 250B 500B 750B 1000B +// +// [offset,len) |-----------|------------|-----------|-----------| +func (r *Renderer) buildHeader(name string, size uint64, ranges []Range) (string, error) { + var sb strings.Builder + + // Helper to build a runes slice filled with the provided fill rune. + makeRunes := func(n int, fill rune) []rune { + s := make([]rune, n) + for i := range s { + s[i] = fill + } + return s + } + + // Compose fileOffsetAxis with marker at 0%, 25%, 50%, 75%, 100% of size. + fileOffsetAxisChars := makeRunes(r.plotWidth, '-') + fileOffsetMarkers := []uint64{0, size / 4, size / 2, (size * 3) / 4, size} + for _, off := range fileOffsetMarkers { + if p, err := mapCoord(off, size, r.plotWidth); err == nil { + fileOffsetAxisChars[p] = '|' + } + } + + // File offset labels, placed above the fileOffsetAxis at the size markers. + fileOffsetLabels := makeRunes(r.plotWidth, ' ') + for _, off := range fileOffsetMarkers { + p, err := mapCoord(off, size, r.plotWidth) + if err != nil { + return "", err + } + + offsetLabel := humanReadable(off) + // Center fileOffsetLabel around the fileOffsetMarker. + start := max(p-len(offsetLabel)/2, 0) + if start+len(offsetLabel) > r.plotWidth { + start = max(r.plotWidth-len(offsetLabel), 0) + } + copy(fileOffsetLabels[start:], []rune(offsetLabel)) + } + + // Filename line. + sb.WriteString("Name: ") + sb.WriteString(name) + sb.WriteByte('\n') + + // IO stats. + sb.WriteString(r.buildStats(ranges)) + + // Fileoffset labels just above the fileOffsetAxis. + sb.WriteString(strings.Repeat(" ", r.labelWidth)) + if r.pad > 0 { + sb.WriteString(strings.Repeat(" ", r.pad)) + } + sb.WriteString(string(fileOffsetLabels)) + sb.WriteByte('\n') + + // labelHeader ("[offset,len)") and horizontal tick line. + sb.WriteString(labelHeader) + if r.labelWidth > len(labelHeader) { + sb.WriteString(strings.Repeat(" ", r.labelWidth-len(labelHeader))) + } + if r.pad > 0 { + sb.WriteString(strings.Repeat(" ", r.pad)) + } + sb.WriteString(string(fileOffsetAxisChars)) + sb.WriteByte('\n') + + return sb.String(), nil +} + +// buildRow composes a single plotted row (label + plot cells) for the given range. +func (r *Renderer) buildRow(size uint64, rg Range) (string, error) { + var sb strings.Builder + + // Validate range: do not normalize or clamp; return an error on unexpected values. + if rg.Start > rg.End { + return "", fmt.Errorf("invalid range: start > end: [%d,%d)", rg.Start, rg.End) + } + if rg.End > size { + return "", fmt.Errorf("range extends beyond file size: [%d,%d) size=%d", rg.Start, rg.End, size) + } + + // Build plotting row + cells := make([]string, r.plotWidth) + for j := range cells { + cells[j] = emptyChar + } + + s := rg.Start + e := rg.End - 1 // make end inclusive for plotting + + // Map start/end to columns. Reserve column 0 as a separator when possible by + // mapping into [1, plotWidth-1] when plotWidth > 1. + cs, err := mapCoord(s, size, r.plotWidth-1) + if err != nil { + return "", err + } + ce, err := mapCoord(e, size, r.plotWidth-1) + if err != nil { + return "", err + } + cs = cs + 1 + ce = ce + 1 + + for c := cs; c <= ce; c++ { + cells[c] = blockChar + } + + // Place a vertical separator glyph in column 0. + if r.plotWidth > 0 && cells[0] == emptyChar { + cells[0] = "|" + } + + // Compose label and write. + label := fmt.Sprintf("[%d,%s)", s, humanReadable(e-s+1)) + if len(label) > r.labelWidth { + label = label[:r.labelWidth] + } + if len(label) < r.labelWidth { + label = label + strings.Repeat(" ", r.labelWidth-len(label)) + } + sb.WriteString(label) + if r.pad > 0 { + sb.WriteString(strings.Repeat(" ", r.pad)) + } + + // Write chart. + sb.WriteString(strings.Join(cells, "")) + + return sb.String(), nil +} + +// mapCoord maps an offset in [0, size) to a column in [0, plotWidth). +// If size == 0 returns 0. plotWidth must be >0. +func mapCoord(offset, size uint64, plotWidth int) (int, error) { + if plotWidth <= 0 || size == 0 { + return 0, fmt.Errorf("invalid arguments to mapCoord") + } + frac := float64(offset) / float64(size) + col := int(math.Floor(frac * float64(plotWidth))) + if col < 0 { + return 0, nil + } + if col >= plotWidth { + return plotWidth - 1, nil + } + return col, nil +} diff --git a/internal/workloadinsight/io_renderer_test.go b/internal/workloadinsight/io_renderer_test.go new file mode 100644 index 0000000000..6457a5c220 --- /dev/null +++ b/internal/workloadinsight/io_renderer_test.go @@ -0,0 +1,244 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package workloadinsight + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestIORenderer_NewRenderer(t *testing.T) { + _, err := NewRenderer() + + assert.NoError(t, err, "NewRenderer should not return an error") +} + +func TestIORenderer_NewRendererWithSettings_InvalidSettings(t *testing.T) { + tc := []struct { + name string + plotWidth int + labelWidth int + pad int + }{ + {name: "negative plotWidth", plotWidth: -1, labelWidth: 0, pad: 2}, + {name: "negative labelWidth", plotWidth: 80, labelWidth: -5, pad: 2}, + {name: "negative pad", plotWidth: 80, labelWidth: 0, pad: -3}, + } + + for _, test := range tc { + t.Run(test.name, func(t *testing.T) { + _, err := NewRendererWithSettings(test.plotWidth, test.labelWidth, test.pad) + + assert.Error(t, err, "expected error for invalid settings") + }) + } +} + +func TestIORenderer_NewRendererWithSettings_ValidSettings(t *testing.T) { + tc := []struct { + name string + plotWidth int + labelWidth int + pad int + }{ + {name: "zero labelWidth", plotWidth: 80, labelWidth: 20, pad: 2}, + {name: "positive labelWidth", plotWidth: 100, labelWidth: 15, pad: 4}, + {name: "zero pad", plotWidth: 60, labelWidth: 20, pad: 0}, + } + + for _, test := range tc { + t.Run(test.name, func(t *testing.T) { + _, err := NewRendererWithSettings(test.plotWidth, test.labelWidth, test.pad) + + assert.NoError(t, err, "NewRendererWithSettings should not return an error for valid settings") + }) + } +} + +func TestHumanReadable(t *testing.T) { + tc := []struct { + name string + size uint64 + expected string + }{ + {name: "zero", size: 0, expected: "0B"}, + {name: "one", size: 1, expected: "1B"}, + {name: "five hundred", size: 500, expected: "500B"}, + {name: "one thousand five hundred", size: 1500, expected: "1.5K"}, + {name: "two megabytes", size: 2 * 1024 * 1024, expected: "2.0M"}, + {name: "three gigabytes", size: 3 * 1024 * 1024 * 1024, expected: "3.0G"}, + {name: "one hundred twenty-three million four hundred fifty-six thousand seven hundred eighty-nine", size: 123456789, expected: "117.7M"}, + } + + for _, test := range tc { + t.Run(test.name, func(t *testing.T) { + result := humanReadable(test.size) + + assert.Equal(t, test.expected, result, "humanReadable(%d) should be %s", test.size, test.expected) + }) + } +} + +func TestMapCoord_Valid(t *testing.T) { + tc := []struct { + name string + plotWidth int + fileSize uint64 + offset uint64 + expected int + }{ + {name: "start of file", plotWidth: 80, fileSize: 1000, offset: 0, expected: 0}, + {name: "end of file", plotWidth: 80, fileSize: 1000, offset: 1000, expected: 79}, + {name: "middle of file upper half decimal", plotWidth: 17, fileSize: 335, offset: 43, expected: 2}, + {name: "middle of file lower half decimal", plotWidth: 17, fileSize: 335, offset: 73, expected: 3}, + {name: "three quarters of file", plotWidth: 80, fileSize: 1000, offset: 750, expected: 60}, + } + + for _, test := range tc { + t.Run(test.name, func(t *testing.T) { + result, err := mapCoord(test.offset, test.fileSize, test.plotWidth) + + assert.NoError(t, err, "mapCoord should not return an error") + assert.Equal(t, test.expected, result, "mapCoord(%d, %d, %d) should return %d", + test.offset, test.fileSize, test.plotWidth, test.expected) + }) + } +} + +func TestMapCoord_Invalid(t *testing.T) { + tc := []struct { + name string + plotWidth int + fileSize uint64 + offset uint64 + }{ + {name: "zero plotWidth", plotWidth: 0, fileSize: 1000, offset: 500}, + {name: "zero file size", plotWidth: 80, fileSize: 0, offset: 500}, + } + + for _, test := range tc { + t.Run(test.name, func(t *testing.T) { + _, err := mapCoord(test.offset, test.fileSize, test.plotWidth) + + assert.Error(t, err, "expected error for invalid input to mapCoord") + }) + } +} + +func TestIORenderer_Render(t *testing.T) { + tc := []struct { + name string + plotWidth int + labelWidth int + pad int + expectedOutputFile string + }{ + { + name: "default settings", + plotWidth: 80, + labelWidth: 12, // len(labelHeader) + pad: 2, + expectedOutputFile: "testdata/io_renderer/default_settings.txt", + }, + { + name: "custom settings", + plotWidth: 50, + labelWidth: 20, + pad: 4, + expectedOutputFile: "testdata/io_renderer/custom_settings.txt", + }, + } + + for _, test := range tc { + t.Run(test.name, func(t *testing.T) { + r, err := NewRendererWithSettings(test.plotWidth, test.labelWidth, test.pad) + require.NoError(t, err, "NewRendererWithSettings should not return an error for valid settings") + + name := "demo.txt" + size := uint64(1000) + ranges := []Range{ + {Start: 0, End: 100}, + {Start: 200, End: 300}, + {Start: 400, End: 600}, + {Start: 800, End: 1000}, + } + + out, err := r.Render(name, size, ranges) + // os.WriteFile(test.expectedOutputFile, []byte(out), 0644) // Uncomment to create a new test output file. + + assert.NoError(t, err, "Render should not return an error for valid input") + expectedOutput, err := os.ReadFile(test.expectedOutputFile) + assert.NoError(t, err, "should be able to read golden file: %s", test.expectedOutputFile) + assert.Equal(t, string(expectedOutput), out, "visual output should exactly match the golden ASCII representation for %s", test.name) + }) + } +} + +func TestIORenderer_Render_DifferentFileSizesAndRanges(t *testing.T) { + tc := []struct { + name string + filename string + size uint64 + ranges []Range + expectedOutputFile string + }{ + { + name: "small file", + filename: "small.txt", + size: 500, + ranges: []Range{{Start: 0, End: 100}, {Start: 200, End: 300}}, + expectedOutputFile: "testdata/io_renderer/different_file_sizes_small.txt", + }, + { + name: "medium file", + filename: "medium.txt", + size: 10 * 1024 * 1024, // 10 MB + ranges: []Range{{Start: 5000000, End: 7000000}, {Start: 2000000, End: 8000000}}, + expectedOutputFile: "testdata/io_renderer/different_file_sizes_medium.txt", + }, + { + name: "very large file", + filename: "very_large.txt", + size: 2 * 1024 * 1024 * 1024, // 2 GB + ranges: []Range{{Start: 0, End: 1000000}, {Start: 1500000000, End: 1501000000}}, + expectedOutputFile: "testdata/io_renderer/different_file_sizes_very_large.txt", + }, + { + name: "empty ranges", + filename: "empty_ranges.txt", + size: 1000, + ranges: []Range{}, // No ranges + expectedOutputFile: "testdata/io_renderer/different_file_sizes_empty_ranges.txt", + }, + } + + for _, test := range tc { + t.Run(test.name, func(t *testing.T) { + r, err := NewRenderer() + require.NoError(t, err, "NewRenderer should not return an error") + + out, err := r.Render(test.filename, test.size, test.ranges) + // os.WriteFile(test.expectedOutputFile, []byte(out), 0644) // Uncomment to create a new test output file. + + assert.NoError(t, err, "Render should not return an error for valid input") + expectedOutput, err := os.ReadFile(test.expectedOutputFile) + assert.NoError(t, err, "should be able to read golden file: %s", test.expectedOutputFile) + assert.Equal(t, string(expectedOutput), out, "visual output should exactly match the golden ASCII representation for %s", test.name) + }) + } +} diff --git a/internal/workloadinsight/testdata/io_renderer/custom_settings.txt b/internal/workloadinsight/testdata/io_renderer/custom_settings.txt new file mode 100644 index 0000000000..0dac67850f --- /dev/null +++ b/internal/workloadinsight/testdata/io_renderer/custom_settings.txt @@ -0,0 +1,9 @@ +Name: demo.txt +Total IOs: 4 +IO Size Distributions: (Min: 100B, Median: 200B, Max: 200B, Avg: 150B) + 0B 250B 500B 750B 1000B +[offset,len) |-----------|------------|-----------|-----------| +[0,100B) |█████ +[200,100B) | ██████ +[400,200B) | ███████████ +[800,200B) | ██████████ diff --git a/internal/workloadinsight/testdata/io_renderer/default_settings.txt b/internal/workloadinsight/testdata/io_renderer/default_settings.txt new file mode 100644 index 0000000000..0c43039c5b --- /dev/null +++ b/internal/workloadinsight/testdata/io_renderer/default_settings.txt @@ -0,0 +1,9 @@ +Name: demo.txt +Total IOs: 4 +IO Size Distributions: (Min: 100B, Median: 200B, Max: 200B, Avg: 150B) + 0B 250B 500B 750B 1000B +[offset,len) |-------------------|-------------------|-------------------|------------------| +[0,100B) |████████ +[200,100B) | █████████ +[400,200B) | █████████████████ +[800,200B) | ████████████████ diff --git a/internal/workloadinsight/testdata/io_renderer/different_file_sizes_empty_ranges.txt b/internal/workloadinsight/testdata/io_renderer/different_file_sizes_empty_ranges.txt new file mode 100644 index 0000000000..c4f678389a --- /dev/null +++ b/internal/workloadinsight/testdata/io_renderer/different_file_sizes_empty_ranges.txt @@ -0,0 +1,4 @@ +Name: empty_ranges.txt + 0B 250B 500B 750B 1000B +[offset,len) |------------------------|------------------------|------------------------|-----------------------| + diff --git a/internal/workloadinsight/testdata/io_renderer/different_file_sizes_medium.txt b/internal/workloadinsight/testdata/io_renderer/different_file_sizes_medium.txt new file mode 100644 index 0000000000..20746e983e --- /dev/null +++ b/internal/workloadinsight/testdata/io_renderer/different_file_sizes_medium.txt @@ -0,0 +1,7 @@ +Name: medium.txt +Total IOs: 2 +IO Size Distributions: (Min: 1.9M, Median: 5.7M, Max: 5.7M, Avg: 3.8M) + 0B 2.5M 5.0M 7.5M 10.0M +[offset,len) |------------------------|------------------------|------------------------|-----------------------| +[5000000,1.9M) | ████████████████████ +[2000000,5.7M) | ██████████████████████████████████████████████████████████ diff --git a/internal/workloadinsight/testdata/io_renderer/different_file_sizes_small.txt b/internal/workloadinsight/testdata/io_renderer/different_file_sizes_small.txt new file mode 100644 index 0000000000..1a59664ed9 --- /dev/null +++ b/internal/workloadinsight/testdata/io_renderer/different_file_sizes_small.txt @@ -0,0 +1,7 @@ +Name: small.txt +Total IOs: 2 +IO Size Distributions: (Min: 100B, Median: 100B, Max: 100B, Avg: 100B) + 0B 125B 250B 375B 500B +[offset,len) |------------------------|------------------------|------------------------|-----------------------| +[0,100B) |████████████████████ +[200,100B) | █████████████████████ diff --git a/internal/workloadinsight/testdata/io_renderer/different_file_sizes_very_large.txt b/internal/workloadinsight/testdata/io_renderer/different_file_sizes_very_large.txt new file mode 100644 index 0000000000..1ac052f1c0 --- /dev/null +++ b/internal/workloadinsight/testdata/io_renderer/different_file_sizes_very_large.txt @@ -0,0 +1,7 @@ +Name: very_large.txt +Total IOs: 2 +IO Size Distributions: (Min: 976.6K, Median: 976.6K, Max: 976.6K, Avg: 976.6K) + 0B 512.0M 1.0G 1.5G 2.0G +[offset,len) |------------------------|------------------------|------------------------|-----------------------| +[0,976.6K) |█ +[1500000000,976.6K) | █ diff --git a/main.go b/main.go index 9ba069bcbb..ed342471c1 100644 --- a/main.go +++ b/main.go @@ -22,9 +22,9 @@ package main import ( "log" - "github.com/googlecloudplatform/gcsfuse/v2/cmd" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" - "github.com/googlecloudplatform/gcsfuse/v2/internal/perf" + "github.com/googlecloudplatform/gcsfuse/v3/cmd" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/perf" ) func logPanic() { @@ -40,6 +40,7 @@ func logPanic() { // Refer https://go.dev/blog/generate for details. // //go:generate go run -C tools/config-gen . --paramsFile=../../cfg/params.yaml --outDir=../../cfg --templateDir=templates +//go:generate go run -C tools/metrics-gen . --input=../../metrics/metrics.yaml --outDir=../../metrics func main() { // Common configuration for all commands defer logPanic() diff --git a/metrics/constants.go b/metrics/constants.go new file mode 100644 index 0000000000..5a639b0d36 --- /dev/null +++ b/metrics/constants.go @@ -0,0 +1,29 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +const ( + ReadTypeUnknown int64 = -1 + ReadTypeSequential int64 = 0 + ReadTypeRandom int64 = 1 + ReadTypeParallel int64 = 2 +) + +var ReadTypeNames = map[int64]ReadType{ + ReadTypeUnknown: ReadTypeUnknownAttr, + ReadTypeSequential: ReadTypeSequentialAttr, + ReadTypeRandom: ReadTypeRandomAttr, + ReadTypeParallel: ReadTypeParallelAttr, +} diff --git a/metrics/helper.go b/metrics/helper.go new file mode 100644 index 0000000000..605806481e --- /dev/null +++ b/metrics/helper.go @@ -0,0 +1,22 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +// CaptureGCSReadMetrics is a helper function to encapsulate the logic for recording +// GCS read-related metrics. +func CaptureGCSReadMetrics(mh MetricHandle, readType ReadType, downloadBytes int64) { + mh.GcsReadCount(1, readType) + mh.GcsDownloadBytesCount(downloadBytes, readType) +} diff --git a/metrics/metric_handle.go b/metrics/metric_handle.go new file mode 100644 index 0000000000..7bad7174d7 --- /dev/null +++ b/metrics/metric_handle.go @@ -0,0 +1,245 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// **** DO NOT EDIT - FILE IS AUTO-GENERATED **** +package metrics + +import ( + "context" + "time" +) + +// EntryStatus is a custom type for the entry_status attribute. +type EntryStatus string + +const ( + EntryStatusNegativeAttr EntryStatus = "negative" + EntryStatusPositiveAttr EntryStatus = "positive" +) + +// FsErrorCategory is a custom type for the fs_error_category attribute. +type FsErrorCategory string + +const ( + FsErrorCategoryDEVICEERRORAttr FsErrorCategory = "DEVICE_ERROR" + FsErrorCategoryDIRNOTEMPTYAttr FsErrorCategory = "DIR_NOT_EMPTY" + FsErrorCategoryFILEDIRERRORAttr FsErrorCategory = "FILE_DIR_ERROR" + FsErrorCategoryFILEEXISTSAttr FsErrorCategory = "FILE_EXISTS" + FsErrorCategoryINTERRUPTERRORAttr FsErrorCategory = "INTERRUPT_ERROR" + FsErrorCategoryINVALIDARGUMENTAttr FsErrorCategory = "INVALID_ARGUMENT" + FsErrorCategoryINVALIDOPERATIONAttr FsErrorCategory = "INVALID_OPERATION" + FsErrorCategoryIOERRORAttr FsErrorCategory = "IO_ERROR" + FsErrorCategoryMISCERRORAttr FsErrorCategory = "MISC_ERROR" + FsErrorCategoryNETWORKERRORAttr FsErrorCategory = "NETWORK_ERROR" + FsErrorCategoryNOTADIRAttr FsErrorCategory = "NOT_A_DIR" + FsErrorCategoryNOTIMPLEMENTEDAttr FsErrorCategory = "NOT_IMPLEMENTED" + FsErrorCategoryNOFILEORDIRAttr FsErrorCategory = "NO_FILE_OR_DIR" + FsErrorCategoryPERMERRORAttr FsErrorCategory = "PERM_ERROR" + FsErrorCategoryPROCESSRESOURCEMGMTERRORAttr FsErrorCategory = "PROCESS_RESOURCE_MGMT_ERROR" + FsErrorCategoryTOOMANYOPENFILESAttr FsErrorCategory = "TOO_MANY_OPEN_FILES" +) + +// FsOp is a custom type for the fs_op attribute. +type FsOp string + +const ( + FsOpBatchForgetAttr FsOp = "BatchForget" + FsOpCreateFileAttr FsOp = "CreateFile" + FsOpCreateLinkAttr FsOp = "CreateLink" + FsOpCreateSymlinkAttr FsOp = "CreateSymlink" + FsOpFlushFileAttr FsOp = "FlushFile" + FsOpForgetInodeAttr FsOp = "ForgetInode" + FsOpGetInodeAttributesAttr FsOp = "GetInodeAttributes" + FsOpLookUpInodeAttr FsOp = "LookUpInode" + FsOpMkDirAttr FsOp = "MkDir" + FsOpMkNodeAttr FsOp = "MkNode" + FsOpOpenDirAttr FsOp = "OpenDir" + FsOpOpenFileAttr FsOp = "OpenFile" + FsOpOthersAttr FsOp = "Others" + FsOpReadDirAttr FsOp = "ReadDir" + FsOpReadDirPlusAttr FsOp = "ReadDirPlus" + FsOpReadFileAttr FsOp = "ReadFile" + FsOpReadSymlinkAttr FsOp = "ReadSymlink" + FsOpReleaseDirHandleAttr FsOp = "ReleaseDirHandle" + FsOpReleaseFileHandleAttr FsOp = "ReleaseFileHandle" + FsOpRenameAttr FsOp = "Rename" + FsOpRmDirAttr FsOp = "RmDir" + FsOpSetInodeAttributesAttr FsOp = "SetInodeAttributes" + FsOpSyncFileAttr FsOp = "SyncFile" + FsOpUnlinkAttr FsOp = "Unlink" + FsOpWriteFileAttr FsOp = "WriteFile" +) + +// GcsMethod is a custom type for the gcs_method attribute. +type GcsMethod string + +const ( + GcsMethodComposeObjectsAttr GcsMethod = "ComposeObjects" + GcsMethodCopyObjectAttr GcsMethod = "CopyObject" + GcsMethodCreateAppendableObjectWriterAttr GcsMethod = "CreateAppendableObjectWriter" + GcsMethodCreateFolderAttr GcsMethod = "CreateFolder" + GcsMethodCreateObjectAttr GcsMethod = "CreateObject" + GcsMethodCreateObjectChunkWriterAttr GcsMethod = "CreateObjectChunkWriter" + GcsMethodDeleteFolderAttr GcsMethod = "DeleteFolder" + GcsMethodDeleteObjectAttr GcsMethod = "DeleteObject" + GcsMethodFinalizeUploadAttr GcsMethod = "FinalizeUpload" + GcsMethodFlushPendingWritesAttr GcsMethod = "FlushPendingWrites" + GcsMethodGetFolderAttr GcsMethod = "GetFolder" + GcsMethodListObjectsAttr GcsMethod = "ListObjects" + GcsMethodMoveObjectAttr GcsMethod = "MoveObject" + GcsMethodMultiRangeDownloaderAddAttr GcsMethod = "MultiRangeDownloader::Add" + GcsMethodNewMultiRangeDownloaderAttr GcsMethod = "NewMultiRangeDownloader" + GcsMethodNewReaderAttr GcsMethod = "NewReader" + GcsMethodRenameFolderAttr GcsMethod = "RenameFolder" + GcsMethodStatObjectAttr GcsMethod = "StatObject" + GcsMethodUpdateObjectAttr GcsMethod = "UpdateObject" +) + +// IoMethod is a custom type for the io_method attribute. +type IoMethod string + +const ( + IoMethodClosedAttr IoMethod = "closed" + IoMethodOpenedAttr IoMethod = "opened" +) + +// LookupDetail is a custom type for the lookup_detail attribute. +type LookupDetail string + +const ( + LookupDetailFoundAttr LookupDetail = "found" + LookupDetailNotFoundAttr LookupDetail = "not_found" + LookupDetailTtlExpiredAttr LookupDetail = "ttl_expired" +) + +// OpenMode is a custom type for the open_mode attribute. +type OpenMode string + +const ( + OpenModeOtherAttr OpenMode = "other" + OpenModeReadWriteAttr OpenMode = "read_write" + OpenModeReadWriteAppendAttr OpenMode = "read_write_append" + OpenModeWriteOnlyAttr OpenMode = "write_only" + OpenModeWriteOnlyAppendAttr OpenMode = "write_only_append" +) + +// ReadType is a custom type for the read_type attribute. +type ReadType string + +const ( + ReadTypeBufferedAttr ReadType = "Buffered" + ReadTypeParallelAttr ReadType = "Parallel" + ReadTypeRandomAttr ReadType = "Random" + ReadTypeSequentialAttr ReadType = "Sequential" + ReadTypeUnknownAttr ReadType = "Unknown" +) + +// Reason is a custom type for the reason attribute. +type Reason string + +const ( + ReasonInsufficientMemoryAttr Reason = "insufficient_memory" + ReasonRandomReadDetectedAttr Reason = "random_read_detected" +) + +// RequestType is a custom type for the request_type attribute. +type RequestType string + +const ( + RequestTypeAttr1Attr RequestType = "attr1" + RequestTypeAttr2Attr RequestType = "attr2" +) + +// RetryErrorCategory is a custom type for the retry_error_category attribute. +type RetryErrorCategory string + +const ( + RetryErrorCategoryOTHERERRORSAttr RetryErrorCategory = "OTHER_ERRORS" + RetryErrorCategorySTALLEDREADREQUESTAttr RetryErrorCategory = "STALLED_READ_REQUEST" +) + +// WriteFallbackReason is a custom type for the write_fallback_reason attribute. +type WriteFallbackReason string + +const ( + WriteFallbackReasonConcurrencyLimitBreachedAttr WriteFallbackReason = "concurrency_limit_breached" + WriteFallbackReasonExistingFileAttr WriteFallbackReason = "existing_file" + WriteFallbackReasonOtherAttr WriteFallbackReason = "other" + WriteFallbackReasonOutOfOrderAttr WriteFallbackReason = "out_of_order" +) + +// MetricHandle provides an interface for recording metrics. +// The methods of this interface are auto-generated from metrics.yaml. +// Each method corresponds to a metric defined in metrics.yaml. +type MetricHandle interface { + // BufferedReadFallbackTriggerCount - The cumulative number of times the BufferedReader falls back to a different reader, along with the reason: random_read_detected or insufficient_memory. + BufferedReadFallbackTriggerCount(inc int64, reason Reason) + + // BufferedReadReadLatency - The cumulative distribution of latencies for ReadAt calls served by the buffered reader. + BufferedReadReadLatency(ctx context.Context, latency time.Duration) + + // FileCacheReadBytesCount - The cumulative number of bytes read from file cache along with read type - Sequential/Random + FileCacheReadBytesCount(inc int64, readType ReadType) + + // FileCacheReadCount - Specifies the number of read requests made via file cache along with type - Sequential/Random and cache hit - true/false + FileCacheReadCount(inc int64, cacheHit bool, readType ReadType) + + // FileCacheReadLatencies - The cumulative distribution of the file cache read latencies along with cache hit - true/false. + FileCacheReadLatencies(ctx context.Context, latency time.Duration, cacheHit bool) + + // FsOpsCount - The cumulative number of ops processed by the file system. + FsOpsCount(inc int64, fsOp FsOp) + + // FsOpsErrorCount - The cumulative number of errors generated by file system operations. + FsOpsErrorCount(inc int64, fsErrorCategory FsErrorCategory, fsOp FsOp) + + // FsOpsLatency - The cumulative distribution of file system operation latencies + FsOpsLatency(ctx context.Context, latency time.Duration, fsOp FsOp) + + // FsStreamingWriteFallbackCount - The cumulative number of streaming write fallbacks with reason attached + FsStreamingWriteFallbackCount(inc int64, openMode OpenMode, writeFallbackReason WriteFallbackReason) + + // GcsDownloadBytesCount - The cumulative number of bytes downloaded from GCS along with type - Sequential/Random + GcsDownloadBytesCount(inc int64, readType ReadType) + + // GcsReadBytesCount - The cumulative number of bytes read from GCS objects. + GcsReadBytesCount(inc int64) + + // GcsReadCount - Specifies the number of gcs reads made along with type - Sequential/Random + GcsReadCount(inc int64, readType ReadType) + + // GcsReaderCount - The cumulative number of GCS object readers opened or closed. + GcsReaderCount(inc int64, ioMethod IoMethod) + + // GcsRequestCount - The cumulative number of GCS requests processed along with the GCS method. + GcsRequestCount(inc int64, gcsMethod GcsMethod) + + // GcsRequestLatencies - The cumulative distribution of the GCS request latencies. + GcsRequestLatencies(ctx context.Context, latency time.Duration, gcsMethod GcsMethod) + + // GcsRetryCount - The cumulative number of retry requests made to GCS. + GcsRetryCount(inc int64, retryErrorCategory RetryErrorCategory) + + // MetadataCacheReadCount - Total number of read requests to the metadata cache. Use attributes to analyze hit/miss ratios, entry types, and specific lookup outcomes (e.g., expiration vs. total absence). + MetadataCacheReadCount(inc int64, cacheHit bool, entryStatus EntryStatus, lookupDetail LookupDetail) + + // ReadBlockSizes - The cumulative distribution of read block sizes across different bucket boundaries + ReadBlockSizes(ctx context.Context, value int64) + + // TestUpdownCounter - Test metric for updown counters. + TestUpdownCounter(inc int64) + + // TestUpdownCounterWithAttrs - Test metric for updown counters with attributes. + TestUpdownCounterWithAttrs(inc int64, requestType RequestType) +} diff --git a/metrics/metrics.yaml b/metrics/metrics.yaml new file mode 100644 index 0000000000..572cae412e --- /dev/null +++ b/metrics/metrics.yaml @@ -0,0 +1,311 @@ +- metric-name: "buffered_read/fallback_trigger_count" + description: "The cumulative number of times the BufferedReader falls back to a different reader, along with the reason: random_read_detected or insufficient_memory." + type: "int_counter" + attributes: + - attribute-name: reason + attribute-type: string + values: + - "insufficient_memory" + - "random_read_detected" + +- metric-name: "buffered_read/read_latency" + description: "The cumulative distribution of latencies for ReadAt calls served by the buffered reader." + unit: "us" + type: "time_histogram" + boundaries: µseconds_boundaries + - 50 + - 100 + - 200 + - 400 + - 800 + - 1500 + - 3000 + - 5000 + - 10000 + - 20000 + - 50000 + - 100000 + - 200000 + - 500000 + - 1000000 + - 2000000 + - 5000000 + - 10000000 + - 20000000 + - 50000000 + - 100000000 + - 200000000 + - 500000000 + + +- metric-name: "file_cache/read_bytes_count" + description: "The cumulative number of bytes read from file cache along with read type - Sequential/Random" + unit: "By" + type: "int_counter" + attributes: + - attribute-name: read_type + attribute-type: string + values: &read_types_list + - "Parallel" + - "Random" + - "Sequential" + - "Unknown" + +- metric-name: "file_cache/read_count" + description: "Specifies the number of read requests made via file cache along with type - Sequential/Random and cache hit - true/false" + type: "int_counter" + attributes: + - attribute-name: cache_hit + attribute-type: bool + - attribute-name: read_type + attribute-type: string + values: *read_types_list + +- metric-name: "file_cache/read_latencies" + description: "The cumulative distribution of the file cache read latencies along with cache hit - true/false." + type: "time_histogram" + unit: "us" + boundaries: *microseconds_boundaries + attributes: + - attribute-name: cache_hit + attribute-type: bool + +- metric-name: "fs/ops_count" + description: "The cumulative number of ops processed by the file system." + type: "int_counter" + attributes: + - attribute-name: fs_op + attribute-type: string + values: &fs_ops_list + - "BatchForget" + - "CreateFile" + - "CreateLink" + - "CreateSymlink" + - "FlushFile" + - "ForgetInode" + - "GetInodeAttributes" + - "LookUpInode" + - "MkDir" + - "MkNode" + - "OpenDir" + - "OpenFile" + - "ReadDir" + - "ReadDirPlus" + - "ReadFile" + - "ReadSymlink" + - "ReleaseDirHandle" + - "ReleaseFileHandle" + - "Rename" + - "RmDir" + - "SetInodeAttributes" + - "SyncFile" + - "Unlink" + - "WriteFile" + - "Others" + +- metric-name: "fs/ops_error_count" + description: "The cumulative number of errors generated by file system operations." + type: "int_counter" + attributes: + - attribute-name: fs_error_category + attribute-type: string + values: + - "DEVICE_ERROR" + - "DIR_NOT_EMPTY" + - "FILE_DIR_ERROR" + - "FILE_EXISTS" + - "INTERRUPT_ERROR" + - "INVALID_ARGUMENT" + - "INVALID_OPERATION" + - "IO_ERROR" + - "MISC_ERROR" + - "NETWORK_ERROR" + - "NOT_A_DIR" + - "NOT_IMPLEMENTED" + - "NO_FILE_OR_DIR" + - "PERM_ERROR" + - "PROCESS_RESOURCE_MGMT_ERROR" + - "TOO_MANY_OPEN_FILES" + - attribute-name: fs_op + attribute-type: string + values: *fs_ops_list + +- metric-name: "fs/ops_latency" + description: "The cumulative distribution of file system operation latencies" + type: "time_histogram" + unit: "us" + boundaries: *microseconds_boundaries + attributes: + - attribute-name: fs_op + attribute-type: string + values: *fs_ops_list + +- metric-name: "fs/streaming_write_fallback_count" + description: "The cumulative number of streaming write fallbacks with reason attached" + type: "int_counter" + attributes: + - attribute-name: open_mode # Note: considering only those modes valid for write + attribute-type: string + values: + - "write_only" + - "write_only_append" + - "read_write" + - "read_write_append" + - "other" + - attribute-name: write_fallback_reason + attribute-type: string + values: + - "out_of_order" + - "existing_file" + - "concurrency_limit_breached" + - "other" # tracks any other errors not from above + +- metric-name: "gcs/download_bytes_count" + description: "The cumulative number of bytes downloaded from GCS along with type - Sequential/Random" + unit: "By" + type: "int_counter" + attributes: + - attribute-name: read_type + attribute-type: string + values: + - "Buffered" + - "Parallel" + - "Random" + - "Sequential" + +- metric-name: "gcs/read_bytes_count" + description: "The cumulative number of bytes read from GCS objects." + unit: "By" + type: "int_counter" + +- metric-name: "gcs/read_count" + description: "Specifies the number of gcs reads made along with type - Sequential/Random" + type: "int_counter" + attributes: + - attribute-name: read_type + attribute-type: string + values: *read_types_list + +- metric-name: "gcs/reader_count" + description: "The cumulative number of GCS object readers opened or closed." + type: "int_counter" + attributes: + - attribute-name: io_method + attribute-type: string + values: + - "closed" + - "opened" + +- metric-name: "gcs/request_count" + description: "The cumulative number of GCS requests processed along with the GCS method." + type: "int_counter" + attributes: + - attribute-name: gcs_method + attribute-type: string + values: &gcs_method_list + - "ComposeObjects" + - "CopyObject" + - "CreateAppendableObjectWriter" + - "CreateFolder" + - "CreateObject" + - "CreateObjectChunkWriter" + - "DeleteFolder" + - "DeleteObject" + - "FinalizeUpload" + - "FlushPendingWrites" + - "GetFolder" + - "ListObjects" + - "MoveObject" + - "MultiRangeDownloader::Add" + - "NewMultiRangeDownloader" + - "NewReader" + - "RenameFolder" + - "StatObject" + - "UpdateObject" + +- metric-name: "gcs/request_latencies" + description: "The cumulative distribution of the GCS request latencies." + type: "time_histogram" + unit: "ms" + boundaries: &millisecond_boundaries + - 100 + - 200 + - 400 + - 800 + - 1500 + - 3000 + - 5000 + - 10000 + - 20000 + - 50000 + - 100000 + - 200000 + - 500000 + attributes: + - attribute-name: gcs_method + attribute-type: string + values: *gcs_method_list + +- metric-name: "gcs/retry_count" + description: "The cumulative number of retry requests made to GCS." + type: "int_counter" + attributes: + - attribute-name: retry_error_category + attribute-type: string + values: + - "OTHER_ERRORS" + - "STALLED_READ_REQUEST" + +- metric-name: "metadata_cache/read_count" + description: "Total number of read requests to the metadata cache. Use attributes to analyze hit/miss ratios, entry types, and specific lookup outcomes (e.g., expiration vs. total absence)." + type: "int_counter" + attributes: + - attribute-name: cache_hit + attribute-type: bool + - attribute-name: entry_status + attribute-type: string + values: + - positive + - negative + - attribute-name: lookup_detail + attribute-type: string + values: + - found + - ttl_expired + - not_found + +- metric-name: "read/block_sizes" + description: "The cumulative distribution of read block sizes across different bucket boundaries" + type: "int_histogram" + unit: "By" + boundaries: &bytes_boundaries # in bytes + - 0 # To detect EOF scenarios + - 8192 + - 16384 + - 32768 + - 65536 + - 131072 + - 262144 + - 524288 + - 1048576 + - 2097152 + - 4194304 + - 8388608 + - 16777216 + - 33554432 + - 67108864 + - 134217728 + +- metric-name: "test/updown_counter" + description: "Test metric for updown counters." + type: "int_up_down_counter" + +- metric-name: "test/updown_counter_with_attrs" + description: "Test metric for updown counters with attributes." + type: "int_up_down_counter" + attributes: + - attribute-name: "request_type" + attribute-type: "string" + values: + - "attr1" + - "attr2" diff --git a/metrics/metrics_test_utils.go b/metrics/metrics_test_utils.go new file mode 100644 index 0000000000..748b1ce704 --- /dev/null +++ b/metrics/metrics_test_utils.go @@ -0,0 +1,204 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +type verifyConfig struct { + atLeast bool + subset bool +} + +// VerifyOption defines functional options for metric verification. +type VerifyOption func(*verifyConfig) + +// AtLeast changes the verification to check if the metric value is greater or equal +// to the expected value, rather than exactly equal. +func AtLeast() VerifyOption { + return func(c *verifyConfig) { c.atLeast = true } +} + +// Subset changes the verification to check if the provided attributes are a subset +// of the recorded attributes, rather than an exact match. +func Subset() VerifyOption { + return func(c *verifyConfig) { c.subset = true } +} + +func matchesAttributes(dpAttrs attribute.Set, targetAttrs attribute.Set, subset bool, encoder attribute.Encoder) bool { + if !subset { + return dpAttrs.Encoded(encoder) == targetAttrs.Encoded(encoder) + } + // Subset matching + for _, targetKV := range targetAttrs.ToSlice() { + val, ok := dpAttrs.Value(targetKV.Key) + if !ok || val.Emit() != targetKV.Value.Emit() { + return false + } + } + return true +} + +func verifyValue[T int64 | uint64](t *testing.T, actual T, expected T, atLeast bool, metricName string, attrs attribute.Set) { + if atLeast { + assert.GreaterOrEqual(t, actual, expected, "metric value too low for %s with attributes %v", metricName, attrs) + } else { + assert.Equal(t, expected, actual, "metric value mismatch for %s with attributes %v", metricName, attrs) + } +} + +// VerifyCounterMetric finds a counter metric across all scopes and verifies its value. +// By default, it requires an exact attribute match and an exact value match. +// Use AtLeast() or Subset() options to relax these requirements. +func VerifyCounterMetric(t *testing.T, ctx context.Context, reader *metric.ManualReader, metricName string, attrs attribute.Set, expectedValue int64, options ...VerifyOption) { + t.Helper() + cfg := &verifyConfig{} + for _, opt := range options { + opt(cfg) + } + + var rm metricdata.ResourceMetrics + err := reader.Collect(ctx, &rm) + require.NoError(t, err, "reader.Collect") + encoder := attribute.DefaultEncoder() + + foundMetric := false + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name == metricName { + foundMetric = true + data, ok := m.Data.(metricdata.Sum[int64]) + require.True(t, ok, "metric %s is not a Sum[int64], but %T", metricName, m.Data) + + for _, dp := range data.DataPoints { + if matchesAttributes(dp.Attributes, attrs, cfg.subset, encoder) { + verifyValue(t, dp.Value, expectedValue, cfg.atLeast, metricName, attrs) + return + } + } + } + } + } + + require.True(t, foundMetric, "metric %s not found", metricName) + require.Fail(t, "Data point for attributes %v not found in %s metric", attrs, metricName) +} + +// VerifyHistogramMetric finds a histogram metric across all scopes and verifies its count. +// By default, it requires an exact attribute match and an exact count match. +// Use AtLeast() or Subset() options to relax these requirements. +func VerifyHistogramMetric(t *testing.T, ctx context.Context, reader *metric.ManualReader, metricName string, attrs attribute.Set, expectedCount uint64, options ...VerifyOption) { + t.Helper() + cfg := &verifyConfig{} + for _, opt := range options { + opt(cfg) + } + + var rm metricdata.ResourceMetrics + err := reader.Collect(ctx, &rm) + require.NoError(t, err, "reader.Collect") + encoder := attribute.DefaultEncoder() + + foundMetric := false + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name == metricName { + foundMetric = true + switch data := m.Data.(type) { + case metricdata.Histogram[int64]: + for _, dp := range data.DataPoints { + if matchesAttributes(dp.Attributes, attrs, cfg.subset, encoder) { + verifyValue(t, dp.Count, expectedCount, cfg.atLeast, metricName, attrs) + return + } + } + case metricdata.Histogram[float64]: + for _, dp := range data.DataPoints { + if matchesAttributes(dp.Attributes, attrs, cfg.subset, encoder) { + verifyValue(t, dp.Count, expectedCount, cfg.atLeast, metricName, attrs) + return + } + } + default: + require.Fail(t, "metric %s is not an expected histogram type, but %T", metricName, m.Data) + } + } + } + } + require.True(t, foundMetric, "metric %s not found", metricName) + require.Fail(t, "Data point for attributes %v not found in %s metric", attrs, metricName) +} + +// VerifyHistogramFull finds a histogram metric and fully verifies its state including total count, sum, and bucket distribution. +// expectedBuckets is a map of bucket indices to their expected counts. +func VerifyHistogramFull[T int64 | float64](t *testing.T, ctx context.Context, reader *metric.ManualReader, metricName string, attrs attribute.Set, expectedCount uint64, expectedSum T, expectedBuckets map[int]uint64, options ...VerifyOption) { + t.Helper() + cfg := &verifyConfig{} + for _, opt := range options { + opt(cfg) + } + + var rm metricdata.ResourceMetrics + err := reader.Collect(ctx, &rm) + require.NoError(t, err, "reader.Collect") + encoder := attribute.DefaultEncoder() + + foundMetric := false + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name == metricName { + foundMetric = true + data, ok := m.Data.(metricdata.Histogram[T]) + require.True(t, ok, "metric %s is not of expected histogram type %T, but %T", metricName, data, m.Data) + + for _, dp := range data.DataPoints { + if matchesAttributes(dp.Attributes, attrs, cfg.subset, encoder) { + // Assert total count + require.Equal(t, expectedCount, dp.Count, "Total count mismatch for %s", metricName) + + // Assert total sum + if !cfg.atLeast { + require.Equal(t, expectedSum, dp.Sum, "Total sum mismatch for %s", metricName) + } + + // Assert individual bucket counts + for bucketIdx, expBucketCount := range expectedBuckets { + require.GreaterOrEqual(t, len(dp.BucketCounts), bucketIdx+1, "Bucket index %d out of range for %s", bucketIdx, metricName) + require.Equal(t, expBucketCount, dp.BucketCounts[bucketIdx], "Bucket %d count mismatch for %s", bucketIdx, metricName) + } + + // Verify that sum of all bucket counts matches total count + var totalBucketCount uint64 + for _, count := range dp.BucketCounts { + totalBucketCount += count + } + require.Equal(t, expectedCount, totalBucketCount, "Sum of bucket counts must equal total count for %s", metricName) + return + } + } + } + } + } + require.True(t, foundMetric, "metric %s not found", metricName) + require.Fail(t, "Data point for attributes %v not found in %s metric", attrs, metricName) +} diff --git a/metrics/noop_metrics.go b/metrics/noop_metrics.go new file mode 100644 index 0000000000..d61dd848d4 --- /dev/null +++ b/metrics/noop_metrics.go @@ -0,0 +1,72 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// **** DO NOT EDIT - FILE IS AUTO-GENERATED **** +package metrics + +import ( + "context" + "time" +) + +type noopMetrics struct{} + +func (*noopMetrics) BufferedReadFallbackTriggerCount(inc int64, reason Reason) {} + +func (*noopMetrics) BufferedReadReadLatency(ctx context.Context, latency time.Duration) {} + +func (*noopMetrics) FileCacheReadBytesCount(inc int64, readType ReadType) {} + +func (*noopMetrics) FileCacheReadCount(inc int64, cacheHit bool, readType ReadType) {} + +func (*noopMetrics) FileCacheReadLatencies(ctx context.Context, latency time.Duration, cacheHit bool) { +} + +func (*noopMetrics) FsOpsCount(inc int64, fsOp FsOp) {} + +func (*noopMetrics) FsOpsErrorCount(inc int64, fsErrorCategory FsErrorCategory, fsOp FsOp) {} + +func (*noopMetrics) FsOpsLatency(ctx context.Context, latency time.Duration, fsOp FsOp) {} + +func (*noopMetrics) FsStreamingWriteFallbackCount(inc int64, openMode OpenMode, writeFallbackReason WriteFallbackReason) { +} + +func (*noopMetrics) GcsDownloadBytesCount(inc int64, readType ReadType) {} + +func (*noopMetrics) GcsReadBytesCount(inc int64) {} + +func (*noopMetrics) GcsReadCount(inc int64, readType ReadType) {} + +func (*noopMetrics) GcsReaderCount(inc int64, ioMethod IoMethod) {} + +func (*noopMetrics) GcsRequestCount(inc int64, gcsMethod GcsMethod) {} + +func (*noopMetrics) GcsRequestLatencies(ctx context.Context, latency time.Duration, gcsMethod GcsMethod) { +} + +func (*noopMetrics) GcsRetryCount(inc int64, retryErrorCategory RetryErrorCategory) {} + +func (*noopMetrics) MetadataCacheReadCount(inc int64, cacheHit bool, entryStatus EntryStatus, lookupDetail LookupDetail) { +} + +func (*noopMetrics) ReadBlockSizes(ctx context.Context, value int64) {} + +func (*noopMetrics) TestUpdownCounter(inc int64) {} + +func (*noopMetrics) TestUpdownCounterWithAttrs(inc int64, requestType RequestType) {} + +func NewNoopMetrics() MetricHandle { + var n noopMetrics + return &n +} diff --git a/metrics/otel_metrics.go b/metrics/otel_metrics.go new file mode 100644 index 0000000000..4ee1ee6ec1 --- /dev/null +++ b/metrics/otel_metrics.go @@ -0,0 +1,4363 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// **** DO NOT EDIT - FILE IS AUTO-GENERATED **** + +package metrics + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +const logInterval = 5 * time.Minute + +var ( + unrecognizedAttr atomic.Value + bufferedReadFallbackTriggerCountReasonInsufficientMemoryAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("reason", "insufficient_memory"))) + bufferedReadFallbackTriggerCountReasonRandomReadDetectedAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("reason", "random_read_detected"))) + fileCacheReadBytesCountReadTypeParallelAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("read_type", "Parallel"))) + fileCacheReadBytesCountReadTypeRandomAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("read_type", "Random"))) + fileCacheReadBytesCountReadTypeSequentialAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("read_type", "Sequential"))) + fileCacheReadBytesCountReadTypeUnknownAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("read_type", "Unknown"))) + fileCacheReadCountCacheHitTrueReadTypeParallelAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("read_type", "Parallel"))) + fileCacheReadCountCacheHitTrueReadTypeRandomAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("read_type", "Random"))) + fileCacheReadCountCacheHitTrueReadTypeSequentialAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("read_type", "Sequential"))) + fileCacheReadCountCacheHitTrueReadTypeUnknownAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("read_type", "Unknown"))) + fileCacheReadCountCacheHitFalseReadTypeParallelAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("read_type", "Parallel"))) + fileCacheReadCountCacheHitFalseReadTypeRandomAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("read_type", "Random"))) + fileCacheReadCountCacheHitFalseReadTypeSequentialAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("read_type", "Sequential"))) + fileCacheReadCountCacheHitFalseReadTypeUnknownAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("read_type", "Unknown"))) + fileCacheReadLatenciesCacheHitTrueAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", true))) + fileCacheReadLatenciesCacheHitFalseAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", false))) + fsOpsCountFsOpBatchForgetAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "BatchForget"))) + fsOpsCountFsOpCreateFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "CreateFile"))) + fsOpsCountFsOpCreateLinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "CreateLink"))) + fsOpsCountFsOpCreateSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "CreateSymlink"))) + fsOpsCountFsOpFlushFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "FlushFile"))) + fsOpsCountFsOpForgetInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "ForgetInode"))) + fsOpsCountFsOpGetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "GetInodeAttributes"))) + fsOpsCountFsOpLookUpInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "LookUpInode"))) + fsOpsCountFsOpMkDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "MkDir"))) + fsOpsCountFsOpMkNodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "MkNode"))) + fsOpsCountFsOpOpenDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "OpenDir"))) + fsOpsCountFsOpOpenFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "OpenFile"))) + fsOpsCountFsOpOthersAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "Others"))) + fsOpsCountFsOpReadDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "ReadDir"))) + fsOpsCountFsOpReadDirPlusAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "ReadDirPlus"))) + fsOpsCountFsOpReadFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "ReadFile"))) + fsOpsCountFsOpReadSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "ReadSymlink"))) + fsOpsCountFsOpReleaseDirHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "ReleaseDirHandle"))) + fsOpsCountFsOpReleaseFileHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "ReleaseFileHandle"))) + fsOpsCountFsOpRenameAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "Rename"))) + fsOpsCountFsOpRmDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "RmDir"))) + fsOpsCountFsOpSetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "SetInodeAttributes"))) + fsOpsCountFsOpSyncFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "SyncFile"))) + fsOpsCountFsOpUnlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "Unlink"))) + fsOpsCountFsOpWriteFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "WriteFile"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpBatchForgetAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "BatchForget"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "CreateFile"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateLinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "CreateLink"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "CreateSymlink"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpFlushFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "FlushFile"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpForgetInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "ForgetInode"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpGetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "GetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpLookUpInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "LookUpInode"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpMkDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "MkDir"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpMkNodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "MkNode"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOpenDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "OpenDir"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOpenFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "OpenFile"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOthersAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "Others"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "ReadDir"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadDirPlusAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "ReadDirPlus"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "ReadFile"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "ReadSymlink"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReleaseDirHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "ReleaseDirHandle"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReleaseFileHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "ReleaseFileHandle"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpRenameAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "Rename"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpRmDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "RmDir"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpSetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "SetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpSyncFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "SyncFile"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpUnlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "Unlink"))) + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpWriteFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "WriteFile"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpBatchForgetAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "BatchForget"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "CreateFile"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateLinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "CreateLink"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "CreateSymlink"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpFlushFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "FlushFile"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpForgetInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "ForgetInode"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpGetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "GetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpLookUpInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "LookUpInode"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpMkDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "MkDir"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpMkNodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "MkNode"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOpenDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "OpenDir"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOpenFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "OpenFile"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOthersAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "Others"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "ReadDir"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadDirPlusAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "ReadDirPlus"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "ReadFile"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "ReadSymlink"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReleaseDirHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "ReleaseDirHandle"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReleaseFileHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "ReleaseFileHandle"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpRenameAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "Rename"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpRmDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "RmDir"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpSetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "SetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpSyncFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "SyncFile"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpUnlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "Unlink"))) + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpWriteFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "WriteFile"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpBatchForgetAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "BatchForget"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "CreateFile"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateLinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "CreateLink"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "CreateSymlink"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpFlushFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "FlushFile"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpForgetInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "ForgetInode"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpGetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "GetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpLookUpInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "LookUpInode"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpMkDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "MkDir"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpMkNodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "MkNode"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOpenDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "OpenDir"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOpenFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "OpenFile"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOthersAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "Others"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "ReadDir"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadDirPlusAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "ReadDirPlus"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "ReadFile"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "ReadSymlink"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReleaseDirHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "ReleaseDirHandle"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReleaseFileHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "ReleaseFileHandle"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpRenameAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "Rename"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpRmDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "RmDir"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpSetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "SetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpSyncFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "SyncFile"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpUnlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "Unlink"))) + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpWriteFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "WriteFile"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpBatchForgetAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "BatchForget"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "CreateFile"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateLinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "CreateLink"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "CreateSymlink"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpFlushFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "FlushFile"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpForgetInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "ForgetInode"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpGetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "GetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpLookUpInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "LookUpInode"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpMkDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "MkDir"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpMkNodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "MkNode"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOpenDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "OpenDir"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOpenFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "OpenFile"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOthersAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "Others"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "ReadDir"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadDirPlusAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "ReadDirPlus"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "ReadFile"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "ReadSymlink"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReleaseDirHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "ReleaseDirHandle"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReleaseFileHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "ReleaseFileHandle"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpRenameAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "Rename"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpRmDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "RmDir"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpSetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "SetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpSyncFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "SyncFile"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpUnlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "Unlink"))) + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpWriteFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "WriteFile"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpBatchForgetAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "BatchForget"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "CreateFile"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateLinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "CreateLink"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "CreateSymlink"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpFlushFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "FlushFile"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpForgetInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "ForgetInode"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpGetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "GetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpLookUpInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "LookUpInode"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpMkDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "MkDir"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpMkNodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "MkNode"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOpenDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "OpenDir"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOpenFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "OpenFile"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOthersAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "Others"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "ReadDir"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadDirPlusAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "ReadDirPlus"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "ReadFile"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "ReadSymlink"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReleaseDirHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "ReleaseDirHandle"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReleaseFileHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "ReleaseFileHandle"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpRenameAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "Rename"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpRmDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "RmDir"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpSetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "SetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpSyncFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "SyncFile"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpUnlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "Unlink"))) + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpWriteFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "WriteFile"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpBatchForgetAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "BatchForget"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "CreateFile"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateLinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "CreateLink"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "CreateSymlink"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpFlushFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "FlushFile"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpForgetInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "ForgetInode"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpGetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "GetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpLookUpInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "LookUpInode"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpMkDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "MkDir"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpMkNodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "MkNode"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOpenDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "OpenDir"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOpenFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "OpenFile"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOthersAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "Others"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "ReadDir"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadDirPlusAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "ReadDirPlus"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "ReadFile"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "ReadSymlink"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReleaseDirHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "ReleaseDirHandle"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReleaseFileHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "ReleaseFileHandle"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpRenameAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "Rename"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpRmDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "RmDir"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpSetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "SetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpSyncFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "SyncFile"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpUnlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "Unlink"))) + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpWriteFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "WriteFile"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpBatchForgetAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "BatchForget"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "CreateFile"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateLinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "CreateLink"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "CreateSymlink"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpFlushFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "FlushFile"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpForgetInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "ForgetInode"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpGetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "GetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpLookUpInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "LookUpInode"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpMkDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "MkDir"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpMkNodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "MkNode"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOpenDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "OpenDir"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOpenFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "OpenFile"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOthersAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "Others"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "ReadDir"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadDirPlusAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "ReadDirPlus"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "ReadFile"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "ReadSymlink"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReleaseDirHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "ReleaseDirHandle"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReleaseFileHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "ReleaseFileHandle"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpRenameAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "Rename"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpRmDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "RmDir"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpSetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "SetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpSyncFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "SyncFile"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpUnlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "Unlink"))) + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpWriteFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "WriteFile"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpBatchForgetAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "BatchForget"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "CreateFile"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateLinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "CreateLink"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "CreateSymlink"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpFlushFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "FlushFile"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpForgetInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "ForgetInode"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpGetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "GetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpLookUpInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "LookUpInode"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpMkDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "MkDir"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpMkNodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "MkNode"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpOpenDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "OpenDir"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpOpenFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "OpenFile"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpOthersAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "Others"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "ReadDir"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadDirPlusAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "ReadDirPlus"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "ReadFile"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "ReadSymlink"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReleaseDirHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "ReleaseDirHandle"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReleaseFileHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "ReleaseFileHandle"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpRenameAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "Rename"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpRmDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "RmDir"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpSetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "SetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpSyncFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "SyncFile"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpUnlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "Unlink"))) + fsOpsErrorCountFsErrorCategoryIOERRORFsOpWriteFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "WriteFile"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpBatchForgetAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "BatchForget"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "CreateFile"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateLinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "CreateLink"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "CreateSymlink"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpFlushFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "FlushFile"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpForgetInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "ForgetInode"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpGetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "GetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpLookUpInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "LookUpInode"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpMkDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "MkDir"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpMkNodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "MkNode"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOpenDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "OpenDir"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOpenFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "OpenFile"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOthersAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "Others"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "ReadDir"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadDirPlusAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "ReadDirPlus"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "ReadFile"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "ReadSymlink"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReleaseDirHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "ReleaseDirHandle"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReleaseFileHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "ReleaseFileHandle"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpRenameAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "Rename"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpRmDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "RmDir"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpSetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "SetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpSyncFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "SyncFile"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpUnlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "Unlink"))) + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpWriteFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "WriteFile"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpBatchForgetAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "BatchForget"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "CreateFile"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateLinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "CreateLink"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "CreateSymlink"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpFlushFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "FlushFile"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpForgetInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "ForgetInode"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpGetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "GetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpLookUpInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "LookUpInode"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpMkDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "MkDir"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpMkNodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "MkNode"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOpenDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "OpenDir"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOpenFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "OpenFile"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOthersAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "Others"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "ReadDir"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadDirPlusAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "ReadDirPlus"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "ReadFile"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "ReadSymlink"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReleaseDirHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "ReleaseDirHandle"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReleaseFileHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "ReleaseFileHandle"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpRenameAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "Rename"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpRmDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "RmDir"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpSetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "SetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpSyncFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "SyncFile"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpUnlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "Unlink"))) + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpWriteFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "WriteFile"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpBatchForgetAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "BatchForget"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "CreateFile"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateLinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "CreateLink"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "CreateSymlink"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpFlushFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "FlushFile"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpForgetInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "ForgetInode"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpGetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "GetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpLookUpInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "LookUpInode"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpMkDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "MkDir"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpMkNodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "MkNode"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOpenDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "OpenDir"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOpenFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "OpenFile"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOthersAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "Others"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "ReadDir"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadDirPlusAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "ReadDirPlus"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "ReadFile"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "ReadSymlink"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReleaseDirHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "ReleaseDirHandle"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReleaseFileHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "ReleaseFileHandle"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpRenameAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "Rename"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpRmDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "RmDir"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpSetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "SetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpSyncFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "SyncFile"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpUnlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "Unlink"))) + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpWriteFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "WriteFile"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpBatchForgetAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "BatchForget"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "CreateFile"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateLinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "CreateLink"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "CreateSymlink"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpFlushFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "FlushFile"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpForgetInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "ForgetInode"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpGetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "GetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpLookUpInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "LookUpInode"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpMkDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "MkDir"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpMkNodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "MkNode"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOpenDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "OpenDir"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOpenFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "OpenFile"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOthersAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "Others"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "ReadDir"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadDirPlusAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "ReadDirPlus"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "ReadFile"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "ReadSymlink"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReleaseDirHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "ReleaseDirHandle"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReleaseFileHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "ReleaseFileHandle"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpRenameAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "Rename"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpRmDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "RmDir"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpSetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "SetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpSyncFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "SyncFile"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpUnlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "Unlink"))) + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpWriteFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "WriteFile"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpBatchForgetAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "BatchForget"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "CreateFile"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateLinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "CreateLink"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "CreateSymlink"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpFlushFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "FlushFile"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpForgetInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "ForgetInode"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpGetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "GetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpLookUpInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "LookUpInode"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpMkDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "MkDir"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpMkNodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "MkNode"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOpenDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "OpenDir"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOpenFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "OpenFile"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOthersAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "Others"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "ReadDir"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadDirPlusAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "ReadDirPlus"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "ReadFile"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "ReadSymlink"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReleaseDirHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "ReleaseDirHandle"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReleaseFileHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "ReleaseFileHandle"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpRenameAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "Rename"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpRmDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "RmDir"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpSetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "SetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpSyncFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "SyncFile"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpUnlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "Unlink"))) + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpWriteFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "WriteFile"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpBatchForgetAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "BatchForget"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "CreateFile"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateLinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "CreateLink"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "CreateSymlink"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpFlushFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "FlushFile"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpForgetInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "ForgetInode"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpGetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "GetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpLookUpInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "LookUpInode"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpMkDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "MkDir"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpMkNodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "MkNode"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOpenDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "OpenDir"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOpenFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "OpenFile"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOthersAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "Others"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "ReadDir"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadDirPlusAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "ReadDirPlus"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "ReadFile"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "ReadSymlink"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReleaseDirHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "ReleaseDirHandle"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReleaseFileHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "ReleaseFileHandle"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpRenameAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "Rename"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpRmDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "RmDir"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpSetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "SetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpSyncFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "SyncFile"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpUnlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "Unlink"))) + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpWriteFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "WriteFile"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpBatchForgetAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "BatchForget"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "CreateFile"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateLinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "CreateLink"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "CreateSymlink"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpFlushFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "FlushFile"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpForgetInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "ForgetInode"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpGetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "GetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpLookUpInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "LookUpInode"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpMkDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "MkDir"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpMkNodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "MkNode"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOpenDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "OpenDir"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOpenFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "OpenFile"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOthersAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "Others"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "ReadDir"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadDirPlusAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "ReadDirPlus"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "ReadFile"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "ReadSymlink"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReleaseDirHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "ReleaseDirHandle"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReleaseFileHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "ReleaseFileHandle"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpRenameAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "Rename"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpRmDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "RmDir"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpSetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "SetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpSyncFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "SyncFile"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpUnlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "Unlink"))) + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpWriteFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "WriteFile"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpBatchForgetAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "BatchForget"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "CreateFile"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateLinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "CreateLink"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "CreateSymlink"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpFlushFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "FlushFile"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpForgetInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "ForgetInode"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpGetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "GetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpLookUpInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "LookUpInode"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpMkDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "MkDir"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpMkNodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "MkNode"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOpenDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "OpenDir"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOpenFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "OpenFile"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOthersAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "Others"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "ReadDir"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadDirPlusAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "ReadDirPlus"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "ReadFile"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "ReadSymlink"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReleaseDirHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "ReleaseDirHandle"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReleaseFileHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "ReleaseFileHandle"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpRenameAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "Rename"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpRmDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "RmDir"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpSetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "SetInodeAttributes"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpSyncFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "SyncFile"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpUnlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "Unlink"))) + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpWriteFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "WriteFile"))) + fsOpsLatencyFsOpBatchForgetAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "BatchForget"))) + fsOpsLatencyFsOpCreateFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "CreateFile"))) + fsOpsLatencyFsOpCreateLinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "CreateLink"))) + fsOpsLatencyFsOpCreateSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "CreateSymlink"))) + fsOpsLatencyFsOpFlushFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "FlushFile"))) + fsOpsLatencyFsOpForgetInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "ForgetInode"))) + fsOpsLatencyFsOpGetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "GetInodeAttributes"))) + fsOpsLatencyFsOpLookUpInodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "LookUpInode"))) + fsOpsLatencyFsOpMkDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "MkDir"))) + fsOpsLatencyFsOpMkNodeAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "MkNode"))) + fsOpsLatencyFsOpOpenDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "OpenDir"))) + fsOpsLatencyFsOpOpenFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "OpenFile"))) + fsOpsLatencyFsOpOthersAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "Others"))) + fsOpsLatencyFsOpReadDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "ReadDir"))) + fsOpsLatencyFsOpReadDirPlusAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "ReadDirPlus"))) + fsOpsLatencyFsOpReadFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "ReadFile"))) + fsOpsLatencyFsOpReadSymlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "ReadSymlink"))) + fsOpsLatencyFsOpReleaseDirHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "ReleaseDirHandle"))) + fsOpsLatencyFsOpReleaseFileHandleAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "ReleaseFileHandle"))) + fsOpsLatencyFsOpRenameAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "Rename"))) + fsOpsLatencyFsOpRmDirAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "RmDir"))) + fsOpsLatencyFsOpSetInodeAttributesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "SetInodeAttributes"))) + fsOpsLatencyFsOpSyncFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "SyncFile"))) + fsOpsLatencyFsOpUnlinkAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "Unlink"))) + fsOpsLatencyFsOpWriteFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("fs_op", "WriteFile"))) + fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonConcurrencyLimitBreachedAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("open_mode", "other"), attribute.String("write_fallback_reason", "concurrency_limit_breached"))) + fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonExistingFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("open_mode", "other"), attribute.String("write_fallback_reason", "existing_file"))) + fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonOtherAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("open_mode", "other"), attribute.String("write_fallback_reason", "other"))) + fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonOutOfOrderAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("open_mode", "other"), attribute.String("write_fallback_reason", "out_of_order"))) + fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonConcurrencyLimitBreachedAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("open_mode", "read_write"), attribute.String("write_fallback_reason", "concurrency_limit_breached"))) + fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonExistingFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("open_mode", "read_write"), attribute.String("write_fallback_reason", "existing_file"))) + fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonOtherAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("open_mode", "read_write"), attribute.String("write_fallback_reason", "other"))) + fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonOutOfOrderAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("open_mode", "read_write"), attribute.String("write_fallback_reason", "out_of_order"))) + fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonConcurrencyLimitBreachedAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("open_mode", "read_write_append"), attribute.String("write_fallback_reason", "concurrency_limit_breached"))) + fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonExistingFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("open_mode", "read_write_append"), attribute.String("write_fallback_reason", "existing_file"))) + fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonOtherAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("open_mode", "read_write_append"), attribute.String("write_fallback_reason", "other"))) + fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonOutOfOrderAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("open_mode", "read_write_append"), attribute.String("write_fallback_reason", "out_of_order"))) + fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonConcurrencyLimitBreachedAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("open_mode", "write_only"), attribute.String("write_fallback_reason", "concurrency_limit_breached"))) + fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonExistingFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("open_mode", "write_only"), attribute.String("write_fallback_reason", "existing_file"))) + fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonOtherAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("open_mode", "write_only"), attribute.String("write_fallback_reason", "other"))) + fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonOutOfOrderAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("open_mode", "write_only"), attribute.String("write_fallback_reason", "out_of_order"))) + fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonConcurrencyLimitBreachedAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("open_mode", "write_only_append"), attribute.String("write_fallback_reason", "concurrency_limit_breached"))) + fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonExistingFileAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("open_mode", "write_only_append"), attribute.String("write_fallback_reason", "existing_file"))) + fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonOtherAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("open_mode", "write_only_append"), attribute.String("write_fallback_reason", "other"))) + fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonOutOfOrderAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("open_mode", "write_only_append"), attribute.String("write_fallback_reason", "out_of_order"))) + gcsDownloadBytesCountReadTypeBufferedAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("read_type", "Buffered"))) + gcsDownloadBytesCountReadTypeParallelAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("read_type", "Parallel"))) + gcsDownloadBytesCountReadTypeRandomAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("read_type", "Random"))) + gcsDownloadBytesCountReadTypeSequentialAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("read_type", "Sequential"))) + gcsReadCountReadTypeParallelAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("read_type", "Parallel"))) + gcsReadCountReadTypeRandomAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("read_type", "Random"))) + gcsReadCountReadTypeSequentialAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("read_type", "Sequential"))) + gcsReadCountReadTypeUnknownAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("read_type", "Unknown"))) + gcsReaderCountIoMethodClosedAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("io_method", "closed"))) + gcsReaderCountIoMethodOpenedAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("io_method", "opened"))) + gcsRequestCountGcsMethodComposeObjectsAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "ComposeObjects"))) + gcsRequestCountGcsMethodCopyObjectAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "CopyObject"))) + gcsRequestCountGcsMethodCreateAppendableObjectWriterAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "CreateAppendableObjectWriter"))) + gcsRequestCountGcsMethodCreateFolderAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "CreateFolder"))) + gcsRequestCountGcsMethodCreateObjectAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "CreateObject"))) + gcsRequestCountGcsMethodCreateObjectChunkWriterAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "CreateObjectChunkWriter"))) + gcsRequestCountGcsMethodDeleteFolderAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "DeleteFolder"))) + gcsRequestCountGcsMethodDeleteObjectAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "DeleteObject"))) + gcsRequestCountGcsMethodFinalizeUploadAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "FinalizeUpload"))) + gcsRequestCountGcsMethodFlushPendingWritesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "FlushPendingWrites"))) + gcsRequestCountGcsMethodGetFolderAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "GetFolder"))) + gcsRequestCountGcsMethodListObjectsAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "ListObjects"))) + gcsRequestCountGcsMethodMoveObjectAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "MoveObject"))) + gcsRequestCountGcsMethodMultiRangeDownloaderAddAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "MultiRangeDownloader::Add"))) + gcsRequestCountGcsMethodNewMultiRangeDownloaderAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "NewMultiRangeDownloader"))) + gcsRequestCountGcsMethodNewReaderAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "NewReader"))) + gcsRequestCountGcsMethodRenameFolderAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "RenameFolder"))) + gcsRequestCountGcsMethodStatObjectAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "StatObject"))) + gcsRequestCountGcsMethodUpdateObjectAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "UpdateObject"))) + gcsRequestLatenciesGcsMethodComposeObjectsAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "ComposeObjects"))) + gcsRequestLatenciesGcsMethodCopyObjectAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "CopyObject"))) + gcsRequestLatenciesGcsMethodCreateAppendableObjectWriterAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "CreateAppendableObjectWriter"))) + gcsRequestLatenciesGcsMethodCreateFolderAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "CreateFolder"))) + gcsRequestLatenciesGcsMethodCreateObjectAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "CreateObject"))) + gcsRequestLatenciesGcsMethodCreateObjectChunkWriterAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "CreateObjectChunkWriter"))) + gcsRequestLatenciesGcsMethodDeleteFolderAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "DeleteFolder"))) + gcsRequestLatenciesGcsMethodDeleteObjectAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "DeleteObject"))) + gcsRequestLatenciesGcsMethodFinalizeUploadAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "FinalizeUpload"))) + gcsRequestLatenciesGcsMethodFlushPendingWritesAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "FlushPendingWrites"))) + gcsRequestLatenciesGcsMethodGetFolderAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "GetFolder"))) + gcsRequestLatenciesGcsMethodListObjectsAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "ListObjects"))) + gcsRequestLatenciesGcsMethodMoveObjectAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "MoveObject"))) + gcsRequestLatenciesGcsMethodMultiRangeDownloaderAddAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "MultiRangeDownloader::Add"))) + gcsRequestLatenciesGcsMethodNewMultiRangeDownloaderAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "NewMultiRangeDownloader"))) + gcsRequestLatenciesGcsMethodNewReaderAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "NewReader"))) + gcsRequestLatenciesGcsMethodRenameFolderAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "RenameFolder"))) + gcsRequestLatenciesGcsMethodStatObjectAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "StatObject"))) + gcsRequestLatenciesGcsMethodUpdateObjectAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("gcs_method", "UpdateObject"))) + gcsRetryCountRetryErrorCategoryOTHERERRORSAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("retry_error_category", "OTHER_ERRORS"))) + gcsRetryCountRetryErrorCategorySTALLEDREADREQUESTAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("retry_error_category", "STALLED_READ_REQUEST"))) + metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailFoundAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("entry_status", "negative"), attribute.String("lookup_detail", "found"))) + metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailNotFoundAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("entry_status", "negative"), attribute.String("lookup_detail", "not_found"))) + metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailTtlExpiredAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("entry_status", "negative"), attribute.String("lookup_detail", "ttl_expired"))) + metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailFoundAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("entry_status", "positive"), attribute.String("lookup_detail", "found"))) + metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailNotFoundAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("entry_status", "positive"), attribute.String("lookup_detail", "not_found"))) + metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailTtlExpiredAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("entry_status", "positive"), attribute.String("lookup_detail", "ttl_expired"))) + metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailFoundAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("entry_status", "negative"), attribute.String("lookup_detail", "found"))) + metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailNotFoundAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("entry_status", "negative"), attribute.String("lookup_detail", "not_found"))) + metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailTtlExpiredAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("entry_status", "negative"), attribute.String("lookup_detail", "ttl_expired"))) + metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailFoundAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("entry_status", "positive"), attribute.String("lookup_detail", "found"))) + metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailNotFoundAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("entry_status", "positive"), attribute.String("lookup_detail", "not_found"))) + metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailTtlExpiredAttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("entry_status", "positive"), attribute.String("lookup_detail", "ttl_expired"))) + testUpdownCounterWithAttrsRequestTypeAttr1AttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("request_type", "attr1"))) + testUpdownCounterWithAttrsRequestTypeAttr2AttrSet = metric.WithAttributeSet(attribute.NewSet(attribute.String("request_type", "attr2"))) +) + +type histogramRecord struct { + ctx context.Context + instrument metric.Int64Histogram + value int64 + attributes metric.RecordOption +} + +type otelMetrics struct { + ch chan histogramRecord + wg *sync.WaitGroup + bufferedReadFallbackTriggerCountReasonInsufficientMemoryAtomic *atomic.Int64 + bufferedReadFallbackTriggerCountReasonRandomReadDetectedAtomic *atomic.Int64 + fileCacheReadBytesCountReadTypeParallelAtomic *atomic.Int64 + fileCacheReadBytesCountReadTypeRandomAtomic *atomic.Int64 + fileCacheReadBytesCountReadTypeSequentialAtomic *atomic.Int64 + fileCacheReadBytesCountReadTypeUnknownAtomic *atomic.Int64 + fileCacheReadCountCacheHitTrueReadTypeParallelAtomic *atomic.Int64 + fileCacheReadCountCacheHitTrueReadTypeRandomAtomic *atomic.Int64 + fileCacheReadCountCacheHitTrueReadTypeSequentialAtomic *atomic.Int64 + fileCacheReadCountCacheHitTrueReadTypeUnknownAtomic *atomic.Int64 + fileCacheReadCountCacheHitFalseReadTypeParallelAtomic *atomic.Int64 + fileCacheReadCountCacheHitFalseReadTypeRandomAtomic *atomic.Int64 + fileCacheReadCountCacheHitFalseReadTypeSequentialAtomic *atomic.Int64 + fileCacheReadCountCacheHitFalseReadTypeUnknownAtomic *atomic.Int64 + fsOpsCountFsOpBatchForgetAtomic *atomic.Int64 + fsOpsCountFsOpCreateFileAtomic *atomic.Int64 + fsOpsCountFsOpCreateLinkAtomic *atomic.Int64 + fsOpsCountFsOpCreateSymlinkAtomic *atomic.Int64 + fsOpsCountFsOpFlushFileAtomic *atomic.Int64 + fsOpsCountFsOpForgetInodeAtomic *atomic.Int64 + fsOpsCountFsOpGetInodeAttributesAtomic *atomic.Int64 + fsOpsCountFsOpLookUpInodeAtomic *atomic.Int64 + fsOpsCountFsOpMkDirAtomic *atomic.Int64 + fsOpsCountFsOpMkNodeAtomic *atomic.Int64 + fsOpsCountFsOpOpenDirAtomic *atomic.Int64 + fsOpsCountFsOpOpenFileAtomic *atomic.Int64 + fsOpsCountFsOpOthersAtomic *atomic.Int64 + fsOpsCountFsOpReadDirAtomic *atomic.Int64 + fsOpsCountFsOpReadDirPlusAtomic *atomic.Int64 + fsOpsCountFsOpReadFileAtomic *atomic.Int64 + fsOpsCountFsOpReadSymlinkAtomic *atomic.Int64 + fsOpsCountFsOpReleaseDirHandleAtomic *atomic.Int64 + fsOpsCountFsOpReleaseFileHandleAtomic *atomic.Int64 + fsOpsCountFsOpRenameAtomic *atomic.Int64 + fsOpsCountFsOpRmDirAtomic *atomic.Int64 + fsOpsCountFsOpSetInodeAttributesAtomic *atomic.Int64 + fsOpsCountFsOpSyncFileAtomic *atomic.Int64 + fsOpsCountFsOpUnlinkAtomic *atomic.Int64 + fsOpsCountFsOpWriteFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpBatchForgetAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateLinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpFlushFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpForgetInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpGetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpLookUpInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpMkDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpMkNodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOpenDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOpenFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOthersAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadDirPlusAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReleaseDirHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReleaseFileHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpRenameAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpRmDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpSetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpSyncFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpUnlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpWriteFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpBatchForgetAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateLinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpFlushFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpForgetInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpGetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpLookUpInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpMkDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpMkNodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOpenDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOpenFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOthersAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadDirPlusAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReleaseDirHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReleaseFileHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpRenameAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpRmDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpSetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpSyncFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpUnlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpWriteFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpBatchForgetAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateLinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpFlushFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpForgetInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpGetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpLookUpInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpMkDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpMkNodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOpenDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOpenFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOthersAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadDirPlusAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReleaseDirHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReleaseFileHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpRenameAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpRmDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpSetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpSyncFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpUnlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpWriteFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpBatchForgetAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateLinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpFlushFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpForgetInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpGetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpLookUpInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpMkDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpMkNodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOpenDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOpenFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOthersAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadDirPlusAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReleaseDirHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReleaseFileHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpRenameAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpRmDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpSetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpSyncFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpUnlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpWriteFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpBatchForgetAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateLinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpFlushFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpForgetInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpGetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpLookUpInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpMkDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpMkNodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOpenDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOpenFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOthersAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadDirPlusAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReleaseDirHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReleaseFileHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpRenameAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpRmDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpSetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpSyncFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpUnlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpWriteFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpBatchForgetAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateLinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpFlushFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpForgetInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpGetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpLookUpInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpMkDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpMkNodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOpenDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOpenFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOthersAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadDirPlusAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReleaseDirHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReleaseFileHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpRenameAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpRmDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpSetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpSyncFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpUnlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpWriteFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpBatchForgetAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateLinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpFlushFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpForgetInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpGetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpLookUpInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpMkDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpMkNodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOpenDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOpenFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOthersAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadDirPlusAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReleaseDirHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReleaseFileHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpRenameAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpRmDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpSetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpSyncFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpUnlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpWriteFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpBatchForgetAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateLinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpFlushFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpForgetInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpGetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpLookUpInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpMkDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpMkNodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpOpenDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpOpenFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpOthersAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadDirPlusAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReleaseDirHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReleaseFileHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpRenameAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpRmDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpSetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpSyncFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpUnlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryIOERRORFsOpWriteFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpBatchForgetAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateLinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpFlushFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpForgetInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpGetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpLookUpInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpMkDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpMkNodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOpenDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOpenFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOthersAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadDirPlusAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReleaseDirHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReleaseFileHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpRenameAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpRmDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpSetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpSyncFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpUnlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpWriteFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpBatchForgetAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateLinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpFlushFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpForgetInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpGetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpLookUpInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpMkDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpMkNodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOpenDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOpenFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOthersAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadDirPlusAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReleaseDirHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReleaseFileHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpRenameAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpRmDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpSetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpSyncFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpUnlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpWriteFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpBatchForgetAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateLinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpFlushFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpForgetInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpGetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpLookUpInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpMkDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpMkNodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOpenDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOpenFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOthersAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadDirPlusAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReleaseDirHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReleaseFileHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpRenameAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpRmDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpSetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpSyncFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpUnlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpWriteFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpBatchForgetAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateLinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpFlushFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpForgetInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpGetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpLookUpInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpMkDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpMkNodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOpenDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOpenFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOthersAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadDirPlusAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReleaseDirHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReleaseFileHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpRenameAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpRmDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpSetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpSyncFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpUnlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpWriteFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpBatchForgetAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateLinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpFlushFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpForgetInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpGetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpLookUpInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpMkDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpMkNodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOpenDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOpenFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOthersAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadDirPlusAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReleaseDirHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReleaseFileHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpRenameAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpRmDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpSetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpSyncFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpUnlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpWriteFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpBatchForgetAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateLinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpFlushFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpForgetInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpGetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpLookUpInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpMkDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpMkNodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOpenDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOpenFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOthersAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadDirPlusAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReleaseDirHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReleaseFileHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpRenameAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpRmDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpSetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpSyncFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpUnlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpWriteFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpBatchForgetAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateLinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpFlushFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpForgetInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpGetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpLookUpInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpMkDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpMkNodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOpenDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOpenFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOthersAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadDirPlusAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReleaseDirHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReleaseFileHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpRenameAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpRmDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpSetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpSyncFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpUnlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpWriteFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpBatchForgetAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateLinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpFlushFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpForgetInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpGetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpLookUpInodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpMkDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpMkNodeAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOpenDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOpenFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOthersAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadDirPlusAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadSymlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReleaseDirHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReleaseFileHandleAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpRenameAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpRmDirAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpSetInodeAttributesAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpSyncFileAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpUnlinkAtomic *atomic.Int64 + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpWriteFileAtomic *atomic.Int64 + fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonConcurrencyLimitBreachedAtomic *atomic.Int64 + fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonExistingFileAtomic *atomic.Int64 + fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonOtherAtomic *atomic.Int64 + fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonOutOfOrderAtomic *atomic.Int64 + fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonConcurrencyLimitBreachedAtomic *atomic.Int64 + fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonExistingFileAtomic *atomic.Int64 + fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonOtherAtomic *atomic.Int64 + fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonOutOfOrderAtomic *atomic.Int64 + fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonConcurrencyLimitBreachedAtomic *atomic.Int64 + fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonExistingFileAtomic *atomic.Int64 + fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonOtherAtomic *atomic.Int64 + fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonOutOfOrderAtomic *atomic.Int64 + fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonConcurrencyLimitBreachedAtomic *atomic.Int64 + fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonExistingFileAtomic *atomic.Int64 + fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonOtherAtomic *atomic.Int64 + fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonOutOfOrderAtomic *atomic.Int64 + fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonConcurrencyLimitBreachedAtomic *atomic.Int64 + fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonExistingFileAtomic *atomic.Int64 + fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonOtherAtomic *atomic.Int64 + fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonOutOfOrderAtomic *atomic.Int64 + gcsDownloadBytesCountReadTypeBufferedAtomic *atomic.Int64 + gcsDownloadBytesCountReadTypeParallelAtomic *atomic.Int64 + gcsDownloadBytesCountReadTypeRandomAtomic *atomic.Int64 + gcsDownloadBytesCountReadTypeSequentialAtomic *atomic.Int64 + gcsReadBytesCountAtomic *atomic.Int64 + gcsReadCountReadTypeParallelAtomic *atomic.Int64 + gcsReadCountReadTypeRandomAtomic *atomic.Int64 + gcsReadCountReadTypeSequentialAtomic *atomic.Int64 + gcsReadCountReadTypeUnknownAtomic *atomic.Int64 + gcsReaderCountIoMethodClosedAtomic *atomic.Int64 + gcsReaderCountIoMethodOpenedAtomic *atomic.Int64 + gcsRequestCountGcsMethodComposeObjectsAtomic *atomic.Int64 + gcsRequestCountGcsMethodCopyObjectAtomic *atomic.Int64 + gcsRequestCountGcsMethodCreateAppendableObjectWriterAtomic *atomic.Int64 + gcsRequestCountGcsMethodCreateFolderAtomic *atomic.Int64 + gcsRequestCountGcsMethodCreateObjectAtomic *atomic.Int64 + gcsRequestCountGcsMethodCreateObjectChunkWriterAtomic *atomic.Int64 + gcsRequestCountGcsMethodDeleteFolderAtomic *atomic.Int64 + gcsRequestCountGcsMethodDeleteObjectAtomic *atomic.Int64 + gcsRequestCountGcsMethodFinalizeUploadAtomic *atomic.Int64 + gcsRequestCountGcsMethodFlushPendingWritesAtomic *atomic.Int64 + gcsRequestCountGcsMethodGetFolderAtomic *atomic.Int64 + gcsRequestCountGcsMethodListObjectsAtomic *atomic.Int64 + gcsRequestCountGcsMethodMoveObjectAtomic *atomic.Int64 + gcsRequestCountGcsMethodMultiRangeDownloaderAddAtomic *atomic.Int64 + gcsRequestCountGcsMethodNewMultiRangeDownloaderAtomic *atomic.Int64 + gcsRequestCountGcsMethodNewReaderAtomic *atomic.Int64 + gcsRequestCountGcsMethodRenameFolderAtomic *atomic.Int64 + gcsRequestCountGcsMethodStatObjectAtomic *atomic.Int64 + gcsRequestCountGcsMethodUpdateObjectAtomic *atomic.Int64 + gcsRetryCountRetryErrorCategoryOTHERERRORSAtomic *atomic.Int64 + gcsRetryCountRetryErrorCategorySTALLEDREADREQUESTAtomic *atomic.Int64 + metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailFoundAtomic *atomic.Int64 + metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailNotFoundAtomic *atomic.Int64 + metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailTtlExpiredAtomic *atomic.Int64 + metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailFoundAtomic *atomic.Int64 + metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailNotFoundAtomic *atomic.Int64 + metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailTtlExpiredAtomic *atomic.Int64 + metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailFoundAtomic *atomic.Int64 + metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailNotFoundAtomic *atomic.Int64 + metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailTtlExpiredAtomic *atomic.Int64 + metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailFoundAtomic *atomic.Int64 + metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailNotFoundAtomic *atomic.Int64 + metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailTtlExpiredAtomic *atomic.Int64 + testUpdownCounterAtomic *atomic.Int64 + testUpdownCounterWithAttrsRequestTypeAttr1Atomic *atomic.Int64 + testUpdownCounterWithAttrsRequestTypeAttr2Atomic *atomic.Int64 + bufferedReadReadLatency metric.Int64Histogram + fileCacheReadLatencies metric.Int64Histogram + fsOpsLatency metric.Int64Histogram + gcsRequestLatencies metric.Int64Histogram + readBlockSizes metric.Int64Histogram +} + +func (o *otelMetrics) BufferedReadFallbackTriggerCount( + inc int64, reason Reason) { + if inc < 0 { + logger.Errorf("Counter metric buffered_read/fallback_trigger_count received a negative increment: %d", inc) + return + } + switch reason { + case ReasonInsufficientMemoryAttr: + o.bufferedReadFallbackTriggerCountReasonInsufficientMemoryAtomic.Add(inc) + case ReasonRandomReadDetectedAttr: + o.bufferedReadFallbackTriggerCountReasonRandomReadDetectedAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(reason)) + return + } +} + +func (o *otelMetrics) BufferedReadReadLatency( + ctx context.Context, latency time.Duration) { + record := histogramRecord{ctx: ctx, instrument: o.bufferedReadReadLatency, value: latency.Microseconds()} + + select { + case o.ch <- record: // Do nothing + default: // Unblock writes to channel if it's full. + } +} + +func (o *otelMetrics) FileCacheReadBytesCount( + inc int64, readType ReadType) { + if inc < 0 { + logger.Errorf("Counter metric file_cache/read_bytes_count received a negative increment: %d", inc) + return + } + switch readType { + case ReadTypeParallelAttr: + o.fileCacheReadBytesCountReadTypeParallelAtomic.Add(inc) + case ReadTypeRandomAttr: + o.fileCacheReadBytesCountReadTypeRandomAtomic.Add(inc) + case ReadTypeSequentialAttr: + o.fileCacheReadBytesCountReadTypeSequentialAtomic.Add(inc) + case ReadTypeUnknownAttr: + o.fileCacheReadBytesCountReadTypeUnknownAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(readType)) + return + } +} + +func (o *otelMetrics) FileCacheReadCount( + inc int64, cacheHit bool, readType ReadType) { + if inc < 0 { + logger.Errorf("Counter metric file_cache/read_count received a negative increment: %d", inc) + return + } + switch cacheHit { + case true: + switch readType { + case ReadTypeParallelAttr: + o.fileCacheReadCountCacheHitTrueReadTypeParallelAtomic.Add(inc) + case ReadTypeRandomAttr: + o.fileCacheReadCountCacheHitTrueReadTypeRandomAtomic.Add(inc) + case ReadTypeSequentialAttr: + o.fileCacheReadCountCacheHitTrueReadTypeSequentialAtomic.Add(inc) + case ReadTypeUnknownAttr: + o.fileCacheReadCountCacheHitTrueReadTypeUnknownAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(readType)) + return + } + case false: + switch readType { + case ReadTypeParallelAttr: + o.fileCacheReadCountCacheHitFalseReadTypeParallelAtomic.Add(inc) + case ReadTypeRandomAttr: + o.fileCacheReadCountCacheHitFalseReadTypeRandomAtomic.Add(inc) + case ReadTypeSequentialAttr: + o.fileCacheReadCountCacheHitFalseReadTypeSequentialAtomic.Add(inc) + case ReadTypeUnknownAttr: + o.fileCacheReadCountCacheHitFalseReadTypeUnknownAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(readType)) + return + } + } +} + +func (o *otelMetrics) FileCacheReadLatencies( + ctx context.Context, latency time.Duration, cacheHit bool) { + var record histogramRecord + switch cacheHit { + case true: + record = histogramRecord{ctx: ctx, instrument: o.fileCacheReadLatencies, value: latency.Microseconds(), attributes: fileCacheReadLatenciesCacheHitTrueAttrSet} + case false: + record = histogramRecord{ctx: ctx, instrument: o.fileCacheReadLatencies, value: latency.Microseconds(), attributes: fileCacheReadLatenciesCacheHitFalseAttrSet} + } + + select { + case o.ch <- record: // Do nothing + default: // Unblock writes to channel if it's full. + } +} + +func (o *otelMetrics) FsOpsCount( + inc int64, fsOp FsOp) { + if inc < 0 { + logger.Errorf("Counter metric fs/ops_count received a negative increment: %d", inc) + return + } + switch fsOp { + case FsOpBatchForgetAttr: + o.fsOpsCountFsOpBatchForgetAtomic.Add(inc) + case FsOpCreateFileAttr: + o.fsOpsCountFsOpCreateFileAtomic.Add(inc) + case FsOpCreateLinkAttr: + o.fsOpsCountFsOpCreateLinkAtomic.Add(inc) + case FsOpCreateSymlinkAttr: + o.fsOpsCountFsOpCreateSymlinkAtomic.Add(inc) + case FsOpFlushFileAttr: + o.fsOpsCountFsOpFlushFileAtomic.Add(inc) + case FsOpForgetInodeAttr: + o.fsOpsCountFsOpForgetInodeAtomic.Add(inc) + case FsOpGetInodeAttributesAttr: + o.fsOpsCountFsOpGetInodeAttributesAtomic.Add(inc) + case FsOpLookUpInodeAttr: + o.fsOpsCountFsOpLookUpInodeAtomic.Add(inc) + case FsOpMkDirAttr: + o.fsOpsCountFsOpMkDirAtomic.Add(inc) + case FsOpMkNodeAttr: + o.fsOpsCountFsOpMkNodeAtomic.Add(inc) + case FsOpOpenDirAttr: + o.fsOpsCountFsOpOpenDirAtomic.Add(inc) + case FsOpOpenFileAttr: + o.fsOpsCountFsOpOpenFileAtomic.Add(inc) + case FsOpOthersAttr: + o.fsOpsCountFsOpOthersAtomic.Add(inc) + case FsOpReadDirAttr: + o.fsOpsCountFsOpReadDirAtomic.Add(inc) + case FsOpReadDirPlusAttr: + o.fsOpsCountFsOpReadDirPlusAtomic.Add(inc) + case FsOpReadFileAttr: + o.fsOpsCountFsOpReadFileAtomic.Add(inc) + case FsOpReadSymlinkAttr: + o.fsOpsCountFsOpReadSymlinkAtomic.Add(inc) + case FsOpReleaseDirHandleAttr: + o.fsOpsCountFsOpReleaseDirHandleAtomic.Add(inc) + case FsOpReleaseFileHandleAttr: + o.fsOpsCountFsOpReleaseFileHandleAtomic.Add(inc) + case FsOpRenameAttr: + o.fsOpsCountFsOpRenameAtomic.Add(inc) + case FsOpRmDirAttr: + o.fsOpsCountFsOpRmDirAtomic.Add(inc) + case FsOpSetInodeAttributesAttr: + o.fsOpsCountFsOpSetInodeAttributesAtomic.Add(inc) + case FsOpSyncFileAttr: + o.fsOpsCountFsOpSyncFileAtomic.Add(inc) + case FsOpUnlinkAttr: + o.fsOpsCountFsOpUnlinkAtomic.Add(inc) + case FsOpWriteFileAttr: + o.fsOpsCountFsOpWriteFileAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(fsOp)) + return + } +} + +func (o *otelMetrics) FsOpsErrorCount( + inc int64, fsErrorCategory FsErrorCategory, fsOp FsOp) { + if inc < 0 { + logger.Errorf("Counter metric fs/ops_error_count received a negative increment: %d", inc) + return + } + switch fsErrorCategory { + case FsErrorCategoryDEVICEERRORAttr: + switch fsOp { + case FsOpBatchForgetAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpBatchForgetAtomic.Add(inc) + case FsOpCreateFileAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateFileAtomic.Add(inc) + case FsOpCreateLinkAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateLinkAtomic.Add(inc) + case FsOpCreateSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateSymlinkAtomic.Add(inc) + case FsOpFlushFileAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpFlushFileAtomic.Add(inc) + case FsOpForgetInodeAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpForgetInodeAtomic.Add(inc) + case FsOpGetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpGetInodeAttributesAtomic.Add(inc) + case FsOpLookUpInodeAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpLookUpInodeAtomic.Add(inc) + case FsOpMkDirAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpMkDirAtomic.Add(inc) + case FsOpMkNodeAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpMkNodeAtomic.Add(inc) + case FsOpOpenDirAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOpenDirAtomic.Add(inc) + case FsOpOpenFileAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOpenFileAtomic.Add(inc) + case FsOpOthersAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOthersAtomic.Add(inc) + case FsOpReadDirAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadDirAtomic.Add(inc) + case FsOpReadDirPlusAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadDirPlusAtomic.Add(inc) + case FsOpReadFileAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadFileAtomic.Add(inc) + case FsOpReadSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadSymlinkAtomic.Add(inc) + case FsOpReleaseDirHandleAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReleaseDirHandleAtomic.Add(inc) + case FsOpReleaseFileHandleAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReleaseFileHandleAtomic.Add(inc) + case FsOpRenameAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpRenameAtomic.Add(inc) + case FsOpRmDirAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpRmDirAtomic.Add(inc) + case FsOpSetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpSetInodeAttributesAtomic.Add(inc) + case FsOpSyncFileAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpSyncFileAtomic.Add(inc) + case FsOpUnlinkAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpUnlinkAtomic.Add(inc) + case FsOpWriteFileAttr: + o.fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpWriteFileAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(fsOp)) + return + } + case FsErrorCategoryDIRNOTEMPTYAttr: + switch fsOp { + case FsOpBatchForgetAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpBatchForgetAtomic.Add(inc) + case FsOpCreateFileAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateFileAtomic.Add(inc) + case FsOpCreateLinkAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateLinkAtomic.Add(inc) + case FsOpCreateSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateSymlinkAtomic.Add(inc) + case FsOpFlushFileAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpFlushFileAtomic.Add(inc) + case FsOpForgetInodeAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpForgetInodeAtomic.Add(inc) + case FsOpGetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpGetInodeAttributesAtomic.Add(inc) + case FsOpLookUpInodeAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpLookUpInodeAtomic.Add(inc) + case FsOpMkDirAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpMkDirAtomic.Add(inc) + case FsOpMkNodeAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpMkNodeAtomic.Add(inc) + case FsOpOpenDirAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOpenDirAtomic.Add(inc) + case FsOpOpenFileAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOpenFileAtomic.Add(inc) + case FsOpOthersAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOthersAtomic.Add(inc) + case FsOpReadDirAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadDirAtomic.Add(inc) + case FsOpReadDirPlusAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadDirPlusAtomic.Add(inc) + case FsOpReadFileAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadFileAtomic.Add(inc) + case FsOpReadSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadSymlinkAtomic.Add(inc) + case FsOpReleaseDirHandleAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReleaseDirHandleAtomic.Add(inc) + case FsOpReleaseFileHandleAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReleaseFileHandleAtomic.Add(inc) + case FsOpRenameAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpRenameAtomic.Add(inc) + case FsOpRmDirAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpRmDirAtomic.Add(inc) + case FsOpSetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpSetInodeAttributesAtomic.Add(inc) + case FsOpSyncFileAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpSyncFileAtomic.Add(inc) + case FsOpUnlinkAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpUnlinkAtomic.Add(inc) + case FsOpWriteFileAttr: + o.fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpWriteFileAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(fsOp)) + return + } + case FsErrorCategoryFILEDIRERRORAttr: + switch fsOp { + case FsOpBatchForgetAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpBatchForgetAtomic.Add(inc) + case FsOpCreateFileAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateFileAtomic.Add(inc) + case FsOpCreateLinkAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateLinkAtomic.Add(inc) + case FsOpCreateSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateSymlinkAtomic.Add(inc) + case FsOpFlushFileAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpFlushFileAtomic.Add(inc) + case FsOpForgetInodeAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpForgetInodeAtomic.Add(inc) + case FsOpGetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpGetInodeAttributesAtomic.Add(inc) + case FsOpLookUpInodeAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpLookUpInodeAtomic.Add(inc) + case FsOpMkDirAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpMkDirAtomic.Add(inc) + case FsOpMkNodeAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpMkNodeAtomic.Add(inc) + case FsOpOpenDirAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOpenDirAtomic.Add(inc) + case FsOpOpenFileAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOpenFileAtomic.Add(inc) + case FsOpOthersAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOthersAtomic.Add(inc) + case FsOpReadDirAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadDirAtomic.Add(inc) + case FsOpReadDirPlusAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadDirPlusAtomic.Add(inc) + case FsOpReadFileAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadFileAtomic.Add(inc) + case FsOpReadSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadSymlinkAtomic.Add(inc) + case FsOpReleaseDirHandleAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReleaseDirHandleAtomic.Add(inc) + case FsOpReleaseFileHandleAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReleaseFileHandleAtomic.Add(inc) + case FsOpRenameAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpRenameAtomic.Add(inc) + case FsOpRmDirAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpRmDirAtomic.Add(inc) + case FsOpSetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpSetInodeAttributesAtomic.Add(inc) + case FsOpSyncFileAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpSyncFileAtomic.Add(inc) + case FsOpUnlinkAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpUnlinkAtomic.Add(inc) + case FsOpWriteFileAttr: + o.fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpWriteFileAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(fsOp)) + return + } + case FsErrorCategoryFILEEXISTSAttr: + switch fsOp { + case FsOpBatchForgetAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpBatchForgetAtomic.Add(inc) + case FsOpCreateFileAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateFileAtomic.Add(inc) + case FsOpCreateLinkAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateLinkAtomic.Add(inc) + case FsOpCreateSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateSymlinkAtomic.Add(inc) + case FsOpFlushFileAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpFlushFileAtomic.Add(inc) + case FsOpForgetInodeAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpForgetInodeAtomic.Add(inc) + case FsOpGetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpGetInodeAttributesAtomic.Add(inc) + case FsOpLookUpInodeAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpLookUpInodeAtomic.Add(inc) + case FsOpMkDirAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpMkDirAtomic.Add(inc) + case FsOpMkNodeAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpMkNodeAtomic.Add(inc) + case FsOpOpenDirAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOpenDirAtomic.Add(inc) + case FsOpOpenFileAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOpenFileAtomic.Add(inc) + case FsOpOthersAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOthersAtomic.Add(inc) + case FsOpReadDirAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadDirAtomic.Add(inc) + case FsOpReadDirPlusAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadDirPlusAtomic.Add(inc) + case FsOpReadFileAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadFileAtomic.Add(inc) + case FsOpReadSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadSymlinkAtomic.Add(inc) + case FsOpReleaseDirHandleAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReleaseDirHandleAtomic.Add(inc) + case FsOpReleaseFileHandleAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReleaseFileHandleAtomic.Add(inc) + case FsOpRenameAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpRenameAtomic.Add(inc) + case FsOpRmDirAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpRmDirAtomic.Add(inc) + case FsOpSetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpSetInodeAttributesAtomic.Add(inc) + case FsOpSyncFileAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpSyncFileAtomic.Add(inc) + case FsOpUnlinkAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpUnlinkAtomic.Add(inc) + case FsOpWriteFileAttr: + o.fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpWriteFileAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(fsOp)) + return + } + case FsErrorCategoryINTERRUPTERRORAttr: + switch fsOp { + case FsOpBatchForgetAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpBatchForgetAtomic.Add(inc) + case FsOpCreateFileAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateFileAtomic.Add(inc) + case FsOpCreateLinkAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateLinkAtomic.Add(inc) + case FsOpCreateSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateSymlinkAtomic.Add(inc) + case FsOpFlushFileAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpFlushFileAtomic.Add(inc) + case FsOpForgetInodeAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpForgetInodeAtomic.Add(inc) + case FsOpGetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpGetInodeAttributesAtomic.Add(inc) + case FsOpLookUpInodeAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpLookUpInodeAtomic.Add(inc) + case FsOpMkDirAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpMkDirAtomic.Add(inc) + case FsOpMkNodeAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpMkNodeAtomic.Add(inc) + case FsOpOpenDirAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOpenDirAtomic.Add(inc) + case FsOpOpenFileAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOpenFileAtomic.Add(inc) + case FsOpOthersAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOthersAtomic.Add(inc) + case FsOpReadDirAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadDirAtomic.Add(inc) + case FsOpReadDirPlusAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadDirPlusAtomic.Add(inc) + case FsOpReadFileAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadFileAtomic.Add(inc) + case FsOpReadSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadSymlinkAtomic.Add(inc) + case FsOpReleaseDirHandleAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReleaseDirHandleAtomic.Add(inc) + case FsOpReleaseFileHandleAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReleaseFileHandleAtomic.Add(inc) + case FsOpRenameAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpRenameAtomic.Add(inc) + case FsOpRmDirAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpRmDirAtomic.Add(inc) + case FsOpSetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpSetInodeAttributesAtomic.Add(inc) + case FsOpSyncFileAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpSyncFileAtomic.Add(inc) + case FsOpUnlinkAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpUnlinkAtomic.Add(inc) + case FsOpWriteFileAttr: + o.fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpWriteFileAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(fsOp)) + return + } + case FsErrorCategoryINVALIDARGUMENTAttr: + switch fsOp { + case FsOpBatchForgetAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpBatchForgetAtomic.Add(inc) + case FsOpCreateFileAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateFileAtomic.Add(inc) + case FsOpCreateLinkAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateLinkAtomic.Add(inc) + case FsOpCreateSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateSymlinkAtomic.Add(inc) + case FsOpFlushFileAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpFlushFileAtomic.Add(inc) + case FsOpForgetInodeAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpForgetInodeAtomic.Add(inc) + case FsOpGetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpGetInodeAttributesAtomic.Add(inc) + case FsOpLookUpInodeAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpLookUpInodeAtomic.Add(inc) + case FsOpMkDirAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpMkDirAtomic.Add(inc) + case FsOpMkNodeAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpMkNodeAtomic.Add(inc) + case FsOpOpenDirAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOpenDirAtomic.Add(inc) + case FsOpOpenFileAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOpenFileAtomic.Add(inc) + case FsOpOthersAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOthersAtomic.Add(inc) + case FsOpReadDirAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadDirAtomic.Add(inc) + case FsOpReadDirPlusAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadDirPlusAtomic.Add(inc) + case FsOpReadFileAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadFileAtomic.Add(inc) + case FsOpReadSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadSymlinkAtomic.Add(inc) + case FsOpReleaseDirHandleAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReleaseDirHandleAtomic.Add(inc) + case FsOpReleaseFileHandleAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReleaseFileHandleAtomic.Add(inc) + case FsOpRenameAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpRenameAtomic.Add(inc) + case FsOpRmDirAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpRmDirAtomic.Add(inc) + case FsOpSetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpSetInodeAttributesAtomic.Add(inc) + case FsOpSyncFileAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpSyncFileAtomic.Add(inc) + case FsOpUnlinkAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpUnlinkAtomic.Add(inc) + case FsOpWriteFileAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpWriteFileAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(fsOp)) + return + } + case FsErrorCategoryINVALIDOPERATIONAttr: + switch fsOp { + case FsOpBatchForgetAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpBatchForgetAtomic.Add(inc) + case FsOpCreateFileAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateFileAtomic.Add(inc) + case FsOpCreateLinkAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateLinkAtomic.Add(inc) + case FsOpCreateSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateSymlinkAtomic.Add(inc) + case FsOpFlushFileAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpFlushFileAtomic.Add(inc) + case FsOpForgetInodeAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpForgetInodeAtomic.Add(inc) + case FsOpGetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpGetInodeAttributesAtomic.Add(inc) + case FsOpLookUpInodeAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpLookUpInodeAtomic.Add(inc) + case FsOpMkDirAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpMkDirAtomic.Add(inc) + case FsOpMkNodeAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpMkNodeAtomic.Add(inc) + case FsOpOpenDirAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOpenDirAtomic.Add(inc) + case FsOpOpenFileAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOpenFileAtomic.Add(inc) + case FsOpOthersAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOthersAtomic.Add(inc) + case FsOpReadDirAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadDirAtomic.Add(inc) + case FsOpReadDirPlusAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadDirPlusAtomic.Add(inc) + case FsOpReadFileAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadFileAtomic.Add(inc) + case FsOpReadSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadSymlinkAtomic.Add(inc) + case FsOpReleaseDirHandleAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReleaseDirHandleAtomic.Add(inc) + case FsOpReleaseFileHandleAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReleaseFileHandleAtomic.Add(inc) + case FsOpRenameAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpRenameAtomic.Add(inc) + case FsOpRmDirAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpRmDirAtomic.Add(inc) + case FsOpSetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpSetInodeAttributesAtomic.Add(inc) + case FsOpSyncFileAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpSyncFileAtomic.Add(inc) + case FsOpUnlinkAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpUnlinkAtomic.Add(inc) + case FsOpWriteFileAttr: + o.fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpWriteFileAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(fsOp)) + return + } + case FsErrorCategoryIOERRORAttr: + switch fsOp { + case FsOpBatchForgetAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpBatchForgetAtomic.Add(inc) + case FsOpCreateFileAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateFileAtomic.Add(inc) + case FsOpCreateLinkAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateLinkAtomic.Add(inc) + case FsOpCreateSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateSymlinkAtomic.Add(inc) + case FsOpFlushFileAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpFlushFileAtomic.Add(inc) + case FsOpForgetInodeAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpForgetInodeAtomic.Add(inc) + case FsOpGetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpGetInodeAttributesAtomic.Add(inc) + case FsOpLookUpInodeAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpLookUpInodeAtomic.Add(inc) + case FsOpMkDirAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpMkDirAtomic.Add(inc) + case FsOpMkNodeAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpMkNodeAtomic.Add(inc) + case FsOpOpenDirAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpOpenDirAtomic.Add(inc) + case FsOpOpenFileAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpOpenFileAtomic.Add(inc) + case FsOpOthersAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpOthersAtomic.Add(inc) + case FsOpReadDirAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadDirAtomic.Add(inc) + case FsOpReadDirPlusAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadDirPlusAtomic.Add(inc) + case FsOpReadFileAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadFileAtomic.Add(inc) + case FsOpReadSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadSymlinkAtomic.Add(inc) + case FsOpReleaseDirHandleAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpReleaseDirHandleAtomic.Add(inc) + case FsOpReleaseFileHandleAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpReleaseFileHandleAtomic.Add(inc) + case FsOpRenameAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpRenameAtomic.Add(inc) + case FsOpRmDirAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpRmDirAtomic.Add(inc) + case FsOpSetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpSetInodeAttributesAtomic.Add(inc) + case FsOpSyncFileAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpSyncFileAtomic.Add(inc) + case FsOpUnlinkAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpUnlinkAtomic.Add(inc) + case FsOpWriteFileAttr: + o.fsOpsErrorCountFsErrorCategoryIOERRORFsOpWriteFileAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(fsOp)) + return + } + case FsErrorCategoryMISCERRORAttr: + switch fsOp { + case FsOpBatchForgetAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpBatchForgetAtomic.Add(inc) + case FsOpCreateFileAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateFileAtomic.Add(inc) + case FsOpCreateLinkAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateLinkAtomic.Add(inc) + case FsOpCreateSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateSymlinkAtomic.Add(inc) + case FsOpFlushFileAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpFlushFileAtomic.Add(inc) + case FsOpForgetInodeAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpForgetInodeAtomic.Add(inc) + case FsOpGetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpGetInodeAttributesAtomic.Add(inc) + case FsOpLookUpInodeAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpLookUpInodeAtomic.Add(inc) + case FsOpMkDirAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpMkDirAtomic.Add(inc) + case FsOpMkNodeAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpMkNodeAtomic.Add(inc) + case FsOpOpenDirAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOpenDirAtomic.Add(inc) + case FsOpOpenFileAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOpenFileAtomic.Add(inc) + case FsOpOthersAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOthersAtomic.Add(inc) + case FsOpReadDirAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadDirAtomic.Add(inc) + case FsOpReadDirPlusAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadDirPlusAtomic.Add(inc) + case FsOpReadFileAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadFileAtomic.Add(inc) + case FsOpReadSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadSymlinkAtomic.Add(inc) + case FsOpReleaseDirHandleAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReleaseDirHandleAtomic.Add(inc) + case FsOpReleaseFileHandleAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReleaseFileHandleAtomic.Add(inc) + case FsOpRenameAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpRenameAtomic.Add(inc) + case FsOpRmDirAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpRmDirAtomic.Add(inc) + case FsOpSetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpSetInodeAttributesAtomic.Add(inc) + case FsOpSyncFileAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpSyncFileAtomic.Add(inc) + case FsOpUnlinkAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpUnlinkAtomic.Add(inc) + case FsOpWriteFileAttr: + o.fsOpsErrorCountFsErrorCategoryMISCERRORFsOpWriteFileAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(fsOp)) + return + } + case FsErrorCategoryNETWORKERRORAttr: + switch fsOp { + case FsOpBatchForgetAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpBatchForgetAtomic.Add(inc) + case FsOpCreateFileAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateFileAtomic.Add(inc) + case FsOpCreateLinkAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateLinkAtomic.Add(inc) + case FsOpCreateSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateSymlinkAtomic.Add(inc) + case FsOpFlushFileAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpFlushFileAtomic.Add(inc) + case FsOpForgetInodeAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpForgetInodeAtomic.Add(inc) + case FsOpGetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpGetInodeAttributesAtomic.Add(inc) + case FsOpLookUpInodeAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpLookUpInodeAtomic.Add(inc) + case FsOpMkDirAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpMkDirAtomic.Add(inc) + case FsOpMkNodeAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpMkNodeAtomic.Add(inc) + case FsOpOpenDirAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOpenDirAtomic.Add(inc) + case FsOpOpenFileAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOpenFileAtomic.Add(inc) + case FsOpOthersAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOthersAtomic.Add(inc) + case FsOpReadDirAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadDirAtomic.Add(inc) + case FsOpReadDirPlusAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadDirPlusAtomic.Add(inc) + case FsOpReadFileAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadFileAtomic.Add(inc) + case FsOpReadSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadSymlinkAtomic.Add(inc) + case FsOpReleaseDirHandleAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReleaseDirHandleAtomic.Add(inc) + case FsOpReleaseFileHandleAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReleaseFileHandleAtomic.Add(inc) + case FsOpRenameAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpRenameAtomic.Add(inc) + case FsOpRmDirAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpRmDirAtomic.Add(inc) + case FsOpSetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpSetInodeAttributesAtomic.Add(inc) + case FsOpSyncFileAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpSyncFileAtomic.Add(inc) + case FsOpUnlinkAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpUnlinkAtomic.Add(inc) + case FsOpWriteFileAttr: + o.fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpWriteFileAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(fsOp)) + return + } + case FsErrorCategoryNOTADIRAttr: + switch fsOp { + case FsOpBatchForgetAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpBatchForgetAtomic.Add(inc) + case FsOpCreateFileAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateFileAtomic.Add(inc) + case FsOpCreateLinkAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateLinkAtomic.Add(inc) + case FsOpCreateSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateSymlinkAtomic.Add(inc) + case FsOpFlushFileAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpFlushFileAtomic.Add(inc) + case FsOpForgetInodeAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpForgetInodeAtomic.Add(inc) + case FsOpGetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpGetInodeAttributesAtomic.Add(inc) + case FsOpLookUpInodeAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpLookUpInodeAtomic.Add(inc) + case FsOpMkDirAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpMkDirAtomic.Add(inc) + case FsOpMkNodeAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpMkNodeAtomic.Add(inc) + case FsOpOpenDirAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOpenDirAtomic.Add(inc) + case FsOpOpenFileAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOpenFileAtomic.Add(inc) + case FsOpOthersAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOthersAtomic.Add(inc) + case FsOpReadDirAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadDirAtomic.Add(inc) + case FsOpReadDirPlusAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadDirPlusAtomic.Add(inc) + case FsOpReadFileAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadFileAtomic.Add(inc) + case FsOpReadSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadSymlinkAtomic.Add(inc) + case FsOpReleaseDirHandleAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReleaseDirHandleAtomic.Add(inc) + case FsOpReleaseFileHandleAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReleaseFileHandleAtomic.Add(inc) + case FsOpRenameAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpRenameAtomic.Add(inc) + case FsOpRmDirAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpRmDirAtomic.Add(inc) + case FsOpSetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpSetInodeAttributesAtomic.Add(inc) + case FsOpSyncFileAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpSyncFileAtomic.Add(inc) + case FsOpUnlinkAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpUnlinkAtomic.Add(inc) + case FsOpWriteFileAttr: + o.fsOpsErrorCountFsErrorCategoryNOTADIRFsOpWriteFileAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(fsOp)) + return + } + case FsErrorCategoryNOTIMPLEMENTEDAttr: + switch fsOp { + case FsOpBatchForgetAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpBatchForgetAtomic.Add(inc) + case FsOpCreateFileAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateFileAtomic.Add(inc) + case FsOpCreateLinkAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateLinkAtomic.Add(inc) + case FsOpCreateSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateSymlinkAtomic.Add(inc) + case FsOpFlushFileAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpFlushFileAtomic.Add(inc) + case FsOpForgetInodeAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpForgetInodeAtomic.Add(inc) + case FsOpGetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpGetInodeAttributesAtomic.Add(inc) + case FsOpLookUpInodeAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpLookUpInodeAtomic.Add(inc) + case FsOpMkDirAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpMkDirAtomic.Add(inc) + case FsOpMkNodeAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpMkNodeAtomic.Add(inc) + case FsOpOpenDirAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOpenDirAtomic.Add(inc) + case FsOpOpenFileAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOpenFileAtomic.Add(inc) + case FsOpOthersAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOthersAtomic.Add(inc) + case FsOpReadDirAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadDirAtomic.Add(inc) + case FsOpReadDirPlusAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadDirPlusAtomic.Add(inc) + case FsOpReadFileAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadFileAtomic.Add(inc) + case FsOpReadSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadSymlinkAtomic.Add(inc) + case FsOpReleaseDirHandleAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReleaseDirHandleAtomic.Add(inc) + case FsOpReleaseFileHandleAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReleaseFileHandleAtomic.Add(inc) + case FsOpRenameAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpRenameAtomic.Add(inc) + case FsOpRmDirAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpRmDirAtomic.Add(inc) + case FsOpSetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpSetInodeAttributesAtomic.Add(inc) + case FsOpSyncFileAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpSyncFileAtomic.Add(inc) + case FsOpUnlinkAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpUnlinkAtomic.Add(inc) + case FsOpWriteFileAttr: + o.fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpWriteFileAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(fsOp)) + return + } + case FsErrorCategoryNOFILEORDIRAttr: + switch fsOp { + case FsOpBatchForgetAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpBatchForgetAtomic.Add(inc) + case FsOpCreateFileAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateFileAtomic.Add(inc) + case FsOpCreateLinkAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateLinkAtomic.Add(inc) + case FsOpCreateSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateSymlinkAtomic.Add(inc) + case FsOpFlushFileAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpFlushFileAtomic.Add(inc) + case FsOpForgetInodeAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpForgetInodeAtomic.Add(inc) + case FsOpGetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpGetInodeAttributesAtomic.Add(inc) + case FsOpLookUpInodeAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpLookUpInodeAtomic.Add(inc) + case FsOpMkDirAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpMkDirAtomic.Add(inc) + case FsOpMkNodeAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpMkNodeAtomic.Add(inc) + case FsOpOpenDirAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOpenDirAtomic.Add(inc) + case FsOpOpenFileAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOpenFileAtomic.Add(inc) + case FsOpOthersAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOthersAtomic.Add(inc) + case FsOpReadDirAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadDirAtomic.Add(inc) + case FsOpReadDirPlusAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadDirPlusAtomic.Add(inc) + case FsOpReadFileAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadFileAtomic.Add(inc) + case FsOpReadSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadSymlinkAtomic.Add(inc) + case FsOpReleaseDirHandleAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReleaseDirHandleAtomic.Add(inc) + case FsOpReleaseFileHandleAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReleaseFileHandleAtomic.Add(inc) + case FsOpRenameAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpRenameAtomic.Add(inc) + case FsOpRmDirAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpRmDirAtomic.Add(inc) + case FsOpSetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpSetInodeAttributesAtomic.Add(inc) + case FsOpSyncFileAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpSyncFileAtomic.Add(inc) + case FsOpUnlinkAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpUnlinkAtomic.Add(inc) + case FsOpWriteFileAttr: + o.fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpWriteFileAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(fsOp)) + return + } + case FsErrorCategoryPERMERRORAttr: + switch fsOp { + case FsOpBatchForgetAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpBatchForgetAtomic.Add(inc) + case FsOpCreateFileAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateFileAtomic.Add(inc) + case FsOpCreateLinkAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateLinkAtomic.Add(inc) + case FsOpCreateSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateSymlinkAtomic.Add(inc) + case FsOpFlushFileAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpFlushFileAtomic.Add(inc) + case FsOpForgetInodeAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpForgetInodeAtomic.Add(inc) + case FsOpGetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpGetInodeAttributesAtomic.Add(inc) + case FsOpLookUpInodeAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpLookUpInodeAtomic.Add(inc) + case FsOpMkDirAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpMkDirAtomic.Add(inc) + case FsOpMkNodeAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpMkNodeAtomic.Add(inc) + case FsOpOpenDirAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOpenDirAtomic.Add(inc) + case FsOpOpenFileAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOpenFileAtomic.Add(inc) + case FsOpOthersAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOthersAtomic.Add(inc) + case FsOpReadDirAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadDirAtomic.Add(inc) + case FsOpReadDirPlusAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadDirPlusAtomic.Add(inc) + case FsOpReadFileAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadFileAtomic.Add(inc) + case FsOpReadSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadSymlinkAtomic.Add(inc) + case FsOpReleaseDirHandleAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReleaseDirHandleAtomic.Add(inc) + case FsOpReleaseFileHandleAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReleaseFileHandleAtomic.Add(inc) + case FsOpRenameAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpRenameAtomic.Add(inc) + case FsOpRmDirAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpRmDirAtomic.Add(inc) + case FsOpSetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpSetInodeAttributesAtomic.Add(inc) + case FsOpSyncFileAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpSyncFileAtomic.Add(inc) + case FsOpUnlinkAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpUnlinkAtomic.Add(inc) + case FsOpWriteFileAttr: + o.fsOpsErrorCountFsErrorCategoryPERMERRORFsOpWriteFileAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(fsOp)) + return + } + case FsErrorCategoryPROCESSRESOURCEMGMTERRORAttr: + switch fsOp { + case FsOpBatchForgetAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpBatchForgetAtomic.Add(inc) + case FsOpCreateFileAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateFileAtomic.Add(inc) + case FsOpCreateLinkAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateLinkAtomic.Add(inc) + case FsOpCreateSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateSymlinkAtomic.Add(inc) + case FsOpFlushFileAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpFlushFileAtomic.Add(inc) + case FsOpForgetInodeAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpForgetInodeAtomic.Add(inc) + case FsOpGetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpGetInodeAttributesAtomic.Add(inc) + case FsOpLookUpInodeAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpLookUpInodeAtomic.Add(inc) + case FsOpMkDirAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpMkDirAtomic.Add(inc) + case FsOpMkNodeAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpMkNodeAtomic.Add(inc) + case FsOpOpenDirAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOpenDirAtomic.Add(inc) + case FsOpOpenFileAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOpenFileAtomic.Add(inc) + case FsOpOthersAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOthersAtomic.Add(inc) + case FsOpReadDirAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadDirAtomic.Add(inc) + case FsOpReadDirPlusAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadDirPlusAtomic.Add(inc) + case FsOpReadFileAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadFileAtomic.Add(inc) + case FsOpReadSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadSymlinkAtomic.Add(inc) + case FsOpReleaseDirHandleAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReleaseDirHandleAtomic.Add(inc) + case FsOpReleaseFileHandleAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReleaseFileHandleAtomic.Add(inc) + case FsOpRenameAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpRenameAtomic.Add(inc) + case FsOpRmDirAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpRmDirAtomic.Add(inc) + case FsOpSetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpSetInodeAttributesAtomic.Add(inc) + case FsOpSyncFileAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpSyncFileAtomic.Add(inc) + case FsOpUnlinkAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpUnlinkAtomic.Add(inc) + case FsOpWriteFileAttr: + o.fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpWriteFileAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(fsOp)) + return + } + case FsErrorCategoryTOOMANYOPENFILESAttr: + switch fsOp { + case FsOpBatchForgetAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpBatchForgetAtomic.Add(inc) + case FsOpCreateFileAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateFileAtomic.Add(inc) + case FsOpCreateLinkAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateLinkAtomic.Add(inc) + case FsOpCreateSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateSymlinkAtomic.Add(inc) + case FsOpFlushFileAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpFlushFileAtomic.Add(inc) + case FsOpForgetInodeAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpForgetInodeAtomic.Add(inc) + case FsOpGetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpGetInodeAttributesAtomic.Add(inc) + case FsOpLookUpInodeAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpLookUpInodeAtomic.Add(inc) + case FsOpMkDirAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpMkDirAtomic.Add(inc) + case FsOpMkNodeAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpMkNodeAtomic.Add(inc) + case FsOpOpenDirAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOpenDirAtomic.Add(inc) + case FsOpOpenFileAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOpenFileAtomic.Add(inc) + case FsOpOthersAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOthersAtomic.Add(inc) + case FsOpReadDirAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadDirAtomic.Add(inc) + case FsOpReadDirPlusAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadDirPlusAtomic.Add(inc) + case FsOpReadFileAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadFileAtomic.Add(inc) + case FsOpReadSymlinkAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadSymlinkAtomic.Add(inc) + case FsOpReleaseDirHandleAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReleaseDirHandleAtomic.Add(inc) + case FsOpReleaseFileHandleAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReleaseFileHandleAtomic.Add(inc) + case FsOpRenameAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpRenameAtomic.Add(inc) + case FsOpRmDirAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpRmDirAtomic.Add(inc) + case FsOpSetInodeAttributesAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpSetInodeAttributesAtomic.Add(inc) + case FsOpSyncFileAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpSyncFileAtomic.Add(inc) + case FsOpUnlinkAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpUnlinkAtomic.Add(inc) + case FsOpWriteFileAttr: + o.fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpWriteFileAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(fsOp)) + return + } + default: + updateUnrecognizedAttribute(string(fsErrorCategory)) + return + } +} + +func (o *otelMetrics) FsOpsLatency( + ctx context.Context, latency time.Duration, fsOp FsOp) { + var record histogramRecord + switch fsOp { + case FsOpBatchForgetAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpBatchForgetAttrSet} + case FsOpCreateFileAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpCreateFileAttrSet} + case FsOpCreateLinkAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpCreateLinkAttrSet} + case FsOpCreateSymlinkAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpCreateSymlinkAttrSet} + case FsOpFlushFileAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpFlushFileAttrSet} + case FsOpForgetInodeAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpForgetInodeAttrSet} + case FsOpGetInodeAttributesAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpGetInodeAttributesAttrSet} + case FsOpLookUpInodeAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpLookUpInodeAttrSet} + case FsOpMkDirAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpMkDirAttrSet} + case FsOpMkNodeAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpMkNodeAttrSet} + case FsOpOpenDirAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpOpenDirAttrSet} + case FsOpOpenFileAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpOpenFileAttrSet} + case FsOpOthersAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpOthersAttrSet} + case FsOpReadDirAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpReadDirAttrSet} + case FsOpReadDirPlusAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpReadDirPlusAttrSet} + case FsOpReadFileAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpReadFileAttrSet} + case FsOpReadSymlinkAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpReadSymlinkAttrSet} + case FsOpReleaseDirHandleAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpReleaseDirHandleAttrSet} + case FsOpReleaseFileHandleAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpReleaseFileHandleAttrSet} + case FsOpRenameAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpRenameAttrSet} + case FsOpRmDirAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpRmDirAttrSet} + case FsOpSetInodeAttributesAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpSetInodeAttributesAttrSet} + case FsOpSyncFileAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpSyncFileAttrSet} + case FsOpUnlinkAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpUnlinkAttrSet} + case FsOpWriteFileAttr: + record = histogramRecord{ctx: ctx, instrument: o.fsOpsLatency, value: latency.Microseconds(), attributes: fsOpsLatencyFsOpWriteFileAttrSet} + default: + updateUnrecognizedAttribute(string(fsOp)) + return + } + + select { + case o.ch <- record: // Do nothing + default: // Unblock writes to channel if it's full. + } +} + +func (o *otelMetrics) FsStreamingWriteFallbackCount( + inc int64, openMode OpenMode, writeFallbackReason WriteFallbackReason) { + if inc < 0 { + logger.Errorf("Counter metric fs/streaming_write_fallback_count received a negative increment: %d", inc) + return + } + switch openMode { + case OpenModeOtherAttr: + switch writeFallbackReason { + case WriteFallbackReasonConcurrencyLimitBreachedAttr: + o.fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonConcurrencyLimitBreachedAtomic.Add(inc) + case WriteFallbackReasonExistingFileAttr: + o.fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonExistingFileAtomic.Add(inc) + case WriteFallbackReasonOtherAttr: + o.fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonOtherAtomic.Add(inc) + case WriteFallbackReasonOutOfOrderAttr: + o.fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonOutOfOrderAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(writeFallbackReason)) + return + } + case OpenModeReadWriteAttr: + switch writeFallbackReason { + case WriteFallbackReasonConcurrencyLimitBreachedAttr: + o.fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonConcurrencyLimitBreachedAtomic.Add(inc) + case WriteFallbackReasonExistingFileAttr: + o.fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonExistingFileAtomic.Add(inc) + case WriteFallbackReasonOtherAttr: + o.fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonOtherAtomic.Add(inc) + case WriteFallbackReasonOutOfOrderAttr: + o.fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonOutOfOrderAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(writeFallbackReason)) + return + } + case OpenModeReadWriteAppendAttr: + switch writeFallbackReason { + case WriteFallbackReasonConcurrencyLimitBreachedAttr: + o.fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonConcurrencyLimitBreachedAtomic.Add(inc) + case WriteFallbackReasonExistingFileAttr: + o.fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonExistingFileAtomic.Add(inc) + case WriteFallbackReasonOtherAttr: + o.fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonOtherAtomic.Add(inc) + case WriteFallbackReasonOutOfOrderAttr: + o.fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonOutOfOrderAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(writeFallbackReason)) + return + } + case OpenModeWriteOnlyAttr: + switch writeFallbackReason { + case WriteFallbackReasonConcurrencyLimitBreachedAttr: + o.fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonConcurrencyLimitBreachedAtomic.Add(inc) + case WriteFallbackReasonExistingFileAttr: + o.fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonExistingFileAtomic.Add(inc) + case WriteFallbackReasonOtherAttr: + o.fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonOtherAtomic.Add(inc) + case WriteFallbackReasonOutOfOrderAttr: + o.fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonOutOfOrderAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(writeFallbackReason)) + return + } + case OpenModeWriteOnlyAppendAttr: + switch writeFallbackReason { + case WriteFallbackReasonConcurrencyLimitBreachedAttr: + o.fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonConcurrencyLimitBreachedAtomic.Add(inc) + case WriteFallbackReasonExistingFileAttr: + o.fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonExistingFileAtomic.Add(inc) + case WriteFallbackReasonOtherAttr: + o.fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonOtherAtomic.Add(inc) + case WriteFallbackReasonOutOfOrderAttr: + o.fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonOutOfOrderAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(writeFallbackReason)) + return + } + default: + updateUnrecognizedAttribute(string(openMode)) + return + } +} + +func (o *otelMetrics) GcsDownloadBytesCount( + inc int64, readType ReadType) { + if inc < 0 { + logger.Errorf("Counter metric gcs/download_bytes_count received a negative increment: %d", inc) + return + } + switch readType { + case ReadTypeBufferedAttr: + o.gcsDownloadBytesCountReadTypeBufferedAtomic.Add(inc) + case ReadTypeParallelAttr: + o.gcsDownloadBytesCountReadTypeParallelAtomic.Add(inc) + case ReadTypeRandomAttr: + o.gcsDownloadBytesCountReadTypeRandomAtomic.Add(inc) + case ReadTypeSequentialAttr: + o.gcsDownloadBytesCountReadTypeSequentialAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(readType)) + return + } +} + +func (o *otelMetrics) GcsReadBytesCount( + inc int64) { + if inc < 0 { + logger.Errorf("Counter metric gcs/read_bytes_count received a negative increment: %d", inc) + return + } + o.gcsReadBytesCountAtomic.Add(inc) +} + +func (o *otelMetrics) GcsReadCount( + inc int64, readType ReadType) { + if inc < 0 { + logger.Errorf("Counter metric gcs/read_count received a negative increment: %d", inc) + return + } + switch readType { + case ReadTypeParallelAttr: + o.gcsReadCountReadTypeParallelAtomic.Add(inc) + case ReadTypeRandomAttr: + o.gcsReadCountReadTypeRandomAtomic.Add(inc) + case ReadTypeSequentialAttr: + o.gcsReadCountReadTypeSequentialAtomic.Add(inc) + case ReadTypeUnknownAttr: + o.gcsReadCountReadTypeUnknownAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(readType)) + return + } +} + +func (o *otelMetrics) GcsReaderCount( + inc int64, ioMethod IoMethod) { + if inc < 0 { + logger.Errorf("Counter metric gcs/reader_count received a negative increment: %d", inc) + return + } + switch ioMethod { + case IoMethodClosedAttr: + o.gcsReaderCountIoMethodClosedAtomic.Add(inc) + case IoMethodOpenedAttr: + o.gcsReaderCountIoMethodOpenedAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(ioMethod)) + return + } +} + +func (o *otelMetrics) GcsRequestCount( + inc int64, gcsMethod GcsMethod) { + if inc < 0 { + logger.Errorf("Counter metric gcs/request_count received a negative increment: %d", inc) + return + } + switch gcsMethod { + case GcsMethodComposeObjectsAttr: + o.gcsRequestCountGcsMethodComposeObjectsAtomic.Add(inc) + case GcsMethodCopyObjectAttr: + o.gcsRequestCountGcsMethodCopyObjectAtomic.Add(inc) + case GcsMethodCreateAppendableObjectWriterAttr: + o.gcsRequestCountGcsMethodCreateAppendableObjectWriterAtomic.Add(inc) + case GcsMethodCreateFolderAttr: + o.gcsRequestCountGcsMethodCreateFolderAtomic.Add(inc) + case GcsMethodCreateObjectAttr: + o.gcsRequestCountGcsMethodCreateObjectAtomic.Add(inc) + case GcsMethodCreateObjectChunkWriterAttr: + o.gcsRequestCountGcsMethodCreateObjectChunkWriterAtomic.Add(inc) + case GcsMethodDeleteFolderAttr: + o.gcsRequestCountGcsMethodDeleteFolderAtomic.Add(inc) + case GcsMethodDeleteObjectAttr: + o.gcsRequestCountGcsMethodDeleteObjectAtomic.Add(inc) + case GcsMethodFinalizeUploadAttr: + o.gcsRequestCountGcsMethodFinalizeUploadAtomic.Add(inc) + case GcsMethodFlushPendingWritesAttr: + o.gcsRequestCountGcsMethodFlushPendingWritesAtomic.Add(inc) + case GcsMethodGetFolderAttr: + o.gcsRequestCountGcsMethodGetFolderAtomic.Add(inc) + case GcsMethodListObjectsAttr: + o.gcsRequestCountGcsMethodListObjectsAtomic.Add(inc) + case GcsMethodMoveObjectAttr: + o.gcsRequestCountGcsMethodMoveObjectAtomic.Add(inc) + case GcsMethodMultiRangeDownloaderAddAttr: + o.gcsRequestCountGcsMethodMultiRangeDownloaderAddAtomic.Add(inc) + case GcsMethodNewMultiRangeDownloaderAttr: + o.gcsRequestCountGcsMethodNewMultiRangeDownloaderAtomic.Add(inc) + case GcsMethodNewReaderAttr: + o.gcsRequestCountGcsMethodNewReaderAtomic.Add(inc) + case GcsMethodRenameFolderAttr: + o.gcsRequestCountGcsMethodRenameFolderAtomic.Add(inc) + case GcsMethodStatObjectAttr: + o.gcsRequestCountGcsMethodStatObjectAtomic.Add(inc) + case GcsMethodUpdateObjectAttr: + o.gcsRequestCountGcsMethodUpdateObjectAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(gcsMethod)) + return + } +} + +func (o *otelMetrics) GcsRequestLatencies( + ctx context.Context, latency time.Duration, gcsMethod GcsMethod) { + var record histogramRecord + switch gcsMethod { + case GcsMethodComposeObjectsAttr: + record = histogramRecord{ctx: ctx, instrument: o.gcsRequestLatencies, value: latency.Milliseconds(), attributes: gcsRequestLatenciesGcsMethodComposeObjectsAttrSet} + case GcsMethodCopyObjectAttr: + record = histogramRecord{ctx: ctx, instrument: o.gcsRequestLatencies, value: latency.Milliseconds(), attributes: gcsRequestLatenciesGcsMethodCopyObjectAttrSet} + case GcsMethodCreateAppendableObjectWriterAttr: + record = histogramRecord{ctx: ctx, instrument: o.gcsRequestLatencies, value: latency.Milliseconds(), attributes: gcsRequestLatenciesGcsMethodCreateAppendableObjectWriterAttrSet} + case GcsMethodCreateFolderAttr: + record = histogramRecord{ctx: ctx, instrument: o.gcsRequestLatencies, value: latency.Milliseconds(), attributes: gcsRequestLatenciesGcsMethodCreateFolderAttrSet} + case GcsMethodCreateObjectAttr: + record = histogramRecord{ctx: ctx, instrument: o.gcsRequestLatencies, value: latency.Milliseconds(), attributes: gcsRequestLatenciesGcsMethodCreateObjectAttrSet} + case GcsMethodCreateObjectChunkWriterAttr: + record = histogramRecord{ctx: ctx, instrument: o.gcsRequestLatencies, value: latency.Milliseconds(), attributes: gcsRequestLatenciesGcsMethodCreateObjectChunkWriterAttrSet} + case GcsMethodDeleteFolderAttr: + record = histogramRecord{ctx: ctx, instrument: o.gcsRequestLatencies, value: latency.Milliseconds(), attributes: gcsRequestLatenciesGcsMethodDeleteFolderAttrSet} + case GcsMethodDeleteObjectAttr: + record = histogramRecord{ctx: ctx, instrument: o.gcsRequestLatencies, value: latency.Milliseconds(), attributes: gcsRequestLatenciesGcsMethodDeleteObjectAttrSet} + case GcsMethodFinalizeUploadAttr: + record = histogramRecord{ctx: ctx, instrument: o.gcsRequestLatencies, value: latency.Milliseconds(), attributes: gcsRequestLatenciesGcsMethodFinalizeUploadAttrSet} + case GcsMethodFlushPendingWritesAttr: + record = histogramRecord{ctx: ctx, instrument: o.gcsRequestLatencies, value: latency.Milliseconds(), attributes: gcsRequestLatenciesGcsMethodFlushPendingWritesAttrSet} + case GcsMethodGetFolderAttr: + record = histogramRecord{ctx: ctx, instrument: o.gcsRequestLatencies, value: latency.Milliseconds(), attributes: gcsRequestLatenciesGcsMethodGetFolderAttrSet} + case GcsMethodListObjectsAttr: + record = histogramRecord{ctx: ctx, instrument: o.gcsRequestLatencies, value: latency.Milliseconds(), attributes: gcsRequestLatenciesGcsMethodListObjectsAttrSet} + case GcsMethodMoveObjectAttr: + record = histogramRecord{ctx: ctx, instrument: o.gcsRequestLatencies, value: latency.Milliseconds(), attributes: gcsRequestLatenciesGcsMethodMoveObjectAttrSet} + case GcsMethodMultiRangeDownloaderAddAttr: + record = histogramRecord{ctx: ctx, instrument: o.gcsRequestLatencies, value: latency.Milliseconds(), attributes: gcsRequestLatenciesGcsMethodMultiRangeDownloaderAddAttrSet} + case GcsMethodNewMultiRangeDownloaderAttr: + record = histogramRecord{ctx: ctx, instrument: o.gcsRequestLatencies, value: latency.Milliseconds(), attributes: gcsRequestLatenciesGcsMethodNewMultiRangeDownloaderAttrSet} + case GcsMethodNewReaderAttr: + record = histogramRecord{ctx: ctx, instrument: o.gcsRequestLatencies, value: latency.Milliseconds(), attributes: gcsRequestLatenciesGcsMethodNewReaderAttrSet} + case GcsMethodRenameFolderAttr: + record = histogramRecord{ctx: ctx, instrument: o.gcsRequestLatencies, value: latency.Milliseconds(), attributes: gcsRequestLatenciesGcsMethodRenameFolderAttrSet} + case GcsMethodStatObjectAttr: + record = histogramRecord{ctx: ctx, instrument: o.gcsRequestLatencies, value: latency.Milliseconds(), attributes: gcsRequestLatenciesGcsMethodStatObjectAttrSet} + case GcsMethodUpdateObjectAttr: + record = histogramRecord{ctx: ctx, instrument: o.gcsRequestLatencies, value: latency.Milliseconds(), attributes: gcsRequestLatenciesGcsMethodUpdateObjectAttrSet} + default: + updateUnrecognizedAttribute(string(gcsMethod)) + return + } + + select { + case o.ch <- record: // Do nothing + default: // Unblock writes to channel if it's full. + } +} + +func (o *otelMetrics) GcsRetryCount( + inc int64, retryErrorCategory RetryErrorCategory) { + if inc < 0 { + logger.Errorf("Counter metric gcs/retry_count received a negative increment: %d", inc) + return + } + switch retryErrorCategory { + case RetryErrorCategoryOTHERERRORSAttr: + o.gcsRetryCountRetryErrorCategoryOTHERERRORSAtomic.Add(inc) + case RetryErrorCategorySTALLEDREADREQUESTAttr: + o.gcsRetryCountRetryErrorCategorySTALLEDREADREQUESTAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(retryErrorCategory)) + return + } +} + +func (o *otelMetrics) MetadataCacheReadCount( + inc int64, cacheHit bool, entryStatus EntryStatus, lookupDetail LookupDetail) { + if inc < 0 { + logger.Errorf("Counter metric metadata_cache/read_count received a negative increment: %d", inc) + return + } + switch cacheHit { + case true: + switch entryStatus { + case EntryStatusNegativeAttr: + switch lookupDetail { + case LookupDetailFoundAttr: + o.metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailFoundAtomic.Add(inc) + case LookupDetailNotFoundAttr: + o.metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailNotFoundAtomic.Add(inc) + case LookupDetailTtlExpiredAttr: + o.metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailTtlExpiredAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(lookupDetail)) + return + } + case EntryStatusPositiveAttr: + switch lookupDetail { + case LookupDetailFoundAttr: + o.metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailFoundAtomic.Add(inc) + case LookupDetailNotFoundAttr: + o.metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailNotFoundAtomic.Add(inc) + case LookupDetailTtlExpiredAttr: + o.metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailTtlExpiredAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(lookupDetail)) + return + } + default: + updateUnrecognizedAttribute(string(entryStatus)) + return + } + case false: + switch entryStatus { + case EntryStatusNegativeAttr: + switch lookupDetail { + case LookupDetailFoundAttr: + o.metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailFoundAtomic.Add(inc) + case LookupDetailNotFoundAttr: + o.metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailNotFoundAtomic.Add(inc) + case LookupDetailTtlExpiredAttr: + o.metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailTtlExpiredAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(lookupDetail)) + return + } + case EntryStatusPositiveAttr: + switch lookupDetail { + case LookupDetailFoundAttr: + o.metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailFoundAtomic.Add(inc) + case LookupDetailNotFoundAttr: + o.metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailNotFoundAtomic.Add(inc) + case LookupDetailTtlExpiredAttr: + o.metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailTtlExpiredAtomic.Add(inc) + default: + updateUnrecognizedAttribute(string(lookupDetail)) + return + } + default: + updateUnrecognizedAttribute(string(entryStatus)) + return + } + } +} + +func (o *otelMetrics) ReadBlockSizes( + ctx context.Context, value int64) { + record := histogramRecord{ctx: ctx, instrument: o.readBlockSizes, value: value} + + select { + case o.ch <- record: // Do nothing + default: // Unblock writes to channel if it's full. + } +} + +func (o *otelMetrics) TestUpdownCounter( + inc int64) { + o.testUpdownCounterAtomic.Add(inc) +} + +func (o *otelMetrics) TestUpdownCounterWithAttrs( + inc int64, requestType RequestType) { + switch requestType { + case RequestTypeAttr1Attr: + o.testUpdownCounterWithAttrsRequestTypeAttr1Atomic.Add(inc) + case RequestTypeAttr2Attr: + o.testUpdownCounterWithAttrsRequestTypeAttr2Atomic.Add(inc) + default: + updateUnrecognizedAttribute(string(requestType)) + return + } +} + +func NewOTelMetrics(ctx context.Context, workers int, bufferSize int) (*otelMetrics, error) { + ch := make(chan histogramRecord, bufferSize) + var wg sync.WaitGroup + startSampledLogging(ctx) + for range workers { + wg.Add(1) + go func() { + defer wg.Done() + for record := range ch { + if record.attributes != nil { + record.instrument.Record(record.ctx, record.value, record.attributes) + } else { + record.instrument.Record(record.ctx, record.value) + } + } + }() + } + meter := otel.Meter("gcsfuse") + var bufferedReadFallbackTriggerCountReasonInsufficientMemoryAtomic, + bufferedReadFallbackTriggerCountReasonRandomReadDetectedAtomic atomic.Int64 + + var fileCacheReadBytesCountReadTypeParallelAtomic, + fileCacheReadBytesCountReadTypeRandomAtomic, + fileCacheReadBytesCountReadTypeSequentialAtomic, + fileCacheReadBytesCountReadTypeUnknownAtomic atomic.Int64 + + var fileCacheReadCountCacheHitTrueReadTypeParallelAtomic, + fileCacheReadCountCacheHitTrueReadTypeRandomAtomic, + fileCacheReadCountCacheHitTrueReadTypeSequentialAtomic, + fileCacheReadCountCacheHitTrueReadTypeUnknownAtomic, + fileCacheReadCountCacheHitFalseReadTypeParallelAtomic, + fileCacheReadCountCacheHitFalseReadTypeRandomAtomic, + fileCacheReadCountCacheHitFalseReadTypeSequentialAtomic, + fileCacheReadCountCacheHitFalseReadTypeUnknownAtomic atomic.Int64 + + var fsOpsCountFsOpBatchForgetAtomic, + fsOpsCountFsOpCreateFileAtomic, + fsOpsCountFsOpCreateLinkAtomic, + fsOpsCountFsOpCreateSymlinkAtomic, + fsOpsCountFsOpFlushFileAtomic, + fsOpsCountFsOpForgetInodeAtomic, + fsOpsCountFsOpGetInodeAttributesAtomic, + fsOpsCountFsOpLookUpInodeAtomic, + fsOpsCountFsOpMkDirAtomic, + fsOpsCountFsOpMkNodeAtomic, + fsOpsCountFsOpOpenDirAtomic, + fsOpsCountFsOpOpenFileAtomic, + fsOpsCountFsOpOthersAtomic, + fsOpsCountFsOpReadDirAtomic, + fsOpsCountFsOpReadDirPlusAtomic, + fsOpsCountFsOpReadFileAtomic, + fsOpsCountFsOpReadSymlinkAtomic, + fsOpsCountFsOpReleaseDirHandleAtomic, + fsOpsCountFsOpReleaseFileHandleAtomic, + fsOpsCountFsOpRenameAtomic, + fsOpsCountFsOpRmDirAtomic, + fsOpsCountFsOpSetInodeAttributesAtomic, + fsOpsCountFsOpSyncFileAtomic, + fsOpsCountFsOpUnlinkAtomic, + fsOpsCountFsOpWriteFileAtomic atomic.Int64 + + var fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpWriteFileAtomic atomic.Int64 + + var fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonConcurrencyLimitBreachedAtomic, + fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonExistingFileAtomic, + fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonOtherAtomic, + fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonOutOfOrderAtomic, + fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonConcurrencyLimitBreachedAtomic, + fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonExistingFileAtomic, + fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonOtherAtomic, + fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonOutOfOrderAtomic, + fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonConcurrencyLimitBreachedAtomic, + fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonExistingFileAtomic, + fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonOtherAtomic, + fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonOutOfOrderAtomic, + fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonConcurrencyLimitBreachedAtomic, + fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonExistingFileAtomic, + fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonOtherAtomic, + fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonOutOfOrderAtomic, + fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonConcurrencyLimitBreachedAtomic, + fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonExistingFileAtomic, + fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonOtherAtomic, + fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonOutOfOrderAtomic atomic.Int64 + + var gcsDownloadBytesCountReadTypeBufferedAtomic, + gcsDownloadBytesCountReadTypeParallelAtomic, + gcsDownloadBytesCountReadTypeRandomAtomic, + gcsDownloadBytesCountReadTypeSequentialAtomic atomic.Int64 + + var gcsReadBytesCountAtomic atomic.Int64 + + var gcsReadCountReadTypeParallelAtomic, + gcsReadCountReadTypeRandomAtomic, + gcsReadCountReadTypeSequentialAtomic, + gcsReadCountReadTypeUnknownAtomic atomic.Int64 + + var gcsReaderCountIoMethodClosedAtomic, + gcsReaderCountIoMethodOpenedAtomic atomic.Int64 + + var gcsRequestCountGcsMethodComposeObjectsAtomic, + gcsRequestCountGcsMethodCopyObjectAtomic, + gcsRequestCountGcsMethodCreateAppendableObjectWriterAtomic, + gcsRequestCountGcsMethodCreateFolderAtomic, + gcsRequestCountGcsMethodCreateObjectAtomic, + gcsRequestCountGcsMethodCreateObjectChunkWriterAtomic, + gcsRequestCountGcsMethodDeleteFolderAtomic, + gcsRequestCountGcsMethodDeleteObjectAtomic, + gcsRequestCountGcsMethodFinalizeUploadAtomic, + gcsRequestCountGcsMethodFlushPendingWritesAtomic, + gcsRequestCountGcsMethodGetFolderAtomic, + gcsRequestCountGcsMethodListObjectsAtomic, + gcsRequestCountGcsMethodMoveObjectAtomic, + gcsRequestCountGcsMethodMultiRangeDownloaderAddAtomic, + gcsRequestCountGcsMethodNewMultiRangeDownloaderAtomic, + gcsRequestCountGcsMethodNewReaderAtomic, + gcsRequestCountGcsMethodRenameFolderAtomic, + gcsRequestCountGcsMethodStatObjectAtomic, + gcsRequestCountGcsMethodUpdateObjectAtomic atomic.Int64 + + var gcsRetryCountRetryErrorCategoryOTHERERRORSAtomic, + gcsRetryCountRetryErrorCategorySTALLEDREADREQUESTAtomic atomic.Int64 + + var metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailFoundAtomic, + metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailNotFoundAtomic, + metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailTtlExpiredAtomic, + metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailFoundAtomic, + metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailNotFoundAtomic, + metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailTtlExpiredAtomic, + metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailFoundAtomic, + metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailNotFoundAtomic, + metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailTtlExpiredAtomic, + metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailFoundAtomic, + metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailNotFoundAtomic, + metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailTtlExpiredAtomic atomic.Int64 + + var testUpdownCounterAtomic atomic.Int64 + + var testUpdownCounterWithAttrsRequestTypeAttr1Atomic, + testUpdownCounterWithAttrsRequestTypeAttr2Atomic atomic.Int64 + + _, err0 := meter.Int64ObservableCounter("buffered_read/fallback_trigger_count", + metric.WithDescription("The cumulative number of times the BufferedReader falls back to a different reader, along with the reason: random_read_detected or insufficient_memory."), + metric.WithUnit(""), + metric.WithInt64Callback(func(_ context.Context, obsrv metric.Int64Observer) error { + conditionallyObserve(obsrv, &bufferedReadFallbackTriggerCountReasonInsufficientMemoryAtomic, bufferedReadFallbackTriggerCountReasonInsufficientMemoryAttrSet) + conditionallyObserve(obsrv, &bufferedReadFallbackTriggerCountReasonRandomReadDetectedAtomic, bufferedReadFallbackTriggerCountReasonRandomReadDetectedAttrSet) + return nil + })) + + bufferedReadReadLatency, err1 := meter.Int64Histogram("buffered_read/read_latency", + metric.WithDescription("The cumulative distribution of latencies for ReadAt calls served by the buffered reader."), + metric.WithUnit("us"), + metric.WithExplicitBucketBoundaries(50, 100, 200, 400, 800, 1500, 3000, 5000, 10000, 20000, 50000, 100000, 200000, 500000, 1000000, 2000000, 5000000, 10000000, 20000000, 50000000, 100000000, 200000000, 500000000)) + + _, err2 := meter.Int64ObservableCounter("file_cache/read_bytes_count", + metric.WithDescription("The cumulative number of bytes read from file cache along with read type - Sequential/Random"), + metric.WithUnit("By"), + metric.WithInt64Callback(func(_ context.Context, obsrv metric.Int64Observer) error { + conditionallyObserve(obsrv, &fileCacheReadBytesCountReadTypeParallelAtomic, fileCacheReadBytesCountReadTypeParallelAttrSet) + conditionallyObserve(obsrv, &fileCacheReadBytesCountReadTypeRandomAtomic, fileCacheReadBytesCountReadTypeRandomAttrSet) + conditionallyObserve(obsrv, &fileCacheReadBytesCountReadTypeSequentialAtomic, fileCacheReadBytesCountReadTypeSequentialAttrSet) + conditionallyObserve(obsrv, &fileCacheReadBytesCountReadTypeUnknownAtomic, fileCacheReadBytesCountReadTypeUnknownAttrSet) + return nil + })) + + _, err3 := meter.Int64ObservableCounter("file_cache/read_count", + metric.WithDescription("Specifies the number of read requests made via file cache along with type - Sequential/Random and cache hit - true/false"), + metric.WithUnit(""), + metric.WithInt64Callback(func(_ context.Context, obsrv metric.Int64Observer) error { + conditionallyObserve(obsrv, &fileCacheReadCountCacheHitTrueReadTypeParallelAtomic, fileCacheReadCountCacheHitTrueReadTypeParallelAttrSet) + conditionallyObserve(obsrv, &fileCacheReadCountCacheHitTrueReadTypeRandomAtomic, fileCacheReadCountCacheHitTrueReadTypeRandomAttrSet) + conditionallyObserve(obsrv, &fileCacheReadCountCacheHitTrueReadTypeSequentialAtomic, fileCacheReadCountCacheHitTrueReadTypeSequentialAttrSet) + conditionallyObserve(obsrv, &fileCacheReadCountCacheHitTrueReadTypeUnknownAtomic, fileCacheReadCountCacheHitTrueReadTypeUnknownAttrSet) + conditionallyObserve(obsrv, &fileCacheReadCountCacheHitFalseReadTypeParallelAtomic, fileCacheReadCountCacheHitFalseReadTypeParallelAttrSet) + conditionallyObserve(obsrv, &fileCacheReadCountCacheHitFalseReadTypeRandomAtomic, fileCacheReadCountCacheHitFalseReadTypeRandomAttrSet) + conditionallyObserve(obsrv, &fileCacheReadCountCacheHitFalseReadTypeSequentialAtomic, fileCacheReadCountCacheHitFalseReadTypeSequentialAttrSet) + conditionallyObserve(obsrv, &fileCacheReadCountCacheHitFalseReadTypeUnknownAtomic, fileCacheReadCountCacheHitFalseReadTypeUnknownAttrSet) + return nil + })) + + fileCacheReadLatencies, err4 := meter.Int64Histogram("file_cache/read_latencies", + metric.WithDescription("The cumulative distribution of the file cache read latencies along with cache hit - true/false."), + metric.WithUnit("us"), + metric.WithExplicitBucketBoundaries(50, 100, 200, 400, 800, 1500, 3000, 5000, 10000, 20000, 50000, 100000, 200000, 500000, 1000000, 2000000, 5000000, 10000000, 20000000, 50000000, 100000000, 200000000, 500000000)) + + _, err5 := meter.Int64ObservableCounter("fs/ops_count", + metric.WithDescription("The cumulative number of ops processed by the file system."), + metric.WithUnit(""), + metric.WithInt64Callback(func(_ context.Context, obsrv metric.Int64Observer) error { + conditionallyObserve(obsrv, &fsOpsCountFsOpBatchForgetAtomic, fsOpsCountFsOpBatchForgetAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpCreateFileAtomic, fsOpsCountFsOpCreateFileAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpCreateLinkAtomic, fsOpsCountFsOpCreateLinkAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpCreateSymlinkAtomic, fsOpsCountFsOpCreateSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpFlushFileAtomic, fsOpsCountFsOpFlushFileAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpForgetInodeAtomic, fsOpsCountFsOpForgetInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpGetInodeAttributesAtomic, fsOpsCountFsOpGetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpLookUpInodeAtomic, fsOpsCountFsOpLookUpInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpMkDirAtomic, fsOpsCountFsOpMkDirAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpMkNodeAtomic, fsOpsCountFsOpMkNodeAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpOpenDirAtomic, fsOpsCountFsOpOpenDirAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpOpenFileAtomic, fsOpsCountFsOpOpenFileAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpOthersAtomic, fsOpsCountFsOpOthersAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpReadDirAtomic, fsOpsCountFsOpReadDirAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpReadDirPlusAtomic, fsOpsCountFsOpReadDirPlusAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpReadFileAtomic, fsOpsCountFsOpReadFileAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpReadSymlinkAtomic, fsOpsCountFsOpReadSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpReleaseDirHandleAtomic, fsOpsCountFsOpReleaseDirHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpReleaseFileHandleAtomic, fsOpsCountFsOpReleaseFileHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpRenameAtomic, fsOpsCountFsOpRenameAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpRmDirAtomic, fsOpsCountFsOpRmDirAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpSetInodeAttributesAtomic, fsOpsCountFsOpSetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpSyncFileAtomic, fsOpsCountFsOpSyncFileAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpUnlinkAtomic, fsOpsCountFsOpUnlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsCountFsOpWriteFileAtomic, fsOpsCountFsOpWriteFileAttrSet) + return nil + })) + + _, err6 := meter.Int64ObservableCounter("fs/ops_error_count", + metric.WithDescription("The cumulative number of errors generated by file system operations."), + metric.WithUnit(""), + metric.WithInt64Callback(func(_ context.Context, obsrv metric.Int64Observer) error { + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpBatchForgetAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpBatchForgetAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateFileAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateLinkAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateLinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateSymlinkAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpFlushFileAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpFlushFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpForgetInodeAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpForgetInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpGetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpGetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpLookUpInodeAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpLookUpInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpMkDirAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpMkDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpMkNodeAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpMkNodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOpenDirAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOpenDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOpenFileAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOpenFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOthersAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOthersAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadDirAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadDirPlusAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadDirPlusAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadFileAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadSymlinkAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReleaseDirHandleAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReleaseDirHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReleaseFileHandleAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReleaseFileHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpRenameAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpRenameAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpRmDirAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpRmDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpSetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpSetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpSyncFileAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpSyncFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpUnlinkAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpUnlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpWriteFileAtomic, fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpWriteFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpBatchForgetAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpBatchForgetAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateFileAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateLinkAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateLinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateSymlinkAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpFlushFileAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpFlushFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpForgetInodeAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpForgetInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpGetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpGetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpLookUpInodeAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpLookUpInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpMkDirAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpMkDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpMkNodeAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpMkNodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOpenDirAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOpenDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOpenFileAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOpenFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOthersAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOthersAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadDirAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadDirPlusAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadDirPlusAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadFileAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadSymlinkAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReleaseDirHandleAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReleaseDirHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReleaseFileHandleAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReleaseFileHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpRenameAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpRenameAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpRmDirAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpRmDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpSetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpSetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpSyncFileAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpSyncFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpUnlinkAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpUnlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpWriteFileAtomic, fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpWriteFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpBatchForgetAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpBatchForgetAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateFileAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateLinkAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateLinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateSymlinkAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpFlushFileAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpFlushFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpForgetInodeAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpForgetInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpGetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpGetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpLookUpInodeAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpLookUpInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpMkDirAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpMkDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpMkNodeAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpMkNodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOpenDirAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOpenDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOpenFileAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOpenFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOthersAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOthersAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadDirAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadDirPlusAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadDirPlusAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadFileAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadSymlinkAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReleaseDirHandleAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReleaseDirHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReleaseFileHandleAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReleaseFileHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpRenameAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpRenameAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpRmDirAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpRmDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpSetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpSetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpSyncFileAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpSyncFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpUnlinkAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpUnlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpWriteFileAtomic, fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpWriteFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpBatchForgetAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpBatchForgetAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateFileAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateLinkAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateLinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateSymlinkAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpFlushFileAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpFlushFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpForgetInodeAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpForgetInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpGetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpGetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpLookUpInodeAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpLookUpInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpMkDirAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpMkDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpMkNodeAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpMkNodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOpenDirAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOpenDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOpenFileAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOpenFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOthersAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOthersAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadDirAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadDirPlusAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadDirPlusAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadFileAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadSymlinkAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReleaseDirHandleAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReleaseDirHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReleaseFileHandleAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReleaseFileHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpRenameAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpRenameAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpRmDirAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpRmDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpSetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpSetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpSyncFileAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpSyncFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpUnlinkAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpUnlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpWriteFileAtomic, fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpWriteFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpBatchForgetAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpBatchForgetAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateFileAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateLinkAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateLinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateSymlinkAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpFlushFileAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpFlushFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpForgetInodeAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpForgetInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpGetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpGetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpLookUpInodeAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpLookUpInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpMkDirAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpMkDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpMkNodeAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpMkNodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOpenDirAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOpenDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOpenFileAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOpenFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOthersAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOthersAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadDirAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadDirPlusAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadDirPlusAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadFileAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadSymlinkAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReleaseDirHandleAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReleaseDirHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReleaseFileHandleAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReleaseFileHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpRenameAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpRenameAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpRmDirAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpRmDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpSetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpSetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpSyncFileAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpSyncFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpUnlinkAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpUnlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpWriteFileAtomic, fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpWriteFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpBatchForgetAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpBatchForgetAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateFileAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateLinkAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateLinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateSymlinkAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpFlushFileAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpFlushFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpForgetInodeAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpForgetInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpGetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpGetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpLookUpInodeAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpLookUpInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpMkDirAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpMkDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpMkNodeAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpMkNodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOpenDirAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOpenDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOpenFileAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOpenFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOthersAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOthersAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadDirAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadDirPlusAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadDirPlusAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadFileAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadSymlinkAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReleaseDirHandleAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReleaseDirHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReleaseFileHandleAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReleaseFileHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpRenameAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpRenameAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpRmDirAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpRmDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpSetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpSetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpSyncFileAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpSyncFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpUnlinkAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpUnlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpWriteFileAtomic, fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpWriteFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpBatchForgetAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpBatchForgetAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateFileAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateLinkAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateLinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateSymlinkAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpFlushFileAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpFlushFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpForgetInodeAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpForgetInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpGetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpGetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpLookUpInodeAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpLookUpInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpMkDirAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpMkDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpMkNodeAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpMkNodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOpenDirAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOpenDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOpenFileAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOpenFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOthersAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOthersAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadDirAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadDirPlusAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadDirPlusAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadFileAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadSymlinkAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReleaseDirHandleAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReleaseDirHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReleaseFileHandleAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReleaseFileHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpRenameAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpRenameAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpRmDirAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpRmDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpSetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpSetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpSyncFileAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpSyncFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpUnlinkAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpUnlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpWriteFileAtomic, fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpWriteFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpBatchForgetAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpBatchForgetAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateFileAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateLinkAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateLinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateSymlinkAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpFlushFileAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpFlushFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpForgetInodeAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpForgetInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpGetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpGetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpLookUpInodeAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpLookUpInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpMkDirAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpMkDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpMkNodeAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpMkNodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpOpenDirAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpOpenDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpOpenFileAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpOpenFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpOthersAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpOthersAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadDirAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadDirPlusAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadDirPlusAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadFileAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadSymlinkAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpReleaseDirHandleAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpReleaseDirHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpReleaseFileHandleAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpReleaseFileHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpRenameAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpRenameAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpRmDirAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpRmDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpSetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpSetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpSyncFileAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpSyncFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpUnlinkAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpUnlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryIOERRORFsOpWriteFileAtomic, fsOpsErrorCountFsErrorCategoryIOERRORFsOpWriteFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpBatchForgetAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpBatchForgetAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateFileAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateLinkAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateLinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateSymlinkAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpFlushFileAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpFlushFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpForgetInodeAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpForgetInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpGetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpGetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpLookUpInodeAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpLookUpInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpMkDirAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpMkDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpMkNodeAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpMkNodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOpenDirAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOpenDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOpenFileAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOpenFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOthersAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOthersAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadDirAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadDirPlusAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadDirPlusAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadFileAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadSymlinkAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReleaseDirHandleAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReleaseDirHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReleaseFileHandleAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReleaseFileHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpRenameAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpRenameAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpRmDirAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpRmDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpSetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpSetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpSyncFileAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpSyncFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpUnlinkAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpUnlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpWriteFileAtomic, fsOpsErrorCountFsErrorCategoryMISCERRORFsOpWriteFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpBatchForgetAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpBatchForgetAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateFileAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateLinkAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateLinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateSymlinkAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpFlushFileAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpFlushFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpForgetInodeAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpForgetInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpGetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpGetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpLookUpInodeAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpLookUpInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpMkDirAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpMkDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpMkNodeAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpMkNodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOpenDirAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOpenDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOpenFileAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOpenFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOthersAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOthersAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadDirAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadDirPlusAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadDirPlusAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadFileAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadSymlinkAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReleaseDirHandleAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReleaseDirHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReleaseFileHandleAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReleaseFileHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpRenameAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpRenameAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpRmDirAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpRmDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpSetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpSetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpSyncFileAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpSyncFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpUnlinkAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpUnlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpWriteFileAtomic, fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpWriteFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpBatchForgetAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpBatchForgetAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateFileAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateLinkAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateLinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateSymlinkAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpFlushFileAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpFlushFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpForgetInodeAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpForgetInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpGetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpGetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpLookUpInodeAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpLookUpInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpMkDirAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpMkDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpMkNodeAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpMkNodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOpenDirAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOpenDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOpenFileAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOpenFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOthersAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOthersAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadDirAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadDirPlusAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadDirPlusAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadFileAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadSymlinkAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReleaseDirHandleAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReleaseDirHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReleaseFileHandleAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReleaseFileHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpRenameAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpRenameAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpRmDirAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpRmDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpSetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpSetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpSyncFileAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpSyncFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpUnlinkAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpUnlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpWriteFileAtomic, fsOpsErrorCountFsErrorCategoryNOTADIRFsOpWriteFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpBatchForgetAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpBatchForgetAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateFileAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateLinkAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateLinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateSymlinkAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpFlushFileAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpFlushFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpForgetInodeAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpForgetInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpGetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpGetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpLookUpInodeAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpLookUpInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpMkDirAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpMkDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpMkNodeAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpMkNodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOpenDirAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOpenDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOpenFileAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOpenFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOthersAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOthersAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadDirAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadDirPlusAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadDirPlusAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadFileAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadSymlinkAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReleaseDirHandleAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReleaseDirHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReleaseFileHandleAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReleaseFileHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpRenameAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpRenameAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpRmDirAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpRmDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpSetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpSetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpSyncFileAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpSyncFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpUnlinkAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpUnlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpWriteFileAtomic, fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpWriteFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpBatchForgetAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpBatchForgetAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateFileAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateLinkAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateLinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateSymlinkAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpFlushFileAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpFlushFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpForgetInodeAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpForgetInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpGetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpGetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpLookUpInodeAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpLookUpInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpMkDirAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpMkDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpMkNodeAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpMkNodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOpenDirAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOpenDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOpenFileAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOpenFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOthersAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOthersAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadDirAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadDirPlusAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadDirPlusAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadFileAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadSymlinkAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReleaseDirHandleAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReleaseDirHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReleaseFileHandleAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReleaseFileHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpRenameAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpRenameAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpRmDirAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpRmDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpSetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpSetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpSyncFileAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpSyncFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpUnlinkAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpUnlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpWriteFileAtomic, fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpWriteFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpBatchForgetAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpBatchForgetAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateFileAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateLinkAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateLinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateSymlinkAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpFlushFileAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpFlushFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpForgetInodeAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpForgetInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpGetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpGetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpLookUpInodeAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpLookUpInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpMkDirAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpMkDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpMkNodeAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpMkNodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOpenDirAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOpenDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOpenFileAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOpenFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOthersAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOthersAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadDirAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadDirPlusAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadDirPlusAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadFileAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadSymlinkAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReleaseDirHandleAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReleaseDirHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReleaseFileHandleAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReleaseFileHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpRenameAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpRenameAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpRmDirAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpRmDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpSetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpSetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpSyncFileAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpSyncFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpUnlinkAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpUnlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpWriteFileAtomic, fsOpsErrorCountFsErrorCategoryPERMERRORFsOpWriteFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpBatchForgetAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpBatchForgetAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateFileAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateLinkAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateLinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateSymlinkAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpFlushFileAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpFlushFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpForgetInodeAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpForgetInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpGetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpGetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpLookUpInodeAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpLookUpInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpMkDirAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpMkDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpMkNodeAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpMkNodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOpenDirAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOpenDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOpenFileAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOpenFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOthersAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOthersAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadDirAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadDirPlusAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadDirPlusAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadFileAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadSymlinkAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReleaseDirHandleAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReleaseDirHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReleaseFileHandleAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReleaseFileHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpRenameAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpRenameAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpRmDirAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpRmDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpSetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpSetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpSyncFileAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpSyncFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpUnlinkAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpUnlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpWriteFileAtomic, fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpWriteFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpBatchForgetAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpBatchForgetAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateFileAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateLinkAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateLinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateSymlinkAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpFlushFileAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpFlushFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpForgetInodeAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpForgetInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpGetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpGetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpLookUpInodeAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpLookUpInodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpMkDirAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpMkDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpMkNodeAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpMkNodeAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOpenDirAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOpenDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOpenFileAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOpenFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOthersAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOthersAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadDirAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadDirPlusAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadDirPlusAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadFileAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadSymlinkAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadSymlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReleaseDirHandleAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReleaseDirHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReleaseFileHandleAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReleaseFileHandleAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpRenameAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpRenameAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpRmDirAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpRmDirAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpSetInodeAttributesAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpSetInodeAttributesAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpSyncFileAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpSyncFileAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpUnlinkAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpUnlinkAttrSet) + conditionallyObserve(obsrv, &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpWriteFileAtomic, fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpWriteFileAttrSet) + return nil + })) + + fsOpsLatency, err7 := meter.Int64Histogram("fs/ops_latency", + metric.WithDescription("The cumulative distribution of file system operation latencies"), + metric.WithUnit("us"), + metric.WithExplicitBucketBoundaries(50, 100, 200, 400, 800, 1500, 3000, 5000, 10000, 20000, 50000, 100000, 200000, 500000, 1000000, 2000000, 5000000, 10000000, 20000000, 50000000, 100000000, 200000000, 500000000)) + + _, err8 := meter.Int64ObservableCounter("fs/streaming_write_fallback_count", + metric.WithDescription("The cumulative number of streaming write fallbacks with reason attached"), + metric.WithUnit(""), + metric.WithInt64Callback(func(_ context.Context, obsrv metric.Int64Observer) error { + conditionallyObserve(obsrv, &fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonConcurrencyLimitBreachedAtomic, fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonConcurrencyLimitBreachedAttrSet) + conditionallyObserve(obsrv, &fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonExistingFileAtomic, fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonExistingFileAttrSet) + conditionallyObserve(obsrv, &fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonOtherAtomic, fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonOtherAttrSet) + conditionallyObserve(obsrv, &fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonOutOfOrderAtomic, fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonOutOfOrderAttrSet) + conditionallyObserve(obsrv, &fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonConcurrencyLimitBreachedAtomic, fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonConcurrencyLimitBreachedAttrSet) + conditionallyObserve(obsrv, &fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonExistingFileAtomic, fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonExistingFileAttrSet) + conditionallyObserve(obsrv, &fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonOtherAtomic, fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonOtherAttrSet) + conditionallyObserve(obsrv, &fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonOutOfOrderAtomic, fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonOutOfOrderAttrSet) + conditionallyObserve(obsrv, &fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonConcurrencyLimitBreachedAtomic, fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonConcurrencyLimitBreachedAttrSet) + conditionallyObserve(obsrv, &fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonExistingFileAtomic, fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonExistingFileAttrSet) + conditionallyObserve(obsrv, &fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonOtherAtomic, fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonOtherAttrSet) + conditionallyObserve(obsrv, &fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonOutOfOrderAtomic, fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonOutOfOrderAttrSet) + conditionallyObserve(obsrv, &fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonConcurrencyLimitBreachedAtomic, fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonConcurrencyLimitBreachedAttrSet) + conditionallyObserve(obsrv, &fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonExistingFileAtomic, fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonExistingFileAttrSet) + conditionallyObserve(obsrv, &fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonOtherAtomic, fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonOtherAttrSet) + conditionallyObserve(obsrv, &fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonOutOfOrderAtomic, fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonOutOfOrderAttrSet) + conditionallyObserve(obsrv, &fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonConcurrencyLimitBreachedAtomic, fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonConcurrencyLimitBreachedAttrSet) + conditionallyObserve(obsrv, &fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonExistingFileAtomic, fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonExistingFileAttrSet) + conditionallyObserve(obsrv, &fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonOtherAtomic, fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonOtherAttrSet) + conditionallyObserve(obsrv, &fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonOutOfOrderAtomic, fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonOutOfOrderAttrSet) + return nil + })) + + _, err9 := meter.Int64ObservableCounter("gcs/download_bytes_count", + metric.WithDescription("The cumulative number of bytes downloaded from GCS along with type - Sequential/Random"), + metric.WithUnit("By"), + metric.WithInt64Callback(func(_ context.Context, obsrv metric.Int64Observer) error { + conditionallyObserve(obsrv, &gcsDownloadBytesCountReadTypeBufferedAtomic, gcsDownloadBytesCountReadTypeBufferedAttrSet) + conditionallyObserve(obsrv, &gcsDownloadBytesCountReadTypeParallelAtomic, gcsDownloadBytesCountReadTypeParallelAttrSet) + conditionallyObserve(obsrv, &gcsDownloadBytesCountReadTypeRandomAtomic, gcsDownloadBytesCountReadTypeRandomAttrSet) + conditionallyObserve(obsrv, &gcsDownloadBytesCountReadTypeSequentialAtomic, gcsDownloadBytesCountReadTypeSequentialAttrSet) + return nil + })) + + _, err10 := meter.Int64ObservableCounter("gcs/read_bytes_count", + metric.WithDescription("The cumulative number of bytes read from GCS objects."), + metric.WithUnit("By"), + metric.WithInt64Callback(func(_ context.Context, obsrv metric.Int64Observer) error { + conditionallyObserve(obsrv, &gcsReadBytesCountAtomic) + return nil + })) + + _, err11 := meter.Int64ObservableCounter("gcs/read_count", + metric.WithDescription("Specifies the number of gcs reads made along with type - Sequential/Random"), + metric.WithUnit(""), + metric.WithInt64Callback(func(_ context.Context, obsrv metric.Int64Observer) error { + conditionallyObserve(obsrv, &gcsReadCountReadTypeParallelAtomic, gcsReadCountReadTypeParallelAttrSet) + conditionallyObserve(obsrv, &gcsReadCountReadTypeRandomAtomic, gcsReadCountReadTypeRandomAttrSet) + conditionallyObserve(obsrv, &gcsReadCountReadTypeSequentialAtomic, gcsReadCountReadTypeSequentialAttrSet) + conditionallyObserve(obsrv, &gcsReadCountReadTypeUnknownAtomic, gcsReadCountReadTypeUnknownAttrSet) + return nil + })) + + _, err12 := meter.Int64ObservableCounter("gcs/reader_count", + metric.WithDescription("The cumulative number of GCS object readers opened or closed."), + metric.WithUnit(""), + metric.WithInt64Callback(func(_ context.Context, obsrv metric.Int64Observer) error { + conditionallyObserve(obsrv, &gcsReaderCountIoMethodClosedAtomic, gcsReaderCountIoMethodClosedAttrSet) + conditionallyObserve(obsrv, &gcsReaderCountIoMethodOpenedAtomic, gcsReaderCountIoMethodOpenedAttrSet) + return nil + })) + + _, err13 := meter.Int64ObservableCounter("gcs/request_count", + metric.WithDescription("The cumulative number of GCS requests processed along with the GCS method."), + metric.WithUnit(""), + metric.WithInt64Callback(func(_ context.Context, obsrv metric.Int64Observer) error { + conditionallyObserve(obsrv, &gcsRequestCountGcsMethodComposeObjectsAtomic, gcsRequestCountGcsMethodComposeObjectsAttrSet) + conditionallyObserve(obsrv, &gcsRequestCountGcsMethodCopyObjectAtomic, gcsRequestCountGcsMethodCopyObjectAttrSet) + conditionallyObserve(obsrv, &gcsRequestCountGcsMethodCreateAppendableObjectWriterAtomic, gcsRequestCountGcsMethodCreateAppendableObjectWriterAttrSet) + conditionallyObserve(obsrv, &gcsRequestCountGcsMethodCreateFolderAtomic, gcsRequestCountGcsMethodCreateFolderAttrSet) + conditionallyObserve(obsrv, &gcsRequestCountGcsMethodCreateObjectAtomic, gcsRequestCountGcsMethodCreateObjectAttrSet) + conditionallyObserve(obsrv, &gcsRequestCountGcsMethodCreateObjectChunkWriterAtomic, gcsRequestCountGcsMethodCreateObjectChunkWriterAttrSet) + conditionallyObserve(obsrv, &gcsRequestCountGcsMethodDeleteFolderAtomic, gcsRequestCountGcsMethodDeleteFolderAttrSet) + conditionallyObserve(obsrv, &gcsRequestCountGcsMethodDeleteObjectAtomic, gcsRequestCountGcsMethodDeleteObjectAttrSet) + conditionallyObserve(obsrv, &gcsRequestCountGcsMethodFinalizeUploadAtomic, gcsRequestCountGcsMethodFinalizeUploadAttrSet) + conditionallyObserve(obsrv, &gcsRequestCountGcsMethodFlushPendingWritesAtomic, gcsRequestCountGcsMethodFlushPendingWritesAttrSet) + conditionallyObserve(obsrv, &gcsRequestCountGcsMethodGetFolderAtomic, gcsRequestCountGcsMethodGetFolderAttrSet) + conditionallyObserve(obsrv, &gcsRequestCountGcsMethodListObjectsAtomic, gcsRequestCountGcsMethodListObjectsAttrSet) + conditionallyObserve(obsrv, &gcsRequestCountGcsMethodMoveObjectAtomic, gcsRequestCountGcsMethodMoveObjectAttrSet) + conditionallyObserve(obsrv, &gcsRequestCountGcsMethodMultiRangeDownloaderAddAtomic, gcsRequestCountGcsMethodMultiRangeDownloaderAddAttrSet) + conditionallyObserve(obsrv, &gcsRequestCountGcsMethodNewMultiRangeDownloaderAtomic, gcsRequestCountGcsMethodNewMultiRangeDownloaderAttrSet) + conditionallyObserve(obsrv, &gcsRequestCountGcsMethodNewReaderAtomic, gcsRequestCountGcsMethodNewReaderAttrSet) + conditionallyObserve(obsrv, &gcsRequestCountGcsMethodRenameFolderAtomic, gcsRequestCountGcsMethodRenameFolderAttrSet) + conditionallyObserve(obsrv, &gcsRequestCountGcsMethodStatObjectAtomic, gcsRequestCountGcsMethodStatObjectAttrSet) + conditionallyObserve(obsrv, &gcsRequestCountGcsMethodUpdateObjectAtomic, gcsRequestCountGcsMethodUpdateObjectAttrSet) + return nil + })) + + gcsRequestLatencies, err14 := meter.Int64Histogram("gcs/request_latencies", + metric.WithDescription("The cumulative distribution of the GCS request latencies."), + metric.WithUnit("ms"), + metric.WithExplicitBucketBoundaries(100, 200, 400, 800, 1500, 3000, 5000, 10000, 20000, 50000, 100000, 200000, 500000)) + + _, err15 := meter.Int64ObservableCounter("gcs/retry_count", + metric.WithDescription("The cumulative number of retry requests made to GCS."), + metric.WithUnit(""), + metric.WithInt64Callback(func(_ context.Context, obsrv metric.Int64Observer) error { + conditionallyObserve(obsrv, &gcsRetryCountRetryErrorCategoryOTHERERRORSAtomic, gcsRetryCountRetryErrorCategoryOTHERERRORSAttrSet) + conditionallyObserve(obsrv, &gcsRetryCountRetryErrorCategorySTALLEDREADREQUESTAtomic, gcsRetryCountRetryErrorCategorySTALLEDREADREQUESTAttrSet) + return nil + })) + + _, err16 := meter.Int64ObservableCounter("metadata_cache/read_count", + metric.WithDescription("Total number of read requests to the metadata cache. Use attributes to analyze hit/miss ratios, entry types, and specific lookup outcomes (e.g., expiration vs. total absence)."), + metric.WithUnit(""), + metric.WithInt64Callback(func(_ context.Context, obsrv metric.Int64Observer) error { + conditionallyObserve(obsrv, &metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailFoundAtomic, metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailFoundAttrSet) + conditionallyObserve(obsrv, &metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailNotFoundAtomic, metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailNotFoundAttrSet) + conditionallyObserve(obsrv, &metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailTtlExpiredAtomic, metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailTtlExpiredAttrSet) + conditionallyObserve(obsrv, &metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailFoundAtomic, metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailFoundAttrSet) + conditionallyObserve(obsrv, &metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailNotFoundAtomic, metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailNotFoundAttrSet) + conditionallyObserve(obsrv, &metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailTtlExpiredAtomic, metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailTtlExpiredAttrSet) + conditionallyObserve(obsrv, &metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailFoundAtomic, metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailFoundAttrSet) + conditionallyObserve(obsrv, &metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailNotFoundAtomic, metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailNotFoundAttrSet) + conditionallyObserve(obsrv, &metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailTtlExpiredAtomic, metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailTtlExpiredAttrSet) + conditionallyObserve(obsrv, &metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailFoundAtomic, metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailFoundAttrSet) + conditionallyObserve(obsrv, &metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailNotFoundAtomic, metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailNotFoundAttrSet) + conditionallyObserve(obsrv, &metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailTtlExpiredAtomic, metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailTtlExpiredAttrSet) + return nil + })) + + readBlockSizes, err17 := meter.Int64Histogram("read/block_sizes", + metric.WithDescription("The cumulative distribution of read block sizes across different bucket boundaries"), + metric.WithUnit("By"), + metric.WithExplicitBucketBoundaries(0, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576, 2097152, 4194304, 8388608, 16777216, 33554432, 67108864, 134217728)) + + _, err18 := meter.Int64ObservableUpDownCounter("test/updown_counter", + metric.WithDescription("Test metric for updown counters."), + metric.WithUnit(""), + metric.WithInt64Callback(func(_ context.Context, obsrv metric.Int64Observer) error { + observeUpDownCounter(obsrv, &testUpdownCounterAtomic) + return nil + })) + + _, err19 := meter.Int64ObservableUpDownCounter("test/updown_counter_with_attrs", + metric.WithDescription("Test metric for updown counters with attributes."), + metric.WithUnit(""), + metric.WithInt64Callback(func(_ context.Context, obsrv metric.Int64Observer) error { + observeUpDownCounter(obsrv, &testUpdownCounterWithAttrsRequestTypeAttr1Atomic, testUpdownCounterWithAttrsRequestTypeAttr1AttrSet) + observeUpDownCounter(obsrv, &testUpdownCounterWithAttrsRequestTypeAttr2Atomic, testUpdownCounterWithAttrsRequestTypeAttr2AttrSet) + return nil + })) + + errs := []error{err0, err1, err2, err3, err4, err5, err6, err7, err8, err9, err10, err11, err12, err13, err14, err15, err16, err17, err18, err19} + if err := errors.Join(errs...); err != nil { + return nil, err + } + + return &otelMetrics{ + ch: ch, + wg: &wg, + bufferedReadFallbackTriggerCountReasonInsufficientMemoryAtomic: &bufferedReadFallbackTriggerCountReasonInsufficientMemoryAtomic, + bufferedReadFallbackTriggerCountReasonRandomReadDetectedAtomic: &bufferedReadFallbackTriggerCountReasonRandomReadDetectedAtomic, + bufferedReadReadLatency: bufferedReadReadLatency, + fileCacheReadBytesCountReadTypeParallelAtomic: &fileCacheReadBytesCountReadTypeParallelAtomic, + fileCacheReadBytesCountReadTypeRandomAtomic: &fileCacheReadBytesCountReadTypeRandomAtomic, + fileCacheReadBytesCountReadTypeSequentialAtomic: &fileCacheReadBytesCountReadTypeSequentialAtomic, + fileCacheReadBytesCountReadTypeUnknownAtomic: &fileCacheReadBytesCountReadTypeUnknownAtomic, + fileCacheReadCountCacheHitTrueReadTypeParallelAtomic: &fileCacheReadCountCacheHitTrueReadTypeParallelAtomic, + fileCacheReadCountCacheHitTrueReadTypeRandomAtomic: &fileCacheReadCountCacheHitTrueReadTypeRandomAtomic, + fileCacheReadCountCacheHitTrueReadTypeSequentialAtomic: &fileCacheReadCountCacheHitTrueReadTypeSequentialAtomic, + fileCacheReadCountCacheHitTrueReadTypeUnknownAtomic: &fileCacheReadCountCacheHitTrueReadTypeUnknownAtomic, + fileCacheReadCountCacheHitFalseReadTypeParallelAtomic: &fileCacheReadCountCacheHitFalseReadTypeParallelAtomic, + fileCacheReadCountCacheHitFalseReadTypeRandomAtomic: &fileCacheReadCountCacheHitFalseReadTypeRandomAtomic, + fileCacheReadCountCacheHitFalseReadTypeSequentialAtomic: &fileCacheReadCountCacheHitFalseReadTypeSequentialAtomic, + fileCacheReadCountCacheHitFalseReadTypeUnknownAtomic: &fileCacheReadCountCacheHitFalseReadTypeUnknownAtomic, + fileCacheReadLatencies: fileCacheReadLatencies, + fsOpsCountFsOpBatchForgetAtomic: &fsOpsCountFsOpBatchForgetAtomic, + fsOpsCountFsOpCreateFileAtomic: &fsOpsCountFsOpCreateFileAtomic, + fsOpsCountFsOpCreateLinkAtomic: &fsOpsCountFsOpCreateLinkAtomic, + fsOpsCountFsOpCreateSymlinkAtomic: &fsOpsCountFsOpCreateSymlinkAtomic, + fsOpsCountFsOpFlushFileAtomic: &fsOpsCountFsOpFlushFileAtomic, + fsOpsCountFsOpForgetInodeAtomic: &fsOpsCountFsOpForgetInodeAtomic, + fsOpsCountFsOpGetInodeAttributesAtomic: &fsOpsCountFsOpGetInodeAttributesAtomic, + fsOpsCountFsOpLookUpInodeAtomic: &fsOpsCountFsOpLookUpInodeAtomic, + fsOpsCountFsOpMkDirAtomic: &fsOpsCountFsOpMkDirAtomic, + fsOpsCountFsOpMkNodeAtomic: &fsOpsCountFsOpMkNodeAtomic, + fsOpsCountFsOpOpenDirAtomic: &fsOpsCountFsOpOpenDirAtomic, + fsOpsCountFsOpOpenFileAtomic: &fsOpsCountFsOpOpenFileAtomic, + fsOpsCountFsOpOthersAtomic: &fsOpsCountFsOpOthersAtomic, + fsOpsCountFsOpReadDirAtomic: &fsOpsCountFsOpReadDirAtomic, + fsOpsCountFsOpReadDirPlusAtomic: &fsOpsCountFsOpReadDirPlusAtomic, + fsOpsCountFsOpReadFileAtomic: &fsOpsCountFsOpReadFileAtomic, + fsOpsCountFsOpReadSymlinkAtomic: &fsOpsCountFsOpReadSymlinkAtomic, + fsOpsCountFsOpReleaseDirHandleAtomic: &fsOpsCountFsOpReleaseDirHandleAtomic, + fsOpsCountFsOpReleaseFileHandleAtomic: &fsOpsCountFsOpReleaseFileHandleAtomic, + fsOpsCountFsOpRenameAtomic: &fsOpsCountFsOpRenameAtomic, + fsOpsCountFsOpRmDirAtomic: &fsOpsCountFsOpRmDirAtomic, + fsOpsCountFsOpSetInodeAttributesAtomic: &fsOpsCountFsOpSetInodeAttributesAtomic, + fsOpsCountFsOpSyncFileAtomic: &fsOpsCountFsOpSyncFileAtomic, + fsOpsCountFsOpUnlinkAtomic: &fsOpsCountFsOpUnlinkAtomic, + fsOpsCountFsOpWriteFileAtomic: &fsOpsCountFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpBatchForgetAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateFileAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateLinkAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpFlushFileAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpForgetInodeAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpGetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpLookUpInodeAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpMkDirAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpMkNodeAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOpenDirAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOpenFileAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOthersAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadDirAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadDirPlusAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadFileAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReleaseDirHandleAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReleaseFileHandleAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpRenameAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpRmDirAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpSetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpSyncFileAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpUnlinkAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpWriteFileAtomic: &fsOpsErrorCountFsErrorCategoryDEVICEERRORFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpBatchForgetAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateFileAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateLinkAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpFlushFileAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpForgetInodeAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpGetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpLookUpInodeAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpMkDirAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpMkNodeAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOpenDirAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOpenFileAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOthersAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadDirAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadDirPlusAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadFileAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReleaseDirHandleAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReleaseFileHandleAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpRenameAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpRmDirAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpSetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpSyncFileAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpUnlinkAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpWriteFileAtomic: &fsOpsErrorCountFsErrorCategoryDIRNOTEMPTYFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpBatchForgetAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateFileAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateLinkAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpFlushFileAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpForgetInodeAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpGetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpLookUpInodeAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpMkDirAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpMkNodeAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOpenDirAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOpenFileAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOthersAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadDirAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadDirPlusAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadFileAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReleaseDirHandleAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReleaseFileHandleAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpRenameAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpRmDirAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpSetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpSyncFileAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpUnlinkAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpWriteFileAtomic: &fsOpsErrorCountFsErrorCategoryFILEDIRERRORFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpBatchForgetAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateFileAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateLinkAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpFlushFileAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpForgetInodeAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpGetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpLookUpInodeAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpMkDirAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpMkNodeAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOpenDirAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOpenFileAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOthersAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadDirAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadDirPlusAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadFileAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReleaseDirHandleAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReleaseFileHandleAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpRenameAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpRmDirAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpSetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpSyncFileAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpUnlinkAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpWriteFileAtomic: &fsOpsErrorCountFsErrorCategoryFILEEXISTSFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpBatchForgetAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateFileAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateLinkAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpFlushFileAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpForgetInodeAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpGetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpLookUpInodeAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpMkDirAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpMkNodeAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOpenDirAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOpenFileAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOthersAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadDirAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadDirPlusAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadFileAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReleaseDirHandleAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReleaseFileHandleAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpRenameAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpRmDirAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpSetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpSyncFileAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpUnlinkAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpWriteFileAtomic: &fsOpsErrorCountFsErrorCategoryINTERRUPTERRORFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpBatchForgetAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateFileAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateLinkAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpFlushFileAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpForgetInodeAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpGetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpLookUpInodeAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpMkDirAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpMkNodeAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOpenDirAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOpenFileAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOthersAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadDirAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadDirPlusAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadFileAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReleaseDirHandleAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReleaseFileHandleAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpRenameAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpRmDirAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpSetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpSyncFileAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpUnlinkAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpWriteFileAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDARGUMENTFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpBatchForgetAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateFileAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateLinkAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpFlushFileAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpForgetInodeAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpGetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpLookUpInodeAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpMkDirAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpMkNodeAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOpenDirAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOpenFileAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOthersAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadDirAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadDirPlusAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadFileAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReleaseDirHandleAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReleaseFileHandleAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpRenameAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpRmDirAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpSetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpSyncFileAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpUnlinkAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpWriteFileAtomic: &fsOpsErrorCountFsErrorCategoryINVALIDOPERATIONFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpBatchForgetAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateFileAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateLinkAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpFlushFileAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpForgetInodeAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpGetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpLookUpInodeAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpMkDirAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpMkNodeAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpOpenDirAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpOpenFileAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpOthersAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadDirAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadDirPlusAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadFileAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReleaseDirHandleAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpReleaseFileHandleAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpRenameAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpRmDirAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpSetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpSyncFileAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpUnlinkAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryIOERRORFsOpWriteFileAtomic: &fsOpsErrorCountFsErrorCategoryIOERRORFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpBatchForgetAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateFileAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateLinkAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpFlushFileAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpForgetInodeAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpGetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpLookUpInodeAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpMkDirAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpMkNodeAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOpenDirAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOpenFileAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOthersAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadDirAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadDirPlusAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadFileAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReleaseDirHandleAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReleaseFileHandleAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpRenameAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpRmDirAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpSetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpSyncFileAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpUnlinkAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryMISCERRORFsOpWriteFileAtomic: &fsOpsErrorCountFsErrorCategoryMISCERRORFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpBatchForgetAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateFileAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateLinkAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpFlushFileAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpForgetInodeAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpGetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpLookUpInodeAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpMkDirAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpMkNodeAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOpenDirAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOpenFileAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOthersAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadDirAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadDirPlusAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadFileAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReleaseDirHandleAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReleaseFileHandleAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpRenameAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpRmDirAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpSetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpSyncFileAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpUnlinkAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpWriteFileAtomic: &fsOpsErrorCountFsErrorCategoryNETWORKERRORFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpBatchForgetAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateFileAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateLinkAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpFlushFileAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpForgetInodeAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpGetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpLookUpInodeAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpMkDirAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpMkNodeAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOpenDirAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOpenFileAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOthersAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadDirAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadDirPlusAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadFileAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReleaseDirHandleAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReleaseFileHandleAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpRenameAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpRmDirAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpSetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpSyncFileAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpUnlinkAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryNOTADIRFsOpWriteFileAtomic: &fsOpsErrorCountFsErrorCategoryNOTADIRFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpBatchForgetAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateFileAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateLinkAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpFlushFileAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpForgetInodeAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpGetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpLookUpInodeAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpMkDirAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpMkNodeAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOpenDirAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOpenFileAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOthersAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadDirAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadDirPlusAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadFileAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReleaseDirHandleAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReleaseFileHandleAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpRenameAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpRmDirAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpSetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpSyncFileAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpUnlinkAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpWriteFileAtomic: &fsOpsErrorCountFsErrorCategoryNOTIMPLEMENTEDFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpBatchForgetAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateFileAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateLinkAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpFlushFileAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpForgetInodeAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpGetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpLookUpInodeAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpMkDirAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpMkNodeAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOpenDirAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOpenFileAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOthersAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadDirAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadDirPlusAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadFileAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReleaseDirHandleAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReleaseFileHandleAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpRenameAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpRmDirAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpSetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpSyncFileAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpUnlinkAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpWriteFileAtomic: &fsOpsErrorCountFsErrorCategoryNOFILEORDIRFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpBatchForgetAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateFileAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateLinkAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpFlushFileAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpForgetInodeAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpGetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpLookUpInodeAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpMkDirAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpMkNodeAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOpenDirAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOpenFileAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOthersAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadDirAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadDirPlusAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadFileAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReleaseDirHandleAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReleaseFileHandleAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpRenameAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpRmDirAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpSetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpSyncFileAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpUnlinkAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryPERMERRORFsOpWriteFileAtomic: &fsOpsErrorCountFsErrorCategoryPERMERRORFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpBatchForgetAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateFileAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateLinkAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpFlushFileAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpForgetInodeAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpGetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpLookUpInodeAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpMkDirAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpMkNodeAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOpenDirAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOpenFileAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOthersAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadDirAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadDirPlusAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadFileAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReleaseDirHandleAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReleaseFileHandleAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpRenameAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpRmDirAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpSetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpSyncFileAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpUnlinkAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpWriteFileAtomic: &fsOpsErrorCountFsErrorCategoryPROCESSRESOURCEMGMTERRORFsOpWriteFileAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpBatchForgetAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpBatchForgetAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateFileAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateFileAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateLinkAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateLinkAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpCreateSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpFlushFileAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpFlushFileAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpForgetInodeAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpForgetInodeAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpGetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpGetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpLookUpInodeAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpLookUpInodeAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpMkDirAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpMkDirAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpMkNodeAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpMkNodeAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOpenDirAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOpenDirAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOpenFileAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOpenFileAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOthersAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpOthersAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadDirAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadDirAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadDirPlusAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadDirPlusAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadFileAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadFileAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadSymlinkAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReadSymlinkAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReleaseDirHandleAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReleaseDirHandleAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReleaseFileHandleAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpReleaseFileHandleAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpRenameAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpRenameAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpRmDirAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpRmDirAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpSetInodeAttributesAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpSetInodeAttributesAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpSyncFileAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpSyncFileAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpUnlinkAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpUnlinkAtomic, + fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpWriteFileAtomic: &fsOpsErrorCountFsErrorCategoryTOOMANYOPENFILESFsOpWriteFileAtomic, + fsOpsLatency: fsOpsLatency, + fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonConcurrencyLimitBreachedAtomic: &fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonConcurrencyLimitBreachedAtomic, + fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonExistingFileAtomic: &fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonExistingFileAtomic, + fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonOtherAtomic: &fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonOtherAtomic, + fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonOutOfOrderAtomic: &fsStreamingWriteFallbackCountOpenModeOtherWriteFallbackReasonOutOfOrderAtomic, + fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonConcurrencyLimitBreachedAtomic: &fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonConcurrencyLimitBreachedAtomic, + fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonExistingFileAtomic: &fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonExistingFileAtomic, + fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonOtherAtomic: &fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonOtherAtomic, + fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonOutOfOrderAtomic: &fsStreamingWriteFallbackCountOpenModeReadWriteWriteFallbackReasonOutOfOrderAtomic, + fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonConcurrencyLimitBreachedAtomic: &fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonConcurrencyLimitBreachedAtomic, + fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonExistingFileAtomic: &fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonExistingFileAtomic, + fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonOtherAtomic: &fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonOtherAtomic, + fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonOutOfOrderAtomic: &fsStreamingWriteFallbackCountOpenModeReadWriteAppendWriteFallbackReasonOutOfOrderAtomic, + fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonConcurrencyLimitBreachedAtomic: &fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonConcurrencyLimitBreachedAtomic, + fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonExistingFileAtomic: &fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonExistingFileAtomic, + fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonOtherAtomic: &fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonOtherAtomic, + fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonOutOfOrderAtomic: &fsStreamingWriteFallbackCountOpenModeWriteOnlyWriteFallbackReasonOutOfOrderAtomic, + fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonConcurrencyLimitBreachedAtomic: &fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonConcurrencyLimitBreachedAtomic, + fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonExistingFileAtomic: &fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonExistingFileAtomic, + fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonOtherAtomic: &fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonOtherAtomic, + fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonOutOfOrderAtomic: &fsStreamingWriteFallbackCountOpenModeWriteOnlyAppendWriteFallbackReasonOutOfOrderAtomic, + gcsDownloadBytesCountReadTypeBufferedAtomic: &gcsDownloadBytesCountReadTypeBufferedAtomic, + gcsDownloadBytesCountReadTypeParallelAtomic: &gcsDownloadBytesCountReadTypeParallelAtomic, + gcsDownloadBytesCountReadTypeRandomAtomic: &gcsDownloadBytesCountReadTypeRandomAtomic, + gcsDownloadBytesCountReadTypeSequentialAtomic: &gcsDownloadBytesCountReadTypeSequentialAtomic, + gcsReadBytesCountAtomic: &gcsReadBytesCountAtomic, + gcsReadCountReadTypeParallelAtomic: &gcsReadCountReadTypeParallelAtomic, + gcsReadCountReadTypeRandomAtomic: &gcsReadCountReadTypeRandomAtomic, + gcsReadCountReadTypeSequentialAtomic: &gcsReadCountReadTypeSequentialAtomic, + gcsReadCountReadTypeUnknownAtomic: &gcsReadCountReadTypeUnknownAtomic, + gcsReaderCountIoMethodClosedAtomic: &gcsReaderCountIoMethodClosedAtomic, + gcsReaderCountIoMethodOpenedAtomic: &gcsReaderCountIoMethodOpenedAtomic, + gcsRequestCountGcsMethodComposeObjectsAtomic: &gcsRequestCountGcsMethodComposeObjectsAtomic, + gcsRequestCountGcsMethodCopyObjectAtomic: &gcsRequestCountGcsMethodCopyObjectAtomic, + gcsRequestCountGcsMethodCreateAppendableObjectWriterAtomic: &gcsRequestCountGcsMethodCreateAppendableObjectWriterAtomic, + gcsRequestCountGcsMethodCreateFolderAtomic: &gcsRequestCountGcsMethodCreateFolderAtomic, + gcsRequestCountGcsMethodCreateObjectAtomic: &gcsRequestCountGcsMethodCreateObjectAtomic, + gcsRequestCountGcsMethodCreateObjectChunkWriterAtomic: &gcsRequestCountGcsMethodCreateObjectChunkWriterAtomic, + gcsRequestCountGcsMethodDeleteFolderAtomic: &gcsRequestCountGcsMethodDeleteFolderAtomic, + gcsRequestCountGcsMethodDeleteObjectAtomic: &gcsRequestCountGcsMethodDeleteObjectAtomic, + gcsRequestCountGcsMethodFinalizeUploadAtomic: &gcsRequestCountGcsMethodFinalizeUploadAtomic, + gcsRequestCountGcsMethodFlushPendingWritesAtomic: &gcsRequestCountGcsMethodFlushPendingWritesAtomic, + gcsRequestCountGcsMethodGetFolderAtomic: &gcsRequestCountGcsMethodGetFolderAtomic, + gcsRequestCountGcsMethodListObjectsAtomic: &gcsRequestCountGcsMethodListObjectsAtomic, + gcsRequestCountGcsMethodMoveObjectAtomic: &gcsRequestCountGcsMethodMoveObjectAtomic, + gcsRequestCountGcsMethodMultiRangeDownloaderAddAtomic: &gcsRequestCountGcsMethodMultiRangeDownloaderAddAtomic, + gcsRequestCountGcsMethodNewMultiRangeDownloaderAtomic: &gcsRequestCountGcsMethodNewMultiRangeDownloaderAtomic, + gcsRequestCountGcsMethodNewReaderAtomic: &gcsRequestCountGcsMethodNewReaderAtomic, + gcsRequestCountGcsMethodRenameFolderAtomic: &gcsRequestCountGcsMethodRenameFolderAtomic, + gcsRequestCountGcsMethodStatObjectAtomic: &gcsRequestCountGcsMethodStatObjectAtomic, + gcsRequestCountGcsMethodUpdateObjectAtomic: &gcsRequestCountGcsMethodUpdateObjectAtomic, + gcsRequestLatencies: gcsRequestLatencies, + gcsRetryCountRetryErrorCategoryOTHERERRORSAtomic: &gcsRetryCountRetryErrorCategoryOTHERERRORSAtomic, + gcsRetryCountRetryErrorCategorySTALLEDREADREQUESTAtomic: &gcsRetryCountRetryErrorCategorySTALLEDREADREQUESTAtomic, + metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailFoundAtomic: &metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailFoundAtomic, + metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailNotFoundAtomic: &metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailNotFoundAtomic, + metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailTtlExpiredAtomic: &metadataCacheReadCountCacheHitTrueEntryStatusNegativeLookupDetailTtlExpiredAtomic, + metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailFoundAtomic: &metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailFoundAtomic, + metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailNotFoundAtomic: &metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailNotFoundAtomic, + metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailTtlExpiredAtomic: &metadataCacheReadCountCacheHitTrueEntryStatusPositiveLookupDetailTtlExpiredAtomic, + metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailFoundAtomic: &metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailFoundAtomic, + metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailNotFoundAtomic: &metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailNotFoundAtomic, + metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailTtlExpiredAtomic: &metadataCacheReadCountCacheHitFalseEntryStatusNegativeLookupDetailTtlExpiredAtomic, + metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailFoundAtomic: &metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailFoundAtomic, + metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailNotFoundAtomic: &metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailNotFoundAtomic, + metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailTtlExpiredAtomic: &metadataCacheReadCountCacheHitFalseEntryStatusPositiveLookupDetailTtlExpiredAtomic, + readBlockSizes: readBlockSizes, + testUpdownCounterAtomic: &testUpdownCounterAtomic, + testUpdownCounterWithAttrsRequestTypeAttr1Atomic: &testUpdownCounterWithAttrsRequestTypeAttr1Atomic, + testUpdownCounterWithAttrsRequestTypeAttr2Atomic: &testUpdownCounterWithAttrsRequestTypeAttr2Atomic, + }, nil +} + +func (o *otelMetrics) Close() { + close(o.ch) + o.wg.Wait() +} + +func conditionallyObserve(obsrv metric.Int64Observer, counter *atomic.Int64, obsrvOptions ...metric.ObserveOption) { + if val := counter.Load(); val > 0 { + obsrv.Observe(val, obsrvOptions...) + } +} + +func observeUpDownCounter(obsrv metric.Int64Observer, counter *atomic.Int64, obsrvOptions ...metric.ObserveOption) { + obsrv.Observe(counter.Load(), obsrvOptions...) +} + +func updateUnrecognizedAttribute(newValue string) { + unrecognizedAttr.CompareAndSwap("", newValue) +} + +// StartSampledLogging starts a goroutine that logs unrecognized attributes periodically. +func startSampledLogging(ctx context.Context) { + // Init the atomic.Value + unrecognizedAttr.Store("") + + go func() { + ticker := time.NewTicker(logInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + logUnrecognizedAttribute() + } + } + }() +} + +// logUnrecognizedAttribute retrieves and logs any unrecognized attributes. +func logUnrecognizedAttribute() { + // Atomically load and reset the attribute name, then generate a log + // if an unrecognized attribute was encountered. + if currentAttr := unrecognizedAttr.Swap("").(string); currentAttr != "" { + logger.Tracef("Attribute %s is not declared", currentAttr) + } +} diff --git a/metrics/otel_metrics_test.go b/metrics/otel_metrics_test.go new file mode 100644 index 0000000000..4367d34d07 --- /dev/null +++ b/metrics/otel_metrics_test.go @@ -0,0 +1,5772 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// **** DO NOT EDIT - FILE IS AUTO-GENERATED **** + +package metrics + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +// metricValueMap maps attribute sets to metric values. +type metricValueMap map[string]int64 + +// metricHistogramMap maps attribute sets to histogram data points. +type metricHistogramMap map[string]metricdata.HistogramDataPoint[int64] + +func waitForMetricsProcessing() { + time.Sleep(5 * time.Millisecond) +} + +func setupOTel(ctx context.Context, t *testing.T) (*otelMetrics, *metric.ManualReader) { + t.Helper() + reader := metric.NewManualReader() + provider := metric.NewMeterProvider(metric.WithReader(reader)) + otel.SetMeterProvider(provider) + + m, err := NewOTelMetrics(ctx, 10, 100) + require.NoError(t, err) + return m, reader +} + +// gatherHistogramMetrics collects all histogram metrics from the reader. +// It returns a map where the key is the metric name, and the value is another map. +// The inner map's key is a string representation of the attributes, +// and the value is the metricdata.HistogramDataPoint. +func gatherHistogramMetrics(ctx context.Context, t *testing.T, rd *metric.ManualReader) map[string]map[string]metricdata.HistogramDataPoint[int64] { + t.Helper() + var rm metricdata.ResourceMetrics + err := rd.Collect(ctx, &rm) + require.NoError(t, err) + + results := make(map[string]map[string]metricdata.HistogramDataPoint[int64]) + encoder := attribute.DefaultEncoder() // Using default encoder + + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + // We are interested in Histogram[int64]. + hist, ok := m.Data.(metricdata.Histogram[int64]) + if !ok { + continue + } + + metricMap := make(metricHistogramMap) + for _, dp := range hist.DataPoints { + if dp.Count == 0 { + continue + } + + metricMap[dp.Attributes.Encoded(encoder)] = dp + } + + if len(metricMap) > 0 { + results[m.Name] = metricMap + } + } + } + + return results +} + +// gatherNonZeroCounterMetrics collects all non-zero counter metrics from the reader. +// It returns a map where the key is the metric name, and the value is another map. +// The inner map's key is a string representation of the attributes, +// and the value is the metric's value. +func gatherNonZeroCounterMetrics(ctx context.Context, t *testing.T, rd *metric.ManualReader) map[string]map[string]int64 { + t.Helper() + var rm metricdata.ResourceMetrics + err := rd.Collect(ctx, &rm) + require.NoError(t, err) + + results := make(map[string]map[string]int64) + encoder := attribute.DefaultEncoder() + + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + // We are interested in Sum[int64] which corresponds to int_counter. + sum, ok := m.Data.(metricdata.Sum[int64]) + if !ok { + continue + } + + metricMap := make(metricValueMap) + for _, dp := range sum.DataPoints { + if dp.Value == 0 { + continue + } + + metricMap[dp.Attributes.Encoded(encoder)] = dp.Value + } + + if len(metricMap) > 0 { + results[m.Name] = metricMap + } + } + } + + return results +} + +func TestBufferedReadFallbackTriggerCount(t *testing.T) { + tests := []struct { + name string + f func(m *otelMetrics) + expected map[attribute.Set]int64 + }{ + { + name: "reason_insufficient_memory", + f: func(m *otelMetrics) { + m.BufferedReadFallbackTriggerCount(5, "insufficient_memory") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("reason", "insufficient_memory")): 5, + }, + }, + { + name: "reason_random_read_detected", + f: func(m *otelMetrics) { + m.BufferedReadFallbackTriggerCount(5, "random_read_detected") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("reason", "random_read_detected")): 5, + }, + }, { + name: "multiple_attributes_summed", + f: func(m *otelMetrics) { + m.BufferedReadFallbackTriggerCount(5, "insufficient_memory") + m.BufferedReadFallbackTriggerCount(2, "random_read_detected") + m.BufferedReadFallbackTriggerCount(3, "insufficient_memory") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("reason", "insufficient_memory")): 8, + attribute.NewSet(attribute.String("reason", "random_read_detected")): 2, + }, + }, + { + name: "negative_increment", + f: func(m *otelMetrics) { + m.BufferedReadFallbackTriggerCount(-5, "insufficient_memory") + m.BufferedReadFallbackTriggerCount(2, "insufficient_memory") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("reason", "insufficient_memory")): 2}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + + tc.f(m) + waitForMetricsProcessing() + + metrics := gatherNonZeroCounterMetrics(ctx, t, rd) + metric, ok := metrics["buffered_read/fallback_trigger_count"] + if len(tc.expected) == 0 { + assert.False(t, ok, "buffered_read/fallback_trigger_count metric should not be found") + return + } + require.True(t, ok, "buffered_read/fallback_trigger_count metric not found") + expectedMap := make(map[string]int64) + for k, v := range tc.expected { + expectedMap[k.Encoded(encoder)] = v + } + assert.Equal(t, expectedMap, metric) + }) + } +} + +func TestBufferedReadReadLatency(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + var totalLatency time.Duration + latencies := []time.Duration{100 * time.Microsecond, 200 * time.Microsecond} + + for _, latency := range latencies { + m.BufferedReadReadLatency(ctx, latency) + totalLatency += latency + } + waitForMetricsProcessing() + + metrics := gatherHistogramMetrics(ctx, t, rd) + metric, ok := metrics["buffered_read/read_latency"] + require.True(t, ok, "buffered_read/read_latency metric not found") + + s := attribute.NewSet() + expectedKey := s.Encoded(encoder) + dp, ok := metric[expectedKey] + require.True(t, ok, "DataPoint not found for key: %s", expectedKey) + assert.Equal(t, uint64(len(latencies)), dp.Count) + assert.Equal(t, totalLatency.Microseconds(), dp.Sum) +} + +func TestFileCacheReadBytesCount(t *testing.T) { + tests := []struct { + name string + f func(m *otelMetrics) + expected map[attribute.Set]int64 + }{ + { + name: "read_type_Parallel", + f: func(m *otelMetrics) { + m.FileCacheReadBytesCount(5, "Parallel") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("read_type", "Parallel")): 5, + }, + }, + { + name: "read_type_Random", + f: func(m *otelMetrics) { + m.FileCacheReadBytesCount(5, "Random") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("read_type", "Random")): 5, + }, + }, + { + name: "read_type_Sequential", + f: func(m *otelMetrics) { + m.FileCacheReadBytesCount(5, "Sequential") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("read_type", "Sequential")): 5, + }, + }, + { + name: "read_type_Unknown", + f: func(m *otelMetrics) { + m.FileCacheReadBytesCount(5, "Unknown") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("read_type", "Unknown")): 5, + }, + }, { + name: "multiple_attributes_summed", + f: func(m *otelMetrics) { + m.FileCacheReadBytesCount(5, "Parallel") + m.FileCacheReadBytesCount(2, "Random") + m.FileCacheReadBytesCount(3, "Parallel") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("read_type", "Parallel")): 8, + attribute.NewSet(attribute.String("read_type", "Random")): 2, + }, + }, + { + name: "negative_increment", + f: func(m *otelMetrics) { + m.FileCacheReadBytesCount(-5, "Parallel") + m.FileCacheReadBytesCount(2, "Parallel") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("read_type", "Parallel")): 2}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + + tc.f(m) + waitForMetricsProcessing() + + metrics := gatherNonZeroCounterMetrics(ctx, t, rd) + metric, ok := metrics["file_cache/read_bytes_count"] + if len(tc.expected) == 0 { + assert.False(t, ok, "file_cache/read_bytes_count metric should not be found") + return + } + require.True(t, ok, "file_cache/read_bytes_count metric not found") + expectedMap := make(map[string]int64) + for k, v := range tc.expected { + expectedMap[k.Encoded(encoder)] = v + } + assert.Equal(t, expectedMap, metric) + }) + } +} + +func TestFileCacheReadCount(t *testing.T) { + tests := []struct { + name string + f func(m *otelMetrics) + expected map[attribute.Set]int64 + }{ + { + name: "cache_hit_true_read_type_Parallel", + f: func(m *otelMetrics) { + m.FileCacheReadCount(5, true, "Parallel") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("read_type", "Parallel")): 5, + }, + }, + { + name: "cache_hit_true_read_type_Random", + f: func(m *otelMetrics) { + m.FileCacheReadCount(5, true, "Random") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("read_type", "Random")): 5, + }, + }, + { + name: "cache_hit_true_read_type_Sequential", + f: func(m *otelMetrics) { + m.FileCacheReadCount(5, true, "Sequential") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("read_type", "Sequential")): 5, + }, + }, + { + name: "cache_hit_true_read_type_Unknown", + f: func(m *otelMetrics) { + m.FileCacheReadCount(5, true, "Unknown") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("read_type", "Unknown")): 5, + }, + }, + { + name: "cache_hit_false_read_type_Parallel", + f: func(m *otelMetrics) { + m.FileCacheReadCount(5, false, "Parallel") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("read_type", "Parallel")): 5, + }, + }, + { + name: "cache_hit_false_read_type_Random", + f: func(m *otelMetrics) { + m.FileCacheReadCount(5, false, "Random") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("read_type", "Random")): 5, + }, + }, + { + name: "cache_hit_false_read_type_Sequential", + f: func(m *otelMetrics) { + m.FileCacheReadCount(5, false, "Sequential") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("read_type", "Sequential")): 5, + }, + }, + { + name: "cache_hit_false_read_type_Unknown", + f: func(m *otelMetrics) { + m.FileCacheReadCount(5, false, "Unknown") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("read_type", "Unknown")): 5, + }, + }, { + name: "multiple_attributes_summed", + f: func(m *otelMetrics) { + m.FileCacheReadCount(5, true, "Parallel") + m.FileCacheReadCount(2, true, "Random") + m.FileCacheReadCount(3, true, "Parallel") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("read_type", "Parallel")): 8, + attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("read_type", "Random")): 2, + }, + }, + { + name: "negative_increment", + f: func(m *otelMetrics) { + m.FileCacheReadCount(-5, true, "Parallel") + m.FileCacheReadCount(2, true, "Parallel") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("read_type", "Parallel")): 2}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + + tc.f(m) + waitForMetricsProcessing() + + metrics := gatherNonZeroCounterMetrics(ctx, t, rd) + metric, ok := metrics["file_cache/read_count"] + if len(tc.expected) == 0 { + assert.False(t, ok, "file_cache/read_count metric should not be found") + return + } + require.True(t, ok, "file_cache/read_count metric not found") + expectedMap := make(map[string]int64) + for k, v := range tc.expected { + expectedMap[k.Encoded(encoder)] = v + } + assert.Equal(t, expectedMap, metric) + }) + } +} + +func TestFileCacheReadLatencies(t *testing.T) { + tests := []struct { + name string + latencies []time.Duration + cacheHit bool + }{ + { + name: "cache_hit_true", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + cacheHit: true, + }, + { + name: "cache_hit_false", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + cacheHit: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + var totalLatency time.Duration + + for _, latency := range tc.latencies { + m.FileCacheReadLatencies(ctx, latency, tc.cacheHit) + totalLatency += latency + } + waitForMetricsProcessing() + + metrics := gatherHistogramMetrics(ctx, t, rd) + metric, ok := metrics["file_cache/read_latencies"] + require.True(t, ok, "file_cache/read_latencies metric not found") + + attrs := []attribute.KeyValue{ + attribute.Bool("cache_hit", tc.cacheHit), + } + s := attribute.NewSet(attrs...) + expectedKey := s.Encoded(encoder) + dp, ok := metric[expectedKey] + require.True(t, ok, "DataPoint not found for key: %s", expectedKey) + assert.Equal(t, uint64(len(tc.latencies)), dp.Count) + assert.Equal(t, totalLatency.Microseconds(), dp.Sum) + }) + } +} + +func TestFsOpsCount(t *testing.T) { + tests := []struct { + name string + f func(m *otelMetrics) + expected map[attribute.Set]int64 + }{ + { + name: "fs_op_BatchForget", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "BatchForget") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "BatchForget")): 5, + }, + }, + { + name: "fs_op_CreateFile", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "CreateFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "CreateFile")): 5, + }, + }, + { + name: "fs_op_CreateLink", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "CreateLink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "CreateLink")): 5, + }, + }, + { + name: "fs_op_CreateSymlink", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "CreateSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "CreateSymlink")): 5, + }, + }, + { + name: "fs_op_FlushFile", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "FlushFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "FlushFile")): 5, + }, + }, + { + name: "fs_op_ForgetInode", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "ForgetInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "ForgetInode")): 5, + }, + }, + { + name: "fs_op_GetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "GetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "GetInodeAttributes")): 5, + }, + }, + { + name: "fs_op_LookUpInode", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "LookUpInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "LookUpInode")): 5, + }, + }, + { + name: "fs_op_MkDir", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "MkDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "MkDir")): 5, + }, + }, + { + name: "fs_op_MkNode", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "MkNode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "MkNode")): 5, + }, + }, + { + name: "fs_op_OpenDir", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "OpenDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "OpenDir")): 5, + }, + }, + { + name: "fs_op_OpenFile", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "OpenFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "OpenFile")): 5, + }, + }, + { + name: "fs_op_Others", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "Others") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "Others")): 5, + }, + }, + { + name: "fs_op_ReadDir", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "ReadDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "ReadDir")): 5, + }, + }, + { + name: "fs_op_ReadDirPlus", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "ReadDirPlus") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "ReadDirPlus")): 5, + }, + }, + { + name: "fs_op_ReadFile", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "ReadFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "ReadFile")): 5, + }, + }, + { + name: "fs_op_ReadSymlink", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "ReadSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "ReadSymlink")): 5, + }, + }, + { + name: "fs_op_ReleaseDirHandle", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "ReleaseDirHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "ReleaseDirHandle")): 5, + }, + }, + { + name: "fs_op_ReleaseFileHandle", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "ReleaseFileHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "ReleaseFileHandle")): 5, + }, + }, + { + name: "fs_op_Rename", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "Rename") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "Rename")): 5, + }, + }, + { + name: "fs_op_RmDir", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "RmDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "RmDir")): 5, + }, + }, + { + name: "fs_op_SetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "SetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "SetInodeAttributes")): 5, + }, + }, + { + name: "fs_op_SyncFile", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "SyncFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "SyncFile")): 5, + }, + }, + { + name: "fs_op_Unlink", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "Unlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "Unlink")): 5, + }, + }, + { + name: "fs_op_WriteFile", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "WriteFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_op", "WriteFile")): 5, + }, + }, { + name: "multiple_attributes_summed", + f: func(m *otelMetrics) { + m.FsOpsCount(5, "BatchForget") + m.FsOpsCount(2, "CreateFile") + m.FsOpsCount(3, "BatchForget") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("fs_op", "BatchForget")): 8, + attribute.NewSet(attribute.String("fs_op", "CreateFile")): 2, + }, + }, + { + name: "negative_increment", + f: func(m *otelMetrics) { + m.FsOpsCount(-5, "BatchForget") + m.FsOpsCount(2, "BatchForget") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("fs_op", "BatchForget")): 2}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + + tc.f(m) + waitForMetricsProcessing() + + metrics := gatherNonZeroCounterMetrics(ctx, t, rd) + metric, ok := metrics["fs/ops_count"] + if len(tc.expected) == 0 { + assert.False(t, ok, "fs/ops_count metric should not be found") + return + } + require.True(t, ok, "fs/ops_count metric not found") + expectedMap := make(map[string]int64) + for k, v := range tc.expected { + expectedMap[k.Encoded(encoder)] = v + } + assert.Equal(t, expectedMap, metric) + }) + } +} + +func TestFsOpsErrorCount(t *testing.T) { + tests := []struct { + name string + f func(m *otelMetrics) + expected map[attribute.Set]int64 + }{ + { + name: "fs_error_category_DEVICE_ERROR_fs_op_BatchForget", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "BatchForget") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "BatchForget")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_CreateFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "CreateFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "CreateFile")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_CreateLink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "CreateLink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "CreateLink")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_CreateSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "CreateSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "CreateSymlink")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_FlushFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "FlushFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "FlushFile")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_ForgetInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "ForgetInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "ForgetInode")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_GetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "GetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "GetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_LookUpInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "LookUpInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "LookUpInode")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_MkDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "MkDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "MkDir")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_MkNode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "MkNode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "MkNode")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_OpenDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "OpenDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "OpenDir")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_OpenFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "OpenFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "OpenFile")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_Others", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "Others") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "Others")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_ReadDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "ReadDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "ReadDir")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_ReadDirPlus", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "ReadDirPlus") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "ReadDirPlus")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_ReadFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "ReadFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "ReadFile")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_ReadSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "ReadSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "ReadSymlink")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_ReleaseDirHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "ReleaseDirHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "ReleaseDirHandle")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_ReleaseFileHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "ReleaseFileHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "ReleaseFileHandle")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_Rename", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "Rename") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "Rename")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_RmDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "RmDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "RmDir")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_SetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "SetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "SetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_SyncFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "SyncFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "SyncFile")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_Unlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "Unlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "Unlink")): 5, + }, + }, + { + name: "fs_error_category_DEVICE_ERROR_fs_op_WriteFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "WriteFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "WriteFile")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_BatchForget", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "BatchForget") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "BatchForget")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_CreateFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "CreateFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "CreateFile")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_CreateLink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "CreateLink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "CreateLink")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_CreateSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "CreateSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "CreateSymlink")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_FlushFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "FlushFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "FlushFile")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_ForgetInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "ForgetInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "ForgetInode")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_GetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "GetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "GetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_LookUpInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "LookUpInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "LookUpInode")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_MkDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "MkDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "MkDir")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_MkNode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "MkNode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "MkNode")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_OpenDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "OpenDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "OpenDir")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_OpenFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "OpenFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "OpenFile")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_Others", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "Others") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "Others")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_ReadDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "ReadDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "ReadDir")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_ReadDirPlus", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "ReadDirPlus") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "ReadDirPlus")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_ReadFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "ReadFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "ReadFile")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_ReadSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "ReadSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "ReadSymlink")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_ReleaseDirHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "ReleaseDirHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "ReleaseDirHandle")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_ReleaseFileHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "ReleaseFileHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "ReleaseFileHandle")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_Rename", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "Rename") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "Rename")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_RmDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "RmDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "RmDir")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_SetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "SetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "SetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_SyncFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "SyncFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "SyncFile")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_Unlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "Unlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "Unlink")): 5, + }, + }, + { + name: "fs_error_category_DIR_NOT_EMPTY_fs_op_WriteFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DIR_NOT_EMPTY", "WriteFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "DIR_NOT_EMPTY"), attribute.String("fs_op", "WriteFile")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_BatchForget", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "BatchForget") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "BatchForget")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_CreateFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "CreateFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "CreateFile")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_CreateLink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "CreateLink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "CreateLink")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_CreateSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "CreateSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "CreateSymlink")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_FlushFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "FlushFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "FlushFile")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_ForgetInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "ForgetInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "ForgetInode")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_GetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "GetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "GetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_LookUpInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "LookUpInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "LookUpInode")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_MkDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "MkDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "MkDir")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_MkNode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "MkNode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "MkNode")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_OpenDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "OpenDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "OpenDir")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_OpenFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "OpenFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "OpenFile")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_Others", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "Others") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "Others")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_ReadDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "ReadDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "ReadDir")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_ReadDirPlus", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "ReadDirPlus") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "ReadDirPlus")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_ReadFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "ReadFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "ReadFile")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_ReadSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "ReadSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "ReadSymlink")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_ReleaseDirHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "ReleaseDirHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "ReleaseDirHandle")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_ReleaseFileHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "ReleaseFileHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "ReleaseFileHandle")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_Rename", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "Rename") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "Rename")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_RmDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "RmDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "RmDir")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_SetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "SetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "SetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_SyncFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "SyncFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "SyncFile")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_Unlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "Unlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "Unlink")): 5, + }, + }, + { + name: "fs_error_category_FILE_DIR_ERROR_fs_op_WriteFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_DIR_ERROR", "WriteFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_DIR_ERROR"), attribute.String("fs_op", "WriteFile")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_BatchForget", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "BatchForget") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "BatchForget")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_CreateFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "CreateFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "CreateFile")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_CreateLink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "CreateLink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "CreateLink")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_CreateSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "CreateSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "CreateSymlink")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_FlushFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "FlushFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "FlushFile")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_ForgetInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "ForgetInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "ForgetInode")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_GetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "GetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "GetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_LookUpInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "LookUpInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "LookUpInode")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_MkDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "MkDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "MkDir")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_MkNode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "MkNode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "MkNode")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_OpenDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "OpenDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "OpenDir")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_OpenFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "OpenFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "OpenFile")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_Others", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "Others") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "Others")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_ReadDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "ReadDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "ReadDir")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_ReadDirPlus", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "ReadDirPlus") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "ReadDirPlus")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_ReadFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "ReadFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "ReadFile")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_ReadSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "ReadSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "ReadSymlink")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_ReleaseDirHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "ReleaseDirHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "ReleaseDirHandle")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_ReleaseFileHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "ReleaseFileHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "ReleaseFileHandle")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_Rename", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "Rename") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "Rename")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_RmDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "RmDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "RmDir")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_SetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "SetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "SetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_SyncFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "SyncFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "SyncFile")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_Unlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "Unlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "Unlink")): 5, + }, + }, + { + name: "fs_error_category_FILE_EXISTS_fs_op_WriteFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "FILE_EXISTS", "WriteFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "FILE_EXISTS"), attribute.String("fs_op", "WriteFile")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_BatchForget", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "BatchForget") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "BatchForget")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_CreateFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "CreateFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "CreateFile")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_CreateLink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "CreateLink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "CreateLink")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_CreateSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "CreateSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "CreateSymlink")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_FlushFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "FlushFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "FlushFile")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_ForgetInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "ForgetInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "ForgetInode")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_GetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "GetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "GetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_LookUpInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "LookUpInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "LookUpInode")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_MkDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "MkDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "MkDir")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_MkNode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "MkNode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "MkNode")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_OpenDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "OpenDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "OpenDir")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_OpenFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "OpenFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "OpenFile")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_Others", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "Others") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "Others")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_ReadDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "ReadDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "ReadDir")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_ReadDirPlus", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "ReadDirPlus") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "ReadDirPlus")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_ReadFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "ReadFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "ReadFile")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_ReadSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "ReadSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "ReadSymlink")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_ReleaseDirHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "ReleaseDirHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "ReleaseDirHandle")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_ReleaseFileHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "ReleaseFileHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "ReleaseFileHandle")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_Rename", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "Rename") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "Rename")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_RmDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "RmDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "RmDir")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_SetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "SetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "SetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_SyncFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "SyncFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "SyncFile")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_Unlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "Unlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "Unlink")): 5, + }, + }, + { + name: "fs_error_category_INTERRUPT_ERROR_fs_op_WriteFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INTERRUPT_ERROR", "WriteFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INTERRUPT_ERROR"), attribute.String("fs_op", "WriteFile")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_BatchForget", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "BatchForget") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "BatchForget")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_CreateFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "CreateFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "CreateFile")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_CreateLink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "CreateLink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "CreateLink")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_CreateSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "CreateSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "CreateSymlink")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_FlushFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "FlushFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "FlushFile")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_ForgetInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "ForgetInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "ForgetInode")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_GetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "GetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "GetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_LookUpInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "LookUpInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "LookUpInode")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_MkDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "MkDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "MkDir")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_MkNode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "MkNode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "MkNode")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_OpenDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "OpenDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "OpenDir")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_OpenFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "OpenFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "OpenFile")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_Others", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "Others") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "Others")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_ReadDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "ReadDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "ReadDir")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_ReadDirPlus", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "ReadDirPlus") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "ReadDirPlus")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_ReadFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "ReadFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "ReadFile")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_ReadSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "ReadSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "ReadSymlink")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_ReleaseDirHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "ReleaseDirHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "ReleaseDirHandle")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_ReleaseFileHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "ReleaseFileHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "ReleaseFileHandle")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_Rename", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "Rename") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "Rename")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_RmDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "RmDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "RmDir")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_SetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "SetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "SetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_SyncFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "SyncFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "SyncFile")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_Unlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "Unlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "Unlink")): 5, + }, + }, + { + name: "fs_error_category_INVALID_ARGUMENT_fs_op_WriteFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_ARGUMENT", "WriteFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_ARGUMENT"), attribute.String("fs_op", "WriteFile")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_BatchForget", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "BatchForget") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "BatchForget")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_CreateFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "CreateFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "CreateFile")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_CreateLink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "CreateLink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "CreateLink")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_CreateSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "CreateSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "CreateSymlink")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_FlushFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "FlushFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "FlushFile")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_ForgetInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "ForgetInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "ForgetInode")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_GetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "GetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "GetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_LookUpInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "LookUpInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "LookUpInode")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_MkDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "MkDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "MkDir")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_MkNode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "MkNode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "MkNode")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_OpenDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "OpenDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "OpenDir")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_OpenFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "OpenFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "OpenFile")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_Others", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "Others") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "Others")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_ReadDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "ReadDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "ReadDir")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_ReadDirPlus", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "ReadDirPlus") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "ReadDirPlus")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_ReadFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "ReadFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "ReadFile")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_ReadSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "ReadSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "ReadSymlink")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_ReleaseDirHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "ReleaseDirHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "ReleaseDirHandle")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_ReleaseFileHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "ReleaseFileHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "ReleaseFileHandle")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_Rename", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "Rename") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "Rename")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_RmDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "RmDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "RmDir")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_SetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "SetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "SetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_SyncFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "SyncFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "SyncFile")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_Unlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "Unlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "Unlink")): 5, + }, + }, + { + name: "fs_error_category_INVALID_OPERATION_fs_op_WriteFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "INVALID_OPERATION", "WriteFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "INVALID_OPERATION"), attribute.String("fs_op", "WriteFile")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_BatchForget", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "BatchForget") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "BatchForget")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_CreateFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "CreateFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "CreateFile")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_CreateLink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "CreateLink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "CreateLink")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_CreateSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "CreateSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "CreateSymlink")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_FlushFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "FlushFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "FlushFile")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_ForgetInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "ForgetInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "ForgetInode")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_GetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "GetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "GetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_LookUpInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "LookUpInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "LookUpInode")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_MkDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "MkDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "MkDir")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_MkNode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "MkNode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "MkNode")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_OpenDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "OpenDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "OpenDir")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_OpenFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "OpenFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "OpenFile")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_Others", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "Others") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "Others")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_ReadDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "ReadDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "ReadDir")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_ReadDirPlus", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "ReadDirPlus") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "ReadDirPlus")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_ReadFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "ReadFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "ReadFile")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_ReadSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "ReadSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "ReadSymlink")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_ReleaseDirHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "ReleaseDirHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "ReleaseDirHandle")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_ReleaseFileHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "ReleaseFileHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "ReleaseFileHandle")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_Rename", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "Rename") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "Rename")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_RmDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "RmDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "RmDir")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_SetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "SetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "SetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_SyncFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "SyncFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "SyncFile")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_Unlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "Unlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "Unlink")): 5, + }, + }, + { + name: "fs_error_category_IO_ERROR_fs_op_WriteFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "IO_ERROR", "WriteFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "IO_ERROR"), attribute.String("fs_op", "WriteFile")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_BatchForget", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "BatchForget") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "BatchForget")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_CreateFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "CreateFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "CreateFile")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_CreateLink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "CreateLink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "CreateLink")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_CreateSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "CreateSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "CreateSymlink")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_FlushFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "FlushFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "FlushFile")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_ForgetInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "ForgetInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "ForgetInode")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_GetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "GetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "GetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_LookUpInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "LookUpInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "LookUpInode")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_MkDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "MkDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "MkDir")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_MkNode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "MkNode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "MkNode")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_OpenDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "OpenDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "OpenDir")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_OpenFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "OpenFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "OpenFile")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_Others", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "Others") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "Others")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_ReadDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "ReadDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "ReadDir")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_ReadDirPlus", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "ReadDirPlus") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "ReadDirPlus")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_ReadFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "ReadFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "ReadFile")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_ReadSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "ReadSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "ReadSymlink")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_ReleaseDirHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "ReleaseDirHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "ReleaseDirHandle")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_ReleaseFileHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "ReleaseFileHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "ReleaseFileHandle")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_Rename", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "Rename") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "Rename")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_RmDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "RmDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "RmDir")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_SetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "SetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "SetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_SyncFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "SyncFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "SyncFile")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_Unlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "Unlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "Unlink")): 5, + }, + }, + { + name: "fs_error_category_MISC_ERROR_fs_op_WriteFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "MISC_ERROR", "WriteFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "MISC_ERROR"), attribute.String("fs_op", "WriteFile")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_BatchForget", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "BatchForget") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "BatchForget")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_CreateFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "CreateFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "CreateFile")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_CreateLink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "CreateLink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "CreateLink")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_CreateSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "CreateSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "CreateSymlink")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_FlushFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "FlushFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "FlushFile")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_ForgetInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "ForgetInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "ForgetInode")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_GetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "GetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "GetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_LookUpInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "LookUpInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "LookUpInode")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_MkDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "MkDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "MkDir")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_MkNode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "MkNode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "MkNode")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_OpenDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "OpenDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "OpenDir")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_OpenFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "OpenFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "OpenFile")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_Others", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "Others") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "Others")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_ReadDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "ReadDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "ReadDir")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_ReadDirPlus", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "ReadDirPlus") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "ReadDirPlus")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_ReadFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "ReadFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "ReadFile")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_ReadSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "ReadSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "ReadSymlink")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_ReleaseDirHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "ReleaseDirHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "ReleaseDirHandle")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_ReleaseFileHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "ReleaseFileHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "ReleaseFileHandle")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_Rename", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "Rename") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "Rename")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_RmDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "RmDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "RmDir")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_SetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "SetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "SetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_SyncFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "SyncFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "SyncFile")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_Unlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "Unlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "Unlink")): 5, + }, + }, + { + name: "fs_error_category_NETWORK_ERROR_fs_op_WriteFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NETWORK_ERROR", "WriteFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NETWORK_ERROR"), attribute.String("fs_op", "WriteFile")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_BatchForget", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "BatchForget") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "BatchForget")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_CreateFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "CreateFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "CreateFile")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_CreateLink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "CreateLink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "CreateLink")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_CreateSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "CreateSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "CreateSymlink")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_FlushFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "FlushFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "FlushFile")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_ForgetInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "ForgetInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "ForgetInode")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_GetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "GetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "GetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_LookUpInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "LookUpInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "LookUpInode")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_MkDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "MkDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "MkDir")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_MkNode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "MkNode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "MkNode")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_OpenDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "OpenDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "OpenDir")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_OpenFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "OpenFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "OpenFile")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_Others", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "Others") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "Others")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_ReadDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "ReadDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "ReadDir")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_ReadDirPlus", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "ReadDirPlus") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "ReadDirPlus")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_ReadFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "ReadFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "ReadFile")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_ReadSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "ReadSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "ReadSymlink")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_ReleaseDirHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "ReleaseDirHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "ReleaseDirHandle")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_ReleaseFileHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "ReleaseFileHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "ReleaseFileHandle")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_Rename", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "Rename") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "Rename")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_RmDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "RmDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "RmDir")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_SetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "SetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "SetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_SyncFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "SyncFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "SyncFile")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_Unlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "Unlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "Unlink")): 5, + }, + }, + { + name: "fs_error_category_NOT_A_DIR_fs_op_WriteFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_A_DIR", "WriteFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_A_DIR"), attribute.String("fs_op", "WriteFile")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_BatchForget", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "BatchForget") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "BatchForget")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_CreateFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "CreateFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "CreateFile")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_CreateLink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "CreateLink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "CreateLink")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_CreateSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "CreateSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "CreateSymlink")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_FlushFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "FlushFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "FlushFile")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_ForgetInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "ForgetInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "ForgetInode")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_GetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "GetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "GetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_LookUpInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "LookUpInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "LookUpInode")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_MkDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "MkDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "MkDir")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_MkNode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "MkNode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "MkNode")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_OpenDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "OpenDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "OpenDir")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_OpenFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "OpenFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "OpenFile")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_Others", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "Others") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "Others")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_ReadDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "ReadDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "ReadDir")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_ReadDirPlus", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "ReadDirPlus") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "ReadDirPlus")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_ReadFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "ReadFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "ReadFile")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_ReadSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "ReadSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "ReadSymlink")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_ReleaseDirHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "ReleaseDirHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "ReleaseDirHandle")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_ReleaseFileHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "ReleaseFileHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "ReleaseFileHandle")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_Rename", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "Rename") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "Rename")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_RmDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "RmDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "RmDir")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_SetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "SetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "SetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_SyncFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "SyncFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "SyncFile")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_Unlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "Unlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "Unlink")): 5, + }, + }, + { + name: "fs_error_category_NOT_IMPLEMENTED_fs_op_WriteFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NOT_IMPLEMENTED", "WriteFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NOT_IMPLEMENTED"), attribute.String("fs_op", "WriteFile")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_BatchForget", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "BatchForget") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "BatchForget")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_CreateFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "CreateFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "CreateFile")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_CreateLink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "CreateLink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "CreateLink")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_CreateSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "CreateSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "CreateSymlink")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_FlushFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "FlushFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "FlushFile")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_ForgetInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "ForgetInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "ForgetInode")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_GetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "GetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "GetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_LookUpInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "LookUpInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "LookUpInode")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_MkDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "MkDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "MkDir")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_MkNode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "MkNode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "MkNode")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_OpenDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "OpenDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "OpenDir")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_OpenFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "OpenFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "OpenFile")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_Others", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "Others") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "Others")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_ReadDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "ReadDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "ReadDir")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_ReadDirPlus", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "ReadDirPlus") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "ReadDirPlus")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_ReadFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "ReadFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "ReadFile")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_ReadSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "ReadSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "ReadSymlink")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_ReleaseDirHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "ReleaseDirHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "ReleaseDirHandle")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_ReleaseFileHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "ReleaseFileHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "ReleaseFileHandle")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_Rename", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "Rename") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "Rename")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_RmDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "RmDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "RmDir")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_SetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "SetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "SetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_SyncFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "SyncFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "SyncFile")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_Unlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "Unlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "Unlink")): 5, + }, + }, + { + name: "fs_error_category_NO_FILE_OR_DIR_fs_op_WriteFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "NO_FILE_OR_DIR", "WriteFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "NO_FILE_OR_DIR"), attribute.String("fs_op", "WriteFile")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_BatchForget", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "BatchForget") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "BatchForget")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_CreateFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "CreateFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "CreateFile")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_CreateLink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "CreateLink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "CreateLink")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_CreateSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "CreateSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "CreateSymlink")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_FlushFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "FlushFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "FlushFile")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_ForgetInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "ForgetInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "ForgetInode")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_GetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "GetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "GetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_LookUpInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "LookUpInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "LookUpInode")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_MkDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "MkDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "MkDir")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_MkNode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "MkNode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "MkNode")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_OpenDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "OpenDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "OpenDir")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_OpenFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "OpenFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "OpenFile")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_Others", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "Others") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "Others")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_ReadDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "ReadDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "ReadDir")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_ReadDirPlus", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "ReadDirPlus") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "ReadDirPlus")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_ReadFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "ReadFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "ReadFile")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_ReadSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "ReadSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "ReadSymlink")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_ReleaseDirHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "ReleaseDirHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "ReleaseDirHandle")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_ReleaseFileHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "ReleaseFileHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "ReleaseFileHandle")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_Rename", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "Rename") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "Rename")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_RmDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "RmDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "RmDir")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_SetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "SetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "SetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_SyncFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "SyncFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "SyncFile")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_Unlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "Unlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "Unlink")): 5, + }, + }, + { + name: "fs_error_category_PERM_ERROR_fs_op_WriteFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PERM_ERROR", "WriteFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PERM_ERROR"), attribute.String("fs_op", "WriteFile")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_BatchForget", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "BatchForget") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "BatchForget")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_CreateFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "CreateFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "CreateFile")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_CreateLink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "CreateLink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "CreateLink")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_CreateSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "CreateSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "CreateSymlink")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_FlushFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "FlushFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "FlushFile")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_ForgetInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "ForgetInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "ForgetInode")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_GetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "GetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "GetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_LookUpInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "LookUpInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "LookUpInode")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_MkDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "MkDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "MkDir")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_MkNode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "MkNode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "MkNode")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_OpenDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "OpenDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "OpenDir")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_OpenFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "OpenFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "OpenFile")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_Others", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "Others") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "Others")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_ReadDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "ReadDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "ReadDir")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_ReadDirPlus", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "ReadDirPlus") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "ReadDirPlus")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_ReadFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "ReadFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "ReadFile")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_ReadSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "ReadSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "ReadSymlink")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_ReleaseDirHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "ReleaseDirHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "ReleaseDirHandle")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_ReleaseFileHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "ReleaseFileHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "ReleaseFileHandle")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_Rename", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "Rename") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "Rename")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_RmDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "RmDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "RmDir")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_SetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "SetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "SetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_SyncFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "SyncFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "SyncFile")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_Unlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "Unlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "Unlink")): 5, + }, + }, + { + name: "fs_error_category_PROCESS_RESOURCE_MGMT_ERROR_fs_op_WriteFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "PROCESS_RESOURCE_MGMT_ERROR", "WriteFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "PROCESS_RESOURCE_MGMT_ERROR"), attribute.String("fs_op", "WriteFile")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_BatchForget", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "BatchForget") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "BatchForget")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_CreateFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "CreateFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "CreateFile")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_CreateLink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "CreateLink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "CreateLink")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_CreateSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "CreateSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "CreateSymlink")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_FlushFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "FlushFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "FlushFile")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_ForgetInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "ForgetInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "ForgetInode")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_GetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "GetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "GetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_LookUpInode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "LookUpInode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "LookUpInode")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_MkDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "MkDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "MkDir")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_MkNode", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "MkNode") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "MkNode")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_OpenDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "OpenDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "OpenDir")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_OpenFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "OpenFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "OpenFile")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_Others", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "Others") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "Others")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_ReadDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "ReadDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "ReadDir")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_ReadDirPlus", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "ReadDirPlus") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "ReadDirPlus")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_ReadFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "ReadFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "ReadFile")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_ReadSymlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "ReadSymlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "ReadSymlink")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_ReleaseDirHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "ReleaseDirHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "ReleaseDirHandle")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_ReleaseFileHandle", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "ReleaseFileHandle") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "ReleaseFileHandle")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_Rename", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "Rename") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "Rename")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_RmDir", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "RmDir") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "RmDir")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_SetInodeAttributes", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "SetInodeAttributes") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "SetInodeAttributes")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_SyncFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "SyncFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "SyncFile")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_Unlink", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "Unlink") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "Unlink")): 5, + }, + }, + { + name: "fs_error_category_TOO_MANY_OPEN_FILES_fs_op_WriteFile", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "TOO_MANY_OPEN_FILES", "WriteFile") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("fs_error_category", "TOO_MANY_OPEN_FILES"), attribute.String("fs_op", "WriteFile")): 5, + }, + }, { + name: "multiple_attributes_summed", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(5, "DEVICE_ERROR", "BatchForget") + m.FsOpsErrorCount(2, "DEVICE_ERROR", "CreateFile") + m.FsOpsErrorCount(3, "DEVICE_ERROR", "BatchForget") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "BatchForget")): 8, + attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "CreateFile")): 2, + }, + }, + { + name: "negative_increment", + f: func(m *otelMetrics) { + m.FsOpsErrorCount(-5, "DEVICE_ERROR", "BatchForget") + m.FsOpsErrorCount(2, "DEVICE_ERROR", "BatchForget") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("fs_error_category", "DEVICE_ERROR"), attribute.String("fs_op", "BatchForget")): 2}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + + tc.f(m) + waitForMetricsProcessing() + + metrics := gatherNonZeroCounterMetrics(ctx, t, rd) + metric, ok := metrics["fs/ops_error_count"] + if len(tc.expected) == 0 { + assert.False(t, ok, "fs/ops_error_count metric should not be found") + return + } + require.True(t, ok, "fs/ops_error_count metric not found") + expectedMap := make(map[string]int64) + for k, v := range tc.expected { + expectedMap[k.Encoded(encoder)] = v + } + assert.Equal(t, expectedMap, metric) + }) + } +} + +func TestFsOpsLatency(t *testing.T) { + tests := []struct { + name string + latencies []time.Duration + fsOp FsOp + }{ + { + name: "fs_op_BatchForget", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "BatchForget", + }, + { + name: "fs_op_CreateFile", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "CreateFile", + }, + { + name: "fs_op_CreateLink", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "CreateLink", + }, + { + name: "fs_op_CreateSymlink", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "CreateSymlink", + }, + { + name: "fs_op_FlushFile", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "FlushFile", + }, + { + name: "fs_op_ForgetInode", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "ForgetInode", + }, + { + name: "fs_op_GetInodeAttributes", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "GetInodeAttributes", + }, + { + name: "fs_op_LookUpInode", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "LookUpInode", + }, + { + name: "fs_op_MkDir", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "MkDir", + }, + { + name: "fs_op_MkNode", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "MkNode", + }, + { + name: "fs_op_OpenDir", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "OpenDir", + }, + { + name: "fs_op_OpenFile", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "OpenFile", + }, + { + name: "fs_op_Others", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "Others", + }, + { + name: "fs_op_ReadDir", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "ReadDir", + }, + { + name: "fs_op_ReadDirPlus", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "ReadDirPlus", + }, + { + name: "fs_op_ReadFile", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "ReadFile", + }, + { + name: "fs_op_ReadSymlink", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "ReadSymlink", + }, + { + name: "fs_op_ReleaseDirHandle", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "ReleaseDirHandle", + }, + { + name: "fs_op_ReleaseFileHandle", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "ReleaseFileHandle", + }, + { + name: "fs_op_Rename", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "Rename", + }, + { + name: "fs_op_RmDir", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "RmDir", + }, + { + name: "fs_op_SetInodeAttributes", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "SetInodeAttributes", + }, + { + name: "fs_op_SyncFile", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "SyncFile", + }, + { + name: "fs_op_Unlink", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "Unlink", + }, + { + name: "fs_op_WriteFile", + latencies: []time.Duration{100 * time.Microsecond, 200 * time.Microsecond}, + fsOp: "WriteFile", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + var totalLatency time.Duration + + for _, latency := range tc.latencies { + m.FsOpsLatency(ctx, latency, tc.fsOp) + totalLatency += latency + } + waitForMetricsProcessing() + + metrics := gatherHistogramMetrics(ctx, t, rd) + metric, ok := metrics["fs/ops_latency"] + require.True(t, ok, "fs/ops_latency metric not found") + + attrs := []attribute.KeyValue{ + attribute.String("fs_op", string(tc.fsOp)), + } + s := attribute.NewSet(attrs...) + expectedKey := s.Encoded(encoder) + dp, ok := metric[expectedKey] + require.True(t, ok, "DataPoint not found for key: %s", expectedKey) + assert.Equal(t, uint64(len(tc.latencies)), dp.Count) + assert.Equal(t, totalLatency.Microseconds(), dp.Sum) + }) + } +} + +func TestFsStreamingWriteFallbackCount(t *testing.T) { + tests := []struct { + name string + f func(m *otelMetrics) + expected map[attribute.Set]int64 + }{ + { + name: "open_mode_other_write_fallback_reason_concurrency_limit_breached", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(5, "other", "concurrency_limit_breached") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("open_mode", "other"), attribute.String("write_fallback_reason", "concurrency_limit_breached")): 5, + }, + }, + { + name: "open_mode_other_write_fallback_reason_existing_file", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(5, "other", "existing_file") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("open_mode", "other"), attribute.String("write_fallback_reason", "existing_file")): 5, + }, + }, + { + name: "open_mode_other_write_fallback_reason_other", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(5, "other", "other") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("open_mode", "other"), attribute.String("write_fallback_reason", "other")): 5, + }, + }, + { + name: "open_mode_other_write_fallback_reason_out_of_order", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(5, "other", "out_of_order") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("open_mode", "other"), attribute.String("write_fallback_reason", "out_of_order")): 5, + }, + }, + { + name: "open_mode_read_write_write_fallback_reason_concurrency_limit_breached", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(5, "read_write", "concurrency_limit_breached") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("open_mode", "read_write"), attribute.String("write_fallback_reason", "concurrency_limit_breached")): 5, + }, + }, + { + name: "open_mode_read_write_write_fallback_reason_existing_file", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(5, "read_write", "existing_file") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("open_mode", "read_write"), attribute.String("write_fallback_reason", "existing_file")): 5, + }, + }, + { + name: "open_mode_read_write_write_fallback_reason_other", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(5, "read_write", "other") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("open_mode", "read_write"), attribute.String("write_fallback_reason", "other")): 5, + }, + }, + { + name: "open_mode_read_write_write_fallback_reason_out_of_order", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(5, "read_write", "out_of_order") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("open_mode", "read_write"), attribute.String("write_fallback_reason", "out_of_order")): 5, + }, + }, + { + name: "open_mode_read_write_append_write_fallback_reason_concurrency_limit_breached", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(5, "read_write_append", "concurrency_limit_breached") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("open_mode", "read_write_append"), attribute.String("write_fallback_reason", "concurrency_limit_breached")): 5, + }, + }, + { + name: "open_mode_read_write_append_write_fallback_reason_existing_file", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(5, "read_write_append", "existing_file") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("open_mode", "read_write_append"), attribute.String("write_fallback_reason", "existing_file")): 5, + }, + }, + { + name: "open_mode_read_write_append_write_fallback_reason_other", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(5, "read_write_append", "other") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("open_mode", "read_write_append"), attribute.String("write_fallback_reason", "other")): 5, + }, + }, + { + name: "open_mode_read_write_append_write_fallback_reason_out_of_order", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(5, "read_write_append", "out_of_order") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("open_mode", "read_write_append"), attribute.String("write_fallback_reason", "out_of_order")): 5, + }, + }, + { + name: "open_mode_write_only_write_fallback_reason_concurrency_limit_breached", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(5, "write_only", "concurrency_limit_breached") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("open_mode", "write_only"), attribute.String("write_fallback_reason", "concurrency_limit_breached")): 5, + }, + }, + { + name: "open_mode_write_only_write_fallback_reason_existing_file", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(5, "write_only", "existing_file") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("open_mode", "write_only"), attribute.String("write_fallback_reason", "existing_file")): 5, + }, + }, + { + name: "open_mode_write_only_write_fallback_reason_other", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(5, "write_only", "other") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("open_mode", "write_only"), attribute.String("write_fallback_reason", "other")): 5, + }, + }, + { + name: "open_mode_write_only_write_fallback_reason_out_of_order", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(5, "write_only", "out_of_order") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("open_mode", "write_only"), attribute.String("write_fallback_reason", "out_of_order")): 5, + }, + }, + { + name: "open_mode_write_only_append_write_fallback_reason_concurrency_limit_breached", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(5, "write_only_append", "concurrency_limit_breached") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("open_mode", "write_only_append"), attribute.String("write_fallback_reason", "concurrency_limit_breached")): 5, + }, + }, + { + name: "open_mode_write_only_append_write_fallback_reason_existing_file", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(5, "write_only_append", "existing_file") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("open_mode", "write_only_append"), attribute.String("write_fallback_reason", "existing_file")): 5, + }, + }, + { + name: "open_mode_write_only_append_write_fallback_reason_other", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(5, "write_only_append", "other") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("open_mode", "write_only_append"), attribute.String("write_fallback_reason", "other")): 5, + }, + }, + { + name: "open_mode_write_only_append_write_fallback_reason_out_of_order", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(5, "write_only_append", "out_of_order") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("open_mode", "write_only_append"), attribute.String("write_fallback_reason", "out_of_order")): 5, + }, + }, { + name: "multiple_attributes_summed", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(5, "other", "concurrency_limit_breached") + m.FsStreamingWriteFallbackCount(2, "other", "existing_file") + m.FsStreamingWriteFallbackCount(3, "other", "concurrency_limit_breached") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("open_mode", "other"), attribute.String("write_fallback_reason", "concurrency_limit_breached")): 8, + attribute.NewSet(attribute.String("open_mode", "other"), attribute.String("write_fallback_reason", "existing_file")): 2, + }, + }, + { + name: "negative_increment", + f: func(m *otelMetrics) { + m.FsStreamingWriteFallbackCount(-5, "other", "concurrency_limit_breached") + m.FsStreamingWriteFallbackCount(2, "other", "concurrency_limit_breached") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("open_mode", "other"), attribute.String("write_fallback_reason", "concurrency_limit_breached")): 2}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + + tc.f(m) + waitForMetricsProcessing() + + metrics := gatherNonZeroCounterMetrics(ctx, t, rd) + metric, ok := metrics["fs/streaming_write_fallback_count"] + if len(tc.expected) == 0 { + assert.False(t, ok, "fs/streaming_write_fallback_count metric should not be found") + return + } + require.True(t, ok, "fs/streaming_write_fallback_count metric not found") + expectedMap := make(map[string]int64) + for k, v := range tc.expected { + expectedMap[k.Encoded(encoder)] = v + } + assert.Equal(t, expectedMap, metric) + }) + } +} + +func TestGcsDownloadBytesCount(t *testing.T) { + tests := []struct { + name string + f func(m *otelMetrics) + expected map[attribute.Set]int64 + }{ + { + name: "read_type_Buffered", + f: func(m *otelMetrics) { + m.GcsDownloadBytesCount(5, "Buffered") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("read_type", "Buffered")): 5, + }, + }, + { + name: "read_type_Parallel", + f: func(m *otelMetrics) { + m.GcsDownloadBytesCount(5, "Parallel") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("read_type", "Parallel")): 5, + }, + }, + { + name: "read_type_Random", + f: func(m *otelMetrics) { + m.GcsDownloadBytesCount(5, "Random") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("read_type", "Random")): 5, + }, + }, + { + name: "read_type_Sequential", + f: func(m *otelMetrics) { + m.GcsDownloadBytesCount(5, "Sequential") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("read_type", "Sequential")): 5, + }, + }, { + name: "multiple_attributes_summed", + f: func(m *otelMetrics) { + m.GcsDownloadBytesCount(5, "Buffered") + m.GcsDownloadBytesCount(2, "Parallel") + m.GcsDownloadBytesCount(3, "Buffered") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("read_type", "Buffered")): 8, + attribute.NewSet(attribute.String("read_type", "Parallel")): 2, + }, + }, + { + name: "negative_increment", + f: func(m *otelMetrics) { + m.GcsDownloadBytesCount(-5, "Buffered") + m.GcsDownloadBytesCount(2, "Buffered") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("read_type", "Buffered")): 2}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + + tc.f(m) + waitForMetricsProcessing() + + metrics := gatherNonZeroCounterMetrics(ctx, t, rd) + metric, ok := metrics["gcs/download_bytes_count"] + if len(tc.expected) == 0 { + assert.False(t, ok, "gcs/download_bytes_count metric should not be found") + return + } + require.True(t, ok, "gcs/download_bytes_count metric not found") + expectedMap := make(map[string]int64) + for k, v := range tc.expected { + expectedMap[k.Encoded(encoder)] = v + } + assert.Equal(t, expectedMap, metric) + }) + } +} + +func TestGcsReadBytesCount(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + + m.GcsReadBytesCount(1024) + m.GcsReadBytesCount(2048) + waitForMetricsProcessing() + + metrics := gatherNonZeroCounterMetrics(ctx, t, rd) + metric, ok := metrics["gcs/read_bytes_count"] + require.True(t, ok, "gcs/read_bytes_count metric not found") + s := attribute.NewSet() + assert.Equal(t, map[string]int64{s.Encoded(encoder): 3072}, metric, "Positive increments should be summed.") + + // Test negative increment + m.GcsReadBytesCount(-100) + waitForMetricsProcessing() + + metrics = gatherNonZeroCounterMetrics(ctx, t, rd) + metric, ok = metrics["gcs/read_bytes_count"] + require.True(t, ok, "gcs/read_bytes_count metric not found after negative increment") + assert.Equal(t, map[string]int64{s.Encoded(encoder): 3072}, metric, "Negative increment should not change the metric value.") +} + +func TestGcsReadCount(t *testing.T) { + tests := []struct { + name string + f func(m *otelMetrics) + expected map[attribute.Set]int64 + }{ + { + name: "read_type_Parallel", + f: func(m *otelMetrics) { + m.GcsReadCount(5, "Parallel") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("read_type", "Parallel")): 5, + }, + }, + { + name: "read_type_Random", + f: func(m *otelMetrics) { + m.GcsReadCount(5, "Random") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("read_type", "Random")): 5, + }, + }, + { + name: "read_type_Sequential", + f: func(m *otelMetrics) { + m.GcsReadCount(5, "Sequential") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("read_type", "Sequential")): 5, + }, + }, + { + name: "read_type_Unknown", + f: func(m *otelMetrics) { + m.GcsReadCount(5, "Unknown") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("read_type", "Unknown")): 5, + }, + }, { + name: "multiple_attributes_summed", + f: func(m *otelMetrics) { + m.GcsReadCount(5, "Parallel") + m.GcsReadCount(2, "Random") + m.GcsReadCount(3, "Parallel") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("read_type", "Parallel")): 8, + attribute.NewSet(attribute.String("read_type", "Random")): 2, + }, + }, + { + name: "negative_increment", + f: func(m *otelMetrics) { + m.GcsReadCount(-5, "Parallel") + m.GcsReadCount(2, "Parallel") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("read_type", "Parallel")): 2}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + + tc.f(m) + waitForMetricsProcessing() + + metrics := gatherNonZeroCounterMetrics(ctx, t, rd) + metric, ok := metrics["gcs/read_count"] + if len(tc.expected) == 0 { + assert.False(t, ok, "gcs/read_count metric should not be found") + return + } + require.True(t, ok, "gcs/read_count metric not found") + expectedMap := make(map[string]int64) + for k, v := range tc.expected { + expectedMap[k.Encoded(encoder)] = v + } + assert.Equal(t, expectedMap, metric) + }) + } +} + +func TestGcsReaderCount(t *testing.T) { + tests := []struct { + name string + f func(m *otelMetrics) + expected map[attribute.Set]int64 + }{ + { + name: "io_method_closed", + f: func(m *otelMetrics) { + m.GcsReaderCount(5, "closed") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("io_method", "closed")): 5, + }, + }, + { + name: "io_method_opened", + f: func(m *otelMetrics) { + m.GcsReaderCount(5, "opened") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("io_method", "opened")): 5, + }, + }, { + name: "multiple_attributes_summed", + f: func(m *otelMetrics) { + m.GcsReaderCount(5, "closed") + m.GcsReaderCount(2, "opened") + m.GcsReaderCount(3, "closed") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("io_method", "closed")): 8, + attribute.NewSet(attribute.String("io_method", "opened")): 2, + }, + }, + { + name: "negative_increment", + f: func(m *otelMetrics) { + m.GcsReaderCount(-5, "closed") + m.GcsReaderCount(2, "closed") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("io_method", "closed")): 2}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + + tc.f(m) + waitForMetricsProcessing() + + metrics := gatherNonZeroCounterMetrics(ctx, t, rd) + metric, ok := metrics["gcs/reader_count"] + if len(tc.expected) == 0 { + assert.False(t, ok, "gcs/reader_count metric should not be found") + return + } + require.True(t, ok, "gcs/reader_count metric not found") + expectedMap := make(map[string]int64) + for k, v := range tc.expected { + expectedMap[k.Encoded(encoder)] = v + } + assert.Equal(t, expectedMap, metric) + }) + } +} + +func TestGcsRequestCount(t *testing.T) { + tests := []struct { + name string + f func(m *otelMetrics) + expected map[attribute.Set]int64 + }{ + { + name: "gcs_method_ComposeObjects", + f: func(m *otelMetrics) { + m.GcsRequestCount(5, "ComposeObjects") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("gcs_method", "ComposeObjects")): 5, + }, + }, + { + name: "gcs_method_CopyObject", + f: func(m *otelMetrics) { + m.GcsRequestCount(5, "CopyObject") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("gcs_method", "CopyObject")): 5, + }, + }, + { + name: "gcs_method_CreateAppendableObjectWriter", + f: func(m *otelMetrics) { + m.GcsRequestCount(5, "CreateAppendableObjectWriter") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("gcs_method", "CreateAppendableObjectWriter")): 5, + }, + }, + { + name: "gcs_method_CreateFolder", + f: func(m *otelMetrics) { + m.GcsRequestCount(5, "CreateFolder") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("gcs_method", "CreateFolder")): 5, + }, + }, + { + name: "gcs_method_CreateObject", + f: func(m *otelMetrics) { + m.GcsRequestCount(5, "CreateObject") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("gcs_method", "CreateObject")): 5, + }, + }, + { + name: "gcs_method_CreateObjectChunkWriter", + f: func(m *otelMetrics) { + m.GcsRequestCount(5, "CreateObjectChunkWriter") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("gcs_method", "CreateObjectChunkWriter")): 5, + }, + }, + { + name: "gcs_method_DeleteFolder", + f: func(m *otelMetrics) { + m.GcsRequestCount(5, "DeleteFolder") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("gcs_method", "DeleteFolder")): 5, + }, + }, + { + name: "gcs_method_DeleteObject", + f: func(m *otelMetrics) { + m.GcsRequestCount(5, "DeleteObject") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("gcs_method", "DeleteObject")): 5, + }, + }, + { + name: "gcs_method_FinalizeUpload", + f: func(m *otelMetrics) { + m.GcsRequestCount(5, "FinalizeUpload") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("gcs_method", "FinalizeUpload")): 5, + }, + }, + { + name: "gcs_method_FlushPendingWrites", + f: func(m *otelMetrics) { + m.GcsRequestCount(5, "FlushPendingWrites") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("gcs_method", "FlushPendingWrites")): 5, + }, + }, + { + name: "gcs_method_GetFolder", + f: func(m *otelMetrics) { + m.GcsRequestCount(5, "GetFolder") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("gcs_method", "GetFolder")): 5, + }, + }, + { + name: "gcs_method_ListObjects", + f: func(m *otelMetrics) { + m.GcsRequestCount(5, "ListObjects") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("gcs_method", "ListObjects")): 5, + }, + }, + { + name: "gcs_method_MoveObject", + f: func(m *otelMetrics) { + m.GcsRequestCount(5, "MoveObject") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("gcs_method", "MoveObject")): 5, + }, + }, + { + name: "gcs_method_MultiRangeDownloader::Add", + f: func(m *otelMetrics) { + m.GcsRequestCount(5, "MultiRangeDownloader::Add") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("gcs_method", "MultiRangeDownloader::Add")): 5, + }, + }, + { + name: "gcs_method_NewMultiRangeDownloader", + f: func(m *otelMetrics) { + m.GcsRequestCount(5, "NewMultiRangeDownloader") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("gcs_method", "NewMultiRangeDownloader")): 5, + }, + }, + { + name: "gcs_method_NewReader", + f: func(m *otelMetrics) { + m.GcsRequestCount(5, "NewReader") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("gcs_method", "NewReader")): 5, + }, + }, + { + name: "gcs_method_RenameFolder", + f: func(m *otelMetrics) { + m.GcsRequestCount(5, "RenameFolder") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("gcs_method", "RenameFolder")): 5, + }, + }, + { + name: "gcs_method_StatObject", + f: func(m *otelMetrics) { + m.GcsRequestCount(5, "StatObject") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("gcs_method", "StatObject")): 5, + }, + }, + { + name: "gcs_method_UpdateObject", + f: func(m *otelMetrics) { + m.GcsRequestCount(5, "UpdateObject") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("gcs_method", "UpdateObject")): 5, + }, + }, { + name: "multiple_attributes_summed", + f: func(m *otelMetrics) { + m.GcsRequestCount(5, "ComposeObjects") + m.GcsRequestCount(2, "CopyObject") + m.GcsRequestCount(3, "ComposeObjects") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("gcs_method", "ComposeObjects")): 8, + attribute.NewSet(attribute.String("gcs_method", "CopyObject")): 2, + }, + }, + { + name: "negative_increment", + f: func(m *otelMetrics) { + m.GcsRequestCount(-5, "ComposeObjects") + m.GcsRequestCount(2, "ComposeObjects") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("gcs_method", "ComposeObjects")): 2}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + + tc.f(m) + waitForMetricsProcessing() + + metrics := gatherNonZeroCounterMetrics(ctx, t, rd) + metric, ok := metrics["gcs/request_count"] + if len(tc.expected) == 0 { + assert.False(t, ok, "gcs/request_count metric should not be found") + return + } + require.True(t, ok, "gcs/request_count metric not found") + expectedMap := make(map[string]int64) + for k, v := range tc.expected { + expectedMap[k.Encoded(encoder)] = v + } + assert.Equal(t, expectedMap, metric) + }) + } +} + +func TestGcsRequestLatencies(t *testing.T) { + tests := []struct { + name string + latencies []time.Duration + gcsMethod GcsMethod + }{ + { + name: "gcs_method_ComposeObjects", + latencies: []time.Duration{100 * time.Millisecond, 200 * time.Millisecond}, + gcsMethod: "ComposeObjects", + }, + { + name: "gcs_method_CopyObject", + latencies: []time.Duration{100 * time.Millisecond, 200 * time.Millisecond}, + gcsMethod: "CopyObject", + }, + { + name: "gcs_method_CreateAppendableObjectWriter", + latencies: []time.Duration{100 * time.Millisecond, 200 * time.Millisecond}, + gcsMethod: "CreateAppendableObjectWriter", + }, + { + name: "gcs_method_CreateFolder", + latencies: []time.Duration{100 * time.Millisecond, 200 * time.Millisecond}, + gcsMethod: "CreateFolder", + }, + { + name: "gcs_method_CreateObject", + latencies: []time.Duration{100 * time.Millisecond, 200 * time.Millisecond}, + gcsMethod: "CreateObject", + }, + { + name: "gcs_method_CreateObjectChunkWriter", + latencies: []time.Duration{100 * time.Millisecond, 200 * time.Millisecond}, + gcsMethod: "CreateObjectChunkWriter", + }, + { + name: "gcs_method_DeleteFolder", + latencies: []time.Duration{100 * time.Millisecond, 200 * time.Millisecond}, + gcsMethod: "DeleteFolder", + }, + { + name: "gcs_method_DeleteObject", + latencies: []time.Duration{100 * time.Millisecond, 200 * time.Millisecond}, + gcsMethod: "DeleteObject", + }, + { + name: "gcs_method_FinalizeUpload", + latencies: []time.Duration{100 * time.Millisecond, 200 * time.Millisecond}, + gcsMethod: "FinalizeUpload", + }, + { + name: "gcs_method_FlushPendingWrites", + latencies: []time.Duration{100 * time.Millisecond, 200 * time.Millisecond}, + gcsMethod: "FlushPendingWrites", + }, + { + name: "gcs_method_GetFolder", + latencies: []time.Duration{100 * time.Millisecond, 200 * time.Millisecond}, + gcsMethod: "GetFolder", + }, + { + name: "gcs_method_ListObjects", + latencies: []time.Duration{100 * time.Millisecond, 200 * time.Millisecond}, + gcsMethod: "ListObjects", + }, + { + name: "gcs_method_MoveObject", + latencies: []time.Duration{100 * time.Millisecond, 200 * time.Millisecond}, + gcsMethod: "MoveObject", + }, + { + name: "gcs_method_MultiRangeDownloader::Add", + latencies: []time.Duration{100 * time.Millisecond, 200 * time.Millisecond}, + gcsMethod: "MultiRangeDownloader::Add", + }, + { + name: "gcs_method_NewMultiRangeDownloader", + latencies: []time.Duration{100 * time.Millisecond, 200 * time.Millisecond}, + gcsMethod: "NewMultiRangeDownloader", + }, + { + name: "gcs_method_NewReader", + latencies: []time.Duration{100 * time.Millisecond, 200 * time.Millisecond}, + gcsMethod: "NewReader", + }, + { + name: "gcs_method_RenameFolder", + latencies: []time.Duration{100 * time.Millisecond, 200 * time.Millisecond}, + gcsMethod: "RenameFolder", + }, + { + name: "gcs_method_StatObject", + latencies: []time.Duration{100 * time.Millisecond, 200 * time.Millisecond}, + gcsMethod: "StatObject", + }, + { + name: "gcs_method_UpdateObject", + latencies: []time.Duration{100 * time.Millisecond, 200 * time.Millisecond}, + gcsMethod: "UpdateObject", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + var totalLatency time.Duration + + for _, latency := range tc.latencies { + m.GcsRequestLatencies(ctx, latency, tc.gcsMethod) + totalLatency += latency + } + waitForMetricsProcessing() + + metrics := gatherHistogramMetrics(ctx, t, rd) + metric, ok := metrics["gcs/request_latencies"] + require.True(t, ok, "gcs/request_latencies metric not found") + + attrs := []attribute.KeyValue{ + attribute.String("gcs_method", string(tc.gcsMethod)), + } + s := attribute.NewSet(attrs...) + expectedKey := s.Encoded(encoder) + dp, ok := metric[expectedKey] + require.True(t, ok, "DataPoint not found for key: %s", expectedKey) + assert.Equal(t, uint64(len(tc.latencies)), dp.Count) + assert.Equal(t, totalLatency.Milliseconds(), dp.Sum) + }) + } +} + +func TestGcsRetryCount(t *testing.T) { + tests := []struct { + name string + f func(m *otelMetrics) + expected map[attribute.Set]int64 + }{ + { + name: "retry_error_category_OTHER_ERRORS", + f: func(m *otelMetrics) { + m.GcsRetryCount(5, "OTHER_ERRORS") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("retry_error_category", "OTHER_ERRORS")): 5, + }, + }, + { + name: "retry_error_category_STALLED_READ_REQUEST", + f: func(m *otelMetrics) { + m.GcsRetryCount(5, "STALLED_READ_REQUEST") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("retry_error_category", "STALLED_READ_REQUEST")): 5, + }, + }, { + name: "multiple_attributes_summed", + f: func(m *otelMetrics) { + m.GcsRetryCount(5, "OTHER_ERRORS") + m.GcsRetryCount(2, "STALLED_READ_REQUEST") + m.GcsRetryCount(3, "OTHER_ERRORS") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("retry_error_category", "OTHER_ERRORS")): 8, + attribute.NewSet(attribute.String("retry_error_category", "STALLED_READ_REQUEST")): 2, + }, + }, + { + name: "negative_increment", + f: func(m *otelMetrics) { + m.GcsRetryCount(-5, "OTHER_ERRORS") + m.GcsRetryCount(2, "OTHER_ERRORS") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("retry_error_category", "OTHER_ERRORS")): 2}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + + tc.f(m) + waitForMetricsProcessing() + + metrics := gatherNonZeroCounterMetrics(ctx, t, rd) + metric, ok := metrics["gcs/retry_count"] + if len(tc.expected) == 0 { + assert.False(t, ok, "gcs/retry_count metric should not be found") + return + } + require.True(t, ok, "gcs/retry_count metric not found") + expectedMap := make(map[string]int64) + for k, v := range tc.expected { + expectedMap[k.Encoded(encoder)] = v + } + assert.Equal(t, expectedMap, metric) + }) + } +} + +func TestMetadataCacheReadCount(t *testing.T) { + tests := []struct { + name string + f func(m *otelMetrics) + expected map[attribute.Set]int64 + }{ + { + name: "cache_hit_true_entry_status_negative_lookup_detail_found", + f: func(m *otelMetrics) { + m.MetadataCacheReadCount(5, true, "negative", "found") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("entry_status", "negative"), attribute.String("lookup_detail", "found")): 5, + }, + }, + { + name: "cache_hit_true_entry_status_negative_lookup_detail_not_found", + f: func(m *otelMetrics) { + m.MetadataCacheReadCount(5, true, "negative", "not_found") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("entry_status", "negative"), attribute.String("lookup_detail", "not_found")): 5, + }, + }, + { + name: "cache_hit_true_entry_status_negative_lookup_detail_ttl_expired", + f: func(m *otelMetrics) { + m.MetadataCacheReadCount(5, true, "negative", "ttl_expired") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("entry_status", "negative"), attribute.String("lookup_detail", "ttl_expired")): 5, + }, + }, + { + name: "cache_hit_true_entry_status_positive_lookup_detail_found", + f: func(m *otelMetrics) { + m.MetadataCacheReadCount(5, true, "positive", "found") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("entry_status", "positive"), attribute.String("lookup_detail", "found")): 5, + }, + }, + { + name: "cache_hit_true_entry_status_positive_lookup_detail_not_found", + f: func(m *otelMetrics) { + m.MetadataCacheReadCount(5, true, "positive", "not_found") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("entry_status", "positive"), attribute.String("lookup_detail", "not_found")): 5, + }, + }, + { + name: "cache_hit_true_entry_status_positive_lookup_detail_ttl_expired", + f: func(m *otelMetrics) { + m.MetadataCacheReadCount(5, true, "positive", "ttl_expired") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("entry_status", "positive"), attribute.String("lookup_detail", "ttl_expired")): 5, + }, + }, + { + name: "cache_hit_false_entry_status_negative_lookup_detail_found", + f: func(m *otelMetrics) { + m.MetadataCacheReadCount(5, false, "negative", "found") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("entry_status", "negative"), attribute.String("lookup_detail", "found")): 5, + }, + }, + { + name: "cache_hit_false_entry_status_negative_lookup_detail_not_found", + f: func(m *otelMetrics) { + m.MetadataCacheReadCount(5, false, "negative", "not_found") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("entry_status", "negative"), attribute.String("lookup_detail", "not_found")): 5, + }, + }, + { + name: "cache_hit_false_entry_status_negative_lookup_detail_ttl_expired", + f: func(m *otelMetrics) { + m.MetadataCacheReadCount(5, false, "negative", "ttl_expired") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("entry_status", "negative"), attribute.String("lookup_detail", "ttl_expired")): 5, + }, + }, + { + name: "cache_hit_false_entry_status_positive_lookup_detail_found", + f: func(m *otelMetrics) { + m.MetadataCacheReadCount(5, false, "positive", "found") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("entry_status", "positive"), attribute.String("lookup_detail", "found")): 5, + }, + }, + { + name: "cache_hit_false_entry_status_positive_lookup_detail_not_found", + f: func(m *otelMetrics) { + m.MetadataCacheReadCount(5, false, "positive", "not_found") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("entry_status", "positive"), attribute.String("lookup_detail", "not_found")): 5, + }, + }, + { + name: "cache_hit_false_entry_status_positive_lookup_detail_ttl_expired", + f: func(m *otelMetrics) { + m.MetadataCacheReadCount(5, false, "positive", "ttl_expired") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.Bool("cache_hit", false), attribute.String("entry_status", "positive"), attribute.String("lookup_detail", "ttl_expired")): 5, + }, + }, { + name: "multiple_attributes_summed", + f: func(m *otelMetrics) { + m.MetadataCacheReadCount(5, true, "negative", "found") + m.MetadataCacheReadCount(2, true, "negative", "not_found") + m.MetadataCacheReadCount(3, true, "negative", "found") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("entry_status", "negative"), attribute.String("lookup_detail", "found")): 8, + attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("entry_status", "negative"), attribute.String("lookup_detail", "not_found")): 2, + }, + }, + { + name: "negative_increment", + f: func(m *otelMetrics) { + m.MetadataCacheReadCount(-5, true, "negative", "found") + m.MetadataCacheReadCount(2, true, "negative", "found") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.Bool("cache_hit", true), attribute.String("entry_status", "negative"), attribute.String("lookup_detail", "found")): 2}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + + tc.f(m) + waitForMetricsProcessing() + + metrics := gatherNonZeroCounterMetrics(ctx, t, rd) + metric, ok := metrics["metadata_cache/read_count"] + if len(tc.expected) == 0 { + assert.False(t, ok, "metadata_cache/read_count metric should not be found") + return + } + require.True(t, ok, "metadata_cache/read_count metric not found") + expectedMap := make(map[string]int64) + for k, v := range tc.expected { + expectedMap[k.Encoded(encoder)] = v + } + assert.Equal(t, expectedMap, metric) + }) + } +} + +func TestReadBlockSizes(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + var totalValue int64 + values := []int64{100, 200} + + for _, value := range values { + m.ReadBlockSizes(ctx, value) + totalValue += value + } + waitForMetricsProcessing() + + metrics := gatherHistogramMetrics(ctx, t, rd) + metric, ok := metrics["read/block_sizes"] + require.True(t, ok, "read/block_sizes metric not found") + + s := attribute.NewSet() + expectedKey := s.Encoded(encoder) + dp, ok := metric[expectedKey] + require.True(t, ok, "DataPoint not found for key: %s", expectedKey) + assert.Equal(t, uint64(len(values)), dp.Count) + assert.Equal(t, totalValue, dp.Sum) +} + +func TestTestUpdownCounter(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + + m.TestUpdownCounter(1024) + m.TestUpdownCounter(2048) + waitForMetricsProcessing() + + metrics := gatherNonZeroCounterMetrics(ctx, t, rd) + metric, ok := metrics["test/updown_counter"] + require.True(t, ok, "test/updown_counter metric not found") + s := attribute.NewSet() + assert.Equal(t, map[string]int64{s.Encoded(encoder): 3072}, metric, "Positive increments should be summed.") + + // Test negative increment + m.TestUpdownCounter(-100) + waitForMetricsProcessing() + + metrics = gatherNonZeroCounterMetrics(ctx, t, rd) + metric, ok = metrics["test/updown_counter"] + require.True(t, ok, "test/updown_counter metric not found after negative increment") + assert.Equal(t, map[string]int64{s.Encoded(encoder): 2972}, metric, "Negative increment should change the metric value.") +} + +func TestTestUpdownCounterWithAttrs(t *testing.T) { + tests := []struct { + name string + f func(m *otelMetrics) + expected map[attribute.Set]int64 + }{ + { + name: "request_type_attr1", + f: func(m *otelMetrics) { + m.TestUpdownCounterWithAttrs(5, "attr1") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("request_type", "attr1")): 5, + }, + }, + { + name: "request_type_attr2", + f: func(m *otelMetrics) { + m.TestUpdownCounterWithAttrs(5, "attr2") + }, + expected: map[attribute.Set]int64{ + attribute.NewSet(attribute.String("request_type", "attr2")): 5, + }, + }, { + name: "multiple_attributes_summed", + f: func(m *otelMetrics) { + m.TestUpdownCounterWithAttrs(5, "attr1") + m.TestUpdownCounterWithAttrs(2, "attr2") + m.TestUpdownCounterWithAttrs(3, "attr1") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("request_type", "attr1")): 8, + attribute.NewSet(attribute.String("request_type", "attr2")): 2, + }, + }, + { + name: "negative_increment", + f: func(m *otelMetrics) { + m.TestUpdownCounterWithAttrs(-5, "attr1") + m.TestUpdownCounterWithAttrs(2, "attr1") + }, + expected: map[attribute.Set]int64{attribute.NewSet(attribute.String("request_type", "attr1")): -3}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + + tc.f(m) + waitForMetricsProcessing() + + metrics := gatherNonZeroCounterMetrics(ctx, t, rd) + metric, ok := metrics["test/updown_counter_with_attrs"] + if len(tc.expected) == 0 { + assert.False(t, ok, "test/updown_counter_with_attrs metric should not be found") + return + } + require.True(t, ok, "test/updown_counter_with_attrs metric not found") + expectedMap := make(map[string]int64) + for k, v := range tc.expected { + expectedMap[k.Encoded(encoder)] = v + } + assert.Equal(t, expectedMap, metric) + }) + } +} diff --git a/perfmetrics/scripts/README.md b/perfmetrics/scripts/README.md index f09354c719..6d0ea88944 100644 --- a/perfmetrics/scripts/README.md +++ b/perfmetrics/scripts/README.md @@ -47,7 +47,7 @@ pip install --require-hashes -r requirements.txt --user ``` 8. Create a service account by following this [documentation](https://cloud.google.com/iam/docs/creating-managing-service-accounts). Generate your service account key, `creds.json` by following [this doc](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#iam-service-account-keys-create-console) and upload the file on your GCS bucket `your-bucket-name`. If using an old credentials file, make sure that it is not expired. Run the following command to copy it into `gsheet` directory: ```bash -gsutil cp gs://your-bucket-name/creds.json ./gsheet +gcloud storage cp gs://your-bucket-name/creds.json ./gsheet ``` 9. Create a Google Sheet with id `your-gsheet-id` by copying this [Google Sheet](https://docs.google.com/spreadsheets/d/1IJIjWuEs7cL6eYqPmlVaEGdclr6MSiaKJdnFXXC5tg8/). 10. Share the above copied Google Sheet with your service account(created in step 8) diff --git a/perfmetrics/scripts/bigquery/requirements.txt b/perfmetrics/scripts/bigquery/requirements.txt index 4ff1114522..889250f74b 100644 --- a/perfmetrics/scripts/bigquery/requirements.txt +++ b/perfmetrics/scripts/bigquery/requirements.txt @@ -273,20 +273,29 @@ proto-plus==1.22.2 \ --hash=sha256:0e8cda3d5a634d9895b75c573c9352c16486cb75deb0e078b5fda34db4243165 \ --hash=sha256:de34e52d6c9c6fcd704192f09767cb561bb4ee64e70eede20b0834d841f0be4d # via google-cloud-bigquery -protobuf==4.23.2 \ - --hash=sha256:09310bce43353b46d73ba7e3bca78273b9bc50349509b9698e64d288c6372c2a \ - --hash=sha256:20874e7ca4436f683b64ebdbee2129a5a2c301579a67d1a7dda2cdf62fb7f5f7 \ - --hash=sha256:25e3370eda26469b58b602e29dff069cfaae8eaa0ef4550039cc5ef8dc004511 \ - --hash=sha256:281342ea5eb631c86697e1e048cb7e73b8a4e85f3299a128c116f05f5c668f8f \ - --hash=sha256:384dd44cb4c43f2ccddd3645389a23ae61aeb8cfa15ca3a0f60e7c3ea09b28b3 \ - --hash=sha256:54a533b971288af3b9926e53850c7eb186886c0c84e61daa8444385a4720297f \ - --hash=sha256:6c081863c379bb1741be8f8193e893511312b1d7329b4a75445d1ea9955be69e \ - --hash=sha256:86df87016d290143c7ce3be3ad52d055714ebaebb57cc659c387e76cfacd81aa \ - --hash=sha256:8da6070310d634c99c0db7df48f10da495cc283fd9e9234877f0cd182d43ab7f \ - --hash=sha256:b2cfab63a230b39ae603834718db74ac11e52bccaaf19bf20f5cce1a84cf76df \ - --hash=sha256:c52cfcbfba8eb791255edd675c1fe6056f723bf832fa67f0442218f8817c076e \ - --hash=sha256:ce744938406de1e64b91410f473736e815f28c3b71201302612a68bf01517fea \ - --hash=sha256:efabbbbac1ab519a514579ba9ec52f006c28ae19d97915951f69fa70da2c9e91 +protobuf==4.25.8 \ + --hash=sha256:077ff8badf2acf8bc474406706ad890466274191a48d0abd3bd6987107c9cde5 \ + --hash=sha256:077ff8badf2acf8bc474406706ad890466274191a48d0abd3bd6987107c9cde5 \ + --hash=sha256:15a0af558aa3b13efef102ae6e4f3efac06f1eea11afb3a57db2901447d9fb59 \ + --hash=sha256:15a0af558aa3b13efef102ae6e4f3efac06f1eea11afb3a57db2901447d9fb59 \ + --hash=sha256:27d498ffd1f21fb81d987a041c32d07857d1d107909f5134ba3350e1ce80a4af \ + --hash=sha256:27d498ffd1f21fb81d987a041c32d07857d1d107909f5134ba3350e1ce80a4af \ + --hash=sha256:504435d831565f7cfac9f0714440028907f1975e4bed228e58e72ecfff58a1e0 \ + --hash=sha256:504435d831565f7cfac9f0714440028907f1975e4bed228e58e72ecfff58a1e0 \ + --hash=sha256:6135cf8affe1fc6f76cced2641e4ea8d3e59518d1f24ae41ba97bcad82d397cd \ + --hash=sha256:6135cf8affe1fc6f76cced2641e4ea8d3e59518d1f24ae41ba97bcad82d397cd \ + --hash=sha256:83e6e54e93d2b696a92cad6e6efc924f3850f82b52e1563778dfab8b355101b0 \ + --hash=sha256:83e6e54e93d2b696a92cad6e6efc924f3850f82b52e1563778dfab8b355101b0 \ + --hash=sha256:9ad7ef62d92baf5a8654fbb88dac7fa5594cfa70fd3440488a5ca3bfc6d795a7 \ + --hash=sha256:9ad7ef62d92baf5a8654fbb88dac7fa5594cfa70fd3440488a5ca3bfc6d795a7 \ + --hash=sha256:bd551eb1fe1d7e92c1af1d75bdfa572eff1ab0e5bf1736716814cdccdb2360f9 \ + --hash=sha256:bd551eb1fe1d7e92c1af1d75bdfa572eff1ab0e5bf1736716814cdccdb2360f9 \ + --hash=sha256:ca809b42f4444f144f2115c4c1a747b9a404d590f18f37e9402422033e464e0f \ + --hash=sha256:ca809b42f4444f144f2115c4c1a747b9a404d590f18f37e9402422033e464e0f \ + --hash=sha256:d552c53d0415449c8d17ced5c341caba0d89dbf433698e1436c8fa0aae7808a3 \ + --hash=sha256:d552c53d0415449c8d17ced5c341caba0d89dbf433698e1436c8fa0aae7808a3 \ + --hash=sha256:f4510b93a3bec6eba8fd8f1093e9d7fb0d4a24d1a81377c10c0e5bbfe9e4ed24 \ + --hash=sha256:f4510b93a3bec6eba8fd8f1093e9d7fb0d4a24d1a81377c10c0e5bbfe9e4ed24 # via # google-api-core # google-cloud-bigquery @@ -307,9 +316,9 @@ python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via google-cloud-bigquery -requests==2.32.2 \ - --hash=sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289 \ - --hash=sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c +requests==2.33.0 \ + --hash=sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b \ + --hash=sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652 # via # google-api-core # google-cloud-bigquery diff --git a/perfmetrics/scripts/build_and_install_gcsfuse.sh b/perfmetrics/scripts/build_and_install_gcsfuse.sh index 7e7035c861..bf3f84d00d 100755 --- a/perfmetrics/scripts/build_and_install_gcsfuse.sh +++ b/perfmetrics/scripts/build_and_install_gcsfuse.sh @@ -16,22 +16,35 @@ # This script will build gcsfuse package on given commitId or branch and install it on the machine. # This will stop execution when any command will have non-zero status. set -e -# e.g. architecture=arm64 or amd64 + +# --- Determine architecture (e.g., amd64, arm64) --- architecture=$(dpkg --print-architecture) -echo "Installing docker..." -sudo mkdir -p /etc/apt/keyrings -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg -echo \ - "deb [arch=${architecture} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ - $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null -sudo apt-get update -sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin -y +# Install Docker if it's not already present or if this is a Kokoro environment as it has old docker version. +if ! command -v docker &> /dev/null || [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then + echo "Installing Docker..." + sudo mkdir -p /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + echo \ + "deb [arch=${architecture} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin -y +else + echo "Docker is already installed. Skipping Docker installation." +fi + +# --- Build and install gcsfuse --- echo "Building and installing gcsfuse..." -# $1 refers to branch or commit-id on which we want to build package. branch=$1 -# Build the gcsfuse package using the same commands used during release. +if [ -z "$branch" ]; then + echo "Usage: $0 <branch-or-commit-id>" + exit 1 +fi + GCSFUSE_VERSION=0.0.0 + +# Build the gcsfuse package using Docker sudo docker buildx build --load ./tools/package_gcsfuse_docker/ -t gcsfuse:$branch --build-arg ARCHITECTURE=${architecture} --build-arg GCSFUSE_VERSION=$GCSFUSE_VERSION --build-arg BRANCH_NAME=$branch --platform=linux/${architecture} sudo docker run -v $HOME/release:/release gcsfuse:$branch cp -r /packages /release/ sudo dpkg -i $HOME/release/packages/gcsfuse_${GCSFUSE_VERSION}_${architecture}.deb diff --git a/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh b/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh index f68377b275..b2cd2eb8ee 100755 --- a/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh @@ -13,82 +13,196 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -e -sudo apt-get update +# Exit on error, treat unset variables as errors, and propagate pipeline errors. +set -euo pipefail +# ----------------------------------------------------------------- +# Environment Setup +# ----------------------------------------------------------------- +sudo apt-get update echo "Installing git" -sudo apt-get install git +sudo apt-get install -y git cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse" -echo "Building and installing gcsfuse" -# Get the latest commitId of yesterday in the log file. Build gcsfuse and run +# Get the latest commitId of yesterday in the log file. commitId=$(git log --before='yesterday 23:59:59' --max-count=1 --pretty=%H) -./perfmetrics/scripts/build_and_install_gcsfuse.sh $commitId - -# Mounting gcs bucket -cd "./perfmetrics/scripts/" - -echo Installing Bigquery module requirements... -pip install --require-hashes -r bigquery/requirements.txt --user - -# Upload data to the gsheet only when it runs through kokoro. -UPLOAD_FLAGS="" -if [ "${KOKORO_JOB_TYPE}" == "RELEASE" ] || [ "${KOKORO_JOB_TYPE}" == "CONTINUOUS_INTEGRATION" ] || [ "${KOKORO_JOB_TYPE}" == "PRESUBMIT_GITHUB" ] || [ "${KOKORO_JOB_TYPE}" == "SUB_JOB" ]; -then - UPLOAD_FLAGS="--upload_gs" -fi -run_load_test_and_fetch_metrics(){ - fio_flags=$1 - gcsfuse_flags="$COMMON_MOUNT_FLAGS $fio_flags" - bucket_name=$2 - spreadsheet_id=$3 - - # Executing perf tests - ./run_load_test_and_fetch_metrics.sh "$gcsfuse_flags" "$UPLOAD_FLAGS" "$bucket_name" "$spreadsheet_id" +# ----------------------------------------------------------------- +# Helper function to calculate and print execution time +# ----------------------------------------------------------------- +print_duration() { + local task_name="$1" + local start_time="$2" + local end_time=$SECONDS + local duration=$((end_time - start_time)) + echo "=================================================================" + echo "⏱️ EXECUTION TIME - ${task_name}: ${duration} seconds" + echo "=================================================================" } -run_ls_benchmark(){ - ls_flags=$1 - gcsfuse_flags="$COMMON_MOUNT_FLAGS $ls_flags" - spreadsheet_id="$2" - config_file="$3" +# Record start time of the entire script +TOTAL_START=$SECONDS - cd "./ls_metrics" - ./run_ls_benchmark.sh "$gcsfuse_flags" "$UPLOAD_FLAGS" "$spreadsheet_id" "$config_file" - cd "../" +# Trap to always print total execution time on exit +exit_handler() { + print_duration "Total Execution Time" "$TOTAL_START" } - -COMMON_MOUNT_FLAGS="--debug_fuse --debug_gcs --log-format \"text\"" - -# Testing for flat bucket. -LOG_FILE_FIO_TESTS=${KOKORO_ARTIFACTS_DIR}/gcsfuse-logs-fio-flat.txt -LOG_FILE_LS_TESTS=${KOKORO_ARTIFACTS_DIR}/gcsfuse-logs-ls-flat.txt -GCSFUSE_FIO_FLAGS="--implicit-dirs --stackdriver-export-interval=30s --log-file $LOG_FILE_FIO_TESTS" -GCSFUSE_LS_FLAGS="--implicit-dirs --log-file $LOG_FILE_LS_TESTS" -BUCKET_NAME="periodic-perf-tests" -SPREADSHEET_ID='1kvHv1OBCzr9GnFxRu9RTJC7jjQjc9M4rAiDnhyak2Sg' -LIST_CONFIG_FILE="config.json" -run_load_test_and_fetch_metrics "$GCSFUSE_FIO_FLAGS" "$BUCKET_NAME" "$SPREADSHEET_ID" -run_ls_benchmark "$GCSFUSE_LS_FLAGS" "$SPREADSHEET_ID" "$LIST_CONFIG_FILE" - -# Testing for hns bucket. -LOG_FILE_FIO_TESTS=${KOKORO_ARTIFACTS_DIR}/gcsfuse-logs-fio-hns.txt -LOG_FILE_LS_TESTS=${KOKORO_ARTIFACTS_DIR}/gcsfuse-logs-ls-hns.txt -GCSFUSE_FIO_FLAGS="--stackdriver-export-interval=30s --log-file $LOG_FILE_FIO_TESTS" -GCSFUSE_LS_FLAGS="--log-file $LOG_FILE_LS_TESTS" -BUCKET_NAME="periodic-perf-tests-hns" -SPREADSHEET_ID='1wXRGYyAWvasU8U4KaP7NGPHEvgiOSgMd1sCLxsQUwf0' -LIST_CONFIG_FILE="config-hns.json" -run_load_test_and_fetch_metrics "$GCSFUSE_FIO_FLAGS" "$BUCKET_NAME" "$SPREADSHEET_ID" -run_ls_benchmark "$GCSFUSE_LS_FLAGS" "$SPREADSHEET_ID" "$LIST_CONFIG_FILE" - -#Running the rename benchmark script. -cd "./hns_rename_folders_metrics" -./run_rename_benchmark.sh $UPLOAD_FLAGS - - +trap exit_handler EXIT + +# ================================================================= +# 1) DISTRIBUTED READ BENCHMARK +# ================================================================= +if [ "${BENCHMARK_TYPE:-}" == "distributed_benchmark_read" ]; then + TOOLS_DIR="${KOKORO_ARTIFACTS_DIR}/github/gcsfuse-tools" + PERF_BENCHMARKS_FAILED=0 + + if [ -d "$TOOLS_DIR" ]; then + echo "Running Distributed READ Micro-Benchmark from gcsfuse-tools..." + START_TIME=$SECONDS + "$TOOLS_DIR/distributed-micro-benchmark/kokoro_run.sh" --commit "$commitId" --read || PERF_BENCHMARKS_FAILED=1 + print_duration "Distributed READ Benchmark" "$START_TIME" + else + echo "ERROR: gcsfuse-tools directory not found!" + PERF_BENCHMARKS_FAILED=1 + fi + + if [ $PERF_BENCHMARKS_FAILED -ne 0 ]; then + echo "Distributed READ benchmarks have failed." + exit 1 + fi + +# ================================================================= +# 2) DISTRIBUTED WRITE BENCHMARK +# ================================================================= +elif [ "${BENCHMARK_TYPE:-}" == "distributed_benchmark_write" ]; then + TOOLS_DIR="${KOKORO_ARTIFACTS_DIR}/github/gcsfuse-tools" + PERF_BENCHMARKS_FAILED=0 + + if [ -d "$TOOLS_DIR" ]; then + echo "Running Distributed WRITE Micro-Benchmark from gcsfuse-tools..." + START_TIME=$SECONDS + "$TOOLS_DIR/distributed-micro-benchmark/kokoro_run.sh" --commit "$commitId" --write || PERF_BENCHMARKS_FAILED=1 + print_duration "Distributed WRITE Benchmark" "$START_TIME" + else + echo "ERROR: gcsfuse-tools directory not found!" + PERF_BENCHMARKS_FAILED=1 + fi + + if [ $PERF_BENCHMARKS_FAILED -ne 0 ]; then + echo "Distributed WRITE benchmarks have failed." + exit 1 + fi + +# ================================================================= +# 3) LOCAL PERFORMANCE TESTS +# ================================================================= +elif [ "${BENCHMARK_TYPE:-}" == "local_tests" ]; then + # --- Execute local performance tests --- + echo "Building and installing gcsfuse..." + BUILD_START=$SECONDS + ./perfmetrics/scripts/build_and_install_gcsfuse.sh "$commitId" + print_duration "Build and Install GCSFuse" "$BUILD_START" + + cd "./perfmetrics/scripts/" + echo "Installing Bigquery module requirements..." + pip install --require-hashes -r bigquery/requirements.txt --user + + UPLOAD_FLAGS="" + if [ "${KOKORO_JOB_TYPE:-}" == "RELEASE" ] || \ + [ "${KOKORO_JOB_TYPE:-}" == "CONTINUOUS_INTEGRATION" ] || \ + [ "${KOKORO_JOB_TYPE:-}" == "PRESUBMIT_GITHUB" ] || \ + [ "${KOKORO_JOB_TYPE:-}" == "SUB_JOB" ]; then + UPLOAD_FLAGS="--upload_gs" + fi + + COMMON_MOUNT_FLAGS="--debug_fuse --debug_gcs --log-format \"text\"" + + run_load_test_and_fetch_metrics() { + local FIO_START=$SECONDS + local fio_flags="$1" + local bucket_name="$2" + local spreadsheet_id="$3" + local gcsfuse_flags="$COMMON_MOUNT_FLAGS $fio_flags" + + echo "Starting FIO Load Test on $bucket_name..." + ./run_load_test_and_fetch_metrics.sh "$gcsfuse_flags" "$UPLOAD_FLAGS" "$bucket_name" "$spreadsheet_id" + print_duration "FIO Load Test ($bucket_name)" "$FIO_START" + } + + run_ls_benchmark() { + local LS_START=$SECONDS + local ls_flags="$1" + local spreadsheet_id="$2" + local config_file="$3" + local gcsfuse_flags="$COMMON_MOUNT_FLAGS $ls_flags" + + echo "Starting LS Benchmark with $config_file..." + cd "./ls_metrics" + ./run_ls_benchmark.sh "$gcsfuse_flags" "$UPLOAD_FLAGS" "$spreadsheet_id" "$config_file" + cd "../" + print_duration "LS Benchmark ($config_file)" "$LS_START" + } + + # --- Flat Bucket Tests --- + echo "Starting Flat Bucket Tests..." + FLAT_START=$SECONDS + + LOG_FILE_FIO_TESTS="${KOKORO_ARTIFACTS_DIR}/gcsfuse-logs-fio-flat.txt" + LOG_FILE_LS_TESTS="${KOKORO_ARTIFACTS_DIR}/gcsfuse-logs-ls-flat.txt" + GCSFUSE_FIO_FLAGS="--implicit-dirs --stackdriver-export-interval=30s --log-file $LOG_FILE_FIO_TESTS" + GCSFUSE_LS_FLAGS="--implicit-dirs --log-file $LOG_FILE_LS_TESTS" + BUCKET_NAME="periodic-perf-tests" + SPREADSHEET_ID='1kvHv1OBCzr9GnFxRu9RTJC7jjQjc9M4rAiDnhyak2Sg' + LIST_CONFIG_FILE="config.json" + + run_load_test_and_fetch_metrics "$GCSFUSE_FIO_FLAGS" "$BUCKET_NAME" "$SPREADSHEET_ID" + run_ls_benchmark "$GCSFUSE_LS_FLAGS" "$SPREADSHEET_ID" "$LIST_CONFIG_FILE" + + print_duration "Flat Bucket Benchmarks (Total)" "$FLAT_START" + + # --- HNS Bucket Tests --- + echo "Starting HNS Bucket Tests..." + HNS_START=$SECONDS + + LOG_FILE_FIO_TESTS="${KOKORO_ARTIFACTS_DIR}/gcsfuse-logs-fio-hns.txt" + LOG_FILE_LS_TESTS="${KOKORO_ARTIFACTS_DIR}/gcsfuse-logs-ls-hns.txt" + GCSFUSE_FIO_FLAGS="--stackdriver-export-interval=30s --log-file $LOG_FILE_FIO_TESTS" + GCSFUSE_LS_FLAGS="--log-file $LOG_FILE_LS_TESTS" + BUCKET_NAME="periodic-perf-tests-hns" + SPREADSHEET_ID='1wXRGYyAWvasU8U4KaP7NGPHEvgiOSgMd1sCLxsQUwf0' + LIST_CONFIG_FILE="config-hns.json" + + run_load_test_and_fetch_metrics "$GCSFUSE_FIO_FLAGS" "$BUCKET_NAME" "$SPREADSHEET_ID" + run_ls_benchmark "$GCSFUSE_LS_FLAGS" "$SPREADSHEET_ID" "$LIST_CONFIG_FILE" + + print_duration "HNS Bucket Benchmarks (Total)" "$HNS_START" + + # --- Rename Benchmark --- + echo "Starting Rename Benchmark..." + RENAME_START=$SECONDS + + cd "./hns_rename_folders_metrics" + ./run_rename_benchmark.sh $UPLOAD_FLAGS + + print_duration "Rename Benchmark" "$RENAME_START" + +# ================================================================= +# 4) ZONAL PERFORMANCE TESTS +# ================================================================= +elif [ "${BENCHMARK_TYPE:-}" == "distributed_benchmark_zonal" ]; then + echo "Running Zonal Performance Tests..." + START_TIME=$SECONDS + + # TODO: Add upcoming zonal performance tests. + echo "Zonal tests scaffolding ready." + + print_duration "Zonal Performance Tests" "$START_TIME" + +else + echo "Unknown or unspecified BENCHMARK_TYPE: ${BENCHMARK_TYPE:-}" + exit 1 +fi # TODO: Testing for hns bucket with client protocol set to grpc. To be done when # includeFolderAsPrefixes is supported in grpc. # TODO: Testing for hns bucket with client protocol set to grpc with grpc-conn-pool-size diff --git a/perfmetrics/scripts/continuous_test/gcp_ubuntu/continuous.cfg b/perfmetrics/scripts/continuous_test/gcp_ubuntu/continuous.cfg index 8c4967c5f7..078db7b121 100644 --- a/perfmetrics/scripts/continuous_test/gcp_ubuntu/continuous.cfg +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/continuous.cfg @@ -19,9 +19,11 @@ action { regex: "gcsfuse-logs-fio-flat.txt" regex: "gcsfuse-logs-ls-flat.txt" regex: "github/gcsfuse/perfmetrics/scripts/fio-output.json" + regex: "**/*sponge_log.*" strip_prefix: "github/gcsfuse/perfmetrics/scripts" } } +# Increase timeout to 6 hours +timeout_mins: 360 build_file: "gcsfuse/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh" - diff --git a/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/build.sh b/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/build.sh index f84b9e86e5..f9575c8177 100755 --- a/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/build.sh +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/build.sh @@ -13,20 +13,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This script will run e2e tests. -# This will stop execution when any command will have non-zero status. -set -e +# Script to run e2e tests for regional or zonal buckets if env variable RUN_TESTS_WITH_ZONAL_BUCKET is set to 'true'. +# Exit on error, treat unset variables as errors, and propagate pipeline errors. +set -euo pipefail -readonly RUN_E2E_TESTS_ON_INSTALLED_PACKAGE=true -readonly SKIP_NON_ESSENTIAL_TESTS_ON_PACKAGE=false -readonly RUN_TEST_ON_TPC_ENDPOINT=false -readonly PROJECT_ID="gcs-fuse-test-ml" -readonly BUCKET_LOCATION=us-central1 +if [[ $# -gt 0 ]]; then + echo "This script requires no argument. Pass env variable RUN_TESTS_WITH_ZONAL_BUCKET set to 'true' to run this script for zonal buckets." + exit 1 +fi -# This flag, if set true, will indicate to the underlying script, to customize for a presubmit-run. -readonly RUN_TESTS_WITH_PRESUBMIT_FLAG=false +readonly BUCKET_LOCATION="us-central1" cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse" + echo "Building and installing gcsfuse..." # Get the latest commitId of yesterday in the log file. Build gcsfuse and run commitId=$(git log --before='yesterday 23:59:59' --max-count=1 --pretty=%H) @@ -35,6 +34,15 @@ commitId=$(git log --before='yesterday 23:59:59' --max-count=1 --pretty=%H) # To execute tests for a specific commitId, ensure you've checked out from that commitId first. git checkout $commitId -echo "Running e2e tests on installed package...." -# $1 argument is refering to value of testInstalledPackage -./tools/integration_tests/run_e2e_tests.sh $RUN_E2E_TESTS_ON_INSTALLED_PACKAGE $SKIP_NON_ESSENTIAL_TESTS_ON_PACKAGE $BUCKET_LOCATION $RUN_TEST_ON_TPC_ENDPOINT $RUN_TESTS_WITH_PRESUBMIT_FLAG +if [[ "${RUN_TESTS_WITH_ZONAL_BUCKET-}" == "true" ]]; then + echo "Running zonal e2e tests on installed package...." + bash ./tools/integration_tests/improved_run_e2e_tests.sh --bucket-location="$BUCKET_LOCATION" --test-installed-package --zonal +else + if [[ -n "${RUN_TESTS_WITH_ZONAL_BUCKET-}" ]]; then + echo "Warning: RUN_TESTS_WITH_ZONAL_BUCKET is set to '${RUN_TESTS_WITH_ZONAL_BUCKET}', which is not 'true'. Running regional tests." + else + echo "RUN_TESTS_WITH_ZONAL_BUCKET is not set. Running regional tests by default." + fi + echo "Running regional e2e tests on installed package...." + bash ./tools/integration_tests/improved_run_e2e_tests.sh --bucket-location="$BUCKET_LOCATION" --test-installed-package +fi diff --git a/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/checkpoint-tests.cfg b/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/checkpoint-tests.cfg new file mode 100644 index 0000000000..de400298dc --- /dev/null +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/checkpoint-tests.cfg @@ -0,0 +1,23 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +action { + define_artifacts { + regex: "gcsfuse_logs/*" + regex: "**/*sponge_log.*" + strip_prefix: "github/gcsfuse/perfmetrics/scripts" + } +} + +build_file: "gcsfuse/perfmetrics/scripts/ml_tests/checkpoint/Jax/run_checkpoints.sh" diff --git a/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/e2e-tests-master-zb.cfg b/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/e2e-tests-master-zb.cfg new file mode 100644 index 0000000000..99cc07894b --- /dev/null +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/e2e-tests-master-zb.cfg @@ -0,0 +1,29 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +action { + define_artifacts { + regex: "gcsfuse-failed-integration-test-logs-*" + strip_prefix: "github/gcsfuse/perfmetrics/scripts" + regex: "**/*sponge_log.*" + regex: "proxy*" + } +} + +env_vars { + key: "RUN_TESTS_WITH_ZONAL_BUCKET" + value: "true" +} + +build_file: "gcsfuse/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/build.sh" diff --git a/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/e2e-tests-master.cfg b/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/e2e-tests-master.cfg index d954cc48f0..74eba8277f 100644 --- a/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/e2e-tests-master.cfg +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/e2e-tests-master.cfg @@ -15,6 +15,8 @@ action { define_artifacts { regex: "gcsfuse-failed-integration-test-logs-*" + regex: "proxy-server-failed-integration-test-logs-*" + regex: "**/*sponge_log.*" strip_prefix: "github/gcsfuse/perfmetrics/scripts" } } diff --git a/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/e2e-tests-release.cfg b/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/e2e-tests-release.cfg index d954cc48f0..74eba8277f 100644 --- a/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/e2e-tests-release.cfg +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/e2e-tests-release.cfg @@ -15,6 +15,8 @@ action { define_artifacts { regex: "gcsfuse-failed-integration-test-logs-*" + regex: "proxy-server-failed-integration-test-logs-*" + regex: "**/*sponge_log.*" strip_prefix: "github/gcsfuse/perfmetrics/scripts" } } diff --git a/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/e2e-tests-tpc.cfg b/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/e2e-tests-tpc.cfg index 6417584ab8..96250f3386 100644 --- a/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/e2e-tests-tpc.cfg +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/e2e-tests-tpc.cfg @@ -15,6 +15,7 @@ action { define_artifacts { regex: "gcsfuse-failed-integration-test-logs-*" + regex: "**/*sponge_log.*" strip_prefix: "github/gcsfuse/perfmetrics/scripts" } } diff --git a/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/tpc_build.sh b/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/tpc_build.sh old mode 100644 new mode 100755 index 01bd1dac22..cdb3845fdb --- a/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/tpc_build.sh +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/tpc_build.sh @@ -13,30 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This script will run e2e tests for tpc. -# This will stop execution when any command will have non-zero status. -set -e +# Script to run e2e tests for tpc universe. +# Exit on error, treat unset variables as errors, and propagate pipeline errors. +set -euo pipefail + +if [[ $# -gt 0 ]]; then + echo "This script requires no argument" + exit 1 +fi -readonly RUN_E2E_TESTS_ON_INSTALLED_PACKAGE=true -readonly SKIP_NON_ESSENTIAL_TESTS_ON_PACKAGE=true -readonly RUN_TEST_ON_TPC_ENDPOINT=true # TPC project id readonly PROJECT_ID="tpczero-system:gcsfuse-test-project" readonly BUCKET_LOCATION="u-us-prp1" -# This flag, if set true, will indicate to underlying script to customize for a presubmit run. -readonly RUN_TESTS_WITH_PRESUBMIT_FLAG=false - cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse" -# Upgrade gcloud version -gcloud version -wget -O gcloud.tar.gz https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz -q -sudo tar xzf gcloud.tar.gz && sudo cp -r google-cloud-sdk /usr/local && sudo rm -r google-cloud-sdk -sudo /usr/local/google-cloud-sdk/install.sh -export PATH=/usr/local/google-cloud-sdk/bin:$PATH -echo 'export PATH=/usr/local/google-cloud-sdk/bin:$PATH' >> ~/.bashrc -gcloud version && rm gcloud.tar.gz +# Install latest gcloud. +./perfmetrics/scripts/install_latest_gcloud.sh +export PATH="/usr/local/google-cloud-sdk/bin:$PATH" # Copy the key file for the TPC service account to use for authentication. gcloud storage cp gs://gcsfuse-tpc-tests/creds.json /tmp/sa.key.json @@ -59,8 +53,7 @@ gcloud auth activate-service-account --key-file=/tmp/sa.key.json gcloud config set project $PROJECT_ID set +e -# $1 argument is refering to value of testInstalledPackage -./tools/integration_tests/run_e2e_tests.sh $RUN_E2E_TESTS_ON_INSTALLED_PACKAGE $SKIP_NON_ESSENTIAL_TESTS_ON_PACKAGE $BUCKET_LOCATION $RUN_TEST_ON_TPC_ENDPOINT $RUN_TESTS_WITH_PRESUBMIT_FLAG +bash ./tools/integration_tests/improved_run_e2e_tests.sh --bucket-location=$BUCKET_LOCATION --test-installed-package --skip-non-essential-tests --test-on-tpc-endpoint exit_code=$? set -e diff --git a/perfmetrics/scripts/ml_tests/pytorch/v1_12/dino/setup_host_and_run_container.sh b/perfmetrics/scripts/continuous_test/gcp_ubuntu/local_tests.cfg old mode 100755 new mode 100644 similarity index 51% rename from perfmetrics/scripts/ml_tests/pytorch/v1_12/dino/setup_host_and_run_container.sh rename to perfmetrics/scripts/continuous_test/gcp_ubuntu/local_tests.cfg index 98c874b25c..523a717772 --- a/perfmetrics/scripts/ml_tests/pytorch/v1_12/dino/setup_host_and_run_container.sh +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/local_tests.cfg @@ -1,5 +1,4 @@ -#!/bin/bash -# Copyright 2023 Google LLC +# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,16 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This will stop execution when any command will have non-zero status. -set -e +action { + define_artifacts { + regex: "gcsfuse-logs-fio-hns.txt" + regex: "gcsfuse-logs-ls-hns.txt" + regex: "gcsfuse-logs-fio-flat.txt" + regex: "gcsfuse-logs-ls-flat.txt" + regex: "github/gcsfuse/perfmetrics/scripts/fio-output.json" + regex: "**/*sponge_log.*" + strip_prefix: "github/gcsfuse/perfmetrics/scripts" + } +} -cd "$HOME/github/gcsfuse/perfmetrics/scripts" +timeout_mins: 300 +build_file: "gcsfuse/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh" -echo "Setting up the machine with Docker and Nvidia Driver" -# Driver version for A100 GPUs is 450.172.01 -DRIVER_VERSION="450.172.01" -source ml_tests/setup_host.sh $DRIVER_VERSION - -PYTORCH_VERSION="v1_12" -BUCKET_TYPE=$1 -source ml_tests/pytorch/run_container.sh $PYTORCH_VERSION $BUCKET_TYPE +env_vars { + key: "BENCHMARK_TYPE" + value: "local_tests" +} diff --git a/perfmetrics/scripts/continuous_test/gcp_ubuntu/micro_benchmarks/build.sh b/perfmetrics/scripts/continuous_test/gcp_ubuntu/micro_benchmarks/build.sh new file mode 100644 index 0000000000..e198c7b809 --- /dev/null +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/micro_benchmarks/build.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +VM_NAME="periodic-micro-benchmark-tests" +ZONE="us-west1-b" +REPO_DIR="~/github/gcsfuse" +MOUNTED_DIR="$REPO_DIR/perfmetrics/scripts/micro_benchmarks/gcs" +TEST_SCRIPT_PATH="github/gcsfuse/perfmetrics/scripts/micro_benchmarks/run_microbenchmark.sh" +GCSFUSE_REPO="https://github.com/GoogleCloudPlatform/gcsfuse.git" + +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" +} + +run_script_on_vm() { + log "Running benchmark script on VM with clean setup..." + + sudo gcloud compute ssh "$VM_NAME" --zone "$ZONE" --internal-ip --command " + set -euxo pipefail + + MOUNTED_DIR=\"$MOUNTED_DIR\" + GCSFUSE_REPO=\"$GCSFUSE_REPO\" + TEST_SCRIPT_PATH=\"$TEST_SCRIPT_PATH\" + + sudo apt-get update -y + sudo apt-get install -y git + + # Unmount if gcsfuse mount exists + if mountpoint -q \"\$MOUNTED_DIR\"; then + echo \"\$MOUNTED_DIR is mounted. Attempting to unmount...\" + sudo fusermount -u \"\$MOUNTED_DIR\" || sudo umount \"\$MOUNTED_DIR\" + fi + + # Clean up any existing repo + rm -rf ~/github + + # Clone fresh repo + mkdir -p ~/github + git clone \"\$GCSFUSE_REPO\" ~/github/gcsfuse + cd ~/github/gcsfuse + commitId=\$(git log --before='yesterday 23:59:59' --max-count=1 --pretty=%H) + git checkout \$commitId + + # Run benchmark + echo \"Triggering benchmark script...\" + bash ~/\$TEST_SCRIPT_PATH + " + + log "Benchmark script executed successfully on VM." +} + +# ---- Main Execution ---- +run_script_on_vm diff --git a/perfmetrics/scripts/load_tests/python/requirements.in b/perfmetrics/scripts/continuous_test/gcp_ubuntu/micro_benchmarks/continuous.cfg similarity index 81% rename from perfmetrics/scripts/load_tests/python/requirements.in rename to perfmetrics/scripts/continuous_test/gcp_ubuntu/micro_benchmarks/continuous.cfg index dbe870d622..75bbaa39bc 100644 --- a/perfmetrics/scripts/load_tests/python/requirements.in +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/micro_benchmarks/continuous.cfg @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,5 +11,5 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -tensorflow==2.17.0 +build_file: "gcsfuse/perfmetrics/scripts/continuous_test/gcp_ubuntu/micro_benchmarks/build.sh" diff --git a/perfmetrics/scripts/continuous_test/ml_tests/pytorch/v2/dino/continuous.cfg b/perfmetrics/scripts/continuous_test/gcp_ubuntu/read_distributed.cfg similarity index 73% rename from perfmetrics/scripts/continuous_test/ml_tests/pytorch/v2/dino/continuous.cfg rename to perfmetrics/scripts/continuous_test/gcp_ubuntu/read_distributed.cfg index 26bf076565..16388333aa 100644 --- a/perfmetrics/scripts/continuous_test/ml_tests/pytorch/v2/dino/continuous.cfg +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/read_distributed.cfg @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -build_file: "gcsfuse/perfmetrics/scripts/continuous_test/ml_tests/pytorch/v2/dino/build.sh" +timeout_mins: 300 +build_file: "gcsfuse/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh" -# 2 hours timeout. -timeout_mins: 60 +env_vars { + key: "BENCHMARK_TYPE" + value: "distributed_benchmark_read" +} diff --git a/perfmetrics/scripts/continuous_test/ml_tests/tf/resnet_hns/continuous.cfg b/perfmetrics/scripts/continuous_test/gcp_ubuntu/write_distributed.cfg similarity index 73% rename from perfmetrics/scripts/continuous_test/ml_tests/tf/resnet_hns/continuous.cfg rename to perfmetrics/scripts/continuous_test/gcp_ubuntu/write_distributed.cfg index 39ba104f27..2083ef1a25 100644 --- a/perfmetrics/scripts/continuous_test/ml_tests/tf/resnet_hns/continuous.cfg +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/write_distributed.cfg @@ -1,4 +1,4 @@ -# Copyright 2024 Google LLC +# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Config file for kokoro test -build_file: "gcsfuse/perfmetrics/scripts/continuous_test/ml_tests/tf/resnet_hns/build.sh" +timeout_mins: 300 +build_file: "gcsfuse/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh" -# 1 hours timeout. -timeout_mins: 60 +env_vars { + key: "BENCHMARK_TYPE" + value: "distributed_benchmark_write" +} diff --git a/perfmetrics/scripts/continuous_test/ml_tests/pytorch/v2_hns/dino/continuous.cfg b/perfmetrics/scripts/continuous_test/gcp_ubuntu/zonal_distributed.cfg similarity index 73% rename from perfmetrics/scripts/continuous_test/ml_tests/pytorch/v2_hns/dino/continuous.cfg rename to perfmetrics/scripts/continuous_test/gcp_ubuntu/zonal_distributed.cfg index 6fcfa505cd..a7c6242e3c 100644 --- a/perfmetrics/scripts/continuous_test/ml_tests/pytorch/v2_hns/dino/continuous.cfg +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/zonal_distributed.cfg @@ -1,4 +1,4 @@ -# Copyright 2024 Google LLC +# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -build_file: "gcsfuse/perfmetrics/scripts/continuous_test/ml_tests/pytorch/v2_hns/dino/build.sh" +timeout_mins: 300 +build_file: "gcsfuse/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh" -# 1 hour timeout. -timeout_mins: 60 +env_vars { + key: "BENCHMARK_TYPE" + value: "distributed_benchmark_zonal" +} diff --git a/perfmetrics/scripts/load_tests/python/load_generator/__init__.py b/perfmetrics/scripts/continuous_test/gke/common/__init__.py similarity index 95% rename from perfmetrics/scripts/load_tests/python/load_generator/__init__.py rename to perfmetrics/scripts/continuous_test/gke/common/__init__.py index 1dc90d1848..0a2669d7a2 100644 --- a/perfmetrics/scripts/load_tests/python/load_generator/__init__.py +++ b/perfmetrics/scripts/continuous_test/gke/common/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/perfmetrics/scripts/continuous_test/gke/common/utils.py b/perfmetrics/scripts/continuous_test/gke/common/utils.py new file mode 100644 index 0000000000..4a25a0796d --- /dev/null +++ b/perfmetrics/scripts/continuous_test/gke/common/utils.py @@ -0,0 +1,591 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Common utilities for GKE tests.""" + +import asyncio +import os +import shlex +import subprocess +import sys + +# The Maximum Transmission Unit (MTU) for the network. +DEFAULT_MTU = 8896 +# Default configuration for testing on TPU +DEFAULT_PROJECT_ID = "gcs-fuse-test-ml" +DEFAULT_ZONE = "europe-west4-a" +DEFAULT_RESERVATION_NAME = "cloudtpu-20260521143000-1388945208" + + +async def run_command_async(command_list, check=True, cwd=None): + """Runs a command asynchronously, preventing command injection. + + Args: + command_list: A list of strings representing the command and its + arguments. + check: If True, raises CalledProcessError if the command returns a + non-zero exit code. + cwd: The working directory to run the command in. + + Returns: + A tuple containing (stdout, stderr, returncode). + + Raises: + subprocess.CalledProcessError: If the command fails and check is True. + """ + command_str = " ".join(map(shlex.quote, command_list)) + print(f"Executing command: {command_str}") + process = await asyncio.create_subprocess_exec( + *command_list, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=cwd, + ) + stdout, stderr = await process.communicate() + stdout_decoded = stdout.decode().strip() + stderr_decoded = stderr.decode().strip() + + if check and process.returncode != 0: + raise subprocess.CalledProcessError( + process.returncode, command_str, stdout_decoded, stderr_decoded + ) + + print(stdout_decoded) + print(stderr_decoded, file=sys.stderr) + sys.stdout.flush() + return stdout_decoded, stderr_decoded, process.returncode + + +async def check_prerequisites(): + """Checks for required command-line tools. + + Verifies that gcloud, git, make, and kubectl are installed. If kubectl is + missing, it attempts to install it using 'gcloud components install'. + Exits the script if any other required tool is not found. + """ + await run_command_async( + ["sudo","apt","install","-y","apt-transport-https","ca-certificates","gnupg","curl"] + ) + + # Pipe curl output to gpg + curl_process = await asyncio.create_subprocess_exec( + "curl", + "https://packages.cloud.google.com/apt/doc/apt-key.gpg", + stdout=asyncio.subprocess.PIPE, + ) + gpg_process = await asyncio.create_subprocess_exec( + "sudo", "gpg", "--yes", "--dearmor", "-o", "/usr/share/keyrings/cloud.google.gpg", + stdin=asyncio.subprocess.PIPE, + ) + await gpg_process.communicate(input=await curl_process.stdout.read()) + + # Pipe echo output to tee + echo_process = await asyncio.create_subprocess_exec( + "echo", + "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main", + stdout=asyncio.subprocess.PIPE, + ) + tee_process = await asyncio.create_subprocess_exec( + "sudo", + "tee", + "/etc/apt/sources.list.d/google-cloud-sdk.list", + stdin=asyncio.subprocess.PIPE, + ) + await tee_process.communicate(input=await echo_process.stdout.read()) + + await run_command_async(["sudo", "apt", "update", "-y"]) + + print("Checking for required tools...") + tools = { + "gcloud": ["gcloud", "--version"], + "git": ["git", "--version"], + "make": ["make", "--version"], + "kubectl": ["kubectl", "version", "--client=true"], + "gke-gcloud-auth-plugin": ["gke-gcloud-auth-plugin", "--version"], + } + + for tool, version_cmd in tools.items(): + try: + await run_command_async(version_cmd) + except (FileNotFoundError, subprocess.CalledProcessError): + if tool == "gcloud": + print("gcloud not found. Attempting to install...") + try: + await run_command_async( + ["sudo", "apt", "install", "-y", "google-cloud-sdk"] + ) + # Re-check after installation + await run_command_async(version_cmd) + except ( + FileNotFoundError, + subprocess.CalledProcessError, + ) as install_e: + print( + f"Error: Failed to install gcloud: {install_e}", file=sys.stderr + ) + sys.exit(1) + + if tool == "make": + print("make not found. Attempting to install...") + try: + await run_command_async(["sudo", "apt", "install", "-y", "make"]) + await run_command_async(version_cmd) + except ( + FileNotFoundError, + subprocess.CalledProcessError, + ) as install_e: + print(f"Error: Failed to install make: {install_e}", file=sys.stderr) + sys.exit(1) + + if tool == "kubectl": + print("kubectl not found. Attempting to install...") + try: + await run_command_async( + ["sudo", "snap", "install", "kubectl", "--classic"] + ) + except (FileNotFoundError, subprocess.CalledProcessError) as e: + print(f"Error: Failed to install kubectl: {e}", file=sys.stderr) + sys.exit(1) + + elif tool == "gke-gcloud-auth-plugin": + print("gke-gcloud-auth-plugin not found. Attempting to install...") + try: + await run_command_async([ + "sudo", + "apt", + "install", + "-y", + "google-cloud-sdk-gke-gcloud-auth-plugin", + ]) + except (FileNotFoundError, subprocess.CalledProcessError) as e: + print( + f"Error: Failed to install gke-gcloud-auth-plugin: {e}", + file=sys.stderr, + ) + sys.exit(1) + + else: + print( + f"Error: Required tool '{tool}' is not installed. Please install it" + " before running.", + file=sys.stderr, + ) + sys.exit(1) + print("All required tools are installed.") + + +async def setup_gke_cluster( + project_id, + zone, + cluster_name, + network_name, + subnet_name, + region, + machine_type, + node_pool_name, + reservation_name=None, +): + """Sets up the GKE cluster and required node pool. + + This function ensures a GKE cluster and a specific node pool are ready for + the test. It will create the cluster, network, and node pool if they + don't exist. If the node pool exists but is unhealthy, it will be recreated. + + Args: + project_id: The Google Cloud project ID. + zone: The GCP zone for the cluster and node pool. + cluster_name: The name of the GKE cluster. + network_name: The name of the VPC network. + subnet_name: The name of the VPC subnet. + region: The GCP region for the network. + machine_type: The machine type for the node pool. + node_pool_name: The name of the node pool. + reservation_name: The specific reservation to use for the nodes. + """ + print(f"Setting up GKE cluster '{cluster_name}' in zone '{zone}'...") + if await get_cluster_async(project_id, zone, cluster_name): + print(f"Cluster '{cluster_name}' already exists.") + if await get_node_pool_async( + project_id, zone, cluster_name, node_pool_name + ): + print(f"Node pool '{node_pool_name}' exists.") + if not await is_node_pool_healthy_async( + project_id, zone, cluster_name, node_pool_name + ): + print(f"Node pool '{node_pool_name}' is unhealthy. Recreating...") + await delete_node_pool_async( + project_id, zone, cluster_name, node_pool_name + ) + await create_node_pool_async( + project_id, + zone, + cluster_name, + node_pool_name, + machine_type, + reservation_name, + ) + else: + print(f"Creating node pool '{node_pool_name}'...") + await create_node_pool_async( + project_id, + zone, + cluster_name, + node_pool_name, + machine_type, + reservation_name, + ) + else: + print(f"Creating network '{network_name}' and subnet '{subnet_name}'...") + await create_network(project_id, network_name, subnet_name, region) + print(f"Creating cluster '{cluster_name}'...") + cmd = [ + "gcloud", + "container", + "clusters", + "create", + cluster_name, + f"--project={project_id}", + f"--zone={zone}", + f"--network={network_name}", + f"--subnetwork={subnet_name}", + f"--workload-pool={project_id}.svc.id.goog", + "--addons=GcsFuseCsiDriver", + "--num-nodes=1", + ] + await run_command_async(cmd) + print(f"Creating node pool '{node_pool_name}'...") + await create_node_pool_async( + project_id, + zone, + cluster_name, + node_pool_name, + machine_type, + reservation_name, + ) + + # Get credentials for the cluster to allow kubectl to connect. + print("Fetching cluster endpoint and auth data.") + await run_command_async([ + "gcloud", + "container", + "clusters", + "get-credentials", + cluster_name, + f"--project={project_id}", + f"--zone={zone}", + ]) + print("GKE cluster setup complete.") + + +async def get_cluster_async(project_id, zone, cluster_name): + """Checks if a GKE cluster exists. + + Args: + project_id: The Google Cloud project ID. + zone: The GCP zone where the cluster is located. + cluster_name: The name of the GKE cluster. + + Returns: + True if the cluster exists, False otherwise. + """ + cmd = [ + "gcloud", + "container", + "clusters", + "describe", + cluster_name, + f"--project={project_id}", + f"--zone={zone}", + "--format=value(name)", + ] + _, _, returncode = await run_command_async(cmd, check=False) + return returncode == 0 + + +async def get_node_pool_async(project_id, zone, cluster_name, node_pool_name): + """Checks if a node pool exists in a GKE cluster. + + Args: + project_id: The Google Cloud project ID. + zone: The GCP zone where the cluster is located. + cluster_name: The name of the GKE cluster. + node_pool_name: The name of the node pool. + + Returns: + True if the node pool exists, False otherwise. + """ + cmd = [ + "gcloud", + "container", + "node-pools", + "describe", + node_pool_name, + f"--project={project_id}", + f"--zone={zone}", + f"--cluster={cluster_name}", + "--format=value(name)", + ] + _, _, returncode = await run_command_async(cmd, check=False) + return returncode == 0 + + +async def is_node_pool_healthy_async( + project_id, zone, cluster_name, node_pool_name +): + """Checks if a node pool's status is RUNNING. + + Args: + project_id: The Google Cloud project ID. + zone: The GCP zone where the cluster is located. + cluster_name: The name of the GKE cluster. + node_pool_name: The name of the node pool. + + Returns: + True if the node pool status is 'RUNNING', False otherwise. + """ + cmd = [ + "gcloud", + "container", + "node-pools", + "describe", + node_pool_name, + f"--project={project_id}", + f"--zone={zone}", + f"--cluster={cluster_name}", + "--format=value(status)", + ] + status, _, returncode = await run_command_async(cmd, check=False) + return returncode == 0 and status == "RUNNING" + + +async def create_node_pool_async( + project_id, + zone, + cluster_name, + node_pool_name, + machine_type, + reservation_name=None, +): + """Creates a new node pool. + + Args: + project_id: The Google Cloud project ID. + zone: The GCP zone for the node pool. + cluster_name: The name of the GKE cluster. + node_pool_name: The name for the new node pool. + machine_type: The machine type for the nodes in the pool. + reservation_name: The specific reservation to use for the nodes. + """ + cmd = [ + "gcloud", + "container", + "node-pools", + "create", + node_pool_name, + f"--project={project_id}", + f"--cluster={cluster_name}", + f"--zone={zone}", + f"--machine-type={machine_type}", + "--num-nodes=1", + "--scopes=https://www.googleapis.com/auth/cloud-platform", + ] + if reservation_name: + cmd.extend([ + f"--reservation-affinity=specific", + f"--reservation={reservation_name}", + ]) + await run_command_async(cmd) + + +async def delete_node_pool_async( + project_id, zone, cluster_name, node_pool_name +): + """Deletes an existing node pool. + + Args: + project_id: The Google Cloud project ID. + zone: The GCP zone where the node pool is located. + cluster_name: The name of the GKE cluster. + node_pool_name: The name of the node pool to delete. + """ + cmd = [ + "gcloud", + "container", + "node-pools", + "delete", + node_pool_name, + f"--project={project_id}", + f"--cluster={cluster_name}", + f"--zone={zone}", + "--quiet", + ] + await run_command_async(cmd, check=False) + + +async def create_network(project_id, network_name, subnet_name, region): + """Creates a new network and subnet if they don't exist. + + Args: + project_id: The Google Cloud project ID. + network_name: The name for the new VPC network. + subnet_name: The name for the new subnet. + region: The GCP region for the subnet. + """ + await run_command_async( + [ + "gcloud", + "compute", + "networks", + "create", + network_name, + f"--project={project_id}", + "--subnet-mode=custom", + f"--mtu={DEFAULT_MTU}", + ], + check=False, + ) + await run_command_async( + [ + "gcloud", + "compute", + "networks", + "subnets", + "create", + subnet_name, + f"--project={project_id}", + f"--network={network_name}", + "--range=10.0.0.0/24", + f"--region={region}", + ], + check=False, + ) + + +async def cleanup(project_id, zone, cluster_name, network_name, subnet_name): + """Cleans up the created GKE, network, and firewall resources. + + Args: + project_id: The Google Cloud project ID. + zone: The GCP zone where the resources are located. + cluster_name: The name of the GKE cluster to delete. + network_name: The name of the VPC network to delete. + subnet_name: The name of the subnet to delete. + """ + print("Cleaning up GKE and network resources...") + # First, delete the cluster, which is the primary user of the firewall rules. + await run_command_async( + [ + "gcloud", + "container", + "clusters", + "delete", + cluster_name, + f"--project={project_id}", + f"--zone={zone}", + "--quiet", + ], + check=False, + ) + + # Find and delete firewall rules associated with the network. + print(f"Finding and deleting firewall rules for network '{network_name}'...") + list_fw_cmd = [ + "gcloud", + "compute", + "firewall-rules", + "list", + f"--project={project_id}", + f"--filter=network~/{network_name}$", + "--format=value(name)", + ] + fw_rules_str, _, returncode = await run_command_async( + list_fw_cmd, check=False + ) + if returncode == 0 and fw_rules_str: + fw_rules = fw_rules_str.splitlines() + delete_tasks = [] + for rule in fw_rules: + print(f"Deleting firewall rule: {rule}") + delete_fw_cmd = [ + "gcloud", + "compute", + "firewall-rules", + "delete", + rule, + f"--project={project_id}", + "--quiet", + ] + delete_tasks.append(run_command_async(delete_fw_cmd, check=False)) + if delete_tasks: + await asyncio.gather(*delete_tasks) + + # Now, delete the subnetwork and network. + print(f"Deleting subnetwork '{subnet_name}'...") + await run_command_async( + [ + "gcloud", + "compute", + "networks", + "subnets", + "delete", + subnet_name, + f"--project={project_id}", + f"--region={zone.rsplit('-', 1)[0]}", + "--quiet", + ], + check=False, + ) + + print(f"Deleting network '{network_name}'...") + await run_command_async( + [ + "gcloud", + "compute", + "networks", + "delete", + network_name, + f"--project={project_id}", + "--quiet", + ], + check=False, + ) + + print("Cleanup complete.") + + +# GCSFuse Build and Deploy +async def build_gcsfuse_image(project_id, branch, temp_dir, staging_version): + """Clones GCSFuse and builds the CSI driver image. + + Args: + branch: The git branch or tag of the GCSFuse repository to use. + temp_dir: A temporary directory to clone the repository into. + """ + gcsfuse_dir = os.path.join(temp_dir, "gcsfuse") + await run_command_async([ + "git", + "clone", + "--depth=1", + "-b", + branch, + "https://github.com/GoogleCloudPlatform/gcsfuse.git", + gcsfuse_dir, + ]) + build_cmd = [ + "make", + "build-csi", + f"PROJECT={project_id}", + f"STAGINGVERSION={staging_version}", + ] + await run_command_async(build_cmd, cwd=gcsfuse_dir) diff --git a/perfmetrics/scripts/continuous_test/gke/machine_type_test/README.md b/perfmetrics/scripts/continuous_test/gke/machine_type_test/README.md new file mode 100644 index 0000000000..acff5d01bb --- /dev/null +++ b/perfmetrics/scripts/continuous_test/gke/machine_type_test/README.md @@ -0,0 +1,143 @@ +# GKE Machine Type Test + +This script automates the process of running the Machine Type Test on a GKE +cluster. It handles the entire workflow, including GKE cluster setup, GCSFuse +CSI driver building, workload execution, result gathering, and resource cleanup. + +## Overview + +The `run.py` script automates the end-to-end testing process on Google +Kubernetes Engine (GKE). It handles: +1. **Cluster Management**: Creating/configuring GKE clusters and node pools (including Workload Identity). +2. **Driver Build**: Building the GCSFuse CSI driver from source. +3. **Workload Deployment**: Deploying a Kubernetes Pod to run `go test` integration tests. +4. **Result Verification**: Checking test success/failure. +5. **Cleanup**: Removing cloud resources. + +## Prerequisites + +Before running the script, ensure you have the following tools installed and +configured. The script will check for these and attempt to install `kubectl` if +it's missing. + +### Tools + +- `gcloud`: The Google Cloud CLI, authenticated with a project. Ensure the + following APIs are enabled in your project: + - Kubernetes Engine API (`container.googleapis.com`) + - Cloud Storage API (`storage.googleapis.com`) +- `kubectl`: The Kubernetes command-line tool. +- `git`: The version control system. +- `make`: The build automation tool. +- `python3` with the `asyncio` library (standard in Python 3.7+). + +### Workload Identity Setup (Critical) + +The test uses GKE Workload Identity Federation. You must grant the Kubernetes +Service Account (KSA) direct access to the GCS bucket. For this test, we use the +`default` KSA in the `default` namespace. + +**1. Grant Bucket Permissions to the KSA Principal:** You need the **Project +Number** (not ID) of the project hosting the GKE cluster. + +```bash +# Get Project Number +PROJECT_NUMBER=$(gcloud projects describe <PROJECT_ID> --format="value(projectNumber)") + +# Grant Storage Object User role to the 'default' KSA principal +gcloud storage buckets add-iam-policy-binding gs://<BUCKET_NAME> \ + --member="principal://iam.googleapis.com/projects/${PROJECT_NUMBER}/locations/global/workloadIdentityPools/<PROJECT_ID>.svc.id.goog/subject/ns/default/sa/default" \ + --role=roles/storage.objectUser \ + --project=<BUCKET_PROJECT_ID> +``` + +*Note: This has already been configured for the test environment +(project: `gcs-fuse-test-ml`, bucket: `gcsfuse_gke_machine_type_test_flat_euw4`).* + +## Workflow + +The script performs the following steps: + +1. **Prerequisite Check**: Verifies that `gcloud`, `git`, `make`, `kubectl`, + and `gke-gcloud-auth-plugin` are installed. +2. **VPC Network and Subnet Setup**: Creates a VPC network and subnet if they + don't already exist. +3. **GKE Cluster Setup**: Creates a new GKE cluster with a dedicated node pool + if one doesn't already exist. If the node pool is unhealthy, it's recreated. +4. **Build GCSFuse CSI Driver**: Concurrently with cluster setup, it clones the + specified GCSFuse repository branch and builds the GCSFuse CSI driver + container image. +5. **Run Test**: Deploys the test workload as a Kubernetes Pod. + * It automatically selects the appropriate node-pool based on the machine + type. +6. **Gather Results**: Fetches the logs from the completed test pod. +7. **Evaluate Success**: Checks if the pod completed successfully. +8. **Cleanup**: Deletes the GKE cluster and other created resources like the + VPC network, subnet, and associated firewall rules, unless the + `--no_cleanup` flag is specified. + +## Usage + +The script is controlled via command-line arguments. + +``` +usage: run.py [-h] [--project_id PROJECT_ID] --bucket_name BUCKET_NAME [--zone ZONE] \ + [--cluster_name CLUSTER_NAME] [--network_name NETWORK_NAME] \ + [--subnet_name SUBNET_NAME] [--machine_type MACHINE_TYPE] \ + [--node_pool_name NODE_POOL_NAME] [--gcsfuse_branch GCSFUSE_BRANCH] \ + [--reservation_name RESERVATION_NAME] [--no_cleanup] \ + [--pod_timeout_seconds POD_TIMEOUT_SECONDS] [--skip_csi_driver_build] +``` + +### Argument Reference + +Argument | Description | Default Value +:------------------------ | :--------------------------------------------------------------- | :------------ +`--project_id` | Google Cloud project ID. | `gcs-fuse-test-ml` (Env: `PROJECT_ID`) +`--bucket_name` | **(Required)** GCS bucket name for the workload. | `None` (Env: `BUCKET_NAME`) +`--zone` | GCP zone. | `europe-west4-a` (Env: `ZONE`) +`--cluster_name` | GKE cluster name. | `gke-machine-type-test-cluster` +`--network_name` | VPC network name. | `gke-machine-type-test-network-<ZONE>` +`--subnet_name` | VPC subnet name. | `gke-machine-type-test-subnet-<ZONE>` +`--machine_type` | Machine type for the node pool. | `ct6e-standard-4t` (TPU v6e) +`--node_pool_name` | Node pool name. | `ct6e-pool` +`--gcsfuse_branch` | GCSFuse branch to build. | `master` +`--reservation_name` | Specific reservation to use for the nodes. | `cloudtpu-20260521143000-1388945208` +`--no_cleanup` | If set, resources will NOT be deleted after the test. | `False` +`--skip_csi_driver_build` | If set, skips building the CSI driver image (assumes it exists). | `False` +`--pod_timeout_seconds` | Timeout in seconds for the pod to complete. | `1800` (30 mins) + +## Examples + +To run the test with default settings (TPU v6e), you only need to provide the +required arguments: + +```bash +python3 perfmetrics/scripts/continuous_test/gke/machine_type_test/run.py \ + --project_id "your-gcp-project-id" \ + --bucket_name "your-gcs-bucket-name" \ + --zone "us-central1-a" +``` + +To run on a **TPU Machine Type** (`ct6e-standard-4t`) using a reservation and a +specific branch: + +```bash +python3 perfmetrics/scripts/continuous_test/gke/machine_type_test/run.py \ + --project_id "your-gcp-project-id" \ + --bucket_name "your-gcs-bucket-name" \ + --zone "europe-west4-a" \ + --machine_type ct6e-standard-4t \ + --node_pool_name tpu-v6-pool \ + --reservation_name "your-reservation-name" \ + --gcsfuse_branch "my-feature-branch" \ + --no_cleanup \ + --skip_csi_driver_build +``` + +## Troubleshooting + +* **403 Forbidden**: Check Workload Identity setup. Ensure `default` KSA + exists and has the read/write access to the bucket. +* **Init:ErrImagePull**: Check if the CSI driver image exists in GCR/Artifact + Registry. If running locally, you might need to authenticate docker. diff --git a/perfmetrics/scripts/continuous_test/gke/machine_type_test/continuous.cfg b/perfmetrics/scripts/continuous_test/gke/machine_type_test/continuous.cfg new file mode 100644 index 0000000000..106662c5d5 --- /dev/null +++ b/perfmetrics/scripts/continuous_test/gke/machine_type_test/continuous.cfg @@ -0,0 +1,20 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +env_vars { + key:"BUCKET_NAME" + value: "gcsfuse_gke_machine_type_test_flat_euw4" +} + +build_file: "gcsfuse/perfmetrics/scripts/continuous_test/gke/machine_type_test/run.py" diff --git a/perfmetrics/scripts/continuous_test/gke/machine_type_test/pod_tpu.yaml.template b/perfmetrics/scripts/continuous_test/gke/machine_type_test/pod_tpu.yaml.template new file mode 100644 index 0000000000..4f84a659d2 --- /dev/null +++ b/perfmetrics/scripts/continuous_test/gke/machine_type_test/pod_tpu.yaml.template @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gcsfuse-gke-machine-type-test + annotations: + gke-gcsfuse/volumes: "true" +spec: + restartPolicy: Never + nodeSelector: + cloud.google.com/gke-tpu-topology: 2x2 + cloud.google.com/gke-tpu-accelerator: tpu-v6e-slice + containers: + - name: gke-gcsfuse-sidecar + image: gcr.io/$project_id/cloudbuild-gcsfuse-csi/gcs-fuse-csi-driver-sidecar-mounter:$staging_version + - name: machine-type-test + image: ubuntu:24.04 + imagePullPolicy: Always + resources: + limits: + google.com/tpu: 4 + env: + - name: GCSFUSE_BRANCH + value: "$gcsfuse_branch" + - name: BUCKET_NAME + value: "$bucket_name" + command: ["/bin/bash", "/scripts/run_test.sh"] + volumeMounts: + - name: data-vol + mountPath: /data_mnt + - name: scripts-vol + mountPath: /scripts + serviceAccountName: default + volumes: + - name: data-vol + csi: + driver: gcsfuse.csi.storage.gke.io + volumeAttributes: + bucketName: "$bucket_name" + mountOptions: "" + - name: scripts-vol + configMap: + name: machine-type-test-scripts + diff --git a/perfmetrics/scripts/continuous_test/gke/machine_type_test/run.py b/perfmetrics/scripts/continuous_test/gke/machine_type_test/run.py new file mode 100755 index 0000000000..b75d1a44c8 --- /dev/null +++ b/perfmetrics/scripts/continuous_test/gke/machine_type_test/run.py @@ -0,0 +1,492 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run GKE Machine-type test. + +This script automates the process of running the Machine-type test on a GKE +cluster. +It performs the following steps: +1. Checks for prerequisite tools (gcloud, git, make, kubectl). +2. Sets up a GKE cluster with a specific node pool if it doesn't exist. +3. Builds a GCSFuse CSI driver image from a specified git branch. +4. Deploys a Kubernetes pod that runs the test workload. +5. Streams logs from the Kubernetes pod. +6. Determines if the test passed based on the pod exit status. +7. Cleans up all created cloud resources (GKE cluster, network, etc.). +""" + +import argparse +import asyncio +from datetime import datetime +import os +import shlex +from string import Template +import subprocess +import sys +import tempfile + +# Add the parent directory to sys.path to allow imports from common +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.abspath(os.path.join(SCRIPT_DIR, ".."))) +from common import utils + +# The prefix prow-gob-internal-boskos- is needed to allow passing machine-type from gke csi driver to gcsfuse, +# bypassing the check at +# https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/blob/15afd00dcc2cfe0f9753ddc53c81631ff037c3f2/pkg/csi_driver/utils.go#L532. +STAGING_VERSION = "prow-gob-internal-boskos-machine-type-test" + + +async def set_up_bucket_permissions( + project_id, zone, cluster_name, bucket_name +): + # Authenticate kubectl + await utils.run_command_async([ + "gcloud", + "container", + "clusters", + "get-credentials", + cluster_name, + f"--project={project_id}", + f"--zone={zone}", + ]) + + # Fetch current-context + await utils.run_command_async([ + "kubectl", + "config", + "current-context", + ]) + + # Ensure default KSA exists + await utils.run_command_async( + ["kubectl", "create", "serviceaccount", "default", "--namespace=default"], + check=False, + ) + + # Set current-context + await utils.run_command_async([ + "kubectl", + "config", + "set-context", + "--current", + "--namespace=default", + ]) + + # Get Project Number + project_number, _, _ = await utils.run_command_async( + [ + "gcloud", + "projects", + "describe", + project_id, + "--format=value(projectNumber)", + ], + check=True, + ) + + # Grant Storage objectUser role to the 'default' KSA principal + principal = f"principal://iam.googleapis.com/projects/{project_number}/locations/global/workloadIdentityPools/{project_id}.svc.id.goog/subject/ns/default/sa/default" + print( + f"Granting roles/storage.objectUser to {principal} on bucket" + f" {bucket_name}..." + ) + + await utils.run_command_async([ + "gcloud", + "storage", + "buckets", + "add-iam-policy-binding", + f"gs://{bucket_name}", + f"--member={principal}", + "--role=roles/storage.objectUser", + f"--project={project_id}", + ]) + + +def is_tpu_machine_type(machine_type): + """Checks if the machine type is a TPU machine type.""" + # Heuristic: check for "ct" (Cloud TPU) or "tpu" in the name. + return machine_type.startswith("ct") or "tpu" in machine_type + + +# Workload Execution and Result Gathering +async def execute_test_workload( + project_id, + zone, + cluster_name, + bucket_name, + timestamp, + staging_version, + pod_timeout_seconds, + machine_type, + gcsfuse_branch, +): + """Executes the workload pod, gathers results, and cleans up workload resources. + + This function creates a Kubernetes Pod to run the test. + It waits for the pod to complete, collects its logs, + and then deletes the created Kubernetes resources. + + Args: + project_id: The Google Cloud project ID. + zone: The GCP zone of the cluster. + cluster_name: The name of the GKE cluster. + bucket_name: The GCS bucket to use for the test. + timestamp: A unique timestamp string for manifest naming. + staging_version: The version tag for the GCSFuse CSI driver image. + pod_timeout_seconds: The timeout in seconds for the pod to complete. + machine_type: The machine type of the node pool. + gcsfuse_branch: The gcsfuse branch to clone. + + Returns: + True if the test passed, False otherwise. + """ + print(f"Executing workload for machine type: {machine_type}...") + + if not is_tpu_machine_type(machine_type): + raise ValueError( + f"Machine type {machine_type} is not supported. Only TPU machine types" + " are supported." + ) + + template_file = "pod_tpu.yaml.template" + + template_path = os.path.join(SCRIPT_DIR, template_file) + print(f"Using pod template: {template_path}") + + with open(template_path, "r") as f: + pod_template = Template(f.read()) + + # Use timestamp to make pod name unique to avoid conflict + pod_name = f"gcsfuse-gke-machine-type-test-{timestamp}" + print(f"Pod name: {pod_name}") + + manifest = pod_template.safe_substitute( + project_id=project_id, + bucket_name=bucket_name, + staging_version=staging_version, + gcsfuse_branch=gcsfuse_branch, + machine_type=machine_type, + ) + # Update the pod name in the manifest content dynamically + manifest = manifest.replace( + "name: gcsfuse-gke-machine-type-test", f"name: {pod_name}" + ) + + manifest_filename = f"manifest-{timestamp}.yaml" + + try: + await utils.run_command_async([ + "kubectl", + "create", + "configmap", + "machine-type-test-scripts", + f"--from-file={os.path.join(SCRIPT_DIR, 'run_test.sh')}", + ]) + + with open(manifest_filename, "w") as f: + f.write(manifest) + + # Check if pod exists and delete it (just in case, though name is unique now) + # We ignore the error if it doesn't exist + print("Checking for existing pod...") + await utils.run_command_async( + ["kubectl", "delete", "pod", pod_name, "--ignore-not-found=true"], + check=False, + ) + + print(f"Applying manifest: {manifest_filename}") + await utils.run_command_async(["kubectl", "apply", "-f", manifest_filename]) + + start_time = datetime.now() + pod_finished = False + success = False + + print( + f"Waiting for pod {pod_name} to complete (timeout:" + f" {pod_timeout_seconds}s)..." + ) + + # Wait for pod to be schedulable/running to avoid immediate exit of logs -f + while (datetime.now() - start_time).total_seconds() < pod_timeout_seconds: + status, stderr, _ = await utils.run_command_async( + [ + "kubectl", + "get", + "pod", + pod_name, + "-o", + "jsonpath='{.status.phase}'", + ], + check=False, + ) + # jsonpath output comes with quotes, remove them + status = status.strip("'") + if status in ["Running", "Succeeded", "Failed"]: + break + await asyncio.sleep(5) + + # Stream logs using subprocess.Popen (blocking loop) + # This waits for the pod to finish (as kubectl logs -f exits on termination) + log_process = subprocess.Popen( + ["kubectl", "logs", "-f", pod_name, "-c", "machine-type-test"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + + while True: + line = log_process.stdout.readline() + if not line and log_process.poll() is not None: + break + if line: + print(line, end="") + + # Wait for the pod status to update to Succeeded or Failed + # kubectl logs -f exits when the container stops, but the pod status update might be delayed. + for _ in range(12): # Retry for up to 60 seconds (12 * 5s) + status, stderr, _ = await utils.run_command_async( + [ + "kubectl", + "get", + "pod", + pod_name, + "-o", + "jsonpath='{.status.phase}'", + ], + check=False, + ) + status = status.strip("'") + if status in ["Succeeded", "Failed"]: + break + print(f"Pod status is '{status}'. Waiting for final status...") + await asyncio.sleep(5) + + if status == "Succeeded": + print(f"Pod {pod_name} succeeded.") + success = True + elif status == "Failed": + print(f"Pod {pod_name} failed.") + success = False + else: + # If logs finished but pod is not Succeeded/Failed (e.g. timeout or crash loop?), check timeout + if (datetime.now() - start_time).total_seconds() > pod_timeout_seconds: + print( + f"Pod did not complete within {pod_timeout_seconds / 60} minutes.", + file=sys.stderr, + ) + success = False + else: + # Fallback: assume failure if logs ended but status isn't clear? + # Or maybe it just finished and status update is pending? + # Let's assume if logs exited, the container stopped. + success = False + + return success + finally: + print("Cleaning up pod resources...") + await utils.run_command_async( + ["kubectl", "delete", "configmap", "machine-type-test-scripts"], + check=False, + ) + await utils.run_command_async( + ["kubectl", "delete", "-f", manifest_filename], check=False + ) + if os.path.exists(manifest_filename): + os.remove(manifest_filename) + + +# Main function +async def main(): + """Parses arguments, orchestrates the test execution, and handles cleanup. + + This is the main entry point of the script. + """ + parser = argparse.ArgumentParser( + description="Run GKE Machine-type test.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--project_id", + default=os.environ.get("PROJECT_ID", utils.DEFAULT_PROJECT_ID), + help="Google Cloud project ID. Can also be set with PROJECT_ID env var.", + ) + parser.add_argument( + "--bucket_name", + required=os.environ.get("BUCKET_NAME") is None, + default=os.environ.get("BUCKET_NAME"), + help=( + "GCS bucket name for the workload. Can also be set with BUCKET_NAME" + " env var." + ), + ) + parser.add_argument( + "--zone", + default=os.environ.get("ZONE", utils.DEFAULT_ZONE), + help="GCP zone. Can also be set with ZONE env var.", + ) + parser.add_argument( + "--cluster_name", + default=os.environ.get("CLUSTER_NAME", "gke-machine-type-test-cluster"), + help="GKE cluster name. Can also be set with CLUSTER_NAME env var.", + ) + parser.add_argument( + "--network_name", + default=os.environ.get("NETWORK_NAME", "gke-machine-type-test-network"), + help="VPC network name. Can also be set with NETWORK_NAME env var.", + ) + parser.add_argument( + "--subnet_name", + default=os.environ.get("SUBNET_NAME", "gke-machine-type-test-subnet"), + help="VPC subnet name. Can also be set with SUBNET_NAME env var.", + ) + parser.add_argument( + "--machine_type", + default=os.environ.get("MACHINE_TYPE", "ct6e-standard-4t"), + help="Machine type. Can also be set with MACHINE_TYPE env var.", + ) + parser.add_argument( + "--node_pool_name", + default=os.environ.get("NODE_POOL_NAME", "ct6e-pool"), + help="Node pool name. Can also be set with NODE_POOL_NAME env var.", + ) + parser.add_argument( + "--gcsfuse_branch", + default=os.environ.get("GCSFUSE_BRANCH", "master"), + help=( + "GCSFuse branch or tag to build. Can also be set with GCSFUSE_BRANCH" + " env var." + ), + ) + parser.add_argument( + "--reservation_name", + default=os.environ.get("RESERVATION_NAME", utils.DEFAULT_RESERVATION_NAME), + help=( + "The specific reservation to use for the nodes. Can also be set with" + " RESERVATION_NAME env var." + ), + ) + parser.add_argument( + "--no_cleanup", + action="store_true", + default=os.environ.get("NO_CLEANUP", "False").lower() in ("true", "1"), + help=( + "Don't clean up resources after. Can also be set with NO_CLEANUP=true" + " env var." + ), + ) + parser.add_argument( + "--pod_timeout_seconds", + type=int, + default=int(os.environ.get("POD_TIMEOUT_SECONDS", 1800)), + help=( + "Timeout in seconds for the test pod to complete. Can also be" + " set with POD_TIMEOUT_SECONDS env var." + ), + ) + parser.add_argument( + "--skip_csi_driver_build", + action="store_true", + default=os.environ.get("SKIP_CSI_DRIVER_BUILD", "False").lower() + in ("true", "1"), + help=( + "Skip building the CSI driver. Can also be set with" + " SKIP_CSI_DRIVER_BUILD=true env var." + ), + ) + args = parser.parse_args() + + # Append zone to default network and subnet names to avoid collisions + if args.network_name == "gke-machine-type-test-network": + args.network_name = f"{args.network_name}-{args.zone}" + if args.subnet_name == "gke-machine-type-test-subnet": + args.subnet_name = f"{args.subnet_name}-{args.zone}" + + await utils.check_prerequisites() + + timestamp = datetime.now().strftime("%Y%m%d%H%M%S") + return_code = 0 + with tempfile.TemporaryDirectory() as temp_dir: + try: + if args.skip_csi_driver_build: + await utils.setup_gke_cluster( + args.project_id, + args.zone, + args.cluster_name, + args.network_name, + args.subnet_name, + args.zone.rsplit("-", 1)[0], + args.machine_type, + args.node_pool_name, + args.reservation_name, + ) + else: + setup_task = asyncio.create_task( + utils.setup_gke_cluster( + args.project_id, + args.zone, + args.cluster_name, + args.network_name, + args.subnet_name, + args.zone.rsplit("-", 1)[0], + args.machine_type, + args.node_pool_name, + args.reservation_name, + ) + ) + build_task = asyncio.create_task( + utils.build_gcsfuse_image( + args.project_id, args.gcsfuse_branch, temp_dir, STAGING_VERSION + ) + ) + await asyncio.gather(setup_task, build_task) + + await set_up_bucket_permissions( + args.project_id, args.zone, args.cluster_name, args.bucket_name + ) + + success = await execute_test_workload( + args.project_id, + args.zone, + args.cluster_name, + args.bucket_name, + timestamp, + STAGING_VERSION, + args.pod_timeout_seconds, + args.machine_type, + args.gcsfuse_branch, + ) + + if success: + print("Test passed successfully.") + else: + print("Test failed.", file=sys.stderr) + return_code = 1 + finally: + if not args.no_cleanup: + await utils.cleanup( + args.project_id, + args.zone, + args.cluster_name, + args.network_name, + args.subnet_name, + ) + + sys.exit(return_code) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/perfmetrics/scripts/continuous_test/gke/machine_type_test/run_test.sh b/perfmetrics/scripts/continuous_test/gke/machine_type_test/run_test.sh new file mode 100644 index 0000000000..ac1be26e6a --- /dev/null +++ b/perfmetrics/scripts/continuous_test/gke/machine_type_test/run_test.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +echo "Step 1: Container started. Updating apt and installing dependencies..." +apt-get update && apt-get install -y wget git build-essential ca-certificates sudo + +echo "Step 2: Cloning GCSFuse repo..." +git clone -b "$GCSFUSE_BRANCH" https://github.com/GoogleCloudPlatform/gcsfuse.git +cd gcsfuse + +GO_VERSION=$(cat .go-version | tr -d '[:space:]') +if [[ ! "$GO_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Error: Invalid Go version format in .go-version" + exit 1 +fi +echo "Step 3: Installing Go..." +./perfmetrics/scripts/install_go.sh "$GO_VERSION" +export PATH="/usr/local/go/bin:$PATH" +echo "Go version installed:" +go version + +echo "Step 4: Running tests ..." +# These tests are chosen to verify that machine-type is correctly passed by +# CSI Driver to GCSFuse and GCSFuse is correctly accepting it and triggering optimization flags +# like implicit-dirs and rename-dir-limit for high-performance machine-type as expected. +go test -v ./tools/integration_tests/flag_optimizations/... --integrationTest --mountedDirectory=/data_mnt --testbucket="$BUCKET_NAME" -run "TestImplicitDirsEnabled|TestRenameDirLimitSet" + +echo "Step 5: Test finished successfully." diff --git a/perfmetrics/scripts/continuous_test/gke/orbax_benchmark/README.md b/perfmetrics/scripts/continuous_test/gke/orbax_benchmark/README.md new file mode 100644 index 0000000000..57b96ad10a --- /dev/null +++ b/perfmetrics/scripts/continuous_test/gke/orbax_benchmark/README.md @@ -0,0 +1,84 @@ +# GKE Orbax Benchmark + +This script automates the process of running the Orbax benchmark on a GKE cluster. It handles the entire workflow, including GKE cluster setup, GCSFuse CSI driver building, workload execution, result gathering, and resource cleanup. + +## Prerequisites + +Before running the script, ensure you have the following tools installed and configured. The script will check for these and attempt to install `kubectl` if it's missing. + +- `gcloud`: The Google Cloud CLI, authenticated with a project. Ensure the following APIs are enabled in your project: + - Kubernetes Engine API (`container.googleapis.com`) + - Cloud Storage API (`storage.googleapis.com`) +- `kubectl`: The Kubernetes command-line tool. +- `git`: The version control system. +- `make`: The build automation tool. +- `python3` with the `asyncio` library (standard in Python 3.7+). + +## Workflow + +The script performs the following steps: + +1. **Prerequisite Check**: Verifies that `gcloud`, `git`, `make`, and `kubectl` are installed. +2. **VPC Network and Subnet Setup**: Creates a VPC network and subnet if they don't already exist. +3. **GKE Cluster Setup**: Creates a new GKE cluster with a dedicated node pool if one doesn't already exist. If the node pool is unhealthy, it's recreated. +4. **Build GCSFuse CSI Driver**: Concurrently with cluster setup, it clones the specified GCSFuse repository branch and builds the GCSFuse CSI driver container image. +5. **Run Benchmark**: Deploys the Orbax benchmark workload as a Kubernetes Pod. +6. **Gather and Parse Results**: Fetches the logs from the completed benchmark pod and parses them to extract throughput metrics. +7. **Evaluate Performance**: Compares the results against a performance threshold to determine if the benchmark passed. +8. **Cleanup**: Deletes the GKE cluster and other created resources like the VPC network, subnet, and associated firewall rules, unless the `--no_cleanup` flag is specified. + +## Usage + +The script is controlled via command-line arguments. + +``` +usage: run_benchmark.py [-h] [--project_id PROJECT_ID] --bucket_name BUCKET_NAME [--zone ZONE] \ + [--cluster_name CLUSTER_NAME] [--network_name NETWORK_NAME] \ + [--subnet_name SUBNET_NAME] [--machine_type MACHINE_TYPE] \ + [--node_pool_name NODE_POOL_NAME] [--gcsfuse_branch GCSFUSE_BRANCH] \ + [--reservation_name RESERVATION_NAME] [--no_cleanup] \ + [--iterations ITERATIONS] \ + [--performance_threshold_gbps PERFORMANCE_THRESHOLD_GBPS] \ + [--pod_timeout_seconds POD_TIMEOUT_SECONDS] [--skip_csi_driver_build] +``` + +### Argument Reference + +Argument | Description | Default Value +:------------------------ | :--------------------------------------------------------------- | :------------ +`--project_id` | Google Cloud project ID. | `gcs-fuse-test-ml` (Env: `PROJECT_ID`) +`--bucket_name` | **(Required)** GCS bucket name for the workload. | `None` (Env: `BUCKET_NAME`) +`--zone` | GCP zone. | `europe-west4-a` (Env: `ZONE`) +`--cluster_name` | GKE cluster name. | `gke-orbax-benchmark-cluster` +`--network_name` | VPC network name. | `gke-orbax-benchmark-network-<ZONE>` +`--subnet_name` | VPC subnet name. | `gke-orbax-benchmark-subnet-<ZONE>` +`--machine_type` | Machine type for the node pool. | `ct6e-standard-4t` (TPU v6e) +`--node_pool_name` | Node pool name. | `ct6e-pool` +`--gcsfuse_branch` | GCSFuse branch to build. | `master` +`--reservation_name` | Specific reservation to use for the nodes. | `cloudtpu-20260521143000-1388945208` +`--no_cleanup` | If set, resources will NOT be deleted after the test. | `False` +`--iterations` | Number of iterations for the benchmark. | `20` +`--performance_threshold_gbps` | Minimum throughput in GB/s for a successful iteration. | `13.0` +`--pod_timeout_seconds` | Timeout in seconds for the pod to complete. | `1800` (30 mins) +`--skip_csi_driver_build` | If set, skips building the CSI driver image (assumes it exists). | `False` +``` + +## Example + +To run the benchmark with default settings, you only need to provide your Google Cloud project ID and the GCS bucket name for the workload: + +```bash +python3 perfmetrics/scripts/gke_orbax_benchmark/run_benchmark.py \ + --project_id "your-gcp-project-id" \ + --bucket_name "your-gcs-bucket-name" +``` + +To run on a specific GCSFuse branch and prevent cleanup after the run: + +```bash +python3 perfmetrics/scripts/gke_orbax_benchmark/run_benchmark.py \ + --project_id "your-gcp-project-id" \ + --bucket_name "your-gcs-bucket-name" \ + --gcsfuse_branch "my-feature-branch" \ + --no_cleanup +``` diff --git a/perfmetrics/scripts/continuous_test/ml_tests/pytorch/v1_12/dino/continuous.cfg b/perfmetrics/scripts/continuous_test/gke/orbax_benchmark/continuous.cfg similarity index 74% rename from perfmetrics/scripts/continuous_test/ml_tests/pytorch/v1_12/dino/continuous.cfg rename to perfmetrics/scripts/continuous_test/gke/orbax_benchmark/continuous.cfg index 03cd9cb47b..7bdd325aaf 100644 --- a/perfmetrics/scripts/continuous_test/ml_tests/pytorch/v1_12/dino/continuous.cfg +++ b/perfmetrics/scripts/continuous_test/gke/orbax_benchmark/continuous.cfg @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -build_file: "gcsfuse/perfmetrics/scripts/continuous_test/ml_tests/pytorch/v1_12/dino/build.sh" +env_vars { + key:"BUCKET_NAME" + value: "llama_europe_west4" +} -# 2 hours timeout. -timeout_mins: 60 +build_file: "gcsfuse/perfmetrics/scripts/continuous_test/gke/orbax_benchmark/run_benchmark.py" diff --git a/perfmetrics/scripts/continuous_test/gke/orbax_benchmark/pod.yaml.template b/perfmetrics/scripts/continuous_test/gke/orbax_benchmark/pod.yaml.template new file mode 100644 index 0000000000..c3835e3a7c --- /dev/null +++ b/perfmetrics/scripts/continuous_test/gke/orbax_benchmark/pod.yaml.template @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gcsfuse-test + annotations: + gke-gcsfuse/volumes: "true" +spec: + restartPolicy: Never + hostNetwork: true + nodeSelector: + cloud.google.com/gke-tpu-topology: 2x2 + cloud.google.com/gke-tpu-accelerator: tpu-v6e-slice + initContainers: + - name: prep-caches + securityContext: + privileged: true + image: ubuntu:22.04 + command: ["bash", "-c"] + args: + - | + echo always > /sys/kernel/mm/transparent_hugepage/enabled + volumeMounts: + - name: sys-mm + mountPath: /sys/kernel/mm + containers: + - name: gke-gcsfuse-sidecar + image: gcr.io/$project_id/cloudbuild-gcsfuse-csi/gcs-fuse-csi-driver-sidecar-mounter:$staging_version + - name: load-test + resources: + limits: + # Make sure you run on a node with these configurations (ct6e-standard-8t) + google.com/tpu: 4 + image: gcr.io/$project_id/kislayk_orbax_workload:latest + imagePullPolicy: Always + command: ["bash", "-c"] + args: + - | + for i in $(seq 1 $iterations); do + echo 3 > /proc/sys/vm/drop_caches + /env/bin/python3 /app/test_load.py load-test --path /orbax_mnt/0/items --num 1 --backend numpy + done + ports: + - containerPort: 5201 + hostPort: 5201 + securityContext: # for cache dropping in the benchmarking tests. + privileged: true + volumeMounts: + - name: python-script + mountPath: /app + - name: data-vol + mountPath: /orbax_mnt + serviceAccountName: default + volumes: + - name: sys-mm + hostPath: + path: /sys/kernel/mm + type: Directory + - name: python-script + configMap: + name: orbax-benchmark + - name: data-vol + csi: + driver: gcsfuse.csi.storage.gke.io + volumeAttributes: + bucketName: "$bucket_name" + mountOptions: "read_ahead_kb=1024,disable-autoconfig,client-protocol=$client_protocol" + hostNetworkPodKSA: "true" + diff --git a/perfmetrics/scripts/continuous_test/gke/orbax_benchmark/requirements.txt b/perfmetrics/scripts/continuous_test/gke/orbax_benchmark/requirements.txt new file mode 100644 index 0000000000..c3726e8bfe --- /dev/null +++ b/perfmetrics/scripts/continuous_test/gke/orbax_benchmark/requirements.txt @@ -0,0 +1 @@ +pyyaml diff --git a/perfmetrics/scripts/continuous_test/gke/orbax_benchmark/run_benchmark.py b/perfmetrics/scripts/continuous_test/gke/orbax_benchmark/run_benchmark.py new file mode 100644 index 0000000000..37db3cdcc5 --- /dev/null +++ b/perfmetrics/scripts/continuous_test/gke/orbax_benchmark/run_benchmark.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run GKE Orbax benchmark. + +This script automates the process of running the Orbax benchmark on a GKE cluster. +It performs the following steps: +1. Checks for prerequisite tools (gcloud, git, make, kubectl). +2. Sets up a GKE cluster with a specific node pool if it doesn't exist. +3. Builds a GCSFuse CSI driver image from a specified git branch. +4. Deploys a Kubernetes pod that runs the benchmark workload. +5. Parses the benchmark results (throughput) from the pod logs. +6. Determines if the benchmark passed based on a performance threshold. +7. Cleans up all created cloud resources (GKE cluster, network, etc.). +""" + +import argparse +import asyncio +import os +import re +import subprocess +import sys +import tempfile +from datetime import datetime +from string import Template + +# Add the parent directory to sys.path to allow imports from common +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.dirname(SCRIPT_DIR)) +from common import utils + +# The prefix prow-gob-internal-boskos- is needed to allow passing machine-type from gke csi driver to gcsfuse, +# bypassing the check at +# https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/blob/15afd00dcc2cfe0f9753ddc53c81631ff037c3f2/pkg/csi_driver/utils.go#L532. +STAGING_VERSION = "prow-gob-internal-boskos-orbax-benchmark" + + +def parse_all_gbytes_per_sec(logs): + """Parses logs to find and extract all gbytes_per_sec values. + + Args: + logs: A string containing the log output from the benchmark pod. + + Returns: + A list of float values representing the 'gbytes_per_sec' found in the logs. + """ + values = [] + for line in logs.splitlines(): + match = re.search(r"gbytes_per_sec: ([\d.]+) Bytes/s", line) + if match: + gbytes_per_sec = float(match.group(1)) + print(f"Extracted gbytes_per_sec: {gbytes_per_sec}") + values.append(gbytes_per_sec) + if not values: + print("gbytes_per_sec not found in logs.", file=sys.stderr) + return values + +# Workload Execution and Result Gathering +async def execute_workload_and_gather_results(project_id, zone, cluster_name, bucket_name, timestamp, iterations, staging_version, pod_timeout_seconds, client_protocol): + """Executes the workload pod, gathers results, and cleans up workload resources. + + This function creates a Kubernetes ConfigMap and a Pod to run the benchmark. + It waits for the pod to complete, collects its logs, parses the throughput + results, and then deletes the created Kubernetes resources. + + Args: + project_id: The Google Cloud project ID. + zone: The GCP zone of the cluster. + cluster_name: The name of the GKE cluster. + bucket_name: The GCS bucket to use for the benchmark. + timestamp: A unique timestamp string for manifest naming. + iterations: The number of benchmark iterations to run inside the pod. + staging_version: The version tag for the GCSFuse CSI driver image. + pod_timeout_seconds: The timeout in seconds for the pod to complete. + client_protocol: The client protocol to use for GCS (e.g. grpc or http1). + + Returns: + A list of throughput values (float) parsed from the pod logs. + """ + await utils.run_command_async(["kubectl", "create", "configmap", "orbax-benchmark", f"--from-file={os.path.join(SCRIPT_DIR, 'test_load.py')}"]) + + template_path = os.path.join(SCRIPT_DIR, "pod.yaml.template") + with open(template_path, "r") as f: + pod_template = Template(f.read()) + + manifest = pod_template.safe_substitute(project_id=project_id, bucket_name=bucket_name, iterations=iterations, staging_version=staging_version, client_protocol=client_protocol) + manifest_filename = f"manifest-{timestamp}.yaml" + pod_name = f"gcsfuse-test" + + try: + with open(manifest_filename, "w") as f: + f.write(manifest) + await utils.run_command_async(["kubectl", "apply", "-f", manifest_filename]) + + start_time = datetime.now() + pod_finished = False + while (datetime.now() - start_time).total_seconds() < pod_timeout_seconds: + status, stderr, _ = await utils.run_command_async(["kubectl", "get", "pod", pod_name, "-o", "jsonpath='{.status.phase}'"], check=False) + if "Succeeded" in status or "Failed" in status: + pod_finished = True + break + await asyncio.sleep(10) + + if not pod_finished: + raise TimeoutError(f"Pod did not complete within {pod_timeout_seconds / 60} minutes.") + + logs, _, _ = await utils.run_command_async(["kubectl", "logs", pod_name], check=False) + if logs: + return parse_all_gbytes_per_sec(logs) + return [] + finally: + await utils.run_command_async(["kubectl", "delete", "configmap", "orbax-benchmark"]) + await utils.run_command_async(["kubectl", "delete", "-f", manifest_filename], check=False) + if os.path.exists(manifest_filename): + os.remove(manifest_filename) + + +# Main function +async def main(): + """Parses arguments, orchestrates the benchmark execution, and handles cleanup. + + This is the main entry point of the script. + """ + parser = argparse.ArgumentParser(description="Run GKE Orbax benchmark.") + parser.add_argument("--project_id", default=os.environ.get("PROJECT_ID", utils.DEFAULT_PROJECT_ID), help="Google Cloud project ID. Can also be set with PROJECT_ID env var.") + parser.add_argument("--bucket_name", required=os.environ.get("BUCKET_NAME") is None, default=os.environ.get("BUCKET_NAME"), help="GCS bucket name for the workload. Can also be set with BUCKET_NAME env var.") + parser.add_argument("--zone", default=os.environ.get("ZONE", utils.DEFAULT_ZONE), help="GCP zone. Can also be set with ZONE env var.") + parser.add_argument("--cluster_name", default=os.environ.get("CLUSTER_NAME", "gke-orbax-benchmark-cluster"), help="GKE cluster name. Can also be set with CLUSTER_NAME env var.") + parser.add_argument("--network_name", default=os.environ.get("NETWORK_NAME", "gke-orbax-benchmark-network"), help="VPC network name. Can also be set with NETWORK_NAME env var.") + parser.add_argument("--subnet_name", default=os.environ.get("SUBNET_NAME", "gke-orbax-benchmark-subnet"), help="VPC subnet name. Can also be set with SUBNET_NAME env var.") + parser.add_argument("--machine_type", default=os.environ.get("MACHINE_TYPE", "ct6e-standard-4t"), help="Machine type. Can also be set with MACHINE_TYPE env var.") + parser.add_argument("--node_pool_name", default=os.environ.get("NODE_POOL_NAME", "ct6e-pool"), help="Node pool name. Can also be set with NODE_POOL_NAME env var.") + parser.add_argument("--gcsfuse_branch", default=os.environ.get("GCSFUSE_BRANCH", "master"), help="GCSFuse branch or tag to build. Can also be set with GCSFUSE_BRANCH env var.") + parser.add_argument("--reservation_name", default=os.environ.get("RESERVATION_NAME", utils.DEFAULT_RESERVATION_NAME), help="The specific reservation to use for the nodes. Can also be set with RESERVATION_NAME env var.") + parser.add_argument("--no_cleanup", action="store_true", default=os.environ.get("NO_CLEANUP", "False").lower() in ("true", "1"), help="Don't clean up resources after. Can also be set with NO_CLEANUP=true env var.") + parser.add_argument("--iterations", type=int, default=int(os.environ.get("ITERATIONS", 20)), help="Number of iterations for the benchmark. Can also be set with ITERATIONS env var.") + parser.add_argument("--performance_threshold_gbps", type=float, default=float(os.environ.get("PERFORMANCE_THRESHOLD_GBPS", 13.0)), help="Minimum throughput in GB/s for a successful iteration. Can also be set with PERFORMANCE_THRESHOLD_GBPS env var.") + parser.add_argument("--pod_timeout_seconds", type=int, default=int(os.environ.get("POD_TIMEOUT_SECONDS", 1800)), help="Timeout in seconds for the benchmark pod to complete. Can also be set with POD_TIMEOUT_SECONDS env var.") + parser.add_argument("--skip_csi_driver_build", action="store_true", default=os.environ.get("SKIP_CSI_DRIVER_BUILD", "False").lower() in ("true", "1"), help="Skip building the CSI driver. Can also be set with SKIP_CSI_DRIVER_BUILD=true env var.") + parser.add_argument("--client_protocol", default=os.environ.get("CLIENT_PROTOCOL", "http1"), help="The client protocol to use for GCS. Can also be set with CLIENT_PROTOCOL env var.") + args = parser.parse_args() + + # Append zone to default network and subnet names to avoid collisions + if args.network_name == "gke-orbax-benchmark-network": + args.network_name = f"{args.network_name}-{args.zone}" + if args.subnet_name == "gke-orbax-benchmark-subnet": + args.subnet_name = f"{args.subnet_name}-{args.zone}" + + await utils.check_prerequisites() + + timestamp = datetime.now().strftime("%Y%m%d%H%M%S") + with tempfile.TemporaryDirectory() as temp_dir: + try: + if args.skip_csi_driver_build: + await utils.setup_gke_cluster(args.project_id, args.zone, args.cluster_name, args.network_name, args.subnet_name, args.zone.rsplit('-', 1)[0], args.machine_type, args.node_pool_name, args.reservation_name) + else: + setup_task = asyncio.create_task(utils.setup_gke_cluster(args.project_id, args.zone, args.cluster_name, args.network_name, args.subnet_name, args.zone.rsplit('-', 1)[0], args.machine_type, args.node_pool_name, args.reservation_name)) + build_task = asyncio.create_task(utils.build_gcsfuse_image(args.project_id,args.gcsfuse_branch, temp_dir, STAGING_VERSION)) + await asyncio.gather(setup_task, build_task) + + throughputs = await execute_workload_and_gather_results(args.project_id, args.zone, args.cluster_name, args.bucket_name, timestamp, args.iterations, STAGING_VERSION, args.pod_timeout_seconds, args.client_protocol) + + if not throughputs: + print("No throughput data was collected.", file=sys.stderr) + if not args.no_cleanup: + await utils.cleanup(args.project_id, args.zone, args.cluster_name, args.network_name, args.subnet_name) + sys.exit(-1) + + successful_iterations = sum(1 for t in throughputs if t >= args.performance_threshold_gbps) + if successful_iterations < (len(throughputs) * 5)/8: # At least 5/8th of the iterations must meet the threshold. + print(f"Benchmark failed: Only {successful_iterations}/{len(throughputs)} iterations were >= {args.performance_threshold_gbps} gbytes/sec.", file=sys.stderr) + if not args.no_cleanup: + await utils.cleanup(args.project_id, args.zone, args.cluster_name, args.network_name, args.subnet_name) + sys.exit(-1) + + print(f"Benchmark successful: {successful_iterations}/{len(throughputs)} iterations met the performance threshold ({args.performance_threshold_gbps} GB/s).") + finally: + if not args.no_cleanup: + await utils.cleanup(args.project_id, args.zone, args.cluster_name, args.network_name, args.subnet_name) + +if __name__ == "__main__": + asyncio.run(main()) + diff --git a/perfmetrics/scripts/continuous_test/gke/orbax_benchmark/test_load.py b/perfmetrics/scripts/continuous_test/gke/orbax_benchmark/test_load.py new file mode 100644 index 0000000000..b03ca45697 --- /dev/null +++ b/perfmetrics/scripts/continuous_test/gke/orbax_benchmark/test_load.py @@ -0,0 +1,199 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# test_load.py +"""A script for benchmarking Orbax checkpoint loading and resaving.""" + +import os +import re +import time +from absl import app +from absl import logging +import click +from etils import epath +import jax +import numpy as np +import orbax.checkpoint as ocp + + +def set_no_of_jax_cpu(num_cpu_devices): + """Configures JAX to use a specific number of CPU devices. + + This must be called before any JAX operations. It sets the `XLA_FLAGS` + environment variable to force JAX to recognize the specified number of CPUs. + + Args: + num_cpu_devices: The number of CPU devices to configure for JAX. + """ + # This session needs to be run before any JAX code. + jax.config.update('jax_platforms', 'cpu') + xla_flags = os.getenv('XLA_FLAGS', '') + xla_flags = re.sub( + r'--xla_force_host_platform_device_count=\S+', '', xla_flags + ).split() + os.environ['XLA_FLAGS'] = ' '.join( + [f'--xla_force_host_platform_device_count={num_cpu_devices}'] + xla_flags + ) + + +def load_ckpt(path, backend): + """Loads an Orbax checkpoint from the given path. + + This function measures the time it takes to restore a checkpoint. It also + handles updating the sharding information in the checkpoint metadata to match + the target backend devices. + + Args: + path: The path to the checkpoint to load. + backend: The JAX backend to use ('cpu', 'tpu', or 'numpy'). + + Returns: + A tuple containing: + - elapsed (float): The time taken to load the checkpoint in seconds. + - restored: The restored checkpoint data. + """ + def update_devices(x): + """Updates sharding metadata to match the target backend devices.""" + if isinstance(x, ocp.metadata.ArrayMetadata): + if isinstance(x.sharding, ocp.metadata.NamedShardingMetadata): + if backend in ('cpu', 'tpu'): + mesh = jax.sharding.Mesh( + np.asarray(jax.devices(backend=backend)), x.sharding.axis_names + ) + pspec = jax.sharding.PartitionSpec(*x.sharding.partition_spec) + sharding = jax.sharding.NamedSharding(mesh, pspec) + x.sharding = ocp.metadata.NamedShardingMetadata.from_jax_sharding( + sharding + ) + else: + x.sharding = None + else: + if backend in ('cpu', 'tpu'): + # assume sharding + if ( + len(x.shape) + and x.shape[0] % jax.device_count(backend=backend) != 0 + ): + raise ValueError(f'Unable to shard shape={x.shape}') + + mesh = jax.sharding.Mesh( + jax.devices(backend=backend), axis_names=('x',) + ) + if len(x.shape): + pspec = jax.sharding.PartitionSpec('x') + else: + pspec = jax.sharding.PartitionSpec() + sharding = jax.sharding.NamedSharding(mesh, pspec) + x.sharding = ocp.metadata.NamedShardingMetadata.from_jax_sharding( + sharding + ) + return x + + with ocp.StandardCheckpointer(restore_concurrent_gb= int(os.environ['RESTORE_CONCURRENT_GB']) if 'RESTORE_CONCURRENT_GB' in os.environ else 512) as ckptr: + metadata = ckptr.metadata(path) + items = jax.tree.map(update_devices, metadata) + items = jax.tree.map(ocp.utils.to_shape_dtype_struct, items) + + def restore(): + return ckptr.restore(path, target=items) + + stime = time.time() + restored = restore() + etime = time.time() + elapsed = etime - stime + + restored_types = set() + jax.tree.map(lambda x: restored_types.add(type(x)), restored) + logging.info('restored_types = %s', restored_types) + + return elapsed, restored + + +def save_ckpt(item, path): + """Saves a checkpoint using Orbax. + + Args: + item: The data (e.g., a PyTree of arrays) to save as a checkpoint. + path: The path where the checkpoint will be saved. + """ + with ocp.StandardCheckpointer() as ckptr: + ckptr.save(path, state=item) + saved_sizes = [] + jax.tree.map( + lambda x: saved_sizes.append(np.prod(x.shape)) * x.dtype.itemsize, item + ) + logging.info(f'Saved {np.sum(saved_sizes) / (1000**3)} GB') + + +@click.group() +def cli(): + """Command-line interface for Orbax benchmark tests.""" + pass + + +@click.command() +@click.option('--path', type=str, default=None, help='path to load') +@click.option( + '--backend', + default='cpu', + type=click.Choice(['cpu', 'tpu', 'numpy']), + help='backend to use', +) +@click.option('--num', default=5, help='Number of iterations.') +@click.option('--cpuno', default=4, help='Number of cpus.') +def load_test(path, backend, num, cpuno): + """Load the checkpoint and time it.""" + if backend == 'cpu': + set_no_of_jax_cpu(cpuno) + if backend in ('cpu', 'tpu'): + logging.info('Loading with %s', jax.devices(backend=backend)) + jax.clear_caches() + elapsed_times = [] + for i in range(num): + elapsed, _ = load_ckpt(path, backend) + print(f'Loop_{i}: Took {elapsed}s to load') + elapsed_times.append(elapsed) + + print(f'Average elapsed time: {np.mean(elapsed_times)}s') + + +@click.command() +@click.option('--path', type=str, default=None, help='input path to load') +@click.option('--output', type=str, default=None, help='output path to save') +@click.option( + '--backend', + default='cpu', + type=click.Choice(['cpu', 'tpu', 'numpy']), + help='backend to use', +) +@click.option('--cpuno', default=4, help='Number of cpus.') +def resave(path, output, backend, cpuno): + """Load the checkpoint from `path` and save it to `output`.""" + if backend == 'cpu': + set_no_of_jax_cpu(cpuno) + if backend in ('cpu', 'tpu'): + logging.info('Loading with %s', jax.devices(backend=backend)) + jax.clear_caches() + elapsed, restored = load_ckpt(path, backend) + print(f'Took {elapsed}s to load') + + save_ckpt(restored, output) + + +cli.add_command(load_test) +cli.add_command(resave) + +if __name__ == '__main__': + logging.set_verbosity(logging.INFO) + cli() \ No newline at end of file diff --git a/perfmetrics/scripts/continuous_test/gke/orbax_benchmark_grpc/continuous.cfg b/perfmetrics/scripts/continuous_test/gke/orbax_benchmark_grpc/continuous.cfg new file mode 100644 index 0000000000..7e0f7e68d9 --- /dev/null +++ b/perfmetrics/scripts/continuous_test/gke/orbax_benchmark_grpc/continuous.cfg @@ -0,0 +1,35 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +env_vars { + key:"BUCKET_NAME" + value: "llama_europe_west4" +} +env_vars { + key:"CLIENT_PROTOCOL" + value: "grpc" +} +env_vars { + key:"CLUSTER_NAME" + value: "gke-orbax-benchmark-grpc" +} +env_vars { + key:"NETWORK_NAME" + value: "gke-orbax-benchmark-grpc-network" +} +env_vars { + key:"SUBNET_NAME" + value: "gke-orbax-benchmark-grpc-subnet" +} +build_file: "gcsfuse/perfmetrics/scripts/continuous_test/gke/orbax_benchmark/run_benchmark.py" diff --git a/perfmetrics/scripts/continuous_test/ml_tests/pytorch/v1_12/dino/build.sh b/perfmetrics/scripts/continuous_test/ml_tests/pytorch/v1_12/dino/build.sh deleted file mode 100755 index 49eb29cc1d..0000000000 --- a/perfmetrics/scripts/continuous_test/ml_tests/pytorch/v1_12/dino/build.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This will stop execution when any command will have non-zero status. -set -e - -VM_NAME="pytorch-dino-7d" -ZONE_NAME="us-west1-b" -ARTIFACTS_BUCKET_PATH="gs://gcsfuse-ml-tests-logs/ci_artifacts/pytorch/v1_12/dino" -TEST_SCRIPT_PATH="github/gcsfuse/perfmetrics/scripts/ml_tests/pytorch/v1_12/dino/setup_host_and_run_container.sh" -PYTORCH_VERSION="v1_12" -BUCKET_TYPE="non-hns" - -cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse/perfmetrics/scripts/continuous_test/ml_tests/" - -source run_and_manage_test.sh $VM_NAME $ZONE_NAME $ARTIFACTS_BUCKET_PATH $TEST_SCRIPT_PATH $PYTORCH_VERSION $BUCKET_TYPE diff --git a/perfmetrics/scripts/continuous_test/ml_tests/pytorch/v2/dino/build.sh b/perfmetrics/scripts/continuous_test/ml_tests/pytorch/v2/dino/build.sh deleted file mode 100755 index 183a588094..0000000000 --- a/perfmetrics/scripts/continuous_test/ml_tests/pytorch/v2/dino/build.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This will stop execution when any command will have non-zero status. -set -e - -VM_NAME="pytorch2-dino-7d-a100-gpu" -ZONE_NAME="asia-northeast1-a" -ARTIFACTS_BUCKET_PATH="gs://gcsfuse-ml-tests-logs/ci_artifacts/pytorch/v2/dino" -TEST_SCRIPT_PATH="github/gcsfuse/perfmetrics/scripts/ml_tests/pytorch/v2/dino/setup_host_and_run_container.sh" -PYTORCH_VERSION="v2" -BUCKET_TYPE="non-hns" - -cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse/perfmetrics/scripts/continuous_test/ml_tests/" - -source run_and_manage_test.sh $VM_NAME $ZONE_NAME $ARTIFACTS_BUCKET_PATH $TEST_SCRIPT_PATH $PYTORCH_VERSION $BUCKET_TYPE diff --git a/perfmetrics/scripts/continuous_test/ml_tests/pytorch/v2_hns/dino/build.sh b/perfmetrics/scripts/continuous_test/ml_tests/pytorch/v2_hns/dino/build.sh deleted file mode 100755 index 5c2ca5ff1a..0000000000 --- a/perfmetrics/scripts/continuous_test/ml_tests/pytorch/v2_hns/dino/build.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This will stop execution when any command will have non-zero status. -set -e - -VM_NAME="pytorch2-dino-7d-a100-gpu-hns-bucket" -ZONE_NAME="us-central1-f" -ARTIFACTS_BUCKET_PATH="gs://gcsfuse-ml-tests-logs/ci_artifacts/pytorch/v2_hns/dino" -TEST_SCRIPT_PATH="github/gcsfuse/perfmetrics/scripts/ml_tests/pytorch/v2/dino/setup_host_and_run_container.sh" -PYTORCH_VERSION="v2" -BUCKET_TYPE="hns" - -cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse/perfmetrics/scripts/continuous_test/ml_tests/" - -source run_and_manage_test.sh $VM_NAME $ZONE_NAME $ARTIFACTS_BUCKET_PATH $TEST_SCRIPT_PATH $PYTORCH_VERSION $BUCKET_TYPE diff --git a/perfmetrics/scripts/continuous_test/ml_tests/run_and_manage_test.sh b/perfmetrics/scripts/continuous_test/ml_tests/run_and_manage_test.sh deleted file mode 100755 index 7b9f8e9a93..0000000000 --- a/perfmetrics/scripts/continuous_test/ml_tests/run_and_manage_test.sh +++ /dev/null @@ -1,221 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This will stop execution when any command will have non-zero status. -set -e - -# 7.5 days of timeout for running test -TIMEOUT=$(echo "7.5*24*60*60" | bc) -GCP_PROJECT="gcs-fuse-test" -# Name of test VM. -VM_NAME=$1 -# Zone of test VM. -ZONE_NAME=$2 -# Bucket path where the test VM artifacts should be saved. -ARTIFACTS_BUCKET_PATH=$3 -# Path of test script relative to $HOME inside test VM. -TEST_SCRIPT_PATH=$4 -# pytorch version -PYTORCH_VERSION=$5 -BUCKET_TYPE=$6 -RESERVATION="projects/$GCP_PROJECT/reservations/ai-ml-tests-2gpus" - -function initialize_ssh_key () { - echo "Delete existing ssh keys " - # This is required to avoid issue: https://github.com/kyma-project/test-infra/issues/93 - for i in $(sudo gcloud compute os-login ssh-keys list | grep -v FINGERPRINT); do sudo gcloud compute os-login ssh-keys remove --key $i; done - - # Requires running first ssh command with --quiet option to initialize keys. - # Otherwise it prompts for yes and no. - sudo gcloud compute ssh $VM_NAME --zone $ZONE_NAME --internal-ip --quiet --command "echo 'Running from VM'" -} - -function delete_existing_vm_and_create_new () { - ( - set +e - - echo "Deleting VM $VM_NAME in zone $ZONE_NAME." - sudo gcloud compute instances delete $VM_NAME --zone $ZONE_NAME --quiet - if [ $? -eq 0 ]; - then - echo "Machine deleted successfully !" - else - echo "Machine was not deleted as it doesn't exist." - fi - ) - - echo "Wait for 30 seconds for old VM to be deleted" - sleep 30s - - # NVIDIA A100 40GB GPU type machine is currently unavailable due to global shortage. - # Create NVIDIA L4 machines which are available on us-west1-1 zone. - if [ $PYTORCH_VERSION == "v2" ]; - then - RESERVATION="projects/$GCP_PROJECT/reservations/ai-ml-tests-pytorch2-2gpu" - fi - - if [ $BUCKET_TYPE == "hns" ]; - then - RESERVATION="projects/$GCP_PROJECT/reservations/ai-ml-tests-hns-bucket" - fi - - echo "Creating VM $VM_NAME in zone $ZONE_NAME" - # The below command creates VM using the reservation 'ai-ml-tests' - sudo gcloud compute instances create $VM_NAME \ - --project=$GCP_PROJECT\ - --zone=$ZONE_NAME \ - --machine-type=a2-highgpu-2g\ - --network-interface=network-tier=PREMIUM,nic-type=GVNIC,stack-type=IPV4_ONLY,subnet=default \ - --metadata=enable-osconfig=TRUE,enable-oslogin=true \ - --maintenance-policy=TERMINATE \ - --provisioning-model=STANDARD \ - --service-account=927584127901-compute@developer.gserviceaccount.com \ - --scopes=https://www.googleapis.com/auth/cloud-platform \ - --accelerator=count=2,type=nvidia-tesla-a100 \ - --create-disk=auto-delete=yes,boot=yes,device-name=$VM_NAME,image=projects/ubuntu-os-cloud/global/images/ubuntu-2004-focal-v20241115,mode=rw,size=150,type=projects/$GCP_PROJECT/zones/$ZONE_NAME/diskTypes/pd-balanced \ - --no-shielded-secure-boot \ - --shielded-vtpm \ - --shielded-integrity-monitoring \ - --labels=goog-ops-agent-policy=v2-x86-template-1-0-0,goog-ec-src=vm_add-gcloud \ - --reservation-affinity=specific \ - --reservation=$RESERVATION - - echo "Wait for 30 seconds for new VM to be initialised" - sleep 30s - - initialize_ssh_key -} - -# Takes commit id of on-going test run ($1) and copies artifacts to GCS bucket. -function copy_run_artifacts_to_gcs () { - ( - # We don't want to exit if failure occurs while copying GCSFuse logs because - # gsutil always gives error (even the files are copied) while uploading - # files that are changing while uploading and gcsfuse logs changes when the - # test is running. - set +e - echo "Copying GCSFuse and test logs to GCS bucket for the run $1" - sudo gcloud compute ssh $VM_NAME --zone $ZONE_NAME --internal-ip --command "gsutil rsync -R -d \$HOME/github/gcsfuse/container_artifacts/ $ARTIFACTS_BUCKET_PATH/$1/container_artifacts" - sudo gcloud compute ssh $VM_NAME --zone $ZONE_NAME --internal-ip --command "gsutil cp \$HOME/build.out $ARTIFACTS_BUCKET_PATH/$1/build.out" - sudo gcloud compute ssh $VM_NAME --zone $ZONE_NAME --internal-ip --command "gsutil cp \$HOME/build.err $ARTIFACTS_BUCKET_PATH/$1/build.err" - echo "\n" - ) - echo "Also, copy the status, commit and start time to $1 artifacts location in GCS bucket" - sudo gcloud compute ssh $VM_NAME --zone $ZONE_NAME --internal-ip --command "gsutil cp $ARTIFACTS_BUCKET_PATH/status.txt $ARTIFACTS_BUCKET_PATH/$1/" - sudo gcloud compute ssh $VM_NAME --zone $ZONE_NAME --internal-ip --command "gsutil cp $ARTIFACTS_BUCKET_PATH/commit.txt $ARTIFACTS_BUCKET_PATH/$1/" - sudo gcloud compute ssh $VM_NAME --zone $ZONE_NAME --internal-ip --command "gsutil cp $ARTIFACTS_BUCKET_PATH/start_time.txt $ARTIFACTS_BUCKET_PATH/$1/" - echo "\n" -} - -# Takes commit id of on-going test run ($1) and cat the artifacts to kokoro build. -function cat_run_artifacts () { - echo "Below is the stdout of build on test VM" - gsutil cat $ARTIFACTS_BUCKET_PATH/$1/build.out - - echo "Below is the stderr of build on test VM" - gsutil cat $ARTIFACTS_BUCKET_PATH/$1/build.err -} - -# Echo status of on-going test run. -function get_run_status () { - status=$(gsutil cat $ARTIFACTS_BUCKET_PATH/status.txt) - echo $status -} - -# Echo commit id of on-going test run. -function get_run_commit_id () { - commit_id=$(gsutil cat $ARTIFACTS_BUCKET_PATH/commit.txt) - echo $commit_id -} - -sudo gcloud config set project $GCP_PROJECT -current_status=$(get_run_status) -echo "The current status is $current_status" -exit_status=0 - -# Transitions: -# START to START: If model run is not triggerred due to some error. -# START to RUNNING: If model is successfully triggerred on GPU. This state is -# changed by setup_host.sh that runs inside docker container of test VM. -if [ $current_status == "START" ]; -then - echo "Update commit Id for the run" - commit_id=$(git rev-parse HEAD) - echo $commit_id > commit.txt - gsutil cp commit.txt $ARTIFACTS_BUCKET_PATH/ - - delete_existing_vm_and_create_new - - echo "Clone the gcsfuse repo on test VM" - sudo gcloud compute ssh $VM_NAME --zone $ZONE_NAME --internal-ip --command "mkdir github; cd github; git clone https://github.com/GoogleCloudPlatform/gcsfuse.git; cd gcsfuse; git checkout master;" - echo "Trigger the build script on test VM" - sudo gcloud compute ssh $VM_NAME --zone $ZONE_NAME --internal-ip --command "bash \$HOME/$TEST_SCRIPT_PATH $BUCKET_TYPE 1> \$HOME/build.out 2> \$HOME/build.err &" - echo "Wait for 15 minutes for test VM to setup for test and to change the status from START to RUNNING." - sleep 900s - - # If the model is still not running after waiting, then the build should fail. - if [ $(get_run_status) != "RUNNING" ]; - then - echo "The model has not started." - exit_status=1 - fi -# If the current state is running, then check for timeout. If timed out then the -# build should fail. -# Transitions: RUNNING TO ERROR: If the model fails. -# RUNNING TO COMPLETE: If the model succeeds. -# The above transitions are done by docker container running inside test VM. -elif [ $current_status == "RUNNING" ]; -then - # Check for timeout. - start_time=$(gsutil cat $ARTIFACTS_BUCKET_PATH/start_time.txt) - current_time=$(date +"%s") - time_elapsed=$(expr $current_time - $start_time) - if (( $(echo "$time_elapsed > $TIMEOUT" | bc -l) )); - then - echo "The tests have time out, start_time was $start_time, current time is $current_time, time elapsed is $time_elapsed" - exit_status=1 - fi -# Fail the build if the current state is ERROR. This state is set by docker -# container running inside test VM if model fails. -# Transitions: ERROR TO START: This has to be changed manually when the model/ -# error is fixed. -elif [ $current_status == "ERROR" ]; -then - exit_status=1 -# Transitions: COMPLETE TO START: Once the current run is complete, mark the -# state as START. -# The status "COMPLETE" is set by docker container inside test VM when the model -# is successfully trained. -elif [ $current_status == "COMPLETE" ]; -then - exit_status=0 -else - echo "Unknown state in status file. Please check." - exit 1 -fi - -initialize_ssh_key -commit_id=$(get_run_commit_id) -copy_run_artifacts_to_gcs $commit_id -cat_run_artifacts $commit_id - -# Change status back to start -if [ $current_status == "COMPLETE" ]; -then - echo "START" > status.txt - gsutil cp status.txt $ARTIFACTS_BUCKET_PATH/ -fi - -exit $exit_status \ No newline at end of file diff --git a/perfmetrics/scripts/continuous_test/ml_tests/tf/resnet/build.sh b/perfmetrics/scripts/continuous_test/ml_tests/tf/resnet/build.sh deleted file mode 100755 index ec68b28a85..0000000000 --- a/perfmetrics/scripts/continuous_test/ml_tests/tf/resnet/build.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This will stop execution when any command will have non-zero status. -set -e - -VM_NAME="tf-resnet-7d" -ZONE_NAME="us-west1-b" -ARTIFACTS_BUCKET_PATH="gs://gcsfuse-ml-tests-logs/ci_artifacts/tf/resnet" -TEST_SCRIPT_PATH="github/gcsfuse/perfmetrics/scripts/ml_tests/tf/resnet/setup_host_and_run_model.sh" -BUCKET_TYPE="non-hns" - -cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse/perfmetrics/scripts/continuous_test/ml_tests/" - -source run_and_manage_test.sh $VM_NAME $ZONE_NAME $ARTIFACTS_BUCKET_PATH $TEST_SCRIPT_PATH "" $BUCKET_TYPE diff --git a/perfmetrics/scripts/continuous_test/ml_tests/tf/resnet/continuous.cfg b/perfmetrics/scripts/continuous_test/ml_tests/tf/resnet/continuous.cfg deleted file mode 100644 index e3165af009..0000000000 --- a/perfmetrics/scripts/continuous_test/ml_tests/tf/resnet/continuous.cfg +++ /dev/null @@ -1,5 +0,0 @@ -# Config file for kokoro test -build_file: "gcsfuse/perfmetrics/scripts/continuous_test/ml_tests/tf/resnet/build.sh" - -# 2 hours timeout. -timeout_mins: 60 diff --git a/perfmetrics/scripts/continuous_test/ml_tests/tf/resnet_hns/build.sh b/perfmetrics/scripts/continuous_test/ml_tests/tf/resnet_hns/build.sh deleted file mode 100755 index 9871e70529..0000000000 --- a/perfmetrics/scripts/continuous_test/ml_tests/tf/resnet_hns/build.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This will stop execution when any command will have non-zero status. -set -e - -VM_NAME="tf-resnet-7d-a100-gpu-hns-bucket" -ZONE_NAME="us-central1-f" -ARTIFACTS_BUCKET_PATH="gs://gcsfuse-ml-tests-logs/ci_artifacts/tf/resnet_hns" -TEST_SCRIPT_PATH="github/gcsfuse/perfmetrics/scripts/ml_tests/tf/resnet/setup_host_and_run_model.sh" -BUCKET_TYPE="hns" - -cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse/perfmetrics/scripts/continuous_test/ml_tests/" - -source run_and_manage_test.sh $VM_NAME $ZONE_NAME $ARTIFACTS_BUCKET_PATH $TEST_SCRIPT_PATH "" $BUCKET_TYPE diff --git a/perfmetrics/scripts/generate_files.py b/perfmetrics/scripts/generate_files.py index 913199b66c..ca0f3fb5ab 100644 --- a/perfmetrics/scripts/generate_files.py +++ b/perfmetrics/scripts/generate_files.py @@ -78,7 +78,7 @@ def generate_files_and_upload_to_gcs_bucket(destination_blob_name, num_of_files, # Uploading batch files to GCS if upload_to_gcs_bucket: process = Popen( - 'gsutil -m cp -r {}/* {}'.format(TEMPORARY_DIRECTORY, + 'gcloud storage cp --recursive {}/* {}'.format(TEMPORARY_DIRECTORY, destination_blob_name), shell=True) process.communicate() @@ -121,13 +121,13 @@ def generate_files_and_upload_to_gcs_bucket(destination_blob_name, num_of_files, args = parser.parse_args(argv[1:]) - # Checking that gsutil is installed: - logmessage('Checking whether gsutil is installed.\n') - process = Popen('gsutil version', shell=True) + # Checking that gcloud is installed: + logmessage('Checking whether gcloud is installed.\n') + process = Popen('gcloud -v', shell=True) process.communicate() exit_code = process.wait() if(exit_code != 0): - print('Gsutil not installed.') + print('gcloud not installed.') subprocess.call('bash', shell=True) config = configparser.ConfigParser() @@ -184,4 +184,3 @@ def generate_files_and_upload_to_gcs_bucket(destination_blob_name, num_of_files, subprocess.call(['rm', '-r', TEMPORARY_DIRECTORY]) logmessage('Process complete.\n') - diff --git a/perfmetrics/scripts/hns_rename_folders_metrics/requirements.txt b/perfmetrics/scripts/hns_rename_folders_metrics/requirements.txt index cafda1ff02..fee52467d1 100644 --- a/perfmetrics/scripts/hns_rename_folders_metrics/requirements.txt +++ b/perfmetrics/scripts/hns_rename_folders_metrics/requirements.txt @@ -7,7 +7,7 @@ argparse==1.4.0 \ --hash=sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4 \ --hash=sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314 - # via -r requirements.in + # via -r hns_rename_folders_metrics/requirements.in cachetools==5.5.0 \ --hash=sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292 \ --hash=sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a @@ -121,12 +121,12 @@ google-api-core[grpc]==2.19.2 \ google-api-python-client==2.145.0 \ --hash=sha256:8b84dde11aaccadc127e4846f5cd932331d804ea324e353131595e3f25376e97 \ --hash=sha256:d74da1358f3f2d63daf3c6f26bd96d89652051183bc87cf10a56ceb2a70beb50 - # via -r requirements.in + # via -r hns_rename_folders_metrics/requirements.in google-auth==2.34.0 \ --hash=sha256:72fd4733b80b6d777dcde515628a9eb4a577339437012874ea286bca7261ee65 \ --hash=sha256:8eb87396435c19b20d32abd2f984e31c191a15284af72eb922f10e5bde9c04cc # via - # -r requirements.in + # -r hns_rename_folders_metrics/requirements.in # google-api-core # google-api-python-client # google-auth-httplib2 @@ -138,7 +138,7 @@ google-auth-httplib2==0.2.0 \ google-cloud-monitoring==2.22.2 \ --hash=sha256:3f07aead6a80a894c5f8e151f1cccf78478eab14e14294f4b83aaa3f478b5c4e \ --hash=sha256:9fc22dac48d14dd1c7fb83ee4a54f7a57bf9852ee6b5c9dca9e3bb093b04c7ee - # via -r requirements.in + # via -r hns_rename_folders_metrics/requirements.in googleapis-common-protos==1.65.0 \ --hash=sha256:2972e6c496f435b92590fd54045060867f3fe9be2c82ab148fc8885035479a63 \ --hash=sha256:334a29d07cddc3aa01dee4988f9afd9b2916ee2ff49d6b757155dc0d197852c0 @@ -238,7 +238,7 @@ numpy==1.24.4 \ --hash=sha256:ed094d4f0c177b1b8e7aa9cba7d6ceed51c0e569a5318ac0ca9a090680a6a1b1 \ --hash=sha256:f136bab9c2cfd8da131132c2cf6cc27331dd6fae65f95f69dcd4ae3c3639c810 \ --hash=sha256:f3a86ed21e4f87050382c7bc96571755193c4c1392490744ac73d660e8f564a9 - # via -r requirements.in + # via -r hns_rename_folders_metrics/requirements.in proto-plus==1.24.0 \ --hash=sha256:30b72a5ecafe4406b0d339db35b56c4059064e69227b8c3bda7462397f966445 \ --hash=sha256:402576830425e5f6ce4c2a6702400ac79897dab0b4343821aa5188b0fab81a12 @@ -275,9 +275,9 @@ pyparsing==3.1.4 \ --hash=sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c \ --hash=sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032 # via httplib2 -requests==2.32.3 \ - --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ - --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 +requests==2.33.0 \ + --hash=sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b \ + --hash=sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652 # via google-api-core rsa==4.9 \ --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \ @@ -285,7 +285,7 @@ rsa==4.9 \ # via google-auth statistics==1.0.3.5 \ --hash=sha256:2dc379b80b07bf2ddd5488cad06b2b9531da4dd31edb04dc9ec0dc226486c138 - # via -r requirements.in + # via -r hns_rename_folders_metrics/requirements.in uritemplate==4.1.1 \ --hash=sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0 \ --hash=sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e diff --git a/perfmetrics/scripts/hns_rename_folders_metrics/run_rename_benchmark.sh b/perfmetrics/scripts/hns_rename_folders_metrics/run_rename_benchmark.sh index fd1f102ed2..563326f411 100755 --- a/perfmetrics/scripts/hns_rename_folders_metrics/run_rename_benchmark.sh +++ b/perfmetrics/scripts/hns_rename_folders_metrics/run_rename_benchmark.sh @@ -21,19 +21,16 @@ echo Installing requirements.. pip install --require-hashes -r requirements.txt --user echo Running script.. -echo "Installing the Cloud Monitoring agent on VM ...." -curl -sSO https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh -sudo bash add-monitoring-agent-repo.sh --also-install - echo "Installing Ops Agent on Vm" curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh sudo bash add-google-cloud-ops-agent-repo.sh --also-install UPLOAD_FLAGS=$1 -gsutil cp gs://periodic-perf-tests/creds.json ../gsheet/ +gcloud storage cp gs://periodic-perf-tests/creds.json ../gsheet/ -echo "Upgrading gcloud version" -../upgrade_gcloud.sh +# Install latest gcloud. +../install_latest_gcloud.sh +export PATH="/usr/local/google-cloud-sdk/bin:$PATH" #echo "Running renaming benchmark on flat bucket" #python3 renaming_benchmark.py config-flat.json flat "$UPLOAD_FLAGS" diff --git a/perfmetrics/scripts/install_bash.sh b/perfmetrics/scripts/install_bash.sh new file mode 100755 index 0000000000..a226fe4e6a --- /dev/null +++ b/perfmetrics/scripts/install_bash.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script to install a specific version of GNU Bash to /usr/local/bin/bash +# Usage: install_bash.go <bash-version> + +# Exit on error, treat unset variables as errors, and propagate pipeline errors. +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "This script requires exactly one argument." + echo "Usage: $0 <bash-version>" + echo "Example: $0 5.3" + exit 1 +fi + +BASH_VERSION="$1" +INSTALL_DIR="/usr/local/" # Installation directory + +# Function to install dependencies like gcc, make and wget if not present +install_dependencies() { + if ! command -v gcc &>/dev/null || ! command -v make &>/dev/null || ! command -v wget &>/dev/null; then + echo "GCC or make not found. Attempting to install build tools..." + if command -v apt-get &>/dev/null; then + sudo apt-get update && sudo apt-get install -y build-essential wget + elif command -v dnf &>/dev/null; then + sudo dnf install -y gcc make wget + elif command -v yum &>/dev/null; then + sudo yum install -y gcc make wget + else + echo "Error: Could not find a known package manager (apt, dnf, yum)." + echo "Please install gcc and make manually before running this script." + exit 1 + fi + echo "Build tools installed successfully." + fi +} + +# Function to download, compile, and install Bash +install_bash() { + ( + set -euo pipefail + local temp_dir + temp_dir=$(mktemp -d /tmp/bash_install_src.XXXXXX) + pushd "$temp_dir" + + wget -q "https://ftp.gnu.org/gnu/bash/bash-${BASH_VERSION}.tar.gz" + tar -xzf "bash-${BASH_VERSION}.tar.gz" + cd "bash-${BASH_VERSION}" + ./configure --prefix="$INSTALL_DIR" --enable-readline + make -s -j"$(nproc 2>/dev/null || echo 1)" + sudo make install + + popd + rm -rf "$temp_dir" + ) +} + +echo "Installing bash version ${BASH_VERSION} to ${INSTALL_DIR}bin/bash" +INSTALLATION_LOG=$(mktemp /tmp/bash_install_log.XXXXXX) + +# Installing dependencies before installing Bash +install_dependencies +set +e +install_bash >"$INSTALLATION_LOG" 2>&1 +installation_status=$? +set -e +if [[ $installation_status -ne 0 ]]; then + echo "Error: Bash version ${BASH_VERSION} installation failed." + cat "$INSTALLATION_LOG" + rm -f "$INSTALLATION_LOG" + exit 1 +else + echo "Bash ${BASH_VERSION} installed successfully." + echo "Checking bash version at ${INSTALL_DIR}bin/bash:" + "${INSTALL_DIR}bin/bash" --version + rm -f "$INSTALLATION_LOG" +fi diff --git a/perfmetrics/scripts/install_go.sh b/perfmetrics/scripts/install_go.sh new file mode 100755 index 0000000000..151ab88217 --- /dev/null +++ b/perfmetrics/scripts/install_go.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script to install a specific go version to /usr/local +# Usage: install_go.sh <go-version> + +# Exit on error, treat unset variables as errors, and propagate pipeline errors. +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "This script requires exactly one argument." + echo "Usage: $0 <go-version>" + echo "Example: $0 1.24.0" + exit 1 +fi + +# Source common utilities for OS and Arch detection +SCRIPT_DIR=$(dirname "$(readlink -f "$0")") +if [ -f "${SCRIPT_DIR}/os_utils.sh" ]; then + source "${SCRIPT_DIR}/os_utils.sh" +else + echo "Error: os_utils.sh not found in ${SCRIPT_DIR}" + exit 1 +fi + +GO_VERSION="$1" +INSTALL_DIR="/usr/local" # Installation directory + +# Function to download, extract, and install go +install_go() { + local temp_dir architecture os_id system_arch + + if ! os_id=$(get_os_id); then + log_error "Failed to detect OS ID." + return 1 + fi + architecture=$(get_go_arch) + system_arch=$(uname -m) + + if [ "$architecture" == "unsupported" ]; then + echo "Unsupported architecture: $system_arch" + return 1 + fi + echo "Detected architecture: $system_arch mapped to Go arch: $architecture" + + echo "Installing dependencies for Go installation..." + install_packages_by_os "$os_id" "wget" "tar" || { + echo "Warning: Could not install dependencies via package manager. Proceeding with existing tools." + } + + temp_dir=$(mktemp -d /tmp/go_install_src.XXXXXX) + pushd "$temp_dir" + + local url="https://go.dev/dl/go${GO_VERSION}.linux-${architecture}.tar.gz" + echo "Downloading from: $url" + wget -O go_tar.tar.gz "https://go.dev/dl/go${GO_VERSION}.linux-${architecture}.tar.gz" -q + + sudo rm -rf "${INSTALL_DIR}/go" # Remove previous installation. + sudo tar -C "$INSTALL_DIR" -xzf go_tar.tar.gz + + popd + sudo rm -rf "$temp_dir" +} + +echo "Installing Go version ${GO_VERSION} to ${INSTALL_DIR}" +INSTALLATION_LOG=$(mktemp /tmp/go_install_log.XXXXXX) +if ! install_go > "$INSTALLATION_LOG" 2>&1; then + echo "Go version ${GO_VERSION} installation failed." + echo "--- Installation Log ---" + cat "$INSTALLATION_LOG" + echo "------------------------" + rm -f "$INSTALLATION_LOG" + exit 1 +else + echo "Go version ${GO_VERSION} installed successfully." + # If this script is run in background or different shell then + # export PATH needs to be called from the shell or use absolute go path + # or permanently add this to path variable in bashrc. + export PATH="${INSTALL_DIR}/go/bin:$PATH" + + # Verify installation + if ! command -v go >/dev/null 2>&1; then + echo "Error: 'go' command not found after installation. Check path." + cat "$INSTALLATION_LOG" + exit 1 + fi + + echo "Go version is: $(go version)" + echo "Go is present at: $(which go)" + rm -f "$INSTALLATION_LOG" +fi diff --git a/perfmetrics/scripts/install_latest_gcloud.sh b/perfmetrics/scripts/install_latest_gcloud.sh new file mode 100755 index 0000000000..ad75441a46 --- /dev/null +++ b/perfmetrics/scripts/install_latest_gcloud.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script to install latest version of gcloud along with alpha components + +# Exit on error, treat unset variables as errors, and propagate pipeline errors. +set -euo pipefail +set -x + +if [[ $# -ne 0 ]]; then + echo "This script requires no argument." + echo "Usage: $0" + exit 1 +fi + +INSTALL_DIR="/usr/local" # Installation directory + +install_latest_gcloud() { + # Upgrade Python first, as gcloud requires a version between 3.9 and 3.13. + # The upgrade_python3.sh script installs Python 3.11.9. + "$(dirname "$0")/upgrade_python3.sh" + # Set CLOUDSDK_PYTHON to point to the newly installed Python executable. + export CLOUDSDK_PYTHON="$HOME/.local/python-3.11.9/bin/python3.11" + + local temp_dir + temp_dir=$(mktemp -d /tmp/gcloud_install_src.XXXXXX) + pushd "$temp_dir" + + wget -O gcloud.tar.gz https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz -q + sudo rm -rf "${INSTALL_DIR}/google-cloud-sdk" # Remove existing gcloud installation + sudo tar -C "$INSTALL_DIR" -xzf gcloud.tar.gz + # Use `sudo env` to pass the CLOUDSDK_PYTHON variable to the gcloud commands. + sudo env CLOUDSDK_PYTHON="$CLOUDSDK_PYTHON" "${INSTALL_DIR}/google-cloud-sdk/install.sh" -q + sudo env CLOUDSDK_PYTHON="$CLOUDSDK_PYTHON" "${INSTALL_DIR}/google-cloud-sdk/bin/gcloud" components update -q + sudo env CLOUDSDK_PYTHON="$CLOUDSDK_PYTHON" "${INSTALL_DIR}/google-cloud-sdk/bin/gcloud" components install alpha -q + popd + sudo rm -rf "$temp_dir" +} + +echo "Installing latest gcloud version to ${INSTALL_DIR}" +INSTALLATION_LOG=$(mktemp /tmp/gcloud_install_log.XXXXXX) +if ! install_latest_gcloud >"$INSTALLATION_LOG" 2>&1; then + echo "latest gcloud installation failed." + cat "$INSTALLATION_LOG" + rm -f "$INSTALLATION_LOG" + exit 1 +else + echo "latest gcloud installed successfully." + echo "gcloud Version is:" + export PATH="/usr/local/google-cloud-sdk/bin:$PATH" + gcloud version + echo "Gcloud is present at: $( (which gcloud) )" + rm -f "$INSTALLATION_LOG" +fi diff --git a/perfmetrics/scripts/load_tests/python/README.md b/perfmetrics/scripts/load_tests/python/README.md deleted file mode 100644 index 928d00a008..0000000000 --- a/perfmetrics/scripts/load_tests/python/README.md +++ /dev/null @@ -1,146 +0,0 @@ -# Python load testing tool - -Python load testing tool is to generate load on the machine it is run using a -given task and then report the latencies of performing the task over the span -of load test. -A task is a piece of python code that is executed in multiple threads -in multiple processes. Users can define their own tasks, pass them to the tool -and pass different flags to configure the load testing. - -Example usage: -``` -python3 load_test.py --tasks-python-file-path /path/to/task/module.py ---num-processes 32 --num-threads 2 --output-dir /dir/for/test/output ---run-time 60 -``` -In the above example usage, two threads are spawned in each of the 32 processes -where each thread runs the task defined in /path/to/task/module.py in a -continuous loop till 60 seconds. After 60 seconds, the results containing -latencies are saved in /dir/for/test/output. - -# Prerequisites - -* python3 -* python packages mentioned in [requirements.txt] - -[requirements.txt]: ./requirements.txt - -# Supported flags -[load_test.py] is the script used that can be used to run load test. The script -accepts many flags to customize load testing configuration. Below are some -important flags that can be passed to the script: -* ```--tasks-python-file-path```: Path to python module (file) containing task -classes implementing task.LoadTestTask. -* ```--tasks-yaml-file-path```: Path to yaml file containing configurations for -tasks. Note: Configurations in this file can only be of defined and recognised -tasks. To know about the recognises types, see [sample_tasks.yaml]. -* ```--num-processes```: Number of processes to spawn in load tests with - --num-threads threads where each thread runs the task. -* ```--num-threads-per-process```: Number of threads to run in each process -spawned for load test. Each thread runs the task in a loop depending and -terminate depending upon other flags. -* ```--run-time```: Duration in seconds for which to run the load test. -* ```--output-dir```: Path to directory where you want to save the output of -load tests. One file is created for each task with which load test is performed. - -For more details on the supported flags, their default values and uses, please -run load_test.py script with ```--help``` flag. - -[load_test.py]: ./load_test.py -[sample_tasks.yaml]: ./sample_tasks.yaml - -# Output metrics -The output of load test contains the following metrics: -* General: Start time, end time, actual run time, tasks count. -* Latencies: Min, mean, max latencies and 25th, 50th, 95th and 99th percentiles -of latencies of task performed over span of load test. - -The output of load test performed using task with name SampleTask is saved -in the file ```output-dir/SampleTask.json```. - -# How to run - -## Custom task -Let's say we want to run a CPU intensive task parallely with 40 processes for 5 -minutes (300s) and save the result in file: ```~/output/CPUTask.json``` - -### Steps: -* Make sure the prerequisites are installed. -* Set ```PYTHONPATH = gcsfuse/perfmetrics/scripts/load_tests/python/``` -* Create a module for task class and implement LoadTestTask class in it i.e. -define task method. E.g. -``` -from load_generator import task - -class CPUTask(task.LoadTestTask): - - def task(self, process_id, thread_id): - s = 0 - for i in range(1000000): - s = s + process_id * thread_id - return s -``` -cpu_task.py -* Run the following command: -``` -python3 load_test.py --tasks-python-file-path cpu_task.py --num-processes 40 ---output-dir ~/output --run-time 300 -``` -* The latencies of CPU task performed over the span of load test is saved in -```~/output/CPUTask.json```. - -## Predefined tasks -The following [tasks] are predefined in tasks directory: -* python_os.py: Tasks to read files from disk python's native open api. Can be -used with GCSFuse if disk is mounted using GCSFuse. -* tf_gfile.py: Tasks to read files from GCS using tf's tf.io.gfile.Gfile api. -Can be used with GCSFuse or GCS files. -* tf_data.py: Tasks to read files from GCS using tf's tf.data api. Can be used -with GCSFuse or GCS files. - -For more details on the tasks, please refer to the module level -description of files. - -### Steps: -* Make sure the prerequisites are installed. -* Set ```PYTHONPATH = gcsfuse/perfmetrics/scripts/load_tests/python/``` -* Create a yaml file containing configs for predefined tasks. E.g. -``` ---- -200mb_os: - task_type: python_os_read - file_path_format: ./gcs/200mb/read.{process_id} - file_size: 200M - -200mb_tf_data: - task_type: tf_data_read - file_path_format: gs://load-test-bucket-gcs/200mb/read.{file_num}.tfrecord - file_size: 200M - num_files: 3072 -``` -read_tasks.yaml. - -For more details on the supported parameters in configs of predefined tasks, -please refer to [sample_tasks.yaml] -* Run the following command: -``` -python3 load_test.py --tasks-yaml-file-path read_tasks.yaml --num-processes 40 ---output-dir ~/output --run-time 300 -``` -* The latencies of read tasks performed over the span of load test is saved in - ```~/output/200mb_os.json``` & ```~/output/200mb_tf_data.json```. - -[sample_tasks.yaml]: ./sample_tasks.yaml -[tasks]: tasks - -# Miscellaneous -* GCSFuse has to be mounted for using with [python_os.py] tasks. -* It is recommended to keep --num-processes and --num-threads as 1 for -[tf_data.py] tasks as the parallelism is inside those tasks. -* All the tasks defined under [tasks] directory are marked as read/write tasks. -So, load_test.py script tries to create files before running actual load tests. -* Task using tf apis require gcloud login on machine. - -[python_os.py]: tasks/python_os.py -[tf_data.py]: tasks/tf_data.py -[tasks]: tasks diff --git a/perfmetrics/scripts/load_tests/python/__init__.py b/perfmetrics/scripts/load_tests/python/__init__.py deleted file mode 100644 index 1dc90d1848..0000000000 --- a/perfmetrics/scripts/load_tests/python/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/perfmetrics/scripts/load_tests/python/load_generator/constants.py b/perfmetrics/scripts/load_tests/python/load_generator/constants.py deleted file mode 100644 index a738df22e5..0000000000 --- a/perfmetrics/scripts/load_tests/python/load_generator/constants.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Module for constants to be used in load generator module and script. -""" -# Sizes -KB = 1024 -MB = 1024 * KB -GB = 1024 * MB - -# Used in showing percentage of load test completed (time-wise). e.g 0.25 -# represents load test has run 25% of its total run time. -TIME_LOADING_PERCENTAGES = (0.25, 0.50, 0.75) - -# Metrics names -START_TIME = 'start_time' -END_TIME = 'end_time' -TASKS_RESULTS = 'tasks_results' -PRE_TASKS_RESULTS = 'pre_tasks_results' -POST_TASKS_RESULTS = 'post_tasks_results' -TASKS_LAT_STATS = 'tasks_lat_stats' -PRE_TASKS_LAT_STATS = 'pre_tasks_lat_stats' -POST_TASKS_LAT_STATS = 'post_tasks_lat_stats' -MIN = 'min' -MAX = 'max' -MEAN = 'mean' -PER_25 = 'per_25' -PER_50 = 'per_50' -PER_90 = 'per_90' -PER_95 = 'per_95' -PER_99 = 'per_99' -TASKS_COUNT = 'tasks_count' -ACTUAL_RUN_TIME = 'actual_run_time' diff --git a/perfmetrics/scripts/load_tests/python/load_generator/load_generator.py b/perfmetrics/scripts/load_tests/python/load_generator/load_generator.py deleted file mode 100644 index df50b7ae55..0000000000 --- a/perfmetrics/scripts/load_tests/python/load_generator/load_generator.py +++ /dev/null @@ -1,392 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Load Generator for performing load test with a given task (in python). - -Contains the class for generating load with the help of a task written in -python. - -Example: - - task_obj = ReadTask() - lg_obj = LoadGenerator(...) - lg_obj.pre_load_generation(...) - observations = lg_obj.generate_load(task_obj) - metrics = lg_obj.post_load_generation(observations, ...) -""" -import logging -import sys -import threading -import time -import json -import multiprocessing -import numpy as np -from dataclasses import dataclass -from typing import Any -from load_generator import constants as lg_const - - -@dataclass(frozen=True) -class TaskExecutionResult: - """Dataclass for a single task (pre, task & post) execution result. - - process_id: Integer value denoting the process id in which the task was run. - thread_id: Integer value denoting the thread id in process_id in which the - task was run. - start_time: Float value denoting the time when the task was started. - end_time: Float value denoting the time when the task was ended. - result: Any type of value that the task returns - """ - process_id: int - thread_id: int - start_time: float - end_time: float - result: Any - - -class LoadGenerator: - """Generates load using a given task. - - Generates load on CPU and other resources depending upon the given task. This - class also provides default implementation of post load test task to get - latencies of tasks. Classes derived from this class can define - their own pre- and post-load test tasks and run before and after actual load - test. - - Args: - num_processes: An integer defining the number of processes to run during - load test. - num_threads_per_process: An integer defining the number of threads to run - in each process during load test. - run_time: An integer defining the number of seconds to run the load test - for. - num_executions_per_thread: An integer defining the number of times the given - task to be run inside each thread of each process during load test. - """ - - def __init__(self, - num_processes, - num_threads_per_process, - run_time=sys.maxsize, - num_executions_per_thread=sys.maxsize): - self.num_processes = num_processes - self.num_threads_per_process = num_threads_per_process - self.run_time = min(sys.maxsize, run_time) - self.num_executions_per_thread = min(sys.maxsize, num_executions_per_thread) - - if (self.run_time == sys.maxsize) & \ - (self.num_executions_per_thread == sys.maxsize): - raise ValueError('Out of run_time and num_executions_per_thread ' - 'arguments, one has to be passed.') - - self.total_num_tasks = sys.maxsize - if self.num_executions_per_thread != sys.maxsize: - self.total_num_tasks = self.num_executions_per_thread * \ - self.num_threads_per_process * self.num_processes - - def pre_load_generation(self): - """Task to perform before running load test. - """ - pass - - def generate_load(self, task): - """Performs load test using the given task. - - The load is generated on CPU and other resources (depending upon the task - used) by running process(s) and thread(s) where task runs inside thread(s) - and thread(s) runs inside process(s). - - Args: - task: Implementation of task.LoadTestTask. - - Returns: - Returns start_time, end_time of load test, latencies and results of all - the tasks performed over the span of load test. - - Raises: - RuntimeError: If the given task is not completed even once during the - course of load test. - """ - tasks_results_queue = multiprocessing.Manager().Queue() - pre_tasks_results_queue = multiprocessing.Manager().Queue() - post_tasks_results_queue = multiprocessing.Manager().Queue() - - processes = [] - for process_id in range(self.num_processes): - process = multiprocessing.Process( - target=LoadGenerator._process_task, - args=(task, process_id, self.num_threads_per_process, - self.num_executions_per_thread, pre_tasks_results_queue, - tasks_results_queue, post_tasks_results_queue)) - processes.append(process) - - # Initialize checkpoints to show completion of load test i.e. 25%, 50% etc. - # Note: Completion checkpoints are shown only when self.run_time is set. - # E.g. if self.run_time is 60 then the load test will inform that 50% of - # load test is completed after 30 seconds. - log_loading = self.run_time != sys.maxsize - loading_checkpoints = list( - map(lambda t: (t * self.run_time), lg_const.TIME_LOADING_PERCENTAGES)) - curr_loading_idx = 0 - - for process in processes: - process.start() - logging.debug('%s number of processes started for task %s', len(processes), - task.task_name) - - start_time = curr_time = time.time() - loading_checkpoints = [t + start_time for t in loading_checkpoints] - # Loop till the condition of termination of load test is not met. The - # condition is either the load test has run for self.run_time or completed - # the total number of tasks assigned. - while ((curr_time - start_time) < self.run_time) & \ - (tasks_results_queue.qsize() < self.total_num_tasks): - # Sleep so that the looping is not very fast. 0.1 is decided on - # discretion with the intention that time duration shouldn't be very - # small or shouldn't be very large. - time.sleep(0.1) - curr_time = time.time() - if log_loading & (curr_loading_idx < len(loading_checkpoints)) and ( - curr_time >= loading_checkpoints[curr_loading_idx]): - logging.info('Load test completed %s%% for task: %s', - lg_const.TIME_LOADING_PERCENTAGES[curr_loading_idx] * 100, - task.task_name) - curr_loading_idx = curr_loading_idx + 1 - logging.info('Load test completed 100%% for task: %s', task.task_name) - - for process in processes: - process.terminate() - logging.debug('%s number of processes terminated for task %s', - len(processes), task.task_name) - - # Raise error if not even a single task is completed - if tasks_results_queue.qsize() < 1: - raise RuntimeError('Not even a single task is completed. Pass higher ' - 'value to --run-time flag or check the task.') - return { - lg_const.START_TIME: - start_time, - lg_const.END_TIME: - curr_time, - lg_const.TASKS_RESULTS: - self._convert_multiprocessing_queue_to_list(tasks_results_queue), - lg_const.PRE_TASKS_RESULTS: - self._convert_multiprocessing_queue_to_list(pre_tasks_results_queue - ), - lg_const.POST_TASKS_RESULTS: - self._convert_multiprocessing_queue_to_list(post_tasks_results_queue - ), - } - - def post_load_generation(self, - observations, - output_file=None, - print_metrics=True): - """Task to perform after load testing. - - In this default implementation, latency metrics are computed. It can also - dump and print the metrics. - - Args: - observations: Observations collected during load generation. - output_file: String path where the metrics are dumped in JSON. - print_metrics: Bool, whether to print the metrics on console or not. - - Returns: - Default metrics (latencies of tasks) in a dictionary. - """ - metrics = self._compute_default_post_test_metrics(observations) - - # Dump metrics. - if output_file: - self._dump_metrics_into_json(metrics, output_file) - - # Print metrics on console - if print_metrics: - self._print_default_metrics(metrics) - - return metrics - - @staticmethod - def _thread_task(task, process_id, thread_id, num_executions_per_thread, - pre_tasks_results_queue, tasks_results_queue, - post_tasks_results_queue): - """Task run in threads spawned during the load test. - - The task used for the load test is run inside this thread. Pre- and - post-task are also run inside this thread. - This method is kept as protected as it is not used in other classes and - static because class methods can't be passed as target of thread. - """ - cnt = 0 - tasks = [task.pre_task, task.task, task.post_task] - queues = [ - pre_tasks_results_queue, tasks_results_queue, post_tasks_results_queue - ] - while cnt < num_executions_per_thread: - for curr_task, curr_queue in zip(tasks, queues): - start_time = time.time() - result = curr_task(process_id, thread_id) - end_time = time.time() - curr_queue.put( - TaskExecutionResult( - process_id=process_id, - thread_id=thread_id, - start_time=start_time, - end_time=end_time, - result=result)) - cnt = cnt + 1 - - @staticmethod - def _process_task(task, process_id, num_threads_per_process, - num_executions_per_thread, pre_tasks_results_queue, - tasks_results_queue, post_tasks_results_queue): - """Task run in processes spawned during the load test. - - It spawns num_threads_per_process number of threads where each thread runs - the task of load test. - This method is kept as protected as it is not used in other classes and - static because class methods can't be passed as target of process. - """ - # Spawn threads that will run task of load test. - threads = [] - for thread_num in range(num_threads_per_process): - threads.append( - threading.Thread( - target=LoadGenerator._thread_task, - args=(task, process_id, thread_num, num_executions_per_thread, - pre_tasks_results_queue, tasks_results_queue, - post_tasks_results_queue))) - - for thread in threads: - # Thread is kept as daemon, so that it is killed when the parent process - # is killed. - thread.daemon = True - thread.start() - logging.debug('Threads started for process number: %s', process_id) - - for thread in threads: - thread.join() - logging.debug('Threads tasks completed for process number: %s', process_id) - - def _convert_multiprocessing_queue_to_list(self, mp_queue): - """Converts the multiprocessing queue to list. - """ - queue_size = mp_queue.qsize() - return [mp_queue.get() for _ in range(queue_size)] - - def _compute_percentiles(self, data_pts): - """Compute percentiles for given data points. - - Args: - data_pts: List of integer data points. - - Returns: - Dictionary containing 25, 50, 90, 95 & 99 percentiles along with min, - max and mean. - """ - np_array = np.array(data_pts) - return { - lg_const.MIN: min(data_pts), - lg_const.MEAN: np.mean(np_array), - lg_const.MAX: max(data_pts), - lg_const.PER_25: np.percentile(np_array, 25), - lg_const.PER_50: np.percentile(np_array, 50), - lg_const.PER_90: np.percentile(np_array, 90), - lg_const.PER_95: np.percentile(np_array, 95), - lg_const.PER_99: np.percentile(np_array, 99) - } - - def _compute_default_post_test_metrics(self, observations): - """Computes default post load test metrics using observations. - - Computes latency related metrics (percentiles, min, max and mean) of tasks - performed over the course of load test. - """ - # Time stamps - start_time = observations[lg_const.START_TIME] - end_time = observations[lg_const.END_TIME] - actual_run_time = end_time - start_time - - # Latency stats - latency_stats_names = [ - lg_const.PRE_TASKS_LAT_STATS, lg_const.TASKS_LAT_STATS, - lg_const.POST_TASKS_LAT_STATS - ] - result_names = [ - lg_const.PRE_TASKS_RESULTS, lg_const.TASKS_RESULTS, - lg_const.POST_TASKS_RESULTS - ] - latency_stats = {} - for stat_name, result_name in zip(latency_stats_names, result_names): - lat_pts = [ - result.end_time - result.start_time - for result in observations[result_name] - ] - lat_pers = self._compute_percentiles(lat_pts) - latency_stats[stat_name] = lat_pers - - metrics = { - lg_const.START_TIME: start_time, - lg_const.END_TIME: end_time, - lg_const.ACTUAL_RUN_TIME: actual_run_time, - lg_const.TASKS_COUNT: len(observations[lg_const.TASKS_RESULTS]) - } - metrics.update(latency_stats) - return metrics - - def _dump_metrics_into_json(self, metrics, output_file): - """Dumps given metrics as JSON file in UTF-8 format given output directory. - """ - with open(output_file, 'w', encoding='utf-8') as f_p: - json.dump(metrics, f_p) - - def _print_default_metrics(self, metrics): - """Prints given default metrics to console. - """ - actual_run_time = metrics[lg_const.ACTUAL_RUN_TIME] - latency_stats_names = [ - lg_const.PRE_TASKS_LAT_STATS, lg_const.TASKS_LAT_STATS, - lg_const.POST_TASKS_LAT_STATS - ] - task_type_names = ['Pre', 'Task', 'Post'] - - # Time metrics - print('\nTime: ') - print('\tStart time (epoch): ', metrics[lg_const.START_TIME]) - print('\tEnd time (epoch): ', metrics[lg_const.END_TIME]) - print('\tActual run time (in seconds): ', actual_run_time) - - # Task related - print('\nTasks: ') - print('\tTasks count: ', metrics[lg_const.TASKS_COUNT]) - print('\tTasks per sec: ', metrics[lg_const.TASKS_COUNT] / actual_run_time) - - # Latency metrics - print('\nTasks latencies: ') - for task_type, latency_stat_name in zip(task_type_names, - latency_stats_names): - print('\t', task_type, ': ') - print('\t\tMin (in seconds): ', metrics[latency_stat_name][lg_const.MIN]) - print('\t\tMean (in seconds): ', metrics[latency_stat_name][lg_const.MAX]) - print('\t\t25th Percentile (in seconds): ', - metrics[latency_stat_name][lg_const.PER_25]) - print('\t\t50th Percentile (in seconds): ', - metrics[latency_stat_name][lg_const.PER_50]) - print('\t\t90th Percentile (in seconds): ', - metrics[latency_stat_name][lg_const.PER_90]) - print('\t\t95th Percentile (in seconds): ', - metrics[latency_stat_name][lg_const.PER_95]) - print('\t\t99th Percentile (in seconds): ', - metrics[latency_stat_name][lg_const.PER_99]) - print('\t\tMax (in seconds): ', metrics[latency_stat_name][lg_const.MAX]) diff --git a/perfmetrics/scripts/load_tests/python/load_generator/task.py b/perfmetrics/scripts/load_tests/python/load_generator/task.py deleted file mode 100644 index 946a8cae4f..0000000000 --- a/perfmetrics/scripts/load_tests/python/load_generator/task.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains abstract class to represent task in load test. - -Example: - - lg_obj = lg.LoadGenerator(...) - task_obj = TaskImplementingLoadTestTask(...) - metrics = lg_obj.generate_load(task_obj) -""" -from abc import ABC, abstractmethod - - -class LoadTestTask(ABC): - """Abstract class to represent task in load test. - - pre_task represents the task to be performed before performing the actual - task and post_task represents the task to be performed after the actual task. - However, pre_task and post_task are not mandatory for a task in a typical load - test. - """ - - def __init__(self): - # A name given to task. It is recommended to keep a unique name for every - # task implemented in same module. - self.task_name = self.__class__.__name__ - - def pre_task(self, process_id, thread_id): - """Task to be always performed before the actual task in a load test. - - process_id & thread_id are kept as arguments in case the task logic - requires them. - - Args: - process_id: The process number assigned by the load generator to - the process running this task. - thread_id: The process number assigned by the load generator to - the thread running this task. - - Returns: - Result of the task. Type can be anything depending upon the use-case. - """ - pass - - @abstractmethod - def task(self, process_id, thread_id): - """Actual task in a load test. - - process_id & thread_id are kept as arguments in case the task logic - requires them. - - Args: - process_id: The process number assigned by the load generator to - the process running this task. - thread_id: The process number assigned by the load generator to - the thread running this task. - - Returns: - Result of the task. Type can be anything depending upon the use-case. - """ - pass - - def post_task(self, process_id, thread_id): - """Task to be always performed after the actual task in a load test. - - process_id & thread_id are kept as arguments in case the task logic - requires them. - - Args: - process_id: The process number assigned by the load generator to - the process running this task. - thread_id: The process number assigned by the load generator to - the thread running this task. - - Returns: - Result of the task. Type can be anything depending upon the use-case. - """ - pass diff --git a/perfmetrics/scripts/load_tests/python/load_test.py b/perfmetrics/scripts/load_tests/python/load_test.py deleted file mode 100644 index e864716276..0000000000 --- a/perfmetrics/scripts/load_tests/python/load_test.py +++ /dev/null @@ -1,360 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Script to run load tests with given tasks and configuration. - -The script requires tasks to run in load tests which can be written in a python -module or YAML file and then path of that file can be passed to this load script -using --tasks-python-file-path or --tasks-yaml-file path flag. -Apart from the tasks, the load test can also be customized in multiple ways -like number of processes, threads inside each process and run time. -To know more about the flags supported in script, run: - python3 load_test.py --help - -Example: - python3 load_test.py --task-python-file-path tasks/python_os.py - --num-processes 40 --num-threads 1 --run-time 90 --output-dir ~/output -""" -import logging -import importlib -import importlib.machinery -import importlib.util -import inspect -import sys -import os -import time -import argparse -import yaml -import re - -from load_generator import load_generator as lg -from load_generator import task -from load_generator import constants as lg_const -from tasks import python_os -from tasks import tf_data -from tasks import tf_gfile - -READ_WRITE_TASK_TYPES = [ - python_os.PYTHON_OS_READ, tf_data.TF_DATA_READ, tf_gfile.TF_GFILE_READ -] -KNOWN_TASK_TYPES = READ_WRITE_TASK_TYPES -SIZE_ABBREV_TO_SIZE = {'K': lg_const.KB, 'M': lg_const.MB, 'G': lg_const.GB} - - -def parse_args(): - """Parses the command line arguments and returns them. - - Returns: - Arguments passed through command line set as attributes in args object. - """ - parser = argparse.ArgumentParser( - description='Script to do Load testing with ' - 'given task using multiprocessing and ' - 'multithreading on CPU and ' - 'other resources.', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument( - '--tasks-python-file-path', - type=str, - help='Path to python module (file) containing task classes implementing ' - 'the base task.LoadTestTask') - group.add_argument( - '--tasks-yaml-file-path', - type=str, - help='Path to yaml file containing configs for known (predefined) ' - f'tasks: {KNOWN_TASK_TYPES}') - parser.add_argument( - '--task-names', - type=str, - default='', - help='Comma separated name(s) of tasks (task.LoadTestTask.task_name) in ' - 'the --tasks-python-file-path or --tasks-yaml-file-path. Load test is ' - 'conducted only with those tasks. If empty string or nothing is passed ' - "then load test is conducted with all tasks. E.g. 'TaskA, TaskB', " - "'TaskA,TaskB'.") - parser.add_argument( - '--output-dir', - type=str, - default=None, - help='Path to directory where you want to save the output of load tests. ' - ) - parser.add_argument( - '--num-processes', - type=int, - default=1, - help='Number of processes to spawn in load tests with ' - '--num-threads-per-process threads where each thread runs the task.') - parser.add_argument( - '--num-threads-per-process', - type=int, - default=1, - help='Number of threads to run in each process spawned for load test. ' - 'Each thread runs the task in a loop and terminate depending upon other ' - 'flags.') - parser.add_argument( - '--run-time', - type=int, - default=600, - help='Duration in seconds for which to run the load test. Note: The load ' - 'test may terminate before depending upon value of ' - '--num-executions-per-thread flags passed.') - parser.add_argument( - '--num-executions-per-thread', - type=int, - default=sys.maxsize, - help='Total number of times the given task to be performed inside each ' - 'thread of each process of load test. Note: It is possible that the task ' - 'is not performed given number of times if --run-time is not enough.') - parser.add_argument( - '--start-delay', - type=int, - default=5, - help='Time in seconds to wait before conducting load test on a task.') - parser.add_argument( - '--debug', - action='store_true', - help='Prints debug logs along with info logs.') - args = parser.parse_args() - args.task_names = args.task_names.replace(' ', '').split(',') - args.task_names = [el for el in args.task_names if len(el)] - return args - - -def import_module_using_src_code_path(src_code_path): - """Imports the python module from its path to source code at runtime. - - Args: - src_code_path: String path to source code of module. - - Returns: - Imported python module - """ - module_name = src_code_path.split('/')[-1].replace('.py', '') - loader = importlib.machinery.SourceFileLoader(module_name, src_code_path) - spec = importlib.util.spec_from_loader(loader.name, loader) - mod = importlib.util.module_from_spec(spec) - loader.exec_module(mod) - return mod - - -def get_tasks_from_python_file_path(python_file_path): - """Get tasks defined in given python module from its file path. - - Args: - python_file_path: String file path to python module. - - Returns: - List of task objects (task.LoadTestTask) - """ - mod = import_module_using_src_code_path(python_file_path) - task_objs = [] - for _, cls in inspect.getmembers(mod, inspect.isclass): - # Skip classes imported in the task file - if cls.__module__ != mod.__name__: - continue - # Skip classes that are not of type task.LoadTestTask or don't - # have implementation. - if (not issubclass(cls, task.LoadTestTask)) or (inspect.isabstract(cls)): - continue - task_objs.append(cls()) - return task_objs - - -def parse_file_size_str(file_size_str): - """Gives file size in bytes from size string. - - Args: - file_size_str: String file size. - - Returns: - Size of file in bytes. - - Raises: - ValueError: If file size string is not of recognised format. - """ - if not bool(re.fullmatch(r'[0-9]+[kKmMgB]', file_size_str)): - raise ValueError('The file size str set in config is not of recognised ' - f'format: {file_size_str}') - return int(file_size_str[:-1]) * \ - SIZE_ABBREV_TO_SIZE[file_size_str[-1].upper()] - - -def get_task_from_config(task_name, config): - """Identify, creates and returns task from config defining task. - - Args: - task_name: String name of task. - config: Dictionary defining config for task. - - Returns: - Task object instantiated using config. - - Raises: - ValueError: If the task_type in config is not recognised and defined. - """ - task_type = config['task_type'] - config.pop('task_type') - config['file_size'] = parse_file_size_str(config['file_size']) - if 'block_size' in config: - config['block_size'] = parse_file_size_str(config['block_size']) - task_cls = None - if task_type == python_os.PYTHON_OS_READ: - task_cls = python_os.OSRead - elif task_type == tf_data.TF_DATA_READ: - task_cls = tf_data.TFDataRead - elif task_type == tf_gfile.TF_GFILE_READ: - task_cls = tf_gfile.TFGFileRead - else: - raise ValueError(f'Given task type {task_type} is not in known types ' - '{KNOWN_TASK_TYPES}') - return task_cls(task_name=task_name, **config) - - -def get_tasks_from_yaml_file_path(yaml_file_path): - """Gives tasks corresponding to configs in given yaml from its file path. - - Args: - yaml_file_path: String path to yaml file. - - Returns: - Task objects instantiated using configs defined in yaml file. - """ - task_configs = {} - with open(yaml_file_path, 'r', encoding='utf-8') as yaml_fh: - task_configs = yaml.safe_load(yaml_fh) - - task_objs = [] - for task_name, config in task_configs.items(): - task_objs.append(get_task_from_config(task_name, config)) - return task_objs - - -class LoadGeneratorWithFileCreation(lg.LoadGenerator): - """Custom load generator derived from load_generator.LoadGenerator. - - This implementation has pre- and post- load test methods for read and write - files task types. (READ_WRITE_TASK_TYPES) - See base class for more details. - """ - - def pre_load_generation(self, task_obj): - """Pre- load test function to create files for read and write tasks. - - Calls task_obj.create_files method if task has create_files method - implemented. create_files method is implemented in case of - READ_WRITE_TASK_TYPES. - - Args: - task_obj: Task object on which load testing to be performed. - - Returns: - None - """ - if getattr(task_obj, 'task_type', '') in READ_WRITE_TASK_TYPES and \ - hasattr(task_obj, 'create_files'): - task_obj.create_files(self.num_processes) - - def post_load_generation(self, observations, output_file, print_metrics, - task_obj): - """Custom implementation of post-load test function. - - This function adds avg_computed_net_bw on top of default post load test - implementation. - avg_computed_net_bw = (SUM(values returned by task.LoadGenerator.task()) / - actual_run_time). - See base class implementation for more details. - """ - metrics = super().post_load_generation(observations, output_file, - print_metrics) - # only run custom logic for read and write tasks - if getattr(task_obj, 'task_type', '') not in READ_WRITE_TASK_TYPES: - return metrics - - # compute bandwidth from task results - total_io_bytes = sum( - (task_result.result - for task_result in observations[lg_const.TASKS_RESULTS])) - avg_computed_net_bw = total_io_bytes / metrics[lg_const.ACTUAL_RUN_TIME] - avg_computed_net_bw = avg_computed_net_bw / lg_const.MB - metrics.update({'avg_computed_net_bw': avg_computed_net_bw}) - - # Re-dump the metrics to same file. - if output_file: - self._dump_metrics_into_json(metrics, output_file) - - if print_metrics: - # print additional metrics - print('\nNetwork bandwidth (computed by Sum(task response) / actual ' - 'run time):') - print('\tAvg. bandwidth (MiB/sec): ', metrics['avg_computed_net_bw'], - '\n') - return metrics - - -def main(): - args = parse_args() - - logging.getLogger().setLevel(logging.INFO) - if args.debug: - logging.getLogger().setLevel(logging.DEBUG) - - logging.info('Initialising Load Generator...') - lg_obj = LoadGeneratorWithFileCreation( - num_processes=args.num_processes, - num_threads_per_process=args.num_threads_per_process, - run_time=args.run_time, - num_executions_per_thread=args.num_executions_per_thread) - - task_objs = [] - if args.tasks_python_file_path: - task_objs = get_tasks_from_python_file_path(args.tasks_python_file_path) - else: - task_objs = get_tasks_from_yaml_file_path(args.tasks_yaml_file_path) - - # keep only those tasks passed in args.task_names. Note: If task_names = '' - # then all tasks are kept. - filtered_task_objs = [] - for task_obj in task_objs: - if len(args.task_names) == 0 or task_obj.task_name in args.task_names: - filtered_task_objs.append(task_obj) - - logging.info('Starting load generation...') - load_test_results = [] - for task_obj in filtered_task_objs: - logging.info('\nSleeping for: %s seconds', args.start_delay) - time.sleep(args.start_delay) - - logging.info('\nRunning pre load test task for: %s', task_obj.task_name) - lg_obj.pre_load_generation(task_obj=task_obj) - - logging.info('Generating load for: %s', task_obj.task_name) - observations = lg_obj.generate_load(task_obj) - - output_file = None - if args.output_dir and (not os.path.exists(args.output_dir)): - os.makedirs(args.output_dir) - output_file = os.path.join(args.output_dir, f'{task_obj.task_name}.json') - - logging.info('Running post load test task for: %s', task_obj.task_name) - metrics = lg_obj.post_load_generation( - observations, - output_file=output_file, - print_metrics=True, - task_obj=task_obj) - load_test_results.append((task_obj.task_name, metrics)) - logging.info('Load test completed for task: %s', task_obj.task_name) - - -if __name__ == '__main__': - main() diff --git a/perfmetrics/scripts/load_tests/python/requirements.txt b/perfmetrics/scripts/load_tests/python/requirements.txt deleted file mode 100644 index 256fd54c16..0000000000 --- a/perfmetrics/scripts/load_tests/python/requirements.txt +++ /dev/null @@ -1,582 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --generate-hashes ./requirements.in -# -absl-py==1.4.0 \ - --hash=sha256:0d3fe606adfa4f7db64792dd4c7aee4ee0c38ab75dfd353b7a83ed3e957fcb47 \ - --hash=sha256:d2c244d01048ba476e7c080bd2c6df5e141d211de80223460d5b3b8a2a58433d - # via - # keras - # tensorboard - # tensorflow -astunparse==1.6.3 \ - --hash=sha256:5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872 \ - --hash=sha256:c2652417f2c8b5bb325c885ae329bdf3f86424075c4fd1a128674bc6fba4b8e8 - # via tensorflow -certifi==2024.7.4 \ - --hash=sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b \ - --hash=sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90 - # via requests -charset-normalizer==3.1.0 \ - --hash=sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6 \ - --hash=sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1 \ - --hash=sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e \ - --hash=sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373 \ - --hash=sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62 \ - --hash=sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230 \ - --hash=sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be \ - --hash=sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c \ - --hash=sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0 \ - --hash=sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448 \ - --hash=sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f \ - --hash=sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649 \ - --hash=sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d \ - --hash=sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0 \ - --hash=sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706 \ - --hash=sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a \ - --hash=sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59 \ - --hash=sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23 \ - --hash=sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5 \ - --hash=sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb \ - --hash=sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e \ - --hash=sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e \ - --hash=sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c \ - --hash=sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28 \ - --hash=sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d \ - --hash=sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41 \ - --hash=sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974 \ - --hash=sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce \ - --hash=sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f \ - --hash=sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1 \ - --hash=sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d \ - --hash=sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8 \ - --hash=sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017 \ - --hash=sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31 \ - --hash=sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7 \ - --hash=sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8 \ - --hash=sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e \ - --hash=sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14 \ - --hash=sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd \ - --hash=sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d \ - --hash=sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795 \ - --hash=sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b \ - --hash=sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b \ - --hash=sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b \ - --hash=sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203 \ - --hash=sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f \ - --hash=sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19 \ - --hash=sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1 \ - --hash=sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a \ - --hash=sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac \ - --hash=sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9 \ - --hash=sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0 \ - --hash=sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137 \ - --hash=sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f \ - --hash=sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6 \ - --hash=sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5 \ - --hash=sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909 \ - --hash=sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f \ - --hash=sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0 \ - --hash=sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324 \ - --hash=sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755 \ - --hash=sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb \ - --hash=sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854 \ - --hash=sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c \ - --hash=sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60 \ - --hash=sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84 \ - --hash=sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0 \ - --hash=sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b \ - --hash=sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1 \ - --hash=sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531 \ - --hash=sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1 \ - --hash=sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11 \ - --hash=sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326 \ - --hash=sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df \ - --hash=sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab - # via requests -flatbuffers==24.3.25 \ - --hash=sha256:8dbdec58f935f3765e4f7f3cf635ac3a77f83568138d6a2311f524ec96364812 \ - --hash=sha256:de2ec5b203f21441716617f38443e0a8ebf3d25bf0d9c0bb0ce68fa00ad546a4 - # via tensorflow -gast==0.4.0 \ - --hash=sha256:40feb7b8b8434785585ab224d1568b857edb18297e5a3047f1ba012bc83b42c1 \ - --hash=sha256:b7adcdd5adbebf1adf17378da5ba3f543684dbec47b1cda1f3997e573cd542c4 - # via tensorflow -google-pasta==0.2.0 \ - --hash=sha256:4612951da876b1a10fe3960d7226f0c7682cf901e16ac06e473b267a5afa8954 \ - --hash=sha256:b32482794a366b5366a32c92a9a9201b107821889935a02b3e51f6b432ea84ed \ - --hash=sha256:c9f2c8dfc8f96d0d5808299920721be30c9eec37f2389f28904f454565c8a16e - # via tensorflow -grpcio==1.63.0 \ - --hash=sha256:01799e8649f9e94ba7db1aeb3452188048b0019dc37696b0f5ce212c87c560c3 \ - --hash=sha256:0697563d1d84d6985e40ec5ec596ff41b52abb3fd91ec240e8cb44a63b895094 \ - --hash=sha256:08e1559fd3b3b4468486b26b0af64a3904a8dbc78d8d936af9c1cf9636eb3e8b \ - --hash=sha256:166e5c460e5d7d4656ff9e63b13e1f6029b122104c1633d5f37eaea348d7356d \ - --hash=sha256:1ff737cf29b5b801619f10e59b581869e32f400159e8b12d7a97e7e3bdeee6a2 \ - --hash=sha256:219bb1848cd2c90348c79ed0a6b0ea51866bc7e72fa6e205e459fedab5770172 \ - --hash=sha256:259e11932230d70ef24a21b9fb5bb947eb4703f57865a404054400ee92f42f5d \ - --hash=sha256:2e93aca840c29d4ab5db93f94ed0a0ca899e241f2e8aec6334ab3575dc46125c \ - --hash=sha256:3a6d1f9ea965e750db7b4ee6f9fdef5fdf135abe8a249e75d84b0a3e0c668a1b \ - --hash=sha256:50344663068041b34a992c19c600236e7abb42d6ec32567916b87b4c8b8833b3 \ - --hash=sha256:56cdf96ff82e3cc90dbe8bac260352993f23e8e256e063c327b6cf9c88daf7a9 \ - --hash=sha256:5c039ef01516039fa39da8a8a43a95b64e288f79f42a17e6c2904a02a319b357 \ - --hash=sha256:6426e1fb92d006e47476d42b8f240c1d916a6d4423c5258ccc5b105e43438f61 \ - --hash=sha256:65bf975639a1f93bee63ca60d2e4951f1b543f498d581869922910a476ead2f5 \ - --hash=sha256:6a1a3642d76f887aa4009d92f71eb37809abceb3b7b5a1eec9c554a246f20e3a \ - --hash=sha256:6ef0ad92873672a2a3767cb827b64741c363ebaa27e7f21659e4e31f4d750280 \ - --hash=sha256:756fed02dacd24e8f488f295a913f250b56b98fb793f41d5b2de6c44fb762434 \ - --hash=sha256:75f701ff645858a2b16bc8c9fc68af215a8bb2d5a9b647448129de6e85d52bce \ - --hash=sha256:8064d986d3a64ba21e498b9a376cbc5d6ab2e8ab0e288d39f266f0fca169b90d \ - --hash=sha256:878b1d88d0137df60e6b09b74cdb73db123f9579232c8456f53e9abc4f62eb3c \ - --hash=sha256:8f3f6883ce54a7a5f47db43289a0a4c776487912de1a0e2cc83fdaec9685cc9f \ - --hash=sha256:91b73d3f1340fefa1e1716c8c1ec9930c676d6b10a3513ab6c26004cb02d8b3f \ - --hash=sha256:93a46794cc96c3a674cdfb59ef9ce84d46185fe9421baf2268ccb556f8f81f57 \ - --hash=sha256:93f45f27f516548e23e4ec3fbab21b060416007dbe768a111fc4611464cc773f \ - --hash=sha256:9e350cb096e5c67832e9b6e018cf8a0d2a53b2a958f6251615173165269a91b0 \ - --hash=sha256:a2d60cd1d58817bc5985fae6168d8b5655c4981d448d0f5b6194bbcc038090d2 \ - --hash=sha256:a3abfe0b0f6798dedd2e9e92e881d9acd0fdb62ae27dcbbfa7654a57e24060c0 \ - --hash=sha256:a44624aad77bf8ca198c55af811fd28f2b3eaf0a50ec5b57b06c034416ef2d0a \ - --hash=sha256:a7b19dfc74d0be7032ca1eda0ed545e582ee46cd65c162f9e9fc6b26ef827dc6 \ - --hash=sha256:ad2ac8903b2eae071055a927ef74121ed52d69468e91d9bcbd028bd0e554be6d \ - --hash=sha256:b005292369d9c1f80bf70c1db1c17c6c342da7576f1c689e8eee4fb0c256af85 \ - --hash=sha256:b2e44f59316716532a993ca2966636df6fbe7be4ab6f099de6815570ebe4383a \ - --hash=sha256:b3afbd9d6827fa6f475a4f91db55e441113f6d3eb9b7ebb8fb806e5bb6d6bd0d \ - --hash=sha256:b416252ac5588d9dfb8a30a191451adbf534e9ce5f56bb02cd193f12d8845b7f \ - --hash=sha256:b5194775fec7dc3dbd6a935102bb156cd2c35efe1685b0a46c67b927c74f0cfb \ - --hash=sha256:cacdef0348a08e475a721967f48206a2254a1b26ee7637638d9e081761a5ba86 \ - --hash=sha256:cd1e68776262dd44dedd7381b1a0ad09d9930ffb405f737d64f505eb7f77d6c7 \ - --hash=sha256:cdcda1156dcc41e042d1e899ba1f5c2e9f3cd7625b3d6ebfa619806a4c1aadda \ - --hash=sha256:cf8dae9cc0412cb86c8de5a8f3be395c5119a370f3ce2e69c8b7d46bb9872c8d \ - --hash=sha256:d2497769895bb03efe3187fb1888fc20e98a5f18b3d14b606167dacda5789434 \ - --hash=sha256:e3b77eaefc74d7eb861d3ffbdf91b50a1bb1639514ebe764c47773b833fa2d91 \ - --hash=sha256:e48cee31bc5f5a31fb2f3b573764bd563aaa5472342860edcc7039525b53e46a \ - --hash=sha256:e4cbb2100ee46d024c45920d16e888ee5d3cf47c66e316210bc236d5bebc42b3 \ - --hash=sha256:f28f8b2db7b86c77916829d64ab21ff49a9d8289ea1564a2b2a3a8ed9ffcccd3 \ - --hash=sha256:f3023e14805c61bc439fb40ca545ac3d5740ce66120a678a3c6c2c55b70343d1 \ - --hash=sha256:fdf348ae69c6ff484402cfdb14e18c1b0054ac2420079d575c53a60b9b2853ae - # via - # tensorboard - # tensorflow -h5py==3.11.0 \ - --hash=sha256:083e0329ae534a264940d6513f47f5ada617da536d8dccbafc3026aefc33c90e \ - --hash=sha256:1625fd24ad6cfc9c1ccd44a66dac2396e7ee74940776792772819fc69f3a3731 \ - --hash=sha256:21dbdc5343f53b2e25404673c4f00a3335aef25521bd5fa8c707ec3833934892 \ - --hash=sha256:52c416f8eb0daae39dabe71415cb531f95dce2d81e1f61a74537a50c63b28ab3 \ - --hash=sha256:55106b04e2c83dfb73dc8732e9abad69d83a436b5b82b773481d95d17b9685e1 \ - --hash=sha256:67462d0669f8f5459529de179f7771bd697389fcb3faab54d63bf788599a48ea \ - --hash=sha256:6c4b760082626120031d7902cd983d8c1f424cdba2809f1067511ef283629d4b \ - --hash=sha256:731839240c59ba219d4cb3bc5880d438248533366f102402cfa0621b71796b62 \ - --hash=sha256:754c0c2e373d13d6309f408325343b642eb0f40f1a6ad21779cfa9502209e150 \ - --hash=sha256:75bd7b3d93fbeee40860fd70cdc88df4464e06b70a5ad9ce1446f5f32eb84007 \ - --hash=sha256:77b19a40788e3e362b54af4dcf9e6fde59ca016db2c61360aa30b47c7b7cef00 \ - --hash=sha256:7b7e8f78072a2edec87c9836f25f34203fd492a4475709a18b417a33cfb21fa9 \ - --hash=sha256:8ec9df3dd2018904c4cc06331951e274f3f3fd091e6d6cc350aaa90fa9b42a76 \ - --hash=sha256:a76cae64080210389a571c7d13c94a1a6cf8cb75153044fd1f822a962c97aeab \ - --hash=sha256:aa6ae84a14103e8dc19266ef4c3e5d7c00b68f21d07f2966f0ca7bdb6c2761fb \ - --hash=sha256:bbd732a08187a9e2a6ecf9e8af713f1d68256ee0f7c8b652a32795670fb481ba \ - --hash=sha256:c072655ad1d5fe9ef462445d3e77a8166cbfa5e599045f8aa3c19b75315f10e5 \ - --hash=sha256:d9c944d364688f827dc889cf83f1fca311caf4fa50b19f009d1f2b525edd33a3 \ - --hash=sha256:ef4e2f338fc763f50a8113890f455e1a70acd42a4d083370ceb80c463d803972 \ - --hash=sha256:f3736fe21da2b7d8a13fe8fe415f1272d2a1ccdeff4849c1421d2fb30fd533bc \ - --hash=sha256:f4e025e852754ca833401777c25888acb96889ee2c27e7e629a19aee288833f0 - # via - # keras - # tensorflow -idna==3.7 \ - --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ - --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 - # via requests -keras==3.5.0 \ - --hash=sha256:53ae4f9472ec9d9c6941c82a3fda86969724ace3b7630a94ba0a1f17ba1065c3 \ - --hash=sha256:d37a3c623935713473ceb25241b52bce9d1e0ff5b36e5d0f6f47ed55f8500c9a - # via tensorflow -libclang==16.0.0 \ - --hash=sha256:2adce42ae652f312245b8f4eda6f30b4076fb61f7619f2dfd0a0c31dee4c32b9 \ - --hash=sha256:65258a6bb3e7dc31dc9b26f8d42f53c9d3b959643ade291fcd1aef4855303ca6 \ - --hash=sha256:7b6686b67a0daa84b4c614bcc119578329fc4fbb52b919565b7376b507c4793b \ - --hash=sha256:a043138caaf2cb076ebb060c6281ec95612926645d425c691991fc9df00e8a24 \ - --hash=sha256:af55a4aa86fdfe6b2ec68bc8cfe5fdac6c448d591ca7648be86ca17099b41ca8 \ - --hash=sha256:bf4628fc4da7a1dd06a244f9b8e121c5ec68076a763c59d6b13cbb103acc935b \ - --hash=sha256:eb59652cb0559c0e71784ff4c8ba24c14644becc907b1446563ecfaa622d523b \ - --hash=sha256:ee20bf93e3dd330f71fc50cdbf13b92ced0aec8e540be64251db53502a9b33f7 - # via tensorflow -markdown==3.4.3 \ - --hash=sha256:065fd4df22da73a625f14890dd77eb8040edcbd68794bcd35943be14490608b2 \ - --hash=sha256:8bf101198e004dc93e84a12a7395e31aac6a9c9942848ae1d99b9d72cf9b3520 - # via tensorboard -markdown-it-py==3.0.0 \ - --hash=sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1 \ - --hash=sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb - # via rich -markupsafe==2.1.2 \ - --hash=sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed \ - --hash=sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc \ - --hash=sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2 \ - --hash=sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460 \ - --hash=sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7 \ - --hash=sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0 \ - --hash=sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1 \ - --hash=sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa \ - --hash=sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03 \ - --hash=sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323 \ - --hash=sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65 \ - --hash=sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013 \ - --hash=sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036 \ - --hash=sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f \ - --hash=sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4 \ - --hash=sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419 \ - --hash=sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2 \ - --hash=sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619 \ - --hash=sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a \ - --hash=sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a \ - --hash=sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd \ - --hash=sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7 \ - --hash=sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666 \ - --hash=sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65 \ - --hash=sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859 \ - --hash=sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625 \ - --hash=sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff \ - --hash=sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156 \ - --hash=sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd \ - --hash=sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba \ - --hash=sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f \ - --hash=sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1 \ - --hash=sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094 \ - --hash=sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a \ - --hash=sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513 \ - --hash=sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed \ - --hash=sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d \ - --hash=sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3 \ - --hash=sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147 \ - --hash=sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c \ - --hash=sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603 \ - --hash=sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601 \ - --hash=sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a \ - --hash=sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1 \ - --hash=sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d \ - --hash=sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3 \ - --hash=sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54 \ - --hash=sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2 \ - --hash=sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6 \ - --hash=sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58 - # via werkzeug -mdurl==0.1.2 \ - --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ - --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba - # via markdown-it-py -ml-dtypes==0.4.0 \ - --hash=sha256:03e7cda6ef164eed0abb31df69d2c00c3a5ab3e2610b6d4c42183a43329c72a5 \ - --hash=sha256:2bb83fd064db43e67e67d021e547698af4c8d5c6190f2e9b1c53c09f6ff5531d \ - --hash=sha256:3b67ec73a697c88c1122038e0de46520e48dc2ec876d42cf61bc5efe3c0b7675 \ - --hash=sha256:41affb38fdfe146e3db226cf2953021184d6f0c4ffab52136613e9601706e368 \ - --hash=sha256:43cf4356a0fe2eeac6d289018d0734e17a403bdf1fd911953c125dd0358edcc0 \ - --hash=sha256:723af6346447268a3cf0b7356e963d80ecb5732b5279b2aa3fa4b9fc8297c85e \ - --hash=sha256:75b4faf99d0711b81f393db36d210b4255fd419f6f790bc6c1b461f95ffb7a9e \ - --hash=sha256:93afe37f3a879d652ec9ef1fc47612388890660a2657fbb5747256c3b818fd81 \ - --hash=sha256:a15d96d090aebb55ee85173d1775ae325a001aab607a76c8ea0b964ccd6b5364 \ - --hash=sha256:ad6849a2db386b38e4d54fe13eb3293464561780531a918f8ef4c8169170dd49 \ - --hash=sha256:bdf689be7351cc3c95110c910c1b864002f113e682e44508910c849e144f3df1 \ - --hash=sha256:c83e4d443962d891d51669ff241d5aaad10a8d3d37a81c5532a45419885d591c \ - --hash=sha256:e1e2f4237b459a63c97c2c9f449baa637d7e4c20addff6a9bac486f22432f3b6 \ - --hash=sha256:eaa32979ebfde3a0d7c947cafbf79edc1ec77ac05ad0780ee86c1d8df70f2259 \ - --hash=sha256:eaf197e72f4f7176a19fe3cb8b61846b38c6757607e7bf9cd4b1d84cd3e74deb \ - --hash=sha256:ee9f91d4c4f9959a7e1051c141dc565f39e54435618152219769e24f5e9a4d06 \ - --hash=sha256:f1724ddcdf5edbaf615a62110af47407f1719b8d02e68ccee60683acb5f74da1 - # via - # keras - # tensorflow -namex==0.0.8 \ - --hash=sha256:32a50f6c565c0bb10aa76298c959507abdc0e850efe085dc38f3440fcb3aa90b \ - --hash=sha256:7ddb6c2bb0e753a311b7590f84f6da659dd0c05e65cb89d519d54c0a250c0487 - # via keras -numpy==1.23.5 \ - --hash=sha256:01dd17cbb340bf0fc23981e52e1d18a9d4050792e8fb8363cecbf066a84b827d \ - --hash=sha256:06005a2ef6014e9956c09ba07654f9837d9e26696a0470e42beedadb78c11b07 \ - --hash=sha256:09b7847f7e83ca37c6e627682f145856de331049013853f344f37b0c9690e3df \ - --hash=sha256:0aaee12d8883552fadfc41e96b4c82ee7d794949e2a7c3b3a7201e968c7ecab9 \ - --hash=sha256:0cbe9848fad08baf71de1a39e12d1b6310f1d5b2d0ea4de051058e6e1076852d \ - --hash=sha256:1b1766d6f397c18153d40015ddfc79ddb715cabadc04d2d228d4e5a8bc4ded1a \ - --hash=sha256:33161613d2269025873025b33e879825ec7b1d831317e68f4f2f0f84ed14c719 \ - --hash=sha256:5039f55555e1eab31124a5768898c9e22c25a65c1e0037f4d7c495a45778c9f2 \ - --hash=sha256:522e26bbf6377e4d76403826ed689c295b0b238f46c28a7251ab94716da0b280 \ - --hash=sha256:56e454c7833e94ec9769fa0f86e6ff8e42ee38ce0ce1fa4cbb747ea7e06d56aa \ - --hash=sha256:58f545efd1108e647604a1b5aa809591ccd2540f468a880bedb97247e72db387 \ - --hash=sha256:5e05b1c973a9f858c74367553e236f287e749465f773328c8ef31abe18f691e1 \ - --hash=sha256:7903ba8ab592b82014713c491f6c5d3a1cde5b4a3bf116404e08f5b52f6daf43 \ - --hash=sha256:8969bfd28e85c81f3f94eb4a66bc2cf1dbdc5c18efc320af34bffc54d6b1e38f \ - --hash=sha256:92c8c1e89a1f5028a4c6d9e3ccbe311b6ba53694811269b992c0b224269e2398 \ - --hash=sha256:9c88793f78fca17da0145455f0d7826bcb9f37da4764af27ac945488116efe63 \ - --hash=sha256:a7ac231a08bb37f852849bbb387a20a57574a97cfc7b6cabb488a4fc8be176de \ - --hash=sha256:abdde9f795cf292fb9651ed48185503a2ff29be87770c3b8e2a14b0cd7aa16f8 \ - --hash=sha256:af1da88f6bc3d2338ebbf0e22fe487821ea4d8e89053e25fa59d1d79786e7481 \ - --hash=sha256:b2a9ab7c279c91974f756c84c365a669a887efa287365a8e2c418f8b3ba73fb0 \ - --hash=sha256:bf837dc63ba5c06dc8797c398db1e223a466c7ece27a1f7b5232ba3466aafe3d \ - --hash=sha256:ca51fcfcc5f9354c45f400059e88bc09215fb71a48d3768fb80e357f3b457e1e \ - --hash=sha256:ce571367b6dfe60af04e04a1834ca2dc5f46004ac1cc756fb95319f64c095a96 \ - --hash=sha256:d208a0f8729f3fb790ed18a003f3a57895b989b40ea4dce4717e9cf4af62c6bb \ - --hash=sha256:dbee87b469018961d1ad79b1a5d50c0ae850000b639bcb1b694e9981083243b6 \ - --hash=sha256:e9f4c4e51567b616be64e05d517c79a8a22f3606499941d97bb76f2ca59f982d \ - --hash=sha256:f063b69b090c9d918f9df0a12116029e274daf0181df392839661c4c7ec9018a \ - --hash=sha256:f9a909a8bae284d46bbfdefbdd4a262ba19d3bc9921b1e76126b1d21c3c34135 - # via - # h5py - # keras - # ml-dtypes - # opt-einsum - # tensorboard - # tensorflow -opt-einsum==3.3.0 \ - --hash=sha256:2455e59e3947d3c275477df7f5205b30635e266fe6dc300e3d9f9646bfcea147 \ - --hash=sha256:59f6475f77bbc37dcf7cd748519c0ec60722e91e63ca114e68821c0c54a46549 - # via tensorflow -optree==0.12.1 \ - --hash=sha256:06d6ef39b3ef9920d6cdb6d3d1d2804a37092d24dc406c4cb9b46cd6c9a44e89 \ - --hash=sha256:154738def491199d3fbcd919437315728e0a1caeaf4ec06688c76ef9d56e5ed6 \ - --hash=sha256:1b2fe5c04c218698a53ed2d4b7372f1989df8cf0a61d616e6f384770d8a5fb1c \ - --hash=sha256:1d76905bced5cf569d23dc4890341fae2fa257cce58a492a1603afcdc5969ae7 \ - --hash=sha256:1f8baf0ad6b58843d24fa8caf079cf1f0c33cc3658263cff960b5c1d0cc53bc8 \ - --hash=sha256:23afe4aae42336bdf8cf4fba35c56593405bf8f8e163627f722205b3bf0d9310 \ - --hash=sha256:24d74a9d97d7bdbdbb30356850f204950c39ab8fad7f273ed29d1feda19060b2 \ - --hash=sha256:27ae426745931ae1c2ccd7a78b27f9b7402167e0600fa62e2ef1cd58727e7b94 \ - --hash=sha256:2a1a9905d2d917d5aff775283e0a59be2c6b529a219241c248d50b3ad51c6cce \ - --hash=sha256:2d4d8e024b841f99907b2340fee7ac9994fbe300383a9af6c93578d12861a969 \ - --hash=sha256:2de1297b2bf019379ab86103e31caa97c8a08628f0c8b58cd7709f9048c589eb \ - --hash=sha256:349aafac463642979f7fe7ca3aa9e2fa8a5a0f81ef7af6946a075b797673e600 \ - --hash=sha256:35ca77b810cf5959e6930d56534ecbecc4300f5e5fa14b977030265c1c8eab6c \ - --hash=sha256:3e323744d083bd8b4648c9ff2383f01bfbc33098656d56fdd984b2263ef905f3 \ - --hash=sha256:404cf2decd8fb6a1a8f6fef623c98873cdf7ae086aeb8909d104cd321d829ba0 \ - --hash=sha256:409ef6f3656299923d722509849d83607bb3e5c621dcfe6aa90ace85665e9b54 \ - --hash=sha256:411a21eca034ddb98eb80e6c4bf552fc46b8d8ab7c4d250446d74d31a251a684 \ - --hash=sha256:42025da0bac19cc6de756fe64511f15baffb3fa7d8402e54aab035c02903eb5c \ - --hash=sha256:47db001a224382493ae7a8df16e7a9668e971fc129970d137995421aa6b06f8f \ - --hash=sha256:4b32f39988bfe6e76eeefb335da529e614145f7f1dfa8583fbc4aca8a72f504b \ - --hash=sha256:4ee926120887404e92877c99714b960bc29f572e8db69fd2e934022d80452f91 \ - --hash=sha256:50893bd088bdb3e2f07ee481dafd848b483bea1a19cc978f2309139314e5bc7d \ - --hash=sha256:509bddd38dae8c4e8d6b988f514b7a9fe803ca916b11af67b40520f0b1eeeaef \ - --hash=sha256:562036d3de15204ed1a88d9fc262a7e1c20964d22ef132069e20dbd88215f983 \ - --hash=sha256:5bfe3d3e47e10b528f9324d446c871bfad7d0be8c2bd2a2fbc3ddf1600ae8558 \ - --hash=sha256:5c2f2e0e3978558bc8f7df8c5a999674097dd0dc71363210783eb8d7a6da8ef9 \ - --hash=sha256:5f24b0a8b181a90a778cadc942a79336d29f0c164704d58cd20989bf7d0bea1c \ - --hash=sha256:606983f4696d81128e205a1c34d0c9f3fe6ae12f6c26ed5e8ab3722d6f378ec2 \ - --hash=sha256:62d232a344c14b8e94fdd6de1acf2c0b05954b05d6bb346bddb13c38be37dc09 \ - --hash=sha256:646842f8a2de2caaacc32a8c91f8031a93eda145ac9c915bb0fd2ad5249c14b7 \ - --hash=sha256:6d90fb28d52725352858013cafe34d98d90ab1bb86b5d8dc29d420e9bbc5706b \ - --hash=sha256:76a2240e7482355966a73c6c701e3d1f148420a77849c78d175d3b08bf06ff36 \ - --hash=sha256:7a71dd58522cd6258b61b639092ac7a2631d881f039ef968b31dfd555e513591 \ - --hash=sha256:7e79eedd9406c59d542482768e490795dc6b6f1a014c7852d29d9fd61749bf94 \ - --hash=sha256:80e0d4eba4a65d4c6f2002ed949142a40933b8185523894659c26c34693c4086 \ - --hash=sha256:8513d6dd71807abb1037a5b5bc66b45c21afb42e9c90961fa5e762cea3943ab2 \ - --hash=sha256:88d01ce6f78f209cad8dc4cf2d3222d7056cac93612abfd6beb40ab43a131769 \ - --hash=sha256:9280452c11da0872ec57be5d8f153207d6303b3cbf26115b2bf6d2b8157a5343 \ - --hash=sha256:9c473988b2d8fd7edc3958e6c7cb1d3f92afb7bcaff53b76a8f41cf4f3a24709 \ - --hash=sha256:a11e58d7c0a71a48d74ca0a6715f4c0932c6f9409ba93d600e3326df4cf778ae \ - --hash=sha256:a49d3cfec1a51463b63e11c889bb00207c4e040016833cd202871ad946116925 \ - --hash=sha256:a55a79c1c72f73259532e4cbe9ff65bed9663064747db02591fb4714fe742d2e \ - --hash=sha256:a67842cd1c5c83d74863872f06fe6ed64e44279c0378267a9805567fe3c38591 \ - --hash=sha256:aadb26d68f1d7871507f84846d8844aa94f47402d5277ce19cfe5102bb5df9e9 \ - --hash=sha256:afa0051335c6032ee4dfc212952dcfb3b23fe59bcd70f56d25a214e7585cd62c \ - --hash=sha256:b1ca00bdfe4da8068c2773b7ac4c8c96d3f61b8d21eba6a8642dab23ee631b0d \ - --hash=sha256:b43c09cf9dd28aed2efc163f4bb4808d7fad54250812960bf349399ba6972e16 \ - --hash=sha256:b890ba0a21049addf589c314c85e98a68d3dfc84e3954491e9ce60f60cb7b0e7 \ - --hash=sha256:ba6aed8b9684c5804a5e2d6b246c3b4a68bab793b6829d369ba1c53734852a0c \ - --hash=sha256:bd207b43e71fb3f8c315e2e4a5444f48317b2108889e96279d5426bca730a47e \ - --hash=sha256:c987931bd31d0f28cbff28925a875639170534a36ce178a40020aca0769d9549 \ - --hash=sha256:ce7cb233e87a2dc127b8ec82bd61f098e6ff1e57d0a09dc110a17b38bfd73034 \ - --hash=sha256:cefd4f4c7596cdd4c95dca431bc41284a43ebd7056e739480f157789aa34579d \ - --hash=sha256:d0950ee245db2c40824362def1efc15621a6492419628cec1fac0061818420f7 \ - --hash=sha256:d313303a1ce36ea55c3a96fc375c5cc64a9ab814ab2677ce64e4a7d755a9b1d0 \ - --hash=sha256:d913122454d0e3f10dc25a1b598eaf588d225372f41ece3ad4d508bddd363e4d \ - --hash=sha256:da37e6cc669a9840844722edb3f8dd5b4f07e99b0e8c9196089cb49af70c7b75 \ - --hash=sha256:e124f30daf79d51b1bbbda7e74d01e637fa47aff4aa64cb082b88057535daa64 \ - --hash=sha256:e2027217c3acaf44e5f5aabe01ba0cbf33066f3f6df870881ddf597965f80db0 \ - --hash=sha256:e20b5569369a5f1e8faa2604799b91a1941fe17b5de8afc84c8c23ff66d8e585 \ - --hash=sha256:e8046cbbcd5f7494ba7c6811e44a6d2867216f2bdb7cef980a9a62e31d39270c \ - --hash=sha256:eb968d3cc1db8944f220f1a67c9db043b86b47ace90ce3cfd23f3e6500baeb65 \ - --hash=sha256:efffa3814ab8e3aaf7bf88495e4b6d263de9689d6f02dfa4490f8f64736806ac \ - --hash=sha256:f0460f025bf1c08f2c008b5e3628d849fcb5810345222e57879cd248fec7f9f7 \ - --hash=sha256:f65a31d7cfab2fed2bc29ab6eabcf4205dec6e0ee3cfb7006336c4f76d78fb0e \ - --hash=sha256:f6b98b80b1259e9817aca701beba616ce33e43e856e7d644f7e0f582b8e45565 \ - --hash=sha256:fc1ec38d1ec43bb8358ab058c3220a70b7bfb56f2bb625f41cb09d117a0d6150 \ - --hash=sha256:fd3ead0c64d22d692284d96c27d5091e682b002ffe5a52afacc9f1fcc8ae3180 - # via keras -packaging==23.1 \ - --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \ - --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f - # via - # keras - # tensorboard - # tensorflow -protobuf==4.23.0 \ - --hash=sha256:03eee35b60317112a72d19c54d0bff7bc58ff12fea4cd7b018232bd99758ffdf \ - --hash=sha256:2b94bd6df92d71bd1234a2ffe7ce96ddf6d10cf637a18d6b55ad0a89fbb7fc21 \ - --hash=sha256:36f5370a930cb77c8ad2f4135590c672d0d2c72d4a707c7d0058dce4b4b4a598 \ - --hash=sha256:5f1eba1da2a2f3f7df469fccddef3cc060b8a16cfe3cc65961ad36b4dbcf59c5 \ - --hash=sha256:6c16657d6717a0c62d5d740cb354fbad1b0d8cb811669e06fc1caa0ff4799ddd \ - --hash=sha256:6fe180b56e1169d72ecc4acbd39186339aed20af5384531b8e8979b02bbee159 \ - --hash=sha256:7cb5b9a05ce52c6a782bb97de52679bd3438ff2b7460eff5da348db65650f227 \ - --hash=sha256:9744e934ea5855d12191040ea198eaf704ac78665d365a89d9572e3b627c2688 \ - --hash=sha256:9f5a0fbfcdcc364f3986f9ed9f8bb1328fb84114fd790423ff3d7fdb0f85c2d1 \ - --hash=sha256:baca40d067dddd62141a129f244703160d278648b569e90bb0e3753067644711 \ - --hash=sha256:d5a35ff54e3f62e8fc7be02bb0d2fbc212bba1a5a9cc2748090690093996f07b \ - --hash=sha256:e62fb869762b4ba18666370e2f8a18f17f8ab92dd4467295c6d38be6f8fef60b \ - --hash=sha256:ebde3a023b8e11bfa6c890ef34cd6a8b47d586f26135e86c21344fe433daf2e2 - # via - # tensorboard - # tensorflow -pygments==2.18.0 \ - --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \ - --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a - # via rich -requests==2.31.0 \ - --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ - --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 - # via tensorflow -rich==13.8.0 \ - --hash=sha256:2e85306a063b9492dffc86278197a60cbece75bcb766022f3436f567cae11bdc \ - --hash=sha256:a5ac1f1cd448ade0d59cc3356f7db7a7ccda2c8cbae9c7a90c28ff463d3e91f4 - # via keras -six==1.16.0 \ - --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ - --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 - # via - # astunparse - # google-pasta - # tensorboard - # tensorflow -tensorboard==2.17.1 \ - --hash=sha256:253701a224000eeca01eee6f7e978aea7b408f60b91eb0babdb04e78947b773e - # via tensorflow -tensorboard-data-server==0.7.0 \ - --hash=sha256:64aa1be7c23e80b1a42c13b686eb0875bb70f5e755f4d2b8de5c1d880cf2267f \ - --hash=sha256:753d4214799b31da7b6d93837959abebbc6afa86e69eacf1e9a317a48daa31eb \ - --hash=sha256:eb7fa518737944dbf4f0cf83c2e40a7ac346bf91be2e6a0215de98be74e85454 - # via tensorboard -tensorflow==2.17.0 \ - --hash=sha256:0ad7bfea6afb4ded3928ca5b24df9fda876cea4904c103a5163fcc0c3483e7a4 \ - --hash=sha256:147c93ded4cb7e500a65d3c26d74744ff41660db7a8afe2b00d1d08bf329b4ec \ - --hash=sha256:278bc80642d799adf08dc4e04f291aab603bba7457d50c1f9bc191ebbca83f43 \ - --hash=sha256:4ae8e6746deb2ec807b902ba26d62fcffb6a6b53555a1a5906ec00416c5e4175 \ - --hash=sha256:515fe5ae8a9bc50312575412b08515f3ca66514c155078e0707bdffbea75d783 \ - --hash=sha256:72adfef0ee39dd641627906fd7b244fcf21bdd8a87216a998ed74d9c74653aff \ - --hash=sha256:8339777b1b5ebd8ffadaa8196f786e65fbb081a371d8e87b52f24563392d8552 \ - --hash=sha256:8f80d11ad3766570deb6ff47d2bed2d166f51399ca08205e38ef024345571d6f \ - --hash=sha256:97f89e95d68b4b46e1072243b9f315c3b340e27cc07b1e1988e2ca97ad844305 \ - --hash=sha256:b36683ac28af20abc3a548c72bf4537b00df1b1f3dd39d59df3873fefaf26f15 \ - --hash=sha256:ca82f98ea38fa6c9e08ccc69eb6c2fab5b35b30a8999115b8b63b6f02fc69d9d \ - --hash=sha256:dde37cff74ed22b8fa2eea944805b001ae38e96adc989666422bdea34f4e2d47 \ - --hash=sha256:e46090587f69e33637d17d7c3d94a790cac7d4bc5ff5ecbf3e71fdc6982fe96e \ - --hash=sha256:e8d26d6c24ccfb139db1306599257ca8f5cfe254ef2d023bfb667f374a17a64d \ - --hash=sha256:ee18b4fcd627c5e872eabb25092af6c808b6ec77948662c88fc5c89a60eb0211 \ - --hash=sha256:ef615c133cf4d592a073feda634ccbeb521a554be57de74f8c318d38febbeab5 - # via -r requirements.in -tensorflow-io-gcs-filesystem==0.32.0 \ - --hash=sha256:045d51bba586390d0545fcd8a18727d62b175eb142f6f4c6d719d39de40774cd \ - --hash=sha256:05e65d3cb6c93a7929b384d86c6369c63cbbab8a770440a3d95e094878403f9f \ - --hash=sha256:122be149e5f6a030f5c2901be0cc3cb07619232f7b03889e2cdf3da1c0d4f92f \ - --hash=sha256:1ce80e1555d6ee88dda67feddf366cc8b30252b5837a7a17303df7b06a71fc2e \ - --hash=sha256:21de7dcc06eb1e7de3c022b0072d90ba35ef886578149663437aa7a6fb5bf6b3 \ - --hash=sha256:28202492d904a6e280cf27560791e87ac1c7566000db82065d63a70c27008af2 \ - --hash=sha256:336d9b3fe6b55aea149c4f6aa1fd6ffaf27d4e5c37e55a182340b47caba38846 \ - --hash=sha256:5635df0bbe40f971dc1b946e3372744b0bdfda45c38ffcd28ef53a32bb8da4da \ - --hash=sha256:74a7e25e83d4117a7ebb09a3f247553a5497393ab48c3ee0cf0d17b405026817 \ - --hash=sha256:79fdd02103b8ae9f8b89af41f744c013fa1caaea709de19833917795e3063857 \ - --hash=sha256:7f15fd22e592661b10de317be2f42a0f84be7bfc5e6a565fcfcb04b60d625b78 \ - --hash=sha256:8214cdf85bea694160f9035ff395221c1e25e119784ccb4c104919b1f5dec84e \ - --hash=sha256:842f5f09cd756bdb3b4d0b5571b3a6f72fd534d42da938b9acf0ef462995eada \ - --hash=sha256:db682e9a510c27dd35710ba5a2c62c371e25b727741b2fe3a920355fa501e947 - # via tensorflow -termcolor==2.3.0 \ - --hash=sha256:3afb05607b89aed0ffe25202399ee0867ad4d3cb4180d98aaf8eefa6a5f7d475 \ - --hash=sha256:b5b08f68937f138fe92f6c089b99f1e2da0ae56c52b78bf7075fd95420fd9a5a - # via tensorflow -typing-extensions==4.5.0 \ - --hash=sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb \ - --hash=sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4 - # via - # optree - # tensorflow -urllib3==1.26.18 \ - --hash=sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07 \ - --hash=sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0 - # via requests -werkzeug==3.0.6 \ - --hash=sha256:1bc0c2310d2fbb07b1dd1105eba2f7af72f322e1e455f2f93c993bee8c8a5f17 \ - --hash=sha256:a8dd59d4de28ca70471a34cba79bed5f7ef2e036a76b3ab0835474246eb41f8d - # via tensorboard -wheel==0.40.0 \ - --hash=sha256:cd1196f3faee2b31968d626e1731c94f99cbdb67cf5a46e4f5656cbee7738873 \ - --hash=sha256:d236b20e7cb522daf2390fa84c55eea81c5c30190f90f29ae2ca1ad8355bf247 - # via astunparse -wrapt==1.14.1 \ - --hash=sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3 \ - --hash=sha256:01c205616a89d09827986bc4e859bcabd64f5a0662a7fe95e0d359424e0e071b \ - --hash=sha256:02b41b633c6261feff8ddd8d11c711df6842aba629fdd3da10249a53211a72c4 \ - --hash=sha256:07f7a7d0f388028b2df1d916e94bbb40624c59b48ecc6cbc232546706fac74c2 \ - --hash=sha256:11871514607b15cfeb87c547a49bca19fde402f32e2b1c24a632506c0a756656 \ - --hash=sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3 \ - --hash=sha256:21ac0156c4b089b330b7666db40feee30a5d52634cc4560e1905d6529a3897ff \ - --hash=sha256:257fd78c513e0fb5cdbe058c27a0624c9884e735bbd131935fd49e9fe719d310 \ - --hash=sha256:2b39d38039a1fdad98c87279b48bc5dce2c0ca0d73483b12cb72aa9609278e8a \ - --hash=sha256:2cf71233a0ed05ccdabe209c606fe0bac7379fdcf687f39b944420d2a09fdb57 \ - --hash=sha256:2fe803deacd09a233e4762a1adcea5db5d31e6be577a43352936179d14d90069 \ - --hash=sha256:3232822c7d98d23895ccc443bbdf57c7412c5a65996c30442ebe6ed3df335383 \ - --hash=sha256:34aa51c45f28ba7f12accd624225e2b1e5a3a45206aa191f6f9aac931d9d56fe \ - --hash=sha256:36f582d0c6bc99d5f39cd3ac2a9062e57f3cf606ade29a0a0d6b323462f4dd87 \ - --hash=sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d \ - --hash=sha256:40e7bc81c9e2b2734ea4bc1aceb8a8f0ceaac7c5299bc5d69e37c44d9081d43b \ - --hash=sha256:43ca3bbbe97af00f49efb06e352eae40434ca9d915906f77def219b88e85d907 \ - --hash=sha256:4fcc4649dc762cddacd193e6b55bc02edca674067f5f98166d7713b193932b7f \ - --hash=sha256:5a0f54ce2c092aaf439813735584b9537cad479575a09892b8352fea5e988dc0 \ - --hash=sha256:5a9a0d155deafd9448baff28c08e150d9b24ff010e899311ddd63c45c2445e28 \ - --hash=sha256:5b02d65b9ccf0ef6c34cba6cf5bf2aab1bb2f49c6090bafeecc9cd81ad4ea1c1 \ - --hash=sha256:60db23fa423575eeb65ea430cee741acb7c26a1365d103f7b0f6ec412b893853 \ - --hash=sha256:642c2e7a804fcf18c222e1060df25fc210b9c58db7c91416fb055897fc27e8cc \ - --hash=sha256:6a9a25751acb379b466ff6be78a315e2b439d4c94c1e99cb7266d40a537995d3 \ - --hash=sha256:6b1a564e6cb69922c7fe3a678b9f9a3c54e72b469875aa8018f18b4d1dd1adf3 \ - --hash=sha256:6d323e1554b3d22cfc03cd3243b5bb815a51f5249fdcbb86fda4bf62bab9e164 \ - --hash=sha256:6e743de5e9c3d1b7185870f480587b75b1cb604832e380d64f9504a0535912d1 \ - --hash=sha256:709fe01086a55cf79d20f741f39325018f4df051ef39fe921b1ebe780a66184c \ - --hash=sha256:7b7c050ae976e286906dd3f26009e117eb000fb2cf3533398c5ad9ccc86867b1 \ - --hash=sha256:7d2872609603cb35ca513d7404a94d6d608fc13211563571117046c9d2bcc3d7 \ - --hash=sha256:7ef58fb89674095bfc57c4069e95d7a31cfdc0939e2a579882ac7d55aadfd2a1 \ - --hash=sha256:80bb5c256f1415f747011dc3604b59bc1f91c6e7150bd7db03b19170ee06b320 \ - --hash=sha256:81b19725065dcb43df02b37e03278c011a09e49757287dca60c5aecdd5a0b8ed \ - --hash=sha256:833b58d5d0b7e5b9832869f039203389ac7cbf01765639c7309fd50ef619e0b1 \ - --hash=sha256:88bd7b6bd70a5b6803c1abf6bca012f7ed963e58c68d76ee20b9d751c74a3248 \ - --hash=sha256:8ad85f7f4e20964db4daadcab70b47ab05c7c1cf2a7c1e51087bfaa83831854c \ - --hash=sha256:8c0ce1e99116d5ab21355d8ebe53d9460366704ea38ae4d9f6933188f327b456 \ - --hash=sha256:8d649d616e5c6a678b26d15ece345354f7c2286acd6db868e65fcc5ff7c24a77 \ - --hash=sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef \ - --hash=sha256:9736af4641846491aedb3c3f56b9bc5568d92b0692303b5a305301a95dfd38b1 \ - --hash=sha256:988635d122aaf2bdcef9e795435662bcd65b02f4f4c1ae37fbee7401c440b3a7 \ - --hash=sha256:9cca3c2cdadb362116235fdbd411735de4328c61425b0aa9f872fd76d02c4e86 \ - --hash=sha256:9e0fd32e0148dd5dea6af5fee42beb949098564cc23211a88d799e434255a1f4 \ - --hash=sha256:9f3e6f9e05148ff90002b884fbc2a86bd303ae847e472f44ecc06c2cd2fcdb2d \ - --hash=sha256:a85d2b46be66a71bedde836d9e41859879cc54a2a04fad1191eb50c2066f6e9d \ - --hash=sha256:a9a52172be0b5aae932bef82a79ec0a0ce87288c7d132946d645eba03f0ad8a8 \ - --hash=sha256:aa31fdcc33fef9eb2552cbcbfee7773d5a6792c137b359e82879c101e98584c5 \ - --hash=sha256:b014c23646a467558be7da3d6b9fa409b2c567d2110599b7cf9a0c5992b3b471 \ - --hash=sha256:b21bb4c09ffabfa0e85e3a6b623e19b80e7acd709b9f91452b8297ace2a8ab00 \ - --hash=sha256:b5901a312f4d14c59918c221323068fad0540e34324925c8475263841dbdfe68 \ - --hash=sha256:b9b7a708dd92306328117d8c4b62e2194d00c365f18eff11a9b53c6f923b01e3 \ - --hash=sha256:d1967f46ea8f2db647c786e78d8cc7e4313dbd1b0aca360592d8027b8508e24d \ - --hash=sha256:d52a25136894c63de15a35bc0bdc5adb4b0e173b9c0d07a2be9d3ca64a332735 \ - --hash=sha256:d77c85fedff92cf788face9bfa3ebaa364448ebb1d765302e9af11bf449ca36d \ - --hash=sha256:d79d7d5dc8a32b7093e81e97dad755127ff77bcc899e845f41bf71747af0c569 \ - --hash=sha256:dbcda74c67263139358f4d188ae5faae95c30929281bc6866d00573783c422b7 \ - --hash=sha256:ddaea91abf8b0d13443f6dac52e89051a5063c7d014710dcb4d4abb2ff811a59 \ - --hash=sha256:dee0ce50c6a2dd9056c20db781e9c1cfd33e77d2d569f5d1d9321c641bb903d5 \ - --hash=sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb \ - --hash=sha256:e2f83e18fe2f4c9e7db597e988f72712c0c3676d337d8b101f6758107c42425b \ - --hash=sha256:e3fb1677c720409d5f671e39bac6c9e0e422584e5f518bfd50aa4cbbea02433f \ - --hash=sha256:ee2b1b1769f6707a8a445162ea16dddf74285c3964f605877a20e38545c3c462 \ - --hash=sha256:ee6acae74a2b91865910eef5e7de37dc6895ad96fa23603d1d27ea69df545015 \ - --hash=sha256:ef3f72c9666bba2bab70d2a8b79f2c6d2c1a42a7f7e2b0ec83bb2f9e383950af - # via tensorflow - -# WARNING: The following packages were not pinned, but pip requires them to be -# pinned when the requirements file includes hashes and the requirement is not -# satisfied by a package already installed. Consider using the --allow-unsafe flag. -# setuptools diff --git a/perfmetrics/scripts/load_tests/python/sample_tasks.yaml b/perfmetrics/scripts/load_tests/python/sample_tasks.yaml deleted file mode 100644 index 6d34775fc1..0000000000 --- a/perfmetrics/scripts/load_tests/python/sample_tasks.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# Sample YAML file containing example configs for recognised and predefined -# tasks. - ---- -# Sample task for reading file using Python OS native open api. -256kb_os: # [Required] Name of the task (task_name). - # Task type. Fixed for reading file using python os native open api. - task_type: python_os_read # [Required] - # Local file path. Can only contain {process_id} and {thread_id} in format. - file_path_format: ./gcs/256kb/read.{process_id} # [Required] - # K for 1024, M for 1024 * K, G for 1024 * M. - file_size: 256K # [Required] - block_size: 16K # [Optional] [Default = file_size] - -# Sample task for reading file using tensorflow's tf.io.gfile.GFile api. -1mb_tf_gfile: # [Required] Name of the task (task_name). - # Task type. Fixed for reading file using tensorflow's tf.io.gfile.GFile api. - task_type: tf_gfile_read # [Required] - # Local file path/GCS path (gs://). Can only contain {process_id} and - # {thread_id} in format. - file_path_format: gs://load-test-bucket/1mb/read.{process_id} # [Required] - # K for 1024, M for 1024 * K, G for 1024 * M. - file_size: 1M # [Required] - block_size: 16K # [Optional] [Default = file_size] - -# Sample task for reading file using tensorflow's tf.data api. -# For parallelism in tf.data, tweak num_parallel_calls in task file and always -# pass 1 to --num-processes & --num-threads -100mb_tf_data: # [Required] Name of the task (task_name). - # Task type. Fixed for reading file using tensorflow's tf.data api. - task_type: tf_data_read # [Required] - # Local file path/GCS path (gs://). Can only contain {file_num} in path. - file_path_format: ./gcs/100mb/read.{file_num}.tfrecord # [Required] - # K for 1024, M for 1024 * K, G for 1024 * M. - file_size: 100M # [Required] - # Number of files to read in one task. - num_files: 3 # [Required] - # Prefetch value in tf.data call. - prefetch: 50 # [Optional] [Default = -1 (AUTOTUNE)] - # Parallelism in tf.data calls. - num_parallel_calls: 100 # [Optional] [Default = -1 (AUTOTUNE)] - # Shard value in tf.data call - shard: 100 # [Optional] [Default = 1] - diff --git a/perfmetrics/scripts/load_tests/python/tasks/__init__.py b/perfmetrics/scripts/load_tests/python/tasks/__init__.py deleted file mode 100644 index 1dc90d1848..0000000000 --- a/perfmetrics/scripts/load_tests/python/tasks/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/perfmetrics/scripts/load_tests/python/tasks/python_os.py b/perfmetrics/scripts/load_tests/python/tasks/python_os.py deleted file mode 100644 index 1e2cba273d..0000000000 --- a/perfmetrics/scripts/load_tests/python/tasks/python_os.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains task which reads data from files using python's native open api. - -Example: - - lg_obj = load_generator.LoadGenerator(...) - task_obj = python_os.OSRead256KB(...) - observations = lg_obj.generate_load(task_obj) -""" -import os -import logging -from load_generator import task - -PYTHON_OS_READ = 'python_os_read' - - -class OSRead(task.LoadTestTask): - """Task class for reading file from disk using python's native open api. - - The task reads file with os.O_DIRECT i.e. bypassing page cache. - - Args: - task_name: String name assigned to task. - file_path_format: String format of file path. The format can contain - 'process_id' and 'thread_id' to be filled. E.g. gcs/256kb/read.{process_id} - file_size: Integer size of file in bytes to be read. - block_size: Integer size of block in bytes. File is read block by block. - If block size is not passed or -1, then by default it takes file size as - block size. - """ - - def __init__(self, task_name, file_path_format, file_size, block_size=-1): - super().__init__() - self.task_type = PYTHON_OS_READ - self.task_name = task_name - self.file_path_format = file_path_format - self.file_size = file_size - # Keep file size as default block size - self.block_size = block_size - if self.block_size == -1: - self.block_size = file_size - - def task(self, process_id, thread_id): - """Reads file of given size from given file path and with given block size. - - See base class for more details. - - Returns: - Integer denoting the size of content read in bytes. - """ - file_path = self.file_path_format.format( - process_id=process_id, thread_id=thread_id) - my_file = os.open(file_path, os.O_DIRECT) - content_len = 0 - with open(my_file, 'rb') as f_p: - for _ in range(0, self.file_size, self.block_size): - content = f_p.read(self.block_size) - content_len = content_len + len(content) - f_p.close() - return content_len - - def create_files(self, num_processes): - """Creates num_processes number of files to be used in load testing. - - Args: - num_processes: Integer denoting number of processes in load test. - - Returns: - None - - Raises: - RuntimeError: If the path under which the files to be created doesn't - exist. - """ - # Create one file per process for read and write tasks. - if not os.path.exists(os.path.dirname(self.file_path_format)): - raise RuntimeError('Directory containing files for task not exists.') - - logging.info( - 'One file is created per process of size %s using the format ' - '%s', self.file_size, self.file_path_format) - for process_id in range(num_processes): - file_path = self.file_path_format.format(process_id=process_id) - self._create_binary_file(file_path, self.file_size) - - def _create_binary_file(self, file_path, file_size): - """Creates binary file of given file size in bytes at given path. - - Doesn't create a file if file is already present and has same file size. - """ - if os.path.exists(file_path) and os.path.getsize(file_path) == file_size: - return - logging.info('Creating file %s of size %s.', file_path, file_size) - with open(file_path, 'wb') as f_p: - f_p.truncate(file_size) diff --git a/perfmetrics/scripts/load_tests/python/tasks/tf_data.py b/perfmetrics/scripts/load_tests/python/tasks/tf_data.py deleted file mode 100644 index 471c01e0f2..0000000000 --- a/perfmetrics/scripts/load_tests/python/tasks/tf_data.py +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains tasks which reads data from files using tensorflow's tf.data api. - -Example: - - lg_obj = load_generator.LoadGenerator(...) - task_obj = tf_data.TFDataRead(...) - observations = lg_obj.generate_load(task_obj) -""" -import os -import logging -import tensorflow as tf -from load_generator import task - -# .tfrecord file contains multiple TFRecords in it. This defines the size of -# that single TFRecord in bytes. -SINGLE_RECORD_SIZE = 256 * 1024 -TF_DATA_READ = 'tf_data_read' - - -class TFDataRead(task.LoadTestTask): - """Task class for reading tfrecord file using tensorflow's tf.data api. - - tf.data: https://www.tensorflow.org/guide/data - tf.data.TFRecordDataset.shard: - https://www.tensorflow.org/api_docs/python/tf/data/Dataset#shard - Note: The same class can be used with tf's internal GCS client if format of - file path starts with gs:// and with GCSFuse if local GCSFuse mounted path - is passed. - - For details on prefetch, num_parallel_calls & shard arguments, please refer - to tf.data public documentation. - - Args: - task_name: String name assigned to task. - file_path_format: String format of file path. Must contain 'file_num' to be - formatted. E.g. gcs/256kb/read.{file_num}. - file_size: Integer size of file in bytes to be read. - num_files: Integer number of files to be read in one task. - prefetch: Integer value of prefetch. Default to AUTOTUNE(-1). - num_parallel_calls = Integer value of num_parallel_calls. Default to - AUTOTUNE(-1). - shard: Integer value of shard. Default to AUTOTUNE(-1). - """ - - def __init__(self, - task_name, - file_path_format, - file_size, - num_files, - prefetch=-1, - num_parallel_calls=-1, - shard=1): - super().__init__() - self.task_type = TF_DATA_READ - self.task_name = task_name - self.file_path_format = file_path_format - self.file_size = file_size - self.num_files = num_files - self.prefetch = prefetch - self.num_parallel_calls = num_parallel_calls - self.shard = shard - self.file_names = [ - self.file_path_format.format(file_num=file_num) - for file_num in range(self.num_files) - ] - - def pre_task(self, process_id, thread_id): #pylint: disable=unused-argument - """Pre-task to clear OS's page caches in load testing. - """ - # Clear the page cache as there is no way to bypass cache in tf.data. - # Note: This takes time and hence decrease the average bandwidth. - os.system("sudo sh -c 'sync; echo 3 > /proc/sys/vm/drop_caches'") - - def task(self, process_id, thread_id): #pylint: disable=unused-argument - """Reads TFRecord files using tensorflow's tf.data method. - - See base class for more details. - - Returns: - Integer denoting the size of content read in bytes. - """ - content_len = 0 - files_dataset = tf.data.Dataset.from_tensor_slices(self.file_names) - - def tfrecord(path): - return tf.data.TFRecordDataset(path).prefetch(self.prefetch).shard( - self.shard, 0) - - dataset = files_dataset.interleave( - tfrecord, - cycle_length=self.num_parallel_calls, - num_parallel_calls=self.num_parallel_calls) - for record in dataset: - content_len = content_len + len(record.numpy()) - - return content_len * self.shard - - def create_files(self, num_processes): #pylint: disable=unused-argument - """Creates self.num_files number of files to be used in load testing. - """ - logging.info('Creating %s bytes TFRecord files using the format ' - '%s', self.file_size, self.file_path_format) - - default_file_path = self.file_path_format.format(file_num=0) - self._create_tfrecord_file(default_file_path, self.file_size) - for file_num in range(1, self.num_files): - file_path = self.file_path_format.format(file_num=file_num) - if not tf.io.gfile.exists(file_path): - logging.info('Creating TFRecord file %s of size %s.', file_path, - self.file_size) - tf.io.gfile.copy(default_file_path, file_path) - - def _create_tfrecord_file(self, file_path, file_size): - """Creates .tfrecord file of given file size in bytes at given file path. - - Doesn't create a file if file is already present. - """ - # We only check existence in this case because actual TFRecord's file size - # is not exactly equal to file_size - if tf.io.gfile.exists(file_path): - return - - logging.info('Creating TFRecord file %s of size %s.', file_path, file_size) - content = b'\t' * SINGLE_RECORD_SIZE - writer = tf.io.TFRecordWriter(file_path) - for _ in range(0, file_size, len(content)): - writer.write(content) - writer.close() diff --git a/perfmetrics/scripts/load_tests/python/tasks/tf_gfile.py b/perfmetrics/scripts/load_tests/python/tasks/tf_gfile.py deleted file mode 100644 index e5294f51e3..0000000000 --- a/perfmetrics/scripts/load_tests/python/tasks/tf_gfile.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains tasks that read data from files using tensorflow's tf.io.gfile api. - -Example: - - lg_obj = load_generator.LoadGenerator(...) - task_obj = python_os.TFGFileRead(...) - observations = lg_obj.generate_load(task_obj) -""" -import logging -import tensorflow as tf -from load_generator import task - -TF_GFILE_READ = 'tf_gfile_read' - - -class TFGFileRead(task.LoadTestTask): - """Task class for reading GCS file using tensorflow's tf.io.gfile api. - - tf.io.gfile: https://www.tensorflow.org/api_docs/python/tf/io/gfile/GFile - Note: The same class can be used with tf's internal GCS client if format of - file path starts with gs:// and with GCSFuse if local GCSFuse mounted path - is passed. - - Args: - task_name: String name assigned to task. - file_path_format: String format of file path. The format can contain - 'process_id' and 'thread_id' to be filled. E.g. gcs/256kb/read.{process_id} - file_size: Integer size of file in bytes to be read. - block_size: Integer size of block in bytes. File is read block by block. - If block size is not passed or -1, then by default it takes file size as - block size - """ - - def __init__(self, task_name, file_path_format, file_size, block_size=-1): - super().__init__() - self.task_type = TF_GFILE_READ - self.task_name = task_name - self.file_path_format = file_path_format - self.file_size = file_size - # Keep file size as default block size - self.block_size = block_size - if self.block_size == -1: - self.block_size = file_size - - def task(self, process_id, thread_id): - """Reads file of given size from given GCS file path and block size. - - See base class for more details. - - Returns: - Integer denoting the size of content read in bytes. - """ - file_path = self.file_path_format.format( - process_id=process_id, thread_id=thread_id) - content_len = 0 - with tf.io.gfile.GFile(file_path, 'rb') as fp: - for _ in range(0, self.file_size, self.block_size): - content = fp.read(self.block_size) - content_len = content_len + len(content) - fp.close() - return content_len - - def create_files(self, num_processes): - """Creates num_processes number of GCS files to be used in load testing. - - Args: - num_processes: Integer denoting number of processes in load test. - - Returns: - None - """ - logging.info( - 'One file is created per process of size %s using the format ' - '%s', self.file_size, self.file_path_format) - for process_id in range(num_processes): - file_path = self.file_path_format.format(process_id=process_id) - self._create_binary_file(file_path, self.file_size) - - def _create_binary_file(self, file_path, file_size): - """Creates binary file of given file size in bytes at given GCS path. - - Doesn't create a file if file is already present and has same file size. - """ - if tf.io.gfile.exists(file_path) and tf.io.gfile.stat( - file_path).length == file_size: - return - logging.info('Creating file %s of size %s.', file_path, file_size) - with tf.io.gfile.GFile(file_path, 'wb') as f_p: - content = b'\t' * file_size - f_p.write(content) diff --git a/perfmetrics/scripts/ls_metrics/README.md b/perfmetrics/scripts/ls_metrics/README.md index 103f308a3e..346821dc4d 100644 --- a/perfmetrics/scripts/ls_metrics/README.md +++ b/perfmetrics/scripts/ls_metrics/README.md @@ -38,7 +38,7 @@ pip install --require-hashes -r requirements.txt --user 2. Install the required packages as mentioned in the above section. 3. Create a service account by following this [documentation](https://cloud.google.com/iam/docs/creating-managing-service-accounts). Generate your service account key, `creds.json` by following [this doc](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#iam-service-account-keys-create-console) and upload the file on your GCS bucket `your-bucket-name`. If using an old credentials file, make sure that it is not expired. Run the following command to copy it into `gsheet` directory: ```bash -gsutil cp gs://your-bucket-name/creds.json ../gsheet +gcloud storage cp gs://your-bucket-name/creds.json ../gsheet ``` 4. Create a Google Sheet with id `your-gsheet-id` by copying this [Google Sheet](https://docs.google.com/spreadsheets/d/1IJIjWuEs7cL6eYqPmlVaEGdclr6MSiaKJdnFXXC5tg8/). 5. Share the above copied Google Sheet with your service account(created in step 2) diff --git a/perfmetrics/scripts/ls_metrics/listing_benchmark.py b/perfmetrics/scripts/ls_metrics/listing_benchmark.py index 08f5fe2db9..b569943a7b 100644 --- a/perfmetrics/scripts/ls_metrics/listing_benchmark.py +++ b/perfmetrics/scripts/ls_metrics/listing_benchmark.py @@ -336,7 +336,7 @@ def _list_directory(path) -> list: """ contents = subprocess.check_output( - 'gsutil -m ls {}'.format(path), shell=True) + 'gcloud storage ls {}'.format(path), shell=True) contents_url = contents.decode('utf-8').split('\n')[:-1] return contents_url @@ -554,7 +554,7 @@ def _export_to_bigquery(test_type, config_id, start_time_build, ls_data): Creating a new one.\n""") log.info('Deleting previously present directories in the GCS bucket.\n') subprocess.call( - 'gsutil -m rm -r gs://{}/*'.format(directory_structure.name), + 'gcloud storage rm --recursive gs://{}/*'.format(directory_structure.name), shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) # Creating a temp directory which will be needed by the generate_files diff --git a/perfmetrics/scripts/ls_metrics/requirements.in b/perfmetrics/scripts/ls_metrics/requirements.in index bc922835bb..92bd8b750e 100644 --- a/perfmetrics/scripts/ls_metrics/requirements.in +++ b/perfmetrics/scripts/ls_metrics/requirements.in @@ -16,4 +16,4 @@ configparser statistics numpy mock -protobuf==4.21.* +protobuf==5.29.* diff --git a/perfmetrics/scripts/ls_metrics/requirements.txt b/perfmetrics/scripts/ls_metrics/requirements.txt index d9de0a2745..a8dbb9cb8e 100644 --- a/perfmetrics/scripts/ls_metrics/requirements.txt +++ b/perfmetrics/scripts/ls_metrics/requirements.txt @@ -50,21 +50,18 @@ numpy==1.24.3 \ --hash=sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c \ --hash=sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b # via -r requirements.in -protobuf==4.21.12 \ - --hash=sha256:1f22ac0ca65bb70a876060d96d914dae09ac98d114294f77584b0d2644fa9c30 \ - --hash=sha256:237216c3326d46808a9f7c26fd1bd4b20015fb6867dc5d263a493ef9a539293b \ - --hash=sha256:27f4d15021da6d2b706ddc3860fac0a5ddaba34ab679dc182b60a8bb4e1121cc \ - --hash=sha256:299ea899484ee6f44604deb71f424234f654606b983cb496ea2a53e3c63ab791 \ - --hash=sha256:3d164928ff0727d97022957c2b849250ca0e64777ee31efd7d6de2e07c494717 \ - --hash=sha256:6ab80df09e3208f742c98443b6166bcb70d65f52cfeb67357d52032ea1ae9bec \ - --hash=sha256:78a28c9fa223998472886c77042e9b9afb6fe4242bd2a2a5aced88e3f4422aa7 \ - --hash=sha256:7cd532c4566d0e6feafecc1059d04c7915aec8e182d1cf7adee8b24ef1e2e6ab \ - --hash=sha256:89f9149e4a0169cddfc44c74f230d7743002e3aa0b9472d8c28f0388102fc4c2 \ - --hash=sha256:a53fd3f03e578553623272dc46ac2f189de23862e68565e83dde203d41b76fc5 \ - --hash=sha256:b135410244ebe777db80298297a97fbb4c862c881b4403b71bac9d4107d61fd1 \ - --hash=sha256:b98d0148f84e3a3c569e19f52103ca1feacdac0d2df8d6533cf983d1fda28462 \ - --hash=sha256:d1736130bce8cf131ac7957fa26880ca19227d4ad68b4888b3be0dea1f95df97 \ - --hash=sha256:f45460f9ee70a0ec1b6694c6e4e348ad2019275680bd68a1d9314b8c7e01e574 +protobuf==5.29.6 \ + --hash=sha256:36ade6ff88212e91aef4e687a971a11d7d24d6948a66751abc1b3238648f5d05 \ + --hash=sha256:62e8a3114992c7c647bce37dcc93647575fc52d50e48de30c6fcb28a6a291eb1 \ + --hash=sha256:6b9edb641441b2da9fa8f428760fc136a49cf97a52076010cf22a2ff73438a86 \ + --hash=sha256:76e07e6567f8baf827137e8d5b8204b6c7b6488bbbff1bf0a72b383f77999c18 \ + --hash=sha256:7e6ad413275be172f67fdee0f43484b6de5a904cc1c3ea9804cb6fe2ff366eda \ + --hash=sha256:831e2da16b6cc9d8f1654c041dd594eda43391affd3c03a91bea7f7f6da106d6 \ + --hash=sha256:a8866b2cff111f0f863c1b3b9e7572dc7eaea23a7fae27f6fc613304046483e6 \ + --hash=sha256:b5a169e664b4057183a34bdc424540e86eea47560f3c123a0d64de4e137f9269 \ + --hash=sha256:cb4c86de9cd8a7f3a256b9744220d87b847371c6b2f10bde87768918ef33ba49 \ + --hash=sha256:da9ee6a5424b6b30fd5e45c5ea663aef540ca95f9ad99d1e887e819cdf9b8723 \ + --hash=sha256:e3387f44798ac1106af0233c04fb8abf543772ff241169946f698b3a9a3d3ab9 # via -r requirements.in statistics==1.0.3.5 \ --hash=sha256:2dc379b80b07bf2ddd5488cad06b2b9531da4dd31edb04dc9ec0dc226486c138 diff --git a/perfmetrics/scripts/micro_benchmarks/helper.py b/perfmetrics/scripts/micro_benchmarks/helper.py new file mode 100644 index 0000000000..7b582bee72 --- /dev/null +++ b/perfmetrics/scripts/micro_benchmarks/helper.py @@ -0,0 +1,206 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from datetime import datetime, timedelta +from google.cloud import bigquery +import os +import subprocess +import pandas as pd + +PROJECT_ID = "gcs-fuse-test-ml" +DATASET_ID = "benchmark_results" +TABLE_ID = "gcsfuse_benchmarks" + +def mount_bucket(mount_dir: str, bucket_name: str, flags: str) -> bool: + """ + Mounts a Google Cloud Storage (GCS) bucket using gcsfuse. + + This function attempts to create the mount directory (if it doesn't exist), + then runs the `gcsfuse` command to mount the specified GCS bucket using the provided flags. + + Args: + mount_dir (str): The local directory where the GCS bucket should be mounted. + bucket_name (str): The name of the GCS bucket to mount. + flags (str): Additional flags or options to pass to the `gcsfuse` command + (e.g., "--implicit-dirs"). + + Returns: + bool: + - True if the mount operation succeeded. + - False if the `gcsfuse` command failed (e.g., due to permissions, bucket issues, etc.). + + Prints: + Status messages indicating success or failure of the mount operation. + """ + os.makedirs(mount_dir, exist_ok=True) + cmd = f"gcsfuse {flags} {bucket_name} {mount_dir}" + print(f"Mounting: {cmd}") + try: + subprocess.run(cmd, shell=True, check=True) + print(f"Successfully mounted {bucket_name} at {mount_dir}") + return True + except subprocess.CalledProcessError as e: + print(f"Failed to mount {bucket_name} at {mount_dir}: {e}") + return False + + +def unmount_gcs_directory(mount_point: str) -> bool: + """ + Unmounts a GCS bucket that was mounted with gcsfuse. + + Args: + mount_point (str): The local mount point directory. + + Prints: + Success or failure message based on the unmount operation. + + Returns: + bool: + - True if the mount operation succeeded. + - False if the `fusermount` command failed. + """ + try: + subprocess.run(["fusermount", "-u", mount_point], check=True) + print(f"Successfully unmounted {mount_point}") + return True + except subprocess.CalledProcessError as e: + print(f"Failed to unmount {mount_point}: {e}. Ensure the directory is correctly mounted.") + return False + + +def log_to_bigquery(start_time_sec: float, duration_sec: float, total_bytes: int, gcsfuse_config: str, workload_type: str) -> None: + """Logs performance metrics to a BigQuery table. + + This function calculates bandwidth, creates a pandas DataFrame with the + provided data, converts the data to the appropriate types, and then inserts + the data into the specified BigQuery table. If the table does not exist, + this query can be used to create it: + + CREATE TABLE `your-project-id.benchmark_results.gcsfuse_benchmarks` ( + start_time TIMESTAMP, + duration_seconds FLOAT64, + bandwidth_mbps FLOAT64, + gcsfuse_config STRING, + workload_type STRING + ); + + Args: + duration_sec (float): Duration of the operation in seconds. + total_bytes (int): Total data processed in bytes. + gcsfuse_config (str): Configuration flags used with gcsfuse. + workload_type (str): Type of workload (e.g., "read", "write"). + + Prints: + Performance metrics and confirmation of successful logging. + """ + bandwidth_mbps = total_bytes / duration_sec / 1000 / 1000 + print(f"Duration: {duration_sec:.2f}s | Data: {total_bytes / (1000 ** 3):.2f} GB | Bandwidth: {bandwidth_mbps:.2f} MB/s") + + client = bigquery.Client(project=PROJECT_ID) + table_ref = client.dataset(DATASET_ID).table(TABLE_ID) + + df = pd.DataFrame([{ + "start_time": datetime.fromtimestamp(start_time_sec), + "duration_seconds": duration_sec, + "bandwidth_mbps": bandwidth_mbps, + "gcsfuse_config": gcsfuse_config, + "workload_type": workload_type, + }]) + + df['start_time'] = pd.to_datetime(df['start_time']) + df['duration_seconds'] = df['duration_seconds'].astype(float) + df['bandwidth_mbps'] = df['bandwidth_mbps'].astype(float) + + client.load_table_from_dataframe(df, table_ref).result() + print("Successfully logged data to BigQuery.") + + +def get_last_n_days_bandwidth_entries( + client: bigquery.Client, + table_ref: bigquery.TableReference, + workload_type: str, + days: int = 3 +) -> list[float]: + """ + Fetches bandwidth measurements (in MB/s) for a given workload type + from the last 'n' days of records in the specified BigQuery table. + + Args: + client (bigquery.Client): Authenticated BigQuery client instance. + table_ref (bigquery.TableReference): Reference to the BigQuery table. + workload_type (str): Type of workload to filter for (e.g., "read" or "write"). + days (int): Number of past days to look back from the current time. + + Returns: + list[float]: A list of bandwidth values (in MB/s). Returns an empty list if no data is found or an error occurs. + """ + full_table_name = f"`{table_ref.project}.{table_ref.dataset_id}.{table_ref.table_id}`" + time_ago = datetime.now() - timedelta(days=days) + time_ago_str = time_ago.strftime('%Y-%m-%d %H:%M:%S') + + query = f""" + SELECT bandwidth_mbps + FROM {full_table_name} + WHERE start_time >= TIMESTAMP('{time_ago_str}') + AND workload_type = '{workload_type}' + ORDER BY start_time DESC + """ + + bandwidths = [] + try: + query_job = client.query(query) + rows = query_job.result() + for row in rows: + bandwidths.append(row.bandwidth_mbps) + except Exception as e: + print(f"Error fetching bandwidth entries for the past {days} days: {e}") + + return bandwidths + + +def check_and_alert_bandwidth(bandwidth_threshold_mbps: float, workload_type: str) -> None: + """ + Validates current bandwidth performance for a workload by comparing it against + a historical average from the past 3 days. If the average bandwidth is below + the defined threshold, the function prints a warning and exits with status code 1. + + Args: + bandwidth_threshold_mbps (float): Minimum acceptable bandwidth (in MB/s). + workload_type (str): The type of workload being evaluated (e.g., "read" or "write"). + """ + client = bigquery.Client(project=PROJECT_ID) + table_ref = client.dataset(DATASET_ID).table(TABLE_ID) + + print("\n--- Bandwidth Validation: Comparing Against Last 3 Days Average ---") + last_three_days_bandwidths = get_last_n_days_bandwidth_entries( + client, table_ref, workload_type, days=3 + ) + + if last_three_days_bandwidths: + avg_past_bandwidth = sum(last_three_days_bandwidths) / len(last_three_days_bandwidths) + print(f"Workload Type : {workload_type}") + print(f"3-Day Average : {avg_past_bandwidth:.2f} MB/s") + print(f"Configured Threshold: {bandwidth_threshold_mbps:.2f} MB/s") + + if avg_past_bandwidth < bandwidth_threshold_mbps: + print("FAILURE: 3-day average bandwidth is below the threshold.") + print("\n----------------------------------\n") + sys.exit(1) # Fail the Kokoro build + else: + print("Bandwidth is within acceptable range.") + else: + print(f"No recent data available for '{workload_type}' workload in the last 3 days.") + + print("\n----------------------------------\n") diff --git a/perfmetrics/scripts/micro_benchmarks/helper_test.py b/perfmetrics/scripts/micro_benchmarks/helper_test.py new file mode 100644 index 0000000000..f95033fa9b --- /dev/null +++ b/perfmetrics/scripts/micro_benchmarks/helper_test.py @@ -0,0 +1,97 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from unittest.mock import patch, MagicMock +import helper +import subprocess + +class TestHelperFunctions(unittest.TestCase): + @patch("helper.subprocess.run") + @patch("helper.os.makedirs") + def test_mount_bucket_success(self, mock_makedirs, mock_run): + result = helper.mount_bucket("/mnt/success", "bucket", "--flags") + + self.assertTrue(result) + + @patch("helper.subprocess.run", side_effect=subprocess.CalledProcessError(1, "gcsfuse")) + @patch("helper.os.makedirs") + def test_mount_bucket_failure(self, mock_makedirs, mock_run): + result = helper.mount_bucket("/mnt/fail", "bucket", "--flags") + + self.assertFalse(result) + + @patch('subprocess.run') + def test_unmount_success(self, mock_run): + mount_point = "/mnt/gcs_test" + mock_run.return_value = subprocess.CompletedProcess(args=["fusermount", "-u", mount_point], returncode=0) + + result = helper.unmount_gcs_directory(mount_point) + + mock_run.assert_called_once_with(["fusermount", "-u", mount_point], check=True) + self.assertTrue(result) + + @patch('subprocess.run', side_effect=subprocess.CalledProcessError(1, ["fusermount", "-u", "/mnt/gcs_test"])) + def test_unmount_failure(self, mock_run): + mount_point = "/mnt/gcs_test" + + result = helper.unmount_gcs_directory(mount_point) + + mock_run.assert_called_once_with(["fusermount", "-u", mount_point], check=True) + self.assertFalse(result) + + @patch("helper.bigquery.Client") # Only patch bigquery.Client + def test_log_to_bigquery(self, mock_bq_client): + duration = 10 + start_time = 0 + total_bytes = 100 * 1000 * 1000 # 100 MB + flags = "--implicit-dirs" + workload_type = "write" + # Setup BigQuery mock + mock_bq_instance = MagicMock() + mock_bq_client.return_value = mock_bq_instance + mock_dataset = mock_bq_instance.dataset.return_value + mock_bq_instance.load_table_from_dataframe.return_value.result.return_value = None + + helper.log_to_bigquery(start_time, duration, total_bytes, flags, workload_type) + + mock_bq_instance.dataset.assert_called_with("benchmark_results") + mock_dataset.table.assert_called_with("gcsfuse_benchmarks") + mock_bq_instance.load_table_from_dataframe.assert_called_once() + mock_bq_instance.load_table_from_dataframe().result.assert_called_once() + + @patch("helper.bigquery.Client") + def test_log_to_bigquery_failure(self, mock_client_cls): + # Create a mock client and a mock load job + mock_client = MagicMock() + mock_load_job = MagicMock() + mock_client.load_table_from_dataframe.return_value = mock_load_job + # Simulate failure on .result() + mock_load_job.result.side_effect = Exception("BigQuery load failed") + # Assign our mock client + mock_client_cls.return_value = mock_client + + with self.assertRaises(Exception) as context: + helper.log_to_bigquery( + start_time_sec=0, + duration_sec=10.0, + total_bytes=100_000_000, + gcsfuse_config="--implicit-dirs", + workload_type="read" + ) + + self.assertIn("BigQuery load failed", str(context.exception)) + +if __name__ == "__main__": + unittest.main() diff --git a/perfmetrics/scripts/micro_benchmarks/read_single_thread.py b/perfmetrics/scripts/micro_benchmarks/read_single_thread.py new file mode 100644 index 0000000000..d12c8f1855 --- /dev/null +++ b/perfmetrics/scripts/micro_benchmarks/read_single_thread.py @@ -0,0 +1,154 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import subprocess +import time +import argparse +from google.cloud import storage +import helper + +MOUNT_DIR = "gcs" +FILE_PREFIX = "testfile_read" + +def check_and_create_files(bucket_name: str, total_files: int, file_size_gb: int): + """ + Ensures that the specified number of files exist in the given GCS bucket. + If a file is missing or its not given size, it is (re)created and uploaded. + + Args: + bucket_name (str): Name of the GCS bucket. + total_files (int): Number of files to check or create. + file_size_gb (int): Expected size of each file in gigabytes (GB, base-10). + """ + client = storage.Client() + bucket = client.get_bucket(bucket_name) + expected_size = file_size_gb * (10 ** 9) # Use base-10 GB + + print(f"Ensuring all {total_files} files exist in gs://{bucket_name}...") + + for i in range(total_files): + fname = f"{FILE_PREFIX}_{file_size_gb}_{i}.bin" + blob = bucket.blob(fname) + + blob_exists = blob.exists() + # Use a default value for blob.size if it's None (e.g., if blob doesn't exist) + if blob_exists: + blob.reload() + current_blob_size = blob.size if blob_exists and blob.size is not None else 0 + + if not blob_exists or current_blob_size != expected_size: + if blob_exists: + print(f"{fname} exists but has size {current_blob_size} bytes (expected ~{expected_size}). Re-uploading...") + + print(f" Creating {file_size_gb}GB dummy file {fname}...") + local_path = f"/tmp/{fname}" + + try: + # Use subprocess.run to execute fallocate + gb_in_bytes = file_size_gb * 10**9 + subprocess.run(f"fallocate -l {gb_in_bytes} {local_path}", shell=True, check=True) + + except subprocess.CalledProcessError as e: + print(f"Error creating dummy file {local_path}: {e}") + continue # Skip to the next file if creation fails + + try: + blob.upload_from_filename(local_path) + print(f"Uploaded {fname} to gs://{bucket_name}/{fname}") + except Exception as e: # Catch broader exceptions for upload issues + print(f"Error uploading {fname} to GCS: {e}") + finally: + # Ensure local file is removed even if upload fails + if os.path.exists(local_path): + os.remove(local_path) + else: + print(f"{fname} already exists with acceptable size.") + +def read_all_files(total_files: int, file_size_gb: int) -> int: + """ + Reads a specified number of files, calculates, and returns the total number + of bytes read across all files. + + The files are expected to be named with a common prefix and index suffix: + {FILE_PREFIX}_{file_size_gb}_{i}.bin, located inside the directory MOUNT_DIR. + + Args: + total_files (int): The number of files to read. + file_size_gb (int): file size in gb + Returns: + int: The total number of bytes read from all files. + + Raises: + RuntimeError: If any error occurs while reading any file, including + file not found, permission issues, or other IO errors. + """ + total_bytes = 0 + for i in range(total_files): + path = os.path.join(MOUNT_DIR, f"{FILE_PREFIX}_{file_size_gb}_{i}.bin") + try: + with open(path, "rb") as f: + total_bytes += len(f.read()) + except Exception as e: # catch all exceptions + print(f"Error reading file {path}: {e}") + raise RuntimeError(f"Failed to read file: {path}") from e + + return total_bytes + +def main(): + parser = argparse.ArgumentParser(description="Measure GCS read bandwidth via gcsfuse.") + parser.add_argument("--bucket", required=True, help="GCS bucket name") + parser.add_argument("--gcsfuse-config", default="--implicit-dirs", help="GCSFuse mount flags") + parser.add_argument("--total-files", type=int, default=10, help="Number of files to read") + parser.add_argument("--file-size-gb", type=int, default=15, help="Size of each file in GB") + + args = parser.parse_args() + workflow_type = f"READ_{args.total_files}_{args.file_size_gb}GB_SINGLE_THREAD" + + # Mount the bucket + helper.mount_bucket(MOUNT_DIR, args.bucket, args.gcsfuse_config) + + # Ensure test files exist + check_and_create_files(args.bucket, args.total_files, args.file_size_gb) + + print(f"Starting read of {args.total_files} files...") + start = time.time() + try: + total_bytes = read_all_files(args.total_files, args.file_size_gb) + print(f"Total bytes read from {args.total_files} files: {total_bytes}") + except RuntimeError as e: + print(f"Failed during file read: {e}") + helper.unmount_gcs_directory(MOUNT_DIR) + sys.exit(1) # Exit with error status + duration = time.time() - start + + # Unmount after test + helper.unmount_gcs_directory(MOUNT_DIR) + + # Log to BigQuery + helper.log_to_bigquery( + start_time_sec=start, + duration_sec=duration, + total_bytes=total_bytes, + gcsfuse_config=args.gcsfuse_config, + workload_type=workflow_type, + ) + + # TODO: Remove this once alerts are configured. + # 160 MB/s is the minimum threshold based on the 3-runs average bandwidth + helper.check_and_alert_bandwidth(160, workflow_type) + +if __name__ == "__main__": + main() diff --git a/perfmetrics/scripts/micro_benchmarks/read_single_thread_test.py b/perfmetrics/scripts/micro_benchmarks/read_single_thread_test.py new file mode 100644 index 0000000000..cd18593ad9 --- /dev/null +++ b/perfmetrics/scripts/micro_benchmarks/read_single_thread_test.py @@ -0,0 +1,163 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from unittest import mock +import read_single_thread + +class TestReadFiles(unittest.TestCase): + + @mock.patch("builtins.open", new_callable=mock.mock_open, read_data=b"abc") + @mock.patch("os.path.join", side_effect=lambda a, b: f"{a}/{b}") + def test_reads_all_files_success(self, mock_join, mock_file): + total_files = 3 + file_size = len(b"abc") + expected_bytes = 3 * file_size + + result = read_single_thread.read_all_files(total_files, file_size) + + self.assertEqual(result, expected_bytes) + actual_calls = mock_file.call_args_list + expected_calls = [ + mock.call(f"{read_single_thread.MOUNT_DIR}/{read_single_thread.FILE_PREFIX}_{file_size}_{i}.bin", "rb") + for i in range(total_files) + ] + self.assertEqual(actual_calls, expected_calls) + + @mock.patch("builtins.open", side_effect=FileNotFoundError("File not found")) + @mock.patch("os.path.join", side_effect=lambda a, b: f"{a}/{b}") + def test_file_not_found_raises_runtime_error(self, mock_join, mock_file): + total_files = 1 + + with self.assertRaises(RuntimeError) as cm: + read_single_thread.read_all_files(total_files, 0) + + self.assertIn("Failed to read file", str(cm.exception)) + + @mock.patch("builtins.open", side_effect=PermissionError("Permission denied")) + @mock.patch("os.path.join", side_effect=lambda a, b: f"{a}/{b}") + def test_permission_error_raises_runtime_error(self, mock_join, mock_file): + total_files = 1 + + with self.assertRaises(RuntimeError) as cm: + read_single_thread.read_all_files(total_files, 0) + + self.assertIn("Failed to read file", str(cm.exception)) + + @mock.patch("builtins.open", new_callable=mock.mock_open, read_data=b"data") + @mock.patch("os.path.join", side_effect=lambda a, b: f"{a}/{b}") + def test_partial_failure(self, mock_join, mock_file): + # Simulate first file reads correctly, second file throws IOError + def side_effect(path, mode="rb"): + if path.endswith("file_0.bin"): + return mock.mock_open(read_data=b"data").return_value + else: + raise IOError("Read error") + mock_file.side_effect = side_effect + + with self.assertRaises(RuntimeError) as cm: + read_single_thread.read_all_files(2, 0) + + self.assertIn("Failed to read file", str(cm.exception)) + + @mock.patch('google.cloud.storage.Client') + @mock.patch('subprocess.run') + @mock.patch('os.remove') + @mock.patch('builtins.print') + def test_upload_failure(self, mock_print, mock_os_remove, mock_subprocess_run, MockClient): + """ + Tests the scenario where blob.upload_from_filename fails. + Verifies that os.remove is still called (cleanup). + """ + mock_blob = mock.MagicMock() + mock_blob.exists.return_value = False + mock_blob.upload_from_filename.side_effect = Exception("GCS upload error") # Simulate upload failure + mock_bucket = mock.MagicMock() + mock_bucket.blob.return_value = mock_blob + MockClient.return_value.get_bucket.return_value = mock_bucket + # Mock os.path.exists for the finally block in the function + with mock.patch('os.path.exists', return_value=True): + bucket_name = "test-bucket" + total_files = 1 + file_size_gb = 1 + + read_single_thread.check_and_create_files(bucket_name, total_files, file_size_gb) + + mock_subprocess_run.assert_called_once() + mock_blob.upload_from_filename.assert_called_once() + mock_os_remove.assert_called_once() # Should be called for cleanup + mock_print.assert_any_call(mock.ANY) # Check if print was called + mock_print.assert_any_call(f"Error uploading {read_single_thread.FILE_PREFIX}_{file_size_gb}_0.bin to GCS: GCS upload error") + + @mock.patch("read_single_thread.os.remove") + @mock.patch("read_single_thread.os.path.exists", return_value=True) + @mock.patch("read_single_thread.subprocess.run") + @mock.patch("read_single_thread.storage.Client") + def test_file_missing_triggers_creation_and_upload(self, mock_storage_client, mock_subprocess_run, mock_path_exists, mock_os_remove): + mock_bucket = mock.MagicMock() + mock_blob = mock.MagicMock() + mock_blob.exists.return_value = False + mock_blob.size = None + mock_bucket.blob.return_value = mock_blob + mock_storage_client.return_value.get_bucket.return_value = mock_bucket + + read_single_thread.check_and_create_files("test-bucket", total_files=1, file_size_gb=1) + + mock_subprocess_run.assert_called_once() # fallocate + mock_blob.upload_from_filename.assert_called_once() + mock_os_remove.assert_called_once() + + @mock.patch("read_single_thread.os.remove") + @mock.patch("read_single_thread.os.path.exists", return_value=True) + @mock.patch("read_single_thread.subprocess.run") + @mock.patch("read_single_thread.storage.Client") + def test_file_too_small_triggers_upload(self, mock_storage_client, mock_subprocess_run, mock_path_exists, mock_os_remove): + mock_bucket = mock.MagicMock() + mock_blob = mock.MagicMock() + mock_blob.exists.return_value = True + mock_blob.size = 1 * (10**9) - 200 * (2**20) # 200 MiB under expected + mock_bucket.blob.return_value = mock_blob + mock_storage_client.return_value.get_bucket.return_value = mock_bucket + + read_single_thread.check_and_create_files("test-bucket", total_files=1, file_size_gb=1) + + mock_subprocess_run.assert_called_once() + mock_blob.upload_from_filename.assert_called_once() + mock_os_remove.assert_called_once() + + @mock.patch("read_single_thread.os.remove") + @mock.patch("read_single_thread.os.path.exists", return_value=True) + @mock.patch("read_single_thread.subprocess.run") + @mock.patch("read_single_thread.storage.Client") + def test_file_equal_file_size_not_trigger_upload(self, mock_storage_client, mock_subprocess_run, mock_path_exists, mock_os_remove): + mock_bucket = mock.MagicMock() + mock_blob = mock.MagicMock() + # Simulate blob already existing with correct size (1 GB) + mock_blob.exists.return_value = True + mock_blob.size = 1 * (10**9) + # Setup bucket and blob mocks + mock_bucket.blob.return_value = mock_blob + mock_storage_client.return_value.get_bucket.return_value = mock_bucket + + # Call function under test + read_single_thread.check_and_create_files("test-bucket", total_files=1, file_size_gb=1) + + # Assert that upload was not triggered + mock_blob.upload_from_filename.assert_not_called() + # subprocess.run should not be called either + mock_subprocess_run.assert_not_called() + + +if __name__ == '__main__': + unittest.main(argv=['first-arg-is-ignored'], exit=False) # For running in environments like notebooks diff --git a/perfmetrics/scripts/micro_benchmarks/requirements.in b/perfmetrics/scripts/micro_benchmarks/requirements.in new file mode 100644 index 0000000000..49c89a2364 --- /dev/null +++ b/perfmetrics/scripts/micro_benchmarks/requirements.in @@ -0,0 +1,8 @@ +pandas +google-cloud-bigquery +google-cloud-storage +pyarrow +pandas_gbq +google-crc32c +psutil +setuptools diff --git a/perfmetrics/scripts/micro_benchmarks/requirements.txt b/perfmetrics/scripts/micro_benchmarks/requirements.txt new file mode 100644 index 0000000000..b626ad4917 --- /dev/null +++ b/perfmetrics/scripts/micro_benchmarks/requirements.txt @@ -0,0 +1,596 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --allow-unsafe --generate-hashes requirements.in +# +cachetools==6.2.2 \ + --hash=sha256:6c09c98183bf58560c97b2abfcedcbaf6a896a490f534b031b661d3723b45ace \ + --hash=sha256:8e6d266b25e539df852251cfd6f990b4bc3a141db73b939058d809ebd2590fc6 + # via google-auth +certifi==2025.11.12 \ + --hash=sha256:97de8790030bbd5c2d96b7ec782fc2f7820ef8dba6db909ccf95449f2d062d4b \ + --hash=sha256:d8ab5478f2ecd78af242878415affce761ca6bc54a22a27e026d7c25357c3316 + # via requests +charset-normalizer==3.4.4 \ + --hash=sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad \ + --hash=sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93 \ + --hash=sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394 \ + --hash=sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89 \ + --hash=sha256:0f04b14ffe5fdc8c4933862d8306109a2c51e0704acfa35d51598eb45a1e89fc \ + --hash=sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86 \ + --hash=sha256:194f08cbb32dc406d6e1aea671a68be0823673db2832b38405deba2fb0d88f63 \ + --hash=sha256:1bee1e43c28aa63cb16e5c14e582580546b08e535299b8b6158a7c9c768a1f3d \ + --hash=sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f \ + --hash=sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8 \ + --hash=sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0 \ + --hash=sha256:2677acec1a2f8ef614c6888b5b4ae4060cc184174a938ed4e8ef690e15d3e505 \ + --hash=sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161 \ + --hash=sha256:2aaba3b0819274cc41757a1da876f810a3e4d7b6eb25699253a4effef9e8e4af \ + --hash=sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152 \ + --hash=sha256:2c9d3c380143a1fedbff95a312aa798578371eb29da42106a29019368a475318 \ + --hash=sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72 \ + --hash=sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4 \ + --hash=sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e \ + --hash=sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3 \ + --hash=sha256:44c2a8734b333e0578090c4cd6b16f275e07aa6614ca8715e6c038e865e70576 \ + --hash=sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c \ + --hash=sha256:4902828217069c3c5c71094537a8e623f5d097858ac6ca8252f7b4d10b7560f1 \ + --hash=sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8 \ + --hash=sha256:4fe7859a4e3e8457458e2ff592f15ccb02f3da787fcd31e0183879c3ad4692a1 \ + --hash=sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2 \ + --hash=sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44 \ + --hash=sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26 \ + --hash=sha256:5947809c8a2417be3267efc979c47d76a079758166f7d43ef5ae8e9f92751f88 \ + --hash=sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016 \ + --hash=sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede \ + --hash=sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf \ + --hash=sha256:5cb4d72eea50c8868f5288b7f7f33ed276118325c1dfd3957089f6b519e1382a \ + --hash=sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc \ + --hash=sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0 \ + --hash=sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84 \ + --hash=sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db \ + --hash=sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1 \ + --hash=sha256:6aee717dcfead04c6eb1ce3bd29ac1e22663cdea57f943c87d1eab9a025438d7 \ + --hash=sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed \ + --hash=sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8 \ + --hash=sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133 \ + --hash=sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e \ + --hash=sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef \ + --hash=sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14 \ + --hash=sha256:778d2e08eda00f4256d7f672ca9fef386071c9202f5e4607920b86d7803387f2 \ + --hash=sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0 \ + --hash=sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d \ + --hash=sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828 \ + --hash=sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f \ + --hash=sha256:7c308f7e26e4363d79df40ca5b2be1c6ba9f02bdbccfed5abddb7859a6ce72cf \ + --hash=sha256:7fa17817dc5625de8a027cb8b26d9fefa3ea28c8253929b8d6649e705d2835b6 \ + --hash=sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328 \ + --hash=sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090 \ + --hash=sha256:837c2ce8c5a65a2035be9b3569c684358dfbf109fd3b6969630a87535495ceaa \ + --hash=sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381 \ + --hash=sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c \ + --hash=sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb \ + --hash=sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc \ + --hash=sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a \ + --hash=sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec \ + --hash=sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc \ + --hash=sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac \ + --hash=sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e \ + --hash=sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313 \ + --hash=sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569 \ + --hash=sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3 \ + --hash=sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d \ + --hash=sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525 \ + --hash=sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894 \ + --hash=sha256:a8bf8d0f749c5757af2142fe7903a9df1d2e8aa3841559b2bad34b08d0e2bcf3 \ + --hash=sha256:a9768c477b9d7bd54bc0c86dbaebdec6f03306675526c9927c0e8a04e8f94af9 \ + --hash=sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a \ + --hash=sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9 \ + --hash=sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14 \ + --hash=sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25 \ + --hash=sha256:b5d84d37db046c5ca74ee7bb47dd6cbc13f80665fdde3e8040bdd3fb015ecb50 \ + --hash=sha256:b7cf1017d601aa35e6bb650b6ad28652c9cd78ee6caff19f3c28d03e1c80acbf \ + --hash=sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1 \ + --hash=sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3 \ + --hash=sha256:c4ef880e27901b6cc782f1b95f82da9313c0eb95c3af699103088fa0ac3ce9ac \ + --hash=sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e \ + --hash=sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815 \ + --hash=sha256:cb01158d8b88ee68f15949894ccc6712278243d95f344770fa7593fa2d94410c \ + --hash=sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6 \ + --hash=sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6 \ + --hash=sha256:cd09d08005f958f370f539f186d10aec3377d55b9eeb0d796025d4886119d76e \ + --hash=sha256:cd4b7ca9984e5e7985c12bc60a6f173f3c958eae74f3ef6624bb6b26e2abbae4 \ + --hash=sha256:ce8a0633f41a967713a59c4139d29110c07e826d131a316b50ce11b1d79b4f84 \ + --hash=sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69 \ + --hash=sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15 \ + --hash=sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191 \ + --hash=sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0 \ + --hash=sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897 \ + --hash=sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd \ + --hash=sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2 \ + --hash=sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794 \ + --hash=sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d \ + --hash=sha256:e912091979546adf63357d7e2ccff9b44f026c075aeaf25a52d0e95ad2281074 \ + --hash=sha256:eaabd426fe94daf8fd157c32e571c85cb12e66692f15516a83a03264b08d06c3 \ + --hash=sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224 \ + --hash=sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838 \ + --hash=sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a \ + --hash=sha256:f155a433c2ec037d4e8df17d18922c3a0d9b3232a396690f17175d2946f0218d \ + --hash=sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d \ + --hash=sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f \ + --hash=sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8 \ + --hash=sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490 \ + --hash=sha256:f8e160feb2aed042cd657a72acc0b481212ed28b1b9a95c0cee1621b524e1966 \ + --hash=sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9 \ + --hash=sha256:fa09f53c465e532f4d3db095e0c55b615f010ad81803d383195b6b5ca6cbf5f3 \ + --hash=sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e \ + --hash=sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608 + # via requests +db-dtypes==1.4.4 \ + --hash=sha256:26f53db5df1acd746b88c5647913a1b20f731c0af1b11abcb6bec5365f31098a \ + --hash=sha256:32c13039982656a8598a0835f25f0e07e34c9a423e471ee60c2553240b7fcf1e + # via pandas-gbq +google-api-core[grpc]==2.28.1 \ + --hash=sha256:2b405df02d68e68ce0fbc138559e6036559e685159d148ae5861013dc201baf8 \ + --hash=sha256:4021b0f8ceb77a6fb4de6fde4502cecab45062e66ff4f2895169e0b35bc9466c + # via + # google-cloud-bigquery + # google-cloud-core + # google-cloud-storage + # pandas-gbq +google-auth==2.41.1 \ + --hash=sha256:754843be95575b9a19c604a848a41be03f7f2afd8c019f716dc1f51ee41c639d \ + --hash=sha256:b76b7b1f9e61f0cb7e88870d14f6a94aeef248959ef6992670efee37709cbfd2 + # via + # google-api-core + # google-auth-oauthlib + # google-cloud-bigquery + # google-cloud-core + # google-cloud-storage + # pandas-gbq + # pydata-google-auth +google-auth-oauthlib==1.2.3 \ + --hash=sha256:7c0940e037677f25e71999607493640d071212e7f3c15aa0febea4c47a5a0680 \ + --hash=sha256:eb09e450d3cc789ecbc2b3529cb94a713673fd5f7a22c718ad91cf75aedc2ea4 + # via + # pandas-gbq + # pydata-google-auth +google-cloud-bigquery==3.38.0 \ + --hash=sha256:8afcb7116f5eac849097a344eb8bfda78b7cfaae128e60e019193dd483873520 \ + --hash=sha256:e06e93ff7b245b239945ef59cb59616057598d369edac457ebf292bd61984da6 + # via + # -r micro_benchmarks/requirements.in + # pandas-gbq +google-cloud-core==2.5.0 \ + --hash=sha256:67d977b41ae6c7211ee830c7912e41003ea8194bff15ae7d72fd6f51e57acabc \ + --hash=sha256:7c1b7ef5c92311717bd05301aa1a91ffbc565673d3b0b4163a52d8413a186963 + # via + # google-cloud-bigquery + # google-cloud-storage +google-cloud-storage==3.6.0 \ + --hash=sha256:29cc6b9a6c0fc9cdad071e375d540a5a50fbc9a7fad8300fa02fb904f6fe2ca2 \ + --hash=sha256:5decbdddd63b7d1fc3e266a393ad6453d2e27d172bd982b1e2f15481668db097 + # via -r micro_benchmarks/requirements.in +google-crc32c==1.7.1 \ + --hash=sha256:0f99eaa09a9a7e642a61e06742856eec8b19fc0037832e03f941fe7cf0c8e4db \ + --hash=sha256:19eafa0e4af11b0a4eb3974483d55d2d77ad1911e6cf6f832e1574f6781fd337 \ + --hash=sha256:1c67ca0a1f5b56162951a9dae987988679a7db682d6f97ce0f6381ebf0fbea4c \ + --hash=sha256:1f2b3522222746fff0e04a9bd0a23ea003ba3cccc8cf21385c564deb1f223242 \ + --hash=sha256:22beacf83baaf59f9d3ab2bbb4db0fb018da8e5aebdce07ef9f09fce8220285e \ + --hash=sha256:2bff2305f98846f3e825dbeec9ee406f89da7962accdb29356e4eadc251bd472 \ + --hash=sha256:2d73a68a653c57281401871dd4aeebbb6af3191dcac751a76ce430df4d403194 \ + --hash=sha256:32d1da0d74ec5634a05f53ef7df18fc646666a25efaaca9fc7dcfd4caf1d98c3 \ + --hash=sha256:3bda0fcb632d390e3ea8b6b07bf6b4f4a66c9d02dcd6fbf7ba00a197c143f582 \ + --hash=sha256:6335de12921f06e1f774d0dd1fbea6bf610abe0887a1638f64d694013138be5d \ + --hash=sha256:6b211ddaf20f7ebeec5c333448582c224a7c90a9d98826fbab82c0ddc11348e6 \ + --hash=sha256:6efb97eb4369d52593ad6f75e7e10d053cf00c48983f7a973105bc70b0ac4d82 \ + --hash=sha256:6fbab4b935989e2c3610371963ba1b86afb09537fd0c633049be82afe153ac06 \ + --hash=sha256:713121af19f1a617054c41f952294764e0c5443d5a5d9034b2cd60f5dd7e0349 \ + --hash=sha256:754561c6c66e89d55754106739e22fdaa93fafa8da7221b29c8b8e8270c6ec8a \ + --hash=sha256:7cc81b3a2fbd932a4313eb53cc7d9dde424088ca3a0337160f35d91826880c1d \ + --hash=sha256:85fef7fae11494e747c9fd1359a527e5970fc9603c90764843caabd3a16a0a48 \ + --hash=sha256:905a385140bf492ac300026717af339790921f411c0dfd9aa5a9e69a08ed32eb \ + --hash=sha256:9fc196f0b8d8bd2789352c6a522db03f89e83a0ed6b64315923c396d7a932315 \ + --hash=sha256:a8e9afc74168b0b2232fb32dd202c93e46b7d5e4bf03e66ba5dc273bb3559589 \ + --hash=sha256:b07d48faf8292b4db7c3d64ab86f950c2e94e93a11fd47271c28ba458e4a0d76 \ + --hash=sha256:b6d86616faaea68101195c6bdc40c494e4d76f41e07a37ffdef270879c15fb65 \ + --hash=sha256:b7491bdc0c7564fcf48c0179d2048ab2f7c7ba36b84ccd3a3e1c3f7a72d3bba6 \ + --hash=sha256:bb5e35dcd8552f76eed9461a23de1030920a3c953c1982f324be8f97946e7127 \ + --hash=sha256:d68e17bad8f7dd9a49181a1f5a8f4b251c6dbc8cc96fb79f1d321dfd57d66f53 \ + --hash=sha256:dcdf5a64adb747610140572ed18d011896e3b9ae5195f2514b7ff678c80f1603 \ + --hash=sha256:df8b38bdaf1629d62d51be8bdd04888f37c451564c2042d36e5812da9eff3c35 \ + --hash=sha256:e10554d4abc5238823112c2ad7e4560f96c7bf3820b202660373d769d9e6e4c9 \ + --hash=sha256:e42e20a83a29aa2709a0cf271c7f8aefaa23b7ab52e53b322585297bb94d4638 \ + --hash=sha256:ed66cbe1ed9cbaaad9392b5259b3eba4a9e565420d734e6238813c428c3336c9 \ + --hash=sha256:ee6547b657621b6cbed3562ea7826c3e11cab01cd33b74e1f677690652883e77 \ + --hash=sha256:f2226b6a8da04f1d9e61d3e357f2460b9551c5e6950071437e122c958a18ae14 \ + --hash=sha256:fa8136cc14dd27f34a3221c0f16fd42d8a40e4778273e61a3c19aedaa44daf6b \ + --hash=sha256:fc5319db92daa516b653600794d5b9f9439a9a121f3e162f94b0e1891c7933cb + # via + # -r micro_benchmarks/requirements.in + # google-cloud-storage + # google-resumable-media +google-resumable-media==2.8.0 \ + --hash=sha256:dd14a116af303845a8d932ddae161a26e86cc229645bc98b39f026f9b1717582 \ + --hash=sha256:f1157ed8b46994d60a1bc432544db62352043113684d4e030ee02e77ebe9a1ae + # via + # google-cloud-bigquery + # google-cloud-storage +googleapis-common-protos==1.72.0 \ + --hash=sha256:4299c5a82d5ae1a9702ada957347726b167f9f8d1fc352477702a1e851ff4038 \ + --hash=sha256:e55a601c1b32b52d7a3e65f43563e2aa61bcd737998ee672ac9b951cd49319f5 + # via + # google-api-core + # grpcio-status +grpcio==1.76.0 \ + --hash=sha256:035d90bc79eaa4bed83f524331d55e35820725c9fbb00ffa1904d5550ed7ede3 \ + --hash=sha256:04bbe1bfe3a68bbfd4e52402ab7d4eb59d72d02647ae2042204326cf4bbad280 \ + --hash=sha256:063065249d9e7e0782d03d2bca50787f53bd0fb89a67de9a7b521c4a01f1989b \ + --hash=sha256:06c3d6b076e7b593905d04fdba6a0525711b3466f43b3400266f04ff735de0cd \ + --hash=sha256:08caea849a9d3c71a542827d6df9d5a69067b0a1efbea8a855633ff5d9571465 \ + --hash=sha256:0aaa82d0813fd4c8e589fac9b65d7dd88702555f702fb10417f96e2a2a6d4c0f \ + --hash=sha256:0b7604868b38c1bfd5cf72d768aedd7db41d78cb6a4a18585e33fb0f9f2363fd \ + --hash=sha256:0c37db8606c258e2ee0c56b78c62fc9dee0e901b5dbdcf816c2dd4ad652b8b0c \ + --hash=sha256:1c9b93f79f48b03ada57ea24725d83a30284a012ec27eab2cf7e50a550cbbbcc \ + --hash=sha256:2107b0c024d1b35f4083f11245c0e23846ae64d02f40b2b226684840260ed054 \ + --hash=sha256:2229ae655ec4e8999599469559e97630185fdd53ae1e8997d147b7c9b2b72cba \ + --hash=sha256:25a18e9810fbc7e7f03ec2516addc116a957f8cbb8cbc95ccc80faa072743d03 \ + --hash=sha256:26ef06c73eb53267c2b319f43e6634c7556ea37672029241a056629af27c10e2 \ + --hash=sha256:2e1743fbd7f5fa713a1b0a8ac8ebabf0ec980b5d8809ec358d488e273b9cf02a \ + --hash=sha256:32483fe2aab2c3794101c2a159070584e5db11d0aa091b2c0ea9c4fc43d0d749 \ + --hash=sha256:3bf0f392c0b806905ed174dcd8bdd5e418a40d5567a05615a030a5aeddea692d \ + --hash=sha256:3e2a27c89eb9ac3d81ec8835e12414d73536c6e620355d65102503064a4ed6eb \ + --hash=sha256:40ad3afe81676fd9ec6d9d406eda00933f218038433980aa19d401490e46ecde \ + --hash=sha256:4215d3a102bd95e2e11b5395c78562967959824156af11fa93d18fdd18050990 \ + --hash=sha256:45d59a649a82df5718fd9527ce775fd66d1af35e6d31abdcdc906a49c6822958 \ + --hash=sha256:45e0111e73f43f735d70786557dc38141185072d7ff8dc1829d6a77ac1471468 \ + --hash=sha256:479496325ce554792dba6548fae3df31a72cef7bad71ca2e12b0e58f9b336bfc \ + --hash=sha256:490fa6d203992c47c7b9e4a9d39003a0c2bcc1c9aa3c058730884bbbb0ee9f09 \ + --hash=sha256:49ce47231818806067aea3324d4bf13825b658ad662d3b25fada0bdad9b8a6af \ + --hash=sha256:4baf3cbe2f0be3289eb68ac8ae771156971848bb8aaff60bad42005539431980 \ + --hash=sha256:522175aba7af9113c48ec10cc471b9b9bd4f6ceb36aeb4544a8e2c80ed9d252d \ + --hash=sha256:5e8571632780e08526f118f74170ad8d50fb0a48c23a746bef2a6ebade3abd6f \ + --hash=sha256:615ba64c208aaceb5ec83bfdce7728b80bfeb8be97562944836a7a0a9647d882 \ + --hash=sha256:61f69297cba3950a524f61c7c8ee12e55c486cb5f7db47ff9dcee33da6f0d3ae \ + --hash=sha256:65a20de41e85648e00305c1bb09a3598f840422e522277641145a32d42dcefcc \ + --hash=sha256:6a15c17af8839b6801d554263c546c69c4d7718ad4321e3166175b37eaacca77 \ + --hash=sha256:747fa73efa9b8b1488a95d0ba1039c8e2dca0f741612d80415b1e1c560febf4e \ + --hash=sha256:7be78388d6da1a25c0d5ec506523db58b18be22d9c37d8d3a32c08be4987bd73 \ + --hash=sha256:81fd9652b37b36f16138611c7e884eb82e0cec137c40d3ef7c3f9b3ed00f6ed8 \ + --hash=sha256:83d57312a58dcfe2a3a0f9d1389b299438909a02db60e2f2ea2ae2d8034909d3 \ + --hash=sha256:8843114c0cfce61b40ad48df65abcfc00d4dba82eae8718fab5352390848c5da \ + --hash=sha256:8cc3309d8e08fd79089e13ed4819d0af72aa935dd8f435a195fd152796752ff2 \ + --hash=sha256:8ebe63ee5f8fa4296b1b8cfc743f870d10e902ca18afc65c68cf46fd39bb0783 \ + --hash=sha256:8eddfb4d203a237da6f3cc8a540dad0517d274b5a1e9e636fd8d2c79b5c1d397 \ + --hash=sha256:922fa70ba549fce362d2e2871ab542082d66e2aaf0c19480ea453905b01f384e \ + --hash=sha256:931091142fd8cc14edccc0845a79248bc155425eee9a98b2db2ea4f00a235a42 \ + --hash=sha256:971fd5a1d6e62e00d945423a567e42eb1fa678ba89072832185ca836a94daaa6 \ + --hash=sha256:980a846182ce88c4f2f7e2c22c56aefd515daeb36149d1c897f83cf57999e0b6 \ + --hash=sha256:9d9adda641db7207e800a7f089068f6f645959f2df27e870ee81d44701dd9db3 \ + --hash=sha256:9f8f757bebaaea112c00dba718fc0d3260052ce714e25804a03f93f5d1c6cc11 \ + --hash=sha256:a6ae758eb08088d36812dd5d9af7a9859c05b1e0f714470ea243694b49278e7b \ + --hash=sha256:a8c2cf1209497cf659a667d7dea88985e834c24b7c3b605e6254cbb5076d985c \ + --hash=sha256:acab0277c40eff7143c2323190ea57b9ee5fd353d8190ee9652369fae735668a \ + --hash=sha256:b331680e46239e090f5b3cead313cc772f6caa7d0fc8de349337563125361a4a \ + --hash=sha256:c088e7a90b6017307f423efbb9d1ba97a22aa2170876223f9709e9d1de0b5347 \ + --hash=sha256:d099566accf23d21037f18a2a63d323075bebace807742e4b0ac210971d4dd70 \ + --hash=sha256:d388087771c837cdb6515539f43b9d4bf0b0f23593a24054ac16f7a960be16f4 \ + --hash=sha256:dcfe41187da8992c5f40aa8c5ec086fa3672834d2be57a32384c08d5a05b4c00 \ + --hash=sha256:e6d1db20594d9daba22f90da738b1a0441a7427552cc6e2e3d1297aeddc00378 \ + --hash=sha256:ebea5cc3aa8ea72e04df9913492f9a96d9348db876f9dda3ad729cfedf7ac416 \ + --hash=sha256:ebebf83299b0cb1721a8859ea98f3a77811e35dce7609c5c963b9ad90728f886 \ + --hash=sha256:f0e34c2079d47ae9f6188211db9e777c619a21d4faba6977774e8fa43b085e48 \ + --hash=sha256:f92f88e6c033db65a5ae3d97905c8fea9c725b63e28d5a75cb73b49bda5024d8 \ + --hash=sha256:f9f7bd5faab55f47231ad8dba7787866b69f5e93bc306e3915606779bbfb4ba8 \ + --hash=sha256:fd5ef5932f6475c436c4a55e4336ebbe47bd3272be04964a03d316bbf4afbcbc \ + --hash=sha256:ff8a59ea85a1f2191a0ffcc61298c571bc566332f82e5f5be1b83c9d8e668a62 + # via + # google-api-core + # grpcio-status +grpcio-status==1.76.0 \ + --hash=sha256:25fcbfec74c15d1a1cb5da3fab8ee9672852dc16a5a9eeb5baf7d7a9952943cd \ + --hash=sha256:380568794055a8efbbd8871162df92012e0228a5f6dffaf57f2a00c534103b18 + # via google-api-core +idna==3.11 \ + --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea \ + --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902 + # via requests +numpy==2.2.6 \ + --hash=sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff \ + --hash=sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47 \ + --hash=sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84 \ + --hash=sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d \ + --hash=sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6 \ + --hash=sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f \ + --hash=sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b \ + --hash=sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49 \ + --hash=sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163 \ + --hash=sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571 \ + --hash=sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42 \ + --hash=sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff \ + --hash=sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491 \ + --hash=sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4 \ + --hash=sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566 \ + --hash=sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf \ + --hash=sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40 \ + --hash=sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd \ + --hash=sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06 \ + --hash=sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282 \ + --hash=sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680 \ + --hash=sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db \ + --hash=sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3 \ + --hash=sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90 \ + --hash=sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1 \ + --hash=sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289 \ + --hash=sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab \ + --hash=sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c \ + --hash=sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d \ + --hash=sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb \ + --hash=sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d \ + --hash=sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a \ + --hash=sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf \ + --hash=sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1 \ + --hash=sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2 \ + --hash=sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a \ + --hash=sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543 \ + --hash=sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00 \ + --hash=sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c \ + --hash=sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f \ + --hash=sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd \ + --hash=sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868 \ + --hash=sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303 \ + --hash=sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83 \ + --hash=sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3 \ + --hash=sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d \ + --hash=sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87 \ + --hash=sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa \ + --hash=sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f \ + --hash=sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae \ + --hash=sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda \ + --hash=sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915 \ + --hash=sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249 \ + --hash=sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de \ + --hash=sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8 + # via + # db-dtypes + # pandas + # pandas-gbq +oauthlib==3.3.1 \ + --hash=sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9 \ + --hash=sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1 + # via requests-oauthlib +packaging==25.0 \ + --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 \ + --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f + # via + # db-dtypes + # google-cloud-bigquery + # pandas-gbq +pandas==2.3.3 \ + --hash=sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7 \ + --hash=sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593 \ + --hash=sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5 \ + --hash=sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791 \ + --hash=sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73 \ + --hash=sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec \ + --hash=sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4 \ + --hash=sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5 \ + --hash=sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac \ + --hash=sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084 \ + --hash=sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c \ + --hash=sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87 \ + --hash=sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35 \ + --hash=sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250 \ + --hash=sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c \ + --hash=sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826 \ + --hash=sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9 \ + --hash=sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713 \ + --hash=sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1 \ + --hash=sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523 \ + --hash=sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3 \ + --hash=sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78 \ + --hash=sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53 \ + --hash=sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c \ + --hash=sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21 \ + --hash=sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5 \ + --hash=sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff \ + --hash=sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45 \ + --hash=sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110 \ + --hash=sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493 \ + --hash=sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b \ + --hash=sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450 \ + --hash=sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86 \ + --hash=sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8 \ + --hash=sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98 \ + --hash=sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89 \ + --hash=sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66 \ + --hash=sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b \ + --hash=sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8 \ + --hash=sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29 \ + --hash=sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6 \ + --hash=sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc \ + --hash=sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2 \ + --hash=sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788 \ + --hash=sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa \ + --hash=sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151 \ + --hash=sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838 \ + --hash=sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b \ + --hash=sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a \ + --hash=sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d \ + --hash=sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908 \ + --hash=sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0 \ + --hash=sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b \ + --hash=sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c \ + --hash=sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee + # via + # -r micro_benchmarks/requirements.in + # db-dtypes + # pandas-gbq +pandas-gbq==0.31.0 \ + --hash=sha256:0a999cfb06c83efa54d36731dda5748d83daef1445cb23ec03d137d23af7fd5c \ + --hash=sha256:cb7aa43bba052631aed66d4464820657aa037b203e9ee1a413437171ab9ab867 + # via -r micro_benchmarks/requirements.in +proto-plus==1.26.1 \ + --hash=sha256:13285478c2dcf2abb829db158e1047e2f1e8d63a077d94263c2b88b043c75a66 \ + --hash=sha256:21a515a4c4c0088a773899e23c7bbade3d18f9c66c73edd4c7ee3816bc96a012 + # via google-api-core +protobuf==6.33.5 \ + --hash=sha256:3093804752167bcab3998bec9f1048baae6e29505adaf1afd14a37bddede533c \ + --hash=sha256:69915a973dd0f60f31a08b8318b73eab2bd6a392c79184b3612226b0a3f8ec02 \ + --hash=sha256:6ddcac2a081f8b7b9642c09406bc6a4290128fce5f471cddd165960bb9119e5c \ + --hash=sha256:8afa18e1d6d20af15b417e728e9f60f3aa108ee76f23c3b2c07a2c3b546d3afd \ + --hash=sha256:8f04fa32763dcdb4973d537d6b54e615cc61108c7cb38fe59310c3192d29510a \ + --hash=sha256:9b71e0281f36f179d00cbcb119cb19dec4d14a81393e5ea220f64b286173e190 \ + --hash=sha256:a3157e62729aafb8df6da2c03aa5c0937c7266c626ce11a278b6eb7963c4e37c \ + --hash=sha256:a5cb85982d95d906df1e2210e58f8e4f1e3cdc088e52c921a041f9c9a0386de5 \ + --hash=sha256:cbf16ba3350fb7b889fca858fb215967792dc125b35c7976ca4818bee3521cf0 \ + --hash=sha256:d71b040839446bac0f4d162e758bea99c8251161dae9d0983a3b88dee345153b + # via + # google-api-core + # googleapis-common-protos + # grpcio-status + # proto-plus +psutil==7.1.3 \ + --hash=sha256:0005da714eee687b4b8decd3d6cc7c6db36215c9e74e5ad2264b90c3df7d92dc \ + --hash=sha256:1068c303be3a72f8e18e412c5b2a8f6d31750fb152f9cb106b54090296c9d251 \ + --hash=sha256:18349c5c24b06ac5612c0428ec2a0331c26443d259e2a0144a9b24b4395b58fa \ + --hash=sha256:19644c85dcb987e35eeeaefdc3915d059dac7bd1167cdcdbf27e0ce2df0c08c0 \ + --hash=sha256:2bdbcd0e58ca14996a42adf3621a6244f1bb2e2e528886959c72cf1e326677ab \ + --hash=sha256:31d77fcedb7529f27bb3a0472bea9334349f9a04160e8e6e5020f22c59893264 \ + --hash=sha256:3792983e23b69843aea49c8f5b8f115572c5ab64c153bada5270086a2123c7e7 \ + --hash=sha256:3bb428f9f05c1225a558f53e30ccbad9930b11c3fc206836242de1091d3e7dd3 \ + --hash=sha256:56d974e02ca2c8eb4812c3f76c30e28836fffc311d55d979f1465c1feeb2b68b \ + --hash=sha256:6c86281738d77335af7aec228328e944b30930899ea760ecf33a4dba66be5e74 \ + --hash=sha256:8f33a3702e167783a9213db10ad29650ebf383946e91bc77f28a5eb083496bc9 \ + --hash=sha256:95ef04cf2e5ba0ab9eaafc4a11eaae91b44f4ef5541acd2ee91d9108d00d59a7 \ + --hash=sha256:ad81425efc5e75da3f39b3e636293360ad8d0b49bed7df824c79764fb4ba9b8b \ + --hash=sha256:b403da1df4d6d43973dc004d19cee3b848e998ae3154cc8097d139b77156c353 \ + --hash=sha256:bc31fa00f1fbc3c3802141eede66f3a2d51d89716a194bf2cd6fc68310a19880 \ + --hash=sha256:bd0d69cee829226a761e92f28140bec9a5ee9d5b4fb4b0cc589068dbfff559b1 \ + --hash=sha256:c525ffa774fe4496282fb0b1187725793de3e7c6b29e41562733cae9ada151ee \ + --hash=sha256:f39c2c19fe824b47484b96f9692932248a54c43799a84282cfe58d05a6449efd \ + --hash=sha256:fac9cd332c67f4422504297889da5ab7e05fd11e3c4392140f7370f4208ded1f + # via + # -r micro_benchmarks/requirements.in + # pandas-gbq +pyarrow==22.0.0 \ + --hash=sha256:001ea83a58024818826a9e3f89bf9310a114f7e26dfe404a4c32686f97bd7901 \ + --hash=sha256:00626d9dc0f5ef3a75fe63fd68b9c7c8302d2b5bbc7f74ecaedba83447a24f84 \ + --hash=sha256:0c34fe18094686194f204a3b1787a27456897d8a2d62caf84b61e8dfbc0252ae \ + --hash=sha256:12fe549c9b10ac98c91cf791d2945e878875d95508e1a5d14091a7aaa66d9cf8 \ + --hash=sha256:1a812a5b727bc09c3d7ea072c4eebf657c2f7066155506ba31ebf4792f88f016 \ + --hash=sha256:252be4a05f9d9185bb8c18e83764ebcfea7185076c07a7a662253af3a8c07941 \ + --hash=sha256:334f900ff08ce0423407af97e6c26ad5d4e3b0763645559ece6fbf3747d6a8f5 \ + --hash=sha256:35ad0f0378c9359b3f297299c3309778bb03b8612f987399a0333a560b43862d \ + --hash=sha256:3d600dc583260d845c7d8a6db540339dd883081925da2bd1c5cb808f720b3cd9 \ + --hash=sha256:3e294c5eadfb93d78b0763e859a0c16d4051fc1c5231ae8956d61cb0b5666f5a \ + --hash=sha256:3e739edd001b04f654b166204fc7a9de896cf6007eaff33409ee9e50ceaff754 \ + --hash=sha256:44729980b6c50a5f2bfcc2668d36c569ce17f8b17bccaf470c4313dcbbf13c9d \ + --hash=sha256:44d2d26cda26d18f7af7db71453b7b783788322d756e81730acb98f24eb90ace \ + --hash=sha256:4c19236ae2402a8663a2c8f21f1870a03cc57f0bef7e4b6eb3238cc82944de80 \ + --hash=sha256:69763ab2445f632d90b504a815a2a033f74332997052b721002298ed6de40f2e \ + --hash=sha256:6dda1ddac033d27421c20d7a7943eec60be44e0db4e079f33cc5af3b8280ccde \ + --hash=sha256:6f9762274496c244d951c819348afbcf212714902742225f649cf02823a6a10f \ + --hash=sha256:710624ab925dc2b05a6229d47f6f0dac1c1155e6ed559be7109f684eba048a48 \ + --hash=sha256:7388ac685cab5b279a41dfe0a6ccd99e4dbf322edfb63e02fc0443bf24134e91 \ + --hash=sha256:77718810bd3066158db1e95a63c160ad7ce08c6b0710bc656055033e39cdad88 \ + --hash=sha256:7a820d8ae11facf32585507c11f04e3f38343c1e784c9b5a8b1da5c930547fe2 \ + --hash=sha256:8382ad21458075c2e66a82a29d650f963ce51c7708c7c0ff313a8c206c4fd5e8 \ + --hash=sha256:84378110dd9a6c06323b41b56e129c504d157d1a983ce8f5443761eb5256bafc \ + --hash=sha256:854794239111d2b88b40b6ef92aa478024d1e5074f364033e73e21e3f76b25e0 \ + --hash=sha256:92843c305330aa94a36e706c16209cd4df274693e777ca47112617db7d0ef3d7 \ + --hash=sha256:9bddc2cade6561f6820d4cd73f99a0243532ad506bc510a75a5a65a522b2d74d \ + --hash=sha256:a4893d31e5ef780b6edcaf63122df0f8d321088bb0dee4c8c06eccb1ca28d145 \ + --hash=sha256:a9d9ffdc2ab696f6b15b4d1f7cec6658e1d788124418cb30030afbae31c64746 \ + --hash=sha256:ac93252226cf288753d8b46280f4edf3433bf9508b6977f8dd8526b521a1bbb9 \ + --hash=sha256:b41f37cabfe2463232684de44bad753d6be08a7a072f6a83447eeaf0e4d2a215 \ + --hash=sha256:b883fe6fd85adad7932b3271c38ac289c65b7337c2c132e9569f9d3940620730 \ + --hash=sha256:b9d71701ce97c95480fecb0039ec5bb889e75f110da72005743451339262f4ce \ + --hash=sha256:ba95112d15fd4f1105fb2402c4eab9068f0554435e9b7085924bcfaac2cc306f \ + --hash=sha256:bba208d9c7decf9961998edf5c65e3ea4355d5818dd6cd0f6809bec1afb951cc \ + --hash=sha256:bd0d42297ace400d8febe55f13fdf46e86754842b860c978dfec16f081e5c653 \ + --hash=sha256:bea79263d55c24a32b0d79c00a1c58bb2ee5f0757ed95656b01c0fb310c5af3d \ + --hash=sha256:c064e28361c05d72eed8e744c9605cbd6d2bb7481a511c74071fd9b24bc65d7d \ + --hash=sha256:c3200cb41cdbc65156e5f8c908d739b0dfed57e890329413da2748d1a2cd1a4e \ + --hash=sha256:c6c791b09c57ed76a18b03f2631753a4960eefbbca80f846da8baefc6491fcfe \ + --hash=sha256:c6ec3675d98915bf1ec8b3c7986422682f7232ea76cad276f4c8abd5b7319b70 \ + --hash=sha256:ce20fe000754f477c8a9125543f1936ea5b8867c5406757c224d745ed033e691 \ + --hash=sha256:cedb9dd9358e4ea1d9bce3665ce0797f6adf97ff142c8e25b46ba9cdd508e9b6 \ + --hash=sha256:e0a15757fccb38c410947df156f9749ae4a3c89b2393741a50521f39a8cf202a \ + --hash=sha256:e6e95176209257803a8b3d0394f21604e796dadb643d2f7ca21b66c9c0b30c9a \ + --hash=sha256:e70ff90c64419709d38c8932ea9fe1cc98415c4f87ea8da81719e43f02534bc9 \ + --hash=sha256:ec1a15968a9d80da01e1d30349b2b0d7cc91e96588ee324ce1b5228175043e95 \ + --hash=sha256:ec5d40dd494882704fb876c16fa7261a69791e784ae34e6b5992e977bd2e238c \ + --hash=sha256:f633074f36dbc33d5c05b5dc75371e5660f1dbf9c8b1d95669def05e5425989c \ + --hash=sha256:f7fe3dbe871294ba70d789be16b6e7e52b418311e166e0e3cba9522f0f437fb1 \ + --hash=sha256:f963ba8c3b0199f9d6b794c90ec77545e05eadc83973897a4523c9e8d84e9340 + # via + # -r micro_benchmarks/requirements.in + # db-dtypes + # pandas-gbq +pyasn1==0.6.3 \ + --hash=sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf \ + --hash=sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.2 \ + --hash=sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a \ + --hash=sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6 + # via google-auth +pydata-google-auth==1.9.1 \ + --hash=sha256:0a51ce41c601ca0bc69b8795bf58bedff74b4a6a007c9106c7cbcdec00eaced2 \ + --hash=sha256:75ffce5d106e34b717b31844c1639ea505b7d9550dc23b96fb6c20d086b53fa3 + # via pandas-gbq +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via + # google-cloud-bigquery + # pandas +pytz==2025.2 \ + --hash=sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3 \ + --hash=sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00 + # via pandas +requests==2.33.0 \ + --hash=sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b \ + --hash=sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652 + # via + # google-api-core + # google-cloud-bigquery + # google-cloud-storage + # requests-oauthlib +requests-oauthlib==2.0.0 \ + --hash=sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36 \ + --hash=sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9 + # via google-auth-oauthlib +rsa==4.9.1 \ + --hash=sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762 \ + --hash=sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75 + # via google-auth +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via python-dateutil +typing-extensions==4.15.0 \ + --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ + --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 + # via grpcio +tzdata==2025.2 \ + --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ + --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 + # via pandas +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 + # via requests + +# The following packages are considered to be unsafe in a requirements file: +setuptools==80.9.0 \ + --hash=sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922 \ + --hash=sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c + # via + # -r micro_benchmarks/requirements.in + # pandas-gbq + # pydata-google-auth diff --git a/perfmetrics/scripts/micro_benchmarks/run_microbenchmark.sh b/perfmetrics/scripts/micro_benchmarks/run_microbenchmark.sh new file mode 100755 index 0000000000..fa6f3cd73a --- /dev/null +++ b/perfmetrics/scripts/micro_benchmarks/run_microbenchmark.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail # Exit on error, unset variables are errors, pipe fails propagate + +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" +} + +# --- Constants --- +VENV_DIR="venv" +ARTIFACT_BUCKET_PATH="gcsfuse-kokoro-logs/prod/gcsfuse/gcp_ubuntu/periodic/micro_benchmark" +DATE=$(date +%Y-%m-%d) + +# --- Functions --- +cleanup_mounts() { + log "Cleaning up any stale gcsfuse mounts..." + for mnt in $(mount | grep gcsfuse | awk '{print $3}'); do + log "Unmounting $mnt" + sudo fusermount -u "$mnt" || true + done +} + + +prepare_venv() { + log "Setting up Python virtual environment..." + if [[ ! -d "$VENV_DIR" ]]; then + python3 -m venv "$VENV_DIR" + fi + source "$VENV_DIR/bin/activate" + pip install --require-hashes -r "requirements.txt" +} + +run_benchmark() { + local rw=$1 # "read" or "write" operation type + local script_path=$2 # Path to the benchmark script (e.g., read_single_thread.py) + local file_size_gb=$3 # Size of each file to read/write in GB + local total_files=$4 # Total number of files to process + + echo "Running $rw benchmark with file size $file_size_gb GB and total files $total_files..." + local log_file="/tmp/gcsfuse-logs-single-threaded-${rw}-${file_size_gb}gb-test.txt" + + # Clean old log file if it exists + rm -f "$log_file" + + # Pass log file flag as a string. + local gcsfuse_flags="--log-file $log_file" + + log "Running $rw benchmark..." + if ! python3 "$script_path" --bucket single-threaded-tests \ + --gcsfuse-config "$gcsfuse_flags" \ + --total-files "$total_files" \ + --file-size-gb "$file_size_gb"; then + log "$rw benchmark failed. Copying log to gs://$ARTIFACT_BUCKET_PATH/$DATE" + gcloud storage cp "$log_file" "gs://$ARTIFACT_BUCKET_PATH/$DATE/" + gcloud storage cat "gs://$ARTIFACT_BUCKET_PATH/$DATE/$(basename "$log_file")" + return 1 + fi + + return 0 +} + +# --- Main Script --- +log "Installing dependencies..." +sudo apt-get update -y +sudo apt-get install -y git gnupg python3-venv + +cd "$HOME/github/gcsfuse" +commitId=$(git rev-parse --short HEAD) +./perfmetrics/scripts/build_and_install_gcsfuse.sh $commitId + +cd "perfmetrics/scripts/micro_benchmarks" +# Cleanup previous mounts if any +cleanup_mounts +prepare_venv + +READ_GB=15 +TOTAL_READ_FILES=10 +WRITE_GB=15 +TOTAL_WRITE_FILES=1 +exit_code=0 + +if ! run_benchmark "read" "read_single_thread.py" "$READ_GB" "$TOTAL_READ_FILES"; then + echo "Read benchmark failed." + exit_code=1 +fi + +if ! run_benchmark "write" "write_single_thread.py" "$WRITE_GB" "$TOTAL_WRITE_FILES"; then + echo "Write benchmark failed." + exit_code=1 +fi + +deactivate || true +cleanup_mounts + +if [[ $exit_code -ne 0 ]]; then + log "One or more benchmarks failed." + exit $exit_code +fi + +log "Benchmarks completed successfully." +exit 0 diff --git a/perfmetrics/scripts/micro_benchmarks/write_single_thread.py b/perfmetrics/scripts/micro_benchmarks/write_single_thread.py new file mode 100644 index 0000000000..c67b7c9b0a --- /dev/null +++ b/perfmetrics/scripts/micro_benchmarks/write_single_thread.py @@ -0,0 +1,140 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import argparse +import time +import helper +import sys + +MOUNT_DIR = "gcs" +FILE_PREFIX = "testfile" + +def delete_existing_file(file_path): + """ + Deletes the file at the specified path if it exists. + + Args: + file_path (str): The full path to the file to delete. + + Returns: + bool: True if file deleted or didn't exist; False on error. + """ + try: + if os.path.exists(file_path): + os.remove(file_path) + print(f"{file_path} existed and was cleared.") + return True + except Exception as e: + print(f"Error deleting file {file_path}: {e}") + return False + +def write_random_file(file_path, file_size_in_bytes): + """ + Creates a binary file filled with random data of the specified size. + + Args: + file_path (str): The full path where the file should be created. + file_size_in_bytes (int): The size of the file in bytes. + + Returns: + bool: True on success; False on failure. + """ + try: + with open(file_path, 'wb') as f: + f.write(os.urandom(file_size_in_bytes)) + print(f"Created {file_path} of size {file_size_in_bytes / (1000 ** 3):.4f} GB") + return True + except Exception as e: + print(f"Error writing file {file_path}: {e}") + return False + +def create_files(file_paths, file_size_in_gb): + """ + Writes random data to specified file paths, each of the given size in GB. + + Args: + file_paths (list[str]): List of file paths to create. + file_size_in_gb (float): Size of each file in GB (base 10). + + Returns: + int | None: Total bytes written on success, None on failure. + """ + total_bytes_written = 0 + file_size_in_bytes = int(file_size_in_gb * (1000 ** 3)) + + for file_path in file_paths: + try: + success = write_random_file(file_path, file_size_in_bytes) + if not success: + print("Write failed. Exiting.") + sys.exit(1) + total_bytes_written += file_size_in_bytes + except Exception as e: + print(f"Error creating file {file_path}: {e}") + return None + + print(f"Total bytes written: {total_bytes_written / (1000**3):.4f} GB") + return total_bytes_written + +def main(): + parser = argparse.ArgumentParser(description="Measure GCS write bandwidth via gcsfuse.") + parser.add_argument("--bucket", required=True) + parser.add_argument("--gcsfuse-config", default="--implicit-dirs") + parser.add_argument("--total-files", type=int, default=1) + parser.add_argument("--file-size-gb", type=int, default=15, help="Size of each file in GB") + args = parser.parse_args() + + workflow_type = f"WRITE_{args.total_files}_{args.file_size_gb}GB_SINGLE_THREAD" + helper.mount_bucket(MOUNT_DIR, args.bucket, args.gcsfuse_config) + + # Prepare file paths + file_paths = [ + os.path.join(MOUNT_DIR, f"{FILE_PREFIX}_{args.file_size_gb}_{i}.bin") + for i in range(args.total_files) + ] + + # Delete files if they already exist + for path in file_paths: + success = delete_existing_file(path) + if not success: + print("Delete failed. Exiting.") + sys.exit(1) + + print(f"Starting write of {args.total_files} files...") + start = time.time() + try: + total_bytes = create_files(file_paths, args.file_size_gb) + except RuntimeError as e: + print(f"Failed during file write: {e}") + helper.unmount_gcs_directory(MOUNT_DIR) + sys.exit(1) # Exit with error status + duration = time.time() - start + + helper.unmount_gcs_directory(MOUNT_DIR) + + helper.log_to_bigquery( + start_time_sec=start, + duration_sec=duration, + total_bytes=total_bytes, + gcsfuse_config=args.gcsfuse_config, + workload_type=workflow_type, + ) + + # TODO: Remove this once alerts are configured. + # 80 MB/s is the minimum threshold based on the 3-runs average bandwidth + helper.check_and_alert_bandwidth(80, workflow_type) + +if __name__ == "__main__": + main() diff --git a/perfmetrics/scripts/micro_benchmarks/write_single_thread_test.py b/perfmetrics/scripts/micro_benchmarks/write_single_thread_test.py new file mode 100644 index 0000000000..bf4c81f558 --- /dev/null +++ b/perfmetrics/scripts/micro_benchmarks/write_single_thread_test.py @@ -0,0 +1,83 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +from unittest import mock +from write_single_thread import create_files, delete_existing_file, write_random_file + +class TestWriteFiles(unittest.TestCase): + + @mock.patch("os.path.exists", return_value=True) + @mock.patch("os.remove") + def test_delete_existing_file_success(self, mock_remove, mock_exists): + result = delete_existing_file("/fake/path") + + self.assertTrue(result) + mock_remove.assert_called_once_with("/fake/path") + + @mock.patch("os.path.exists", return_value=True) + @mock.patch("os.remove", side_effect=OSError("Permission denied")) + def test_delete_existing_file_failure(self, mock_remove, mock_exists): + result = delete_existing_file("/fake/path") + + self.assertFalse(result) + mock_remove.assert_called_once_with("/fake/path") + + @mock.patch("os.path.exists", return_value=False) + @mock.patch("os.remove") + def test_delete_existing_file_file_not_exist(self, mock_remove, mock_exists): + # When file doesn't exist, should return True and not call remove + result = delete_existing_file("/fake/path") + + self.assertTrue(result) + mock_remove.assert_not_called() + + @mock.patch("builtins.open", new_callable=mock.mock_open) + @mock.patch("os.urandom", return_value=b'x' * 10) + def test_write_random_file_success(self, mock_urandom, mock_open): + result = write_random_file("/fake/file", 10) + + self.assertTrue(result) + mock_open.assert_called_once_with("/fake/file", "wb") + mock_urandom.assert_called_once_with(10) + + @mock.patch("builtins.open", side_effect=IOError("Disk full")) + def test_write_random_file_failure(self, mock_open): + result = write_random_file("/fake/file", 10) + + self.assertFalse(result) + mock_open.assert_called_once_with("/fake/file", "wb") + + @mock.patch("os.urandom", return_value=b"x" * 10) + @mock.patch("builtins.open", new_callable=mock.mock_open) + def test_create_files_success(self, mock_open_file, mock_urandom): + paths = ["/tmp/file1.bin", "/tmp/file2.bin"] + expected_total = 20 # 2 files * 10 bytes + + total = create_files(paths, file_size_in_gb=1e-8) # ~10 bytes each + + self.assertEqual(total, expected_total) + self.assertEqual(mock_open_file.call_count, 2) + + @mock.patch("builtins.open", side_effect=Exception("write error")) + def test_create_files_failure(self, mock_open_file): + paths = ["/tmp/file1.bin"] + + with self.assertRaises(SystemExit) as cm: + create_files(paths, file_size_in_gb=1e-8) + + self.assertEqual(cm.exception.code, 1) + + +if __name__ == '__main__': + unittest.main(argv=['first-arg-is-ignored'], exit=False) diff --git a/perfmetrics/scripts/ml_tests/checkpoint/Jax/emulated_checkpoints.py b/perfmetrics/scripts/ml_tests/checkpoint/Jax/emulated_checkpoints.py new file mode 100755 index 0000000000..bb25aa0771 --- /dev/null +++ b/perfmetrics/scripts/ml_tests/checkpoint/Jax/emulated_checkpoints.py @@ -0,0 +1,70 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import jax +import jax.numpy as jnp +import optax +from flax import linen as nn +from flax.training import train_state +from flax.training import checkpoints +import argparse + +# Mock model definition. +class SimpleModel(nn.Module): + @nn.compact + def __call__(self, x): + features = [16384, 8192, 4096, 2048, 1024, 512, 256, 128, 1] + for feature in features: + x = nn.Dense(features=feature)(x) + x = nn.relu(x) + return x + +# Mock training step. +def train_step(state, batch): + def loss_fn(params): + preds = state.apply_fn(params, batch['x']) + loss = jnp.mean(jnp.square(preds - batch['y'])) + return loss + + grad_fn = jax.value_and_grad(loss_fn) + loss, grads = grad_fn(state.params) + state = state.apply_gradients(grads=grads) + return state, loss + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Train a simple model and save checkpoints.') + parser.add_argument('--checkpoint_dir', type=str, required=True, help='Directory to save checkpoints') + parser.add_argument('--num_train_steps', type=int, default=2000, help='Number of training steps') # Added argument for num_train_steps + args = parser.parse_args() + + # Sample data. + key = jax.random.PRNGKey(0) + x = jax.random.normal(key, (10, 5)) + y = jax.random.normal(key, (10, 1)) + + # Initialize model and optimizer. + model = SimpleModel() + params = model.init(key, x) + optimizer = optax.adam(learning_rate=0.01) + + # Create train state. + state = train_state.TrainState.create(apply_fn=model.apply, params=params, tx=optimizer) + + # Mock training step. + state, loss = train_step(state, {'x': x, 'y': y}) + + # Save checkpoint to local directory + for step in range(args.num_train_steps): + if step % 200 == 0: + checkpoints.save_checkpoint(args.checkpoint_dir, state, step, keep=100, prefix='checkpoint_') diff --git a/perfmetrics/scripts/ml_tests/checkpoint/Jax/requirements.in b/perfmetrics/scripts/ml_tests/checkpoint/Jax/requirements.in new file mode 100644 index 0000000000..0cf77ec416 --- /dev/null +++ b/perfmetrics/scripts/ml_tests/checkpoint/Jax/requirements.in @@ -0,0 +1,29 @@ +absl-py==2.1.0 +chex==0.1.89 +etils==1.12.0 +flax==0.10.4 +fsspec==2025.2.0 +humanize==4.12.1 +importlib_resources==6.5.2 +jax==0.5.1 +jaxlib==0.5.1 +markdown-it-py==3.0.0 +mdurl==0.1.2 +ml_dtypes==0.5.1 +msgpack==1.1.0 +nest-asyncio==1.6.0 +numpy==2.2.3 +opt_einsum==3.4.0 +optax==0.2.4 +orbax-checkpoint==0.11.6 +protobuf==5.29.6 +Pygments==2.20.0 +PyYAML==6.0.2 +rich==13.9.4 +scipy==1.15.2 +simplejson==3.20.1 +tensorstore==0.1.72 +toolz==1.0.0 +treescope==0.1.9 +typing_extensions==4.12.2 +zipp==3.21.0 \ No newline at end of file diff --git a/perfmetrics/scripts/ml_tests/checkpoint/Jax/requirements.txt b/perfmetrics/scripts/ml_tests/checkpoint/Jax/requirements.txt new file mode 100644 index 0000000000..01cdca0047 --- /dev/null +++ b/perfmetrics/scripts/ml_tests/checkpoint/Jax/requirements.txt @@ -0,0 +1,587 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --generate-hashes requirements.in +# +absl-py==2.1.0 \ + --hash=sha256:526a04eadab8b4ee719ce68f204172ead1027549089702d99b9059f129ff1308 \ + --hash=sha256:7820790efbb316739cde8b4e19357243fc3608a152024288513dd968d7d959ff + # via + # -r requirements.in + # chex + # optax + # orbax-checkpoint +chex==0.1.89 \ + --hash=sha256:145241c27d8944adb634fb7d472a460e1c1b643f561507d4031ad5156ef82dfa \ + --hash=sha256:78f856e6a0a8459edfcbb402c2c044d2b8102eac4b633838cbdfdcdb09c6c8e0 + # via + # -r requirements.in + # optax +etils[epath,epy]==1.12.0 \ + --hash=sha256:67aa7d549f9bee7851e07fbf0e099232b7f867c2825f468d7cbe728ab0d01bd8 \ + --hash=sha256:f80c2ff4289cc504b58b7e7a9f9db8b373a33227e43694a66808bcc81e51ffb8 + # via + # -r requirements.in + # optax + # orbax-checkpoint +flax==0.10.4 \ + --hash=sha256:57ae44d3f111fc85cff9049adb9684ce8ebd44e87bd8ca776ed52422c2d85021 \ + --hash=sha256:8cc83d91654ff943909730e02e858b4cd4577531373f83abe6597c58c581032d + # via -r requirements.in +fsspec==2025.2.0 \ + --hash=sha256:1c24b16eaa0a1798afa0337aa0db9b256718ab2a89c425371f5628d22c3b6afd \ + --hash=sha256:9de2ad9ce1f85e1931858535bc882543171d197001a0a5eb2ddc04f1781ab95b + # via + # -r requirements.in + # etils +humanize==4.12.1 \ + --hash=sha256:1338ba97415c96556758a6e2f65977ed406dddf4620d4c6db9bbdfd07f0f1232 \ + --hash=sha256:86014ca5c52675dffa1d404491952f1f5bf03b07c175a51891a343daebf01fea + # via + # -r requirements.in + # orbax-checkpoint +importlib-resources==6.5.2 \ + --hash=sha256:185f87adef5bcc288449d98fb4fba07cea78bc036455dd44c5fc4a2fe78fed2c \ + --hash=sha256:789cfdc3ed28c78b67a06acb8126751ced69a3d5f79c095a98298cd8a760ccec + # via + # -r requirements.in + # etils +jax==0.5.1 \ + --hash=sha256:4fdfedeccdb974bf36046a2ebab4217b1abb296e7ce6fabef4c2482d956cbbab \ + --hash=sha256:c098f74846ee718165bbfa83521ae10cd52cf50b47f043f8b33a6cfd3c20ddfd + # via + # -r requirements.in + # chex + # flax + # optax + # orbax-checkpoint +jaxlib==0.5.1 \ + --hash=sha256:090fe7d4bc8e19768771dbc8319de9024746bb567f5d9ba379b5102f17658c41 \ + --hash=sha256:0ed6fc1fbee91be70979f05dd523f11ca9de2a14d81a7f7d5aa5e783580587c2 \ + --hash=sha256:15891f22a05d5b7fc273a7e7473f3b76bd3fee7425602db6f63238567343f5e7 \ + --hash=sha256:26ae319630e72b252e97c103ef071496abd5c4794494d9cc063feabad265567a \ + --hash=sha256:2dabccb086476818f737cd0e622fd88dc4f5bb4f0df4c7a5c68c34b366e8847f \ + --hash=sha256:334c49ad411f39a5055c23f139552ae32bd9afe696abbb1cadf9c44ebef607f7 \ + --hash=sha256:3f4f500cd899e437a505dfe8df4f09e45f160163cd44dea12e0b37145656897b \ + --hash=sha256:51e7b59fc40bb270440c5049b3c82f9f7fdadae3199f16818620cfdb80b967af \ + --hash=sha256:5b4ba5aa3f59b5f2e37d525cedd6afe0feecb88416e5f43eb9a709cfded8b250 \ + --hash=sha256:616b6f4d2ee338f1b7061b17b49b064eaace25666b8e0dc4fb48dfa478aa94f3 \ + --hash=sha256:65bc4900a0491dfc6fb9b6a62e8100d121429d58c7429945ec2b424c1a82bf27 \ + --hash=sha256:69f4b9e07ad074d441b9921b7a83aee4f4ffd3d542033fdec50be1456d0611c6 \ + --hash=sha256:80c0ed5446644b383caa3e617540803bb0ef36e5609cd7756d4d2913a8de512e \ + --hash=sha256:8c57fbbe79aa3ce3ac2ec657a7f17867b9b3c2cad885b2c8390567cc9738eee8 \ + --hash=sha256:ae3de28bf9b86781c30a32c88b7cbd1d3222a8c229aa96cbe21055dcd09eb889 \ + --hash=sha256:dc109ffa6873640226c360da793a8a7beaf8c48751ac3fdb29fa6b337901e186 \ + --hash=sha256:df3704f135cff87fd9d41930248925f2f163beed6efeaaabccd97401580dcfd8 + # via + # -r requirements.in + # chex + # jax + # optax +markdown-it-py==3.0.0 \ + --hash=sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1 \ + --hash=sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb + # via + # -r requirements.in + # rich +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via + # -r requirements.in + # markdown-it-py +ml-dtypes==0.5.1 \ + --hash=sha256:023ce2f502efd4d6c1e0472cc58ce3640d051d40e71e27386bed33901e201327 \ + --hash=sha256:05f23447a1c20ddf4dc7c2c661aa9ed93fcb2658f1017c204d1e758714dc28a8 \ + --hash=sha256:12651420130ee7cc13059fc56dac6ad300c3af3848b802d475148c9defd27c23 \ + --hash=sha256:141b2ea2f20bb10802ddca55d91fe21231ef49715cfc971998e8f2a9838f3dbe \ + --hash=sha256:15ad0f3b0323ce96c24637a88a6f44f6713c64032f27277b069f285c3cf66478 \ + --hash=sha256:1b7fbe5571fdf28fd3aaab3ef4aafc847de9ebf263be959958c1ca58ec8eadf5 \ + --hash=sha256:26ebcc69d7b779c8f129393e99732961b5cc33fcff84090451f448c89b0e01b4 \ + --hash=sha256:6f462f5eca22fb66d7ff9c4744a3db4463af06c49816c4b6ac89b16bfcdc592e \ + --hash=sha256:6f76232163b5b9c34291b54621ee60417601e2e4802a188a0ea7157cd9b323f4 \ + --hash=sha256:7000b6e4d8ef07542c05044ec5d8bbae1df083b3f56822c3da63993a113e716f \ + --hash=sha256:810512e2eccdfc3b41eefa3a27402371a3411453a1efc7e9c000318196140fed \ + --hash=sha256:8f2c028954f16ede77902b223a8da2d9cbb3892375b85809a5c3cfb1587960c4 \ + --hash=sha256:9626d0bca1fb387d5791ca36bacbba298c5ef554747b7ebeafefb4564fc83566 \ + --hash=sha256:ac5b58559bb84a95848ed6984eb8013249f90b6bab62aa5acbad876e256002c9 \ + --hash=sha256:ad4953c5eb9c25a56d11a913c2011d7e580a435ef5145f804d98efa14477d390 \ + --hash=sha256:aefedc579ece2f8fb38f876aa7698204ee4c372d0e54f1c1ffa8ca580b54cc60 \ + --hash=sha256:afb2009ac98da274e893e03162f6269398b2b00d947e7057ee2469a921d58135 \ + --hash=sha256:b8a9d46b4df5ae2135a8e8e72b465448ebbc1559997f4f9304a9ecc3413efb5b \ + --hash=sha256:bd73f51957949069573ff783563486339a9285d72e2f36c18e0c1aa9ca7eb190 \ + --hash=sha256:bf9975bda82a99dc935f2ae4c83846d86df8fd6ba179614acac8e686910851da \ + --hash=sha256:c09526488c3a9e8b7a23a388d4974b670a9a3dd40c5c8a61db5593ce9b725bab \ + --hash=sha256:c9945669d3dadf8acb40ec2e57d38c985d8c285ea73af57fc5b09872c516106d \ + --hash=sha256:d13755f8e8445b3870114e5b6240facaa7cb0c3361e54beba3e07fa912a6e12b \ + --hash=sha256:fd918d4e6a4e0c110e2e05be7a7814d10dc1b95872accbf6512b80a109b71ae1 + # via + # -r requirements.in + # jax + # jaxlib + # tensorstore +msgpack==1.1.0 \ + --hash=sha256:06f5fd2f6bb2a7914922d935d3b8bb4a7fff3a9a91cfce6d06c13bc42bec975b \ + --hash=sha256:071603e2f0771c45ad9bc65719291c568d4edf120b44eb36324dcb02a13bfddf \ + --hash=sha256:0907e1a7119b337971a689153665764adc34e89175f9a34793307d9def08e6ca \ + --hash=sha256:0f92a83b84e7c0749e3f12821949d79485971f087604178026085f60ce109330 \ + --hash=sha256:115a7af8ee9e8cddc10f87636767857e7e3717b7a2e97379dc2054712693e90f \ + --hash=sha256:13599f8829cfbe0158f6456374e9eea9f44eee08076291771d8ae93eda56607f \ + --hash=sha256:17fb65dd0bec285907f68b15734a993ad3fc94332b5bb21b0435846228de1f39 \ + --hash=sha256:2137773500afa5494a61b1208619e3871f75f27b03bcfca7b3a7023284140247 \ + --hash=sha256:3180065ec2abbe13a4ad37688b61b99d7f9e012a535b930e0e683ad6bc30155b \ + --hash=sha256:398b713459fea610861c8a7b62a6fec1882759f308ae0795b5413ff6a160cf3c \ + --hash=sha256:3d364a55082fb2a7416f6c63ae383fbd903adb5a6cf78c5b96cc6316dc1cedc7 \ + --hash=sha256:3df7e6b05571b3814361e8464f9304c42d2196808e0119f55d0d3e62cd5ea044 \ + --hash=sha256:41c991beebf175faf352fb940bf2af9ad1fb77fd25f38d9142053914947cdbf6 \ + --hash=sha256:42f754515e0f683f9c79210a5d1cad631ec3d06cea5172214d2176a42e67e19b \ + --hash=sha256:452aff037287acb1d70a804ffd022b21fa2bb7c46bee884dbc864cc9024128a0 \ + --hash=sha256:4676e5be1b472909b2ee6356ff425ebedf5142427842aa06b4dfd5117d1ca8a2 \ + --hash=sha256:46c34e99110762a76e3911fc923222472c9d681f1094096ac4102c18319e6468 \ + --hash=sha256:471e27a5787a2e3f974ba023f9e265a8c7cfd373632247deb225617e3100a3c7 \ + --hash=sha256:4a1964df7b81285d00a84da4e70cb1383f2e665e0f1f2a7027e683956d04b734 \ + --hash=sha256:4b51405e36e075193bc051315dbf29168d6141ae2500ba8cd80a522964e31434 \ + --hash=sha256:4d1b7ff2d6146e16e8bd665ac726a89c74163ef8cd39fa8c1087d4e52d3a2325 \ + --hash=sha256:53258eeb7a80fc46f62fd59c876957a2d0e15e6449a9e71842b6d24419d88ca1 \ + --hash=sha256:534480ee5690ab3cbed89d4c8971a5c631b69a8c0883ecfea96c19118510c846 \ + --hash=sha256:58638690ebd0a06427c5fe1a227bb6b8b9fdc2bd07701bec13c2335c82131a88 \ + --hash=sha256:58dfc47f8b102da61e8949708b3eafc3504509a5728f8b4ddef84bd9e16ad420 \ + --hash=sha256:59caf6a4ed0d164055ccff8fe31eddc0ebc07cf7326a2aaa0dbf7a4001cd823e \ + --hash=sha256:5dbad74103df937e1325cc4bfeaf57713be0b4f15e1c2da43ccdd836393e2ea2 \ + --hash=sha256:5e1da8f11a3dd397f0a32c76165cf0c4eb95b31013a94f6ecc0b280c05c91b59 \ + --hash=sha256:646afc8102935a388ffc3914b336d22d1c2d6209c773f3eb5dd4d6d3b6f8c1cb \ + --hash=sha256:64fc9068d701233effd61b19efb1485587560b66fe57b3e50d29c5d78e7fef68 \ + --hash=sha256:65553c9b6da8166e819a6aa90ad15288599b340f91d18f60b2061f402b9a4915 \ + --hash=sha256:685ec345eefc757a7c8af44a3032734a739f8c45d1b0ac45efc5d8977aa4720f \ + --hash=sha256:6ad622bf7756d5a497d5b6836e7fc3752e2dd6f4c648e24b1803f6048596f701 \ + --hash=sha256:73322a6cc57fcee3c0c57c4463d828e9428275fb85a27aa2aa1a92fdc42afd7b \ + --hash=sha256:74bed8f63f8f14d75eec75cf3d04ad581da6b914001b474a5d3cd3372c8cc27d \ + --hash=sha256:79ec007767b9b56860e0372085f8504db5d06bd6a327a335449508bbee9648fa \ + --hash=sha256:7a946a8992941fea80ed4beae6bff74ffd7ee129a90b4dd5cf9c476a30e9708d \ + --hash=sha256:7ad442d527a7e358a469faf43fda45aaf4ac3249c8310a82f0ccff9164e5dccd \ + --hash=sha256:7c9a35ce2c2573bada929e0b7b3576de647b0defbd25f5139dcdaba0ae35a4cc \ + --hash=sha256:7e7b853bbc44fb03fbdba34feb4bd414322180135e2cb5164f20ce1c9795ee48 \ + --hash=sha256:879a7b7b0ad82481c52d3c7eb99bf6f0645dbdec5134a4bddbd16f3506947feb \ + --hash=sha256:8a706d1e74dd3dea05cb54580d9bd8b2880e9264856ce5068027eed09680aa74 \ + --hash=sha256:8a84efb768fb968381e525eeeb3d92857e4985aacc39f3c47ffd00eb4509315b \ + --hash=sha256:8cf9e8c3a2153934a23ac160cc4cba0ec035f6867c8013cc6077a79823370346 \ + --hash=sha256:8da4bf6d54ceed70e8861f833f83ce0814a2b72102e890cbdfe4b34764cdd66e \ + --hash=sha256:8e59bca908d9ca0de3dc8684f21ebf9a690fe47b6be93236eb40b99af28b6ea6 \ + --hash=sha256:914571a2a5b4e7606997e169f64ce53a8b1e06f2cf2c3a7273aa106236d43dd5 \ + --hash=sha256:a51abd48c6d8ac89e0cfd4fe177c61481aca2d5e7ba42044fd218cfd8ea9899f \ + --hash=sha256:a52a1f3a5af7ba1c9ace055b659189f6c669cf3657095b50f9602af3a3ba0fe5 \ + --hash=sha256:ad33e8400e4ec17ba782f7b9cf868977d867ed784a1f5f2ab46e7ba53b6e1e1b \ + --hash=sha256:b4c01941fd2ff87c2a934ee6055bda4ed353a7846b8d4f341c428109e9fcde8c \ + --hash=sha256:bce7d9e614a04d0883af0b3d4d501171fbfca038f12c77fa838d9f198147a23f \ + --hash=sha256:c40ffa9a15d74e05ba1fe2681ea33b9caffd886675412612d93ab17b58ea2fec \ + --hash=sha256:c5a91481a3cc573ac8c0d9aace09345d989dc4a0202b7fcb312c88c26d4e71a8 \ + --hash=sha256:c921af52214dcbb75e6bdf6a661b23c3e6417f00c603dd2070bccb5c3ef499f5 \ + --hash=sha256:d46cf9e3705ea9485687aa4001a76e44748b609d260af21c4ceea7f2212a501d \ + --hash=sha256:d8ce0b22b890be5d252de90d0e0d119f363012027cf256185fc3d474c44b1b9e \ + --hash=sha256:dd432ccc2c72b914e4cb77afce64aab761c1137cc698be3984eee260bcb2896e \ + --hash=sha256:e0856a2b7e8dcb874be44fea031d22e5b3a19121be92a1e098f46068a11b0870 \ + --hash=sha256:e1f3c3d21f7cf67bcf2da8e494d30a75e4cf60041d98b3f79875afb5b96f3a3f \ + --hash=sha256:f1ba6136e650898082d9d5a5217d5906d1e138024f836ff48691784bbe1adf96 \ + --hash=sha256:f3e9b4936df53b970513eac1758f3882c88658a220b58dcc1e39606dccaaf01c \ + --hash=sha256:f80bc7d47f76089633763f952e67f8214cb7b3ee6bfa489b3cb6a84cfac114cd \ + --hash=sha256:fd2906780f25c8ed5d7b323379f6138524ba793428db5d0e9d226d3fa6aa1788 + # via + # -r requirements.in + # flax + # orbax-checkpoint +nest-asyncio==1.6.0 \ + --hash=sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe \ + --hash=sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c + # via + # -r requirements.in + # orbax-checkpoint +numpy==2.2.3 \ + --hash=sha256:0391ea3622f5c51a2e29708877d56e3d276827ac5447d7f45e9bc4ade8923c52 \ + --hash=sha256:12c045f43b1d2915eca6b880a7f4a256f59d62df4f044788c8ba67709412128d \ + --hash=sha256:136553f123ee2951bfcfbc264acd34a2fc2f29d7cdf610ce7daf672b6fbaa693 \ + --hash=sha256:1402da8e0f435991983d0a9708b779f95a8c98c6b18a171b9f1be09005e64d9d \ + --hash=sha256:16372619ee728ed67a2a606a614f56d3eabc5b86f8b615c79d01957062826ca8 \ + --hash=sha256:1ad78ce7f18ce4e7df1b2ea4019b5817a2f6a8a16e34ff2775f646adce0a5027 \ + --hash=sha256:1b416af7d0ed3271cad0f0a0d0bee0911ed7eba23e66f8424d9f3dfcdcae1304 \ + --hash=sha256:1f45315b2dc58d8a3e7754fe4e38b6fce132dab284a92851e41b2b344f6441c5 \ + --hash=sha256:2376e317111daa0a6739e50f7ee2a6353f768489102308b0d98fcf4a04f7f3b5 \ + --hash=sha256:23c9f4edbf4c065fddb10a4f6e8b6a244342d95966a48820c614891e5059bb50 \ + --hash=sha256:246535e2f7496b7ac85deffe932896a3577be7af8fb7eebe7146444680297e9a \ + --hash=sha256:2e8da03bd561504d9b20e7a12340870dfc206c64ea59b4cfee9fceb95070ee94 \ + --hash=sha256:34c1b7e83f94f3b564b35f480f5652a47007dd91f7c839f404d03279cc8dd021 \ + --hash=sha256:39261798d208c3095ae4f7bc8eaeb3481ea8c6e03dc48028057d3cbdbdb8937e \ + --hash=sha256:3b787adbf04b0db1967798dba8da1af07e387908ed1553a0d6e74c084d1ceafe \ + --hash=sha256:3c2ec8a0f51d60f1e9c0c5ab116b7fc104b165ada3f6c58abf881cb2eb16044d \ + --hash=sha256:435e7a933b9fda8126130b046975a968cc2d833b505475e588339e09f7672890 \ + --hash=sha256:4d8335b5f1b6e2bce120d55fb17064b0262ff29b459e8493d1785c18ae2553b8 \ + --hash=sha256:4d9828d25fb246bedd31e04c9e75714a4087211ac348cb39c8c5f99dbb6683fe \ + --hash=sha256:52659ad2534427dffcc36aac76bebdd02b67e3b7a619ac67543bc9bfe6b7cdb1 \ + --hash=sha256:5266de33d4c3420973cf9ae3b98b54a2a6d53a559310e3236c4b2b06b9c07d4e \ + --hash=sha256:5521a06a3148686d9269c53b09f7d399a5725c47bbb5b35747e1cb76326b714b \ + --hash=sha256:596140185c7fa113563c67c2e894eabe0daea18cf8e33851738c19f70ce86aeb \ + --hash=sha256:5b732c8beef1d7bc2d9e476dbba20aaff6167bf205ad9aa8d30913859e82884b \ + --hash=sha256:5ebeb7ef54a7be11044c33a17b2624abe4307a75893c001a4800857956b41094 \ + --hash=sha256:712a64103d97c404e87d4d7c47fb0c7ff9acccc625ca2002848e0d53288b90ea \ + --hash=sha256:7678556eeb0152cbd1522b684dcd215250885993dd00adb93679ec3c0e6e091c \ + --hash=sha256:77974aba6c1bc26e3c205c2214f0d5b4305bdc719268b93e768ddb17e3fdd636 \ + --hash=sha256:783145835458e60fa97afac25d511d00a1eca94d4a8f3ace9fe2043003c678e4 \ + --hash=sha256:7bfdb06b395385ea9b91bf55c1adf1b297c9fdb531552845ff1d3ea6e40d5aba \ + --hash=sha256:7c8dde0ca2f77828815fd1aedfdf52e59071a5bae30dac3b4da2a335c672149a \ + --hash=sha256:83807d445817326b4bcdaaaf8e8e9f1753da04341eceec705c001ff342002e5d \ + --hash=sha256:87eed225fd415bbae787f93a457af7f5990b92a334e346f72070bf569b9c9c95 \ + --hash=sha256:8fb62fe3d206d72fe1cfe31c4a1106ad2b136fcc1606093aeab314f02930fdf2 \ + --hash=sha256:95172a21038c9b423e68be78fd0be6e1b97674cde269b76fe269a5dfa6fadf0b \ + --hash=sha256:9f48ba6f6c13e5e49f3d3efb1b51c8193215c42ac82610a04624906a9270be6f \ + --hash=sha256:a0c03b6be48aaf92525cccf393265e02773be8fd9551a2f9adbe7db1fa2b60f1 \ + --hash=sha256:a5ae282abe60a2db0fd407072aff4599c279bcd6e9a2475500fc35b00a57c532 \ + --hash=sha256:aee2512827ceb6d7f517c8b85aa5d3923afe8fc7a57d028cffcd522f1c6fd082 \ + --hash=sha256:c8b0451d2ec95010d1db8ca733afc41f659f425b7f608af569711097fd6014e2 \ + --hash=sha256:c9aa4496fd0e17e3843399f533d62857cef5900facf93e735ef65aa4bbc90ef0 \ + --hash=sha256:cbc6472e01952d3d1b2772b720428f8b90e2deea8344e854df22b0618e9cce71 \ + --hash=sha256:cdfe0c22692a30cd830c0755746473ae66c4a8f2e7bd508b35fb3b6a0813d787 \ + --hash=sha256:cf802eef1f0134afb81fef94020351be4fe1d6681aadf9c5e862af6602af64ef \ + --hash=sha256:d42f9c36d06440e34226e8bd65ff065ca0963aeecada587b937011efa02cdc9d \ + --hash=sha256:d5b47c440210c5d1d67e1cf434124e0b5c395eee1f5806fdd89b553ed1acd0a3 \ + --hash=sha256:d9b4a8148c57ecac25a16b0e11798cbe88edf5237b0df99973687dd866f05e1b \ + --hash=sha256:daf43a3d1ea699402c5a850e5313680ac355b4adc9770cd5cfc2940e7861f1bf \ + --hash=sha256:dbdc15f0c81611925f382dfa97b3bd0bc2c1ce19d4fe50482cb0ddc12ba30020 \ + --hash=sha256:deaa09cd492e24fd9b15296844c0ad1b3c976da7907e1c1ed3a0ad21dded6f76 \ + --hash=sha256:e37242f5324ffd9f7ba5acf96d774f9276aa62a966c0bad8dae692deebec7716 \ + --hash=sha256:ed2cf9ed4e8ebc3b754d398cba12f24359f018b416c380f577bbae112ca52fc9 \ + --hash=sha256:f2712c5179f40af9ddc8f6727f2bd910ea0eb50206daea75f58ddd9fa3f715bb \ + --hash=sha256:f4ca91d61a4bf61b0f2228f24bbfa6a9facd5f8af03759fe2a655c50ae2c6610 \ + --hash=sha256:f6b3dfc7661f8842babd8ea07e9897fe3d9b69a1d7e5fbb743e4160f9387833b + # via + # -r requirements.in + # chex + # jax + # jaxlib + # ml-dtypes + # optax + # orbax-checkpoint + # scipy + # tensorstore + # treescope +opt-einsum==3.4.0 \ + --hash=sha256:69bb92469f86a1565195ece4ac0323943e83477171b91d24c35afe028a90d7cd \ + --hash=sha256:96ca72f1b886d148241348783498194c577fa30a8faac108586b14f1ba4473ac + # via + # -r requirements.in + # jax +optax==0.2.4 \ + --hash=sha256:4e05d3d5307e6dde4c319187ae36e6cd3a0c035d4ed25e9e992449a304f47336 \ + --hash=sha256:db35c04e50b52596662efb002334de08c2a0a74971e4da33f467e84fac08886a + # via + # -r requirements.in + # flax +orbax-checkpoint==0.11.6 \ + --hash=sha256:e16a8bbabe7bc0c94f611d115b2b7790183e6847152804a261048160b81b9628 \ + --hash=sha256:fb208012e5d3601ee37b1100fe4331f9982b814df89f572749be9094fa499e1f + # via + # -r requirements.in + # flax +protobuf==5.29.6 \ + --hash=sha256:36ade6ff88212e91aef4e687a971a11d7d24d6948a66751abc1b3238648f5d05 \ + --hash=sha256:62e8a3114992c7c647bce37dcc93647575fc52d50e48de30c6fcb28a6a291eb1 \ + --hash=sha256:6b9edb641441b2da9fa8f428760fc136a49cf97a52076010cf22a2ff73438a86 \ + --hash=sha256:76e07e6567f8baf827137e8d5b8204b6c7b6488bbbff1bf0a72b383f77999c18 \ + --hash=sha256:7e6ad413275be172f67fdee0f43484b6de5a904cc1c3ea9804cb6fe2ff366eda \ + --hash=sha256:831e2da16b6cc9d8f1654c041dd594eda43391affd3c03a91bea7f7f6da106d6 \ + --hash=sha256:a8866b2cff111f0f863c1b3b9e7572dc7eaea23a7fae27f6fc613304046483e6 \ + --hash=sha256:b5a169e664b4057183a34bdc424540e86eea47560f3c123a0d64de4e137f9269 \ + --hash=sha256:cb4c86de9cd8a7f3a256b9744220d87b847371c6b2f10bde87768918ef33ba49 \ + --hash=sha256:da9ee6a5424b6b30fd5e45c5ea663aef540ca95f9ad99d1e887e819cdf9b8723 \ + --hash=sha256:e3387f44798ac1106af0233c04fb8abf543772ff241169946f698b3a9a3d3ab9 + # via + # -r requirements.in + # orbax-checkpoint +pygments==2.20.0 \ + --hash=sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f \ + --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 + # via + # -r requirements.in + # rich +pyyaml==6.0.2 \ + --hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff \ + --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 \ + --hash=sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086 \ + --hash=sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e \ + --hash=sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133 \ + --hash=sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5 \ + --hash=sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484 \ + --hash=sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee \ + --hash=sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5 \ + --hash=sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68 \ + --hash=sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a \ + --hash=sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf \ + --hash=sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99 \ + --hash=sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8 \ + --hash=sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85 \ + --hash=sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19 \ + --hash=sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc \ + --hash=sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a \ + --hash=sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1 \ + --hash=sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317 \ + --hash=sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c \ + --hash=sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631 \ + --hash=sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d \ + --hash=sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652 \ + --hash=sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5 \ + --hash=sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e \ + --hash=sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b \ + --hash=sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8 \ + --hash=sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 \ + --hash=sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706 \ + --hash=sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563 \ + --hash=sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237 \ + --hash=sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b \ + --hash=sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083 \ + --hash=sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180 \ + --hash=sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425 \ + --hash=sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e \ + --hash=sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f \ + --hash=sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725 \ + --hash=sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183 \ + --hash=sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab \ + --hash=sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774 \ + --hash=sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725 \ + --hash=sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e \ + --hash=sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5 \ + --hash=sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d \ + --hash=sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290 \ + --hash=sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44 \ + --hash=sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed \ + --hash=sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4 \ + --hash=sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba \ + --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 \ + --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4 + # via + # -r requirements.in + # flax + # orbax-checkpoint +rich==13.9.4 \ + --hash=sha256:439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098 \ + --hash=sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90 + # via + # -r requirements.in + # flax +scipy==1.15.2 \ + --hash=sha256:01edfac9f0798ad6b46d9c4c9ca0e0ad23dbf0b1eb70e96adb9fa7f525eff0bf \ + --hash=sha256:03205d57a28e18dfd39f0377d5002725bf1f19a46f444108c29bdb246b6c8a11 \ + --hash=sha256:08b57a9336b8e79b305a143c3655cc5bdbe6d5ece3378578888d2afbb51c4e37 \ + --hash=sha256:11e7ad32cf184b74380f43d3c0a706f49358b904fa7d5345f16ddf993609184d \ + --hash=sha256:28a0d2c2075946346e4408b211240764759e0fabaeb08d871639b5f3b1aca8a0 \ + --hash=sha256:2b871df1fe1a3ba85d90e22742b93584f8d2b8e6124f8372ab15c71b73e428b8 \ + --hash=sha256:302093e7dfb120e55515936cb55618ee0b895f8bcaf18ff81eca086c17bd80af \ + --hash=sha256:42dabaaa798e987c425ed76062794e93a243be8f0f20fff6e7a89f4d61cb3d40 \ + --hash=sha256:447ce30cee6a9d5d1379087c9e474628dab3db4a67484be1b7dc3196bfb2fac9 \ + --hash=sha256:4c6676490ad76d1c2894d77f976144b41bd1a4052107902238047fb6a473e971 \ + --hash=sha256:54c462098484e7466362a9f1672d20888f724911a74c22ae35b61f9c5919183d \ + --hash=sha256:597a0c7008b21c035831c39927406c6181bcf8f60a73f36219b69d010aa04737 \ + --hash=sha256:5a6fd6eac1ce74a9f77a7fc724080d507c5812d61e72bd5e4c489b042455865e \ + --hash=sha256:5ea7ed46d437fc52350b028b1d44e002646e28f3e8ddc714011aaf87330f2f32 \ + --hash=sha256:601881dfb761311045b03114c5fe718a12634e5608c3b403737ae463c9885d53 \ + --hash=sha256:62ca1ff3eb513e09ed17a5736929429189adf16d2d740f44e53270cc800ecff1 \ + --hash=sha256:69ea6e56d00977f355c0f84eba69877b6df084516c602d93a33812aa04d90a3d \ + --hash=sha256:6a8e34cf4c188b6dd004654f88586d78f95639e48a25dfae9c5e34a6dc34547e \ + --hash=sha256:6d0194c37037707b2afa7a2f2a924cf7bac3dc292d51b6a925e5fcb89bc5c776 \ + --hash=sha256:6f223753c6ea76983af380787611ae1291e3ceb23917393079dcc746ba60cfb5 \ + --hash=sha256:6f5e296ec63c5da6ba6fa0343ea73fd51b8b3e1a300b0a8cae3ed4b1122c7462 \ + --hash=sha256:7cd5b77413e1855351cdde594eca99c1f4a588c2d63711388b6a1f1c01f62274 \ + --hash=sha256:869269b767d5ee7ea6991ed7e22b3ca1f22de73ab9a49c44bad338b725603301 \ + --hash=sha256:87994da02e73549dfecaed9e09a4f9d58a045a053865679aeb8d6d43747d4df3 \ + --hash=sha256:888307125ea0c4466287191e5606a2c910963405ce9671448ff9c81c53f85f58 \ + --hash=sha256:92233b2df6938147be6fa8824b8136f29a18f016ecde986666be5f4d686a91a4 \ + --hash=sha256:9412f5e408b397ff5641080ed1e798623dbe1ec0d78e72c9eca8992976fa65aa \ + --hash=sha256:9b18aa747da280664642997e65aab1dd19d0c3d17068a04b3fe34e2559196cb9 \ + --hash=sha256:9de9d1416b3d9e7df9923ab23cd2fe714244af10b763975bea9e4f2e81cebd27 \ + --hash=sha256:a2ec871edaa863e8213ea5df811cd600734f6400b4af272e1c011e69401218e9 \ + --hash=sha256:a5080a79dfb9b78b768cebf3c9dcbc7b665c5875793569f48bf0e2b1d7f68f6f \ + --hash=sha256:a8bf5cb4a25046ac61d38f8d3c3426ec11ebc350246a4642f2f315fe95bda655 \ + --hash=sha256:b09ae80010f52efddb15551025f9016c910296cf70adbf03ce2a8704f3a5ad20 \ + --hash=sha256:b5e025e903b4f166ea03b109bb241355b9c42c279ea694d8864d033727205e65 \ + --hash=sha256:bad78d580270a4d32470563ea86c6590b465cb98f83d760ff5b0990cb5518a93 \ + --hash=sha256:bae43364d600fdc3ac327db99659dcb79e6e7ecd279a75fe1266669d9a652828 \ + --hash=sha256:c4697a10da8f8765bb7c83e24a470da5797e37041edfd77fd95ba3811a47c4fd \ + --hash=sha256:c90ebe8aaa4397eaefa8455a8182b164a6cc1d59ad53f79943f266d99f68687f \ + --hash=sha256:cd58a314d92838f7e6f755c8a2167ead4f27e1fd5c1251fd54289569ef3495ec \ + --hash=sha256:cf72ff559a53a6a6d77bd8eefd12a17995ffa44ad86c77a5df96f533d4e6c6bb \ + --hash=sha256:def751dd08243934c884a3221156d63e15234a3155cf25978b0a668409d45eb6 \ + --hash=sha256:e7c68b6a43259ba0aab737237876e5c2c549a031ddb7abc28c7b47f22e202ded \ + --hash=sha256:ecf797d2d798cf7c838c6d98321061eb3e72a74710e6c40540f0e8087e3b499e \ + --hash=sha256:f031846580d9acccd0044efd1a90e6f4df3a6e12b4b6bd694a7bc03a89892b28 \ + --hash=sha256:fb530e4794fc8ea76a4a21ccb67dea33e5e0e60f07fc38a49e821e1eae3b71a0 \ + --hash=sha256:fe8a9eb875d430d81755472c5ba75e84acc980e4a8f6204d402849234d3017db + # via + # -r requirements.in + # jax + # jaxlib +simplejson==3.20.1 \ + --hash=sha256:000602141d0bddfcff60ea6a6e97d5e10c9db6b17fd2d6c66199fa481b6214bb \ + --hash=sha256:03d7a426e416fe0d3337115f04164cd9427eb4256e843a6b8751cacf70abc832 \ + --hash=sha256:03db8cb64154189a92a7786209f24e391644f3a3fa335658be2df2af1960b8d8 \ + --hash=sha256:03ec618ed65caab48e81e3ed29586236a8e57daef792f1f3bb59504a7e98cd10 \ + --hash=sha256:0821871404a537fd0e22eba240c74c0467c28af6cc435903eca394cfc74a0497 \ + --hash=sha256:1190f9a3ce644fd50ec277ac4a98c0517f532cfebdcc4bd975c0979a9f05e1fb \ + --hash=sha256:15c7de4c88ab2fbcb8781a3b982ef883696736134e20b1210bca43fb42ff1acf \ + --hash=sha256:1b9fd15853b90aec3b1739f4471efbf1ac05066a2c7041bf8db821bb73cd2ddc \ + --hash=sha256:1bd6bfe5678d73fbd5328eea6a35216503796428fc47f1237432522febaf3a0c \ + --hash=sha256:272cc767826e924a6bd369ea3dbf18e166ded29059c7a4d64d21a9a22424b5b5 \ + --hash=sha256:299b1007b8101d50d95bc0db1bf5c38dc372e85b504cf77f596462083ee77e3f \ + --hash=sha256:2b6436c48e64378fa844d8c9e58a5ed0352bbcfd4028369a9b46679b7ab79d2d \ + --hash=sha256:2e671dd62051129185d3a9a92c60101f56cbc174854a1a3dfb69114ebd9e1699 \ + --hash=sha256:325b8c107253d3217e89d7b50c71015b5b31e2433e6c5bf38967b2f80630a8ca \ + --hash=sha256:339f407373325a36b7fd744b688ba5bae0666b5d340ec6d98aebc3014bf3d8ea \ + --hash=sha256:3466d2839fdc83e1af42e07b90bc8ff361c4e8796cd66722a40ba14e458faddd \ + --hash=sha256:391345b4157cc4e120027e013bd35c45e2c191e2bf48b8913af488cdc3b9243c \ + --hash=sha256:3c4f0a61cdc05550782ca4a2cdb311ea196c2e6be6b24a09bf71360ca8c3ca9b \ + --hash=sha256:3d7310172d5340febd258cb147f46aae30ad57c445f4d7e1ae8461c10aaf43b0 \ + --hash=sha256:3e7963197d958fcf9e98b212b80977d56c022384621ff463d98afc3b6b1ce7e8 \ + --hash=sha256:455a882ff3f97d810709f7b620007d4e0aca8da71d06fc5c18ba11daf1c4df49 \ + --hash=sha256:463f1fca8fbf23d088e5850fdd0dd4d5faea8900a9f9680270bd98fd649814ca \ + --hash=sha256:4762e05577955312a4c6802f58dd02e040cc79ae59cda510aa1564d84449c102 \ + --hash=sha256:489c3a43116082bad56795215786313832ba3991cca1f55838e52a553f451ab6 \ + --hash=sha256:49d059b8363327eee3c94799dd96782314b2dbd7bcc293b4ad48db69d6f4d362 \ + --hash=sha256:4a586ce4f78cec11f22fe55c5bee0f067e803aab9bad3441afe2181693b5ebb5 \ + --hash=sha256:4a8e197e4cf6d42c2c57e7c52cd7c1e7b3e37c5911df1314fb393320131e2101 \ + --hash=sha256:4a92e948bad8df7fa900ba2ba0667a98303f3db206cbaac574935c332838208e \ + --hash=sha256:51b41f284d603c4380732d7d619f8b34bd04bc4aa0ed0ed5f4ffd0539b14da44 \ + --hash=sha256:5c0de368f3052a59a1acf21f8b2dd28686a9e4eba2da7efae7ed9554cb31e7bc \ + --hash=sha256:627d4486a1ea7edf1f66bb044ace1ce6b4c1698acd1b05353c97ba4864ea2e17 \ + --hash=sha256:652d8eecbb9a3b6461b21ec7cf11fd0acbab144e45e600c817ecf18e4580b99e \ + --hash=sha256:69dd28d4ce38390ea4aaf212902712c0fd1093dc4c1ff67e09687c3c3e15a749 \ + --hash=sha256:6a6dd11ee282937ad749da6f3b8d87952ad585b26e5edfa10da3ae2536c73078 \ + --hash=sha256:6bd09c8c75666e7f62a33d2f1fb57f81da1fcbb19a9fe7d7910b5756e1dd6048 \ + --hash=sha256:6c21f5c026ca633cfffcb6bc1fac2e99f65cb2b24657d3bef21aed9916cc3bbf \ + --hash=sha256:6d4f320c33277a5b715db5bf5b10dae10c19076bd6d66c2843e04bd12d1f1ea5 \ + --hash=sha256:6dd3a1d5aca87bf947f3339b0f8e8e329f1badf548bdbff37fac63c17936da8e \ + --hash=sha256:6e18345c8dda5d699be8166b61f9d80aaee4545b709f1363f60813dc032dac53 \ + --hash=sha256:6e6697a3067d281f01de0fe96fc7cba4ea870d96d7deb7bfcf85186d74456503 \ + --hash=sha256:71b75d448fd0ceb2e7c90e72bb82c41f8462550d48529980bc0bab1d2495bfbb \ + --hash=sha256:71e849e7ceb2178344998cbe5ade101f1b329460243c79c27fbfc51c0447a7c3 \ + --hash=sha256:74a1608f9e6e8c27a4008d70a54270868306d80ed48c9df7872f9f4b8ac87808 \ + --hash=sha256:7551682b60bba3a9e2780742e101cf0a64250e76de7d09b1c4b0c8a7c7cc6834 \ + --hash=sha256:76461ec929282dde4a08061071a47281ad939d0202dc4e63cdd135844e162fbc \ + --hash=sha256:78520f04b7548a5e476b5396c0847e066f1e0a4c0c5e920da1ad65e95f410b11 \ + --hash=sha256:7ceed598e4bacbf5133fe7a418f7991bb2df0683f3ac11fbf9e36a2bc7aa4b85 \ + --hash=sha256:7e9d73f46119240e4f4f07868241749d67d09873f40cb968d639aa9ccc488b86 \ + --hash=sha256:7eaae2b88eb5da53caaffdfa50e2e12022553949b88c0df4f9a9663609373f72 \ + --hash=sha256:87fc623d457173a0213bc9ca4e346b83c9d443f63ed5cca847fb0cacea3cfc95 \ + --hash=sha256:884e6183d16b725e113b83a6fc0230152ab6627d4d36cb05c89c2c5bccfa7bc6 \ + --hash=sha256:88a7baa8211089b9e58d78fbc1b0b322103f3f3d459ff16f03a36cece0d0fcf0 \ + --hash=sha256:896a6c04d7861d507d800da7642479c3547060bf97419d9ef73d98ced8258766 \ + --hash=sha256:8a6c1bbac39fa4a79f83cbf1df6ccd8ff7069582a9fd8db1e52cea073bc2c697 \ + --hash=sha256:8bb98fdf318c05aefd08a92583bd6ee148e93c6756fb1befb7b2d5f27824be78 \ + --hash=sha256:8c09948f1a486a89251ee3a67c9f8c969b379f6ffff1a6064b41fea3bce0a112 \ + --hash=sha256:8d23b7f8d6b72319d6d55a0261089ff621ce87e54731c2d3de6a9bf7be5c028c \ + --hash=sha256:90b573693d1526bed576f6817e2a492eaaef68f088b57d7a9e83d122bbb49e51 \ + --hash=sha256:9a74e70818818981294b8e6956ce3496c5e1bd4726ac864fae473197671f7b85 \ + --hash=sha256:9c079606f461a6e950099167e21e13985147c8a24be8eea66c9ad68f73fad744 \ + --hash=sha256:9daf8cdc7ee8a9e9f7a3b313ba0a003391857e90d0e82fbcd4d614aa05cb7c3b \ + --hash=sha256:9e8eacf6a3491bf76ea91a8d46726368a6be0eb94993f60b8583550baae9439e \ + --hash=sha256:9faceb68fba27ef17eda306e4cd97a7b4b14fdadca5fbb15790ba8b26ebeec0c \ + --hash=sha256:a2cc4f6486f9f515b62f5831ff1888886619b84fc837de68f26d919ba7bbdcbc \ + --hash=sha256:a3c2df555ee4016148fa192e2b9cd9e60bc1d40769366134882685e90aee2a1e \ + --hash=sha256:a7e15b716d09f318c8cda3e20f82fae81684ce3d3acd1d7770fa3007df1769de \ + --hash=sha256:a8011f1dd1d676befcd4d675ebdbfdbbefd3bf350052b956ba8c699fca7d8cef \ + --hash=sha256:ab19c2da8c043607bde4d4ef3a6b633e668a7d2e3d56f40a476a74c5ea71949f \ + --hash=sha256:ab980fcc446ab87ea0879edad41a5c28f2d86020014eb035cf5161e8de4474c6 \ + --hash=sha256:ae6e637dc24f8fee332ed23dd070e81394138e42cd4fd9d0923e5045ba122e27 \ + --hash=sha256:ae81e482476eaa088ef9d0120ae5345de924f23962c0c1e20abbdff597631f87 \ + --hash=sha256:af8377a8af78226e82e3a4349efdde59ffa421ae88be67e18cef915e4023a595 \ + --hash=sha256:b122a19b552b212fc3b5b96fc5ce92333d4a9ac0a800803e1f17ebb16dac4be5 \ + --hash=sha256:b2578bedaedf6294415197b267d4ef678fea336dd78ee2a6d2f4b028e9d07be3 \ + --hash=sha256:b63fdbab29dc3868d6f009a59797cefaba315fd43cd32ddd998ee1da28e50e29 \ + --hash=sha256:bd9577ec1c8c3a43040e3787711e4c257c70035b7551a21854b5dec88dad09e1 \ + --hash=sha256:c02f4868a3a46ffe284a51a88d134dc96feff6079a7115164885331a1ba8ed9f \ + --hash=sha256:c1336ba7bcb722ad487cd265701ff0583c0bb6de638364ca947bb84ecc0015d1 \ + --hash=sha256:c6fdcc9debb711ddd2ad6d69f9386a3d9e8e253234bbb30513e0a7caa9510c51 \ + --hash=sha256:c7edf279c1376f28bf41e916c015a2a08896597869d57d621f55b6a30c7e1e6d \ + --hash=sha256:c939a1e576bded47d7d03aa2afc2ae90b928b2cf1d9dc2070ceec51fd463f430 \ + --hash=sha256:cbbd7b215ad4fc6f058b5dd4c26ee5c59f72e031dfda3ac183d7968a99e4ca3a \ + --hash=sha256:cd2cdead1d3197f0ff43373cf4730213420523ba48697743e135e26f3d179f38 \ + --hash=sha256:cda5c32a98f392909088111ecec23f2b0d39346ceae1a0fea23ab2d1f84ec21d \ + --hash=sha256:ceab2ce2acdc7fbaa433a93006758db6ba9a659e80c4faa13b80b9d2318e9b17 \ + --hash=sha256:d34d04bf90b4cea7c22d8b19091633908f14a096caa301b24c2f3d85b5068fb8 \ + --hash=sha256:d492ed8e92f3a9f9be829205f44b1d0a89af6582f0cf43e0d129fa477b93fe0c \ + --hash=sha256:d8853c269a4c5146ddca4aa7c70e631795e9d11239d5fedb1c6bbc91ffdebcac \ + --hash=sha256:d9202b9de38f12e99a40addd1a8d508a13c77f46d87ab1f9095f154667f4fe81 \ + --hash=sha256:dfe7a9da5fd2a3499436cd350f31539e0a6ded5da6b5b3d422df016444d65e43 \ + --hash=sha256:e041add470e8f8535cc05509485eb7205729a84441f03b25cde80ad48823792e \ + --hash=sha256:e25b2a0c396f3b84fb89573d07b0e1846ed563eb364f2ea8230ca92b8a8cb786 \ + --hash=sha256:e39eaa57c7757daa25bcd21f976c46be443b73dd6c3da47fe5ce7b7048ccefe2 \ + --hash=sha256:e580aa65d5f6c3bf41b9b4afe74be5d5ddba9576701c107c772d936ea2b5043a \ + --hash=sha256:e64139b4ec4f1f24c142ff7dcafe55a22b811a74d86d66560c8815687143037d \ + --hash=sha256:e66712b17d8425bb7ff8968d4c7c7fd5a2dd7bd63728b28356223c000dd2f91f \ + --hash=sha256:e836fb88902799eac8debc2b642300748f4860a197fa3d9ea502112b6bb8e142 \ + --hash=sha256:e91703a4c5fec53e36875ae426ad785f4120bd1d93b65bed4752eeccd1789e0c \ + --hash=sha256:e975aac6a5acd8b510eba58d5591e10a03e3d16c1cf8a8624ca177491f7230f0 \ + --hash=sha256:ec6a1e0a7aff76f0e008bebfa950188b9c50b58c1885d898145f48fc8e189a56 \ + --hash=sha256:ed6a17fd397f0e2b3ad668fc9e19253ed2e3875ad9086bd7f795c29a3223f4a1 \ + --hash=sha256:ede69c765e9901861ad7c6139023b7b7d5807c48a2539d817b4ab40018002d5f \ + --hash=sha256:eea7e2b7d858f6fdfbf0fe3cb846d6bd8a45446865bc09960e51f3d473c2271b \ + --hash=sha256:efd3bc6c6b17e3d4620eb6be5196f0d1c08b6ce7c3101fa8e292b79e0908944b \ + --hash=sha256:f31c4a3a7ab18467ee73a27f3e59158255d1520f3aad74315edde7a940f1be23 \ + --hash=sha256:f4bd49ecde87b0fe9f55cc971449a32832bca9910821f7072bbfae1155eaa007 \ + --hash=sha256:f5272b5866b259fe6c33c4a8c5073bf8b359c3c97b70c298a2f09a69b52c7c41 \ + --hash=sha256:f5aee2a4cb6b146bd17333ac623610f069f34e8f31d2f4f0c1a2186e50c594f0 \ + --hash=sha256:f924b485537b640dc69434565463fd6fc0c68c65a8c6e01a823dd26c9983cf79 \ + --hash=sha256:fc0f523ce923e7f38eb67804bc80e0a028c76d7868500aa3f59225574b5d0453 + # via + # -r requirements.in + # orbax-checkpoint +tensorstore==0.1.72 \ + --hash=sha256:08c5318535aac5e20e247c6e9b43f5887b2293f548de7279650bc73804ccf3ed \ + --hash=sha256:0cd951e593a17babbbde1410cfadb4a04e1cddfa5ace0de5ccb41029223f96b9 \ + --hash=sha256:170172b698fefb4b5507c6cb339ca0b75d56d12ba6a43d9569c61800c1eeb121 \ + --hash=sha256:2fdfa0118be0721c110bcbe7e464758f78d3e14ee8c30a911eb8f4465e6c2e81 \ + --hash=sha256:4a6825cdb6751663ca0bd9abd528ea354ad2199f549bf1f36feac79a6c06efe2 \ + --hash=sha256:599cc7b26b0c96373e89ff5bcf9b76e832802169229680bef985b10011f9bae7 \ + --hash=sha256:5d410c879dc4b34036ec38e20ff05c7e3b0ad5d1eb595412b27a9dbb5e435035 \ + --hash=sha256:5ed6fe937b0433b573c3d6805d0759d33ccc24aa2aba720e4b8ba689c2f9775f \ + --hash=sha256:66c0658689243af0825fff222fb56fdf05a8553bcb3b471dbf18830161302986 \ + --hash=sha256:721d599db0113d75ab6ba1365989bbaf2ab752d7a6268f975c8bfd3a8eb6084b \ + --hash=sha256:763d7f6898711783f199c8226a9c0b259546f5c6d9b4dc0ad3c9e39627060022 \ + --hash=sha256:7c9413f8318a4fa259ec5325f569c0759bccee936df44bd2f7bb35c8afdcdfc8 \ + --hash=sha256:9113d3fcf78c1366688aa90ee7efdc86b57962ea72276944cc57e916a6180749 \ + --hash=sha256:92fac5e2cbc90e5ca8fc72c5bf112816d981e266a3cf9fb1681ba8b3f59537ef \ + --hash=sha256:9c3a36f681ffcc104ba931d471447e8901e64e8cc6913b61792870ff59529961 \ + --hash=sha256:a41b4fe0603943d23472619a8ada70b8d2c9458747fad88b0ce7b29f1ccf4e74 \ + --hash=sha256:a7e7b02da26ca5c95b3c613efd0fe10c082dfa4dc3e9818fefc69e30fe70ea1e \ + --hash=sha256:b71134b85f540e17a1ae65da1fb906781b7470ef0ed71d98d29459325897f574 \ + --hash=sha256:c0f722218f494b1631dbec451b9863f579054e27da2f39aab418db4493694abe \ + --hash=sha256:d5dced3f367308e9fa8e7b72e9e57a4c491fa47c066e035ac33421e2b2408e3f \ + --hash=sha256:ed916b9aeca242a3f367679f65ba376149251ebb28b873becd76c73b688399b6 + # via + # -r requirements.in + # flax + # orbax-checkpoint +toolz==1.0.0 \ + --hash=sha256:292c8f1c4e7516bf9086f8850935c799a874039c8bcf959d47b600e4c44a6236 \ + --hash=sha256:2c86e3d9a04798ac556793bced838816296a2f085017664e4995cb40a1047a02 + # via + # -r requirements.in + # chex +treescope==0.1.9 \ + --hash=sha256:68677013a9f0228212fccf835f3fb037be07ae8b4c5f6f58eefab11198f83cf7 \ + --hash=sha256:ba6cdbdc9c5b52691d5f3bb4c5d5c7daa5627119acac8640b46d37e6aabe63a6 + # via + # -r requirements.in + # flax +typing-extensions==4.12.2 \ + --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ + --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 + # via + # -r requirements.in + # chex + # etils + # flax + # orbax-checkpoint + # rich +zipp==3.21.0 \ + --hash=sha256:2c9958f6430a2040341a52eb608ed6dd93ef4392e02ffe219417c1b28b5dd1f4 \ + --hash=sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931 + # via + # -r requirements.in + # etils diff --git a/perfmetrics/scripts/ml_tests/checkpoint/Jax/run_checkpoints.sh b/perfmetrics/scripts/ml_tests/checkpoint/Jax/run_checkpoints.sh new file mode 100755 index 0000000000..4d7dd6af93 --- /dev/null +++ b/perfmetrics/scripts/ml_tests/checkpoint/Jax/run_checkpoints.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Fail on any error. +set -e +set -x +echo "Running JAX checkpoint tests" + +sudo apt-get update +# Install Git. +echo "Installing git" +sudo apt-get install git +# Install Golang. +version=$(cat "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse/.go-version") +architecture=$(dpkg --print-architecture) +wget -O go_tar.tar.gz https://go.dev/dl/go"${version}".linux-"${architecture}".tar.gz -q +sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf go_tar.tar.gz && rm go_tar.tar.gz +export PATH=$PATH:/usr/local/go/bin + +# Build gcsfuse. +cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse" +# Install latest gcloud version for compatability with HNS bucket. +./perfmetrics/scripts/install_latest_gcloud.sh +export PATH="/usr/local/google-cloud-sdk/bin:$PATH" +export CLOUDSDK_PYTHON="$HOME/.local/python-3.11.9/bin/python3.11" +export PATH="$HOME/.local/python-3.11.9/bin:$PATH" +echo "PATH:" $PATH +echo "CLOUDSDK_PYTHON:" $CLOUDSDK_PYTHON + +CGO_ENABLED=0 go build . + +function mount_gcsfuse_and_run_test() { + # Function to mount GCSFuse. + # Input: + # $1: Bucket name + + local BUCKET_NAME="$1" + # Clean up bucket before run (ignoring the failure if there are no objects to delete). + gcloud alpha storage rm -r gs://${BUCKET_NAME}/** || true + # Create a directory for gcsfuse logs. + mkdir -p "${KOKORO_ARTIFACTS_DIR}/gcsfuse_logs" + local MOUNT_POINT="${HOME}/gcs/${BUCKET_NAME}" + mkdir -p "${MOUNT_POINT}" + + COMMON_FLAGS=(--log-severity=TRACE --enable-streaming-writes --log-file="${KOKORO_ARTIFACTS_DIR}"/gcsfuse_logs/"${BUCKET_NAME}".log) + if [[ "$BUCKET_NAME" =~ "flat" ]]; then + go run . "${COMMON_FLAGS[@]}" --rename-dir-limit=100 "${BUCKET_NAME}" "${MOUNT_POINT}" + else + go run . "${COMMON_FLAGS[@]}" "${BUCKET_NAME}" "${MOUNT_POINT}" + fi + python3.11 ./perfmetrics/scripts/ml_tests/checkpoint/Jax/emulated_checkpoints.py --checkpoint_dir "${MOUNT_POINT}" +} + +# Install pip +curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py +python3.11 get-pip.py +rm get-pip.py +python3.11 -m venv .venv +source .venv/bin/activate +# Install JAX dependencies. +pip install --require-hashes -r ./perfmetrics/scripts/ml_tests/checkpoint/Jax/requirements.txt + +ZONE=$(curl -s -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/zone | cut -d'/' -f4) +# Run tests in parallel on flat, hns and zonal bucket. +FLAT_BUCKET_NAME="jax-emulated-checkpoint-flat-${architecture}" +HNS_BUCKET_NAME="jax-emulated-checkpoint-hns-${architecture}" +ZONAL_BUCKET_NAME="jax-emulated-checkpoint-zonal-${ZONE}-${architecture}" +mount_gcsfuse_and_run_test "${FLAT_BUCKET_NAME}" & +flat_pid=$! +mount_gcsfuse_and_run_test "${HNS_BUCKET_NAME}" & +hns_pid=$! +mount_gcsfuse_and_run_test "${ZONAL_BUCKET_NAME}" & +zonal_pid=$! + +# Wait for all processes to finish and check exit codes +wait "$flat_pid" +flat_status=$? +wait "$hns_pid" +hns_status=$? +wait "$zonal_pid" +zonal_status=$? + +if [[ "$flat_status" -ne 0 ]] || [[ "$hns_status" -ne 0 ]] || [[ "$zonal_status" -ne 0 ]]; then + echo "Checkpoint tests failed" + exit 1 +else + echo "Checkpoint tests completed successfully" +fi diff --git a/perfmetrics/scripts/ml_tests/pytorch/README-usage.md b/perfmetrics/scripts/ml_tests/pytorch/README-usage.md deleted file mode 100644 index 014eef20c2..0000000000 --- a/perfmetrics/scripts/ml_tests/pytorch/README-usage.md +++ /dev/null @@ -1,45 +0,0 @@ -# Execution of Pytorch DINO Model - -As an automation, we run the pytorch dino model in a docker container. In docker, -we use word host to specify the actual VM and container to specify running -docker image. Please find the description of all involved scripts in this -automation with their purpose: - -## File Descriptions: - -### File: perfmetrics/scripts/ml_test/setup_host.sh -By executing this script, we setup the host machine by installing the ops-agent, -docker system, nvidia-driver (gpu based ml training), and some utilities like, -curl, ca-certificates, lsb-release etc. - -### File: perfmetrics/scripts/ml_test/pytorch/dino/setup_container.sh -This script contains the instruction to install gcsfuse, mount GCS-bucket -using gcsfuse, and finally runs the pytorch dino model. - -### File: perfmetrics/scripts/continuous_test/pytorch/{v1_12 or v2}/dino/build.sh -This is the parent script of the above two scripts. Firstly, it sets-up the host -machine after that it creates the docker-image and finally it runs the container -with the inststructions written in the setup_container.sh. - -## Artifacts after the Executions: -After the execution of kokoro job, we copy two types of logs - -(a) GCSFuse logs -(b) Dino model logs. - -### GCSFuse Logs: container_artifacts/gcsfuse_logs -We mount the gcsfuse with debug flags, this folder contains the running gcsfuse -logs. This will be beneficial for debugging purpose. - -### Dino Model Logs: container_artifacts/dino-experiment/ -checkpoint*.pth - Model checkpointing. -log.txt - Contains the model learning parameter value after each epoch. - -### Steps to run the model on VM -1. Create an A2 GPU instance with 8 GPU on GCP console. -2. Create a Working directory, and sets the KOKORO_ARTIFACTS_DIR environment -variable - with current working directory. -3. Create a folder named "github" and clone the gcsfuse repo in that. -4. Run the below script in the current working directory: - **source github/gcsfuse/permetrics/scripts/continuous_test/ml_tests/pytorch/{v1_12 or v2}/dino/build.sh** -5. The above command first setups the host and then start running the model -inside container. diff --git a/perfmetrics/scripts/ml_tests/pytorch/run_container.sh b/perfmetrics/scripts/ml_tests/pytorch/run_container.sh deleted file mode 100644 index cbe793ed15..0000000000 --- a/perfmetrics/scripts/ml_tests/pytorch/run_container.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -# pytorch version (e.g. v1_12, v2) -PYTORCH_VERSION=$1 -BUCKET_TYPE=$2 - -cd "$HOME/github/gcsfuse" -echo "Building docker image containing all pytorch libraries..." -sudo docker build . -f perfmetrics/scripts/ml_tests/pytorch/${PYTORCH_VERSION}/dino/Dockerfile --tag pytorch-gcsfuse --build-arg PYTORCH_VERSION="${PYTORCH_VERSION}" --build-arg BUCKET_TYPE="${BUCKET_TYPE}" - -mkdir -p container_artifacts - -echo "Running the docker image build in the previous step..." -sudo docker run --gpus all --name=pytorch_automation_container --privileged -d -v $HOME/github/gcsfuse/container_artifacts:/pytorch_dino/run_artifacts:rw,rshared \ ---shm-size=128g pytorch-gcsfuse:latest - -# Wait for the script completion as well as logs output. -sudo docker logs -f pytorch_automation_container diff --git a/perfmetrics/scripts/ml_tests/pytorch/run_model.sh b/perfmetrics/scripts/ml_tests/pytorch/run_model.sh deleted file mode 100755 index cdd2689f94..0000000000 --- a/perfmetrics/scripts/ml_tests/pytorch/run_model.sh +++ /dev/null @@ -1,247 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -PYTORCH_VERSION=$1 -BUCKET_TYPE=$2 - -NUM_EPOCHS=80 -TEST_BUCKET="gcsfuse-ml-data" - -# Install golang -wget -O go_tar.tar.gz https://go.dev/dl/go1.23.3.linux-amd64.tar.gz -q -rm -rf /usr/local/go && tar -C /usr/local -xzf go_tar.tar.gz -export PATH=$PATH:/usr/local/go/bin - -# Clone and build the gcsfuse master branch. -git clone https://github.com/GoogleCloudPlatform/gcsfuse.git -cd gcsfuse -CGO_ENABLED=0 go build . -cd - - -# Create a directory for gcsfuse logs -mkdir run_artifacts/gcsfuse_logs - -# We have created a bucket in the asia-northeast1 region to align with the location of our PyTorch 2.0 VM, which is also in asia-northeast1. -if [ ${PYTORCH_VERSION} == "v2" ]; -then - TEST_BUCKET="gcsfuse-ml-data-asia-northeast1" -fi - -config_filename=/tmp/gcsfuse_config.yaml -cat > $config_filename << EOF -logging: - file-path: run_artifacts/gcsfuse.log - format: text - severity: trace - log-rotate: - max-file-size-mb: 1024 - backup-file-count: 3 - compress: true -metadata-cache: - ttl-secs: 1728000 - stat-cache-max-size-mb: 3200 -EOF - -DIR=${PYTORCH_VERSION} -# Enable the enable-hns flag to run tests on the folder APIs with an HNS bucket. -if [ ${BUCKET_TYPE} == "hns" ]; -then - TEST_BUCKET="gcsfuse-ml-data-hns-central1" - echo "enable-hns: true" >> $config_filename - DIR=${DIR}_${BUCKET_TYPE} -elif [ ${BUCKET_TYPE} == "non-hns" ] && [ ${PYTORCH_VERSION} == "v2" ]; -then - #To validate the gRPC client, we have temporarily changed the non-HNS long haul test. - echo "gcs-connection: - client-protocol: grpc - " >> $config_filename -fi - -echo "Created config-file at "$config_filename - -echo "Mounting GCSFuse..." -nohup /pytorch_dino/gcsfuse/gcsfuse --foreground \ - --stackdriver-export-interval=60s \ - --implicit-dirs \ - --config-file $config_filename \ - $TEST_BUCKET gcsfuse_data > "run_artifacts/gcsfuse.out" 2> "run_artifacts/gcsfuse.err" & - -# Update the pytorch library code to bypass the kernel-cache -echo "Updating the pytorch library code to bypass the kernel-cache..." -echo " -def pil_loader(path: str) -> Image.Image: - fd = os.open(path, os.O_DIRECT) - f = os.fdopen(fd, \"rb\") - img = Image.open(f) - rgb_img = img.convert(\"RGB\") - f.close() - return rgb_img -" > bypassed_code.py - -folder_file="/opt/conda/lib/python3.10/site-packages/torchvision/datasets/folder.py" -x=$(grep -n "def pil_loader(path: str) -> Image.Image:" $folder_file | cut -f1 -d ':') -y=$(grep -n "def accimage_loader(path: str) -> Any:" $folder_file | cut -f1 -d ':') -y=$((y - 2)) -lines="$x,$y" -sed -i "$lines"'d' $folder_file -sed -i "$x"'r bypassed_code.py' $folder_file - -# Fix the caching issue - comes when we run the model first time with 8 -# nproc_per_node - by downloading the model in single thread environment. -python -c 'import torch;torch.hub.list("facebookresearch/xcit:main")' - -# (TulsiShah) TODO: Pytorch 2.0 compile mode has issues (https://github.com/pytorch/pytorch/issues/94599), -# which is fixed in pytorch version 2.1.0 (https://github.com/pytorch/pytorch/pull/100071). -# We'll remove this workaround once we update our Docker image to use Pytorch 2.1.0 or greater version. -if [ ${PYTORCH_VERSION} == "v2" ]; -then - allowed_functions_file="/opt/conda/lib/python3.10/site-packages/torch/_dynamo/allowed_functions.py" - # Update the pytorch library code to bypass the kernel-cache - echo "Updating the pytorch library code to Disallow_in_graph distributed API.." - echo " -def _disallowed_function_ids(): - remove = [ - True, - False, - None, - collections.OrderedDict, - copy.copy, - copy.deepcopy, - inspect.signature, - math.__package__, - torch.__builtins__, - torch.autocast_decrement_nesting, - torch.autocast_increment_nesting, - torch.autograd.grad, - torch.clear_autocast_cache, - torch.cuda.current_device, - torch.cuda.amp.autocast_mode.autocast, - torch.cpu.amp.autocast_mode.autocast, - torch.distributions.constraints.is_dependent, - torch.distributions.normal.Normal, - torch.inference_mode, - torch.set_anomaly_enabled, - torch.set_autocast_cache_enabled, - torch.set_autocast_cpu_dtype, - torch.set_autocast_cpu_enabled, - torch.set_autocast_enabled, - torch.set_autocast_gpu_dtype, - torch.autograd.profiler.profile, - warnings.warn, - torch._C._dynamo.eval_frame.unsupported, - ] - # extract all dtypes from torch - dtypes = [ - obj for obj in torch.__dict__.values() if isinstance(obj, type(torch.float32)) - ] - remove += dtypes - storage = [ - obj - for obj in torch.__dict__.values() - if isinstance(obj, type(torch.FloatStorage)) - ] - remove += storage - - # Distributed APIs don't work well with torch.compile. - if torch.distributed.is_available(): - remove.extend( - torch.distributed.distributed_c10d.dynamo_unsupported_distributed_c10d_ops - ) - - return {id(x) for x in remove} -" > disallowed_function.py - - x=$(grep -n "def _disallowed_function_ids():" $allowed_functions_file | cut -f1 -d ':') - y=$(grep -n "def _allowed_function_ids():" $allowed_functions_file | cut -f1 -d ':') - y=$((y - 3)) - lines="$x,$y" - sed -i "$lines"'d' $allowed_functions_file - sed -i "$x"'r disallowed_function.py' $allowed_functions_file - - distributed_c10d_file="/opt/conda/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py" - echo "# This ops are not friendly to TorchDynamo. So, we decide to disallow these ops -# in FX graph, allowing them to run them on eager, with torch.compile. -dynamo_unsupported_distributed_c10d_ops = [ - all_reduce_multigpu, - recv, - all_gather_object, - all_gather_coalesced, - all_to_all_single, - all_reduce, - gather_object, - all_to_all, - all_reduce_coalesced, - gather, - broadcast_object_list, - barrier, - reduce_multigpu, - scatter, - scatter_object_list, - reduce, - reduce_scatter_multigpu, - all_gather, - broadcast_multigpu, - all_gather_multigpu, - reduce_scatter, - all_gather_into_tensor, - broadcast, - reduce_scatter_tensor, - send, -]" >> $distributed_c10d_file -fi - -ARTIFACTS_BUCKET_PATH="gs://gcsfuse-ml-tests-logs/ci_artifacts/pytorch/${DIR}/dino" -echo "Update status file" -echo "RUNNING" > status.txt -gsutil cp status.txt $ARTIFACTS_BUCKET_PATH/ - -echo "Update start time file" -echo $(date +"%s") > start_time.txt -gsutil cp start_time.txt $ARTIFACTS_BUCKET_PATH/ - -( - set +e - # Run the pytorch Dino model - # We need to run it in foreground mode to make the container running. - echo "Running the pytorch dino model..." - experiment=dino_experiment - torchrun \ - --nproc_per_node=2 dino/main_dino.py \ - --arch vit_small \ - --num_workers 20 \ - --data_path gcsfuse_data/imagenet/ILSVRC/Data/CLS-LOC/train/ \ - --output_dir "./run_artifacts/$experiment" \ - --norm_last_layer False \ - --use_fp16 False \ - --clip_grad 0 \ - --epochs $NUM_EPOCHS \ - --global_crops_scale 0.25 1.0 \ - --local_crops_number 10 \ - --local_crops_scale 0.05 0.25 \ - --teacher_temp 0.07 \ - --warmup_teacher_temp_epochs 30 \ - --clip_grad 0 \ - --min_lr 0.00001 - if [ $? -eq 0 ]; - then - echo "Pytorch dino model completed the training successfully!" - echo "COMPLETE" > status.txt - else - echo "Pytorch dino model training failed!" - echo "ERROR" > status.txt - fi -) - -gsutil cp status.txt $ARTIFACTS_BUCKET_PATH/ diff --git a/perfmetrics/scripts/ml_tests/pytorch/v1_12/dino/Dockerfile b/perfmetrics/scripts/ml_tests/pytorch/v1_12/dino/Dockerfile deleted file mode 100644 index 9653d2b3bc..0000000000 --- a/perfmetrics/scripts/ml_tests/pytorch/v1_12/dino/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Image with gcsfuse installed and its package (.deb) -FROM gcr.io/deeplearning-platform-release/pytorch-gpu.1-12 - -# Allow non-root users to specify the allow_other or allow_root mount options -RUN echo "user_allow_other" > /etc/fuse.conf - -RUN pip3 install timm - -WORKDIR "/pytorch_dino/" - -RUN git clone "https://github.com/facebookresearch/dino" - -COPY perfmetrics/scripts/ml_tests/pytorch/run_model.sh ./ - -RUN mkdir -p "run_artifacts" -RUN mkdir -p "gcsfuse_data" - -ARG PYTORCH_VERSION -ARG BUCKET_TYPE -ENV PYTORCH_VERSION=${PYTORCH_VERSION} -ENV BUCKET_TYPE=${BUCKET_TYPE} - -ENTRYPOINT ["/bin/bash", "-c", "./run_model.sh ${PYTORCH_VERSION} ${BUCKET_TYPE}"] diff --git a/perfmetrics/scripts/ml_tests/pytorch/v2/dino/Dockerfile b/perfmetrics/scripts/ml_tests/pytorch/v2/dino/Dockerfile deleted file mode 100644 index b92732b215..0000000000 --- a/perfmetrics/scripts/ml_tests/pytorch/v2/dino/Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Image with gcsfuse installed and its package (.deb) -FROM gcr.io/deeplearning-platform-release/pytorch-gpu.2-0.py310 - -# Allow non-root users to specify the allow_other or allow_root mount options -RUN echo "user_allow_other" > /etc/fuse.conf - -RUN pip3 install timm setuptools==69.5.1 - -WORKDIR "/pytorch_dino/" - -RUN git clone "https://github.com/facebookresearch/dino" - -WORKDIR "/pytorch_dino/dino" -RUN echo '[remote "origin"]' >> .git/config -RUN echo ' fetch = +refs/pull/262/head:refs/remotes/origin/pr/262' >> .git/config - -RUN git fetch origin -RUN git diff origin/main origin/pr/262 > diff.patch -RUN git apply diff.patch - -WORKDIR "/pytorch_dino/" - -COPY perfmetrics/scripts/ml_tests/pytorch/run_model.sh ./ - -RUN mkdir -p "run_artifacts" -RUN mkdir -p "gcsfuse_data" - -ARG PYTORCH_VERSION -ARG BUCKET_TYPE -ENV PYTORCH_VERSION=${PYTORCH_VERSION} -ENV BUCKET_TYPE=${BUCKET_TYPE} - -RUN echo ${BUCKET_TYPE} -ENTRYPOINT ["/bin/bash", "-c", "./run_model.sh ${PYTORCH_VERSION} ${BUCKET_TYPE}"] diff --git a/perfmetrics/scripts/ml_tests/pytorch/v2/dino/setup_host_and_run_container.sh b/perfmetrics/scripts/ml_tests/pytorch/v2/dino/setup_host_and_run_container.sh deleted file mode 100755 index f2f438c16f..0000000000 --- a/perfmetrics/scripts/ml_tests/pytorch/v2/dino/setup_host_and_run_container.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This will stop execution when any command will have non-zero status. -set -e - -cd "$HOME/github/gcsfuse/perfmetrics/scripts" - -echo "Setting up the machine with Docker and Nvidia Driver" -DRIVER_VERSION="550.127.05" -source ml_tests/setup_host.sh $DRIVER_VERSION - - -PYTORCH_VERSION="v2" -BUCKET_TYPE=$1 - -source ml_tests/pytorch/run_container.sh $PYTORCH_VERSION $BUCKET_TYPE diff --git a/perfmetrics/scripts/ml_tests/tf/resnet/Dockerfile b/perfmetrics/scripts/ml_tests/tf/resnet/Dockerfile deleted file mode 100644 index f479bc56cb..0000000000 --- a/perfmetrics/scripts/ml_tests/tf/resnet/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This dockerfile contains deep learning container as base image -# and is used for running tf based resnet18 model on the gpu vms for -# kokoro test - -ARG DLC_IMAGE_NAME - -FROM gcr.io/deeplearning-platform-release/${DLC_IMAGE_NAME} - -RUN sudo apt-get update -RUN sudo apt-get install -y bash ca-certificates - -# Allow non-root users to specify the allow_other or allow_root mount options -RUN echo "user_allow_other" > /etc/fuse.conf - -WORKDIR "/tf_test/" - -COPY ./perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh . -COPY ./perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/resnet_runner.py . - -ARG BUCKET_TYPE -ENV BUCKET_TYPE=${BUCKET_TYPE} - -RUN mkdir -p "myBucket" -ENTRYPOINT ["/bin/bash", "-c", "./setup_container.sh ${BUCKET_TYPE}"] diff --git a/perfmetrics/scripts/ml_tests/tf/resnet/README.md b/perfmetrics/scripts/ml_tests/tf/resnet/README.md deleted file mode 100644 index df1780f78c..0000000000 --- a/perfmetrics/scripts/ml_tests/tf/resnet/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# Kokoro test for tf model reliability using gcsfuse - -This readme contains file descriptions, libraries and model used for the test. - -## Test description - -* We have used tf2.10 framework along with tf-model-garden library (v2.10) -for [resnet18](https://www.tensorflow.org/tfmodels/vision/image_classification) based reliability testing on ImageNet dataset for 3000 epochs -which runs for roughly 14 days. - -## Packages required - -* The [TensorFlow Model Garden](https://github.com/tensorflow/models) is a repository with a number of different -implementations of state-of-the-art (SOTA) models and modeling solutions for TensorFlow users. - -* [Docker engine](https://docs.docker.com/engine/install/ubuntu/) for running deep learning container based image. - -* [Nvidia drivers](https://docs.nvidia.com/datacenter/tesla/tesla-installation-notes/index.html#runfile) for using gpu. -We have used driver version 450.172.01 for experiments. - -## File description - -* build.sh: Entrypoint for kokoro vm. Runs setup_host.sh for installing required nvidia drivers -and docker engine. And starts experiment using setup_scripts/Dockerfile as container image - -* Dockerfile: Uses Deep learning container tf as a base image. - -* setup_container.sh: Entrypoint for the Docker container. Installs gcsfuse and tf-model-garden -library and starts the experiment in the container - -* resnet_runner.py: python script for running resnet18 model using tf-model-garden library. - -## Config changes for running model - -In resnet_runner.py, batch_size can be adjusted on line 34 and number of epochs for the training can be specified in call -to tfm.core.train_lib.run_experiment at line 100 - -## Logging -4 hours of GCSFuse logs with debug flags: --log-severity=TRACE take around 40 GiB of space on disk. -The gcsfuse based logs are stored in directory ${KOKORO_ARTIFACTS_DIR}/github/gcsfuse/container_artifacts/logs -while the gcsfuse errors and output (Mounted successfully) are stored in ${KOKORO_ARTIFACTS_DIR}/github/gcsfuse/container_artifacts/output diff --git a/perfmetrics/scripts/ml_tests/tf/resnet/setup_host_and_run_model.sh b/perfmetrics/scripts/ml_tests/tf/resnet/setup_host_and_run_model.sh deleted file mode 100755 index b84f549c2d..0000000000 --- a/perfmetrics/scripts/ml_tests/tf/resnet/setup_host_and_run_model.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This will stop execution when any command will have non-zero status. -set -e - -BUCKET_TYPE=$1 -cd "$HOME/github/gcsfuse/perfmetrics/scripts" - -echo "Setting up the machine with Docker and Nvidia Driver..." -DRIVER_VERSION="550.127.05" -source ml_tests/setup_host.sh $DRIVER_VERSION - -cd "$HOME/github/gcsfuse/" -mkdir container_artifacts && mkdir container_artifacts/logs && mkdir container_artifacts/output - -echo "Building tf DLC docker image containing all tensorflow libraries..." -sudo docker build . -f perfmetrics/scripts/ml_tests/tf/resnet/Dockerfile -t tf-dlc-gcsfuse --build-arg DLC_IMAGE_NAME=tf-gpu.2-13 --build-arg BUCKET_TYPE="${BUCKET_TYPE}" - -echo "Running the docker image build in the previous step..." -sudo docker run --gpus all --name tf_model_container --privileged -d \ --v $HOME/github/gcsfuse/container_artifacts/logs:/home/logs:rw,rshared \ --v $HOME/github/gcsfuse/container_artifacts/output:/home/output:rw,rshared --shm-size=24g tf-dlc-gcsfuse:latest - -# Wait for the script completion as well as logs output. -sudo docker logs -f tf_model_container diff --git a/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/resnet_runner.py b/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/resnet_runner.py deleted file mode 100644 index 82359ac576..0000000000 --- a/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/resnet_runner.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Python script for running resnet18 model -# Usage: python3 resnet.py - -import pprint -import tempfile -import os - -# from IPython import display -import matplotlib.pyplot as plt - -import tensorflow as tf -import tensorflow_datasets as tfds -import time -import psutil -import tensorflow_models as tfm - -# These are not in the tfm public API for v2.9. They will be available in v2.10 -from official.vision.serving import export_saved_model_lib -import official.core.train_lib - -os.system("sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches'") - -exp_config = tfm.core.exp_factory.get_exp_config('resnet_imagenet') -tfds_name = 'imagenet2012' -ds_info = tfds.builder(tfds_name ).info -print(ds_info) - -# Configure model -exp_config.task.model.num_classes = 1000 -exp_config.task.model.input_size = [224,224,3]#list(ds_info.features["image"].shape) -exp_config.task.model.backbone.resnet.model_id = 18 - -# Configure training and testing data -batch_size = 1024 - -exp_config.task.train_data.input_path = 'myBucket/imagenet2012-tfrecords/train/train*' -exp_config.task.train_data.tfds_split = 'train' -exp_config.task.train_data.global_batch_size = batch_size - -exp_config.task.validation_data.input_path = 'myBucket/imagenet2012-tfrecords/validation/val*' -exp_config.task.validation_data.tfds_split = 'test' -exp_config.task.validation_data.global_batch_size = batch_size - -logical_device_names = [logical_device.name for logical_device in tf.config.list_logical_devices()] - -if 'GPU' in ''.join(logical_device_names): - print('This may be broken in Colab.') - device = 'GPU' -elif 'TPU' in ''.join(logical_device_names): - print('This may be broken in Colab.') - device = 'TPU' -else: - print('Running on CPU is slow, so only train for a few steps.') - device = 'CPU' - -if device=='CPU': - train_steps = 30 - exp_config.trainer.steps_per_loop = 5 -else: - train_steps=1252 - exp_config.trainer.steps_per_loop = 100 - -exp_config.trainer.summary_interval = 100 -exp_config.trainer.checkpoint_interval = train_steps -exp_config.trainer.validation_interval = 1000 -exp_config.trainer.validation_steps = ds_info.splits['test'].num_examples // batch_size -exp_config.trainer.train_steps = train_steps -exp_config.trainer.optimizer_config.learning_rate.type = 'cosine' -exp_config.trainer.optimizer_config.learning_rate.cosine.decay_steps = train_steps -exp_config.trainer.optimizer_config.learning_rate.cosine.initial_learning_rate = 0.1 -exp_config.trainer.optimizer_config.warmup.linear.warmup_steps = 100 - -logical_device_names = [logical_device.name for logical_device in tf.config.list_logical_devices()] - -if exp_config.runtime.mixed_precision_dtype == tf.float16: - tf.keras.mixed_precision.set_global_policy('mixed_float16') - -if 'GPU' in ''.join(logical_device_names): - distribution_strategy = tf.distribute.MirroredStrategy() -elif 'TPU' in ''.join(logical_device_names): - tf.tpu.experimental.initialize_tpu_system() - tpu = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='/device:TPU_SYSTEM:0') - distribution_strategy = tf.distribute.experimental.TPUStrategy(tpu) -else: - print('Warning: this will be really slow.') - distribution_strategy = tf.distribute.OneDeviceStrategy(logical_device_names[0]) - -with distribution_strategy.scope(): - model_dir = tempfile.mkdtemp() - task = tfm.core.task_factory.get_task(exp_config.task, logging_dir=model_dir) - -# Running the model for given number of epochs -model, eval_logs = tfm.core.train_lib.run_experiment( - distribution_strategy=distribution_strategy, - task=task, - mode='train', - params=exp_config, - model_dir=model_dir, - run_post_eval=True, - epochs=675, - clear_kernel_cache=True) diff --git a/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh b/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh deleted file mode 100755 index e5f5ed85d1..0000000000 --- a/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh +++ /dev/null @@ -1,238 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Installs go1.23.3 on the container, builds gcsfuse using log_rotation file -# and installs tf-models-official v2.13.2, makes update to include clear_kernel_cache -# and epochs functionality, and runs the model - -# Install go lang -BUCKET_TYPE=$1 -wget -O go_tar.tar.gz https://go.dev/dl/go1.23.3.linux-amd64.tar.gz -q -sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz && sudo mv go /usr/local -export PATH=$PATH:/usr/local/go/bin - -# Clone the repo and build gcsfuse -git clone "https://github.com/GoogleCloudPlatform/gcsfuse.git" -cd gcsfuse -CGO_ENABLED=0 go build . -cd - - -# Mount the bucket and run in background so that docker doesn't keep running after resnet_runner.py fails -echo "Mounting the bucket" -echo "logging: - file-path: /home/logs/gcsfuse.log - format: text - severity: trace - log-rotate: - max-file-size-mb: 1024 - backup-file-count: 3 - compress: true - " > /tmp/gcsfuse_config.yaml - -TEST_BUCKET="gcsfuse-ml-tf-data" -DIR="resnet" -# Enable the enable-hns flag to run tests on the folder APIs with an HNS bucket. -if [ ${BUCKET_TYPE} == "hns" ]; -then - TEST_BUCKET="gcsfuse-ml-data-hns-central1" - echo "enable-hns: true" >> /tmp/gcsfuse_config.yaml - DIR=${DIR}_${BUCKET_TYPE} -elif [ ${BUCKET_TYPE} == "non-hns" ]; -then - #To validate the gRPC client, we have temporarily changed the non-HNS long haul test. - echo "gcs-connection: - client-protocol: grpc - " >> /tmp/gcsfuse_config.yaml -fi - -nohup gcsfuse/gcsfuse --foreground \ - --implicit-dirs \ - --stackdriver-export-interval 60s \ - --config-file /tmp/gcsfuse_config.yaml \ - $TEST_BUCKET myBucket > /home/output/gcsfuse.out 2> /home/output/gcsfuse.err & - -# Install tensorflow model garden library -pip3 install --user tf-models-official==2.13.2 - -echo "Updating the tensorflow library code to bypass the kernel-cache..." -# Fail building the container image if train_lib.py and controller.py are not at expected location. -if [ -f "/root/.local/lib/python3.10/site-packages/official/core/train_lib.py" ]; then echo "file exists"; else echo "train_lib.py file not present in expected location. Please correct the location. Exiting"; exit 1; fi -if [ -f "/root/.local/lib/python3.10/site-packages/orbit/controller.py" ]; then echo "file exists"; else echo "controller.py file not present in expected location. Please correct the location. Exiting"; exit 1; fi - -# Adding cache clearing functionality and epochs in controller.py -echo " - def train(self, steps: int, checkpoint_at_completion: bool = True, epochs = 1, clear_kernel_cache = False): - \"\"\"Runs training until the specified global step count has been reached. - - This method makes calls to \`self.trainer.train()\` until the global step - count is equal to \`steps\`. It will additionally save checkpoints (if a - \`CheckpointManager\` was passed to \`Controller.__init__\`) and summarize - training output (if \`summary_dir\` is set). - - Args: - steps: The global step count to train up to. - checkpoint_at_completion: Whether to save a checkpoint when this method - returns (regardless of the checkpointing interval). Defaults to \`True\`. - \"\"\" - self._require(\"trainer\", for_method=\"train\") - total_steps = steps - for _ in range(epochs): - # TODO(momernick): Support steps=None or -1 (training to exhaustion). - current_step = self.global_step.numpy() # Cache, since this is expensive. - _log(f\"train | step: {current_step: 6d} | training until step {steps}...\") - while current_step < total_steps: - # Calculates steps to run for the next train loop. - num_steps = min(total_steps - current_step, self.steps_per_loop) - self._train_n_steps(num_steps) - self._maybe_save_checkpoint() - current_step = self.global_step.numpy() - total_steps += steps - - if clear_kernel_cache: - os.system(\"sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches'\") - - if checkpoint_at_completion: - self._maybe_save_checkpoint(check_interval=False) -" > bypassed_code.py - -controller_file="/root/.local/lib/python3.10/site-packages/orbit/controller.py" -x=$(grep -n "def train(self, steps: int, checkpoint_at_completion: bool = True):" $controller_file | cut -f1 -d ':') -y=$(grep -n "def evaluate(self, steps: int = -1)" $controller_file | cut -f1 -d ':') -y=$((y - 2)) -lines="$x,$y" -sed -i "$lines"'d' $controller_file -sed -i "$x"'r bypassed_code.py' $controller_file - -echo " -import os -import time -" > bypassed_code.py - -x=$(grep -n "import time" $controller_file | cut -f1 -d ':') -lines="$x,$x" -sed -i "$lines"'d' $controller_file -sed -i "$x"'r bypassed_code.py' $controller_file - -# Adding params for clear_kernel_cache and epochs in train_lib.py -echo " -def run_experiment( - distribution_strategy: tf.distribute.Strategy, - task: base_task.Task, - mode: str, - params: config_definitions.ExperimentConfig, - model_dir: str, - run_post_eval: bool = False, - save_summary: bool = True, - train_actions: Optional[List[orbit.Action]] = None, - eval_actions: Optional[List[orbit.Action]] = None, - trainer: Optional[base_trainer.Trainer] = None, - controller_cls=orbit.Controller, - epochs: int = 1, - clear_kernel_cache: bool = False -) -> Tuple[tf.keras.Model, Mapping[str, Any]]: - \"\"\"Runs train/eval configured by the experiment params. - - Args: - distribution_strategy: A distribution distribution_strategy. - task: A Task instance. - mode: A 'str', specifying the mode. Can be 'train', 'eval', 'train_and_eval' - or 'continuous_eval'. - params: ExperimentConfig instance. - model_dir: A 'str', a path to store model checkpoints and summaries. - run_post_eval: Whether to run post eval once after training, metrics logs - are returned. - save_summary: Whether to save train and validation summary. - train_actions: Optional list of Orbit train actions. - eval_actions: Optional list of Orbit eval actions. - trainer: the base_trainer.Trainer instance. It should be created within the - strategy.scope(). - controller_cls: The controller class to manage the train and eval process. - Must be a orbit.Controller subclass. - - Returns: - A 2-tuple of (model, eval_logs). - model: \`tf.keras.Model\` instance. - eval_logs: returns eval metrics logs when run_post_eval is set to True, - otherwise, returns {}. - \"\"\" - runner = OrbitExperimentRunner( - distribution_strategy=distribution_strategy, - task=task, - mode=mode, - params=params, - model_dir=model_dir, - run_post_eval=run_post_eval, - save_summary=save_summary, - train_actions=train_actions, - eval_actions=eval_actions, - trainer=trainer, - controller_cls=controller_cls, - ) - return runner.run(epochs=epochs, clear_kernel_cache=clear_kernel_cache) -" > bypassed_code.py - -train_lib_file="/root/.local/lib/python3.10/site-packages/official/core/train_lib.py" -x=$(grep -n "def run_experiment(" $train_lib_file | cut -f1 -d ':') -y=$(grep -n "return runner.run()" $train_lib_file | cut -f1 -d ':') -lines="$x,$y" -sed -i "$lines"'d' $train_lib_file -x=$((x-1)) -sed -i "$x"'r bypassed_code.py' $train_lib_file - -echo " def run(self, epochs=1, clear_kernel_cache=False) -> Tuple[tf.keras.Model, Mapping[str, Any]]:" > bypassed_code.py -x=$(grep -n "def run(self) -> Tuple\[tf.keras.Model, Mapping\[str, Any\]\]:" $train_lib_file | cut -f1 -d ':') -lines="$x,$x" -sed -i "$lines"'d' $train_lib_file -x=$((x-1)) -sed -i "$x"'r bypassed_code.py' $train_lib_file - -echo " - if mode == 'train' or mode == 'train_and_post_eval': - self.controller.train(steps=params.trainer.train_steps, clear_kernel_cache=clear_kernel_cache, epochs=epochs)" > bypassed_code.py -x=$(grep -n "if mode == 'train' or mode == 'train_and_post_eval':" $train_lib_file | cut -f1 -d ':') -y=$(grep -n "self.controller.train(steps=params.trainer.train_steps)" $train_lib_file | cut -f1 -d ':') -lines="$x,$y" -sed -i "$lines"'d' $train_lib_file -x=$((x-1)) -sed -i "$x"'r bypassed_code.py' $train_lib_file - -ARTIFACTS_BUCKET_PATH="gs://gcsfuse-ml-tests-logs/ci_artifacts/tf/${DIR}" -echo "Update status file" -echo "RUNNING" > status.txt -gsutil cp status.txt $ARTIFACTS_BUCKET_PATH/ - -echo "Update start time file" -echo $(date +"%s") > start_time.txt -gsutil cp start_time.txt $ARTIFACTS_BUCKET_PATH/ - -( - set +e - # We need to run it in foreground mode to make the container running. - echo "Running the tensorflow resnet model..." - # Start training the model - python3 -u resnet_runner.py - if [ $? -eq 0 ]; - then - echo "Tensorflow resnet model completed the training successfully!" - echo "COMPLETE" > status.txt - else - echo "Tensorflow resnet model training failed!" - echo "ERROR" > status.txt - fi -) - -gsutil cp status.txt $ARTIFACTS_BUCKET_PATH/ - - diff --git a/perfmetrics/scripts/os_utils.sh b/perfmetrics/scripts/os_utils.sh new file mode 100644 index 0000000000..22f5b6fc43 --- /dev/null +++ b/perfmetrics/scripts/os_utils.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# PREVENT MULTIPLE SOURCING +if [ "${_OS_UTILS_SH_LOADED:-}" = "true" ]; then + return 0 +fi + +_OS_UTILS_SH_LOADED=true + +OS_UTILS_DIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") + +# Detect OS ID from /etc/os-release +get_os_id() { + if [ -f /etc/os-release ]; then + ( . /etc/os-release && echo "$ID" ) + else + echo "Error: /etc/os-release not found. Cannot detect OS." + return 1 + fi +} + +# Detect and map system architecture to Go architecture +get_go_arch() { + local system_arch=$(uname -m) + case "$system_arch" in + x86_64) echo "amd64" ;; + aarch64|arm64) echo "arm64" ;; + *) echo "unsupported" ;; + esac +} + +# Install packages based on OS ID +install_packages_by_os() { + local os_id=$1 + shift + local pkgs=("$@") + + if [ "${#pkgs[@]}" -eq 0 ]; then + return 0 + fi + + case "$os_id" in + ubuntu|debian) + local retry_count=0 + local max_retries=10 + + # Loop to handle the apt lock issue + until sudo apt-get update; do + if [ $retry_count -ge $max_retries ]; then + echo "Error: Could not obtain apt lock after $max_retries attempts." + return 1 + fi + + echo "Waiting for apt lock (Process $(fuser /var/lib/apt/lists/lock 2>/dev/null))..." + sleep 5 + ((retry_count++)) + done + + sudo apt-get install -y "${pkgs[@]}" + ;; + rhel|centos|fedora|almalinux|rocky) + # Map package names for RHEL if necessary + local rhel_pkgs=() + local install_crcmod=false + for pkg in "${pkgs[@]}"; do + if [[ "$pkg" == "python3-dev" ]]; then + rhel_pkgs+=("python3-devel") + elif [[ "$pkg" == "python3-venv" ]]; then + # Skip on RHEL as it's included in base Python + continue + elif [[ "$pkg" == "python3-crcmod" ]]; then + install_crcmod=true + elif [[ "$pkg" == "fuse3" ]]; then + rhel_pkgs+=("fuse") + else + rhel_pkgs+=("$pkg") + fi + done + + # Ensure pip is installed if crcmod needs it + if [ "$install_crcmod" = true ]; then + rhel_pkgs+=("python3-pip") + fi + + sudo yum install -y "${rhel_pkgs[@]}" + + if [ "$install_crcmod" = true ]; then + sudo python3 -m pip install --require-hashes -r "${OS_UTILS_DIR}/../../tools/cd_scripts/requirements.txt" + fi + ;; + arch|manjaro) + # Map package names for Arch + local arch_pkgs=() + for pkg in "${pkgs[@]}"; do + case "$pkg" in + python3|python3-dev) arch_pkgs+=("python") ;; + python3-setuptools) arch_pkgs+=("python-setuptools") ;; + *) arch_pkgs+=("$pkg") ;; + esac + done + sudo pacman -Sy --noconfirm && sudo pacman -S --noconfirm "${arch_pkgs[@]}" + ;; + *) + echo "Error: Unsupported OS ID for package installation: $os_id" + return 1 + ;; + esac +} diff --git a/perfmetrics/scripts/populate_metrics.sh b/perfmetrics/scripts/populate_metrics.sh index 3682bf3661..1be5d94174 100755 --- a/perfmetrics/scripts/populate_metrics.sh +++ b/perfmetrics/scripts/populate_metrics.sh @@ -20,6 +20,6 @@ set -e echo Installing requirements.. pip install --require-hashes -r requirements.txt --user -gsutil cp gs://periodic-perf-tests/creds.json ./gsheet +gcloud storage cp gs://periodic-perf-tests/creds.json ./gsheet echo Fetching results.. python3 populate_vm_metrics.py $1 $2 diff --git a/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh b/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh index 4e4a69ebe7..e3c31ded78 100755 --- a/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh +++ b/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh @@ -13,27 +13,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Running test only for when PR contains execute-perf-test or execute-integration-tests label +# Running test only for when PR contains execute-perf-test, +# execute-integration-tests or execute-checkpoint-test label. readonly EXECUTE_PERF_TEST_LABEL="execute-perf-test" readonly EXECUTE_INTEGRATION_TEST_LABEL="execute-integration-tests" +readonly EXECUTE_INTEGRATION_TEST_LABEL_ON_ZB="execute-integration-tests-on-zb" readonly EXECUTE_PACKAGE_BUILD_TEST_LABEL="execute-package-build-tests" -readonly RUN_E2E_TESTS_ON_INSTALLED_PACKAGE=false -readonly SKIP_NON_ESSENTIAL_TESTS_ON_PACKAGE=false -readonly BUCKET_LOCATION=us-west1 -readonly RUN_TEST_ON_TPC_ENDPOINT=false - -# This flag, if set true, will indicate to underlying script to customize for a presubmit run. -readonly RUN_TESTS_WITH_PRESUBMIT_FLAG=true +readonly EXECUTE_CHECKPOINT_TEST_LABEL="execute-checkpoint-test" +readonly EXECUTE_ORBAX_BENCHMARK_LABEL="execute-orbax-benchmark" +readonly EXECUTE_MACHINE_TYPE_TEST_LABEL="execute-machine-type-test" +readonly BUCKET_LOCATION=us-west4 curl https://api.github.com/repos/GoogleCloudPlatform/gcsfuse/pulls/$KOKORO_GITHUB_PULL_REQUEST_NUMBER >> pr.json perfTest=$(grep "$EXECUTE_PERF_TEST_LABEL" pr.json) -integrationTests=$(grep "$EXECUTE_INTEGRATION_TEST_LABEL" pr.json) +integrationTests=$(grep "\"$EXECUTE_INTEGRATION_TEST_LABEL\"" pr.json) +integrationTestsOnZB=$(grep "\"$EXECUTE_INTEGRATION_TEST_LABEL_ON_ZB\"" pr.json) packageBuildTests=$(grep "$EXECUTE_PACKAGE_BUILD_TEST_LABEL" pr.json) +checkpointTests=$(grep "$EXECUTE_CHECKPOINT_TEST_LABEL" pr.json) +orbaxBenchmarkTest=$(grep "\"$EXECUTE_ORBAX_BENCHMARK_LABEL\"" pr.json) +machineTypeTest=$(grep "\"$EXECUTE_MACHINE_TYPE_TEST_LABEL\"" pr.json) + rm pr.json perfTestStr="$perfTest" integrationTestsStr="$integrationTests" +integrationTestsOnZBStr="$integrationTestsOnZB" packageBuildTestsStr="$packageBuildTests" -if [[ "$perfTestStr" != *"$EXECUTE_PERF_TEST_LABEL"* && "$integrationTestsStr" != *"$EXECUTE_INTEGRATION_TEST_LABEL"* && "$packageBuildTestsStr" != *"$EXECUTE_PACKAGE_BUILD_TEST_LABEL"* ]] +checkpointTestStr="$checkpointTests" +orbaxBenchmarkTestStr="$orbaxBenchmarkTest" +machineTypeTestStr="$machineTypeTest" +if [[ "$perfTestStr" != *"$EXECUTE_PERF_TEST_LABEL"* && "$integrationTestsStr" != *"$EXECUTE_INTEGRATION_TEST_LABEL"* && "$integrationTestsOnZBStr" != *"$EXECUTE_INTEGRATION_TEST_LABEL_ON_ZB"* && "$packageBuildTestsStr" != *"$EXECUTE_PACKAGE_BUILD_TEST_LABEL"* && "$checkpointTestStr" != *"$EXECUTE_CHECKPOINT_TEST_LABEL"* && "$orbaxBenchmarkTestStr" != *"$EXECUTE_ORBAX_BENCHMARK_LABEL"* && "$machineTypeTestStr" != *"$EXECUTE_MACHINE_TYPE_TEST_LABEL"* ]] then echo "No need to execute tests" exit 0 @@ -43,12 +51,16 @@ set -e sudo apt-get update echo Installing git sudo apt-get install git -echo Installing go-lang 1.23.3 -wget -O go_tar.tar.gz https://go.dev/dl/go1.23.3.linux-amd64.tar.gz -q -sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz && sudo mv go /usr/local -export PATH=$PATH:/usr/local/go/bin +# Get the absolute path to the repo root +REPO_ROOT=${KOKORO_ARTIFACTS_DIR}/github/gcsfuse +cd "${REPO_ROOT}" +# Read the version +GO_VERSION=$(cat "${REPO_ROOT}/.go-version") +# Install required go version. +./perfmetrics/scripts/install_go.sh "$GO_VERSION" export CGO_ENABLED=0 -cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse" +export PATH="/usr/local/go/bin:$PATH" + # Fetch PR branch echo '[remote "origin"] fetch = +refs/pull/*/head:refs/remotes/origin/pr/*' >> .git/config @@ -56,7 +68,7 @@ git fetch origin -q function execute_perf_test() { mkdir -p gcs - GCSFUSE_FLAGS="--implicit-dirs" + GCSFUSE_FLAGS="--implicit-dirs --prometheus-port=48341" BUCKET_NAME=presubmit-perf-tests MOUNT_POINT=gcs # The VM will itself exit if the gcsfuse mount fails. @@ -74,14 +86,23 @@ function install_requirements() { echo Installing Bigquery module requirements... pip install --require-hashes -r ./perfmetrics/scripts/bigquery/requirements.txt --user echo Installing libraries to run python script - pip install google-cloud - pip install google-cloud-vision - pip install google-api-python-client - pip install prettytable + pip install --require-hashes -r ./perfmetrics/scripts/presubmit_test/pr_perf_test/requirements.txt --user "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse/perfmetrics/scripts/fio/install_fio.sh" "${KOKORO_ARTIFACTS_DIR}/github" cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse" } +function execute_gke_test() { + local bucket_name=$1 + local script_path=$2 + + echo checkout PR branch + git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER + + export BUCKET_NAME="$bucket_name" + + python3 "$script_path" +} + # execute perf tests. if [[ "$perfTestStr" == *"$EXECUTE_PERF_TEST_LABEL"* ]]; then @@ -105,15 +126,24 @@ then python3 ./perfmetrics/scripts/presubmit/print_results.py fi -# Execute integration tests. -if [[ "$integrationTestsStr" == *"$EXECUTE_INTEGRATION_TEST_LABEL"* ]]; +# Execute integration tests on zonal bucket(s). +if test -n "${integrationTestsOnZBStr}" ; +then + echo checkout PR branch + git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER + + echo "Running e2e tests on zonal bucket(s) ..." + bash ./tools/integration_tests/improved_run_e2e_tests.sh --bucket-location=$BUCKET_LOCATION --presubmit --zonal --track-resource-usage +fi + +# Execute integration tests on non-zonal bucket(s). +if test -n "${integrationTestsStr}" ; then echo checkout PR branch git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER - echo "Running e2e tests...." - # $1 argument is refering to value of testInstalledPackage. - ./tools/integration_tests/run_e2e_tests.sh $RUN_E2E_TESTS_ON_INSTALLED_PACKAGE $SKIP_NON_ESSENTIAL_TESTS_ON_PACKAGE $BUCKET_LOCATION $RUN_TEST_ON_TPC_ENDPOINT $RUN_TESTS_WITH_PRESUBMIT_FLAG + echo "Running e2e tests on non-zonal bucket(s) ..." + bash ./tools/integration_tests/improved_run_e2e_tests.sh --bucket-location=$BUCKET_LOCATION --presubmit --track-resource-usage fi # Execute package build tests. @@ -123,5 +153,29 @@ then git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER echo "Running package build tests...." - ./perfmetrics/scripts/build_and_install_gcsfuse.sh master + ./perfmetrics/scripts/build_and_install_gcsfuse.sh "$(git rev-parse HEAD)" +fi + +# Execute JAX checkpoints tests. +if [[ "$checkpointTestStr" == *"$EXECUTE_CHECKPOINT_TEST_LABEL"* ]]; +then + echo checkout PR branch + git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER + + echo "Running checkpoint tests...." + ./perfmetrics/scripts/ml_tests/checkpoint/Jax/run_checkpoints.sh +fi + +# Execute Orbax benchmark. +if [[ "$orbaxBenchmarkTestStr" == *"$EXECUTE_ORBAX_BENCHMARK_LABEL"* ]]; +then + echo "Running Orbax benchmark..." + execute_gke_test "llama_europe_west4" "perfmetrics/scripts/continuous_test/gke/orbax_benchmark/run_benchmark.py" +fi + +# Execute Machine Type Test. +if [[ "$machineTypeTestStr" == *"$EXECUTE_MACHINE_TYPE_TEST_LABEL"* ]]; +then + echo "Running Machine Type Test..." + execute_gke_test "gcsfuse_gke_machine_type_test_flat_euw4" "perfmetrics/scripts/continuous_test/gke/machine_type_test/run.py" fi diff --git a/perfmetrics/scripts/presubmit_test/pr_perf_test/presubmit.cfg b/perfmetrics/scripts/presubmit_test/pr_perf_test/presubmit.cfg index 69e77006c7..8672372a40 100644 --- a/perfmetrics/scripts/presubmit_test/pr_perf_test/presubmit.cfg +++ b/perfmetrics/scripts/presubmit_test/pr_perf_test/presubmit.cfg @@ -14,6 +14,8 @@ action { define_artifacts { regex: "gcsfuse-failed-integration-test-logs-*" + regex: "gcsfuse_logs/*" + regex: "**/*sponge_log.*" strip_prefix: "github/gcsfuse/perfmetrics/scripts" } } diff --git a/perfmetrics/scripts/presubmit_test/pr_perf_test/requirements.in b/perfmetrics/scripts/presubmit_test/pr_perf_test/requirements.in new file mode 100644 index 0000000000..8feae40dce --- /dev/null +++ b/perfmetrics/scripts/presubmit_test/pr_perf_test/requirements.in @@ -0,0 +1,4 @@ +google-cloud +google-cloud-vision +google-api-python-client +prettytable diff --git a/perfmetrics/scripts/presubmit_test/pr_perf_test/requirements.txt b/perfmetrics/scripts/presubmit_test/pr_perf_test/requirements.txt new file mode 100644 index 0000000000..6674647fb8 --- /dev/null +++ b/perfmetrics/scripts/presubmit_test/pr_perf_test/requirements.txt @@ -0,0 +1,310 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --generate-hashes requirements.in +# +cachetools==6.2.2 \ + --hash=sha256:6c09c98183bf58560c97b2abfcedcbaf6a896a490f534b031b661d3723b45ace \ + --hash=sha256:8e6d266b25e539df852251cfd6f990b4bc3a141db73b939058d809ebd2590fc6 + # via google-auth +certifi==2025.11.12 \ + --hash=sha256:97de8790030bbd5c2d96b7ec782fc2f7820ef8dba6db909ccf95449f2d062d4b \ + --hash=sha256:d8ab5478f2ecd78af242878415affce761ca6bc54a22a27e026d7c25357c3316 + # via requests +charset-normalizer==3.4.4 \ + --hash=sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad \ + --hash=sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93 \ + --hash=sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394 \ + --hash=sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89 \ + --hash=sha256:0f04b14ffe5fdc8c4933862d8306109a2c51e0704acfa35d51598eb45a1e89fc \ + --hash=sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86 \ + --hash=sha256:194f08cbb32dc406d6e1aea671a68be0823673db2832b38405deba2fb0d88f63 \ + --hash=sha256:1bee1e43c28aa63cb16e5c14e582580546b08e535299b8b6158a7c9c768a1f3d \ + --hash=sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f \ + --hash=sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8 \ + --hash=sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0 \ + --hash=sha256:2677acec1a2f8ef614c6888b5b4ae4060cc184174a938ed4e8ef690e15d3e505 \ + --hash=sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161 \ + --hash=sha256:2aaba3b0819274cc41757a1da876f810a3e4d7b6eb25699253a4effef9e8e4af \ + --hash=sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152 \ + --hash=sha256:2c9d3c380143a1fedbff95a312aa798578371eb29da42106a29019368a475318 \ + --hash=sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72 \ + --hash=sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4 \ + --hash=sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e \ + --hash=sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3 \ + --hash=sha256:44c2a8734b333e0578090c4cd6b16f275e07aa6614ca8715e6c038e865e70576 \ + --hash=sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c \ + --hash=sha256:4902828217069c3c5c71094537a8e623f5d097858ac6ca8252f7b4d10b7560f1 \ + --hash=sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8 \ + --hash=sha256:4fe7859a4e3e8457458e2ff592f15ccb02f3da787fcd31e0183879c3ad4692a1 \ + --hash=sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2 \ + --hash=sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44 \ + --hash=sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26 \ + --hash=sha256:5947809c8a2417be3267efc979c47d76a079758166f7d43ef5ae8e9f92751f88 \ + --hash=sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016 \ + --hash=sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede \ + --hash=sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf \ + --hash=sha256:5cb4d72eea50c8868f5288b7f7f33ed276118325c1dfd3957089f6b519e1382a \ + --hash=sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc \ + --hash=sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0 \ + --hash=sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84 \ + --hash=sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db \ + --hash=sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1 \ + --hash=sha256:6aee717dcfead04c6eb1ce3bd29ac1e22663cdea57f943c87d1eab9a025438d7 \ + --hash=sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed \ + --hash=sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8 \ + --hash=sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133 \ + --hash=sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e \ + --hash=sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef \ + --hash=sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14 \ + --hash=sha256:778d2e08eda00f4256d7f672ca9fef386071c9202f5e4607920b86d7803387f2 \ + --hash=sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0 \ + --hash=sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d \ + --hash=sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828 \ + --hash=sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f \ + --hash=sha256:7c308f7e26e4363d79df40ca5b2be1c6ba9f02bdbccfed5abddb7859a6ce72cf \ + --hash=sha256:7fa17817dc5625de8a027cb8b26d9fefa3ea28c8253929b8d6649e705d2835b6 \ + --hash=sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328 \ + --hash=sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090 \ + --hash=sha256:837c2ce8c5a65a2035be9b3569c684358dfbf109fd3b6969630a87535495ceaa \ + --hash=sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381 \ + --hash=sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c \ + --hash=sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb \ + --hash=sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc \ + --hash=sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a \ + --hash=sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec \ + --hash=sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc \ + --hash=sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac \ + --hash=sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e \ + --hash=sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313 \ + --hash=sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569 \ + --hash=sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3 \ + --hash=sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d \ + --hash=sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525 \ + --hash=sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894 \ + --hash=sha256:a8bf8d0f749c5757af2142fe7903a9df1d2e8aa3841559b2bad34b08d0e2bcf3 \ + --hash=sha256:a9768c477b9d7bd54bc0c86dbaebdec6f03306675526c9927c0e8a04e8f94af9 \ + --hash=sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a \ + --hash=sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9 \ + --hash=sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14 \ + --hash=sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25 \ + --hash=sha256:b5d84d37db046c5ca74ee7bb47dd6cbc13f80665fdde3e8040bdd3fb015ecb50 \ + --hash=sha256:b7cf1017d601aa35e6bb650b6ad28652c9cd78ee6caff19f3c28d03e1c80acbf \ + --hash=sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1 \ + --hash=sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3 \ + --hash=sha256:c4ef880e27901b6cc782f1b95f82da9313c0eb95c3af699103088fa0ac3ce9ac \ + --hash=sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e \ + --hash=sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815 \ + --hash=sha256:cb01158d8b88ee68f15949894ccc6712278243d95f344770fa7593fa2d94410c \ + --hash=sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6 \ + --hash=sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6 \ + --hash=sha256:cd09d08005f958f370f539f186d10aec3377d55b9eeb0d796025d4886119d76e \ + --hash=sha256:cd4b7ca9984e5e7985c12bc60a6f173f3c958eae74f3ef6624bb6b26e2abbae4 \ + --hash=sha256:ce8a0633f41a967713a59c4139d29110c07e826d131a316b50ce11b1d79b4f84 \ + --hash=sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69 \ + --hash=sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15 \ + --hash=sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191 \ + --hash=sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0 \ + --hash=sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897 \ + --hash=sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd \ + --hash=sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2 \ + --hash=sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794 \ + --hash=sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d \ + --hash=sha256:e912091979546adf63357d7e2ccff9b44f026c075aeaf25a52d0e95ad2281074 \ + --hash=sha256:eaabd426fe94daf8fd157c32e571c85cb12e66692f15516a83a03264b08d06c3 \ + --hash=sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224 \ + --hash=sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838 \ + --hash=sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a \ + --hash=sha256:f155a433c2ec037d4e8df17d18922c3a0d9b3232a396690f17175d2946f0218d \ + --hash=sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d \ + --hash=sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f \ + --hash=sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8 \ + --hash=sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490 \ + --hash=sha256:f8e160feb2aed042cd657a72acc0b481212ed28b1b9a95c0cee1621b524e1966 \ + --hash=sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9 \ + --hash=sha256:fa09f53c465e532f4d3db095e0c55b615f010ad81803d383195b6b5ca6cbf5f3 \ + --hash=sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e \ + --hash=sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608 + # via requests +google-api-core[grpc]==2.28.1 \ + --hash=sha256:2b405df02d68e68ce0fbc138559e6036559e685159d148ae5861013dc201baf8 \ + --hash=sha256:4021b0f8ceb77a6fb4de6fde4502cecab45062e66ff4f2895169e0b35bc9466c + # via + # google-api-python-client + # google-cloud-vision +google-api-python-client==2.187.0 \ + --hash=sha256:d8d0f6d85d7d1d10bdab32e642312ed572bdc98919f72f831b44b9a9cebba32f \ + --hash=sha256:e98e8e8f49e1b5048c2f8276473d6485febc76c9c47892a8b4d1afa2c9ec8278 + # via -r requirements.in +google-auth==2.43.0 \ + --hash=sha256:88228eee5fc21b62a1b5fe773ca15e67778cb07dc8363adcb4a8827b52d81483 \ + --hash=sha256:af628ba6fa493f75c7e9dbe9373d148ca9f4399b5ea29976519e0a3848eddd16 + # via + # google-api-core + # google-api-python-client + # google-auth-httplib2 + # google-cloud-vision +google-auth-httplib2==0.2.1 \ + --hash=sha256:1be94c611db91c01f9703e7f62b0a59bbd5587a95571c7b6fade510d648bc08b \ + --hash=sha256:5ef03be3927423c87fb69607b42df23a444e434ddb2555b73b3679793187b7de + # via google-api-python-client +google-cloud==0.34.0 \ + --hash=sha256:01430187cf56df10a9ba775dd547393185d4b40741db0ea5889301f8e7a9d5d3 \ + --hash=sha256:fb1ab7b0548fe44b3d538041f0a374505b7f990d448a935ea36649c5ccab5acf + # via -r requirements.in +google-cloud-vision==3.11.0 \ + --hash=sha256:8910f743a87a34058dd6e5e41790be1eb100a0b91c20cc6372a2388b328c8890 \ + --hash=sha256:c3cb57df2cf152ebe62ebaae9b1d5deff5a26aec5bd6e1c7f67e44bf6f4518f4 + # via -r requirements.in +googleapis-common-protos==1.72.0 \ + --hash=sha256:4299c5a82d5ae1a9702ada957347726b167f9f8d1fc352477702a1e851ff4038 \ + --hash=sha256:e55a601c1b32b52d7a3e65f43563e2aa61bcd737998ee672ac9b951cd49319f5 + # via + # google-api-core + # grpcio-status +grpcio==1.76.0 \ + --hash=sha256:035d90bc79eaa4bed83f524331d55e35820725c9fbb00ffa1904d5550ed7ede3 \ + --hash=sha256:04bbe1bfe3a68bbfd4e52402ab7d4eb59d72d02647ae2042204326cf4bbad280 \ + --hash=sha256:063065249d9e7e0782d03d2bca50787f53bd0fb89a67de9a7b521c4a01f1989b \ + --hash=sha256:06c3d6b076e7b593905d04fdba6a0525711b3466f43b3400266f04ff735de0cd \ + --hash=sha256:08caea849a9d3c71a542827d6df9d5a69067b0a1efbea8a855633ff5d9571465 \ + --hash=sha256:0aaa82d0813fd4c8e589fac9b65d7dd88702555f702fb10417f96e2a2a6d4c0f \ + --hash=sha256:0b7604868b38c1bfd5cf72d768aedd7db41d78cb6a4a18585e33fb0f9f2363fd \ + --hash=sha256:0c37db8606c258e2ee0c56b78c62fc9dee0e901b5dbdcf816c2dd4ad652b8b0c \ + --hash=sha256:1c9b93f79f48b03ada57ea24725d83a30284a012ec27eab2cf7e50a550cbbbcc \ + --hash=sha256:2107b0c024d1b35f4083f11245c0e23846ae64d02f40b2b226684840260ed054 \ + --hash=sha256:2229ae655ec4e8999599469559e97630185fdd53ae1e8997d147b7c9b2b72cba \ + --hash=sha256:25a18e9810fbc7e7f03ec2516addc116a957f8cbb8cbc95ccc80faa072743d03 \ + --hash=sha256:26ef06c73eb53267c2b319f43e6634c7556ea37672029241a056629af27c10e2 \ + --hash=sha256:2e1743fbd7f5fa713a1b0a8ac8ebabf0ec980b5d8809ec358d488e273b9cf02a \ + --hash=sha256:32483fe2aab2c3794101c2a159070584e5db11d0aa091b2c0ea9c4fc43d0d749 \ + --hash=sha256:3bf0f392c0b806905ed174dcd8bdd5e418a40d5567a05615a030a5aeddea692d \ + --hash=sha256:3e2a27c89eb9ac3d81ec8835e12414d73536c6e620355d65102503064a4ed6eb \ + --hash=sha256:40ad3afe81676fd9ec6d9d406eda00933f218038433980aa19d401490e46ecde \ + --hash=sha256:4215d3a102bd95e2e11b5395c78562967959824156af11fa93d18fdd18050990 \ + --hash=sha256:45d59a649a82df5718fd9527ce775fd66d1af35e6d31abdcdc906a49c6822958 \ + --hash=sha256:45e0111e73f43f735d70786557dc38141185072d7ff8dc1829d6a77ac1471468 \ + --hash=sha256:479496325ce554792dba6548fae3df31a72cef7bad71ca2e12b0e58f9b336bfc \ + --hash=sha256:490fa6d203992c47c7b9e4a9d39003a0c2bcc1c9aa3c058730884bbbb0ee9f09 \ + --hash=sha256:49ce47231818806067aea3324d4bf13825b658ad662d3b25fada0bdad9b8a6af \ + --hash=sha256:4baf3cbe2f0be3289eb68ac8ae771156971848bb8aaff60bad42005539431980 \ + --hash=sha256:522175aba7af9113c48ec10cc471b9b9bd4f6ceb36aeb4544a8e2c80ed9d252d \ + --hash=sha256:5e8571632780e08526f118f74170ad8d50fb0a48c23a746bef2a6ebade3abd6f \ + --hash=sha256:615ba64c208aaceb5ec83bfdce7728b80bfeb8be97562944836a7a0a9647d882 \ + --hash=sha256:61f69297cba3950a524f61c7c8ee12e55c486cb5f7db47ff9dcee33da6f0d3ae \ + --hash=sha256:65a20de41e85648e00305c1bb09a3598f840422e522277641145a32d42dcefcc \ + --hash=sha256:6a15c17af8839b6801d554263c546c69c4d7718ad4321e3166175b37eaacca77 \ + --hash=sha256:747fa73efa9b8b1488a95d0ba1039c8e2dca0f741612d80415b1e1c560febf4e \ + --hash=sha256:7be78388d6da1a25c0d5ec506523db58b18be22d9c37d8d3a32c08be4987bd73 \ + --hash=sha256:81fd9652b37b36f16138611c7e884eb82e0cec137c40d3ef7c3f9b3ed00f6ed8 \ + --hash=sha256:83d57312a58dcfe2a3a0f9d1389b299438909a02db60e2f2ea2ae2d8034909d3 \ + --hash=sha256:8843114c0cfce61b40ad48df65abcfc00d4dba82eae8718fab5352390848c5da \ + --hash=sha256:8cc3309d8e08fd79089e13ed4819d0af72aa935dd8f435a195fd152796752ff2 \ + --hash=sha256:8ebe63ee5f8fa4296b1b8cfc743f870d10e902ca18afc65c68cf46fd39bb0783 \ + --hash=sha256:8eddfb4d203a237da6f3cc8a540dad0517d274b5a1e9e636fd8d2c79b5c1d397 \ + --hash=sha256:922fa70ba549fce362d2e2871ab542082d66e2aaf0c19480ea453905b01f384e \ + --hash=sha256:931091142fd8cc14edccc0845a79248bc155425eee9a98b2db2ea4f00a235a42 \ + --hash=sha256:971fd5a1d6e62e00d945423a567e42eb1fa678ba89072832185ca836a94daaa6 \ + --hash=sha256:980a846182ce88c4f2f7e2c22c56aefd515daeb36149d1c897f83cf57999e0b6 \ + --hash=sha256:9d9adda641db7207e800a7f089068f6f645959f2df27e870ee81d44701dd9db3 \ + --hash=sha256:9f8f757bebaaea112c00dba718fc0d3260052ce714e25804a03f93f5d1c6cc11 \ + --hash=sha256:a6ae758eb08088d36812dd5d9af7a9859c05b1e0f714470ea243694b49278e7b \ + --hash=sha256:a8c2cf1209497cf659a667d7dea88985e834c24b7c3b605e6254cbb5076d985c \ + --hash=sha256:acab0277c40eff7143c2323190ea57b9ee5fd353d8190ee9652369fae735668a \ + --hash=sha256:b331680e46239e090f5b3cead313cc772f6caa7d0fc8de349337563125361a4a \ + --hash=sha256:c088e7a90b6017307f423efbb9d1ba97a22aa2170876223f9709e9d1de0b5347 \ + --hash=sha256:d099566accf23d21037f18a2a63d323075bebace807742e4b0ac210971d4dd70 \ + --hash=sha256:d388087771c837cdb6515539f43b9d4bf0b0f23593a24054ac16f7a960be16f4 \ + --hash=sha256:dcfe41187da8992c5f40aa8c5ec086fa3672834d2be57a32384c08d5a05b4c00 \ + --hash=sha256:e6d1db20594d9daba22f90da738b1a0441a7427552cc6e2e3d1297aeddc00378 \ + --hash=sha256:ebea5cc3aa8ea72e04df9913492f9a96d9348db876f9dda3ad729cfedf7ac416 \ + --hash=sha256:ebebf83299b0cb1721a8859ea98f3a77811e35dce7609c5c963b9ad90728f886 \ + --hash=sha256:f0e34c2079d47ae9f6188211db9e777c619a21d4faba6977774e8fa43b085e48 \ + --hash=sha256:f92f88e6c033db65a5ae3d97905c8fea9c725b63e28d5a75cb73b49bda5024d8 \ + --hash=sha256:f9f7bd5faab55f47231ad8dba7787866b69f5e93bc306e3915606779bbfb4ba8 \ + --hash=sha256:fd5ef5932f6475c436c4a55e4336ebbe47bd3272be04964a03d316bbf4afbcbc \ + --hash=sha256:ff8a59ea85a1f2191a0ffcc61298c571bc566332f82e5f5be1b83c9d8e668a62 + # via + # google-api-core + # google-cloud-vision + # grpcio-status +grpcio-status==1.76.0 \ + --hash=sha256:25fcbfec74c15d1a1cb5da3fab8ee9672852dc16a5a9eeb5baf7d7a9952943cd \ + --hash=sha256:380568794055a8efbbd8871162df92012e0228a5f6dffaf57f2a00c534103b18 + # via google-api-core +httplib2==0.31.0 \ + --hash=sha256:ac7ab497c50975147d4f7b1ade44becc7df2f8954d42b38b3d69c515f531135c \ + --hash=sha256:b9cd78abea9b4e43a7714c6e0f8b6b8561a6fc1e95d5dbd367f5bf0ef35f5d24 + # via + # google-api-python-client + # google-auth-httplib2 +idna==3.11 \ + --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea \ + --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902 + # via requests +prettytable==3.17.0 \ + --hash=sha256:59f2590776527f3c9e8cf9fe7b66dd215837cca96a9c39567414cbc632e8ddb0 \ + --hash=sha256:aad69b294ddbe3e1f95ef8886a060ed1666a0b83018bbf56295f6f226c43d287 + # via -r requirements.in +proto-plus==1.26.1 \ + --hash=sha256:13285478c2dcf2abb829db158e1047e2f1e8d63a077d94263c2b88b043c75a66 \ + --hash=sha256:21a515a4c4c0088a773899e23c7bbade3d18f9c66c73edd4c7ee3816bc96a012 + # via + # google-api-core + # google-cloud-vision +protobuf==6.33.5 \ + --hash=sha256:3093804752167bcab3998bec9f1048baae6e29505adaf1afd14a37bddede533c \ + --hash=sha256:69915a973dd0f60f31a08b8318b73eab2bd6a392c79184b3612226b0a3f8ec02 \ + --hash=sha256:6ddcac2a081f8b7b9642c09406bc6a4290128fce5f471cddd165960bb9119e5c \ + --hash=sha256:8afa18e1d6d20af15b417e728e9f60f3aa108ee76f23c3b2c07a2c3b546d3afd \ + --hash=sha256:8f04fa32763dcdb4973d537d6b54e615cc61108c7cb38fe59310c3192d29510a \ + --hash=sha256:9b71e0281f36f179d00cbcb119cb19dec4d14a81393e5ea220f64b286173e190 \ + --hash=sha256:a3157e62729aafb8df6da2c03aa5c0937c7266c626ce11a278b6eb7963c4e37c \ + --hash=sha256:a5cb85982d95d906df1e2210e58f8e4f1e3cdc088e52c921a041f9c9a0386de5 \ + --hash=sha256:cbf16ba3350fb7b889fca858fb215967792dc125b35c7976ca4818bee3521cf0 \ + --hash=sha256:d71b040839446bac0f4d162e758bea99c8251161dae9d0983a3b88dee345153b + # via + # google-api-core + # google-cloud-vision + # googleapis-common-protos + # grpcio-status + # proto-plus +pyasn1==0.6.3 \ + --hash=sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf \ + --hash=sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.2 \ + --hash=sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a \ + --hash=sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6 + # via google-auth +pyparsing==3.2.5 \ + --hash=sha256:2df8d5b7b2802ef88e8d016a2eb9c7aeaa923529cd251ed0fe4608275d4105b6 \ + --hash=sha256:e38a4f02064cf41fe6593d328d0512495ad1f3d8a91c4f73fc401b3079a59a5e + # via httplib2 +requests==2.33.0 \ + --hash=sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b \ + --hash=sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652 + # via google-api-core +rsa==4.9.1 \ + --hash=sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762 \ + --hash=sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75 + # via google-auth +typing-extensions==4.15.0 \ + --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ + --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 + # via grpcio +uritemplate==4.2.0 \ + --hash=sha256:480c2ed180878955863323eea31b0ede668795de182617fef9c6ca09e6ec9d0e \ + --hash=sha256:962201ba1c4edcab02e60f9a0d3821e82dfc5d2d6662a21abd533879bdb8a686 + # via google-api-python-client +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 + # via requests +wcwidth==0.2.14 \ + --hash=sha256:4d478375d31bc5395a3c55c40ccdf3354688364cd61c4f6adacaa9215d0b3605 \ + --hash=sha256:a7bb560c8aee30f9957e5f9895805edd20602f2d7f720186dfd906e82b4982e1 + # via prettytable diff --git a/perfmetrics/scripts/read_cache/setup.sh b/perfmetrics/scripts/read_cache/setup.sh index 1cd1665a5a..4a13f3ab59 100755 --- a/perfmetrics/scripts/read_cache/setup.sh +++ b/perfmetrics/scripts/read_cache/setup.sh @@ -63,8 +63,18 @@ sed -i 's/define \+FIO_IO_U_PLAT_GROUP_NR \+\([0-9]\+\)/define FIO_IO_U_PLAT_GRO ./configure && make && sudo make install cd - +# Export WORKING_DIR env variable and add it to ~/.bashrc. +export WORKING_DIR=$WD +echo "export WORKING_DIR=$WD" >> ~/.bashrc + +# Clone gcsfuse to get fio load test script. +if [ ! -d "./gcsfuse" ]; then + git clone -b master https://github.com/GoogleCloudPlatform/gcsfuse.git +fi + # Install and validate go. -version=1.23.3 +# Read the version +version=$(cat "$WORKING_DIR/gcsfuse/.go-version") wget -O go_tar.tar.gz https://go.dev/dl/go${version}.linux-amd64.tar.gz -q sudo rm -rf /usr/local/go tar -xzf go_tar.tar.gz && sudo mv go /usr/local @@ -74,18 +84,9 @@ export PATH=$PATH:/usr/local/go/bin && go version && rm go_tar.tar.gz export PATH=$PATH:$HOME/go/bin/ echo 'export PATH=$PATH:$HOME/go/bin/:/usr/local/go/bin' >> ~/.bashrc -# Export WORKING_DIR env variable and add it to ~/.bashrc. -export WORKING_DIR=$WD -echo "export WORKING_DIR=$WD" >> ~/.bashrc - # Install gcsfuse. CGO_ENABLED=0 go install github.com/googlecloudplatform/gcsfuse@master -# Clone gcsfuse to get fio load test script. -if [ ! -d "./gcsfuse" ]; then - git clone -b master https://github.com/GoogleCloudPlatform/gcsfuse.git -fi - # Mount gcsfuse. $WORKING_DIR/gcsfuse/perfmetrics/scripts/read_cache/mount_gcsfuse.sh diff --git a/perfmetrics/scripts/requirements.txt b/perfmetrics/scripts/requirements.txt index 3bb9448716..7f91d157e3 100644 --- a/perfmetrics/scripts/requirements.txt +++ b/perfmetrics/scripts/requirements.txt @@ -92,27 +92,27 @@ charset-normalizer==3.1.0 \ dataclasses==0.6 \ --hash=sha256:454a69d788c7fda44efd71e259be79577822f5e3f53f029a22d08004e951dc9f \ --hash=sha256:6988bd2b895eef432d562370bb707d540f32f7360ab13da45340101bc2307d84 - # via -r perfmetrics/scripts/requirements.in + # via -r requirements.in exceptiongroup==1.1.1 \ --hash=sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e \ --hash=sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785 - # via -r perfmetrics/scripts/requirements.in + # via -r requirements.in google-api-core[grpc]==2.15.0 \ --hash=sha256:2aa56d2be495551e66bbff7f729b790546f87d5c90e74781aa77233bcb395a8a \ --hash=sha256:abc978a72658f14a2df1e5e12532effe40f94f868f6e23d95133bd6abcca35ca # via - # -r perfmetrics/scripts/requirements.in + # -r requirements.in # google-api-python-client # google-cloud-monitoring google-api-python-client==2.55.0 \ --hash=sha256:1766c700eee14809ca1f7f52868c937755153289ea77ecdfd73dea6910e9a34d \ --hash=sha256:7837094c6d35cc8e680b60ebefe37f117ad521f3b263b0c7a6efa0b2f84cb17a - # via -r perfmetrics/scripts/requirements.in + # via -r requirements.in google-auth==2.26.1 \ --hash=sha256:2c8b55e3e564f298122a02ab7b97458ccfcc5617840beb5d0ac757ada92c9780 \ --hash=sha256:54385acca5c0fbdda510cd8585ba6f3fcb06eeecf8a6ecca39d3ee148b092590 # via - # -r perfmetrics/scripts/requirements.in + # -r requirements.in # google-api-core # google-api-python-client # google-auth-httplib2 @@ -123,16 +123,16 @@ google-auth-httplib2==0.1.0 \ google-cloud==0.34.0 \ --hash=sha256:01430187cf56df10a9ba775dd547393185d4b40741db0ea5889301f8e7a9d5d3 \ --hash=sha256:fb1ab7b0548fe44b3d538041f0a374505b7f990d448a935ea36649c5ccab5acf - # via -r perfmetrics/scripts/requirements.in + # via -r requirements.in google-cloud-monitoring==2.11.1 \ --hash=sha256:3d76463cc7abfd8e339b1d94b3c11facb60b9c5d6d805eb76431e60663cf334c \ --hash=sha256:cc8a4b118b56ce2566ad7dfab56926f35747ca28465dbac21e8f6e258704a16f - # via -r perfmetrics/scripts/requirements.in + # via -r requirements.in googleapis-common-protos==1.59.0 \ --hash=sha256:4168fcb568a826a52f23510412da405abd93f4d23ba544bb68d943b14ba3cb44 \ --hash=sha256:b287dc48449d1d41af0c69f4ea26242b5ae4c3d7249a38b0984c86a4caffff1f # via - # -r perfmetrics/scripts/requirements.in + # -r requirements.in # google-api-core # grpcio-status grpcio==1.54.3 \ @@ -182,7 +182,7 @@ grpcio==1.54.3 \ --hash=sha256:f78fb0d5e3b1f22060aa32872eab185f214b22a278d1763a2ffd7ca04cc16366 \ --hash=sha256:fd5779aab42c92fc0d31ac4f240c99f02007f0310704eb761abd7ad955edf411 # via - # -r perfmetrics/scripts/requirements.in + # -r requirements.in # google-api-core # grpcio-status grpcio-status==1.54.0 \ @@ -211,15 +211,15 @@ pbr==5.11.1 \ --hash=sha256:567f09558bae2b3ab53cb3c1e2e33e726ff3338e7bae3db5dc954b3a44eef12b \ --hash=sha256:aefc51675b0b533d56bb5fd1c8c6c0522fe31896679882e1c4c63d5e4a0fccb3 # via testresources -pluggy==1.0.0 \ - --hash=sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159 \ - --hash=sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3 +pluggy==1.6.0 \ + --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ + --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 # via pytest proto-plus==1.22.1 \ --hash=sha256:6c7dfd122dfef8019ff654746be4f5b1d9c80bba787fe9611b508dd88be3a2fa \ --hash=sha256:ea8982669a23c379f74495bc48e3dcb47c822c484ce8ee1d1d7beb339d4e34c5 # via - # -r perfmetrics/scripts/requirements.in + # -r requirements.in # google-cloud-monitoring protobuf==4.21.6 \ --hash=sha256:07a0bb9cc6114f16a39c866dc28b6e3d96fa4ffb9cc1033057412547e6e75cb9 \ @@ -237,7 +237,7 @@ protobuf==4.21.6 \ --hash=sha256:ba596b9ffb85c909fcfe1b1a23136224ed678af3faf9912d3fa483d5f9813c4e \ --hash=sha256:c7c864148a237f058c739ae7a05a2b403c0dfa4ce7d1f3e5213f352ad52d57c6 # via - # -r perfmetrics/scripts/requirements.in + # -r requirements.in # google-api-core # google-cloud-monitoring # googleapis-common-protos @@ -253,19 +253,23 @@ pyasn1-modules==0.3.0 \ --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d # via google-auth +pygments==2.20.0 \ + --hash=sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f \ + --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 + # via pytest pyparsing==3.0.9 \ --hash=sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb \ --hash=sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc # via httplib2 -pytest==7.3.1 \ - --hash=sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362 \ - --hash=sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3 - # via -r perfmetrics/scripts/requirements.in -requests==2.32.2 \ - --hash=sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289 \ - --hash=sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c +pytest==9.0.3 \ + --hash=sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9 \ + --hash=sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c + # via -r requirements.in +requests==2.33.0 \ + --hash=sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b \ + --hash=sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652 # via - # -r perfmetrics/scripts/requirements.in + # -r requirements.in # google-api-core rsa==4.9 \ --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \ @@ -278,15 +282,15 @@ six==1.16.0 \ testresources==2.0.1 \ --hash=sha256:67a361c3a2412231963b91ab04192209aa91a1aa052f0ab87245dbea889d1282 \ --hash=sha256:ee9d1982154a1e212d4e4bac6b610800bfb558e4fb853572a827bc14a96e4417 - # via -r perfmetrics/scripts/requirements.in + # via -r requirements.in tomli==2.0.1 \ --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f - # via -r perfmetrics/scripts/requirements.in + # via -r requirements.in typing==3.7.4.3 \ --hash=sha256:1187fb9c82fd670d10aa07bbb6cfcfe4bdda42d6fab8d5134f04e8c4d0b71cc9 \ --hash=sha256:283d868f5071ab9ad873e5e52268d611e851c870a2ba354193026f2dfb29d8b5 - # via -r perfmetrics/scripts/requirements.in + # via -r requirements.in uritemplate==4.1.1 \ --hash=sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0 \ --hash=sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e @@ -295,5 +299,5 @@ urllib3==2.2.2 \ --hash=sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472 \ --hash=sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168 # via - # -r perfmetrics/scripts/requirements.in + # -r requirements.in # requests diff --git a/perfmetrics/scripts/run_load_test_and_fetch_metrics.sh b/perfmetrics/scripts/run_load_test_and_fetch_metrics.sh index cebce489ae..d048e5fbb4 100755 --- a/perfmetrics/scripts/run_load_test_and_fetch_metrics.sh +++ b/perfmetrics/scripts/run_load_test_and_fetch_metrics.sh @@ -41,6 +41,6 @@ sudo umount $MOUNT_POINT echo Installing requirements.. pip install --require-hashes -r requirements.txt --user -gsutil cp gs://periodic-perf-tests/creds.json gsheet +gcloud storage cp gs://periodic-perf-tests/creds.json gsheet echo Fetching results.. python3 fetch_and_upload_metrics.py "fio-output${EXPERIMENT_NUMBER}.json" $UPLOAD_FLAGS --spreadsheet_id=$SPREADSHEET_ID diff --git a/perfmetrics/scripts/testing_on_gke/examples/dlio/data-loader/.helmignore b/perfmetrics/scripts/testing_on_gke/examples/dlio/data-loader/.helmignore deleted file mode 100644 index 0e8a0eb36f..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/dlio/data-loader/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/perfmetrics/scripts/testing_on_gke/examples/dlio/data-loader/Chart.yaml b/perfmetrics/scripts/testing_on_gke/examples/dlio/data-loader/Chart.yaml deleted file mode 100644 index b4ab2e5bbf..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/dlio/data-loader/Chart.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2018 The Kubernetes Authors. -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v2 -name: data-loader -description: A Helm chart for DLIO data loading to GCS buckets -type: application -version: 0.1.0 diff --git a/perfmetrics/scripts/testing_on_gke/examples/dlio/data-loader/templates/dlio-data-loader.yaml b/perfmetrics/scripts/testing_on_gke/examples/dlio/data-loader/templates/dlio-data-loader.yaml deleted file mode 100644 index cc2ab15069..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/dlio/data-loader/templates/dlio-data-loader.yaml +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2018 The Kubernetes Authors. -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Pod -metadata: - name: dlio-data-loader-{{ .Values.dlio.numFilesTrain }}-{{ .Values.dlio.recordLength }} - annotations: - gke-gcsfuse/volumes: "true" - gke-gcsfuse/cpu-limit: "0" - gke-gcsfuse/memory-limit: "0" - gke-gcsfuse/ephemeral-storage-limit: "0" -spec: - restartPolicy: Never - nodeSelector: - cloud.google.com/gke-ephemeral-storage-local-ssd: "true" - containers: - - name: dlio-data-loader - image: {{ .Values.image }} - resources: - limits: - cpu: "100" - memory: 400Gi - requests: - cpu: "30" - memory: 300Gi - command: - - "/bin/sh" - - "-c" - - | - echo "Installing gsutil..." - apt-get update && apt-get install -y apt-transport-https ca-certificates gnupg curl - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg - echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list - apt-get update && apt-get install -y google-cloud-cli - - echo "Generating data for file number: {{ .Values.dlio.numFilesTrain }}, file size: {{ .Values.dlio.recordLength }}..." - - mpirun -np 20 dlio_benchmark workload=unet3d_a100 \ - ++workload.workflow.generate_data=True \ - ++workload.workflow.train=False \ - ++workload.dataset.data_folder=/data \ - ++workload.dataset.num_files_train={{ .Values.dlio.numFilesTrain }} \ - ++workload.dataset.record_length={{ .Values.dlio.recordLength }} \ - ++workload.dataset.record_length_stdev=0 \ - ++workload.dataset.record_length_resize=0 - - gsutil -m cp -R /data/train gs://{{ .Values.bucketName }} - mkdir -p /bucket/valid - volumeMounts: - - name: local-dir - mountPath: /data - - name: gcs-fuse-csi-ephemeral - mountPath: /bucket - volumes: - - name: local-dir - emptyDir: {} - - name: gcs-fuse-csi-ephemeral - csi: - driver: gcsfuse.csi.storage.gke.io - volumeAttributes: - bucketName: {{ .Values.bucketName }} diff --git a/perfmetrics/scripts/testing_on_gke/examples/dlio/data-loader/values.yaml b/perfmetrics/scripts/testing_on_gke/examples/dlio/data-loader/values.yaml deleted file mode 100644 index a7ce5a089e..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/dlio/data-loader/values.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2018 The Kubernetes Authors. -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for data-loader. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -image: jiaxun/dlio:v1.0.0 -bucketName: gke-dlio-unet3d-100kb-500k - -dlio: - numFilesTrain: 500000 - recordLength: 102400 diff --git a/perfmetrics/scripts/testing_on_gke/examples/dlio/dlio_workload.py b/perfmetrics/scripts/testing_on_gke/examples/dlio/dlio_workload.py deleted file mode 100644 index 43805fd766..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/dlio/dlio_workload.py +++ /dev/null @@ -1,182 +0,0 @@ -# Copyright 2018 The Kubernetes Authors. -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This file defines a DlioWorkload (a DLIO Unet3d workload) and provides utility for parsing a json - -test-config file for a list of them. -""" - -import json - - -def validateDlioWorkload(workload: dict, name: str): - """Validates the given json workload object.""" - for requiredWorkloadAttribute, expectedType in { - 'bucket': str, - 'gcsfuseMountOptions': str, - 'dlioWorkload': dict, - }.items(): - if requiredWorkloadAttribute not in workload: - print(f"{name} does not have '{requiredWorkloadAttribute}' key in it.") - return False - if not type(workload[requiredWorkloadAttribute]) is expectedType: - print( - f"In {name}, the type of '{requiredWorkloadAttribute}' is of type" - f" '{type(workload[requiredWorkloadAttribute])}', not {expectedType}" - ) - return False - if expectedType == str and ' ' in workload[requiredWorkloadAttribute]: - print(f"{name} has space in the value of '{requiredWorkloadAttribute}'") - return False - - if 'fioWorkload' in workload: - print(f"{name} has 'fioWorkload' key in it, which is unexpected.") - return False - - dlioWorkload = workload['dlioWorkload'] - for requiredAttribute, expectedType in { - 'numFilesTrain': int, - 'recordLength': int, - 'batchSizes': list, - }.items(): - if requiredAttribute not in dlioWorkload: - print( - f'In {name}, dlioWorkload for {name} does not have' - f' {requiredAttribute} in it' - ) - return False - if not type(dlioWorkload[requiredAttribute]) is expectedType: - print( - f'In {name}, dlioWorkload[{requiredAttribute}] is of type' - f' {type(dlioWorkload[requiredAttribute])}, expected:' - f' {expectedType} ' - ) - return False - - batchSizes = dlioWorkload['batchSizes'] - for batchSize in batchSizes: - if not type(batchSize) is int: - print( - f'In {name}, one of the batch-size values in' - f" dlioWorkload['batchSizes'] is '{batchSize}', which is of type" - f' {type(batchSize)}, not int' - ) - return False - if batchSize < 1: - print( - f'In {name}, one of the batch-size values in' - f" dlioWorkload['batchSizes'] is '{batchSize}' < 1, which is not" - ' supported.' - ) - return False - - return True - - -class DlioWorkload: - """DlioWorkload holds data needed to define a DLIO Unet3d workload - - (essentially the data needed to create a job file for DLIO run). - - Members: - 1. scenario (string): One of "local-ssd", "gcsfuse-generic", - "gcsfuse-file-cache" and "gcsfuse-no-file-cache". - 2. numFilesTrain (int): DLIO numFilesTrain argument e.g. 500000 etc. - 3. recordLength (int): DLIO recordLength argument e.g. 100, 1000000 etc. - 4. bucket (str): Name of a GCS bucket to read input files from. - 5. batchSizes (set of ints): a set of ints representing multiple batchsize - values to test. - 6. gcsfuseMountOptions (str): gcsfuse mount options as a single - string in compact stringified format, to be used for the - test scenario "gcsfuse-generic". The individual config/cli flag values should - be separated by comma. Each cli flag should be of the form "<flag>[=<value>]", - while each config-file flag should be of form - "<config>[:<subconfig>[:<subsubconfig>[...]]]:<value>". For example, a legal - value would be: - "implicit-dirs,file_mode=777,file-cache:enable-parallel-downloads:true,metadata-cache:ttl-secs:true". - """ - - def __init__( - self, - scenario: str, - numFilesTrain: int, - recordLength: int, - bucket: str, - batchSizes: list, - gcsfuseMountOptions: str, - ): - self.scenario = scenario - self.numFilesTrain = numFilesTrain - self.recordLength = recordLength - self.bucket = bucket - self.batchSizes = set(batchSizes) - self.gcsfuseMountOptions = gcsfuseMountOptions - - -def ParseTestConfigForDlioWorkloads(testConfigFileName: str): - """Parses the given workload test configuration file for DLIO workloads.""" - print(f'Parsing {testConfigFileName} for DLIO workloads ...') - with open(testConfigFileName) as f: - file = json.load(f) - testConfig = file['TestConfig'] - workloadConfig = testConfig['workloadConfig'] - workloads = workloadConfig['workloads'] - dlioWorkloads = [] - scenarios = ( - ['local-ssd', 'gcsfuse-generic'] - if ('runOnSSD' not in workloadConfig or workloadConfig['runOnSSD']) - else ['gcsfuse-generic'] - ) - for i in range(len(workloads)): - workload = workloads[i] - if not validateDlioWorkload(workload, f'workload#{i}'): - print(f'workloads#{i} is not a valid DLIO workload, so ignoring it.') - else: - for scenario in scenarios: - dlioWorkload = workload['dlioWorkload'] - dlioWorkloads.append( - DlioWorkload( - scenario, - dlioWorkload['numFilesTrain'], - dlioWorkload['recordLength'], - workload['bucket'], - dlioWorkload['batchSizes'], - workload['gcsfuseMountOptions'], - ) - ) - return dlioWorkloads - - -def DlioChartNamePodName( - dlioWorkload: DlioWorkload, instanceID: str, batchSize: int -) -> (str, str, str): - shortenScenario = { - 'local-ssd': 'ssd', - 'gcsfuse-generic': 'gcsfuse', - } - shortForScenario = ( - shortenScenario[dlioWorkload.scenario] - if dlioWorkload.scenario in shortenScenario - else 'other' - ) - - hashOfWorkload = str(hash((instanceID, batchSize, dlioWorkload))).replace( - '-', '' - ) - return ( - f'dlio-unet3d-{shortForScenario}-{dlioWorkload.recordLength}-{hashOfWorkload}', - f'dlio-tester-{shortForScenario}-{dlioWorkload.recordLength}-{hashOfWorkload}', - f'{instanceID}/{dlioWorkload.numFilesTrain}-{dlioWorkload.recordLength}-{batchSize}-{hashOfWorkload}/{dlioWorkload.scenario}', - ) diff --git a/perfmetrics/scripts/testing_on_gke/examples/dlio/dlio_workload_test.py b/perfmetrics/scripts/testing_on_gke/examples/dlio/dlio_workload_test.py deleted file mode 100644 index db80d994c2..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/dlio/dlio_workload_test.py +++ /dev/null @@ -1,260 +0,0 @@ -# Copyright 2018 The Kubernetes Authors. -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This file defines unit tests for functionalities in dlio_workload.py""" - -import unittest -from dlio_workload import DlioWorkload, validateDlioWorkload - - -class DlioWorkloadTest(unittest.TestCase): - - def test_validate_dlio_workload_empty(self): - self.assertFalse(validateDlioWorkload(({}), "empty-dlio-workload")) - - def test_validate_dlio_workload_invalid_missing_bucket(self): - self.assertFalse( - validateDlioWorkload( - ({"dlioWorkload": {}, "gcsfuseMountOptions": ""}), - "invalid-dlio-workload-missing-bucket", - ) - ) - - def test_validate_dlio_workload_invalid_bucket_contains_space(self): - self.assertFalse( - validateDlioWorkload( - ({"dlioWorkload": {}, "gcsfuseMountOptions": "", "bucket": " "}), - "invalid-dlio-workload-bucket-contains-space", - ) - ) - - def test_validate_dlio_workload_invalid_no_dlioWorkloadSpecified(self): - self.assertFalse( - validateDlioWorkload(({"bucket": {}}), "invalid-dlio-workload-2") - ) - - def test_validate_dlio_workload_invalid_commented_out_dlioWorkload(self): - self.assertFalse( - validateDlioWorkload( - ({ - "_dlioWorkload": {}, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }), - "commented-out-dlio-workload", - ) - ) - - def test_validate_dlio_workload_invalid_mixed_dlioWorkload_fioWorkload(self): - self.assertFalse( - validateDlioWorkload( - ({ - "dlioWorkload": {}, - "fioWorkload": {}, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }), - "mixed-dlio/fio-workload", - ) - ) - - def test_validate_dlio_workload_invalid_missing_numFilesTrain(self): - workload = dict({ - "dlioWorkload": { - "recordLength": 10000, - "batchSizes": [100, 200], - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertFalse( - validateDlioWorkload( - workload, "invalid-dlio-workload-missing-numFilesTrain" - ) - ) - - def test_validate_dlio_workload_invalid_unsupported_numFilesTrain(self): - workload = dict({ - "dlioWorkload": { - "numFilesTrain": "1000", - "recordLength": 10000, - "batchSizes": [100, 200], - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertFalse( - validateDlioWorkload( - workload, "invalid-dlio-workload-unsupported-numFilesTrain" - ) - ) - - def test_validate_dlio_workload_invalid_missing_recordLength(self): - workload = dict({ - "dlioWorkload": { - "numFilesTrain": 1000, - "batchSizes": [100, 200], - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertFalse( - validateDlioWorkload( - workload, "invalid-dlio-workload-missing-recordLength" - ) - ) - - def test_validate_dlio_workload_invalid_unsupported_recordLength(self): - workload = dict({ - "dlioWorkload": { - "numFilesTrain": 1000, - "recordLength": "10000", - "batchSizes": [100, 200], - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertFalse( - validateDlioWorkload( - workload, "invalid-dlio-workload-unsupported-recordLength" - ) - ) - - def test_validate_dlio_workload_invalid_missing_gcsfuseMountOptions(self): - workload = dict({ - "dlioWorkload": { - "numFilesTrain": 1000, - "recordLength": 100, - "batchSizes": [100, 200], - }, - "bucket": "dummy-bucket", - }) - self.assertFalse( - validateDlioWorkload( - workload, "invalid-dlio-workload-missing-gcsfuseMountOptions" - ) - ) - - def test_validate_dlio_workload_invalid_unsupported_gcsfuseMountOptions( - self, - ): - workload = dict({ - "dlioWorkload": { - "numFilesTrain": 1000, - "recordLength": 10000, - "batchSizes": [100, 200], - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": 100, - }) - self.assertFalse( - validateDlioWorkload( - workload, "invalid-dlio-workload-unsupported-gcsfuseMountOptions1" - ) - ) - - def test_validate_dlio_workload_invalid_gcsfuseMountOptions_contains_space( - self, - ): - workload = dict({ - "dlioWorkload": { - "numFilesTrain": 1000, - "recordLength": 10000, - "batchSizes": [100, 200], - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "abc def", - }) - self.assertFalse( - validateDlioWorkload( - workload, - "invalid-dlio-workload-unsupported-gcsfuseMountOptions-contains-space", - ) - ) - - def test_validate_dlio_workload_invalid_missing_batchSizes(self): - workload = dict({ - "dlioWorkload": { - "numFilesTrain": 1000, - "recordLength": 10000, - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertFalse( - validateDlioWorkload( - workload, "invalid-dlio-workload-missing-batchSizes" - ) - ) - - def test_validate_dlio_workload_invalid_unsupported_batchSizes1(self): - workload = dict({ - "dlioWorkload": { - "numFilesTrain": 1000, - "recordLength": 10000, - "batchSizes": ["100"], - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertFalse( - validateDlioWorkload( - workload, "invalid-dlio-workload-unsupported-batchSizes1" - ) - ) - - def test_validate_dlio_workload_invalid_unsupported_batchSizes2(self): - workload = dict({ - "dlioWorkload": { - "numFilesTrain": 1000, - "recordLength": 10000, - "batchSizes": [0, -1], - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertFalse( - validateDlioWorkload( - workload, "invalid-dlio-workload-unsupported-batchSizes2" - ) - ) - - def test_validate_dlio_workload_valid_single_batchSize(self): - workload = dict({ - "dlioWorkload": { - "numFilesTrain": 1000, - "recordLength": 10000, - "batchSizes": [100], - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertTrue(validateDlioWorkload(workload, "valid-dlio-workload-2")) - - def test_validate_dlio_workload_valid_multiple_batchSizes(self): - workload = dict({ - "dlioWorkload": { - "numFilesTrain": 1000, - "recordLength": 10000, - "batchSizes": [100, 200], - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertTrue(validateDlioWorkload(workload, "valid-dlio-workload-2")) - - -if __name__ == "__main__": - unittest.main() diff --git a/perfmetrics/scripts/testing_on_gke/examples/dlio/parse_logs.py b/perfmetrics/scripts/testing_on_gke/examples/dlio/parse_logs.py deleted file mode 100644 index 2f8d580016..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/dlio/parse_logs.py +++ /dev/null @@ -1,321 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018 The Kubernetes Authors. -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# standard library imports -import argparse -import json -import os -import pprint -import subprocess -import sys - -# local library imports -sys.path.append("../") -import dlio_workload -from utils.utils import get_memory, get_cpu, unix_to_timestamp, standard_timestamp, is_mash_installed, get_memory_from_monitoring_api, get_cpu_from_monitoring_api, timestamp_to_epoch -from utils.parse_logs_common import ensure_directory_exists, download_gcs_objects, parse_arguments, SUPPORTED_SCENARIOS - -_LOCAL_LOGS_LOCATION = "../../bin/dlio-logs/logs" - -record = { - "pod_name": "", - "epoch": 0, - "scenario": "", - "train_au_percentage": 0, - "duration": 0, - "train_throughput_samples_per_second": 0, - "train_throughput_mb_per_second": 0, - "throughput_over_local_ssd": 0, - "start_epoch": "", - "end_epoch": "", - "start": "", - "end": "", - "highest_memory": 0, - "lowest_memory": 0, - "highest_cpu": 0.0, - "lowest_cpu": 0.0, - "gcsfuse_mount_options": "", -} - - -def downloadDlioOutputs(dlioWorkloads: set, instanceId: str) -> int: - """Downloads instanceId-specific dlio outputs for each dlioWorkload locally. - - Outputs in the bucket are in the following object naming format - (details in ./unet3d-loading-test/templates/dlio-tester.yaml). - gs://<bucket>/logs/<instanceId>/<numFilesTrain>-<recordLength>-<batchSize>-<hash>/<scenario>/per_epoch_stats.json - gs://<bucket>/logs/<instanceId>/<numFilesTrain>-<recordLength>-<batchSize>-<hash>/<scenario>/summary.json - gs://<bucket>/logs/<instanceId>/<numFilesTrain>-<recordLength>-<batchSize>-<hash>/gcsfuse-generic/gcsfuse_mount_options - - These are downloaded locally as: - <_LOCAL_LOGS_LOCATION>/<instanceId>/<numFilesTrain>-<recordLength>-<batchSize>-<hash>/<scenario>/per_epoch_stats.json - <_LOCAL_LOGS_LOCATION>/<instanceId>/<numFilesTrain>-<recordLength>-<batchSize>-<hash>/<scenario>/summary.json - <_LOCAL_LOGS_LOCATION>/<instanceId>/<numFilesTrain>-<recordLength>-<batchSize>-<hash>/gcsfuse-generic/gcsfuse_mount_options - """ - - for dlioWorkload in dlioWorkloads: - srcObjects = f"gs://{dlioWorkload.bucket}/logs/{instanceId}" - print(f"Downloading DLIO logs from the {srcObjects} ...") - returncode, errorStr = download_gcs_objects( - srcObjects, _LOCAL_LOGS_LOCATION - ) - if returncode < 0: - print(f"Failed to download DLIO logs from {srcObjects}: {errorStr}") - return returncode - return 0 - - -def createOutputScenariosFromDownloadedFiles(args: dict) -> dict: - """Creates output records from the downloaded local files. - - The following creates a dict called 'output' - from the downloaded dlio output files, which are in the following format. - - <_LOCAL_LOGS_LOCATION>/<instanceId>/<numFilesTrain>-<recordLength>-<batchSize>-<hash>/<scenario>/per_epoch_stats.json - <_LOCAL_LOGS_LOCATION>/<instanceId>/<numFilesTrain>-<recordLength>-<batchSize>-<hash>/<scenario>/summary.json - <_LOCAL_LOGS_LOCATION>/<instanceId>/<numFilesTrain>-<recordLength>-<batchSize>-<hash>/gcsfuse-generic/gcsfuse_mount_options - - Output dict structure: - "{num_files_train}-{mean_file_size}-{batch_size}": - "mean_file_size": str - "num_files_train": str - "batch_size": str - "records": - "local-ssd": [record1, record2, record3, record4] - "gcsfuse-generic": [record1, record2, record3, record4] - "gcsfuse-file-cache": [record1, record2, record3, record4] - "gcsfuse-no-file-cache": [record1, record2, record3, record4] - """ - - output = {} - for root, _, files in os.walk(_LOCAL_LOGS_LOCATION + "/" + args.instance_id): - print(f"Parsing directory {root} ...") - if files: - # If directory contains gcsfuse_mount_options file, then parse gcsfuse - # mount options from it in record. - gcsfuse_mount_options = "" - gcsfuse_mount_options_file = root + "/gcsfuse_mount_options" - if os.path.isfile(gcsfuse_mount_options_file): - with open(gcsfuse_mount_options_file) as f: - gcsfuse_mount_options = f.read().strip() - - per_epoch_stats_file = root + "/per_epoch_stats.json" - summary_file = root + "/summary.json" - - # Load per_epoch_stats.json . - with open(per_epoch_stats_file, "r") as f: - try: - per_epoch_stats_data = json.load(f) - except: - print(f"failed to json-parse {per_epoch_stats_file}") - continue - - # Load summary.json . - with open(summary_file, "r") as f: - try: - summary_data = json.load(f) - except: - print(f"failed to json-parse {summary_file}") - continue - - for i in range(summary_data["epochs"]): - # Get numFilesTrain, recordLength, batchSize from the file/dir path. - key = root.split("/")[-2] - key_split = key.split("-") - - if key not in output: - output[key] = { - "num_files_train": key_split[-4], - "mean_file_size": key_split[-3], - "batch_size": key_split[-2], - "records": { - "local-ssd": [], - "gcsfuse-generic": [], - "gcsfuse-file-cache": [], - "gcsfuse-no-file-cache": [], - }, - } - - # Create a record for this key. - r = record.copy() - r["pod_name"] = summary_data["hostname"] - r["epoch"] = i + 1 - r["scenario"] = root.split("/")[-1] - r["train_au_percentage"] = round( - summary_data["metric"]["train_au_percentage"][i], 2 - ) - r["duration"] = int(float(per_epoch_stats_data[str(i + 1)]["duration"])) - r["train_throughput_samples_per_second"] = int( - summary_data["metric"]["train_throughput_samples_per_second"][i] - ) - r["train_throughput_mb_per_second"] = int( - r["train_throughput_samples_per_second"] - * int(output[key]["mean_file_size"]) - / (1024**2) - ) - r["start_epoch"] = timestamp_to_epoch( - per_epoch_stats_data[str(i + 1)]["start"] - ) - r["end_epoch"] = timestamp_to_epoch( - per_epoch_stats_data[str(i + 1)]["end"] - ) - r["start"] = standard_timestamp( - per_epoch_stats_data[str(i + 1)]["start"] - ) - r["end"] = standard_timestamp(per_epoch_stats_data[str(i + 1)]["end"]) - - def fetch_cpu_memory_data(): - if r["scenario"] != "local-ssd": - if mash_installed: - r["lowest_memory"], r["highest_memory"] = get_memory( - r["pod_name"], - r["start"], - r["end"], - project_number=args.project_number, - ) - r["lowest_cpu"], r["highest_cpu"] = get_cpu( - r["pod_name"], - r["start"], - r["end"], - project_number=args.project_number, - ) - else: - r["lowest_memory"], r["highest_memory"] = ( - get_memory_from_monitoring_api( - pod_name=r["pod_name"], - start_epoch=r["start_epoch"], - end_epoch=r["end_epoch"], - project_id=args.project_id, - cluster_name=args.cluster_name, - namespace_name=args.namespace_name, - ) - ) - r["lowest_cpu"], r["highest_cpu"] = get_cpu_from_monitoring_api( - pod_name=r["pod_name"], - start_epoch=r["start_epoch"], - end_epoch=r["end_epoch"], - project_id=args.project_id, - cluster_name=args.cluster_name, - namespace_name=args.namespace_name, - ) - - fetch_cpu_memory_data() - - r["gcsfuse_mount_options"] = gcsfuse_mount_options - - # This print is for debugging in case something goes wrong. - pprint.pprint(r) - - # If a slot for record for this particular epoch has not been created yet, - # append enough empty records to make a slot. - while len(output[key]["records"][r["scenario"]]) < i + 1: - output[key]["records"][r["scenario"]].append({}) - - # Insert the record at the appropriate slot. - output[key]["records"][r["scenario"]][i] = r - - return output - - -def writeRecordsToCsvOutputFile(output: dict, output_file_path: str): - with open(output_file_path, "a") as output_file: - # Write a new header row. - output_file.write( - "File Size,File #,Total Size (GB),Batch Size,Scenario,Epoch,Duration" - " (s),GPU Utilization (%),Throughput (sample/s),Throughput" - " (MB/s),Throughput over Local SSD (%),GCSFuse Lowest Memory" - " (MB),GCSFuse Highest Memory (MB),GCSFuse Lowest CPU (core),GCSFuse" - " Highest CPU (core),Pod,Start,End,GcsfuseMountOptions,InstanceID\n" - ) - - for key in output: - record_set = output[key] - total_size = int( - int(record_set["mean_file_size"]) - * int(record_set["num_files_train"]) - / (1024**3) - ) - - for scenario in SUPPORTED_SCENARIOS: - if scenario not in record_set["records"]: - print(f"{scenario} not in output so skipping") - continue - - for i in range(len(record_set["records"][scenario])): - r = record_set["records"][scenario][i] - - try: - if "local-ssd" in record_set["records"] and ( - len(record_set["records"]["local-ssd"]) - == len(record_set["records"][scenario]) - ): - r["throughput_over_local_ssd"] = round( - r["train_throughput_mb_per_second"] - / record_set["records"]["local-ssd"][i][ - "train_throughput_mb_per_second" - ] - * 100, - 2, - ) - else: - r["throughput_over_local_ssd"] = "NA" - - except ZeroDivisionError: - print("Got ZeroDivisionError. Ignoring it.") - r["throughput_over_local_ssd"] = 0 - - except Exception as e: - print( - "Error: failed to parse/write record-set for" - f" scenario: {scenario}, i: {i}, record: {r}, exception: {e}" - ) - continue - - output_file.write( - f"{record_set['mean_file_size']},{record_set['num_files_train']},{total_size},{record_set['batch_size']},{scenario}," - ) - output_file.write( - f"{r['epoch']},{r['duration']},{r['train_au_percentage']},{r['train_throughput_samples_per_second']},{r['train_throughput_mb_per_second']},{r['throughput_over_local_ssd']},{r['lowest_memory']},{r['highest_memory']},{r['lowest_cpu']},{r['highest_cpu']},{r['pod_name']},{r['start']},{r['end']},\"{r['gcsfuse_mount_options']}\",{args.instance_id}\n" - ) - - output_file.close() - - -if __name__ == "__main__": - args = parse_arguments() - ensure_directory_exists(_LOCAL_LOGS_LOCATION) - - dlioWorkloads = dlio_workload.ParseTestConfigForDlioWorkloads( - args.workload_config - ) - downloadDlioOutputs(dlioWorkloads, args.instance_id) - - mash_installed = is_mash_installed() - if not mash_installed: - print("Mash is not installed, will skip parsing CPU and memory usage.") - - output = createOutputScenariosFromDownloadedFiles(args) - - output_file_path = args.output_file - # Create the parent directory of output_file_path if doesn't - # exist already. - ensure_directory_exists(os.path.dirname(output_file_path)) - writeRecordsToCsvOutputFile(output, output_file_path) - print( - "\n\nSuccessfully published outputs of DLIO test runs to" - f" {output_file_path} !!!\n\n" - ) diff --git a/perfmetrics/scripts/testing_on_gke/examples/dlio/run_tests.py b/perfmetrics/scripts/testing_on_gke/examples/dlio/run_tests.py deleted file mode 100644 index eff9cfa263..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/dlio/run_tests.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018 The Kubernetes Authors. -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Generates and deploys helm charts for DLIO workloads. - -This program takes in a json test-config file, finds out valid -DLIO workloads from it and generates and deploys a helm chart for -each valid DLIO workload. -""" - -# system imports -import argparse -import os -import subprocess -import sys - -# local imports from other directories -sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'utils')) -from run_tests_common import escape_commas_in_string, parse_args, run_command, add_iam_role_for_buckets -from utils import UnknownMachineTypeError, resource_limits - -# local imports from same directory -import dlio_workload - - -def createHelmInstallCommands( - dlioWorkloads: set, - instanceId: str, - machineType: str, -) -> list: - """Creates helm install commands for the given dlioWorkload objects.""" - helm_commands = [] - try: - resourceLimits, resourceRequests = resource_limits(machineType) - except UnknownMachineTypeError: - print( - f'Found unknown machine-type: {machineType}, defaulting resource limits' - ' to cpu=0,memory=0' - ) - resourceLimits = {'cpu': 0, 'memory': '0'} - resourceRequests = resourceLimits - - for dlioWorkload in dlioWorkloads: - for batchSize in dlioWorkload.batchSizes: - chartName, podName, outputDirPrefix = dlio_workload.DlioChartNamePodName( - dlioWorkload, instanceId, batchSize - ) - commands = [ - f'helm install {chartName} unet3d-loading-test', - f'--set bucketName={dlioWorkload.bucket}', - f'--set scenario={dlioWorkload.scenario}', - f'--set dlio.numFilesTrain={dlioWorkload.numFilesTrain}', - f'--set dlio.recordLength={dlioWorkload.recordLength}', - f'--set dlio.batchSize={batchSize}', - f'--set instanceId={instanceId}', - ( - '--set' - f' gcsfuse.mountOptions={escape_commas_in_string(dlioWorkload.gcsfuseMountOptions)}' - ), - f'--set nodeType={machineType}', - f'--set podName={podName}', - f'--set outputDirPrefix={outputDirPrefix}', - f"--set resourceLimits.cpu={resourceLimits['cpu']}", - f"--set resourceLimits.memory={resourceLimits['memory']}", - f"--set resourceRequests.cpu={resourceRequests['cpu']}", - f"--set resourceRequests.memory={resourceRequests['memory']}", - ] - - helm_command = ' '.join(commands) - helm_commands.append(helm_command) - return helm_commands - - -def main(args) -> None: - dlioWorkloads = dlio_workload.ParseTestConfigForDlioWorkloads( - args.workload_config - ) - helmInstallCommands = createHelmInstallCommands( - dlioWorkloads, - args.instance_id, - args.machine_type, - ) - buckets = [dlioWorkload.bucket for dlioWorkload in dlioWorkloads] - role = 'roles/storage.objectUser' - add_iam_role_for_buckets( - buckets, - role, - args.project_id, - args.project_number, - args.namespace, - args.ksa, - ) - for helmInstallCommand in helmInstallCommands: - print(f'{helmInstallCommand}') - if not args.dry_run: - run_command(helmInstallCommand) - - -if __name__ == '__main__': - args = parse_args() - main(args) diff --git a/perfmetrics/scripts/testing_on_gke/examples/dlio/unet3d-loading-test/.helmignore b/perfmetrics/scripts/testing_on_gke/examples/dlio/unet3d-loading-test/.helmignore deleted file mode 100644 index 0e8a0eb36f..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/dlio/unet3d-loading-test/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/perfmetrics/scripts/testing_on_gke/examples/dlio/unet3d-loading-test/Chart.yaml b/perfmetrics/scripts/testing_on_gke/examples/dlio/unet3d-loading-test/Chart.yaml deleted file mode 100644 index d5c816d0a8..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/dlio/unet3d-loading-test/Chart.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2018 The Kubernetes Authors. -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v2 -name: unet3d-loading-test -description: A Helm chart for DLIO Unet3D loading test -type: application -version: 0.1.0 diff --git a/perfmetrics/scripts/testing_on_gke/examples/dlio/unet3d-loading-test/templates/dlio-tester.yaml b/perfmetrics/scripts/testing_on_gke/examples/dlio/unet3d-loading-test/templates/dlio-tester.yaml deleted file mode 100644 index 75829fc93b..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/dlio/unet3d-loading-test/templates/dlio-tester.yaml +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright 2018 The Kubernetes Authors. -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Pod -metadata: - name: {{ .Values.podName }} - {{- if ne .Values.scenario "local-ssd" }} - annotations: - gke-gcsfuse/volumes: "true" - gke-gcsfuse/memory-limit: "6Gi" - {{- end }} -spec: - restartPolicy: Never - nodeSelector: - cloud.google.com/gke-ephemeral-storage-local-ssd: "true" - node.kubernetes.io/instance-type: {{ .Values.nodeType }} - containers: - - name: dlio-tester - image: {{ .Values.image }} - resources: - limits: - cpu: {{ .Values.resourceLimits.cpu }} - memory: {{ .Values.resourceLimits.memory }} - requests: - cpu: {{ .Values.resourceRequests.cpu }} - memory: {{ .Values.resourceRequests.memory }} - env: - - name: RDMAV_FORK_SAFE - value: "1" - command: - - "/bin/sh" - - "-c" - - | - # Change the source code of dlio benchmark so that page cache is cleared at every epoch - main_file="dlio_benchmark/main.py" - x=$(grep -n "for epoch in range(1, self.epochs + 1):" $main_file | cut -f1 -d ':') - x=$((x + 1)) - sed -i "${x} i \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ os.system(\"sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches'\")" $main_file - - echo "Installing gsutil..." - apt-get update && apt-get install -y apt-transport-https ca-certificates gnupg curl - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg - echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list - apt-get update && apt-get install -y google-cloud-cli - - {{ if eq .Values.scenario "local-ssd" }} - echo "Generating data for on Local SSD..." - mkdir -p /data - - mpirun -np 20 dlio_benchmark workload=unet3d_a100 \ - ++workload.workflow.generate_data=True \ - ++workload.workflow.train=False \ - ++workload.dataset.data_folder=/data \ - ++workload.dataset.num_files_train={{ .Values.dlio.numFilesTrain }} \ - ++workload.dataset.record_length={{ .Values.dlio.recordLength }} \ - ++workload.dataset.record_length_stdev=0 \ - ++workload.dataset.record_length_resize=0 - - echo "Sleeping 5 minutes to wait for Local SSD RAID to populate data." - sleep 300 - {{ end }} - - outputDir=/logs/{{ .Values.outputDirPrefix }} - - echo "Testing {{ .Values.scenario }}" - mpirun -np 8 dlio_benchmark workload=unet3d_a100 \ - ++workload.train.epochs=4 \ - ++workload.workflow.profiling=True \ - ++workload.profiling.profiler=iostat \ - ++workload.profiling.iostat_devices=[md0] \ - ++workload.dataset.data_folder=/data \ - ++workload.dataset.num_files_train={{ .Values.dlio.numFilesTrain }} \ - ++workload.reader.batch_size={{ .Values.dlio.batchSize }} \ - ++workload.dataset.record_length={{ .Values.dlio.recordLength }} \ - ++workload.reader.read_threads={{ .Values.dlio.readThreads }} \ - ++workload.output.folder=${outputDir} - - # dump the gcsfuse-mount-configuration to a file in output-directory. - {{ if eq .Values.scenario "gcsfuse-generic"}} - echo "{{ .Values.gcsfuse.mountOptions }}" > ${outputDir}/gcsfuse_mount_options - {{ end }} - - gsutil -m cp -R /logs/* gs://{{ .Values.bucketName }}/logs/ - volumeMounts: - - name: dshm - mountPath: /dev/shm - - name: logging-vol - mountPath: /logs - - name: data-vol - mountPath: /data - volumes: - - name: dshm - emptyDir: - medium: Memory - - name: logging-vol - emptyDir: {} - - name: data-vol - {{- if eq .Values.scenario "local-ssd" }} - emptyDir: {} - {{- else if eq .Values.scenario "gcsfuse-generic" }} - csi: - driver: gcsfuse.csi.storage.gke.io - volumeAttributes: - bucketName: {{ .Values.bucketName }} - mountOptions: "{{ .Values.gcsfuse.mountOptions }}" - {{- else if eq .Values.scenario "gcsfuse-file-cache" }} - csi: - driver: gcsfuse.csi.storage.gke.io - volumeAttributes: - bucketName: {{ .Values.bucketName }} - metadataCacheTTLSeconds: "{{ .Values.gcsfuse.metadataCacheTTLSeconds }}" - metadataTypeCacheCapacity: "{{ .Values.gcsfuse.metadataTypeCacheCapacity }}" - metadataStatCacheCapacity: "{{ .Values.gcsfuse.metadataStatCacheCapacity }}" - fileCacheCapacity: "{{ .Values.gcsfuse.fileCacheCapacity }}" - fileCacheForRangeRead: "{{ .Values.gcsfuse.fileCacheForRangeRead }}" - gcsfuseLoggingSeverity: "debug" - mountOptions: implicit-dirs - {{- else }} - csi: - driver: gcsfuse.csi.storage.gke.io - volumeAttributes: - bucketName: {{ .Values.bucketName }} - metadataCacheTTLSeconds: "{{ .Values.gcsfuse.metadataCacheTTLSeconds }}" - metadataTypeCacheCapacity: "{{ .Values.gcsfuse.metadataTypeCacheCapacity }}" - metadataStatCacheCapacity: "{{ .Values.gcsfuse.metadataStatCacheCapacity }}" - gcsfuseLoggingSeverity: "debug" - mountOptions: implicit-dirs - {{- end }} diff --git a/perfmetrics/scripts/testing_on_gke/examples/dlio/unet3d-loading-test/values.yaml b/perfmetrics/scripts/testing_on_gke/examples/dlio/unet3d-loading-test/values.yaml deleted file mode 100644 index 359185fcb9..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/dlio/unet3d-loading-test/values.yaml +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2018 The Kubernetes Authors. -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for unet3d-loading-test. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -image: jiaxun/dlio:v1.2.0 -bucketName: gke-dlio-test-data -# scenario controls the kind of storage that is used for the load testing. local-ssd means directly on LSSD; gcsfuse-generic means on a gcsfuse mount with gcsfuse.mountOptions sent from the caller; gcsfuse-no-file-cache and gcsfuse-file-cache mean as their name suggests. -scenario: local-ssd -nodeType: n2-standard-96 -instanceId: ldap-yyyymmdd-hhmmss -podName: -outputDirPrefix: - -resourceLimits: - cpu: 0 - memory: 0 -resourceRequests: - cpu: 0 - memory: 0 - -dlio: - numFilesTrain: 500000 - recordLength: 102400 - batchSize: 800 - readThreads: 12 - -gcsfuse: - metadataCacheTTLSeconds: "360000" - metadataStatCacheCapacity: "-1" - metadataTypeCacheCapacity: "-1" - fileCacheCapacity: "-1" - fileCacheForRangeRead: "true" - mountOptions: "implicit-dirs" diff --git a/perfmetrics/scripts/testing_on_gke/examples/fio/data-loader/.helmignore b/perfmetrics/scripts/testing_on_gke/examples/fio/data-loader/.helmignore deleted file mode 100644 index 0e8a0eb36f..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/fio/data-loader/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/perfmetrics/scripts/testing_on_gke/examples/fio/data-loader/Chart.yaml b/perfmetrics/scripts/testing_on_gke/examples/fio/data-loader/Chart.yaml deleted file mode 100644 index fc592f9aee..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/fio/data-loader/Chart.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2018 The Kubernetes Authors. -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v2 -name: data-loader -description: A Helm chart for FIO data loading to GCS buckets -type: application -version: 0.1.0 diff --git a/perfmetrics/scripts/testing_on_gke/examples/fio/data-loader/templates/fio-data-loader.yaml b/perfmetrics/scripts/testing_on_gke/examples/fio/data-loader/templates/fio-data-loader.yaml deleted file mode 100644 index a48e84f6c6..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/fio/data-loader/templates/fio-data-loader.yaml +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright 2018 The Kubernetes Authors. -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Pod -metadata: - name: fio-data-loader-{{ .Values.fio.fileSize | lower }} -spec: - restartPolicy: Never - nodeSelector: - cloud.google.com/gke-ephemeral-storage-local-ssd: "true" - containers: - - name: fio-data-loader - image: ubuntu:24.04 - resources: - limits: - cpu: "100" - memory: 400Gi - requests: - cpu: "30" - memory: 300Gi - command: - - "/bin/sh" - - "-c" - - | - echo "Install dependencies..." - apt-get update - apt-get install -y libaio-dev gcc make git wget - - echo "Installing gsutil..." - apt-get update && apt-get install -y apt-transport-https ca-certificates gnupg curl - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg - echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list - apt-get update && apt-get install -y google-cloud-cli - - echo "Installing fio..." - git clone -b fio-3.36 https://github.com/axboe/fio.git - cd fio - sed -i 's/define \+FIO_IO_U_PLAT_GROUP_NR \+\([0-9]\+\)/define FIO_IO_U_PLAT_GROUP_NR 32/g' stat.h - ./configure && make && make install - cd .. - - echo "Generating data for file size: {{ .Values.fio.fileSize }}, file per thread: {{ .Values.fio.filesPerThread }} ..." - filename=/fio_dataloader_job.fio - {{ if eq .Values.fio.fileSize "200G" }} - cat > $filename << EOF - [global] - ioengine=libaio - direct=1 - fadvise_hint=0 - iodepth=64 - invalidate=1 - nrfiles=1 - thread=1 - openfiles=1 - group_reporting=1 - create_serialize=0 - allrandrepeat=1 - numjobs=1 - filename=/data/0 - - [Workload] - bs=1M - filesize=200G - size=2G - rw=read - offset=0 - offset_increment=1% - EOF - {{ else }} - wget -O $filename https://raw.githubusercontent.com/GoogleCloudPlatform/gcsfuse/master/perfmetrics/scripts/job_files/read_cache_load_test.fio - {{ end }} - - NUMJOBS=50 NRFILES={{ .Values.fio.filesPerThread }} FILE_SIZE={{ .Values.fio.fileSize }} BLOCK_SIZE={{ .Values.fio.blockSize }} READ_TYPE=read DIR=/data fio ${filename} --alloc-size=1048576 - - echo "Uploading data to bucket {{ .Values.bucketName }}..." - gsutil -m cp -R /data/* gs://{{ .Values.bucketName }} - volumeMounts: - - name: local-dir - mountPath: /data - volumes: - - name: local-dir - emptyDir: {} diff --git a/perfmetrics/scripts/testing_on_gke/examples/fio/data-loader/values.yaml b/perfmetrics/scripts/testing_on_gke/examples/fio/data-loader/values.yaml deleted file mode 100644 index 5f9a760f9a..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/fio/data-loader/values.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2018 The Kubernetes Authors. -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for data-loader. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -bucketName: gke-fio-64k-1m - -fio: - fileSize: 64K - blockSize: 64K - filesPerThread: "20000" diff --git a/perfmetrics/scripts/testing_on_gke/examples/fio/fio_workload.py b/perfmetrics/scripts/testing_on_gke/examples/fio/fio_workload.py deleted file mode 100644 index 70e5721947..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/fio/fio_workload.py +++ /dev/null @@ -1,218 +0,0 @@ -# Copyright 2018 The Kubernetes Authors. -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This file defines a FioWorkload and provides utility for parsing a json - -test-config file for a list of them. -""" - -import json - - -def validateFioWorkload(workload: dict, name: str): - """Validates the given json workload object.""" - for requiredWorkloadAttribute, expectedType in { - 'bucket': str, - 'gcsfuseMountOptions': str, - 'fioWorkload': dict, - }.items(): - if requiredWorkloadAttribute not in workload: - print(f"{name} does not have '{requiredWorkloadAttribute}' key in it.") - return False - if not type(workload[requiredWorkloadAttribute]) is expectedType: - print( - f"In {name}, the type of '{requiredWorkloadAttribute}' is of type" - f" '{type(workload[requiredWorkloadAttribute])}', not {expectedType}" - ) - return False - if expectedType == str and ' ' in workload[requiredWorkloadAttribute]: - print(f"{name} has space in the value of '{requiredWorkloadAttribute}'") - return False - - if 'dlioWorkload' in workload: - print(f"{name} has 'dlioWorkload' key in it, which is unexpected.") - return False - - fioWorkload = workload['fioWorkload'] - for requiredAttribute, expectedType in { - 'fileSize': str, - 'blockSize': str, - 'filesPerThread': int, - 'numThreads': int, - }.items(): - if requiredAttribute not in fioWorkload: - print(f'In {name}, fioWorkload does not have {requiredAttribute} in it') - return False - if not type(fioWorkload[requiredAttribute]) is expectedType: - print( - f'In {name}, fioWorkload[{requiredAttribute}] is of type' - f' {type(fioWorkload[requiredAttribute])}, expected:' - f' {expectedType} ' - ) - return False - - if 'readTypes' in fioWorkload: - readTypes = fioWorkload['readTypes'] - if not type(readTypes) is list: - print( - f"In {name}, fioWorkload['readTypes'] is of type {type(readTypes)}," - " not 'list'." - ) - return False - for readType in readTypes: - if not type(readType) is str: - print( - f'In {name}, one of the values in' - f" fioWorkload['readTypes'] is '{readType}', which is of type" - f' {type(readType)}, not str' - ) - return False - if not readType == 'read' and not readType == 'randread': - print( - f"In {name}, one of the values in fioWorkload['readTypes'] is" - f" '{readType}' which is not a supported value. Supported values" - ' are read, randread' - ) - return False - - return True - - -class FioWorkload: - """FioWorkload holds data needed to define a FIO workload - - (essentially the data needed to create a job file for FIO run). - - Members: - 1. scenario (string): One of "local-ssd", "gcsfuse-generic", - "gcsfuse-file-cache" and "gcsfuse-no-file-cache". - 2. fileSize (string): fio filesize field in string format e.g. '100', '10K', - '10M' etc. - 3. blockSize (string): equivalent of bs field in fio job file e.g. '8K', - '128K', '1M' etc. - 4. filesPerThreads (int): equivalent of nrfiles in fio job file. Must be - greater than 0. - 5. numThreads (int): equivalent of numjobs in fio job file. Must be greater - than 0. - 6. bucket (string): Name of a GCS bucket to read input files from. - 7. readTypes (set of strings): a set containing multiple values out of - 'read', 'randread'. - 8. gcsfuseMountOptions (str): gcsfuse mount options as a single - string in compact stringified format, to be used for the - test scenario "gcsfuse-generic". The individual config/cli flag values should - be separated by comma. Each cli flag should be of the form "<flag>[=<value>]", - while each config-file flag should be of form - "<config>[:<subconfig>[:<subsubconfig>[...]]]:<value>". For example, a legal - value would be: - "implicit-dirs,file_mode=777,file-cache:enable-parallel-downloads:true,metadata-cache:ttl-secs:true". - """ - - def __init__( - self, - scenario: str, - fileSize: str, - blockSize: str, - filesPerThread: int, - numThreads: int, - bucket: str, - readTypes: list, - gcsfuseMountOptions: str, - ): - self.scenario = scenario - self.fileSize = fileSize - self.blockSize = blockSize - self.filesPerThread = filesPerThread - self.numThreads = numThreads - self.bucket = bucket - self.readTypes = set(readTypes) - self.gcsfuseMountOptions = gcsfuseMountOptions - - def PPrint(self): - print( - f'scenario:{self.scenario}, fileSize:{self.fileSize},' - f' blockSize:{self.blockSize}, filesPerThread:{self.filesPerThread},' - f' numThreads:{self.numThreads}, bucket:{self.bucket},' - f' readTypes:{self.readTypes}, gcsfuseMountOptions:' - f' {gcsfuseMountOptions}' - ) - - -def ParseTestConfigForFioWorkloads(fioTestConfigFile: str): - """Parses the given workload test configuration file for FIO workloads.""" - print(f'Parsing {fioTestConfigFile} for FIO workloads ...') - with open(fioTestConfigFile) as f: - file = json.load(f) - testConfig = file['TestConfig'] - workloadConfig = testConfig['workloadConfig'] - workloads = workloadConfig['workloads'] - fioWorkloads = [] - scenarios = ( - ['local-ssd', 'gcsfuse-generic'] - if ('runOnSSD' not in workloadConfig or workloadConfig['runOnSSD']) - else ['gcsfuse-generic'] - ) - for i in range(len(workloads)): - workload = workloads[i] - if not validateFioWorkload(workload, f'workload#{i}'): - print(f'workloads#{i} is not a valid FIO workload, so ignoring it.') - else: - for scenario in scenarios: - fioWorkload = workload['fioWorkload'] - fioWorkloads.append( - FioWorkload( - scenario, - fioWorkload['fileSize'], - fioWorkload['blockSize'], - fioWorkload['filesPerThread'], - fioWorkload['numThreads'], - workload['bucket'], - ( - fioWorkload['readTypes'] - if 'readTypes' in fioWorkload - else ['read', 'randread'] - ), - workload['gcsfuseMountOptions'], - ) - ) - return fioWorkloads - - -def FioChartNamePodName( - fioWorkload: FioWorkload, instanceID: str, readType: str -) -> (str, str, str): - shortenScenario = { - 'local-ssd': 'ssd', - 'gcsfuse-generic': 'gcsfuse', - } - shortForScenario = ( - shortenScenario[fioWorkload.scenario] - if fioWorkload.scenario in shortenScenario - else 'other' - ) - readTypeToShortReadType = {'read': 'sr', 'randread': 'rr'} - shortForReadType = ( - readTypeToShortReadType[readType] - if readType in readTypeToShortReadType - else 'ur' - ) - - hashOfWorkload = str(hash((fioWorkload, instanceID, readType))).replace( - '-', '' - ) - return ( - f'fio-load-{shortForScenario}-{shortForReadType}-{fioWorkload.fileSize.lower()}-{hashOfWorkload}', - f'fio-tester-{shortForScenario}-{shortForReadType}-{fioWorkload.fileSize.lower()}-{hashOfWorkload}', - f'{instanceID}/{fioWorkload.fileSize}-{fioWorkload.blockSize}-{fioWorkload.numThreads}-{fioWorkload.filesPerThread}-{hashOfWorkload}/{fioWorkload.scenario}/{readType}', - ) diff --git a/perfmetrics/scripts/testing_on_gke/examples/fio/fio_workload_test.py b/perfmetrics/scripts/testing_on_gke/examples/fio/fio_workload_test.py deleted file mode 100644 index 8e85e93b14..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/fio/fio_workload_test.py +++ /dev/null @@ -1,349 +0,0 @@ -# Copyright 2018 The Kubernetes Authors. -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This file defines unit tests for functionalities in fio_workload.py""" - -import unittest -from fio_workload import FioWorkload, validateFioWorkload - - -class FioWorkloadTest(unittest.TestCase): - - def test_validate_fio_workload_empty(self): - self.assertFalse(validateFioWorkload(({}), "empty-fio-workload")) - - def test_validate_fio_workload_invalid_missing_bucket(self): - self.assertFalse( - validateFioWorkload( - ({"fioWorkload": {}, "gcsfuseMountOptions": ""}), - "invalid-fio-workload-missing-bucket", - ) - ) - - def test_validate_fio_workload_invalid_bucket_contains_space(self): - self.assertFalse( - validateFioWorkload( - ({"fioWorkload": {}, "gcsfuseMountOptions": "", "bucket": " "}), - "invalid-fio-workload-bucket-contains-space", - ) - ) - - def test_validate_fio_workload_invalid_no_fioWorkloadSpecified(self): - self.assertFalse( - validateFioWorkload(({"bucket": {}}), "invalid-fio-workload-2") - ) - - def test_validate_fio_workload_invalid_commented_out_fioWorkload(self): - self.assertFalse( - validateFioWorkload( - ({ - "_fioWorkload": {}, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }), - "commented-out-fio-workload", - ) - ) - - def test_validate_fio_workload_invalid_mixed_fioWorkload_dlioWorkload(self): - self.assertFalse( - validateFioWorkload( - ({ - "fioWorkload": {}, - "dlioWorkload": {}, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }), - "mixed-fio/dlio-workload", - ) - ) - - def test_validate_fio_workload_invalid_missing_fileSize(self): - workload = dict({ - "fioWorkload": { - "filesPerThread": 2, - "numThreads": 100, - "blockSize": "1kb", - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertFalse( - validateFioWorkload(workload, "invalid-fio-workload-missing-fileSize") - ) - - def test_validate_fio_workload_invalid_unsupported_fileSize(self): - workload = dict({ - "fioWorkload": { - "fileSize": 1000, - "filesPerThread": 2, - "numThreads": 100, - "blockSize": "1kb", - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertFalse( - validateFioWorkload( - workload, "invalid-fio-workload-unsupported-fileSize" - ) - ) - - def test_validate_fio_workload_invalid_missing_blockSize(self): - workload = dict({ - "fioWorkload": { - "fileSize": "1kb", - "filesPerThread": 2, - "numThreads": 100, - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertFalse( - validateFioWorkload(workload, "invalid-fio-workload-missing-blockSize") - ) - - def test_validate_fio_workload_invalid_unsupported_blockSize(self): - workload = dict({ - "fioWorkload": { - "fileSize": "1kb", - "blockSize": 1000, - "filesPerThread": 2, - "numThreads": 100, - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertFalse( - validateFioWorkload( - workload, "invalid-fio-workload-unsupported-blockSize" - ) - ) - - def test_validate_fio_workload_invalid_missing_filesPerThread(self): - workload = dict({ - "fioWorkload": { - "fileSize": "1kb", - "numThreads": 100, - "blockSize": "1kb", - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertFalse( - validateFioWorkload( - workload, "invalid-fio-workload-missing-filesPerThread" - ) - ) - - def test_validate_fio_workload_invalid_unsupported_filesPerThread(self): - workload = dict({ - "fioWorkload": { - "fileSize": "1kb", - "filesPerThread": "1k", - "numThreads": 100, - "blockSize": "1kb", - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertFalse( - validateFioWorkload( - workload, "invalid-fio-workload-unsupported-filesPerThread" - ) - ) - - def test_validate_fio_workload_invalid_missing_numThreads(self): - workload = dict({ - "fioWorkload": { - "fileSize": "1kb", - "filesPerThread": 2, - "blockSize": "1kb", - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertFalse( - validateFioWorkload(workload, "invalid-fio-workload-missing-numThreads") - ) - - def test_validate_fio_workload_invalid_unsupported_numThreads(self): - workload = dict({ - "fioWorkload": { - "fileSize": "1kb", - "filesPerThread": 2, - "blockSize": "1kb", - "numThreads": "1k", - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertFalse( - validateFioWorkload( - workload, "invalid-fio-workload-unsupported-numThreads" - ) - ) - - def test_validate_fio_workload_invalid_missing_gcsfuseMountOptions(self): - workload = dict({ - "fioWorkload": { - "fileSize": "1kb", - "filesPerThread": 2, - "blockSize": "1kb", - "numThreads": "1k", - }, - "bucket": "dummy-bucket", - }) - self.assertFalse( - validateFioWorkload( - workload, "invalid-fio-workload-missing-gcsfuseMountOptions" - ) - ) - - def test_validate_fio_workload_invalid_unsupported_gcsfuseMountOptions(self): - workload = dict({ - "fioWorkload": { - "fileSize": "1kb", - "filesPerThread": 2, - "blockSize": "1kb", - "numThreads": "1k", - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": 100, - }) - self.assertFalse( - validateFioWorkload( - workload, "invalid-fio-workload-unsupported-numThreads" - ) - ) - - def test_validate_fio_workload_invalid_gcsfuseMountOptions_contains_space( - self, - ): - workload = dict({ - "fioWorkload": { - "fileSize": "1kb", - "filesPerThread": 2, - "blockSize": "1kb", - "numThreads": "1k", - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "abc def", - }) - self.assertFalse( - validateFioWorkload( - workload, - "invalid-fio-workload-unsupported-gcsfuseMountOptions-contains-space", - ) - ) - - def test_validate_fio_workload_invalid_unsupported_readTypes_1(self): - workload = dict({ - "fioWorkload": { - "fileSize": "1kb", - "filesPerThread": 2, - "blockSize": "1kb", - "numThreads": 10, - "readTypes": True, - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertFalse( - validateFioWorkload( - workload, "invalid-fio-workload-unsupported-readTypes-1" - ) - ) - - def test_validate_fio_workload_invalid_unsupported_readTypes_2(self): - workload = dict({ - "fioWorkload": { - "fileSize": "1kb", - "filesPerThread": 2, - "blockSize": "1kb", - "numThreads": 10, - "readTypes": ["read", 1], - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertFalse( - validateFioWorkload( - workload, "invalid-fio-workload-unsupported-readTypes-2" - ) - ) - - def test_validate_fio_workload_invalid_unsupported_readTypes_3(self): - workload = dict({ - "fioWorkload": { - "fileSize": "1kb", - "filesPerThread": 2, - "blockSize": "1kb", - "numThreads": 10, - "readTypes": ["read", "write"], - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertFalse( - validateFioWorkload( - workload, "invalid-fio-workload-unsupported-readTypes-3" - ) - ) - - def test_validate_fio_workload_valid_without_readTypes(self): - workload = dict({ - "fioWorkload": { - "fileSize": "1kb", - "filesPerThread": 2, - "numThreads": 100, - "blockSize": "1kb", - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertTrue(validateFioWorkload(workload, "valid-fio-workload-1")) - - def test_validate_fio_workload_valid_with_readTypes(self): - workload = dict({ - "fioWorkload": { - "fileSize": "1kb", - "filesPerThread": 2, - "numThreads": 100, - "blockSize": "1kb", - "readTypes": ["read", "randread"], - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertTrue(validateFioWorkload(workload, "valid-fio-workload-2")) - - def test_validate_fio_workload_valid_with_single_readType(self): - workload = dict({ - "fioWorkload": { - "fileSize": "1kb", - "filesPerThread": 2, - "numThreads": 100, - "blockSize": "1kb", - "readTypes": ["randread"], - }, - "bucket": "dummy-bucket", - "gcsfuseMountOptions": "implicit-dirs,cache-max-size:-1", - }) - self.assertTrue(validateFioWorkload(workload, "valid-fio-workload-2")) - - -if __name__ == "__main__": - unittest.main() diff --git a/perfmetrics/scripts/testing_on_gke/examples/fio/loading-test/.helmignore b/perfmetrics/scripts/testing_on_gke/examples/fio/loading-test/.helmignore deleted file mode 100644 index 0e8a0eb36f..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/fio/loading-test/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/perfmetrics/scripts/testing_on_gke/examples/fio/loading-test/Chart.yaml b/perfmetrics/scripts/testing_on_gke/examples/fio/loading-test/Chart.yaml deleted file mode 100644 index 94d6a50e7b..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/fio/loading-test/Chart.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2018 The Kubernetes Authors. -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v2 -name: fio-loading-test -description: A Helm chart for FIO loading test -type: application -version: 0.1.0 diff --git a/perfmetrics/scripts/testing_on_gke/examples/fio/loading-test/templates/fio-tester.yaml b/perfmetrics/scripts/testing_on_gke/examples/fio/loading-test/templates/fio-tester.yaml deleted file mode 100644 index 3cbc9e6c78..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/fio/loading-test/templates/fio-tester.yaml +++ /dev/null @@ -1,203 +0,0 @@ -# Copyright 2018 The Kubernetes Authors. -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Pod -metadata: - name: {{ .Values.podName }} - {{- if ne .Values.scenario "local-ssd" }} - annotations: - gke-gcsfuse/volumes: "true" - {{- end }} -spec: - restartPolicy: Never - nodeSelector: - cloud.google.com/gke-ephemeral-storage-local-ssd: "true" - node.kubernetes.io/instance-type: {{ .Values.nodeType }} - containers: - - name: fio-tester - image: {{ .Values.image }} - securityContext: # for cache dropping in the benchmarking tests. - privileged: true - resources: - limits: - cpu: {{ .Values.resourceLimits.cpu }} - memory: {{ .Values.resourceLimits.memory }} - requests: - cpu: {{ .Values.resourceRequests.cpu }} - memory: {{ .Values.resourceRequests.memory }} - command: - - "/bin/sh" - - "-c" - - | - echo "Install dependencies..." - apt-get update - apt-get install -y libaio-dev gcc make git time wget - - no_of_files_per_thread={{ .Values.fio.filesPerThread }} - block_size={{ .Values.fio.blockSize }} - file_size={{ .Values.fio.fileSize }} - num_of_threads={{ .Values.fio.numThreads }} - - {{ if eq .Values.scenario "local-ssd" }} - echo "Installing gsutil..." - apt-get update && apt-get install -y apt-transport-https ca-certificates gnupg curl - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg - echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list - apt-get update && apt-get install -y google-cloud-cli - - gsutil -m cp -R gs://{{ .Values.bucketName }}/* /data - - echo "Sleeping 5 minutes to wait for Local SSD RAID to populate data." - sleep 300 - {{ end }} - - # We are building fio from source because of the issue: https://github.com/axboe/fio/issues/1668. - # The sed command below is to address internal bug b/309563824. - # As recorded in this bug, fio by-default supports - # clat percentile values to be calculated accurately upto only - # 2^(FIO_IO_U_PLAT_GROUP_NR + 5) ns = 17.17 seconds. - # (with default value of FIO_IO_U_PLAT_GROUP_NR = 29). This change increases it upto 32, to allow - # latencies upto 137.44s to be calculated accurately. - git clone -b fio-3.36 https://github.com/axboe/fio.git - cd fio - sed -i 's/define \+FIO_IO_U_PLAT_GROUP_NR \+\([0-9]\+\)/define FIO_IO_U_PLAT_GROUP_NR 32/g' stat.h - ./configure && make && make install - cd .. - - echo "Preparing fio config file..." - filename=/fio_loading_test_job.fio - {{ if eq .Values.fio.fileSize "200G" }} - cat > $filename << EOF - [global] - ioengine=libaio - direct=1 - fadvise_hint=0 - iodepth=64 - invalidate=1 - nrfiles=1 - thread=1 - openfiles=1 - group_reporting=1 - create_serialize=0 - allrandrepeat=1 - numjobs=100 - filename=/data/0 - - [Workload] - bs=1M - filesize=200G - size=2G - rw={{ .Values.fio.readType }} - offset=0 - offset_increment=1% - EOF - {{ else }} - wget -O $filename https://raw.githubusercontent.com/GoogleCloudPlatform/gcsfuse/master/perfmetrics/scripts/job_files/read_cache_load_test.fio - {{ end }} - - echo "Setup default values..." - epoch=4 - read_type={{ .Values.fio.readType }} - pause_in_seconds=20 - workload_dir=/data - - # Cleaning the pagecache, dentries and inode cache before the starting the workload. - echo "Drop page cache..." - echo 3 > /proc/sys/vm/drop_caches - - # Specially for gcsfuse mounted dir: the purpose of this approach is to efficiently - # populate the gcsfuse metadata cache by utilizing the list call, which internally - # works like bulk stat call rather than making individual stat calls. - # And to reduce the logs redirecting the command standard-output to /dev/null. - echo "List workload dir..." - time ls -R $workload_dir 1> /dev/null - - echo "Run fio tests..." - output_dir=/data/fio-output/{{ .Values.outputDirPrefix }} - mkdir -p ${output_dir} - - # dump the gcsfuse-mount-configuration to a file in output-directory. - {{ if eq .Values.scenario "gcsfuse-generic" }} - echo "{{ .Values.gcsfuse.mountOptions }}" > ${output_dir}/gcsfuse_mount_options - {{ end }} - echo "{{ .Values.podName }}" > ${output_dir}/pod_name - - for i in $(seq $epoch); do - echo "[Epoch ${i}] start time:" `date +%s` - free -mh # Memory usage before workload start. - NUMJOBS=$num_of_threads NRFILES=$no_of_files_per_thread FILE_SIZE=$file_size BLOCK_SIZE=$block_size READ_TYPE=$read_type DIR=$workload_dir fio ${filename} --alloc-size=1048576 --output-format=json --output="${output_dir}/epoch${i}.json" - free -mh # Memory usage after workload completion. - echo "[Epoch ${i}] end time:" `date +%s` - - # To free pagecache. - # Intentionally not clearing dentries and inodes: clearing them - # will necessitate the repopulation of the type cache in gcsfuse 2nd epoch onwards. - # Since we use "ls -R workload_dir" to populate the cache (sort of hack to fill the cache quickly) - # efficiently in the first epoch, it does not populate the negative - # entry for the stat cache. - # So just to stop the execution of “ls -R workload_dir” command at the start - # of every epoch, not clearing the inodes. - echo 1 > /proc/sys/vm/drop_caches - - sleep $pause_in_seconds - done - - {{ if eq .Values.scenario "local-ssd" }} - gsutil -m cp -R /data/fio-output/* gs://{{ .Values.bucketName }}/fio-output/ - {{ end }} - - echo "fio job completed!" - volumeMounts: - - name: dshm - mountPath: /dev/shm - - name: data-vol - mountPath: /data - volumes: - - name: dshm - emptyDir: - medium: Memory - - name: data-vol - {{- if eq .Values.scenario "local-ssd" }} - emptyDir: {} - {{- else if eq .Values.scenario "gcsfuse-generic" }} - csi: - driver: gcsfuse.csi.storage.gke.io - volumeAttributes: - bucketName: {{ .Values.bucketName }} - mountOptions: "{{ .Values.gcsfuse.mountOptions }}" - {{- else if eq .Values.scenario "gcsfuse-file-cache" }} - csi: - driver: gcsfuse.csi.storage.gke.io - volumeAttributes: - bucketName: {{ .Values.bucketName }} - metadataCacheTTLSeconds: "{{ .Values.gcsfuse.metadataCacheTTLSeconds }}" - metadataTypeCacheCapacity: "{{ .Values.gcsfuse.metadataTypeCacheCapacity }}" - metadataStatCacheCapacity: "{{ .Values.gcsfuse.metadataStatCacheCapacity }}" - fileCacheCapacity: "{{ .Values.gcsfuse.fileCacheCapacity }}" - fileCacheForRangeRead: "true" - gcsfuseLoggingSeverity: "debug" - mountOptions: implicit-dirs - {{- else }} - csi: - driver: gcsfuse.csi.storage.gke.io - volumeAttributes: - bucketName: {{ .Values.bucketName }} - metadataCacheTTLSeconds: "{{ .Values.gcsfuse.metadataCacheTTLSeconds }}" - metadataTypeCacheCapacity: "{{ .Values.gcsfuse.metadataTypeCacheCapacity }}" - metadataStatCacheCapacity: "{{ .Values.gcsfuse.metadataStatCacheCapacity }}" - gcsfuseLoggingSeverity: "debug" - mountOptions: implicit-dirs - {{- end }} diff --git a/perfmetrics/scripts/testing_on_gke/examples/fio/loading-test/values.yaml b/perfmetrics/scripts/testing_on_gke/examples/fio/loading-test/values.yaml deleted file mode 100644 index 6723168b88..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/fio/loading-test/values.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2018 The Kubernetes Authors. -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for unet3d-loading-test. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -image: ubuntu:24.04 -bucketName: gke-dlio-test-data -# scenario controls the kind of storage that is used for the load testing. local-ssd means directly on LSSD; gcsfuse-generic means on a gcsfuse mount with gcsfuse.mountOptions sent from the caller; gcsfuse-no-file-cache and gcsfuse-file-cache mean as their name suggests. -scenario: local-ssd -nodeType: n2-standard-96 -instanceId: ldap-yyyymmdd-hhmmss -podName: -outputDirPrefix: - -resourceLimits: - cpu: 0 - memory: 0 -resourceRequests: - cpu: 0 - memory: 0 - -fio: - readType: read - fileSize: 64K - blockSize: 64K - filesPerThread: "20000" - numThreads: "50" - -gcsfuse: - metadataCacheTTLSeconds: "6048000" - metadataStatCacheCapacity: "-1" - metadataTypeCacheCapacity: "-1" - fileCacheCapacity: "-1" - fileCacheForRangeRead: "true" - mountOptions: "implicit-dirs" - diff --git a/perfmetrics/scripts/testing_on_gke/examples/fio/parse_logs.py b/perfmetrics/scripts/testing_on_gke/examples/fio/parse_logs.py deleted file mode 100644 index 7d5b5f2074..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/fio/parse_logs.py +++ /dev/null @@ -1,348 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018 The Kubernetes Authors. -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# standard library imports -import argparse -import json -import os -import pprint -import subprocess -import sys - -# local library imports -sys.path.append("../") -import fio_workload -from utils.utils import get_memory, get_cpu, unix_to_timestamp, is_mash_installed, get_memory_from_monitoring_api, get_cpu_from_monitoring_api -from utils.parse_logs_common import ensure_directory_exists, download_gcs_objects, parse_arguments, SUPPORTED_SCENARIOS - -_LOCAL_LOGS_LOCATION = "../../bin/fio-logs" - -record = { - "pod_name": "", - "epoch": 0, - "scenario": "", - "duration": 0, - "IOPS": 0, - "throughput_mb_per_second": 0, - "throughput_over_local_ssd": 0, - "start_epoch": "", - "end_epoch": "", - "start": "", - "end": "", - "highest_memory": 0, - "lowest_memory": 0, - "highest_cpu": 0.0, - "lowest_cpu": 0.0, - "gcsfuse_mount_options": "", - "blockSize": "", - "filesPerThread": 0, - "numThreads": 0, -} - - -def downloadFioOutputs(fioWorkloads: set, instanceId: str) -> int: - """Downloads instanceId-specific fio outputs for each fioWorkload locally. - - Outputs in the bucket are in the following object naming format - (details in ./loading-test/templates/fio-tester.yaml). - gs://<bucket>/fio-output/<instanceId>/<fileSize>-<blockSize>-<numThreads>-<filesPerThread>-<hash>/<scenario>/<readType>/epoch[N].json - gs://<bucket>/fio-output/<instanceId>/<fileSize>-<blockSize>-<numThreads>-<filesPerThread>-<hash>/<scenario>/<readType>/pod_name - gs://<bucket>/fio-output/<instanceId>/<fileSize>-<blockSize>-<numThreads>-<filesPerThread>-<hash>/gcsfuse-generic/<readType>/gcsfuse_mount_options - - These are downloaded locally as: - <_LOCAL_LOGS_LOCATION>/<instanceId>/<fileSize>/<fileSize>-<blockSize>-<numThreads>-<filesPerThread>-<hash>/<scenario>/<readType>/epoch[N].json - <_LOCAL_LOGS_LOCATION>/<instanceId>/<fileSize>/<fileSize>-<blockSize>-<numThreads>-<filesPerThread>-<hash>/<scenario>/<readType>/pod_name - <_LOCAL_LOGS_LOCATION>/<instanceId>/<fileSize>/<fileSize>-<blockSize>-<numThreads>-<filesPerThread>-<hash>/gcsfuse-generic/<readType>/gcsfuse_mount_options - """ - - for fioWorkload in fioWorkloads: - dstDir = ( - _LOCAL_LOGS_LOCATION + "/" + instanceId + "/" + fioWorkload.fileSize - ) - ensure_directory_exists(dstDir) - - srcObjects = f"gs://{fioWorkload.bucket}/fio-output/{instanceId}/*" - print(f"Downloading FIO outputs from {srcObjects} ...") - returncode, errorStr = download_gcs_objects(srcObjects, dstDir) - if returncode < 0: - print(f"Failed to download FIO outputs from {srcObjects}: {errorStr}") - return returncode - return 0 - - -def createOutputScenariosFromDownloadedFiles(args: dict) -> dict: - """Creates output records from the downloaded local files. - - The following creates a dict called 'output' - from the downloaded fio output files, which are in the following format. - - <_LOCAL_LOGS_LOCATION>/<instanceId>/<fileSize>/<fileSize>-<blockSize>-<numThreads>-<filesPerThread>-<hash>/<scenario>/<readType>/epoch[N].json - where N=1-4 - <_LOCAL_LOGS_LOCATION>/<instanceId>/<fileSize>/<fileSize>-<blockSize>-<numThreads>-<filesPerThread>-<hash>/<scenario>/<readType>/pod_name - <_LOCAL_LOGS_LOCATION>/<instanceId>/<fileSize>/<fileSize>-<blockSize>-<numThreads>-<filesPerThread>-<hash>/gcsfuse-generic/<readType>/gcsfuse_mount_options - - Output dict structure: - "{read_type}-{mean_file_size}-{bs}-{numjobs}-{nrfiles}": - "mean_file_size": str - "read_type": str - "records": - "local-ssd": [record1, record2, record3, record4] - "gcsfuse-generic": [record1, record2, record3, record4] - "gcsfuse-file-cache": [record1, record2, record3, record4] - "gcsfuse-no-file-cache": [record1, record2, record3, record4] - """ - - output = {} - for root, _, files in os.walk(_LOCAL_LOGS_LOCATION + "/" + args.instance_id): - print(f"Parsing directory {root} ...") - - if not files: - # ignore intermediate directories. - continue - - # if directory contains gcsfuse_mount_options file, then parse gcsfuse - # mount options from it in record. - gcsfuse_mount_options = "" - gcsfuse_mount_options_file = root + "/gcsfuse_mount_options" - if os.path.isfile(gcsfuse_mount_options_file): - with open(gcsfuse_mount_options_file) as f: - gcsfuse_mount_options = f.read().strip() - print(f"gcsfuse_mount_options={gcsfuse_mount_options}") - - # if directory has files, it must also contain pod_name file, - # and we should extract pod-name from it in the record. - pod_name = "" - pod_name_file = root + "/pod_name" - with open(pod_name_file) as f: - pod_name = f.read().strip() - print(f"pod_name={pod_name}") - - for file in files: - # Ignore non-json files to avoid unnecessary failure. - if not file.endswith(".json"): - continue - - per_epoch_output = root + f"/{file}" - with open(per_epoch_output, "r") as f: - try: - per_epoch_output_data = json.load(f) - except: - print(f"failed to json-parse {per_epoch_output}, so skipping it.") - continue - - # Confirm that the per_epoch_output_data has ["jobs"][0]["job options"]['"bs"] - # for determining blocksize. - if ( - not "jobs" in per_epoch_output_data - or not per_epoch_output_data["jobs"] - or not "job options" in per_epoch_output_data["jobs"][0] - or not "bs" in per_epoch_output_data["jobs"][0]["job options"] - ): - print( - 'Did not find "[jobs][0][job options][bs]" in' - f" {per_epoch_output}, so ignoring this file" - ) - continue - # Confirm that the per_epoch_output_data has ["global options"] for - # determining nrfiles and numjobs in it. - if "global options" not in per_epoch_output_data: - print(f"field: 'global options' missing in {per_epoch_output}") - continue - - # This print is for debugging in case something goes wrong. - print(f"Now parsing file {per_epoch_output} ...") - - # Get fileSize, readType, echo number from the file path. - root_split = root.split("/") - mean_file_size = root_split[-4] - key = root_split[ - -3 - ] # key is unique for a given combination of of fileSize,blockSize,numThreads(numjobs),filesPerThread(nrfiles). - scenario = root_split[-2] - read_type = root_split[-1] - epoch = int(file.split(".")[0][-1]) - - # Get nrfiles,numjobs, blocksize from ["global options"] and ["job options"]. - global_options = per_epoch_output_data["global options"] - nrfiles = int(global_options["nrfiles"]) - numjobs = int(global_options["numjobs"]) - bs = per_epoch_output_data["jobs"][0]["job options"]["bs"] - - # If the record for this key has not been added, create a new entry - # for it. - if key not in output: - output[key] = { - "mean_file_size": mean_file_size, - "read_type": read_type, - "records": { - "local-ssd": [], - "gcsfuse-generic": [], - "gcsfuse-file-cache": [], - "gcsfuse-no-file-cache": [], - }, - } - - # Create a record for this key. - r = record.copy() - bs = per_epoch_output_data["jobs"][0]["job options"]["bs"] - r["pod_name"] = pod_name - r["epoch"] = epoch - r["scenario"] = scenario - r["duration"] = int( - per_epoch_output_data["jobs"][0]["read"]["runtime"] / 1000 - ) - r["IOPS"] = int(per_epoch_output_data["jobs"][0]["read"]["iops"]) - r["throughput_mb_per_second"] = int( - per_epoch_output_data["jobs"][0]["read"]["bw_bytes"] / (1024**2) - ) - r["start_epoch"] = per_epoch_output_data["jobs"][0]["job_start"] // 1000 - r["end_epoch"] = per_epoch_output_data["timestamp_ms"] // 1000 - r["start"] = unix_to_timestamp( - per_epoch_output_data["jobs"][0]["job_start"] - ) - r["end"] = unix_to_timestamp(per_epoch_output_data["timestamp_ms"]) - - if r["scenario"] != "local-ssd": - if mash_installed: - r["lowest_memory"], r["highest_memory"] = get_memory( - r["pod_name"], - r["start"], - r["end"], - project_number=args.project_number, - ) - r["lowest_cpu"], r["highest_cpu"] = get_cpu( - r["pod_name"], - r["start"], - r["end"], - project_number=args.project_number, - ) - else: - r["lowest_memory"], r["highest_memory"] = ( - get_memory_from_monitoring_api( - pod_name=r["pod_name"], - start_epoch=r["start_epoch"], - end_epoch=r["end_epoch"], - project_id=args.project_id, - cluster_name=args.cluster_name, - namespace_name=args.namespace_name, - ) - ) - r["lowest_cpu"], r["highest_cpu"] = get_cpu_from_monitoring_api( - pod_name=r["pod_name"], - start_epoch=r["start_epoch"], - end_epoch=r["end_epoch"], - project_id=args.project_id, - cluster_name=args.cluster_name, - namespace_name=args.namespace_name, - ) - - r["gcsfuse_mount_options"] = gcsfuse_mount_options - r["blockSize"] = bs - r["filesPerThread"] = nrfiles - r["numThreads"] = numjobs - - # This print is for debugging in case something goes wrong. - pprint.pprint(r) - - # If a slot for record for this particular epoch has not been created yet, - # append enough empty records to make a slot. - while len(output[key]["records"][scenario]) < epoch: - output[key]["records"][scenario].append({}) - - # Insert the record at the appropriate slot. - output[key]["records"][scenario][epoch - 1] = r - - return output - - -def writeRecordsToCsvOutputFile(output: dict, output_file_path: str): - with open(output_file_path, "a") as output_file_fwr: - # Write a new header. - output_file_fwr.write( - "File Size,Read Type,Scenario,Epoch,Duration" - " (s),Throughput (MB/s),IOPS,Throughput over Local SSD (%),GCSFuse" - " Lowest" - " Memory (MB),GCSFuse Highest Memory (MB),GCSFuse Lowest CPU" - " (core),GCSFuse Highest CPU" - " (core),Pod,Start,End,GcsfuseMoutOptions,BlockSize,FilesPerThread,NumThreads,InstanceID\n" - ) - - for key in output: - record_set = output[key] - - for scenario in record_set["records"]: - if scenario not in SUPPORTED_SCENARIOS: - print(f"Unknown scenario: {scenario}. Ignoring it...") - continue - - for i in range(len(record_set["records"][scenario])): - r = record_set["records"][scenario][i] - - try: - if ("local-ssd" in record_set["records"]) and ( - len(record_set["records"]["local-ssd"]) - == len(record_set["records"][scenario]) - ): - r["throughput_over_local_ssd"] = round( - r["throughput_mb_per_second"] - / record_set["records"]["local-ssd"][i][ - "throughput_mb_per_second" - ] - * 100, - 2, - ) - else: - r["throughput_over_local_ssd"] = "NA" - - except Exception as e: - print( - "Error: failed to parse/write record-set for" - f" scenario: {scenario}, i: {i}, record: {r}, exception: {e}" - ) - continue - - output_file_fwr.write( - f"{record_set['mean_file_size']},{record_set['read_type']},{scenario},{r['epoch']},{r['duration']},{r['throughput_mb_per_second']},{r['IOPS']},{r['throughput_over_local_ssd']},{r['lowest_memory']},{r['highest_memory']},{r['lowest_cpu']},{r['highest_cpu']},{r['pod_name']},{r['start']},{r['end']},\"{r['gcsfuse_mount_options']}\",{r['blockSize']},{r['filesPerThread']},{r['numThreads']},{args.instance_id}\n" - ) - - output_file_fwr.close() - - -if __name__ == "__main__": - args = parse_arguments() - ensure_directory_exists(_LOCAL_LOGS_LOCATION) - - fioWorkloads = fio_workload.ParseTestConfigForFioWorkloads( - args.workload_config - ) - downloadFioOutputs(fioWorkloads, args.instance_id) - - mash_installed = is_mash_installed() - if not mash_installed: - print("Mash is not installed, will skip parsing CPU and memory usage.") - - output = createOutputScenariosFromDownloadedFiles(args) - - output_file_path = args.output_file - # Create the parent directory of output_file_path if doesn't - # exist already. - ensure_directory_exists(os.path.dirname(output_file_path)) - writeRecordsToCsvOutputFile(output, output_file_path) - print( - "\n\nSuccessfully published outputs of FIO test runs to" - f" {output_file_path} !!!\n\n" - ) diff --git a/perfmetrics/scripts/testing_on_gke/examples/fio/run_tests.py b/perfmetrics/scripts/testing_on_gke/examples/fio/run_tests.py deleted file mode 100644 index e6eceaa999..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/fio/run_tests.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018 The Kubernetes Authors. -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Generates and deploys helm charts for FIO workloads. - -This program takes in a json test-config file, finds out valid FIO workloads in -it and generates and deploys a helm chart for each valid FIO workload. -""" - -# system imports -import argparse -import os -import subprocess -import sys - -# local imports from other directories -sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'utils')) -from run_tests_common import escape_commas_in_string, parse_args, run_command, add_iam_role_for_buckets -from utils import UnknownMachineTypeError, resource_limits - -# local imports from same directory -import fio_workload - - -def createHelmInstallCommands( - fioWorkloads: set, - instanceId: str, - machineType: str, -) -> list: - """Creates helm install commands for the given fioWorkload objects.""" - helm_commands = [] - try: - resourceLimits, resourceRequests = resource_limits(machineType) - except UnknownMachineTypeError: - print( - f'Found unknown machine-type: {machineType}, defaulting resource limits' - ' to cpu=0,memory=0' - ) - resourceLimits = {'cpu': 0, 'memory': '0'} - resourceRequests = resourceLimits - - for fioWorkload in fioWorkloads: - for readType in fioWorkload.readTypes: - chartName, podName, outputDirPrefix = fio_workload.FioChartNamePodName( - fioWorkload, instanceId, readType - ) - commands = [ - f'helm install {chartName} loading-test', - f'--set bucketName={fioWorkload.bucket}', - f'--set scenario={fioWorkload.scenario}', - f'--set fio.readType={readType}', - f'--set fio.fileSize={fioWorkload.fileSize}', - f'--set fio.blockSize={fioWorkload.blockSize}', - f'--set fio.filesPerThread={fioWorkload.filesPerThread}', - f'--set fio.numThreads={fioWorkload.numThreads}', - f'--set instanceId={instanceId}', - ( - '--set' - f' gcsfuse.mountOptions={escape_commas_in_string(fioWorkload.gcsfuseMountOptions)}' - ), - f'--set nodeType={machineType}', - f'--set podName={podName}', - f'--set outputDirPrefix={outputDirPrefix}', - f"--set resourceLimits.cpu={resourceLimits['cpu']}", - f"--set resourceLimits.memory={resourceLimits['memory']}", - f"--set resourceRequests.cpu={resourceRequests['cpu']}", - f"--set resourceRequests.memory={resourceRequests['memory']}", - ] - - helm_command = ' '.join(commands) - helm_commands.append(helm_command) - return helm_commands - - -def main(args) -> None: - fioWorkloads = fio_workload.ParseTestConfigForFioWorkloads( - args.workload_config - ) - helmInstallCommands = createHelmInstallCommands( - fioWorkloads, - args.instance_id, - args.machine_type, - ) - buckets = (fioWorkload.bucket for fioWorkload in fioWorkloads) - role = 'roles/storage.objectUser' - add_iam_role_for_buckets( - buckets, - role, - args.project_id, - args.project_number, - args.namespace, - args.ksa, - ) - for helmInstallCommand in helmInstallCommands: - print(f'{helmInstallCommand}') - if not args.dry_run: - run_command(helmInstallCommand) - - -if __name__ == '__main__': - args = parse_args() - main(args) diff --git a/perfmetrics/scripts/testing_on_gke/examples/requirements.in b/perfmetrics/scripts/testing_on_gke/examples/requirements.in new file mode 100644 index 0000000000..c25d29e047 --- /dev/null +++ b/perfmetrics/scripts/testing_on_gke/examples/requirements.in @@ -0,0 +1,6 @@ +absl-py +google-cloud-storage +google-api-python-client +google-cloud +google-cloud-monitoring +google-cloud-bigquery diff --git a/perfmetrics/scripts/testing_on_gke/examples/requirements.txt b/perfmetrics/scripts/testing_on_gke/examples/requirements.txt new file mode 100644 index 0000000000..6c2ea73c77 --- /dev/null +++ b/perfmetrics/scripts/testing_on_gke/examples/requirements.txt @@ -0,0 +1,385 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --generate-hashes perfmetrics/scripts/testing_on_gke/examples/requirements.in +# +absl-py==2.3.1 \ + --hash=sha256:a97820526f7fbfd2ec1bce83f3f25e3a14840dac0d8e02a0b71cd75db3f77fc9 \ + --hash=sha256:eeecf07f0c2a93ace0772c92e596ace6d3d3996c042b2128459aaae2a76de11d + # via -r requirements.in +cachetools==6.2.2 \ + --hash=sha256:6c09c98183bf58560c97b2abfcedcbaf6a896a490f534b031b661d3723b45ace \ + --hash=sha256:8e6d266b25e539df852251cfd6f990b4bc3a141db73b939058d809ebd2590fc6 + # via google-auth +certifi==2025.11.12 \ + --hash=sha256:97de8790030bbd5c2d96b7ec782fc2f7820ef8dba6db909ccf95449f2d062d4b \ + --hash=sha256:d8ab5478f2ecd78af242878415affce761ca6bc54a22a27e026d7c25357c3316 + # via requests +charset-normalizer==3.4.4 \ + --hash=sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad \ + --hash=sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93 \ + --hash=sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394 \ + --hash=sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89 \ + --hash=sha256:0f04b14ffe5fdc8c4933862d8306109a2c51e0704acfa35d51598eb45a1e89fc \ + --hash=sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86 \ + --hash=sha256:194f08cbb32dc406d6e1aea671a68be0823673db2832b38405deba2fb0d88f63 \ + --hash=sha256:1bee1e43c28aa63cb16e5c14e582580546b08e535299b8b6158a7c9c768a1f3d \ + --hash=sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f \ + --hash=sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8 \ + --hash=sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0 \ + --hash=sha256:2677acec1a2f8ef614c6888b5b4ae4060cc184174a938ed4e8ef690e15d3e505 \ + --hash=sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161 \ + --hash=sha256:2aaba3b0819274cc41757a1da876f810a3e4d7b6eb25699253a4effef9e8e4af \ + --hash=sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152 \ + --hash=sha256:2c9d3c380143a1fedbff95a312aa798578371eb29da42106a29019368a475318 \ + --hash=sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72 \ + --hash=sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4 \ + --hash=sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e \ + --hash=sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3 \ + --hash=sha256:44c2a8734b333e0578090c4cd6b16f275e07aa6614ca8715e6c038e865e70576 \ + --hash=sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c \ + --hash=sha256:4902828217069c3c5c71094537a8e623f5d097858ac6ca8252f7b4d10b7560f1 \ + --hash=sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8 \ + --hash=sha256:4fe7859a4e3e8457458e2ff592f15ccb02f3da787fcd31e0183879c3ad4692a1 \ + --hash=sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2 \ + --hash=sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44 \ + --hash=sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26 \ + --hash=sha256:5947809c8a2417be3267efc979c47d76a079758166f7d43ef5ae8e9f92751f88 \ + --hash=sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016 \ + --hash=sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede \ + --hash=sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf \ + --hash=sha256:5cb4d72eea50c8868f5288b7f7f33ed276118325c1dfd3957089f6b519e1382a \ + --hash=sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc \ + --hash=sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0 \ + --hash=sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84 \ + --hash=sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db \ + --hash=sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1 \ + --hash=sha256:6aee717dcfead04c6eb1ce3bd29ac1e22663cdea57f943c87d1eab9a025438d7 \ + --hash=sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed \ + --hash=sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8 \ + --hash=sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133 \ + --hash=sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e \ + --hash=sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef \ + --hash=sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14 \ + --hash=sha256:778d2e08eda00f4256d7f672ca9fef386071c9202f5e4607920b86d7803387f2 \ + --hash=sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0 \ + --hash=sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d \ + --hash=sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828 \ + --hash=sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f \ + --hash=sha256:7c308f7e26e4363d79df40ca5b2be1c6ba9f02bdbccfed5abddb7859a6ce72cf \ + --hash=sha256:7fa17817dc5625de8a027cb8b26d9fefa3ea28c8253929b8d6649e705d2835b6 \ + --hash=sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328 \ + --hash=sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090 \ + --hash=sha256:837c2ce8c5a65a2035be9b3569c684358dfbf109fd3b6969630a87535495ceaa \ + --hash=sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381 \ + --hash=sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c \ + --hash=sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb \ + --hash=sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc \ + --hash=sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a \ + --hash=sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec \ + --hash=sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc \ + --hash=sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac \ + --hash=sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e \ + --hash=sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313 \ + --hash=sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569 \ + --hash=sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3 \ + --hash=sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d \ + --hash=sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525 \ + --hash=sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894 \ + --hash=sha256:a8bf8d0f749c5757af2142fe7903a9df1d2e8aa3841559b2bad34b08d0e2bcf3 \ + --hash=sha256:a9768c477b9d7bd54bc0c86dbaebdec6f03306675526c9927c0e8a04e8f94af9 \ + --hash=sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a \ + --hash=sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9 \ + --hash=sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14 \ + --hash=sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25 \ + --hash=sha256:b5d84d37db046c5ca74ee7bb47dd6cbc13f80665fdde3e8040bdd3fb015ecb50 \ + --hash=sha256:b7cf1017d601aa35e6bb650b6ad28652c9cd78ee6caff19f3c28d03e1c80acbf \ + --hash=sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1 \ + --hash=sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3 \ + --hash=sha256:c4ef880e27901b6cc782f1b95f82da9313c0eb95c3af699103088fa0ac3ce9ac \ + --hash=sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e \ + --hash=sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815 \ + --hash=sha256:cb01158d8b88ee68f15949894ccc6712278243d95f344770fa7593fa2d94410c \ + --hash=sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6 \ + --hash=sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6 \ + --hash=sha256:cd09d08005f958f370f539f186d10aec3377d55b9eeb0d796025d4886119d76e \ + --hash=sha256:cd4b7ca9984e5e7985c12bc60a6f173f3c958eae74f3ef6624bb6b26e2abbae4 \ + --hash=sha256:ce8a0633f41a967713a59c4139d29110c07e826d131a316b50ce11b1d79b4f84 \ + --hash=sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69 \ + --hash=sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15 \ + --hash=sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191 \ + --hash=sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0 \ + --hash=sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897 \ + --hash=sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd \ + --hash=sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2 \ + --hash=sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794 \ + --hash=sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d \ + --hash=sha256:e912091979546adf63357d7e2ccff9b44f026c075aeaf25a52d0e95ad2281074 \ + --hash=sha256:eaabd426fe94daf8fd157c32e571c85cb12e66692f15516a83a03264b08d06c3 \ + --hash=sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224 \ + --hash=sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838 \ + --hash=sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a \ + --hash=sha256:f155a433c2ec037d4e8df17d18922c3a0d9b3232a396690f17175d2946f0218d \ + --hash=sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d \ + --hash=sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f \ + --hash=sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8 \ + --hash=sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490 \ + --hash=sha256:f8e160feb2aed042cd657a72acc0b481212ed28b1b9a95c0cee1621b524e1966 \ + --hash=sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9 \ + --hash=sha256:fa09f53c465e532f4d3db095e0c55b615f010ad81803d383195b6b5ca6cbf5f3 \ + --hash=sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e \ + --hash=sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608 + # via requests +google-api-core[grpc]==2.28.1 \ + --hash=sha256:2b405df02d68e68ce0fbc138559e6036559e685159d148ae5861013dc201baf8 \ + --hash=sha256:4021b0f8ceb77a6fb4de6fde4502cecab45062e66ff4f2895169e0b35bc9466c + # via + # google-api-python-client + # google-cloud-bigquery + # google-cloud-core + # google-cloud-monitoring + # google-cloud-storage +google-api-python-client==2.187.0 \ + --hash=sha256:d8d0f6d85d7d1d10bdab32e642312ed572bdc98919f72f831b44b9a9cebba32f \ + --hash=sha256:e98e8e8f49e1b5048c2f8276473d6485febc76c9c47892a8b4d1afa2c9ec8278 + # via -r requirements.in +google-auth==2.43.0 \ + --hash=sha256:88228eee5fc21b62a1b5fe773ca15e67778cb07dc8363adcb4a8827b52d81483 \ + --hash=sha256:af628ba6fa493f75c7e9dbe9373d148ca9f4399b5ea29976519e0a3848eddd16 + # via + # google-api-core + # google-api-python-client + # google-auth-httplib2 + # google-cloud-bigquery + # google-cloud-core + # google-cloud-monitoring + # google-cloud-storage +google-auth-httplib2==0.2.1 \ + --hash=sha256:1be94c611db91c01f9703e7f62b0a59bbd5587a95571c7b6fade510d648bc08b \ + --hash=sha256:5ef03be3927423c87fb69607b42df23a444e434ddb2555b73b3679793187b7de + # via google-api-python-client +google-cloud==0.34.0 \ + --hash=sha256:01430187cf56df10a9ba775dd547393185d4b40741db0ea5889301f8e7a9d5d3 \ + --hash=sha256:fb1ab7b0548fe44b3d538041f0a374505b7f990d448a935ea36649c5ccab5acf + # via -r requirements.in +google-cloud-bigquery==3.38.0 \ + --hash=sha256:8afcb7116f5eac849097a344eb8bfda78b7cfaae128e60e019193dd483873520 \ + --hash=sha256:e06e93ff7b245b239945ef59cb59616057598d369edac457ebf292bd61984da6 + # via -r requirements.in +google-cloud-core==2.5.0 \ + --hash=sha256:67d977b41ae6c7211ee830c7912e41003ea8194bff15ae7d72fd6f51e57acabc \ + --hash=sha256:7c1b7ef5c92311717bd05301aa1a91ffbc565673d3b0b4163a52d8413a186963 + # via + # google-cloud-bigquery + # google-cloud-storage +google-cloud-monitoring==2.28.0 \ + --hash=sha256:25175590907e038add644b5b744941d221776342924637095a879973a7c0ac37 \ + --hash=sha256:64f4c57cc465dd51cceffe559f0ec6fa9f96aa6d82790cd8d3af6d5cc3795160 + # via -r requirements.in +google-cloud-storage==3.6.0 \ + --hash=sha256:29cc6b9a6c0fc9cdad071e375d540a5a50fbc9a7fad8300fa02fb904f6fe2ca2 \ + --hash=sha256:5decbdddd63b7d1fc3e266a393ad6453d2e27d172bd982b1e2f15481668db097 + # via -r requirements.in +google-crc32c==1.7.1 \ + --hash=sha256:0f99eaa09a9a7e642a61e06742856eec8b19fc0037832e03f941fe7cf0c8e4db \ + --hash=sha256:19eafa0e4af11b0a4eb3974483d55d2d77ad1911e6cf6f832e1574f6781fd337 \ + --hash=sha256:1c67ca0a1f5b56162951a9dae987988679a7db682d6f97ce0f6381ebf0fbea4c \ + --hash=sha256:1f2b3522222746fff0e04a9bd0a23ea003ba3cccc8cf21385c564deb1f223242 \ + --hash=sha256:22beacf83baaf59f9d3ab2bbb4db0fb018da8e5aebdce07ef9f09fce8220285e \ + --hash=sha256:2bff2305f98846f3e825dbeec9ee406f89da7962accdb29356e4eadc251bd472 \ + --hash=sha256:2d73a68a653c57281401871dd4aeebbb6af3191dcac751a76ce430df4d403194 \ + --hash=sha256:32d1da0d74ec5634a05f53ef7df18fc646666a25efaaca9fc7dcfd4caf1d98c3 \ + --hash=sha256:3bda0fcb632d390e3ea8b6b07bf6b4f4a66c9d02dcd6fbf7ba00a197c143f582 \ + --hash=sha256:6335de12921f06e1f774d0dd1fbea6bf610abe0887a1638f64d694013138be5d \ + --hash=sha256:6b211ddaf20f7ebeec5c333448582c224a7c90a9d98826fbab82c0ddc11348e6 \ + --hash=sha256:6efb97eb4369d52593ad6f75e7e10d053cf00c48983f7a973105bc70b0ac4d82 \ + --hash=sha256:6fbab4b935989e2c3610371963ba1b86afb09537fd0c633049be82afe153ac06 \ + --hash=sha256:713121af19f1a617054c41f952294764e0c5443d5a5d9034b2cd60f5dd7e0349 \ + --hash=sha256:754561c6c66e89d55754106739e22fdaa93fafa8da7221b29c8b8e8270c6ec8a \ + --hash=sha256:7cc81b3a2fbd932a4313eb53cc7d9dde424088ca3a0337160f35d91826880c1d \ + --hash=sha256:85fef7fae11494e747c9fd1359a527e5970fc9603c90764843caabd3a16a0a48 \ + --hash=sha256:905a385140bf492ac300026717af339790921f411c0dfd9aa5a9e69a08ed32eb \ + --hash=sha256:9fc196f0b8d8bd2789352c6a522db03f89e83a0ed6b64315923c396d7a932315 \ + --hash=sha256:a8e9afc74168b0b2232fb32dd202c93e46b7d5e4bf03e66ba5dc273bb3559589 \ + --hash=sha256:b07d48faf8292b4db7c3d64ab86f950c2e94e93a11fd47271c28ba458e4a0d76 \ + --hash=sha256:b6d86616faaea68101195c6bdc40c494e4d76f41e07a37ffdef270879c15fb65 \ + --hash=sha256:b7491bdc0c7564fcf48c0179d2048ab2f7c7ba36b84ccd3a3e1c3f7a72d3bba6 \ + --hash=sha256:bb5e35dcd8552f76eed9461a23de1030920a3c953c1982f324be8f97946e7127 \ + --hash=sha256:d68e17bad8f7dd9a49181a1f5a8f4b251c6dbc8cc96fb79f1d321dfd57d66f53 \ + --hash=sha256:dcdf5a64adb747610140572ed18d011896e3b9ae5195f2514b7ff678c80f1603 \ + --hash=sha256:df8b38bdaf1629d62d51be8bdd04888f37c451564c2042d36e5812da9eff3c35 \ + --hash=sha256:e10554d4abc5238823112c2ad7e4560f96c7bf3820b202660373d769d9e6e4c9 \ + --hash=sha256:e42e20a83a29aa2709a0cf271c7f8aefaa23b7ab52e53b322585297bb94d4638 \ + --hash=sha256:ed66cbe1ed9cbaaad9392b5259b3eba4a9e565420d734e6238813c428c3336c9 \ + --hash=sha256:ee6547b657621b6cbed3562ea7826c3e11cab01cd33b74e1f677690652883e77 \ + --hash=sha256:f2226b6a8da04f1d9e61d3e357f2460b9551c5e6950071437e122c958a18ae14 \ + --hash=sha256:fa8136cc14dd27f34a3221c0f16fd42d8a40e4778273e61a3c19aedaa44daf6b \ + --hash=sha256:fc5319db92daa516b653600794d5b9f9439a9a121f3e162f94b0e1891c7933cb + # via + # google-cloud-storage + # google-resumable-media +google-resumable-media==2.8.0 \ + --hash=sha256:dd14a116af303845a8d932ddae161a26e86cc229645bc98b39f026f9b1717582 \ + --hash=sha256:f1157ed8b46994d60a1bc432544db62352043113684d4e030ee02e77ebe9a1ae + # via + # google-cloud-bigquery + # google-cloud-storage +googleapis-common-protos==1.72.0 \ + --hash=sha256:4299c5a82d5ae1a9702ada957347726b167f9f8d1fc352477702a1e851ff4038 \ + --hash=sha256:e55a601c1b32b52d7a3e65f43563e2aa61bcd737998ee672ac9b951cd49319f5 + # via + # google-api-core + # grpcio-status +grpcio==1.76.0 \ + --hash=sha256:035d90bc79eaa4bed83f524331d55e35820725c9fbb00ffa1904d5550ed7ede3 \ + --hash=sha256:04bbe1bfe3a68bbfd4e52402ab7d4eb59d72d02647ae2042204326cf4bbad280 \ + --hash=sha256:063065249d9e7e0782d03d2bca50787f53bd0fb89a67de9a7b521c4a01f1989b \ + --hash=sha256:06c3d6b076e7b593905d04fdba6a0525711b3466f43b3400266f04ff735de0cd \ + --hash=sha256:08caea849a9d3c71a542827d6df9d5a69067b0a1efbea8a855633ff5d9571465 \ + --hash=sha256:0aaa82d0813fd4c8e589fac9b65d7dd88702555f702fb10417f96e2a2a6d4c0f \ + --hash=sha256:0b7604868b38c1bfd5cf72d768aedd7db41d78cb6a4a18585e33fb0f9f2363fd \ + --hash=sha256:0c37db8606c258e2ee0c56b78c62fc9dee0e901b5dbdcf816c2dd4ad652b8b0c \ + --hash=sha256:1c9b93f79f48b03ada57ea24725d83a30284a012ec27eab2cf7e50a550cbbbcc \ + --hash=sha256:2107b0c024d1b35f4083f11245c0e23846ae64d02f40b2b226684840260ed054 \ + --hash=sha256:2229ae655ec4e8999599469559e97630185fdd53ae1e8997d147b7c9b2b72cba \ + --hash=sha256:25a18e9810fbc7e7f03ec2516addc116a957f8cbb8cbc95ccc80faa072743d03 \ + --hash=sha256:26ef06c73eb53267c2b319f43e6634c7556ea37672029241a056629af27c10e2 \ + --hash=sha256:2e1743fbd7f5fa713a1b0a8ac8ebabf0ec980b5d8809ec358d488e273b9cf02a \ + --hash=sha256:32483fe2aab2c3794101c2a159070584e5db11d0aa091b2c0ea9c4fc43d0d749 \ + --hash=sha256:3bf0f392c0b806905ed174dcd8bdd5e418a40d5567a05615a030a5aeddea692d \ + --hash=sha256:3e2a27c89eb9ac3d81ec8835e12414d73536c6e620355d65102503064a4ed6eb \ + --hash=sha256:40ad3afe81676fd9ec6d9d406eda00933f218038433980aa19d401490e46ecde \ + --hash=sha256:4215d3a102bd95e2e11b5395c78562967959824156af11fa93d18fdd18050990 \ + --hash=sha256:45d59a649a82df5718fd9527ce775fd66d1af35e6d31abdcdc906a49c6822958 \ + --hash=sha256:45e0111e73f43f735d70786557dc38141185072d7ff8dc1829d6a77ac1471468 \ + --hash=sha256:479496325ce554792dba6548fae3df31a72cef7bad71ca2e12b0e58f9b336bfc \ + --hash=sha256:490fa6d203992c47c7b9e4a9d39003a0c2bcc1c9aa3c058730884bbbb0ee9f09 \ + --hash=sha256:49ce47231818806067aea3324d4bf13825b658ad662d3b25fada0bdad9b8a6af \ + --hash=sha256:4baf3cbe2f0be3289eb68ac8ae771156971848bb8aaff60bad42005539431980 \ + --hash=sha256:522175aba7af9113c48ec10cc471b9b9bd4f6ceb36aeb4544a8e2c80ed9d252d \ + --hash=sha256:5e8571632780e08526f118f74170ad8d50fb0a48c23a746bef2a6ebade3abd6f \ + --hash=sha256:615ba64c208aaceb5ec83bfdce7728b80bfeb8be97562944836a7a0a9647d882 \ + --hash=sha256:61f69297cba3950a524f61c7c8ee12e55c486cb5f7db47ff9dcee33da6f0d3ae \ + --hash=sha256:65a20de41e85648e00305c1bb09a3598f840422e522277641145a32d42dcefcc \ + --hash=sha256:6a15c17af8839b6801d554263c546c69c4d7718ad4321e3166175b37eaacca77 \ + --hash=sha256:747fa73efa9b8b1488a95d0ba1039c8e2dca0f741612d80415b1e1c560febf4e \ + --hash=sha256:7be78388d6da1a25c0d5ec506523db58b18be22d9c37d8d3a32c08be4987bd73 \ + --hash=sha256:81fd9652b37b36f16138611c7e884eb82e0cec137c40d3ef7c3f9b3ed00f6ed8 \ + --hash=sha256:83d57312a58dcfe2a3a0f9d1389b299438909a02db60e2f2ea2ae2d8034909d3 \ + --hash=sha256:8843114c0cfce61b40ad48df65abcfc00d4dba82eae8718fab5352390848c5da \ + --hash=sha256:8cc3309d8e08fd79089e13ed4819d0af72aa935dd8f435a195fd152796752ff2 \ + --hash=sha256:8ebe63ee5f8fa4296b1b8cfc743f870d10e902ca18afc65c68cf46fd39bb0783 \ + --hash=sha256:8eddfb4d203a237da6f3cc8a540dad0517d274b5a1e9e636fd8d2c79b5c1d397 \ + --hash=sha256:922fa70ba549fce362d2e2871ab542082d66e2aaf0c19480ea453905b01f384e \ + --hash=sha256:931091142fd8cc14edccc0845a79248bc155425eee9a98b2db2ea4f00a235a42 \ + --hash=sha256:971fd5a1d6e62e00d945423a567e42eb1fa678ba89072832185ca836a94daaa6 \ + --hash=sha256:980a846182ce88c4f2f7e2c22c56aefd515daeb36149d1c897f83cf57999e0b6 \ + --hash=sha256:9d9adda641db7207e800a7f089068f6f645959f2df27e870ee81d44701dd9db3 \ + --hash=sha256:9f8f757bebaaea112c00dba718fc0d3260052ce714e25804a03f93f5d1c6cc11 \ + --hash=sha256:a6ae758eb08088d36812dd5d9af7a9859c05b1e0f714470ea243694b49278e7b \ + --hash=sha256:a8c2cf1209497cf659a667d7dea88985e834c24b7c3b605e6254cbb5076d985c \ + --hash=sha256:acab0277c40eff7143c2323190ea57b9ee5fd353d8190ee9652369fae735668a \ + --hash=sha256:b331680e46239e090f5b3cead313cc772f6caa7d0fc8de349337563125361a4a \ + --hash=sha256:c088e7a90b6017307f423efbb9d1ba97a22aa2170876223f9709e9d1de0b5347 \ + --hash=sha256:d099566accf23d21037f18a2a63d323075bebace807742e4b0ac210971d4dd70 \ + --hash=sha256:d388087771c837cdb6515539f43b9d4bf0b0f23593a24054ac16f7a960be16f4 \ + --hash=sha256:dcfe41187da8992c5f40aa8c5ec086fa3672834d2be57a32384c08d5a05b4c00 \ + --hash=sha256:e6d1db20594d9daba22f90da738b1a0441a7427552cc6e2e3d1297aeddc00378 \ + --hash=sha256:ebea5cc3aa8ea72e04df9913492f9a96d9348db876f9dda3ad729cfedf7ac416 \ + --hash=sha256:ebebf83299b0cb1721a8859ea98f3a77811e35dce7609c5c963b9ad90728f886 \ + --hash=sha256:f0e34c2079d47ae9f6188211db9e777c619a21d4faba6977774e8fa43b085e48 \ + --hash=sha256:f92f88e6c033db65a5ae3d97905c8fea9c725b63e28d5a75cb73b49bda5024d8 \ + --hash=sha256:f9f7bd5faab55f47231ad8dba7787866b69f5e93bc306e3915606779bbfb4ba8 \ + --hash=sha256:fd5ef5932f6475c436c4a55e4336ebbe47bd3272be04964a03d316bbf4afbcbc \ + --hash=sha256:ff8a59ea85a1f2191a0ffcc61298c571bc566332f82e5f5be1b83c9d8e668a62 + # via + # google-api-core + # google-cloud-monitoring + # grpcio-status +grpcio-status==1.76.0 \ + --hash=sha256:25fcbfec74c15d1a1cb5da3fab8ee9672852dc16a5a9eeb5baf7d7a9952943cd \ + --hash=sha256:380568794055a8efbbd8871162df92012e0228a5f6dffaf57f2a00c534103b18 + # via google-api-core +httplib2==0.31.0 \ + --hash=sha256:ac7ab497c50975147d4f7b1ade44becc7df2f8954d42b38b3d69c515f531135c \ + --hash=sha256:b9cd78abea9b4e43a7714c6e0f8b6b8561a6fc1e95d5dbd367f5bf0ef35f5d24 + # via + # google-api-python-client + # google-auth-httplib2 +idna==3.11 \ + --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea \ + --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902 + # via requests +packaging==25.0 \ + --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 \ + --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f + # via google-cloud-bigquery +proto-plus==1.26.1 \ + --hash=sha256:13285478c2dcf2abb829db158e1047e2f1e8d63a077d94263c2b88b043c75a66 \ + --hash=sha256:21a515a4c4c0088a773899e23c7bbade3d18f9c66c73edd4c7ee3816bc96a012 + # via + # google-api-core + # google-cloud-monitoring +protobuf==6.33.5 \ + --hash=sha256:3093804752167bcab3998bec9f1048baae6e29505adaf1afd14a37bddede533c \ + --hash=sha256:69915a973dd0f60f31a08b8318b73eab2bd6a392c79184b3612226b0a3f8ec02 \ + --hash=sha256:6ddcac2a081f8b7b9642c09406bc6a4290128fce5f471cddd165960bb9119e5c \ + --hash=sha256:8afa18e1d6d20af15b417e728e9f60f3aa108ee76f23c3b2c07a2c3b546d3afd \ + --hash=sha256:8f04fa32763dcdb4973d537d6b54e615cc61108c7cb38fe59310c3192d29510a \ + --hash=sha256:9b71e0281f36f179d00cbcb119cb19dec4d14a81393e5ea220f64b286173e190 \ + --hash=sha256:a3157e62729aafb8df6da2c03aa5c0937c7266c626ce11a278b6eb7963c4e37c \ + --hash=sha256:a5cb85982d95d906df1e2210e58f8e4f1e3cdc088e52c921a041f9c9a0386de5 \ + --hash=sha256:cbf16ba3350fb7b889fca858fb215967792dc125b35c7976ca4818bee3521cf0 \ + --hash=sha256:d71b040839446bac0f4d162e758bea99c8251161dae9d0983a3b88dee345153b + # via + # google-api-core + # google-cloud-monitoring + # googleapis-common-protos + # grpcio-status + # proto-plus +pyasn1==0.6.3 \ + --hash=sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf \ + --hash=sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.2 \ + --hash=sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a \ + --hash=sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6 + # via google-auth +pyparsing==3.2.5 \ + --hash=sha256:2df8d5b7b2802ef88e8d016a2eb9c7aeaa923529cd251ed0fe4608275d4105b6 \ + --hash=sha256:e38a4f02064cf41fe6593d328d0512495ad1f3d8a91c4f73fc401b3079a59a5e + # via httplib2 +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via google-cloud-bigquery +requests==2.32.5 \ + --hash=sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 \ + --hash=sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf + # via + # google-api-core + # google-cloud-bigquery + # google-cloud-storage +rsa==4.9.1 \ + --hash=sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762 \ + --hash=sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75 + # via google-auth +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via python-dateutil +typing-extensions==4.15.0 \ + --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ + --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 + # via grpcio +uritemplate==4.2.0 \ + --hash=sha256:480c2ed180878955863323eea31b0ede668795de182617fef9c6ca09e6ec9d0e \ + --hash=sha256:962201ba1c4edcab02e60f9a0d3821e82dfc5d2d6662a21abd533879bdb8a686 + # via google-api-python-client +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 + # via requests diff --git a/perfmetrics/scripts/testing_on_gke/examples/run-automated.sh b/perfmetrics/scripts/testing_on_gke/examples/run-automated.sh deleted file mode 100755 index a1458bb958..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/run-automated.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/bash -# -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script is used purely for automating the run of -# the script run-gke-tests.sh to -# run it periodically as a cron-job. -# -# For your case, add/remove/modify the configuration parameters as you need. -# -# Assumptions for this script to work: -# 1. You have appropriate access to project_id defined below. -# 2. You have the cluster with cluster_name defined below, or enough. -# resources in project_id to create this cluster. - -# Print all shell commands. -set -x - -# Fail if any command fails. -set -e - -# Environment variable USER must be defined. -if test -z ${USER}; then - echo "USER has not been set" - exit 1 -fi - -# Define configuration parameters. -if test -z ${project_id}; then - echo "project_id has not been set." - exit 1 -fi -if test -z "${project_number}"; then - echo "project_number has not been set." - exit 1 -fi -export zone=us-west1-b -if test -z "${cluster_name}"; then - echo "cluster_name has not been set." - exit 1 -fi -export node_pool=default-pool -export machine_type=n2-standard-96 -export num_nodes=7 -export num_ssd=16 -export use_custom_csi_driver=true -export output_dir=. -if test -z "${gcsfuse_branch}"; then - echo "gcsfuse_branch has not been set." - exit 1 -fi -export pod_wait_time_in_seconds=300 -export pod_timeout_in_seconds=64800 -# Pass instance_id from outside to continue previous run, if it got terminated -# somehow (timeout of ssh etc.) -if test -z ${instance_id}; then - export instance_id=$(echo ${USER} | sed 's/_google//' | sed 's/_com//')-$(date +%Y%m%d-%H%M%S) -fi -if test -z "${output_gsheet_id}"; then - echo "output_gsheet_id has not been set." - exit 1 -fi -if test -z "${output_gsheet_keyfile}"; then - echo "output_gsheet_keyfile has not been set." - exit 1 -fi -export force_update_gcsfuse_code=true -# Continue previous run if pods had been scheduled/completed already. -test -n ${only_parse} || export only_parse=false - -# Create a dedicated folder on the machine. -mkdir -pv ~/gke-testing && cd ~/gke-testing -wget https://raw.githubusercontent.com/googlecloudplatform/gcsfuse/${gcsfuse_branch}/perfmetrics/scripts/testing_on_gke/examples/run-gke-tests.sh -O run-gke-tests.sh -chmod +x run-gke-tests.sh - -# Remove previous run's outputs. -rm -rfv log fio/output.csv dlio/output.csv - -# Run the script. -start_time=$(date +%Y-%m-%dT%H:%M:%SZ) -echo "Run started at ${start_time}" -touch log -(./run-gke-tests.sh --debug |& tee -a log) || true -# Use the following if you want to run it in a tmux session instead. -# tmux new-session -d -s ${instance_id} 'bash -c "(./run-gke-tests.sh --debug |& tee -a log); sleep 604800 "' -end_time=$(date +%Y-%m-%dT%H:%M:%SZ) -echo "Run ended at ${end_time}" - -# Some post-run steps to be taken for output collection. -if test -n "${workload_config}"; then - cp ${workload_config} ./workloads.json -else - cp src/gcsfuse/perfmetrics/scripts/testing_on_gke/examples/workloads.json . -fi -git -C src/gcsfuse rev-parse HEAD > gcsfuse_commithash -git -C src/gcs-fuse-csi-driver rev-parse HEAD > gcs_fuse_csi_driver_commithash -# Fetch cloud-logs for this run. This has not been tested yet. -# (gcloud logging read --project=${project_id} 'timestamp>="${start_time}"" AND timestamp<="${end_time}" AND resource.labels.cluster_name="${cluster_name}" ' --order=ASC --format=csv\(timestamp\,resource.labels.pod_name,resource.labels.container_name,"text_payload"\) > cloud_logs.txt) & - -# Upload outputs to GCS after the run. -if test -z "${output_bucket}"; then - echo "output_bucket has not been set." - exit 1 -fi -output_path_uri=gs://${output_bucket}/outputs/${instance_id} -for file in fio/output.csv dlio/output.csv log run-gke-tests.sh workloads.json gcsfuse_commithash gcs_fuse_csi_driver_commithash; do - if test -f ${file} ; then - gcloud storage cp --content-type=text/text ${file} ${output_path_uri}/${file} - fi -done - -# Go back to whichever working directory you were in. -cd - diff --git a/perfmetrics/scripts/testing_on_gke/examples/run-gke-tests.sh b/perfmetrics/scripts/testing_on_gke/examples/run-gke-tests.sh index 2a86065b46..fc0d9647a4 100755 --- a/perfmetrics/scripts/testing_on_gke/examples/run-gke-tests.sh +++ b/perfmetrics/scripts/testing_on_gke/examples/run-gke-tests.sh @@ -36,8 +36,10 @@ fi # Utilities function exitWithSuccess() { exit 0; } function exitWithFailure() { exit 1; } -function echoerror() { >&2 echo "Error: "$@; } -function exitWithError() { echoerror $@ ; exitWithFailure ; } +function echoerror() { >&2 echo "Error: "$@ ; } +function echowarning() { >&2 echo "Warning: "${@} ; } +function exitWithError() { echoerror "$@" ; exitWithFailure ; } +function returnWithError() { echoerror "$@" ; return 1 ; } # Default values, to be used for parameters in case user does not specify them. # GCP related @@ -50,6 +52,7 @@ readonly DEFAULT_NUM_SSD=16 readonly DEFAULT_APPNAMESPACE=default readonly DEFAULT_KSA=default readonly DEFAULT_USE_CUSTOM_CSI_DRIVER=true +readonly DEFAULT_CUSTOM_CSI_DRIVER= # GCSFuse/GKE GCSFuse CSI Driver source code related readonly DEFAULT_SRC_DIR="$(realpath .)/src" readonly csi_driver_github_path=https://github.com/googlecloudplatform/gcs-fuse-csi-driver @@ -57,12 +60,48 @@ readonly csi_driver_branch=main readonly gcsfuse_github_path=https://github.com/googlecloudplatform/gcsfuse readonly DEFAULT_GCSFUSE_BRANCH=garnitin/add-gke-load-testing/v1 # Test runtime configuration -readonly DEFAULT_INSTANCE_ID=${USER}-$(date +%Y%m%d-%H%M%S) # 5 minutes readonly DEFAULT_POD_WAIT_TIME_IN_SECONDS=300 # 1 week readonly DEFAULT_POD_TIMEOUT_IN_SECONDS=604800 readonly DEFAULT_FORCE_UPDATE_GCSFUSE_CODE=false +readonly DEFAULT_ZONAL=false + +# Config for exporting fio outputs to a Bigquery table. +readonly DEFAULT_BQ_PROJECT_ID='gcs-fuse-test-ml' +readonly DEFAULT_BQ_DATASET_ID='gke_test_tool_outputs' +readonly DEFAULT_BQ_TABLE_ID='fio_outputs' + +# Handling of deprecated flag instance_id if it has been passed. +if test -n "${instance_id}" ; then + deprecation_message="instance_id flag is now deprecated, but has been passed (with value \"${instance_id}\"). In future, please use experiment_id instead." + + # If instance_id is set, but experiment_id is not + # set, then let this be only a warning message and pass the value of + # instance_id to experiment_id. + if test -z "${experiment_id}" ; then + echowarning ${deprecation_message}" For now, setting experiment_id=\"${instance_id}\" ." + export experiment_id="${instance_id}" + unset instance_id + else + # Otherwise, halt the run as this is an ambiguous situation. + exitWithError "${deprecation_message}" + fi +fi + +# Create and return a unique experiment_id taking +# into account user's passed experiment_id. +function create_unique_experiment_id() { + new_uuid=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 4 | head -n 1) + local generated_unique_experiment_id=${USER}-$(date +%Y%m%d-%H%M%S)-${new_uuid} + if [ $# -gt 0 ] && [ -n "${1}" ]; then + local user_provided_experiment_id="${1}" + experiment_id=${user_provided_experiment_id// /-}"-"${generated_unique_experiment_id} + else + experiment_id=${generated_unique_experiment_id} + fi + echo "${experiment_id}" +} function printHelp() { echo "Usage guide: " @@ -80,6 +119,7 @@ function printHelp() { echo "machine_type=<machine-type default=\"${DEFAULT_MACHINE_TYPE}\">" echo "num_nodes=<number from 1-8, default=\"${DEFAULT_NUM_NODES}\">" echo "num_ssd=<number from 0-16, default=\"${DEFAULT_NUM_SSD}\">" + echo "custom_csi_driver=<string representing the full path of the csi-driver image hash e.g. gcr.io/<registry-name>:<hash>, default=\"${DEFAULT_CUSTOM_CSI_DRIVER}\". If it is non-empty, then use_custom_csi_driver is assumed true, but a custom driver is not built and the given custom csi driver is used instead. >" echo "use_custom_csi_driver=<true|false, true means build and use a new custom csi driver using gcsfuse code, default=\"${DEFAULT_USE_CUSTOM_CSI_DRIVER}\">" # GCSFuse/GKE GCSFuse CSI Driver source code related echo "src_dir=<\"directory/to/clone/github/repos/if/needed\", used for locally cloning in case gcsfuse_src_dir or csi_src_dir are not passed, default=\"${DEFAULT_SRC_DIR}\">" @@ -89,10 +129,11 @@ function printHelp() { # Test runtime configuration echo "pod_wait_time_in_seconds=<number e.g. 60 for checking pod status every 1 min, default=\"${DEFAULT_POD_WAIT_TIME_IN_SECONDS}\">" echo "pod_timeout_in_seconds=<number e.g. 3600 for timing out pod runs, should be more than the value of pod_wait_time_in_seconds, default=\"${DEFAULT_POD_TIMEOUT_IN_SECONDS}\">" - echo "instance_id=<string, not containing spaces, representing unique id for particular test-run e.g. \"${DEFAULT_INSTANCE_ID}\"" + echo "experiment_id=<Optional description of this particular test-run, it does not need to be unique e.g. \"cache test #43\"" echo "workload_config=<path/to/workload/configuration/file e.g. /a/b/c.json >" echo "output_dir=</absolute/path/to/output/dir, output files will be written at output_dir/fio/output.csv and output_dir/dlio/output.csv>" echo "force_update_gcsfuse_code=<true|false, to force-update the gcsfuse-code to given branch if gcsfuse_src_dir has been set. Default=\"${DEFAULT_FORCE_UPDATE_GCSFUSE_CODE}\">" + echo "zonal=<true|false, to convey that at least one of the buckets in the given workload configuration is a zonal bucket which can't be read/written using gcloud. Default=\"${DEFAULT_ZONAL}\"> " echo "" echo "" echo "" @@ -108,6 +149,17 @@ if ([ $# -gt 0 ] && ([ "$1" == "-help" ] || [ "$1" == "--help" ] || [ "$1" == "- exitWithSuccess fi +verify_csi_driver_image() { + if [[ $# < 1 ]]; then + returnWithError "No arguments passed to verify_csi_driver_image. Expected: \$1=<csi-driver-image> ." + fi + local csi_driver_image=${1} + echo "Checking ${csi_driver_image} ..." + if ! gcloud -q container images describe ${csi_driver_image} >/dev/null; then + returnWithError "${csi_driver_image} is not a valid GCSFuse csi driver image. !!! Please check if you missed adding /gcs-fuse-csi-driver-sidecar-mounter before the hash. !!!" + fi +} + # Set environment variables. # GCP related if test -z "${project_id}"; then @@ -119,7 +171,7 @@ fi test -n "${zone}" || export zone=${DEFAULT_ZONE} # GKE cluster related if test -z "${cluster_name}"; then - exitWithError "${cluster_name} was not set." + exitWithError "cluster_name was not set." fi test -n "${node_pool}" || export node_pool=${DEFAULT_NODE_POOL} test -n "${machine_type}" || export machine_type=${DEFAULT_MACHINE_TYPE} @@ -129,7 +181,34 @@ test -n "${num_ssd}" || export num_ssd=${DEFAULT_NUM_SSD} export appnamespace=${DEFAULT_APPNAMESPACE} # test -n "${ksa}" || export ksa=${DEFAULT_KSA} -test -n "${use_custom_csi_driver}" || export use_custom_csi_driver="${DEFAULT_USE_CUSTOM_CSI_DRIVER}" + +applied_custom_csi_driver= +if test -z "${custom_csi_driver}"; then + echo "custom_csi_driver has not been set, so assuming \"${DEFAULT_CUSTOM_CSI_DRIVER}\" for it ..." + export custom_csi_driver="${DEFAULT_CUSTOM_CSI_DRIVER}" + if test -z "${use_custom_csi_driver}"; then + echo "use_custom_csi_driver has not been set, so assuming \"${DEFAULT_USE_CUSTOM_CSI_DRIVER}\" for it ..." + export use_custom_csi_driver="${DEFAULT_USE_CUSTOM_CSI_DRIVER}" + elif [[ ${use_custom_csi_driver} = "true" ]]; then + echo "User has enabled use_custom_csi_driver, without passing a custom_csi_driver, so a custom driver will be built in this run." + elif [[ ${use_custom_csi_driver} != "false" ]]; then + exitWithError "Unsupported value passed for use_custom_csi_driver: ${use_custom_csi_driver}. Supported values: true/false ." + fi +else + echo "User passed custom_csi_driver=${custom_csi_driver}. This will be used this run." + printf "\nVerifying that ${custom_csi_driver} is a valid GCSFuse csi driver image ...\n\n" + verify_csi_driver_image ${custom_csi_driver} + if test -z "${use_custom_csi_driver}"; then + echo "use_custom_csi_driver has not been set, so setting it to true as custom_csi_driver has been set to \"${custom_csi_driver}\"" + export use_custom_csi_driver=true + elif [[ ${use_custom_csi_driver} = "false" ]]; then + exitWithError "User has disabled use_custom_csi_driver, while passing a custom_csi_driver. This is unsupported." + elif [[ ${use_custom_csi_driver} != "true" ]]; then + exitWithError "Unsupported value passed for use_custom_csi_driver: ${use_custom_csi_driver}. Supported values: true or false ." + fi + applied_custom_csi_driver=${custom_csi_driver} +fi + test -n "${gcsfuse_branch}" || export gcsfuse_branch="${DEFAULT_GCSFUSE_BRANCH}" # GCSFuse/GKE GCSFuse CSI Driver source code related @@ -170,7 +249,24 @@ fi # Test runtime configuration test -n "${pod_wait_time_in_seconds}" || export pod_wait_time_in_seconds="${DEFAULT_POD_WAIT_TIME_IN_SECONDS}" test -n "${pod_timeout_in_seconds}" || export pod_timeout_in_seconds="${DEFAULT_POD_TIMEOUT_IN_SECONDS}" -test -n "${instance_id}" || export instance_id="${DEFAULT_INSTANCE_ID}" + +if test -z ${only_parse} ; then + export only_parse=false +elif [ "$only_parse" != "true" ] && [ "$only_parse" != "false" ]; then + exitWithError "Unexpected value of only_parse: ${only_parse}. Expected: true or false ." +fi + +# If user passes only_parse=true, then expect an experiment_id +# also with it, and use it as it is. +if ${only_parse}; then + if [ -z "${experiment_id}" ]; then + exitWithError "experiment_id not passed with only_parse=true" + fi +else + # create a new experiment_id + export user_passed_experiment_id="${experiment_id}" + export experiment_id=$(create_unique_experiment_id "${user_passed_experiment_id}") +fi if [[ ${pod_timeout_in_seconds} -le ${pod_wait_time_in_seconds} ]]; then exitWithError "pod_timeout_in_seconds (${pod_timeout_in_seconds}) <= pod_wait_time_in_seconds (${pod_wait_time_in_seconds})" @@ -194,6 +290,13 @@ else export output_dir="${gke_testing_dir}"/examples fi +if test -z "${zonal}"; then + echo "env var zonal not set, so assuming ${DEFAULT_ZONAL} for it." + export zonal=${DEFAULT_ZONAL} +elif [[ ${zonal} != "true" && "${zonal}" != "false" ]]; then + exitWithError "env var zonal should be set as false, or true, but received: ${zonal}" +fi + function printRunParameters() { echo "Running $0 with following parameters:" echo "" @@ -210,6 +313,7 @@ function printRunParameters() { echo "appnamespace=\"${appnamespace}\"" echo "ksa=\"${ksa}\"" echo "use_custom_csi_driver=\"${use_custom_csi_driver}\"" + echo "custom_csi_driver=\"${custom_csi_driver}\"" # GCSFuse/GKE GCSFuse CSI Driver source code related echo "src_dir=\"${src_dir}\"" echo "gcsfuse_src_dir=\"${gcsfuse_src_dir}\"" @@ -218,10 +322,14 @@ function printRunParameters() { # Test runtime configuration echo "pod_wait_time_in_seconds=\"${pod_wait_time_in_seconds}\"" echo "pod_timeout_in_seconds=\"${pod_timeout_in_seconds}\"" - echo "instance_id=\"${instance_id}\"" + echo "experiment_id=User passed: \"${user_passed_experiment_id}\", internally created: \"${experiment_id}\"" echo "workload_config=\"${workload_config}\"" echo "output_dir=\"${output_dir}\"" echo "force_update_gcsfuse_code=\"${force_update_gcsfuse_code}\"" + echo "zonal=\"${zonal}\"" + if ${only_parse}; then + echo "only_parse=${only_parse}" + fi echo "" echo "" echo "" @@ -231,36 +339,36 @@ function printRunParameters() { function installDependencies() { printf "\nInstalling dependencies ...\n\n" # Refresh software repositories. - sudo apt-get update + sudo apt-get update >/dev/null # Get some common software dependencies. - sudo apt-get install -y apt-transport-https ca-certificates gnupg curl + sudo apt-get install -y apt-transport-https ca-certificates gnupg curl >/dev/null # Ensure that realpath is installed. - which realpath + which realpath >/dev/null # Ensure that make is installed. - which make || ( sudo apt-get install -y make time && which make ) + which make >/dev/null || ( sudo apt-get install -y make time >/dev/null && which make >/dev/null ) # Ensure that go is installed. which go || (version=1.22.4 && wget -O go_tar.tar.gz https://go.dev/dl/go${version}.linux-amd64.tar.gz 1>/dev/null && sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz 1>/dev/null && sudo mv go /usr/local && echo $PATH && export PATH=$PATH:/usr/local/go/bin && echo $PATH && echo 'export PATH=$PATH:/usr/local/go/bin'>>~/.bashrc && go version) # for some reason, the above is unable to update the value of $PATH, so doing it explicitly below. export PATH=$PATH:/usr/local/go/bin - which go + which go >/dev/null # Ensure that python3 is installed. - which python3 || ( sudo apt-get install -y python3 && which python3 ) + which python3 || ( sudo apt-get install -y python3 >/dev/null && which python3 >/dev/null ) # Install more python tools. - sudo apt-get -y install python3-dev python3-venv python3-pip + sudo apt-get -y install python3-dev python3-venv python3-pip >/dev/null # Enable python virtual environment. python3 -m venv .venv source .venv/bin/activate # Ensure that pip is installed. - sudo apt-get install -y pip + sudo apt-get install -y pip >/dev/null # python3 -m pip install --upgrade pip # python3 -m pip --version # Ensure that python-absl is installed. - pip install absl-py + pip install --require-hashes -r "${src_dir}/perfmetrics/scripts/testing_on_gke/examples/requirements.txt" >/dev/null # Ensure that helm is installed which helm || (cd "${src_dir}" && (test -d "./helm" || git clone https://github.com/helm/helm.git) && cd helm && make && ls -lh bin/ && mkdir -pv ~/bin && cp -fv bin/helm ~/bin/ && chmod +x ~/bin/helm && export PATH=$PATH:$HOME/bin && echo $PATH && which helm && cd - && cd -) # for some reason, the above is unable to update the value of $PATH, so doing it explicitly below. export PATH=$PATH:$HOME/bin - which helm + which helm >/dev/null # Ensure that kubectl is installed if ! which kubectl; then # Install the latest gcloud cli. Find full instructions at https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-access-for-kubectl . @@ -269,42 +377,30 @@ function installDependencies() { # Add the gcloud CLI distribution URI as a package source (Debian 9+ or Ubuntu 18.04+) echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list # Update and install the gcloud CLI - sudo apt-get update - sudo apt-get install -y google-cloud-cli + sudo apt-get update >/dev/null + sudo apt-get install -y google-cloud-cli >/dev/null # install kubectl - gcloud components install kubectl || sudo apt-get install -y kubectl + gcloud components install kubectl >/dev/null || sudo apt-get install -y kubectl >/dev/null kubectl version --client fi # Ensure that gke-gcloud-auth-plugin is installed. - gke-gcloud-auth-plugin --version || ((gcloud components install gke-gcloud-auth-plugin || sudo apt-get install -y google-cloud-cli-gke-gcloud-auth-plugin) && gke-gcloud-auth-plugin --version) + gke-gcloud-auth-plugin --version || ((gcloud components install gke-gcloud-auth-plugin >/dev/null || sudo apt-get install -y google-cloud-cli-gke-gcloud-auth-plugin >/dev/null) && gke-gcloud-auth-plugin --version) # Ensure that docker is installed. if ! which docker ; then - sudo apt install apt-transport-https ca-certificates curl software-properties-common -y + sudo apt install apt-transport-https ca-certificates curl software-properties-common -y >/dev/null curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable" apt-cache policy docker-ce - sudo apt install docker-ce -y + sudo apt install docker-ce -y >/dev/null fi - # Install mash, as it is needed for fetching cpu/memory values for test runs - # in cloudtop. Even if mash install fails, don't panic, go ahead and install - # google-cloud-monitoring as an alternative. - which mash || sudo apt-get install -y monarch-tools || true - # Ensure that gcloud monitoring tools are installed. This is alternative to - # mash on gce vm. - # pip install --upgrade google-cloud-storage - # pip install --ignore-installed --upgrade google-api-python-client - # pip install --ignore-installed --upgrade google-cloud - pip install --upgrade google-cloud-monitoring # Ensure that jq is installed. - which jq || sudo apt-get install -y jq + which jq || sudo apt-get install -y jq >/dev/null # Ensure sudoless docker is installed. if ! docker ps 1>/dev/null ; then echoerror "sudoless docker is not installed on this machine ($(hostname)). Please install sudoless-docker using the following commands and re-run this script ($0)" echoerror "sudo addgroup docker && sudo usermod -aG docker $USER && newgrp docker" return 1 fi - # Install python modules for gsheet. - python3 -m pip install google-api-python-client } # Make sure you have access to the necessary GCP resources. The easiest way to enable it is to use <your-ldap>@google.com as active auth. @@ -389,7 +485,7 @@ function createNewNodePool() { local machine_type=${4} local num_nodes=${5} local num_ssd=${6} - gcloud container node-pools create ${node_pool} --project=${project_id} --cluster ${cluster_name} --ephemeral-storage-local-ssd count=${num_ssd} --network-performance-configs=total-egress-bandwidth-tier=TIER_1 --machine-type ${machine_type} --zone ${zone} --num-nodes ${num_nodes} --workload-metadata=GKE_METADATA + gcloud container node-pools create ${node_pool} --project=${project_id} --cluster ${cluster_name} --ephemeral-storage-local-ssd count=${num_ssd} --network-performance-configs=total-egress-bandwidth-tier=TIER_1 --machine-type ${machine_type} --zone ${zone} --num-nodes ${num_nodes} --workload-metadata=GKE_METADATA --enable-gvnic } function getMachineTypeInNodePool() { @@ -427,9 +523,20 @@ function ensureGkeCluster() { echo "Internally changing machine-type from ${machine_type} to ${existing_machine_type} ..." machine_type=${existing_machine_type} fi - gcloud container clusters update ${cluster_name} --project=${project_id} --location=${zone} --workload-pool=${project_id}.svc.id.goog + cluster_updation_command="gcloud container clusters update ${cluster_name} --project=${project_id} --location=${zone}" + ${cluster_updation_command} --workload-pool=${project_id}.svc.id.goog + # Separating in two update calls as gcloud doesn't support updating these + # two fields in a single call. + if ${zonal}; then + ${cluster_updation_command} --private-ipv6-google-access-type=bidirectional + fi else - gcloud container clusters create ${cluster_name} --project=${project_id} --zone "${zone}" --workload-pool=${project_id}.svc.id.goog --machine-type "${machine_type}" --image-type "COS_CONTAINERD" --num-nodes ${num_nodes} --ephemeral-storage-local-ssd count=${num_ssd} --network-performance-configs=total-egress-bandwidth-tier=TIER_1 --workload-metadata=GKE_METADATA + # Create a new cluster + cluster_creation_args="--project=${project_id} --zone \"${zone}\" --workload-pool=${project_id}.svc.id.goog --machine-type \"${machine_type}\" --image-type \"COS_CONTAINERD\" --num-nodes ${num_nodes} --ephemeral-storage-local-ssd count=${num_ssd} --network-performance-configs=total-egress-bandwidth-tier=TIER_1 --workload-metadata=GKE_METADATA --enable-gvnic \"${extra_args_for_cluster_creation}\"" + if ${zonal}; then + cluster_creation_args+=" --private-ipv6-google-access-type=bidirectional" + fi + gcloud container clusters create ${cluster_name} ${cluster_creation_args} fi } @@ -462,20 +569,12 @@ function ensureRequiredNodePoolConfiguration() { fi } -function enableManagedCsiDriverIfNeeded() { - if ${use_custom_csi_driver}; then - printf "\nDisabling csi add-on ...\n\n" - gcloud -q container clusters update ${cluster_name} \ +function enableManagedCsiDriver() { + printf "\nEnabling csi add-on ...\n\n" + gcloud -q container clusters update ${cluster_name} \ --project=${project_id} \ - --update-addons GcsFuseCsiDriver=DISABLED \ + --update-addons GcsFuseCsiDriver=ENABLED \ --location=${zone} - else - printf "\nEnabling csi add-on ...\n\n" - gcloud -q container clusters update ${cluster_name} \ - --project=${project_id} \ - --update-addons GcsFuseCsiDriver=ENABLED \ - --location=${zone} - fi } function activateCluster() { @@ -513,11 +612,12 @@ function ensureGcsFuseCsiDriverCode() { fi } -function createCustomCsiDriverIfNeeded() { - if ${use_custom_csi_driver}; then - echo "Disabling managed CSI driver ..." - gcloud -q container clusters update ${cluster_name} --project=${project_id} --update-addons GcsFuseCsiDriver=DISABLED --location=${zone} +uuid() { + echo $(uuidgen) | sed -e "s/\-//g" ; +} +function createCustomCsiDriverIfNeeded() { + if ${use_custom_csi_driver} && test -z "${applied_custom_csi_driver}"; then printf "\nCreating a new custom CSI driver ...\n\n" # Create a bucket (if needed) for storing GCSFuse binaries. @@ -550,48 +650,59 @@ function createCustomCsiDriverIfNeeded() { # Build and install csi driver ensureGcsFuseCsiDriverCode cd "${csi_src_dir}" - make uninstall || true make generate-spec-yaml printf "\nBuilding a new custom CSI driver using the above GCSFuse binary ...\n\n" registry=gcr.io/${project_id}/${USER}/${cluster_name} - make build-image-and-push-multi-arch REGISTRY=${registry} GCSFUSE_PATH=gs://${package_bucket} - printf "\nInstalling the new custom CSI driver built above ...\n\n" - make install PROJECT=${project_id} REGISTRY=${registry} - cd - - - # Wait some time after csi driver installation before deploying pods - # to avoid failures caused by 'the webhook failed to inject the - # sidecar container into the Pod spec' error. - printf "\nSleeping 30 seconds after csi custom driver installation before deploying pods ...\n\n" + if ! which uuidgen; then + # try to install uuidgen + sudo apt-get update && sudo apt-get install -y uuid-runtime + # confirm that it got installed. + which uuidgen + fi + # The prefix prow-gob-internal-boskos- is needed to allow passing machine-type from gke csi driver to gcsfuse, + # bypassing the check at + # https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/blob/15afd00dcc2cfe0f9753ddc53c81631ff037c3f2/pkg/csi_driver/utils.go#L532. + stagingversion=prow-gob-internal-boskos-$(uuid) + make build-image-and-push-multi-arch REGISTRY=${registry} GCSFUSE_PATH=gs://${package_bucket} STAGINGVERSION=${stagingversion} + + readonly subregistry=gcs-fuse-csi-driver-sidecar-mounter + applied_custom_csi_driver=${registry}/${subregistry}:${stagingversion} + printf "\n\nCreated custom csi driver \" ${applied_custom_csi_driver} \" . To use it in future runs, please pass environment variable \" custom_csi_driver=${applied_custom_csi_driver} \" .\n\n" + + # Verify that the csi-driver image is a good image to use.. + printf "\nVerifying that ${applied_custom_csi_driver} is a valid GCSFuse csi driver image ...\n\n" sleep 30 + verify_csi_driver_image ${applied_custom_csi_driver} - else - echo "" - echo "Enabling managed CSI driver ..." - gcloud -q container clusters update ${cluster_name} --project=${project_id} --update-addons GcsFuseCsiDriver=ENABLED --location=${zone} + cd - fi } function deleteAllHelmCharts() { - printf "\nDeleting all existing helm charts ...\n\n" + printf "Deleting all existing helm charts ...\n\n" helm ls --namespace=${appnamespace} | tr -s '\t' ' ' | cut -d' ' -f1 | tail -n +2 | while read helmchart; do helm uninstall ${helmchart} --namespace=${appnamespace}; done } function deleteAllPods() { deleteAllHelmCharts - printf "\nDeleting all existing pods ...\n\n" + printf "Deleting all existing pods ...\n\n" kubectl get pods --namespace=${appnamespace} | tail -n +2 | cut -d' ' -f1 | while read podname; do kubectl delete pods/${podname} --namespace=${appnamespace} --grace-period=0 --force || true; done } function deployAllFioHelmCharts() { printf "\nDeploying all fio helm charts ...\n\n" - cd "${gke_testing_dir}"/examples/fio && python3 ./run_tests.py --workload-config "${workload_config}" --instance-id ${instance_id} --machine-type="${machine_type}" --project-id=${project_id} --project-number=${project_number} --namespace=${appnamespace} --ksa=${ksa} && cd - + cd "${gke_testing_dir}"/examples/fio + python3 ./run_tests.py --workload-config "${workload_config}" --experiment-id ${experiment_id} --machine-type="${machine_type}" --project-id=${project_id} --project-number=${project_number} --namespace=${appnamespace} --ksa=${ksa} --custom-csi-driver=${applied_custom_csi_driver} + cd - } function deployAllDlioHelmCharts() { printf "\nDeploying all dlio helm charts ...\n\n" - cd "${gke_testing_dir}"/examples/dlio && python3 ./run_tests.py --workload-config "${workload_config}" --instance-id ${instance_id} --machine-type="${machine_type}" --project-id=${project_id} --project-number=${project_number} --namespace=${appnamespace} --ksa=${ksa} && cd - + cd "${gke_testing_dir}"/examples/dlio + python3 ./run_tests.py --workload-config "${workload_config}" --experiment-id ${experiment_id} --machine-type="${machine_type}" --project-id=${project_id} --project-number=${project_number} --namespace=${appnamespace} --ksa=${ksa} --custom-csi-driver=${applied_custom_csi_driver} + + cd - } function waitTillAllPodsComplete() { @@ -627,54 +738,142 @@ function waitTillAllPodsComplete() { printf "\nAll pods have completed.\n\n" break else - printf "\n${num_noncompleted_pods} pod(s) is/are still pending/running (time till timeout=${time_till_timeout} seconds). Will check again in "${pod_wait_time_in_seconds}" seconds. Sleeping for now.\n\n" - printf "\nYou can take a break too if you want. Just kill this run and connect back to it later, for fetching and parsing outputs, using the following command: \n" - printf " only_parse=true instance_id=${instance_id} project_id=${project_id} project_number=${project_number} zone=${zone} machine_type=${machine_type} use_custom_csi_driver=${use_custom_csi_driver} gcsfuse_src_dir=\"${gcsfuse_src_dir}\" csi_src_dir=\"${csi_src_dir}\" pod_wait_time_in_seconds=${pod_wait_time_in_seconds} pod_timeout_in_seconds=${pod_timeout_in_seconds} workload_config=\"${workload_config}\" cluster_name=${cluster_name} output_dir=\"${output_dir}\" $0 \n" - printf "\nbut remember that this will reset the start-timer for pod timeout.\n\n" - printf "\nTo ssh to any specific pod, use the following command: \n" - printf " gcloud container clusters get-credentials ${cluster_name} --location=${zone}\n" - printf " kubectl config set-context --current --namespace=${appnamespace}\n" - printf " kubectl exec -it pods/<podname> [-c {gke-gcsfuse-sidecar|fio-tester|dlio-tester}] --namespace=${appnamespace} -- /bin/bash \n" - printf "\nTo view cpu/memory usage of different pods/containers: \n" - printf " kubectl top pod [<podname>] --namespace=${appnamespace} [--containers] \n" - printf "\nTo view the latest status of all the pods in this cluster/namespace: \n" - printf " kubectl get pods --namespace=${appnamespace} [-o wide] [--watch] \n" - printf "\nTo output the configuration of all or one of the pods in this cluster/namespace (useful for debugging): \n" - printf " kubectl get [pods or pods/<podname>] --namespace=${appnamespace} -o yaml \n" - printf "\n\n\n" + message="\n${num_noncompleted_pods} pod(s) is/are still pending/running (time till timeout=${time_till_timeout} seconds). Will check again in "${pod_wait_time_in_seconds}" seconds. Sleeping for now.\n\n" + message+="\nYou can take a break too if you want. Just kill this run and connect back to it later, for fetching and parsing outputs, using the following command: \n\n" + message+=" only_parse=true experiment_id=${experiment_id} project_id=${project_id} project_number=${project_number} zone=${zone} machine_type=${machine_type}" + message+=" use_custom_csi_driver=${use_custom_csi_driver}" + if test -n "${custom_csi_driver}"; then + message+=" custom_csi_driver=${custom_csi_driver}" + fi + message+=" gcsfuse_src_dir=\"${gcsfuse_src_dir}\" " + if test -d "${csi_src_dir}"; then + message+="csi_src_dir=\"${csi_src_dir}\" " + fi + message+=" zonal=${zonal} " + message+="pod_wait_time_in_seconds=${pod_wait_time_in_seconds} pod_timeout_in_seconds=${pod_timeout_in_seconds} workload_config=\"${workload_config}\" cluster_name=${cluster_name} output_dir=\"${output_dir}\" $0 \n" + message+="\nbut remember that this will reset the start-timer for pod timeout.\n\n" + message+="\nTo ssh to any specific pod, use the following command: \n" + message+=" gcloud container clusters get-credentials ${cluster_name} --location=${zone}\n" + message+=" kubectl config set-context --current --namespace=${appnamespace}\n" + message+=" kubectl exec -it pods/<podname> [-c {gke-gcsfuse-sidecar|fio-tester|dlio-tester}] --namespace=${appnamespace} -- /bin/bash \n" + message+="\nTo view cpu/memory usage of different pods/containers: \n" + message+=" kubectl top pod [<podname>] --namespace=${appnamespace} [--containers] \n" + message+="\nTo view the latest status of all the pods in this cluster/namespace: \n" + message+=" kubectl get pods --namespace=${appnamespace} [-o wide] [--watch] \n" + message+="\nTo output the configuration of all or one of the pods in this cluster/namespace (useful for debugging): \n" + message+=" kubectl get [pods or pods/<podname>] --namespace=${appnamespace} -o yaml \n" + printf "${message}\n\n\n" fi sleep ${pod_wait_time_in_seconds} unset podslist # necessary to update the value of podslist every iteration done } +# Download all the fio workload outputs for the current experiment-id from the +# given bucket and file-size. +function downloadFioOutputsFromBucket() { + local bucket=$1 + local mountpath=$2/${bucket}-mount + + mkdir -p $mountpath + fusermount -uz $mountpath 2>/dev/null || true + echo "Searching for FIO outputs for experiment ${experiment_id} in gs://${bucket} ..." + + cd $gcsfuse_src_dir + if ! go run $gcsfuse_src_dir --implicit-dirs --o ro $bucket $mountpath > /dev/null ; then + # If fails to mount this bucket, + # Return to original directory before exiting.. + cd - >/dev/null + + exitWithError "Failed to mount bucket ${bucket} to ${mountpath}." + fi + + # Return to original directory. + cd - >/dev/null + + # If the given bucket has the fio outputs for the given experiment-id, then + # copy/download them locally to the appropriate folder. + src_dir="${mountpath}/fio-output/${experiment_id}" + dst_dir="${gcsfuse_src_dir}/perfmetrics/scripts/testing_on_gke/bin/fio-logs/${experiment_id}/${bucket}" + if test -d "${src_dir}" ; then + mkdir -p "${dst_dir}" + echo "Copying all files from \"${src_dir}\" to \"${dst_dir}/\" ... " + cp -rfu "${src_dir}"/* "${dst_dir}"/ + fi + + fusermount -uz "${mountpath}" || true + rm -rf "${mountpath}" +} + +function downloadFioOutputsFromAllBucketsInWorkloadConfig() { + local mountpath=$(realpath mounted) + # Using jquery, find out all the relevant buckets for non-disabled fio + # workloads in the workload-config file and download fio outputs for them all. + cat ${workload_config} | jq 'select(.TestConfig.workloadConfig.workloads[].fioWorkload != null)' | jq -r '.TestConfig.workloadConfig.workloads[] | [.bucket] | @csv' | grep -v " " | sort | uniq | while read bucket; do + bucket=$(echo ${bucket} | tr -d \" ) + if [[ "${bucket}" != "" ]]; then + downloadFioOutputsFromBucket ${bucket} "${mountpath}" + fi + done + rm -rf ${mountpath} +} + +function areThereAnyDLIOWorkloads() { + lines=$(cat ${workload_config} | jq 'select(.TestConfig.workloadConfig.workloads[].dlioWorkload != null)' | jq -r '.TestConfig.workloadConfig.workloads[] | [.bucket, .dlioWorkload.numFilesTrain, .dlioWorkload.recordLength] | @csv' | grep -v " " | sort | uniq) + while read bucket_numFilesTrain_recordLength_combo; do + workload_bucket=$(echo ${bucket_numFilesTrain_recordLength_combo} | cut -d, -f1 | tr -d \") + workload_numFileTrain=$(echo ${bucket_numFilesTrain_recordLength_combo} | cut -d, -f2 | tr -d \") + workload_recordLength=$(echo ${bucket_numFilesTrain_recordLength_combo} | cut -d, -f3 | tr -d \") + if [[ "${workload_bucket}" != "" && "${workload_numFileTrain}" != "" && "${workload_recordLength}" != "" ]]; then + return 0 + fi + done <<< "${lines}" # It's necessary to pass lines this way to while + # to avoid creating a subshell for while-execution, to + # ensure that the above return statement works in the same shell. + + return 1 +} + function fetchAndParseFioOutputs() { printf "\nFetching and parsing fio outputs ...\n\n" cd "${gke_testing_dir}"/examples/fio - python3 parse_logs.py --project-number=${project_number} --workload-config "${workload_config}" --instance-id ${instance_id} --output-file "${output_dir}"/fio/output.csv --project-id=${project_id} --cluster-name=${cluster_name} --namespace-name=${appnamespace} - cd - + parse_logs_args="--project-number=${project_number} --workload-config ${workload_config} --experiment-id ${experiment_id} --output-file ${output_dir}/fio/output.csv --project-id=${project_id} --cluster-name=${cluster_name} --namespace-name=${appnamespace} --bq-project-id=${DEFAULT_BQ_PROJECT_ID} --bq-dataset-id=${DEFAULT_BQ_DATASET_ID} --bq-table-id=${DEFAULT_BQ_TABLE_ID}" + if ${zonal}; then + # Download fio outputs from all buckets using gcsfuse because zonal buckets don't work with gcloud storage cp. + printf "\nDownloading all fio outputs using gcsfuse mount as there are zonal buckets involved ...\n\n" + downloadFioOutputsFromAllBucketsInWorkloadConfig + + python3 parse_logs.py ${parse_logs_args} --predownloaded-output-files + else + python3 parse_logs.py ${parse_logs_args} + fi + cd - >/dev/null } function fetchAndParseDlioOutputs() { printf "\nFetching and parsing dlio outputs ...\n\n" cd "${gke_testing_dir}"/examples/dlio - python3 parse_logs.py --project-number=${project_number} --workload-config "${workload_config}" --instance-id ${instance_id} --output-file "${output_dir}"/dlio/output.csv --project-id=${project_id} --cluster-name=${cluster_name} --namespace-name=${appnamespace} - cd - + python3 parse_logs.py --project-number=${project_number} --workload-config "${workload_config}" --experiment-id ${experiment_id} --output-file "${output_dir}"/dlio/output.csv --project-id=${project_id} --cluster-name=${cluster_name} --namespace-name=${appnamespace} + cd - >/dev/null } # prep printRunParameters installDependencies -# if only_parse is not set or is set as false, then -if test -z ${only_parse} || ! ${only_parse} ; then +# if only_parse is false, then +if ! ${only_parse} ; then validateMachineConfig ${machine_type} ${num_nodes} ${num_ssd} + if ${zonal} && $(areThereAnyDLIOWorkloads); then + exitWithError "DLIO workloads are not supported with zonal buckets as of now." + fi + # GCP configuration ensureGcpAuthsAndConfig ensureGkeCluster # ensureRequiredNodePoolConfiguration - enableManagedCsiDriverIfNeeded + enableManagedCsiDriver activateCluster createKubernetesServiceAccountForCluster @@ -699,3 +898,7 @@ deleteAllPods # parse outputs fetchAndParseFioOutputs fetchAndParseDlioOutputs + +if test -z "${custom_csi_driver}" && test -n "${applied_custom_csi_driver}"; then + printf "\nTo reuse this custom CSI driver in future runs, pass environment variable \" custom_csi_driver=${applied_custom_csi_driver} \" .\n\n" +fi diff --git a/perfmetrics/scripts/testing_on_gke/examples/utils/parse_logs_common.py b/perfmetrics/scripts/testing_on_gke/examples/utils/parse_logs_common.py deleted file mode 100644 index 936efc9da7..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/utils/parse_logs_common.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018 The Kubernetes Authors. -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import os -import subprocess -from typing import Tuple - -SUPPORTED_SCENARIOS = [ - "local-ssd", - "gcsfuse-generic", - "gcsfuse-no-file-cache", - "gcsfuse-file-cache", -] - - -def ensure_directory_exists(dirpath: str): - try: - os.makedirs(dirpath) - except FileExistsError: - pass - - -def download_gcs_objects(src: str, dst: str) -> Tuple[int, str]: - result = subprocess.run( - [ - "gcloud", - "-q", # ignore prompts - "storage", - "cp", - "-r", - "--no-user-output-enabled", # do not print names of objects being copied - src, - dst, - ], - capture_output=False, - text=True, - ) - if result.returncode < 0: - return (result.returncode, f"error: {result.stderr}") - return result.returncode, "" - - -def parse_arguments() -> object: - parser = argparse.ArgumentParser( - prog="DLIO Unet3d test output parser", - description=( - "This program takes in a json workload configuration file and parses" - " it for valid FIO workloads and the locations of their test outputs" - " on GCS. It downloads each such output object locally to" - " {_LOCAL_LOGS_LOCATION} and parses them for FIO test runs, and then" - " dumps their output metrics into a CSV report file." - ), - ) - parser.add_argument( - "--workload-config", - help=( - "A json configuration file to define workloads that were run to" - " generate the outputs that should be parsed." - ), - required=True, - ) - parser.add_argument( - "--project-id", - metavar="GCP Project ID/name", - help=( - "project-id (e.g. gcs-fuse-test) is needed to fetch the cpu/memory" - " utilization data from GCP." - ), - required=True, - ) - parser.add_argument( - "--project-number", - metavar="GCP Project Number", - help=( - "project-number (e.g. 93817472919) is needed to fetch the cpu/memory" - " utilization data from GCP." - ), - required=True, - ) - parser.add_argument( - "--instance-id", - help="unique string ID for current test-run", - required=True, - ) - parser.add_argument( - "--cluster-name", - help="Name of GKE cluster where the current test was run", - required=True, - ) - parser.add_argument( - "--namespace-name", - help="kubernestes namespace used for the current test-run", - required=True, - ) - parser.add_argument( - "-o", - "--output-file", - metavar="Output file (CSV) path", - help="File path of the output metrics (in CSV format)", - default="output.csv", - ) - return parser.parse_args() diff --git a/perfmetrics/scripts/testing_on_gke/examples/utils/run_tests_common.py b/perfmetrics/scripts/testing_on_gke/examples/utils/run_tests_common.py deleted file mode 100644 index f207bb1ca2..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/utils/run_tests_common.py +++ /dev/null @@ -1,156 +0,0 @@ -# Copyright 2018 The Kubernetes Authors. -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Common code for fio/run_tests.py and dlio/run_tests.py""" - -import argparse -import subprocess -import sys - - -def run_command(command: str) -> int: - """Runs the given string command as a subprocess. - - Returns exit-code which would be non-zero for error. - """ - result = subprocess.run( - [word for word in command.split(' ') if (word and not str.isspace(word))], - capture_output=True, - text=True, - ) - print(result.stdout) - print(result.stderr) - return result.returncode - - -def escape_commas_in_string(unescapedStr: str) -> str: - """Returns equivalent string with ',' replaced with '\,' .""" - return unescapedStr.replace(',', '\,') - - -def parse_args(): - parser = argparse.ArgumentParser( - prog='FIO test runner', - description=( - 'This program takes in a json test-config file, finds out valid FIO' - ' workloads from it and generates and deploys a helm chart for each' - ' FIO workload.' - ), - ) - parser.add_argument( - '--workload-config', - metavar='JSON workload configuration file path', - help='Runs FIO tests from this JSON workload configuration file.', - required=True, - ) - parser.add_argument( - '--instance-id', - metavar='A unique string ID to represent the test-run', - help=( - 'Set to a unique string ID for current test-run. Do not put spaces' - ' in it.' - ), - required=True, - ) - parser.add_argument( - '--machine-type', - metavar='Machine-type of the GCE VM or GKE cluster node', - help='Machine-type of the GCE VM or GKE cluster node e.g. n2-standard-32', - required=True, - ) - parser.add_argument( - '--project-id', - metavar='project-id of the user gke cluster', - help='project-id of the user gke cluster e.g. gcs-fuse-test', - required=True, - ) - parser.add_argument( - '--project-number', - metavar='project-number of the user gke cluster', - help='project-number of the user gke cluster e.g. 927584127901', - required=True, - type=int, - ) - parser.add_argument( - '--namespace', - metavar='kubectl namespace of the user', - help='kubectl namespace of the user e.g. default', - required=False, - default='default', - ) - parser.add_argument( - '--ksa', - metavar='kubernetes service account of the user', - help='kubernetes service account of the user e.g. default', - required=False, - default='default', - ) - parser.add_argument( - '-n', - '--dry-run', - action='store_true', - help=( - 'Only print out the test configurations that will run,' - ' not actually run them.' - ), - ) - - args = parser.parse_args() - for argument in [ - 'instance_id', - 'machine_type', - 'project_id', - 'namespace', - 'ksa', - ]: - value = getattr(args, argument) - if not value.strip(): - raise Exception( - f'Argument {argument} (value="{value}") is empty or contains only' - ' spaces.' - ) - if ' ' in value: - raise Exception( - f'Argument {argument} (value="{value}") contains space in it, which' - ' is not supported.' - ) - - return args - - -def add_iam_role_for_buckets( - buckets: set, - role: str, - project_id: str, - project_number: str, - namespace: str, - ksa: str, -): - print( - f'Adding role {role} to all the relevant buckets to' - f' ksa={ksa} in namespace={namespace} ...\n\n' - ) - for bucket in buckets: - command = ( - f'gcloud storage buckets add-iam-policy-binding gs://{bucket} --member' - f' principal://iam.googleapis.com/projects/{project_number}/locations/global/workloadIdentityPools/{project_id}.svc.id.goog/subject/ns/{namespace}/sa/{ksa} --role' - f' {role}' - ) - print(command) - ret = run_command(command) - if ret != 0: - raise Exception( - f'Failed to add role {role} for {bucket}: exit-code={ret}' - ) diff --git a/perfmetrics/scripts/testing_on_gke/examples/utils/run_tests_common_test.py b/perfmetrics/scripts/testing_on_gke/examples/utils/run_tests_common_test.py deleted file mode 100644 index ddf3ca806c..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/utils/run_tests_common_test.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2018 The Kubernetes Authors. -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest -from run_tests_common import escape_commas_in_string - - -class RunTestsCommonTest(unittest.TestCase): - - def test_escape_commas_in_string(self): - tcs = [ - {"input": "a:b,c=d,", "expected_output": "a:b\,c=d\,"}, - {"input": "", "expected_output": ""}, - ] - for tc in tcs: - self.assertEqual( - tc["expected_output"], escape_commas_in_string(tc["input"]) - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/perfmetrics/scripts/testing_on_gke/examples/utils/utils.py b/perfmetrics/scripts/testing_on_gke/examples/utils/utils.py deleted file mode 100644 index 4bdafaf089..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/utils/utils.py +++ /dev/null @@ -1,364 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018 The Kubernetes Authors. -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime, subprocess -import math -import time -from typing import Tuple -from google.cloud import monitoring_v3 - -_GCSFUSE_CONTAINER_NAME = "gke-gcsfuse-sidecar" - - -def is_mash_installed() -> bool: - try: - subprocess.run( - ["mash", "--version"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=True, - ) - return True - except subprocess.CalledProcessError: - return False - except FileNotFoundError: - return False - - -def get_memory( - pod_name: str, start: str, end: str, project_number: int -) -> Tuple[int, int]: - # for some reason, the mash filter does not always work, so we fetch all the metrics for all the pods and filter later. - result = subprocess.run( - [ - "mash", - "--namespace=cloud_prod", - "--output=csv", - ( - "Query(Fetch(Raw('cloud.kubernetes.K8sContainer'," - " 'kubernetes.io/container/memory/used_bytes'), {'project':" - f" '{project_number}', 'metric:memory_type': 'non-evictable'}})|" - " Window(Align('10m'))| GroupBy(['pod_name', 'container_name']," - f" Max()), TimeInterval('{start}', '{end}'), '5s')" - ), - ], - capture_output=True, - text=True, - ) - - data_points_int = [] - data_points_by_pod_container = result.stdout.strip().split("\n") - for data_points in data_points_by_pod_container[1:]: - data_points_split = data_points.split(",") - if len(data_points_split) < 6: - continue - pn = data_points_split[4] - container_name = data_points_split[5] - if pn == pod_name and container_name == _GCSFUSE_CONTAINER_NAME: - try: - data_points_int = [int(d) for d in data_points_split[7:]] - except: - print( - f"failed to parse memory for pod {pod_name}, {start}, {end}, data" - f" {data_points_int}" - ) - break - if not data_points_int: - return 0, 0 - - return int(min(data_points_int) / 1024**2), int( - max(data_points_int) / 1024**2 - ) - - -def get_cpu( - pod_name: str, start: str, end: str, project_number: int -) -> Tuple[float, float]: - # for some reason, the mash filter does not always work, so we fetch all the metrics for all the pods and filter later. - result = subprocess.run( - [ - "mash", - "--namespace=cloud_prod", - "--output=csv", - ( - "Query(Fetch(Raw('cloud.kubernetes.K8sContainer'," - " 'kubernetes.io/container/cpu/core_usage_time'), {'project':" - f" '{project_number}'}})| Window(Rate('10m'))|" - " GroupBy(['pod_name', 'container_name'], Max())," - f" TimeInterval('{start}', '{end}'), '5s')" - ), - ], - capture_output=True, - text=True, - ) - - data_points_float = [] - data_points_by_pod_container = result.stdout.split("\n") - for data_points in data_points_by_pod_container[1:]: - data_points_split = data_points.split(",") - if len(data_points_split) < 6: - continue - pn = data_points_split[4] - container_name = data_points_split[5] - if pn == pod_name and container_name == _GCSFUSE_CONTAINER_NAME: - try: - data_points_float = [float(d) for d in data_points_split[6:]] - except: - print( - f"failed to parse CPU for pod {pod_name}, {start}, {end}, data" - f" {data_points_float}" - ) - - break - - if not data_points_float: - return 0.0, 0.0 - - return round(min(data_points_float), 5), round(max(data_points_float), 5) - - -def unix_to_timestamp(unix_timestamp: int) -> str: - # Convert Unix timestamp to a datetime object (aware of UTC) - datetime_utc = datetime.datetime.fromtimestamp( - unix_timestamp / 1000, tz=datetime.timezone.utc - ) - - # Format the datetime object as a string (if desired) - utc_timestamp_string = datetime_utc.strftime("%Y-%m-%d %H:%M:%S UTC") - - return utc_timestamp_string - - -def standard_timestamp(timestamp: str) -> str: - return timestamp.split(".")[0].replace("T", " ") + " UTC" - - -def timestamp_to_epoch(timestamp: str) -> int: - try: - return int( - datetime.datetime.strptime( - timestamp, "%Y-%m-%dT%H:%M:%S.%f" - ).timestamp() - ) - except ValueError: - return int( - datetime.datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S").timestamp() - ) - - -class UnknownMachineTypeError(Exception): - """Defines custom exception for unknown machine-type scenario. - - It holds value of machineType as str. - """ - - def __init__(self, message, machineType: str): - super().__init__(message) - self.machineType = machineType - - -def resource_limits(nodeType: str) -> Tuple[dict, dict]: - """Returns resource limits and requests for cpu/memory for different machine types.""" - if nodeType == "n2-standard-96": - return {"cpu": 96, "memory": "384Gi"}, {"cpu": 90, "memory": "300Gi"} - elif nodeType == "n2-standard-48": - return {"cpu": 48, "memory": "192Gi"}, {"cpu": 45, "memory": "150Gi"} - elif nodeType == "n2-standard-32": - return {"cpu": 32, "memory": "128Gi"}, {"cpu": 30, "memory": "100Gi"} - elif nodeType == "c3-standard-176" or nodeType == "c3-standard-176-lssd": - return {"cpu": 176, "memory": "704Gi"}, {"cpu": 100, "memory": "400Gi"} - else: - raise UnknownMachineTypeError( - f"Unknown machine-type: {nodeType}. Unable to decide the" - " resource-limits for it.", - nodeType, - ) - - -def _is_relevant_monitoring_result( - result, - cluster_name: str, - pod_name: str, - namespace_name: str, -) -> bool: - return ( - True - if ( - hasattr(result, "resource") - and hasattr(result.resource, "type") - and result.resource.type == "k8s_container" - and hasattr(result.resource, "labels") - and "cluster_name" in result.resource.labels - and result.resource.labels["cluster_name"] == cluster_name - and "pod_name" in result.resource.labels - and result.resource.labels["pod_name"] == pod_name - and "container_name" in result.resource.labels - and result.resource.labels["container_name"] - == _GCSFUSE_CONTAINER_NAME - and "namespace_name" in result.resource.labels - and result.resource.labels["namespace_name"] == namespace_name - and hasattr(result, "points") - ) - else False - ) - - -def get_memory_from_monitoring_api( - project_id: str, - cluster_name: str, - pod_name: str, - namespace_name: str, - start_epoch: int, - end_epoch: int, -) -> Tuple[int, int]: - """Returns min,max memory usage of the given gke-cluster/namespace/pod/container/start/end scenario in MiB .""" - client = monitoring_v3.MetricServiceClient() - project_name = f"projects/{project_id}" - - interval = monitoring_v3.TimeInterval({ - "start_time": {"seconds": start_epoch, "nanos": 0}, - "end_time": {"seconds": end_epoch, "nanos": 0}, - }) - aggregation = monitoring_v3.Aggregation({ - "alignment_period": {"seconds": 600}, # 10 minutes - "per_series_aligner": monitoring_v3.Aggregation.Aligner.ALIGN_MAX, - }) - - results = client.list_time_series( - request={ - "name": project_name, - "filter": ( - 'metric.type = "kubernetes.io/container/memory/used_bytes"' - ' AND metric.labels.memory_type = "non-evictable"' - f" AND resource.labels.cluster_name = {cluster_name}" - f" AND resource.labels.pod_name = {pod_name}" - f" AND resource.labels.container_name = {_GCSFUSE_CONTAINER_NAME}" - f" AND resource.labels.namespace_name = {namespace_name}" - ), - "interval": interval, - "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL, - "aggregation": aggregation, - } - ) - - relevant_results = [ - result - for result in results - if _is_relevant_monitoring_result( - result, - cluster_name, - pod_name, - namespace_name, - ) - ] - return round( - min( - min( - (point.value.int64_value if point.value.int64_value >= 0 else 0) - for point in result.points - ) - for result in relevant_results - ) - / 2**20, # convert to MiB/s - 0, # round to integer. - ), round( - max( - max( - (point.value.int64_value if point.value.int64_value > 0 else 0) - for point in result.points - ) - for result in relevant_results - ) - / 2**20, # convert to MiB/s - 0, # round to integer. - ) - - -def get_cpu_from_monitoring_api( - project_id: str, - cluster_name: str, - pod_name: str, - namespace_name: str, - start_epoch: int, - end_epoch: int, -) -> Tuple[float, float]: - """Returns min,max cpu usage of the given gke-cluster/namespace/pod/container/start/end scenario.""" - client = monitoring_v3.MetricServiceClient() - project_name = f"projects/{project_id}" - - interval = monitoring_v3.TimeInterval({ - "start_time": {"seconds": start_epoch, "nanos": 0}, - "end_time": {"seconds": end_epoch, "nanos": 0}, - }) - aggregation = monitoring_v3.Aggregation({ - "alignment_period": {"seconds": 600}, # 10 minutes - "per_series_aligner": monitoring_v3.Aggregation.Aligner.ALIGN_RATE, - }) - - results = client.list_time_series( - request={ - "name": project_name, - "filter": ( - 'metric.type = "kubernetes.io/container/cpu/core_usage_time"' - f" AND resource.labels.cluster_name = {cluster_name}" - f" AND resource.labels.pod_name = {pod_name}" - f" AND resource.labels.container_name = {_GCSFUSE_CONTAINER_NAME}" - f" AND resource.labels.namespace_name = {namespace_name}" - ), - "interval": interval, - "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL, - "aggregation": aggregation, - } - ) - - relevant_results = [ - result - for result in results - if _is_relevant_monitoring_result( - result, - cluster_name, - pod_name, - namespace_name, - ) - ] - return round( - min( - min( - ( - point.value.double_value - if point.value.double_value != math.nan - else 0 - ) - for point in result.points - ) - for result in relevant_results - ), - 5, # round up to 5 decimal places. - ), round( - max( - max( - ( - point.value.double_value - if point.value.double_value != math.nan - else 0 - ) - for point in result.points - ) - for result in relevant_results - ), - 5, # round up to 5 decimal places. - ) diff --git a/perfmetrics/scripts/testing_on_gke/examples/utils/utils_test.py b/perfmetrics/scripts/testing_on_gke/examples/utils/utils_test.py deleted file mode 100644 index dae89cd41a..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/utils/utils_test.py +++ /dev/null @@ -1,135 +0,0 @@ -"""This file defines unit tests for functionalities in utils.py""" - -# Copyright 2018 The Kubernetes Authors. -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest -import utils -from utils import get_cpu, get_cpu_from_monitoring_api, get_memory, get_memory_from_monitoring_api, timestamp_to_epoch - - -class UtilsTest(unittest.TestCase): - - @classmethod - def setUpClass(self): - self.project_id = 'gcs-fuse-test-ml' - self.project_number = 786757290066 - self.cluster_name = 'gargnitin-gketesting-us-west1b' - self.pod_name = 'fio-tester-gcsfuse-rr-200g-5061221202711042867' - self.namespace_name = 'default' - self.start_epoch = 1727245942 - self.end_epoch = 1727247982 - self.start = '2024-09-25 06:32:22 UTC' - self.end = '2024-09-25 07:06:22 UTC' - - def test_get_memory_methods(self): - low1, high1 = get_memory_from_monitoring_api( - project_id=self.project_id, - cluster_name=self.cluster_name, - pod_name=self.pod_name, - namespace_name=self.namespace_name, - start_epoch=self.start_epoch, - end_epoch=self.end_epoch, - ) - self.assertLessEqual(low1, high1) - self.assertGreater(high1, 0) - - low2, high2 = get_memory( - project_number=self.project_number, - pod_name=self.pod_name, - start=self.start, - end=self.end, - ) - self.assertLessEqual(low2, high2) - self.assertGreater(high2, 0) - - self.assertTrue(high1 >= 0.99 * high2 and high1 <= 1.01 * high2) - - def test_get_cpu_methods(self): - low1, high1 = get_cpu_from_monitoring_api( - project_id=self.project_id, - cluster_name=self.cluster_name, - pod_name=self.pod_name, - namespace_name=self.namespace_name, - start_epoch=self.start_epoch, - end_epoch=self.end_epoch, - ) - self.assertLessEqual(low1, high1) - self.assertGreater(high1, 0) - - low2, high2 = get_cpu( - project_number=self.project_number, - pod_name=self.pod_name, - start=self.start, - end=self.end, - ) - self.assertLessEqual(low2, high2) - self.assertGreater(high2, 0) - - self.assertTrue(high1 >= 0.99 * high2 and high1 <= 1.01 * high2) - - def test_timestamp_to_epoch(self): - self.assertEqual(timestamp_to_epoch('2024-08-21T19:20:25'), 1724268025) - - def test_timestamp_to_epoch_with_nznano(self): - self.assertEqual( - timestamp_to_epoch('2024-08-21T19:20:25.547456'), 1724268025 - ) - - def test_resource_limit(self): - inputs = [ - { - 'nodeType': 'n2-standard-32', - 'expected_limits_cpu': 32, - 'expected_error': False, - }, - { - 'nodeType': 'n2-standard-96', - 'expected_limits_cpu': 96, - 'expected_error': False, - }, - { - 'nodeType': 'n2-standard-96', - 'expected_limits_cpu': 96, - 'expected_error': False, - }, - { - 'nodeType': 'c3-standard-176', - 'expected_limits_cpu': 176, - 'expected_error': False, - }, - { - 'nodeType': 'c3-standard-176-lssd', - 'expected_limits_cpu': 176, - 'expected_error': False, - }, - {'nodeType': 'n2-standard-1', 'expected_error': True}, - {'nodeType': 'unknown-machine-type', 'expected_error': True}, - ] - for input in inputs: - self.assertEqual(dict, type(input)) - try: - resource_limits = utils.resource_limits(input['nodeType']) - self.assertEqual( - input['expected_limits_cpu'], - resource_limits[0]['cpu'], - ) - self.assertFalse(input['expected_error']) - except utils.UnknownMachineTypeError: - self.assertTrue(input['expected_error']) - - -if __name__ == '__main__': - unittest.main() diff --git a/perfmetrics/scripts/testing_on_gke/examples/workloads.json b/perfmetrics/scripts/testing_on_gke/examples/workloads.json deleted file mode 100644 index ac1039d257..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/workloads.json +++ /dev/null @@ -1,179 +0,0 @@ -{ - "_comment": "_ in the starting of element name indicates comment.", - "TestConfig": { - "workloadConfig": { - "_description": "workloadConfig has an optional field runOnSSD (default true if missing), and an array of workloads.", - "runOnSSD": false, - "workloads": [ - { - "_description": "This is a dummy fio workload (missing the 'fioWorkload' field), purely standing as a header and does not execute any workload. For it to execute a fio workload, it must have a valid 'fioWorkload', a valid 'bucket' attribute, and a valid gcsfuseMountOption attribute.", - "_fioWorkload": { - "_description": "Every fioWorkload must have fileSize, filesPerThread, numThreads, and blockSize fields. readTypes is an array of string values 'read' and 'randread'. If readTypes is missing, then it defaults to [\"read\",\"randread\"].", - "fileSize": "64k", - "filesPerThread": 20000, - "numThreads": 50, - "blockSize": "64K", - "readTypes": ["read","randread"] - }, - "gcsfuseMountOptions": "GCSFuse mount-options, in a compact stringified format, to be used for the test scenario gcsfuse-generic. The individual config/cli flag values should be separated by comma. Each cli flag should be of the form <flag>[=<value>], while each config-file flag should be of form <config>[:<subconfig>[:<subsubconfig>[...]]]:<value>. For example, a legal value would be: implicit-dirs,file_mode=777,file-cache:enable-parallel-downloads:true,metadata-cache:ttl-secs:-1 .", - "bucket":"The bucket must have objects with name Workload.{i}/{j} for every i,j where i:0-{numThreads}-1, j:0-{filesPerThread}-1, and each of these objects must be of size {fileSize}. The buckets gke-* are all in us-central1, are owned by GKE team and are in their GCP project(s)." - }, - { - "fioWorkload": { - "fileSize": "64k", - "filesPerThread": 20000, - "numThreads": 50, - "blockSize": "64K", - "readTypes": ["read"] - }, - "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true", - "bucket":"fio-64k-1m-us-west1", - "_bucket_alt2":"fio-64k-1m-us-central1", - "_bucket_alt3":"gke-fio-64k-1m" - }, - { - "fioWorkload": { - "fileSize": "128K", - "filesPerThread": 20000, - "numThreads": 50, - "blockSize": "128K", - "readTypes": ["read"] - }, - "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true", - "bucket":"fio-128k-1m-us-west1", - "_bucket_alt2":"fio-128k-1m-us-central1", - "_bucket_alt3":"gke-fio-128k-1m" - }, - { - "fioWorkload": { - "fileSize": "1M", - "filesPerThread": 20000, - "numThreads": 50, - "blockSize": "256K", - "readTypes": ["read","randread"] - }, - "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true", - "bucket":"fio-1mb-1m-us-west1", - "_bucket_alt2":"fio-1mb-1m-us-central1", - "_bucket_alt3":"gke-fio-1mb-1m" - }, - { - "fioWorkload": { - "fileSize": "100M", - "filesPerThread": 1000, - "numThreads": 50, - "blockSize": "1M" - }, - "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true", - "bucket":"fio-100mb-50k-us-west1", - "_bucket_alt2":"fio-100mb-50k-us-central1", - "_bucket_alt3":"gke-fio-100mb-50k" - }, - { - "fioWorkload": { - "_description": "This workload's job file is configured differently from the rest. It has one file, whis is read in parallel depending on the value of numThreads (only 100 supported right now).", - "fileSize": "200G", - "filesPerThread": 1, - "numThreads": 100, - "blockSize": "1M" - }, - "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true", - "bucket":"fio-200gb-1-us-west1", - "_bucket_alt2":"fio-200gb-1-us-central1", - "_bucket_alt3":"gke-fio-200gb-1" - }, - { - "fioWorkload": { - "fileSize": "10G", - "filesPerThread": 16, - "numThreads": 1, - "blockSize": "1M", - "readTypes": ["read"] - }, - "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true", - "bucket":"fio-10g-500-us-west1", - "_bucket_alt2":"fio-10g-500-us-central1" - }, - { - "fioWorkload": { - "fileSize": "10G", - "filesPerThread": 2, - "numThreads": 8, - "blockSize": "1M", - "readTypes": ["read"] - }, - "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true", - "bucket":"fio-10g-500-us-west1", - "_bucket_alt2":"fio-10g-500-us-central1" - }, - { - "fioWorkload": { - "fileSize": "10G", - "filesPerThread": 1, - "numThreads": 96, - "blockSize": "1M", - "readTypes": ["read"] - }, - "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true", - "bucket":"fio-10g-500-us-west1", - "_bucket_alt2":"fio-10g-500-us-central1" - }, - { - "_description": "This is a dummy dlio workload (missing the 'dlioWorkload' field), purely standing as a header and does not execute any workload. For it to execute a dlio workload, it must have a valid 'dlioWorkload' object and a valid 'bucket' attribute.", - "_dlioWorkload": { - "_description": "Every dlioWorkload must have numFilesTrain, recordLength, and batchSizes fields. batchSizes is an array of integer values", - "numFilesTrain": 500000, - "recordLength": 102400, - "batchSizes": [800,128] - }, - "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true", - "bucket":"The bucket must have objects with name 'train/', 'valid/', and train/img_{i}_of_{numFilesTrain}.npz for every i where i:0-{numFilesTrain}-1 and each train/img_{i}_of_{numFilesTrain}.npz must be of size {recordLength} bytes. The buckets gke-* are all in us-central1, are owned by GKE team and are in their GCP project(s)." - }, - { - "dlioWorkload": { - "numFilesTrain": 500000, - "recordLength": 102400, - "batchSizes": [800,128] - }, - "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true", - "bucket":"dlio-unet3d-100kb-500k-us-west1", - "_bucket_alt2":"dlio-unet3d-100kb-500k-us-central1", - "_bucket_alt3":"gke-dlio-unet3d-100kb-500k" - }, - { - "dlioWorkload": { - "numFilesTrain": 1000000, - "recordLength": 512000, - "batchSizes": [800,128] - }, - "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true", - "bucket":"dlio-unet3d-500kb-1m-us-west1", - "_bucket_alt2":"dlio-unet3d-500kb-1m-us-central1", - "_bucket_alt3":"gke-dlio-unet3d-500kb-1m" - }, - { - "dlioWorkload": { - "numFilesTrain": 100000, - "recordLength": 3145728, - "batchSizes": [200] - }, - "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true", - "bucket":"dlio-unet3d-3mb-100k-us-west1", - "_bucket_alt2":"dlio-unet3d-3mb-100k-us-central1", - "_bucket_alt3":"gke-dlio-unet3d-3mb-100k" - }, - { - "dlioWorkload": { - "numFilesTrain": 5000, - "recordLength": 157286400, - "batchSizes": [4] - }, - "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true", - "bucket":"dlio-unet3d-150mb-5k-us-west1", - "_bucket_alt2":"dlio-unet3d-150mb-5k-us-central1", - "_bucket_alt3":"gke-dlio-unet3d-150mb-5k" - } - ] - } - } -} diff --git a/perfmetrics/scripts/testing_on_gke/examples/workloads_test.json b/perfmetrics/scripts/testing_on_gke/examples/workloads_test.json deleted file mode 100644 index 280bd073c2..0000000000 --- a/perfmetrics/scripts/testing_on_gke/examples/workloads_test.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "_comment": "_ in the starting of element name indicates comment.", - "TestConfig": { - "workloadConfig": { - "_description": "workloadConfig has an optional field runOnSSD (default true if missing), and an array of workloads.", - "runOnSSD": false, - "workloads": [ - { - "_description": "This is a dummy fio workload (missing the 'fioWorkload' field), purely standing as a header and does not execute any workload. For it to execute a fio workload, it must have a valid 'fioWorkload', a valid 'bucket' attribute, and a valid gcsfuseMountOption attribute.", - "_fioWorkload": { - "_description": "Every fioWorkload must have fileSize, filesPerThread, numThreads, and blockSize fields. readTypes is an array of string values 'read' and 'randread'. If readTypes is missing, then it defaults to [\"read\",\"randread\"].", - "fileSize": "64K", - "filesPerThread": 20000, - "numThreads": 50, - "blockSize": "64K", - "readTypes": ["read","randread"] - }, - "gcsfuseMountOptions": "GCSFuse mount-options, in a compact stringified format, to be used for the test scenario gcsfuse-generic. The individual config/cli flag values should be separated by comma. Each cli flag should be of the form <flag>[=<value>], while each config-file flag should be of form <config>[:<subconfig>[:<subsubconfig>[...]]]:<value>. For example, a legal value would be: implicit-dirs,file_mode=777,file-cache:enable-parallel-downloads:true,metadata-cache:ttl-secs:-1 .", - "bucket":"The bucket must have objects with name Workload.{i}/{j} for every i,j where i:0-{numThreads}-1, j:0-{filesPerThread}-1, and each of these objects must be of size {fileSize}. The buckets gke-* are all in us-central1, are owned by GKE team and are in their GCP project(s). For best performance, please ensure that the bucket is in the same google-cloud region and GCP project as that of the GKE cluster used for running this test configuration." - }, - { - "fioWorkload": { - "fileSize": "64K", - "filesPerThread": 100, - "numThreads": 20, - "blockSize": "64K", - "readTypes": ["randread"] - }, - "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true,file-cache:enable-parallel-downloads:true", - "bucket":"fio-64k-1m-us-west1", - "_bucket_alt2":"fio-64k-1m-us-central1", - "_bucket_alt3":"gke-fio-64k-1m" - }, - { - "_description": "This is a dummy dlio workload (missing the 'dlioWorkload' field), purely standing as a header and does not execute any workload. For it to execute a dlio workload, it must have a valid 'dlioWorkload' object and a valid 'bucket' attribute, and a valid gcsfuseMountOption attribute.", - "_dlioWorkload": { - "_description": "Every dlioWorkload must have numFilesTrain, recordLength, and batchSizes fields. batchSizes is an array of integer values", - "numFilesTrain": 500000, - "recordLength": 102400, - "batchSizes": [800,128] - }, - "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true", - "bucket":"The bucket must have objects with name 'train/', 'valid/', and train/img_{i}_of_{numFilesTrain}.npz for every i where i:0-{numFilesTrain}-1 and each train/img_{i}_of_{numFilesTrain}.npz must be of size {recordLength} bytes. The buckets gke-* are all in us-central1, are owned by GKE team and are in their GCP project(s). For best performance, please ensure that the bucket is in the same google-cloud region and GCP project as that of the GKE cluster used for running this test configuration." - }, - { - "dlioWorkload": { - "numFilesTrain": 1000, - "recordLength": 3145728, - "batchSizes": [200] - }, - "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true,file-cache:enable-parallel-downloads:true", - "bucket":"dlio-unet3d-3mb-100k-us-west1", - "_bucket_alt2":"dlio-unet3d-3mb-100k-us-central1", - "_bucket_alt3":"gke-dlio-unet3d-3mb-100k" - } - ] - } - } -} diff --git a/perfmetrics/scripts/upgrade_gcloud.sh b/perfmetrics/scripts/upgrade_gcloud.sh deleted file mode 100755 index 9cc72b833b..0000000000 --- a/perfmetrics/scripts/upgrade_gcloud.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -sudo apt-get update -# Upgrade gcloud version. -# Kokoro machine's outdated gcloud version prevents the use of the "gcloud storage" feature. -gcloud version -wget -O gcloud.tar.gz https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz -q -sudo tar xzf gcloud.tar.gz && sudo cp -r google-cloud-sdk /usr/local && sudo rm -r google-cloud-sdk -sudo /usr/local/google-cloud-sdk/install.sh -export PATH=/usr/local/google-cloud-sdk/bin:$PATH -echo 'export PATH=/usr/local/google-cloud-sdk/bin:$PATH' >> ~/.bashrc -gcloud version && rm gcloud.tar.gz -sudo /usr/local/google-cloud-sdk/bin/gcloud components update -sudo /usr/local/google-cloud-sdk/bin/gcloud components install alpha diff --git a/perfmetrics/scripts/upgrade_python3.sh b/perfmetrics/scripts/upgrade_python3.sh new file mode 100755 index 0000000000..c41ed6aa8f --- /dev/null +++ b/perfmetrics/scripts/upgrade_python3.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#!/bin/bash + +set -euo pipefail +set -x + +PYTHON_VERSION=3.11.9 +INSTALL_PREFIX="$HOME/.local/python-$PYTHON_VERSION" + +if command -v apt-get &> /dev/null; then + # For Debian/Ubuntu-based systems + echo "Installing dependencies for building Python for Debian..." + sudo apt-get update -y > /dev/null + sudo apt-get install -y \ + build-essential zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev \ + libssl-dev libreadline-dev libffi-dev curl libsqlite3-dev \ + libbz2-dev liblzma-dev tk-dev uuid-dev wget > /dev/null +elif command -v yum &> /dev/null; then + # For RHEL/CentOS-based systems + echo "Installing dependencies for building Python on RHEL..." + # For RHEL-based systems, use 'yum' to install packages. + # The "Development Tools" group is equivalent to 'build-essential' on Debian. + # The '-devel' packages provide the necessary header files for compilation. + sudo yum -y groupinstall "Development Tools" > /dev/null + sudo yum -y install \ + zlib-devel ncurses-devel nss-devel openssl-devel \ + readline-devel libffi-devel curl sqlite-devel bzip2-devel \ + xz-devel tk-devel libuuid-devel wget > /dev/null +else + exit 1 +fi + + +# Download and build Python locally +cd /tmp +wget -q https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz +tar -xf Python-${PYTHON_VERSION}.tgz +cd Python-${PYTHON_VERSION} + +echo "Configuring Python build for local install..." +./configure --enable-optimizations --prefix="$INSTALL_PREFIX" > /dev/null + +echo "Building Python $PYTHON_VERSION..." +make -j"$(nproc)" > /dev/null + +echo "Installing Python $PYTHON_VERSION locally at $INSTALL_PREFIX..." +make altinstall > /dev/null + +echo "Python $PYTHON_VERSION installed at $INSTALL_PREFIX/bin/python3.11" +"$INSTALL_PREFIX/bin/python3.11" --version diff --git a/perfmetrics/scripts/vm_metrics/vm_metrics.py b/perfmetrics/scripts/vm_metrics/vm_metrics.py index bd42a7277b..6c6fa0f1cd 100644 --- a/perfmetrics/scripts/vm_metrics/vm_metrics.py +++ b/perfmetrics/scripts/vm_metrics/vm_metrics.py @@ -187,8 +187,8 @@ def _get_metric_filter(type, metric_type, instance, extra_filter): metric_type=metric_type, instance_name=instance) elif (type == 'custom'): metric_filter = ( - 'metric.type = "{metric_type}" AND metric.labels.opencensus_task = ' - 'ends_with("{instance_name}")').format( + 'metric.type = "{metric_type}" AND metadata.system_labels.name = ' + '"{instance_name}"').format( metric_type=metric_type, instance_name=instance) elif (type == 'agent'): # Fetch the instance ID here diff --git a/perfmetrics/scripts/vm_metrics/vm_metrics_test.py b/perfmetrics/scripts/vm_metrics/vm_metrics_test.py index 6ebbdc3baf..c1cd4f2c04 100644 --- a/perfmetrics/scripts/vm_metrics/vm_metrics_test.py +++ b/perfmetrics/scripts/vm_metrics/vm_metrics_test.py @@ -288,7 +288,7 @@ def test_get_metric_filter_compute(self): def test_get_metric_filter_custom(self): metric_type = "metric_type" extra_filter = "extra_filter" - expected_metric_filter = 'metric.type = "{}" AND metric.labels.opencensus_task = ends_with("{}") AND {}'.format(metric_type, TEST_INSTANCE, extra_filter) + expected_metric_filter = 'metric.type = "{}" AND metadata.system_labels.name = "{}" AND {}'.format(metric_type, TEST_INSTANCE, extra_filter) self.assertEqual(vm_metrics._get_metric_filter('custom', metric_type, TEST_INSTANCE, extra_filter), expected_metric_filter) @patch('vm_metrics._get_instance_id') diff --git a/samples/gcsfuse_config/README.md b/samples/gcsfuse_config/README.md new file mode 100644 index 0000000000..4e94d5f004 --- /dev/null +++ b/samples/gcsfuse_config/README.md @@ -0,0 +1,73 @@ +# **GCSFuse Sample Configurations** + +This directory contains sample GCSFuse configuration files optimized for common machine learning workloads. These configurations are intended for use with vanilla GCSFuse and may require minor adjustments for your specific environment. + +The configurations are organized into two main directories: + +* **GPU:** Configurations optimized for GPU-based workloads. +* **TPU:** Configurations optimized for TPU-based workloads. + +Within each directory, you will find configurations tailored for the following workflows: + +1. **Model Training:** Optimized for training machine learning models. +2. **Model Serving/Inference:** Optimized for serving predictions with trained models. +3. **Checkpointing:** Designed for saving model state during training, also applicable to JAX Just-In-Time (JIT) Cache workflows. + +Choose the configuration that best matches your hardware platform (GPU or TPU) and workload. + +## **Deployment Instructions** + +To use these sample configurations, you can use the `gcsfuse` command with the `--config-file` flag. + +For example, to mount a bucket for model training on a GPU machine: + +``` +gcsfuse --config-file GPU/training.yaml <your-bucket-name> <mount-point> +``` + +Replace `<your-bucket-name>` with the name of your GCS bucket and `<mount-point>` with the desired mount point on your local filesystem. + +### **Usage with GCE, SLURM, and Ansible** + +These configurations are designed to be versatile and can be used in various environments, including Google Compute Engine (GCE) instances and SLURM clusters. + +#### **GCE and SLURM** + +In a GCE or SLURM environment, you can use the same `gcsfuse` command to mount your GCS buckets on your compute nodes. It's recommended to automate this process as part of your instance startup scripts or SLURM job prolog scripts. This ensures that the required GCS buckets are mounted and available to your applications before they start. + +Here is an [example](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/51c51f2c83383a8f241cd0ef8a8998413393bff5/examples/hypercompute_clusters/a3u-slurm-ubuntu-gcs/a3u-slurm-ubuntu-gcs.yaml#L193) of using config with Ansible. + +## **Key Considerations** + +* **File Cache:** + + * For GPU-based workloads, it is highly recommended to use a Local SSD (LSSD) for the file cache directory for optimal performance. + * For TPU-based workloads, using a RAM disk for the file cache can provide a significant speed advantage. +* **Metadata Cache TTL:** + + * The sample configurations may use a long or infinite Time-to-Live (TTL) for the metadata cache and there might be consistency implications. More details can be found [here](https://cloud.google.com/storage/docs/cloud-storage-fuse/performance#increase-metadata-cache-values). +* **Pre-populating Metadata Cache:** + + * To improve the performance of subsequent file and directory lookups, you can pre-populate the metadata cache after mounting the bucket by running the following command. More details can be found [here](https://cloud.google.com/storage/docs/cloud-storage-fuse/performance#pre-populate-the-metadata-cache): + +``` + ls -R <mount-point> > /dev/null +``` +* **Tuning `read-ahead-kb`** + + * The `read-ahead-kb` flag controls the size of kernel read-ahead buffer which in turns impacts number of requests to GCSFuse. Tuning this value can significantly improve performance for sequential reads of large files. + * A good starting point for `read-ahead-kb` is to set it to a value slightly larger than the average read size of your application. Typical recommendation is to set it to `1024`. +``` + export GCSFUSEMOUNT=/your/container/mountpoint + echo 1024 | sudo tee /sys/class/bdi/0:$(stat -c "%d" $GCSFUSEMOUNT)/read_ahead_kb +``` + + +## **Prerequisites and Notes** + +* **Placeholders:** Remember to replace all placeholder values within the YAML configuration files with your specific environment details before use. +* **Permissions:** Ensure that the service account or user running GCSFuse has the necessary permissions to access the specified GCS bucket. + +## **Further Information** + +For comprehensive details on GCSFuse configuration and performance tuning, please consult the official documentation: [https://cloud.google.com/storage/docs/gcsfuse](https://cloud.google.com/storage/docs/gcsfuse) diff --git a/samples/gcsfuse_config/gpu/config_file/checkpointing.yaml b/samples/gcsfuse_config/gpu/config_file/checkpointing.yaml new file mode 100644 index 0000000000..6f8a693f9f --- /dev/null +++ b/samples/gcsfuse_config/gpu/config_file/checkpointing.yaml @@ -0,0 +1,12 @@ +implicit-dirs: true # Create implicit directories locally when accessed +cache-dir: /tmp # Use LSSD backing on GPU and RAM Disk backing on TPU +metadata-cache: + negative-ttl-secs: 0 # Disable caching for lookups of files/dirs that don't exist + ttl-secs: -1 # Keep cached metadata (file attributes, types) indefinitely time-wise + stat-cache-max-size-mb: -1 # Allow unlimited size for the file attribute (stat) cache +file-cache: + max-size-mb: -1 # Allow unlimited size for the file content cache + cache-file-for-range-read: true # Cache the entire file when any part is read sequentially + enable-parallel-downloads: true # Use multiple streams to download file content faster +write: + enable-streaming-writes: true # Enable streaming writes diff --git a/samples/gcsfuse_config/gpu/config_file/serving.yaml b/samples/gcsfuse_config/gpu/config_file/serving.yaml new file mode 100644 index 0000000000..6a0416c4c8 --- /dev/null +++ b/samples/gcsfuse_config/gpu/config_file/serving.yaml @@ -0,0 +1,10 @@ +implicit-dirs: true # Create implicit directories locally when accessed +cache-dir: /tmp # Use LSSD backing on GPU and RAM Disk backing on TPU +metadata-cache: + negative-ttl-secs: 0 # Disable caching for lookups of files/dirs that don't exist + ttl-secs: -1 # Keep cached metadata (file attributes, types) indefinitely time-wise + stat-cache-max-size-mb: -1 # Allow unlimited size for the file attribute (stat) cache +file-cache: + max-size-mb: -1 # Allow unlimited size for the file content cache + cache-file-for-range-read: true # Cache the entire file when any part is read sequentially + enable-parallel-downloads: true # Use multiple streams to download file content faster diff --git a/samples/gcsfuse_config/gpu/config_file/training.yaml b/samples/gcsfuse_config/gpu/config_file/training.yaml new file mode 100644 index 0000000000..ed26b733b1 --- /dev/null +++ b/samples/gcsfuse_config/gpu/config_file/training.yaml @@ -0,0 +1,10 @@ +implicit-dirs: true # Create implicit directories locally when accessed +metadata-cache: + negative-ttl-secs: 0 # Disable caching for lookups of files/dirs that don't exist + ttl-secs: -1 # Keep cached metadata (file attributes, types) indefinitely time-wise + stat-cache-max-size-mb: -1 # Allow unlimited size for the file attribute (stat) cache +# if enabling the file cache, uncomment out to use # +# cache-dir: /tmp # Use LSSD backing on GPU and RAM Disk backing on TPU +# file-cache: +# max-size-mb: <DATASET_SIZE> # Allow DATASET_SIZE worth of size for the file content cache +# cache-file-for-range-read: true # Cache the entire file when any part is read sequentially diff --git a/samples/gcsfuse_config/tpu/config_file/checkpointing.yaml b/samples/gcsfuse_config/tpu/config_file/checkpointing.yaml new file mode 100644 index 0000000000..6f8a693f9f --- /dev/null +++ b/samples/gcsfuse_config/tpu/config_file/checkpointing.yaml @@ -0,0 +1,12 @@ +implicit-dirs: true # Create implicit directories locally when accessed +cache-dir: /tmp # Use LSSD backing on GPU and RAM Disk backing on TPU +metadata-cache: + negative-ttl-secs: 0 # Disable caching for lookups of files/dirs that don't exist + ttl-secs: -1 # Keep cached metadata (file attributes, types) indefinitely time-wise + stat-cache-max-size-mb: -1 # Allow unlimited size for the file attribute (stat) cache +file-cache: + max-size-mb: -1 # Allow unlimited size for the file content cache + cache-file-for-range-read: true # Cache the entire file when any part is read sequentially + enable-parallel-downloads: true # Use multiple streams to download file content faster +write: + enable-streaming-writes: true # Enable streaming writes diff --git a/samples/gcsfuse_config/tpu/config_file/serving.yaml b/samples/gcsfuse_config/tpu/config_file/serving.yaml new file mode 100644 index 0000000000..6a0416c4c8 --- /dev/null +++ b/samples/gcsfuse_config/tpu/config_file/serving.yaml @@ -0,0 +1,10 @@ +implicit-dirs: true # Create implicit directories locally when accessed +cache-dir: /tmp # Use LSSD backing on GPU and RAM Disk backing on TPU +metadata-cache: + negative-ttl-secs: 0 # Disable caching for lookups of files/dirs that don't exist + ttl-secs: -1 # Keep cached metadata (file attributes, types) indefinitely time-wise + stat-cache-max-size-mb: -1 # Allow unlimited size for the file attribute (stat) cache +file-cache: + max-size-mb: -1 # Allow unlimited size for the file content cache + cache-file-for-range-read: true # Cache the entire file when any part is read sequentially + enable-parallel-downloads: true # Use multiple streams to download file content faster diff --git a/samples/gcsfuse_config/tpu/config_file/training.yaml b/samples/gcsfuse_config/tpu/config_file/training.yaml new file mode 100644 index 0000000000..c1a15735fa --- /dev/null +++ b/samples/gcsfuse_config/tpu/config_file/training.yaml @@ -0,0 +1,10 @@ +implicit-dirs: true # Create implicit directories locally when accessed +metadata-cache: + negative-ttl-secs: 0 # Disable caching for lookups of files/dirs that don't exist + ttl-secs: -1 # Keep cached metadata (file attributes, types) indefinitely time-wise + stat-cache-max-size-mb: -1 # Allow unlimited size for the file attribute (stat) cache +# if enabling the file cache, uncomment out to use # +# cache-dir: /tmp # Use LSSD backing on GPU and RAM Disk backing on TPU +#file-cache: +# max-size-mb: <DATASET_SIZE> # Allow DATASET_SIZE worth of size for the file content cache +# cache-file-for-range-read: true # Cache the entire file when any part is read sequentially diff --git a/samples/gke-csi-yaml/gpu/README.md b/samples/gke-csi-yaml/gpu/README.md new file mode 100644 index 0000000000..e0ef3a59bc --- /dev/null +++ b/samples/gke-csi-yaml/gpu/README.md @@ -0,0 +1,37 @@ +# GKE GCSFuse CSI Driver Sample Configurations for GPU Workloads + +This directory provides sample Kubernetes YAML configuration files for utilizing the GKE GCSFuse CSI driver, specifically optimized for workloads running on Graphics Processing Units (GPUs). + +The configurations include recommendations tailored for the following common machine learning workflows: + +1. **Model Training:** Configurations suitable for training machine learning models. +2. **Model Serving/Inference:** Configurations optimized for deploying trained models to serve predictions. +3. **Checkpointing:** Configurations designed for saving model state during training processes. These configurations are also applicable to JAX Just-In-Time (JIT) Cache workflows. + +## Deployment Instructions + +**Note: The sample files are for GCSfuse GKE CSI driver running on GKE clusters of GKE version 1.32.2-gke.1297001 or greater.** + +To utilize these sample configurations, follow the specified deployment order. For instance, to set up the serving workload: + +1. **Deploy the PersistentVolume (PV) and PersistentVolumeClaim (PVC):** + Apply the `*-pv.yaml` file first. This step is crucial as the GKE pod admission webhook inspects the PV's volume attributes to apply potential optimizations, such as the injection of sidecar containers, before the pod is scheduled. + ```bash + kubectl apply -f serving-pv.yaml + ``` + +2. **Deploy the Pod:** + After the PV and PVC are successfully created, deploy the pod specification that references the PVC. + ```bash + kubectl apply -f serving-pod.yaml + ``` + +## Prerequisites and Notes + +* **Service Account:** Ensure the specified Kubernetes Service Account (e.g., `<YOUR_K8S_SA>` in the pod YAML) exists and possesses the necessary permissions to access the target Google Cloud Storage bucket *before* deploying the pod. +* **Placeholders:** Replace all placeholder values (e.g., `<customer-namespace>`, `<checkpoint-bucket>`, `<YOUR_K8S_SA>`) within the YAML files with your specific environment details before application. + +## Further Information + +For comprehensive details on performance tuning and best practices for the GCS FUSE CSI driver, please consult the Google Cloud documentation: +[Best practices for performance tuning](https://cloud.google.com/kubernetes-engine/docs/how-to/cloud-storage-fuse-csi-driver-perf#best-practices-for-performance-tuning) diff --git a/samples/gke-csi-yaml/gpu/checkpointing-pod.yaml b/samples/gke-csi-yaml/gpu/checkpointing-pod.yaml new file mode 100644 index 0000000000..21fb1652ba --- /dev/null +++ b/samples/gke-csi-yaml/gpu/checkpointing-pod.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gcs-fuse-csi-example-pod + namespace: <customer-namespace> + annotations: + gke-gcsfuse/volumes: "true" + # gke-gcsfuse/metadata-prefetch-memory-limit: "0" # min GKE version: `1.32.3-gke.1717000` for this annotation to take effect + # gke-gcsfuse/metadata-prefetch-cpu-limit: "0" # min GKE version: `1.32.3-gke.1717000` for this annotation to take effect +spec: + containers: + # Add your workload container spec + ... + volumeMounts: + - name: checkpoint-bucket-vol + mountPath: /checkpoint-data + serviceAccountName: <YOUR_K8S_SA> + volumes: + # In-memory (RAM Disk) for GCSFuse file cache if available. Uncomment to use. + # - name: gke-gcsfuse-cache # GCSFuse file cache backed in Memory (RAM Disk) + # emptyDir: + # medium: Memory + - name: checkpoint-bucket-vol + persistentVolumeClaim: + claimName: checkpoint-bucket-pvc diff --git a/samples/gke-csi-yaml/gpu/checkpointing-pv.yaml b/samples/gke-csi-yaml/gpu/checkpointing-pv.yaml new file mode 100644 index 0000000000..0620932c33 --- /dev/null +++ b/samples/gke-csi-yaml/gpu/checkpointing-pv.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: checkpoint-bucket-pv +spec: + accessModes: + - ReadWriteMany + capacity: + storage: 64Gi + persistentVolumeReclaimPolicy: Retain + storageClassName: gcsfuse-sc # dummy storage class + claimRef: + namespace: <customer-namespace> + name: checkpoint-bucket-pvc + mountOptions: + - implicit-dirs # Create implicit directories locally when accessed + - metadata-cache:negative-ttl-secs:0 # Disable caching for lookups of files/dirs that don't exist + - metadata-cache:ttl-secs:-1 # Keep cached metadata (file attributes, types) indefinitely time-wise + - metadata-cache:stat-cache-max-size-mb:-1 # Allow unlimited size for the file attribute (stat) cache + - file-cache:max-size-mb:-1 # Allow unlimited size for the file content cache + - file-cache:cache-file-for-range-read:true # Cache the entire file when any part is read sequentially + - file-cache:enable-parallel-downloads:true # Use multiple streams to download file content faster + - read_ahead_kb=1024 # Increase kernel read-ahead buffer + - write:enable-streaming-writes:true # Enable streaming writes + csi: + driver: gcsfuse.csi.storage.gke.io + volumeHandle: <checkpoint-bucket> # Name of the GCS bucket to mount + volumeAttributes: + skipCSIBucketAccessCheck: "true" # Bypass the CSI Drivers bucket access check + gcsfuseMetadataPrefetchOnMount: "true" # Fetch GCS metadata immediately at mount time + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: checkpoint-bucket-pvc + namespace: <customer-namespace> +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 64Gi + volumeName: checkpoint-bucket-pv + storageClassName: gcsfuse-sc # dummy storage class diff --git a/samples/gke-csi-yaml/gpu/serving-pod.yaml b/samples/gke-csi-yaml/gpu/serving-pod.yaml new file mode 100644 index 0000000000..512d9943c4 --- /dev/null +++ b/samples/gke-csi-yaml/gpu/serving-pod.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gcs-fuse-csi-example-pod + namespace: <customer-namespace> + annotations: + gke-gcsfuse/volumes: "true" + # gke-gcsfuse/metadata-prefetch-memory-limit: "0" # min GKE version: `1.32.3-gke.1717000` for this annotation to take effect + # gke-gcsfuse/metadata-prefetch-cpu-limit: "0" # min GKE version: `1.32.3-gke.1717000` for this annotation to take effect +spec: + containers: + # Your workload container spec + ... + volumeMounts: + - name: serving-bucket-vol + mountPath: /serving-data + serviceAccountName: <YOUR_K8S_SA> + volumes: + # RAM disk file cache for best performance of Parallel Download. Can use L-SSD if not available memory + - name: gke-gcsfuse-cache # gcsfuse file cache backed by RAM Disk (Memory) + emptyDir: + medium: Memory + - name: serving-bucket-vol + persistentVolumeClaim: + claimName: serving-bucket-pvc diff --git a/samples/gke-csi-yaml/gpu/serving-pv.yaml b/samples/gke-csi-yaml/gpu/serving-pv.yaml new file mode 100644 index 0000000000..7c8f93be93 --- /dev/null +++ b/samples/gke-csi-yaml/gpu/serving-pv.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: serving-bucket-pv +spec: + accessModes: + - ReadWriteMany + capacity: + storage: 64Gi + persistentVolumeReclaimPolicy: Retain + storageClassName: gcsfuse-sc # dummy storage class + claimRef: + namespace: <customer-namespace> + name: serving-bucket-pvc + mountOptions: + - implicit-dirs # Create implicit directories locally when accessed + - metadata-cache:negative-ttl-secs:0 # Disable caching for lookups of files/dirs that don't exist + - metadata-cache:ttl-secs:-1 # Keep cached metadata (file attributes, types) indefinitely time-wise + - metadata-cache:stat-cache-max-size-mb:-1 # Allow unlimited size for the file attribute (stat) cache + - file-cache:max-size-mb:-1 # Allow unlimited size for the file content cache + - file-cache:cache-file-for-range-read:true # Cache the entire file when any part is read sequentially + - file-cache:enable-parallel-downloads:true # Use multiple streams to download file content faster + - read_ahead_kb=1024 # Increase kernel read-ahead buffer + csi: + driver: gcsfuse.csi.storage.gke.io + volumeHandle: <serving-bucket> # Name of the GCS Bucket to mount + volumeAttributes: + skipCSIBucketAccessCheck: "true" # Bypass the CSI Drivers bucket access check + gcsfuseMetadataPrefetchOnMount: "true" # Fetch GCS metadata immediately at mount time +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: serving-bucket-pvc + namespace: <customer-namespace> +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 64Gi + volumeName: serving-bucket-pv + storageClassName: gcsfuse-sc # dummy storage class diff --git a/samples/gke-csi-yaml/gpu/training-pod.yaml b/samples/gke-csi-yaml/gpu/training-pod.yaml new file mode 100644 index 0000000000..53667a71c6 --- /dev/null +++ b/samples/gke-csi-yaml/gpu/training-pod.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gcs-fuse-csi-example-pod + namespace: <customer-namespace> + annotations: + gke-gcsfuse/volumes: "true" + # gke-gcsfuse/metadata-prefetch-memory-limit: "0" # min GKE version: `1.32.3-gke.1717000` for this annotation to take effect + # gke-gcsfuse/metadata-prefetch-cpu-limit: "0" # min GKE version: `1.32.3-gke.1717000` for this annotation to take effect +spec: + containers: + # Your workload container spec + ... + volumeMounts: + - name: training-bucket-vol + mountPath: /training-data + serviceAccountName: <YOUR_K8S_SA> + volumes: + # RAM disk file cache if L-SSD not available. Uncomment to use + # - name: gke-gcsfuse-cache # gcsfuse file cache backed by RAM Disk + # emptyDir: + # medium: Memory + - name: training-bucket-vol + persistentVolumeClaim: + claimName: training-bucket-pvc diff --git a/samples/gke-csi-yaml/gpu/training-pv.yaml b/samples/gke-csi-yaml/gpu/training-pv.yaml new file mode 100644 index 0000000000..0a3f3d589a --- /dev/null +++ b/samples/gke-csi-yaml/gpu/training-pv.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: training-bucket-pv +spec: + accessModes: + - ReadWriteMany + capacity: + storage: 64Gi + persistentVolumeReclaimPolicy: Retain + storageClassName: gcsfuse-sc # dummy storage class + claimRef: + namespace: <customer-namespace> + name: training-bucket-pvc + mountOptions: + - implicit-dirs # Create implicit directories locally when accessed + - metadata-cache:negative-ttl-secs:0 # Disable caching for lookups of files/dirs that don't exist + - metadata-cache:ttl-secs:-1 # Keep cached metadata (file attributes, types) indefinitely time-wise + - metadata-cache:stat-cache-max-size-mb:-1 # Allow unlimited size for the file attribute (stat) cache + # if enabling the file cache, uncomment out to use # + # - file-cache:max-size-mb:-1 # Allow unlimited size for the file content cache + # - file-cache:cache-file-for-range-read:true # Cache the entire file when any part is read sequentially + # - read_ahead_kb=1024 # Increase kernel read-ahead buffer + csi: + driver: gcsfuse.csi.storage.gke.io + volumeHandle: <training-bucket> # Name of the GCS Bucket to mount + volumeAttributes: + skipCSIBucketAccessCheck: "true" # Bypass the CSI Drivers bucket access check + gcsfuseMetadataPrefetchOnMount: "true" # Fetch GCS metadata immediately at mount time +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: training-bucket-pvc + namespace: <customer-namespace> +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 64Gi + volumeName: training-bucket-pv + storageClassName: gcsfuse-sc # dummy storage class diff --git a/samples/gke-csi-yaml/tpu/README.md b/samples/gke-csi-yaml/tpu/README.md new file mode 100644 index 0000000000..be53cfa7bf --- /dev/null +++ b/samples/gke-csi-yaml/tpu/README.md @@ -0,0 +1,37 @@ +# GKE GCSFuse CSI Driver Sample Configurations for TPU Workloads + +This directory provides sample Kubernetes YAML configuration files for utilizing the GKE GCSFuse CSI driver, specifically optimized for workloads running on Tensor Processing Units (TPUs). + +The configurations include recommendations tailored for the following common machine learning workflows: + +1. **Model Training:** Configurations suitable for training machine learning models. +2. **Model Serving/Inference:** Configurations optimized for deploying trained models to serve predictions. +3. **Checkpointing:** Configurations designed for saving model state during training processes. These configurations are also applicable to JAX Just-In-Time (JIT) Cache workflows. + +## Deployment Instructions + +**Note: The sample files are for GCSfuse GKE CSI driver running on GKE clusters of GKE version 1.32.2-gke.1297001 or greater.** + +To utilize these sample configurations, follow the specified deployment order. For instance, to set up the serving workload: + +1. **Deploy the PersistentVolume (PV) and PersistentVolumeClaim (PVC):** + Apply the `*-pv.yaml` file first. This step is crucial as the GKE pod admission webhook inspects the PV's volume attributes to apply potential optimizations, such as the injection of sidecar containers, before the pod is scheduled. + ```bash + kubectl apply -f serving-pv.yaml + ``` + +2. **Deploy the Pod:** + After the PV and PVC are successfully created, deploy the pod specification that references the PVC. + ```bash + kubectl apply -f serving-pod.yaml + ``` + +## Prerequisites and Notes + +* **Service Account:** Ensure the specified Kubernetes Service Account (e.g., `<YOUR_K8S_SA>` in the pod YAML) exists and possesses the necessary permissions to access the target Google Cloud Storage bucket *before* deploying the pod. +* **Placeholders:** Replace all placeholder values (e.g., `<customer-namespace>`, `<checkpoint-bucket>`, `<YOUR_K8S_SA>`) within the YAML files with your specific environment details before application. + +## Further Information + +For comprehensive details on performance tuning and best practices for the GCS FUSE CSI driver, please consult the Google Cloud documentation: +[Best practices for performance tuning](https://cloud.google.com/kubernetes-engine/docs/how-to/cloud-storage-fuse-csi-driver-perf#best-practices-for-performance-tuning) \ No newline at end of file diff --git a/samples/gke-csi-yaml/tpu/checkpointing-pod.yaml b/samples/gke-csi-yaml/tpu/checkpointing-pod.yaml new file mode 100644 index 0000000000..ec7b442d48 --- /dev/null +++ b/samples/gke-csi-yaml/tpu/checkpointing-pod.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gcs-fuse-csi-example-pod + namespace: <customer-namespace> + annotations: + gke-gcsfuse/volumes: "true" + # gke-gcsfuse/metadata-prefetch-memory-limit: "0" # min GKE version: `1.32.3-gke.1717000` for this annotation to take effect + # gke-gcsfuse/metadata-prefetch-cpu-limit: "0" # min GKE version: `1.32.3-gke.1717000` for this annotation to take effect + +spec: + containers: + # Add your workload container spec + ... + volumeMounts: + - name: checkpoint-bucket-vol + mountPath: /checkpoint-data + serviceAccountName: <YOUR_K8S_SA> + volumes: + - name: gke-gcsfuse-cache # GCSFuse file cache backed in Memory (RAM Disk) + emptyDir: + medium: Memory + - name: checkpoint-bucket-vol + persistentVolumeClaim: + claimName: checkpoint-bucket-pvc diff --git a/samples/gke-csi-yaml/tpu/checkpointing-pv.yaml b/samples/gke-csi-yaml/tpu/checkpointing-pv.yaml new file mode 100644 index 0000000000..27ed40d47f --- /dev/null +++ b/samples/gke-csi-yaml/tpu/checkpointing-pv.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: checkpoint-bucket-pv +spec: + accessModes: + - ReadWriteMany + capacity: + storage: 64Gi + persistentVolumeReclaimPolicy: Retain + storageClassName: gcsfuse-sc # dummy storage class + claimRef: + namespace: <customer-namespace> + name: checkpoint-bucket-pvc + mountOptions: + - implicit-dirs # Create implicit directories locally when accessed. + - metadata-cache:negative-ttl-secs:0 # Disable caching for lookups of files/dirs that don't exist. + - metadata-cache:ttl-secs:-1 # Keep cached metadata (file attributes, types) indefinitely time-wise. + - metadata-cache:stat-cache-max-size-mb:-1 # Allow unlimited size for the file attribute (stat) cache. + - file-cache:max-size-mb:-1 # Allow unlimited size for the file content cache. + - file-cache:cache-file-for-range-read:true # Cache the entire file when any part is read sequentially. + - file-cache:enable-parallel-downloads:true # Use multiple streams to download file content faster. + - read_ahead_kb=1024 # Increase kernel read-ahead buffer. + - write:enable-streaming-writes:true # Enable streaming writes. + csi: + driver: gcsfuse.csi.storage.gke.io + volumeHandle: <checkpoint-bucket> # Name of the GCS bucket to mount. + volumeAttributes: + skipCSIBucketAccessCheck: "true" # Bypass the CSI Drivers bucket existence/access check. + gcsfuseMetadataPrefetchOnMount: "true" # Fetch GCS metadata immediately at mount time. + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: checkpoint-bucket-pvc + namespace: <customer-namespace> +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 64Gi + volumeName: checkpoint-bucket-pv + storageClassName: gcsfuse-sc # dummy storage class diff --git a/samples/gke-csi-yaml/tpu/serving-pod.yaml b/samples/gke-csi-yaml/tpu/serving-pod.yaml new file mode 100644 index 0000000000..b7e4b311e5 --- /dev/null +++ b/samples/gke-csi-yaml/tpu/serving-pod.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gcs-fuse-csi-example-pod + namespace: <customer-namespace> + annotations: + gke-gcsfuse/volumes: "true" + # gke-gcsfuse/metadata-prefetch-memory-limit: "0" # min GKE version: `1.32.3-gke.1717000` for this annotation to take effect + # gke-gcsfuse/metadata-prefetch-cpu-limit: "0" # min GKE version: `1.32.3-gke.1717000` for this annotation to take effect + +spec: + containers: + # Your workload container spec + ... + volumeMounts: + - name: serving-bucket-vol + mountPath: /serving-data + serviceAccountName: <YOUR_K8S_SA> + volumes: + # RAM disk file cache for best performance of Parallel Download. Can use L-SSD if not available memory + - name: gke-gcsfuse-cache # gcsfuse file cache backed by RAM Disk (Memory) + emptyDir: + medium: Memory + - name: serving-bucket-vol + persistentVolumeClaim: + claimName: serving-bucket-pvc diff --git a/samples/gke-csi-yaml/tpu/serving-pv.yaml b/samples/gke-csi-yaml/tpu/serving-pv.yaml new file mode 100644 index 0000000000..7c8f93be93 --- /dev/null +++ b/samples/gke-csi-yaml/tpu/serving-pv.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: serving-bucket-pv +spec: + accessModes: + - ReadWriteMany + capacity: + storage: 64Gi + persistentVolumeReclaimPolicy: Retain + storageClassName: gcsfuse-sc # dummy storage class + claimRef: + namespace: <customer-namespace> + name: serving-bucket-pvc + mountOptions: + - implicit-dirs # Create implicit directories locally when accessed + - metadata-cache:negative-ttl-secs:0 # Disable caching for lookups of files/dirs that don't exist + - metadata-cache:ttl-secs:-1 # Keep cached metadata (file attributes, types) indefinitely time-wise + - metadata-cache:stat-cache-max-size-mb:-1 # Allow unlimited size for the file attribute (stat) cache + - file-cache:max-size-mb:-1 # Allow unlimited size for the file content cache + - file-cache:cache-file-for-range-read:true # Cache the entire file when any part is read sequentially + - file-cache:enable-parallel-downloads:true # Use multiple streams to download file content faster + - read_ahead_kb=1024 # Increase kernel read-ahead buffer + csi: + driver: gcsfuse.csi.storage.gke.io + volumeHandle: <serving-bucket> # Name of the GCS Bucket to mount + volumeAttributes: + skipCSIBucketAccessCheck: "true" # Bypass the CSI Drivers bucket access check + gcsfuseMetadataPrefetchOnMount: "true" # Fetch GCS metadata immediately at mount time +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: serving-bucket-pvc + namespace: <customer-namespace> +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 64Gi + volumeName: serving-bucket-pv + storageClassName: gcsfuse-sc # dummy storage class diff --git a/samples/gke-csi-yaml/tpu/training-pod.yaml b/samples/gke-csi-yaml/tpu/training-pod.yaml new file mode 100644 index 0000000000..6ef6768485 --- /dev/null +++ b/samples/gke-csi-yaml/tpu/training-pod.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gcs-fuse-csi-example-pod + namespace: <customer-namespace> + annotations: + gke-gcsfuse/volumes: "true" + # gke-gcsfuse/metadata-prefetch-memory-limit: "0" # min GKE version: `1.32.3-gke.1717000` for this annotation to take effect + # gke-gcsfuse/metadata-prefetch-cpu-limit: "0" # min GKE version: `1.32.3-gke.1717000` for this annotation to take effect +spec: + containers: + # Your workload container spec + ... + volumeMounts: + - name: training-bucket-vol + mountPath: /training-data + serviceAccountName: <YOUR_K8S_SA> + volumes: + # In-Memory (RAM disk) file cache if dataset fits. Uncomment to use + # - name: gke-gcsfuse-cache # gcsfuse file cache backed by RAM Disk + # emptyDir: + # medium: Memory + - name: training-bucket-vol + persistentVolumeClaim: + claimName: training-bucket-pvc diff --git a/samples/gke-csi-yaml/tpu/training-pv.yaml b/samples/gke-csi-yaml/tpu/training-pv.yaml new file mode 100644 index 0000000000..bb87b95ffb --- /dev/null +++ b/samples/gke-csi-yaml/tpu/training-pv.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: training-bucket-pv +spec: + accessModes: + - ReadWriteMany + capacity: + storage: 64Gi + persistentVolumeReclaimPolicy: Retain + storageClassName: gcsfuse-sc # dummy storage class + claimRef: + namespace: <customer-namespace> + name: training-bucket-pvc + mountOptions: + - implicit-dirs # Create implicit directories locally when accessed + - metadata-cache:negative-ttl-secs:0 # Disable caching for lookups of files/dirs that don't exist + - metadata-cache:ttl-secs:-1 # Keep cached metadata (file attributes, types) indefinitely time-wise + - metadata-cache:stat-cache-max-size-mb:-1 # Allow unlimited size for the file attribute (stat) cache + # if enabling the file cache, uncomment out to use # + # - file-cache:max-size-mb:<DATASET_SIZE> # Allow DATASET_SIZE for the file content cache + # - file-cache:cache-file-for-range-read:true # Cache the entire file when any part is read sequentially + # - read_ahead_kb=1024 # Increase kernel read-ahead buffer + csi: + driver: gcsfuse.csi.storage.gke.io + volumeHandle: <training-bucket> # Name of the GCS Bucket to mount + volumeAttributes: + skipCSIBucketAccessCheck: "true" # Bypass the CSI Drivers bucket access check + gcsfuseMetadataPrefetchOnMount: "true" # Fetch GCS metadata immediately at mount time +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: training-bucket-pvc + namespace: <customer-namespace> +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 64Gi + volumeName: training-bucket-pv + storageClassName: gcsfuse-sc # dummy storage class diff --git a/tools/build_gcsfuse/main.go b/tools/build_gcsfuse/main.go index a0e83440a1..aa989a74ea 100644 --- a/tools/build_gcsfuse/main.go +++ b/tools/build_gcsfuse/main.go @@ -35,7 +35,6 @@ package main import ( - "flag" "fmt" "log" "os" @@ -43,6 +42,8 @@ import ( "path" "runtime" "strings" + + "github.com/spf13/pflag" ) // Build release binaries according to the supplied settings, setting up the @@ -50,8 +51,9 @@ import ( // // version is the gcsfuse version being built (e.g. "0.11.1"), or a short git // commit name if this is not for an official release. -func buildBinaries(dstDir, srcDir, version string, buildArgs []string) (err error) { +func buildBinaries(dstDir, srcDir, version, arch string, buildArgs []string) (err error) { osys := runtime.GOOS + // Create the target structure. { dirs := []string{ @@ -118,11 +120,11 @@ func buildBinaries(dstDir, srcDir, version string, buildArgs []string) (err erro outputPath string }{ { - "github.com/googlecloudplatform/gcsfuse/v2", + "github.com/googlecloudplatform/gcsfuse/v3", "bin/gcsfuse", }, { - "github.com/googlecloudplatform/gcsfuse/v2/tools/mount_gcsfuse", + "github.com/googlecloudplatform/gcsfuse/v3/tools/mount_gcsfuse", path.Join("sbin", mountHelperName), }, } @@ -143,7 +145,7 @@ func buildBinaries(dstDir, srcDir, version string, buildArgs []string) (err erro cmd.Args = append( cmd.Args, "-ldflags", - fmt.Sprintf("-X github.com/googlecloudplatform/gcsfuse/v2/common.gcsfuseVersion=%s", version), + fmt.Sprintf("-X github.com/googlecloudplatform/gcsfuse/v3/common.gcsfuseVersion=%s", version), ) cmd.Args = append(cmd.Args, buildArgs...) } @@ -160,6 +162,7 @@ func buildBinaries(dstDir, srcDir, version string, buildArgs []string) (err erro fmt.Sprintf("GOPATH=%s", gopath), fmt.Sprintf("GOCACHE=%s", gocache), "CGO_ENABLED=0", + fmt.Sprintf("GOARCH=%s", arch), ) // Build. @@ -190,10 +193,13 @@ func buildBinaries(dstDir, srcDir, version string, buildArgs []string) (err erro } func run() (err error) { + var arch = pflag.String("arch", runtime.GOARCH, "Target architecture (e.g., amd64, arm64). Defaults to host architecture.") + pflag.Parse() + // Extract arguments. - args := flag.Args() + args := pflag.Args() if len(args) < 3 { - err = fmt.Errorf("usage: %s src_dir dst_dir version [build args]", os.Args[0]) + err = fmt.Errorf("usage: %s [flags] src_dir dst_dir version [build args]", os.Args[0]) return } @@ -203,7 +209,7 @@ func run() (err error) { buildArgs := args[3:] // Build. - err = buildBinaries(dstDir, srcDir, version, buildArgs) + err = buildBinaries(dstDir, srcDir, version, *arch, buildArgs) if err != nil { err = fmt.Errorf("buildBinaries: %w", err) return @@ -214,7 +220,7 @@ func run() (err error) { func main() { log.SetFlags(log.Lmicroseconds) - flag.Parse() + // pflag.Parse() is called in run() err := run() if err != nil { diff --git a/tools/build_gcsfuse/main_test.go b/tools/build_gcsfuse/main_test.go new file mode 100644 index 0000000000..adf5548c06 --- /dev/null +++ b/tools/build_gcsfuse/main_test.go @@ -0,0 +1,66 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "os/exec" + "path" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestVersion(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), "gcsfuse-test") + if err != nil { + t.Fatalf("Error while creating temporary directory: %v", err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + err = buildBinaries(dir, "../../", "99.88.77", runtime.GOARCH, nil) + if err != nil { + t.Fatalf("Error while building binary: %v", err) + } + testCases := []struct { + name string + args string + expected string + }{ + { + name: "Version Flag", + args: "--version", + expected: "gcsfuse version 99.88.77", + }, + { + name: "Version Shorthand with Viper config", + args: "-v", + expected: "gcsfuse version 99.88.77", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cmd := exec.Command(path.Join(dir, "bin/gcsfuse"), tc.args) + + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("Error running gcsfuse with args %v: %v", tc.args, err) + } + + assert.Contains(t, string(output), tc.expected) + }) + } +} diff --git a/tools/cd_scripts/e2e_test.sh b/tools/cd_scripts/e2e_test.sh index ed492db13e..d335e0f8fd 100755 --- a/tools/cd_scripts/e2e_test.sh +++ b/tools/cd_scripts/e2e_test.sh @@ -18,36 +18,227 @@ set -x # Exit immediately if a command exits with a non-zero status. set -e -#details.txt file contains the release version and commit hash of the current release. -gsutil cp gs://gcsfuse-release-packages/version-detail/details.txt . -# Writing VM instance name to details.txt (Format: release-test-<os-name>) -curl http://metadata.google.internal/computeMetadata/v1/instance/name -H "Metadata-Flavor: Google" >> details.txt +# Install wget +if command -v apt-get &> /dev/null; then + # For Debian/Ubuntu-based systems + sudo apt-get update && sudo apt-get install -y wget +elif command -v yum &> /dev/null; then + # For RHEL/CentOS-based systems + sudo yum install -y wget +else + exit 1 +fi -# Based on the os type(from vm instance name) in detail.txt, run the following commands to add starterscriptuser -if grep -q ubuntu details.txt || grep -q debian details.txt; -then -# For ubuntu and debian os - sudo adduser --ingroup google-sudoers --disabled-password --home=/home/starterscriptuser --gecos "" starterscriptuser +# Upgrade gcloud +echo "Upgrade gcloud version" +wget -O gcloud.tar.gz https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz -q +sudo tar xzf gcloud.tar.gz && sudo cp -r google-cloud-sdk /usr/local && sudo rm -r google-cloud-sdk + +# Conditionally install python3 and run gcloud installer with it for all variants of RHEL, Rocky and CENTOS. +INSTALL_COMMAND="sudo /usr/local/google-cloud-sdk/install.sh --quiet" +if [ -f /etc/os-release ]; then + . /etc/os-release + if [[ ($ID == "rhel" || $ID == "rocky" || $ID == "centos") ]]; then + + # Check if we are on version 8 or 9 to install 3.11 + if [[ $VERSION_ID =~ ^[89] ]]; then + echo "Detected version 8/9. Installing Python 3.11..." + sudo yum install -y python311 + PYTHON_BIN="/usr/bin/python3.11" + else + # Default behavior for other versions + sudo yum install -y python3 + PYTHON_BIN="/usr/bin/python3" + fi + + export CLOUDSDK_PYTHON=$PYTHON_BIN + INSTALL_COMMAND="sudo env CLOUDSDK_PYTHON=$PYTHON_BIN /usr/local/google-cloud-sdk/install.sh --quiet" + fi +fi +$INSTALL_COMMAND + +export PATH=/usr/local/google-cloud-sdk/bin:$PATH +gcloud version && rm gcloud.tar.gz + +# Extract the metadata parameters passed, for which we need the zone of the GCE VM +# on which the tests are supposed to run. +ZONE=$(curl -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/zone) +echo "Got ZONE=\"${ZONE}\" from metadata server." +# The format for the above extracted zone is projects/{project-id}/zones/{zone}, thus, from this +# need extracted zone name. +ZONE_NAME=$(basename "$ZONE") + +CUSTOM_BUCKET=$(gcloud compute instances describe "$HOSTNAME" --zone="$ZONE_NAME" --format='get(metadata.custom_bucket)') +RUN_ON_ZB_ONLY=$(gcloud compute instances describe "$HOSTNAME" --zone="$ZONE_NAME" --format='get(metadata.run-on-zb-only)') +RUN_READ_CACHE_TESTS_ONLY=$(gcloud compute instances describe "$HOSTNAME" --zone="$ZONE_NAME" --format='get(metadata.run-read-cache-only)') +RUN_LIGHT_TEST=$(gcloud compute instances describe "$HOSTNAME" --zone="$ZONE_NAME" --format='get(metadata.run-light-test)') + +# If CUSTOM_BUCKET is empty, use the release-packages bucket as default. When a custom bucket is provided, this script +# will use the provided bucket to fetch details.txt file for the runa nd will upload the results to that bucket, +# specially useful for testing. +BUCKET_NAME_TO_USE=${CUSTOM_BUCKET:-"gcsfuse-release-packages"} +echo "BUCKET_NAME_TO_USE set to: \"${BUCKET_NAME_TO_USE}\"" +echo "RUN_ON_ZB_ONLY flag set to : \"${RUN_ON_ZB_ONLY}\"" +echo "RUN_READ_CACHE_TESTS_ONLY flag set to : \"${RUN_READ_CACHE_TESTS_ONLY}\"" +echo "RUN_LIGHT_TEST flag set to : \"${RUN_LIGHT_TEST}\"" + +# Logging the tests being run on the active GCE VM +if [[ "$RUN_ON_ZB_ONLY" == "true" ]]; then + echo "Running integration tests for Zonal bucket only..." else -# For rhel and centos - sudo adduser -g google-sudoers --home-dir=/home/starterscriptuser starterscriptuser + echo "Running integration tests for non-zonal buckets only..." +fi + +# Logging the tests being run on the active GCE VM +if [[ "$RUN_READ_CACHE_TESTS_ONLY" == "true" ]]; then + echo "Running read cache test only..." fi +# Logging the tests being run on the active GCE VM +if [[ "$RUN_LIGHT_TEST" == "true" ]]; then + echo "Running light tests only..." +fi + +#details.txt file contains the release version and commit hash of the current release. +# Using dynamic bucket. +gcloud storage cp gs://${BUCKET_NAME_TO_USE}/version-detail/details.txt . +# Writing VM instance name to details.txt (Format: release-test-<os-name>) +curl http://metadata.google.internal/computeMetadata/v1/instance/name -H "Metadata-Flavor: Google" >>details.txt + +# Function to create the local user +create_user() { + local USERNAME=$1 + local HOMEDIR=$2 + local DETAILS=$3 + if id "${USERNAME}" &>/dev/null; then + echo "User ${USERNAME} already exists." + return 0 + fi + + echo "Creating user ${USERNAME}..." + if grep -qi -E 'ubuntu|debian' $DETAILS; then + # For Ubuntu and Debian + sudo adduser --disabled-password --home "${HOMEDIR}" --gecos "" "${USERNAME}" + elif grep -qi -E 'rhel|centos|rocky' $DETAILS; then + # For RHEL, CentOS, Rocky Linux + sudo adduser --home-dir "${HOMEDIR}" "${USERNAME}" && sudo passwd -d "${USERNAME}" + else + echo "Unsupported OS type in details file." >&2 + return 1 + fi + local exit_code=$? + + if [[ ${exit_code} -eq 0 ]]; then + echo "User ${USERNAME} created successfully." + else + echo "Failed to create user ${USERNAME}." >&2 + fi + return ${exit_code} +} + +# Function to grant sudo access by creating a file in /etc/sudoers.d/ +grant_sudo() { + local USERNAME=$1 + local HOMEDIR=$2 + if ! id "${USERNAME}" &>/dev/null; then + echo "User ${USERNAME} does not exist. Cannot grant sudo." + return 1 + fi + + sudo mkdir -p /etc/sudoers.d/ + SUDOERS_FILE="/etc/sudoers.d/${USERNAME}" + + if sudo test -f "${SUDOERS_FILE}"; then + echo "Sudoers file ${SUDOERS_FILE} already exists." + else + echo "Granting ${USERNAME} NOPASSWD sudo access..." + # Create the sudoers file with the correct content + if ! echo "${USERNAME} ALL=(ALL:ALL) NOPASSWD:ALL" | sudo tee "${SUDOERS_FILE}" > /dev/null; then + echo "Failed to create sudoers file." >&2 + return 1 + fi + + # Set the correct permissions on the sudoers file + if ! sudo chmod 440 "${SUDOERS_FILE}"; then + echo "Failed to set permissions on sudoers file." >&2 + # Attempt to clean up the partially created file + sudo rm -f "${SUDOERS_FILE}" + return 1 + fi + echo "Sudo access granted to ${USERNAME} via ${SUDOERS_FILE}." + fi + return 0 +} +################################################################################ +# Main script execution flow starts here. +# The script will first attempt to create the user specified by $USERNAME. +# If the user creation is successful, it will then proceed to grant sudo +# privileges to the newly created user. +################################################################################ +USERNAME=starterscriptuser +HOMEDIR="/home/${USERNAME}" +DETAILS_FILE=$(pwd)/details.txt + +create_user "$USERNAME" "$HOMEDIR" "$DETAILS_FILE" +grant_sudo "$USERNAME" "$HOMEDIR" + + # Run the following as starterscriptuser sudo -u starterscriptuser bash -c ' # Exit immediately if a command exits with a non-zero status. set -e # Print commands and their arguments as they are executed. + +function cleanup() { + echo "Performing cleanup..." + #Log results based on the collected exit statuses. + log_based_on_exit_status exit_status +} +trap cleanup EXIT set -x +# GCSFuse test suite uses this environment variable to save failure logs at the specified location. +export KOKORO_ARTIFACTS_DIR=/home/starterscriptuser/failure_logs +mkdir -p "$KOKORO_ARTIFACTS_DIR" +# Since we are now operating as the starterscriptuser, we need to set the environment variable for this user again. +export PATH=/usr/local/google-cloud-sdk/bin:$PATH + +# Exporting variables to the sub-shell +export RUN_ON_ZB_ONLY='$RUN_ON_ZB_ONLY' +export RUN_READ_CACHE_TESTS_ONLY='$RUN_READ_CACHE_TESTS_ONLY' +export RUN_LIGHT_TEST='$RUN_LIGHT_TEST' +export BUCKET_NAME_TO_USE='$BUCKET_NAME_TO_USE' #Copy details.txt to starterscriptuser home directory and create logs.txt cd ~/ cp /details.txt . touch logs.txt touch logs-hns.txt +touch logs-zonal.txt +LOG_FILE='~/logs.txt' + +if [[ "$RUN_ON_ZB_ONLY" == "true" ]]; then + LOG_FILE='~/logs-zonal.txt' +fi -echo User: $USER &>> ~/logs.txt -echo Current Working Directory: $(pwd) &>> ~/logs.txt +echo "User: $USER" &>> ${LOG_FILE} +echo "Current Working Directory: $(pwd)" &>> ${LOG_FILE} + +VERSION=$(sed -n 1p ~/details.txt) +COMMIT_HASH=$(sed -n 2p ~/details.txt) +VM_INSTANCE_NAME=$(sed -n 3p ~/details.txt) + +VERSION_CLEAN=$(echo "$VERSION" | sed 's/^v//') +GO_TEST_SHORT_FLAG="" + +# The -short flag was removed in v2.6.0. To ensure backward compatibility when +# testing older versions with newer CD scripts, we conditionally add it. +if [[ "$VERSION_CLEAN" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+) ]]; then + major="${BASH_REMATCH[1]}" + minor="${BASH_REMATCH[2]}" + if [[ "$major" -lt 2 ]] || [[ "$major" -eq 2 && "$minor" -lt 6 ]]; then + GO_TEST_SHORT_FLAG="-short" + fi +fi # Based on the os type in detail.txt, run the following commands for setup @@ -63,9 +254,14 @@ then sudo apt install -y fuse # download and install gcsfuse deb package - gsutil cp gs://gcsfuse-release-packages/v$(sed -n 1p details.txt)/gcsfuse_$(sed -n 1p details.txt)_${architecture}.deb . - sudo dpkg -i gcsfuse_$(sed -n 1p details.txt)_${architecture}.deb |& tee -a ~/logs.txt - + # Only download and install the pre-built deb package if using the default release bucket. + if [[ "${BUCKET_NAME_TO_USE}" == "gcsfuse-release-packages" ]]; then + echo "Downloading pre-built debian package from release bucket..." &>> ${LOG_FILE} + gcloud storage cp gs://${BUCKET_NAME_TO_USE}/v${VERSION}/gcsfuse_${VERSION}_${architecture}.deb . &>> ${LOG_FILE} + sudo dpkg -i gcsfuse_${VERSION}_${architecture}.deb |& tee -a ${LOG_FILE} + else + echo "Custom bucket detected (${BUCKET_NAME_TO_USE}); skipping pre-built debian package installation." &>> ${LOG_FILE} + fi # install wget sudo apt install -y wget @@ -81,9 +277,23 @@ then #install build-essentials sudo apt install -y build-essential else -# For rhel and centos + # For rhel and centos + # Set CLOUDSDK_PYTHON to python3 for gcloud commands to work. + + if [[ -f /etc/os-release ]]; then + # Extract VERSION_ID from /etc/os-release + V_ID=$(awk -F= "/^VERSION_ID=/{print \$2}" /etc/os-release | tr -d "\"") + + # Check if the version starts with 8 or 9 + if [[ "$V_ID" =~ ^[89] ]]; then + export CLOUDSDK_PYTHON="/usr/bin/python3.11" + else + export CLOUDSDK_PYTHON="/usr/bin/python3" + fi + fi + # uname can be aarch or x86_64 - uname=$(uname -i) + uname=$(uname -m) if [[ $uname == "x86_64" ]]; then architecture="amd64" @@ -97,9 +307,14 @@ else #Install fuse sudo yum -y install fuse - #download and install gcsfuse rpm package - gsutil cp gs://gcsfuse-release-packages/v$(sed -n 1p details.txt)/gcsfuse-$(sed -n 1p details.txt)-1.${uname}.rpm . - sudo yum -y localinstall gcsfuse-$(sed -n 1p details.txt)-1.${uname}.rpm + # Only download and install the pre-built rpm package if using the default release bucket. + if [[ "${BUCKET_NAME_TO_USE}" == "gcsfuse-release-packages" ]]; then + echo "Downloading pre-built rpm package from release bucket..." &>> ${LOG_FILE} + gcloud storage cp gs://${BUCKET_NAME_TO_USE}/v${VERSION}/gcsfuse-${VERSION}-1.${uname}.rpm . &>> ${LOG_FILE} + sudo yum -y localinstall gcsfuse-${VERSION}-1.${uname}.rpm &>> ${LOG_FILE} + else + echo "Custom bucket detected (${BUCKET_NAME_TO_USE}); skipping pre-built rpm package installation." &>> ${LOG_FILE} + fi #install wget sudo yum -y install wget @@ -111,18 +326,23 @@ else sudo yum -y install gcc gcc-c++ make fi -# install go -wget -O go_tar.tar.gz https://go.dev/dl/go1.23.3.linux-${architecture}.tar.gz -sudo tar -C /usr/local -xzf go_tar.tar.gz +# Clone and checkout gcsfuse repository +git clone https://github.com/googlecloudplatform/gcsfuse |& tee -a ${LOG_FILE} +cd gcsfuse + +# Install golang. +version=$(cat .go-version) +wget -O go_tar.tar.gz https://go.dev/dl/go${version}.linux-${architecture}.tar.gz +sudo tar -C /usr/local -xzf go_tar.tar.gz && rm go_tar.tar.gz export PATH=${PATH}:/usr/local/go/bin + #Write gcsfuse and go version to log file -gcsfuse --version |& tee -a ~/logs.txt -go version |& tee -a ~/logs.txt +gcsfuse --version |& tee -a ${LOG_FILE} +go version |& tee -a ${LOG_FILE} -# Clone and checkout gcsfuse repo -export PATH=${PATH}:/usr/local/go/bin -git clone https://github.com/googlecloudplatform/gcsfuse |& tee -a ~/logs.txt -cd gcsfuse +# Install latest gcloud. +bash ./perfmetrics/scripts/install_latest_gcloud.sh +export PATH=/usr/local/google-cloud-sdk/bin:$PATH # Installation of crcmod is working through pip only on rhel and centos. # For debian and ubuntu, we are installing through sudo apt. @@ -136,16 +356,22 @@ then pip3 install --require-hashes -r tools/cd_scripts/requirements.txt --user fi -git checkout $(sed -n 2p ~/details.txt) |& tee -a ~/logs.txt +git checkout ${COMMIT_HASH} |& tee -a ${LOG_FILE} +if [[ "${BUCKET_NAME_TO_USE}" != "gcsfuse-release-packages" ]]; then + echo "Installing GCSFuse from source..." + GOOS=linux GOARCH=${architecture} go run tools/build_gcsfuse/main.go . . "${VERSION}" + sudo cp bin/* /usr/bin/ + sudo cp sbin/* /usr/sbin/ +fi #run tests with testbucket flag set +e # Test directory arrays TEST_DIR_PARALLEL=( + "monitoring" "local_file" "log_rotation" "mounting" - "read_cache" "gzip" "write_large_files" "rename_dir_limit" @@ -154,9 +380,22 @@ TEST_DIR_PARALLEL=( "implicit_dir" "interrupt" "operations" - "log_content" "kernel_list_cache" "concurrent_operations" + "mount_timeout" + "stale_handle" + "negative_stat_cache" + "streaming_writes" + "release_version" + # Reenable when b/461334834 is done. + # "readdirplus" + # "dentry_cache" + "buffered_read" + # Disabled because of b/451462914. + #"requester_pays_bucket" + "flag_optimizations" + "unsupported_path" + "symlink_handling" ) # These tests never become parallel as they are changing bucket permissions. @@ -167,112 +406,189 @@ TEST_DIR_NON_PARALLEL=( "list_large_dir" ) +# For Zonal buckets : Test directory arrays +TEST_DIR_PARALLEL_ZONAL=( + buffered_read + concurrent_operations + # Reenable when b/461334834 is done. + # dentry_cache + explicit_dir + flag_optimizations + gzip + implicit_dir + interrupt + kernel_list_cache + local_file + log_rotation + monitoring + mount_timeout + mounting + negative_stat_cache + operations + rapid_appends + read_large_files + # Reenable when b/461334834 is done. + # readdirplus + release_version + rename_dir_limit + stale_handle + streaming_writes + unfinalized_object + write_large_files + "unsupported_path" + "symlink_handling" +) + +# For Zonal Buckets : These tests never become parallel as they are changing bucket permissions. +TEST_DIR_NON_PARALLEL_ZONAL=( + "managed_folders" + "readonly" + "readonly_creds" + "list_large_dir" +) + # Create a temporary file to store the log file name. TEST_LOGS_FILE=$(mktemp) -INTEGRATION_TEST_TIMEOUT=180m +INTEGRATION_TEST_TIMEOUT=240m +# This method runs test packages in sequence.Necessary when the tests involves +# permissions modification etc. +# Arguments: +# $1: BUCKET_NAME (The name of the GCS bucket to use for tests.) +# $2: IS_ZONAL_BUCKET_FLAG (Boolean flag: 'true' if the bucket is zonal, 'false' otherwise.) +# $3: NAME_OF_TEST_DIR_ARRAY (The shell variable name of the array containing test directory.) function run_non_parallel_tests() { - local exit_code=0 - local -n test_array=$1 - local BUCKET_NAME=$2 + if [[ "$#" -ne 3 ]]; then + echo "Incorrect number of arguments passed, Expecting <BUCKET_NAME> + <IS_ZONAL_BUCKET> <TEST_DIR_ARRAY>" + exit 1 + fi + local exit_code=0 # Initialize to 0 for success + local BUCKET_NAME=$1 + local zonal=$2 + if [[ -z $3 ]]; then + return 1 # The name of the test array cannot be empty. + fi + local -n test_array=$3 # Create a nameref to this array. + for test_dir_np in "${test_array[@]}" do test_path_non_parallel="./tools/integration_tests/$test_dir_np" - # To make it clear whether tests are running on a flat or HNS bucket, We kept the log file naming + # To make it clear whether tests are running on a flat or HNS or zonal bucket, We kept the log file naming # convention to include the bucket name as a suffix (e.g., package_name_bucket_name). local log_file="/tmp/${test_dir_np}_${BUCKET_NAME}.log" - echo $log_file >> $TEST_LOGS_FILE - # Executing integration tests - GODEBUG=asyncpreemptoff=1 go test $test_path_non_parallel -p 1 --integrationTest -v --testbucket=$BUCKET_NAME --testInstalledPackage=true -timeout $INTEGRATION_TEST_TIMEOUT > "$log_file" 2>&1 - exit_code_non_parallel=$? - if [ $exit_code_non_parallel != 0 ]; then + echo "$log_file" >> "$TEST_LOGS_FILE" # Use double quotes for log_file + if [[ -d "$test_path_non_parallel" ]]; then + GODEBUG=asyncpreemptoff=1 go test "$test_path_non_parallel" $GO_TEST_SHORT_FLAG -p 1 --zonal="${zonal}" --integrationTest -v --testbucket="$BUCKET_NAME" --testInstalledPackage=true -timeout "$INTEGRATION_TEST_TIMEOUT" > "$log_file" 2>&1 + exit_code_non_parallel=$? + else + echo "Test path $test_path_non_parallel does not exist. Skipping tests." >> "$log_file" + exit_code_non_parallel=0 # Treat as success if test path doesnt exist + fi + if [[ $exit_code_non_parallel -ne 0 ]]; then exit_code=$exit_code_non_parallel fi done return $exit_code } +#This method runs test packages in parallel. +# Arguments: +# $1: BUCKET_NAME (The name of the GCS bucket to use for tests.) +# $2: IS_ZONAL_BUCKET_FLAG (Boolean flag: 'true' if the bucket is zonal, 'false' otherwise.) +# $3: NAME_OF_TEST_DIR_ARRAY (The shell variable name of the array containing test directory.) function run_parallel_tests() { + if [[ "$#" -ne 3 ]]; then + echo "Incorrect number of arguments passed, Expecting <BUCKET_NAME> + <IS_ZONAL_BUCKET> <TEST_DIR_ARRAY>" + exit 1 + fi local exit_code=0 - local -n test_array=$1 - local BUCKET_NAME=$2 + local BUCKET_NAME=$1 + local zonal=$2 + if [[ -z $3 ]]; then + return 1 # The name of the test array cannot be empty. + fi + local -n test_array=$3 # Create a nameref to this array. local pids=() + for test_dir_p in "${test_array[@]}" do test_path_parallel="./tools/integration_tests/$test_dir_p" - # To make it clear whether tests are running on a flat or HNS bucket, We kept the log file naming + # To make it clear whether tests are running on a flat or HNS or zonal bucket, We kept the log file naming # convention to include the bucket name as a suffix (e.g., package_name_bucket_name). local log_file="/tmp/${test_dir_p}_${BUCKET_NAME}.log" - echo $log_file >> $TEST_LOGS_FILE - # Executing integration tests - GODEBUG=asyncpreemptoff=1 go test $test_path_parallel -p 1 --integrationTest -v --testbucket=$BUCKET_NAME --testInstalledPackage=true -timeout $INTEGRATION_TEST_TIMEOUT > "$log_file" 2>&1 & - pid=$! # Store the PID of the background process - pids+=("$pid") # Optionally add the PID to an array for later + echo "$log_file" >> "$TEST_LOGS_FILE" + if [[ -d "$test_path_parallel" ]]; then + GODEBUG=asyncpreemptoff=1 go test "$test_path_parallel" $GO_TEST_SHORT_FLAG -p 1 --zonal="${zonal}" --integrationTest -v --testbucket="$BUCKET_NAME" --testInstalledPackage=true -timeout "$INTEGRATION_TEST_TIMEOUT" > "$log_file" 2>&1 & + pid=$! + pids+=("$pid") + else + echo "Test path $test_path_parallel does not exist. Skipping tests." >> "$log_file" + fi done - # Wait for processes and collect exit codes for pid in "${pids[@]}"; do - wait $pid + wait "$pid" exit_code_parallel=$? - if [ $exit_code_parallel != 0 ]; then + if [[ $exit_code_parallel -ne 0 ]]; then exit_code=$exit_code_parallel fi done return $exit_code } -function run_e2e_tests_for_flat_bucket() { - flat_bucket_name_non_parallel=$(sed -n 3p ~/details.txt) - echo "Flat Bucket name to run tests sequentially: "$flat_bucket_name_non_parallel - - flat_bucket_name_parallel=$(sed -n 3p ~/details.txt)-parallel - echo "Flat Bucket name to run tests parallelly: "$flat_bucket_name_parallel +#Common method to invoke e2e tests on different types of buckets: flat, HNS or Zonal +# Arguments: +# $1: BUCKET-TYPE (flat/hns/zonal) +# $2: TEST_DIR_PARALLEL (list of test packages that can be run in parallel) +# $3: TEST_DIR_NON_PARALLEL (list of test packages that should be run in sequence) +# $4: IS_ZONAL_BUCKET_FLAG (Boolean flag: 'true' if the bucket is zonal, 'false' otherwise.) +function run_e2e_tests() { + if [[ "$#" -ne 4 ]]; then + echo "Incorrect number of arguments passed, Expecting <TESTCASE> + <NAME_OF_PARALLEL_TEST_DIR_ARRAY> <NAME_OF_PARALLEL_TEST_DIR_ARRAY> + <IS_ZONAL_BUCKET_FLAG>" + exit 1 + fi + local testcase=$1 + local -n test_dir_parallel=$2 + local -n test_dir_non_parallel=$3 + local is_zonal=$4 + local overall_exit_code=0 + + prefix=${VM_INSTANCE_NAME} + if [[ "$testcase" != "flat" ]]; then + prefix=${VM_INSTANCE_NAME}-$testcase + fi + + local bkt_non_parallel=$prefix + echo "Bucket name to run non-parallel tests sequentially: $bkt_non_parallel" + + local bkt_parallel=$prefix-parallel + echo "Bucket name to run parallel tests: $bkt_parallel" echo "Running parallel tests..." - run_parallel_tests TEST_DIR_PARALLEL "$flat_bucket_name_parallel" & + run_parallel_tests "$bkt_parallel" "$is_zonal" test_dir_parallel & # Pass the name of the array parallel_tests_pid=$! - echo "Running non parallel tests ..." - run_non_parallel_tests TEST_DIR_NON_PARALLEL "$flat_bucket_name_non_parallel" & - non_parallel_tests_pid=$! - - # Wait for all tests to complete. - wait $parallel_tests_pid - parallel_tests_exit_code=$? - wait $non_parallel_tests_pid - non_parallel_tests_exit_code=$? - - if [ $non_parallel_tests_exit_code != 0 ] || [ $parallel_tests_exit_code != 0 ]; - then - return 1 - fi - return 0 -} - -function run_e2e_tests_for_hns_bucket(){ - hns_bucket_name_non_parallel=$(sed -n 3p ~/details.txt)-hns - echo "HNS Bucket name to run tests sequentially: "$hns_bucket_name_non_parallel - - hns_bucket_name_parallel=$(sed -n 3p ~/details.txt)-hns-parallel - echo "HNS Bucket name to run tests parallelly: "$hns_bucket_name_parallel + echo "Running non parallel tests ..." + run_non_parallel_tests "$bkt_non_parallel" "$is_zonal" test_dir_non_parallel & # Pass the name of the array + non_parallel_tests_pid=$! - echo "Running tests for HNS bucket" - run_parallel_tests TEST_DIR_PARALLEL "$hns_bucket_name_parallel" & - parallel_tests_hns_group_pid=$! - run_non_parallel_tests TEST_DIR_NON_PARALLEL "$hns_bucket_name_non_parallel" & - non_parallel_tests_hns_group_pid=$! + wait "$parallel_tests_pid" + local parallel_tests_exit_code=$? + wait "$non_parallel_tests_pid" + local non_parallel_tests_exit_code=$? - # Wait for all tests to complete. - wait $parallel_tests_hns_group_pid - parallel_tests_hns_group_exit_code=$? - wait $non_parallel_tests_hns_group_pid - non_parallel_tests_hns_group_exit_code=$? + if [[ "$non_parallel_tests_exit_code" -ne 0 ]]; then + overall_exit_code=$non_parallel_tests_exit_code + fi - if [ $parallel_tests_hns_group_exit_code != 0 ] || [ $non_parallel_tests_hns_group_exit_code != 0 ]; - then - return 1 - fi - return 0 + if [[ "$parallel_tests_exit_code" -ne 0 ]]; then + overall_exit_code=$parallel_tests_exit_code + fi + return $overall_exit_code } function gather_test_logs() { @@ -281,9 +597,11 @@ function gather_test_logs() { for test_log_file in "${test_logs_array[@]}" do log_file=${test_log_file} - if [ -f "$log_file" ]; then + if [[ -f "$log_file" ]]; then if [[ "$test_log_file" == *"hns"* ]]; then output_file="$HOME/logs-hns.txt" + elif [[ "$test_log_file" == *"zonal"* ]]; then + output_file="$HOME/logs-zonal.txt" else output_file="$HOME/logs.txt" fi @@ -295,37 +613,109 @@ function gather_test_logs() { done } -echo "Running integration tests for HNS bucket..." -run_e2e_tests_for_hns_bucket & -e2e_tests_hns_bucket_pid=$! +# Function to log test results and upload them to GCS based on exit status. +# Arguments: $1 = name of the associative array containing testcase exit statuses. +function log_based_on_exit_status() { + if [[ "$#" -ne 1 ]]; then + echo "Incorrect number of arguments passed, Expecting <EXIT_STATUS_ARRAY_NAME>" + exit 1 + fi + gather_test_logs + local -n exit_status_array=$1 + + for testcase in "${!exit_status_array[@]}" + do + local logfile="" + local successfile="" + if [[ "$testcase" == "flat" ]]; then + logfile="$HOME/logs.txt" + successfile="$HOME/success.txt" + else + logfile="$HOME/logs-$testcase.txt" + successfile="$HOME/success-$testcase.txt" + fi + if [[ "${exit_status_array["$testcase"]}" != 0 ]]; + then + echo "Test failures detected in $testcase bucket." &>> $logfile + else + touch $successfile + gcloud storage cp $successfile gs://${BUCKET_NAME_TO_USE}/v${VERSION}/${VM_INSTANCE_NAME}/ + fi + gcloud storage cp $logfile gs://${BUCKET_NAME_TO_USE}/v${VERSION}/${VM_INSTANCE_NAME}/ + done + + gcloud storage cp -R "$KOKORO_ARTIFACTS_DIR" gs://${BUCKET_NAME_TO_USE}/v${VERSION}/${VM_INSTANCE_NAME}/ +} + +# Function to run emulator-based E2E tests and log results. +function run_e2e_tests_for_emulator_and_log() { + ./tools/integration_tests/emulator_tests/emulator_tests.sh true > ~/logs-emulator.txt + emulator_test_status=$? + if [ $e2e_tests_emulator_status != 0 ]; + then + echo "Test failures detected in emulator based tests." &>> ~/logs-emulator.txt + else + touch success-emulator.txt + gcloud storage cp success-emulator.txt gs://${BUCKET_NAME_TO_USE}/v${VERSION}/${VM_INSTANCE_NAME}/ + fi + gcloud storage cp ~/logs-emulator.txt gs://${BUCKET_NAME_TO_USE}/v${VERSION}/${VM_INSTANCE_NAME}/ +} + +# Declare an associative array to store the exit status of different test runs. +declare -A exit_status +if [[ "$RUN_LIGHT_TEST" == "true" ]]; then + light_test_dir_non_parallel=("monitoring") + light_test_dir_parallel=() -echo "Running integration tests for FLAT bucket..." -run_e2e_tests_for_flat_bucket & -e2e_tests_flat_bucket_pid=$! + run_e2e_tests "flat" light_test_dir_parallel light_test_dir_non_parallel false + exit_status["flat"]=$? -wait $e2e_tests_flat_bucket_pid -e2e_tests_flat_bucket_status=$? + run_e2e_tests "hns" light_test_dir_parallel light_test_dir_non_parallel false + exit_status["hns"]=$? -wait $e2e_tests_hns_bucket_pid -e2e_tests_hns_bucket_status=$? + run_e2e_tests "zonal" light_test_dir_parallel light_test_dir_non_parallel true + exit_status["zonal"]=$? -gather_test_logs + # Run emulator tests and log their results. + run_e2e_tests_for_emulator_and_log +elif [[ "$RUN_READ_CACHE_TESTS_ONLY" == "true" ]]; then + read_cache_test_dir_parallel=() # Empty for read cache tests only + read_cache_test_dir_non_parallel=("read_cache") -if [ $e2e_tests_flat_bucket_status != 0 ] -then - echo "Test failures detected in FLAT bucket." &>> ~/logs.txt -else - touch success.txt - gsutil cp success.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/ -fi -gsutil cp ~/logs.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/ + # Run E2E tests for flat, HNS, and zonal buckets with only read cache tests. + # Running sequentially due to known limitations of simultaneous execution. + run_e2e_tests "flat" read_cache_test_dir_parallel read_cache_test_dir_non_parallel false + exit_status["flat"]=$? -if [ $e2e_tests_hns_bucket_status != 0 ]; -then - echo "Test failures detected in HNS bucket." &>> ~/logs-hns.txt + run_e2e_tests "hns" read_cache_test_dir_parallel read_cache_test_dir_non_parallel false + exit_status["hns"]=$? + + run_e2e_tests "zonal" read_cache_test_dir_parallel read_cache_test_dir_non_parallel true + exit_status["zonal"]=$? else - touch success-hns.txt - gsutil cp success-hns.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/ + # If not running *only* read cache tests, proceed with full test suites. + if [[ "$RUN_ON_ZB_ONLY" == "true" ]]; then + # If only zonal bucket tests are to be run. + run_e2e_tests "zonal" TEST_DIR_PARALLEL_ZONAL TEST_DIR_NON_PARALLEL_ZONAL true + exit_status["zonal"]=$? + else + # Run flat and HNS tests concurrently in the background. + run_e2e_tests "flat" TEST_DIR_PARALLEL TEST_DIR_NON_PARALLEL false & + flat_test_pid=$! + + run_e2e_tests "hns" TEST_DIR_PARALLEL TEST_DIR_NON_PARALLEL false & + hns_test_pid=$! + + # Wait for PIDs and populate exit_status associative array + wait $flat_test_pid + exit_status["flat"]=$? + + wait $hns_test_pid + exit_status["hns"]=$? + + # Run emulator tests and log their results. + run_e2e_tests_for_emulator_and_log + fi + fi -gsutil cp ~/logs-hns.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/ ' diff --git a/tools/cd_scripts/improved_e2e_test.sh b/tools/cd_scripts/improved_e2e_test.sh new file mode 100755 index 0000000000..a9dea43b04 --- /dev/null +++ b/tools/cd_scripts/improved_e2e_test.sh @@ -0,0 +1,187 @@ +#! /bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Exit on error, treat unset variables as errors, and propagate pipeline errors. +set -euo pipefail + +# Logging Helpers +log_info() { + echo "[INFO] $(date +"%Y-%m-%d %H:%M:%S"): $1" +} + +log_error() { + echo "[ERROR] $(date +"%Y-%m-%d %H:%M:%S"): $1" +} + +# Defaults +LOCAL_RUN=false +RELEASE_PACKAGE_BUCKET="" +RELEASE_VERSION="" +RUN_TESTS_WITH_ZONAL_BUCKET=false + +usage() { + echo "Usage: $0 [--local-run] " + echo " --local-run Pass this flag to run this script for local runs. If this flag is passed then gcsfuse is built" + echo " locally instead of getting installed by pre-built package from release bucket." + echo " --release-package-bucket <bkt> Name of the GCS bucket from which release packages will be fetched." + echo " Release package bucket is required if not running using --local-run" + echo " --release-version <3.0.0> Release version determines from which directory the pre-built package is used from release package bucket." + echo " Release version is required if not running using --local-run" + echo " --zonal Should run tests for zonal bucket (Default: false)" + echo " --output-dir <output-dir> Directory in which all of log files generated by this script will be stored. (Default: /tmp)" + echo " --help Display this help and exit." + exit "$1" +} + +# Define options for getopt +# A long option name followed by a colon indicates it requires an argument. +LONG=local-run,zonal,release-package-bucket:,release-version:,output-dir:,help + +# Parse the options using getopt +# --options "" specifies that there are no short options. +if ! PARSED=$(getopt --options "" --longoptions "$LONG" --name "$0" -- "$@"); then + usage 1 +fi + +# Output directory where all artifacts generated by this script will be stored. +OUTPUT_DIR="" +# Read the parsed options back into the positional parameters. +eval set -- "$PARSED" + +# Loop through the options and assign values to our variables +while (( $# >= 1 )); do + case "$1" in + --release-version) + RELEASE_VERSION="$2" + shift 2 + # Regex breakdown: + # ^ : Start of string + # [0-9]+ : One or more digits + # \. : A literal dot + # $ : End of string + RE="^[0-9]+\.[0-9]+\.[0-9]+$" + if [[ ! $RELEASE_VERSION =~ $RE ]]; then + log_error "--release-version value '$RELEASE_VERSION' is incorrectly formatted." + usage 1 + fi + ;; + --release-package-bucket) + RELEASE_PACKAGE_BUCKET="$2" + shift 2 + ;; + --zonal) + RUN_TESTS_WITH_ZONAL_BUCKET=true + shift + ;; + --output-dir) + OUTPUT_DIR="$2" + shift 2 + ;; + --local-run) + LOCAL_RUN=true + shift + ;; + --help) + usage 0 + ;; + --) + shift + break + ;; + *) + log_error "Unrecognized arguments [$*]." + usage 1 + ;; + esac +done + +# Argument validation +if [[ "$LOCAL_RUN" == "false" ]]; then + if [[ -z "$RELEASE_VERSION" ]]; then + log_error "--release-version required if not running with --local-run" + usage 1 + fi + if [[ -z "$RELEASE_PACKAGE_BUCKET" ]]; then + log_error "--release-package-bucket required if not running with --local-run" + usage 1 + fi +fi + +# Build args for the e2e script +ARGS=() + +# Local Run Validation and gcsfuse package installation. +if ${LOCAL_RUN}; then + log_info "Running script in local mode gcsfuse binary would be built in script from current repository." +else + log_info "Running script with release version package from release bucket for version $RELEASE_VERSION will be installed." + # Identify the OS and Architecture + if [ -f /etc/os-release ]; then + # We source in a subshell to prevent variable pollution, + # then capture only the ID and ID_LIKE fields. + DISTRO_DATA=$( (source /etc/os-release; echo "${ID:-} ${ID_LIKE:-}") ) + # Check for debian or ubuntu in the ID or the ID_LIKE chain + if [[ "$DISTRO_DATA" == *"debian"* ]] || [[ "$DISTRO_DATA" == *"ubuntu"* ]]; then + ARCH=$(dpkg --print-architecture) + ARGS+=( + "--install-package-from-path=gs://${RELEASE_PACKAGE_BUCKET}/v${RELEASE_VERSION}/gcsfuse_${RELEASE_VERSION}_${ARCH}.deb" + ) + elif [[ "$DISTRO_DATA" == *"rhel"* ]] || [[ "$DISTRO_DATA" == *"centos"* ]]; then + # On RHEL-based systems, we use 'arch' or 'uname -m' + # as dpkg is usually not present. + ARCH=$(uname -m) + ARGS+=( + "--install-package-from-path=gs://${RELEASE_PACKAGE_BUCKET}/v${RELEASE_VERSION}/gcsfuse-${RELEASE_VERSION}-1.${ARCH}.rpm" + ) + else + log_error "This script only supports Debian/Ubuntu/rhel/centos based distributions." + log_info "Your distribution is:" + cat /etc/os-release + exit 1 + fi + else + log_error "/etc/os-release not found. Unable to determine distribution" + exit 1 + fi +fi + +# Set parallelism to 4 as it is optimal for all of the release VM(s). +ARGS+=("--package-level-parallelism=4") + +# Set --zonal arg if required +if ${RUN_TESTS_WITH_ZONAL_BUCKET}; then + log_info "Running zonal e2e tests." + ARGS+=("--zonal") +else + log_info "Running regional e2e tests." +fi + +# Fallback to /tmp if OUTPUT_DIR is unset +OUTPUT_DIR="${OUTPUT_DIR:-/tmp}" +mkdir -p "$OUTPUT_DIR" || { + log_error "Failed to create or access output directory '$OUTPUT_DIR'"; + exit 1 +} +ARGS+=("--output-dir=$OUTPUT_DIR") + +# TODO(b/503191432): Enable the cloud_profiler in release testing once it is stable. +# Run all packages except cloud profiler during release testing. +ARGS+=("--run-package=!cloud_profiler") + +# Set --flake-attempts to 3 for release tests. +ARGS+=("--flake-attempts=3") + +# Run the main e2e script +bash ./tools/integration_tests/improved_run_e2e_tests.sh "${ARGS[@]}" diff --git a/tools/cd_scripts/install_test.sh b/tools/cd_scripts/install_test.sh index 9309286edb..982e2363a3 100755 --- a/tools/cd_scripts/install_test.sh +++ b/tools/cd_scripts/install_test.sh @@ -16,10 +16,64 @@ # Print commands and their arguments as they are executed. set -x +PYTHON_VERSION=3.11.9 +INSTALL_PREFIX="$HOME/.local/python-$PYTHON_VERSION" + +# Install common dependencies (Python 3.11, wget, tar) +# This is required for gcloud to function correctly (needs Python 3.10+) +# and for the script to download/extract the SDK. +if command -v apt-get &> /dev/null; then + sudo apt-get update -y > /dev/null + sudo apt-get install -y \ + build-essential zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev \ + libssl-dev libreadline-dev libffi-dev curl libsqlite3-dev \ + libbz2-dev liblzma-dev tk-dev uuid-dev wget tar > /dev/null +elif command -v yum &> /dev/null; then + # For RHEL-based systems, use 'yum' to install packages. + # The "Development Tools" group is equivalent to 'build-essential' on Debian. + # The '-devel' packages provide the necessary header files for compilation. + sudo yum -y groupinstall "Development Tools" > /dev/null + sudo yum -y install \ + zlib-devel ncurses-devel nss-devel openssl-devel \ + readline-devel libffi-devel curl sqlite-devel bzip2-devel \ + xz-devel tk-devel libuuid-devel wget > /dev/null +fi + +# Download and build Python locally +cd /tmp +wget -q https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz +tar -xf Python-${PYTHON_VERSION}.tgz +cd Python-${PYTHON_VERSION} + +echo "Configuring Python build for local install..." +./configure --enable-optimizations --prefix="$INSTALL_PREFIX" > /dev/null + +echo "Building Python $PYTHON_VERSION..." +make -j"$(nproc)" > /dev/null + +echo "Installing Python $PYTHON_VERSION locally at $INSTALL_PREFIX..." +make altinstall > /dev/null + +echo "Python $PYTHON_VERSION installed at $INSTALL_PREFIX/bin/python3.11" +"$INSTALL_PREFIX/bin/python3.11" --version + +# Ensure gcloud uses the newly installed Python 3.11 +export CLOUDSDK_PYTHON="$INSTALL_PREFIX/bin/python3.11" + +echo "Upgrade gcloud version" +gcloud version +wget -O gcloud.tar.gz https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz -q +sudo tar xzf gcloud.tar.gz && sudo cp -r google-cloud-sdk /usr/local && sudo rm -r google-cloud-sdk +sudo /usr/local/google-cloud-sdk/install.sh --quiet +export PATH=/usr/local/google-cloud-sdk/bin:$PATH +gcloud version && rm gcloud.tar.gz + #details.txt file contains the release version and commit hash of the current release. -gsutil cp gs://gcsfuse-release-packages/version-detail/details.txt . +gcloud storage cp gs://gcsfuse-release-packages/version-detail/details.txt . # Writing VM instance name to details.txt (Format: release-test-<os-name>) vm_instance_name=$(curl http://metadata.google.internal/computeMetadata/v1/instance/name -H "Metadata-Flavor: Google") +# first line of details.txt contains the release version in the format MAJOR.MINOR.PATCH +to_release_version=$(sed '1q' details.txt | tr -d '\n') echo $vm_instance_name >> details.txt touch ~/logs.txt @@ -51,8 +105,8 @@ then fi sudo apt-get update - # Install latest released gcsfuse version - sudo apt-get install -y gcsfuse + # Install to be released gcsfuse version (It can be a patch to older version so allow downgrades) + sudo apt-get install -y --allow-downgrades gcsfuse="$to_release_version" >> ~/logs.txt else # For rhel and centos sudo yum install fuse @@ -72,20 +126,20 @@ repo_gpgcheck=0 gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg EOF -sudo yum install -y gcsfuse +# Attempt a install first, falling back to a standard downgrade if to be released is older version than already installed (patch releases). +sudo yum install -y gcsfuse-"$to_release_version" || sudo yum downgrade -y gcsfuse-"$to_release_version" >> ~/logs.txt fi # Verify gcsfuse version (successful installation) gcsfuse --version |& tee version.txt installed_version=$(echo $(sed -n 1p version.txt) | cut -d' ' -f3) if grep -q $installed_version details.txt; then - echo "GCSFuse latest version installed correctly." &>> ~/logs.txt + echo "GCSFuse to be released version installed correctly." &>> ~/logs.txt else - echo "Failure detected in latest gcsfuse version installation." &>> ~/logs.txt + echo "Failure detected in to be released gcsfuse version installation." &>> ~/logs.txt fi - -# Uninstall gcsfuse latest version and install old version +# Uninstall gcsfuse and install old version. if grep -q ubuntu details.txt || grep -q debian details.txt; then sudo apt-get remove -y gcsfuse |& tee -a ~/logs.txt @@ -104,7 +158,7 @@ else echo "Failure detected in GCSFuse old version installation." &>> ~/logs.txt fi -# Upgrade gcsfuse to latest version +# Upgrade gcsfuse to latest version. if grep -q ubuntu details.txt || grep -q debian details.txt; then sudo apt-get install --only-upgrade gcsfuse |& tee -a ~/logs.txt @@ -112,20 +166,26 @@ else sudo yum -y upgrade gcsfuse |& tee -a ~/logs.txt fi +# Verify that gcsfuse has been upgraded to the to_be_released version using version comparison. +# This is to ensure that the correct version is installed after the upgrade. gcsfuse --version |& tee version.txt installed_version=$(echo $(sed -n 1p version.txt) | cut -d' ' -f3) -if grep -q $installed_version details.txt; then - echo "GCSFuse successfully upgraded to latest version $installed_version." &>> ~/logs.txt +# The following command compares the two versions: +# 1. `printf` outputs to_release_version and installed_version on a new line. +# 2. `sort -V` sorts them naturally (version sort). +# 3. `tail -n 1` gets the last line, which is the highest version. +# The condition is true if installed_version is greater than or equal to to_release_version. +if [[ "$(printf '%s\n%s\n' "$to_release_version" "$installed_version" | sort -V | tail -n 1)" == "$installed_version" ]]; then + echo "GCSFuse successfully upgraded to latest version: installed_version ($installed_version), to_release_version: ($to_release_version)" &>> ~/logs.txt else - echo "Failure detected in upgrading to latest gcsfuse version." &>> ~/logs.txt + echo "Failure detected in upgrading to latest gcsfuse version: installed_version ($installed_version), to_release_version: ($to_release_version)" &>> ~/logs.txt fi if grep -q Failure ~/logs.txt; then echo "Test failed" &>> ~/logs.txt ; else touch success.txt - gsutil cp success.txt gs://gcsfuse-release-packages/v$(sed -n 1p details.txt)/installation-test/$(sed -n 3p details.txt)/ ; + gcloud storage cp success.txt gs://gcsfuse-release-packages/v$(sed -n 1p details.txt)/installation-test/$(sed -n 3p details.txt)/ ; fi -gsutil cp ~/logs.txt gs://gcsfuse-release-packages/v$(sed -n 1p details.txt)/installation-test/$(sed -n 3p details.txt)/ - +gcloud storage cp ~/logs.txt gs://gcsfuse-release-packages/v$(sed -n 1p details.txt)/installation-test/$(sed -n 3p details.txt)/ diff --git a/tools/cd_scripts/package_gcsfuse.sh b/tools/cd_scripts/package_gcsfuse.sh index d6c90d677d..ed55c973ad 100755 --- a/tools/cd_scripts/package_gcsfuse.sh +++ b/tools/cd_scripts/package_gcsfuse.sh @@ -25,25 +25,13 @@ # gcsfuse-RELEASE_VERSION-1.x86_64.rpm set -e -# create-gcsfuse-packages VM is a fixed VM in GCP project gcs-fuse-test -# Function to fetch metadata value of the key. +# Function to fetch metadata value of the key from the VM. function fetch_meta_data_value() { metadata_key=$1 - # Fetch metadata value of the key - gcloud compute instances describe create-gcsfuse-packages --zone us-west1-b --flatten="metadata[$metadata_key]" >> metadata.txt - # cat metadata.txt.txt - # --- - # value - x=$(sed '2!d' metadata.txt) - # value(contains preceding spaces) - # Remove spaces - # value - value=$(echo "$x" | sed 's/[[:space:]]//g') - # echo $value - # value - rm metadata.txt - echo $value + # Fetch metadata value directly from the local metadata server + value=$(curl -sfS -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/$metadata_key") + echo "$value" } architecture=$(dpkg --print-architecture) @@ -59,6 +47,10 @@ echo RELEASE_VERSION_TAG="$RELEASE_VERSION_TAG" UPLOAD_BUCKET=$(fetch_meta_data_value "UPLOAD_BUCKET") echo UPLOAD_BUCKET="$UPLOAD_BUCKET" +# Fetch metadata value of the key "COMMIT_HASH" +COMMIT_HASH=$(fetch_meta_data_value "COMMIT_HASH") +echo COMMIT_HASH="$COMMIT_HASH" + sudo apt-get update echo Install docker sudo apt install apt-transport-https ca-certificates curl software-properties-common -y @@ -72,11 +64,11 @@ sudo apt-get install git -y sudo apt-get install qemu-user-static binfmt-support git clone https://github.com/GoogleCloudPlatform/gcsfuse.git cd gcsfuse/tools/package_gcsfuse_docker/ -git checkout "v$RELEASE_VERSION_TAG" +git checkout "$COMMIT_HASH" echo "Building docker for ${architecture} ..." -sudo docker buildx build --load . -t gcsfuse-release-${architecture}:"$RELEASE_VERSION_TAG" --build-arg GCSFUSE_VERSION="$RELEASE_VERSION" --build-arg ARCHITECTURE=${architecture} --build-arg BRANCH_NAME="v$RELEASE_VERSION_TAG" &> docker_${architecture}.log -gsutil cp docker_${architecture}.log gs://"$UPLOAD_BUCKET"/v"$RELEASE_VERSION"/ +sudo docker buildx build --load . -t gcsfuse-release-${architecture}:"$RELEASE_VERSION_TAG" --build-arg GCSFUSE_VERSION="$RELEASE_VERSION" --build-arg ARCHITECTURE=${architecture} --build-arg BRANCH_NAME="$COMMIT_HASH" &> docker_${architecture}.log +gcloud storage cp docker_${architecture}.log gs://"$UPLOAD_BUCKET"/v"$RELEASE_VERSION"/ sudo docker run -v $HOME/gcsfuse/release:/release gcsfuse-release-${architecture}:"$RELEASE_VERSION_TAG" cp -r /packages/. /release/v"$RELEASE_VERSION" echo "Upload files in the bucket ..." -gsutil cp -r $HOME/gcsfuse/release/v"$RELEASE_VERSION" gs://"$UPLOAD_BUCKET"/ +gcloud storage cp --recursive $HOME/gcsfuse/release/v"$RELEASE_VERSION" gs://"$UPLOAD_BUCKET"/ diff --git a/tools/config-gen/flag_template_data_gen.go b/tools/config-gen/flag_template_data_gen.go index af47d2494d..fc7563fad7 100644 --- a/tools/config-gen/flag_template_data_gen.go +++ b/tools/config-gen/flag_template_data_gen.go @@ -18,13 +18,21 @@ package main import ( "fmt" + "strings" "time" + + "golang.org/x/text/cases" + "golang.org/x/text/language" ) type flagTemplateData struct { Param // The pFlag function to invoke in order to add the flag. Fn string + // The path to the field in the Go Config struct, e.g. "MetadataCache.TtlSecs" + GoPath string + // The Go type of the field, e.g. "int64" + GoType string } func computeFlagTemplateData(paramsConfig []Param) ([]flagTemplateData, error) { @@ -39,6 +47,14 @@ func computeFlagTemplateData(paramsConfig []Param) ([]flagTemplateData, error) { return flgTemplate, nil } +func capitalize(name string) string { + var buf strings.Builder + for w := range strings.SplitSeq(name, "-") { + buf.WriteString(cases.Title(language.English).String(w)) + } + return buf.String() +} + func computeFlagTemplateDataForParam(p Param) (flagTemplateData, error) { var defaultValue string var fn string @@ -76,7 +92,7 @@ func computeFlagTemplateDataForParam(p Param) (flagTemplateData, error) { } defaultValue = fmt.Sprintf("%d * time.Nanosecond", dur.Nanoseconds()) fn = "DurationP" - case "octal", "logSeverity", "protocol", "resolvedPath": + case "octal", "logSeverity", "protocol", "resolvedPath", "directPathStrategy": fallthrough case "string": defaultValue = fmt.Sprintf("%q", p.DefaultValue) @@ -93,8 +109,16 @@ func computeFlagTemplateDataForParam(p Param) (flagTemplateData, error) { p.DefaultValue = defaultValue // Usage string safely escaped with Go syntax. p.Usage = fmt.Sprintf("%q", p.Usage) + // Compute the Go field path, e.g., "metadata-cache.ttl-secs" -> "MetadataCache.TtlSecs" + var goPathParts []string + for part := range strings.SplitSeq(p.ConfigPath, ".") { + goPathParts = append(goPathParts, capitalize(part)) + } + goPath := strings.Join(goPathParts, ".") return flagTemplateData{ - Param: p, - Fn: fn, + Param: p, + Fn: fn, + GoPath: goPath, + GoType: getGoDataType(p.Type), // Assumes getGoDataType is available }, nil } diff --git a/tools/config-gen/main.go b/tools/config-gen/main.go index 8bd4e70068..759abb02c5 100644 --- a/tools/config-gen/main.go +++ b/tools/config-gen/main.go @@ -22,8 +22,14 @@ import ( "fmt" "os" "path" + "reflect" "slices" "text/template" // NOLINT + "unicode" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg/shared" + "golang.org/x/text/cases" + "golang.org/x/text/language" ) var ( @@ -32,9 +38,13 @@ var ( templateDir = flag.String("templateDir", ".", "Directory containing the template files") ) +type OptimizationRulesMap = map[string]shared.OptimizationRules + type templateData struct { - TypeTemplateData []typeTemplateData - FlagTemplateData []flagTemplateData + TypeTemplateData []typeTemplateData + FlagTemplateData []flagTemplateData + MachineTypeToGroupMap map[string]string + MachineTypeGroups map[string][]string // Back-ticks are not supported in templates. So, passing as a parameter. Backticks string } @@ -65,15 +75,35 @@ func write(dataObj any, outputFile, templateFile string) (err error) { err = closeErr } }() + // Define the custom function map. + funcMap := template.FuncMap{ + "formatValue": formatValue, + "title": cases.Title(language.English).String, + } file := path.Base(templateFile) var tmpl *template.Template - if tmpl, err = template.New(file).ParseFiles(templateFile); err != nil { + if tmpl, err = template.New(file).Funcs(funcMap).ParseFiles(templateFile); err != nil { return err } return tmpl.Execute(outF, dataObj) } +// invertMachineTypeGroups takes the parsed map of group->machines +// and returns a map of machine->group. +func invertMachineTypeGroups(groups map[string][]string) map[string]string { + inverted := make(map[string]string) + for groupName, machineTypes := range groups { + for _, machineType := range machineTypes { + if alreadyMappedGroup, ok := inverted[machineType]; ok { + panic(fmt.Sprintf("machine type %q mapped to multiple groups, %q and %q", machineType, alreadyMappedGroup, groupName)) + } + inverted[machineType] = groupName + } + } + return inverted +} + func main() { flag.Parse() err := validateFlags() @@ -81,17 +111,17 @@ func main() { panic(err) } - paramsConfig, err := parseParamsConfig() + paramsYAML, err := parseParamsYAML() if err != nil { panic(err) } - td, err := constructTypeTemplateData(paramsConfig) + td, err := constructTypeTemplateData(paramsYAML.Params) if err != nil { panic(err) } - fd, err := computeFlagTemplateData(paramsConfig) + fd, err := computeFlagTemplateData(paramsYAML.Params) if err != nil { panic(err) } @@ -104,15 +134,44 @@ func main() { return cmp.Compare(i.FlagName, j.FlagName) }) - err = write(templateData{ - FlagTemplateData: fd, - TypeTemplateData: td, - Backticks: "`", - }, - path.Join(*outDir, "config.go"), - path.Join(*templateDir, "config.tpl")) - if err != nil { - panic(err) + // Create a map from given machine type to all the machine type groups that it belongs to. + machineTypeToGroupMap := invertMachineTypeGroups(paramsYAML.MachineTypeGroups) + + for _, rootFileName := range []string{"config", "config_test"} { + generatedFilePath := path.Join(*outDir, rootFileName+".go") + templateFilePath := path.Join(*templateDir, rootFileName+".tpl") + err = write(templateData{ + FlagTemplateData: fd, + TypeTemplateData: td, + MachineTypeToGroupMap: machineTypeToGroupMap, + MachineTypeGroups: paramsYAML.MachineTypeGroups, + Backticks: "`", + }, + generatedFilePath, templateFilePath) + if err != nil { + panic(fmt.Sprintf("failed to generate file %q: %v", generatedFilePath, err)) + } } +} +// formatValue is a custom template function that correctly formats values for Go code. +// It adds quotes to strings and leaves other types as-is. +// Special case: if a string looks like a function call (ends with ()), it's output as-is. +func formatValue(v any) string { + rv := reflect.ValueOf(v) + switch rv.Kind() { + case reflect.String: + s := v.(string) + // Check if it looks like a function call - if so, output as-is without quotes + // To make it more robust, check that it starts with an uppercase letter as well. + // As the function shoud be exported only. + if len(s) > 2 && s[len(s)-2:] == "()" && unicode.IsUpper(rune(s[0])) { + return s + } + // Use %q to safely quote strings, e.g., "my-string" + return fmt.Sprintf("%q", v) + default: + // Use %v for other types like int, bool, etc. + return fmt.Sprintf("%v", v) + } } diff --git a/tools/config-gen/main_test.go b/tools/config-gen/main_test.go new file mode 100644 index 0000000000..36ec704b7b --- /dev/null +++ b/tools/config-gen/main_test.go @@ -0,0 +1,99 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInvertMachineTypeGroups(t *testing.T) { + testCases := []struct { + name string + input map[string][]string + expected map[string]string + expectedError bool + }{ + { + name: "EmptyMap", + input: map[string][]string{}, + expected: map[string]string{}, + }, + { + name: "OneGroupOneMachine", + input: map[string][]string{ + "group1": {"machine1"}, + }, + expected: map[string]string{ + "machine1": "group1", + }, + }, + { + name: "OneGroupMultipleMachines", + input: map[string][]string{ + "group1": {"machine1", "machine2"}, + }, + expected: map[string]string{ + "machine1": "group1", + "machine2": "group1", + }, + }, + { + name: "MultipleGroupsOneMachine", + input: map[string][]string{ + "group1": {"machine1"}, + "group2": {"machine1"}, + }, + expectedError: true, + }, + { + name: "MultipleGroupsMultipleMachines", + input: map[string][]string{ + "group1": {"machine1", "machine2"}, + "group2": {"machine2", "machine3"}, + }, + expectedError: true, + }, + { + name: "ComplexCase", + input: map[string][]string{ + "high_cpu": {"c2-standard-8", "c2-standard-16"}, + "high_memory": {"m1-megamem-96", "m2-megamem-416"}, + "general": {"c2-standard-8", "m1-megamem-96"}, + }, + expectedError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + defer func() { + r := recover() + if tc.expectedError && r == nil { + t.Errorf("Expected error, but got none.") + } else if !tc.expectedError && r != nil { + t.Errorf("Unexpectd error: %v", r) + } + }() + machineTypeToGroupMap := invertMachineTypeGroups(tc.input) + if !tc.expectedError { + assert.Equalf(t, machineTypeToGroupMap, tc.expected, "invertMachineTypeGroups() = %v, want %v", machineTypeToGroupMap, tc.expected) + } + }) + } +} diff --git a/tools/config-gen/parser.go b/tools/config-gen/parser.go index d1b4db647d..9b78f86f3e 100644 --- a/tools/config-gen/parser.go +++ b/tools/config-gen/parser.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,14 +17,24 @@ package main import ( - "bytes" "fmt" "os" + "regexp" "slices" + "strings" + "github.com/googlecloudplatform/gcsfuse/v3/cfg/shared" "gopkg.in/yaml.v3" ) +const ( + machineTypeGroupRegexPattern = `^[a-z]+(-[a-z0-9]+)*$` +) + +var ( + machineTypeGroupRegex = regexp.MustCompile(machineTypeGroupRegexPattern) +) + type Param struct { FlagName string `yaml:"flag-name"` Shorthand string @@ -34,28 +44,58 @@ type Param struct { IsDeprecated bool `yaml:"deprecated"` DeprecationWarning string `yaml:"deprecation-warning"` Usage string - HideFlag bool `yaml:"hide-flag"` - HideShorthand bool `yaml:"hide-shorthand"` + HideFlag bool `yaml:"hide-flag"` + HideShorthand bool `yaml:"hide-shorthand"` + Optimizations *shared.OptimizationRules `yaml:"optimizations,omitempty"` +} + +// ParamsYAML mirrors the params.yaml file itself. +type ParamsYAML struct { + Params []Param `yaml:"params"` + MachineTypeGroups map[string][]string `yaml:"machine-type-groups"` +} + +func parseParamsYAMLStr(paramsYAMLStr string) (paramsYAML ParamsYAML, err error) { + dec := yaml.NewDecoder(strings.NewReader(paramsYAMLStr)) + dec.KnownFields(true) + if err = dec.Decode(¶msYAML); err != nil { + return ParamsYAML{}, err + } + if err = validateParams(paramsYAML.Params); err != nil { + return ParamsYAML{}, err + } + if err = validateMachineTypeGroups(paramsYAML.MachineTypeGroups); err != nil { + return ParamsYAML{}, err + } + return paramsYAML, nil } -func parseParamsConfig() ([]Param, error) { +func parseParamsYAML() (ParamsYAML, error) { buf, err := os.ReadFile(*paramsFile) if err != nil { - return nil, err + return ParamsYAML{}, err } - var paramsConfig []Param - dec := yaml.NewDecoder(bytes.NewReader(buf)) - dec.KnownFields(true) - if err = dec.Decode(¶msConfig); err != nil { - return nil, err + return parseParamsYAMLStr(string(buf)) +} + +func checkFlagName(name string) error { + if name == "" { + return fmt.Errorf("flag-name cannot be empty") } - if err = validateParams(paramsConfig); err != nil { - return nil, err + + // A valid name should contain only lower-case characters with hyphens as + // separators. It must start and end with an alphabet. + regex := `^[a-z]+([-_][a-z]+)*$` + if matched, _ := regexp.MatchString(regex, name); !matched { + return fmt.Errorf("flag-name %q does not conform to the regex: %s", name, regex) } - return paramsConfig, nil + return nil } func validateParam(param Param) error { + if err := checkFlagName(param.FlagName); err != nil { + return err + } if param.IsDeprecated && param.DeprecationWarning == "" { return fmt.Errorf("param %s is marked deprecated but deprecation-warning is not set", param.FlagName) } @@ -76,7 +116,7 @@ func validateParam(param Param) error { // Validate the data type. idx := slices.IndexFunc( []string{"int", "float64", "bool", "string", "duration", "octal", "[]int", - "[]string", "logSeverity", "protocol", "resolvedPath"}, + "[]string", "logSeverity", "protocol", "resolvedPath", "directPathStrategy"}, func(dt string) bool { return dt == param.Type }, @@ -85,6 +125,17 @@ func validateParam(param Param) error { return fmt.Errorf("unsupported datatype: %s", param.Type) } + // Validate bucket-based optimizations if present. + if param.Optimizations != nil { + validBucketTypes := []string{"zonal", "hierarchical", "flat", "pirlo"} + for _, bto := range param.Optimizations.BucketTypeOptimization { + if !slices.Contains(validBucketTypes, bto.BucketType) { + return fmt.Errorf("invalid bucket-type %q for flag %s; must be one of: %v", + bto.BucketType, param.FlagName, validBucketTypes) + } + } + } + return nil } @@ -136,3 +187,57 @@ func validateForDuplicates(params []Param, fn func(param Param) string) error { } return nil } + +// validateMachineTypeGroups validates the machine-type-groups map. +func validateMachineTypeGroups(groups map[string][]string) error { + // Temporary map to check if a machine-type belong to multiple groups. + machineTypeToGroupMap := make(map[string]string) + // Note: We can't easily validate that the group names themselves are sorted + // in the YAML file because Go maps do not preserve insertion order. This + // should be enforced through code reviews or a linter. + for groupName, machineTypes := range groups { + // 1. Validate group name format (e.g., kebab-case). + if !machineTypeGroupRegex.MatchString(groupName) { + return fmt.Errorf("group name %q does not conform to machineTypeGroupRegexPattern: %q", groupName, machineTypeGroupRegexPattern) + } + + if len(machineTypes) == 0 { + return fmt.Errorf("group %q must contain at least one machine type", groupName) + } + + // 2. Validate machine types within the group are sorted and unique. + if !slices.IsSorted(machineTypes) { + return fmt.Errorf("machine types in group %q are not sorted alphabetically", groupName) + } + if err := validateForDuplicatesInSortedSlice(machineTypes); err != nil { + return fmt.Errorf("duplicate machine type found in group %q: %w", groupName, err) + } + // Check for cross-group uniqueness for each machine type. + for _, machineType := range machineTypes { + if existingGroup, ok := machineTypeToGroupMap[machineType]; ok { + return fmt.Errorf( + "machine type %q cannot be in multiple groups; it is in both %q and %q", + machineType, + existingGroup, + groupName, + ) + } + machineTypeToGroupMap[machineType] = groupName + } + } + + return nil +} + +// validateForDuplicatesInSortedSlice is a helper to check for duplicates in an already sorted string slice. +func validateForDuplicatesInSortedSlice(items []string) error { + for i, item := range items { + if item == "" { + return fmt.Errorf("item cannot be an empty string") + } + if i > 0 && item == items[i-1] { + return fmt.Errorf("%q is present more than once", item) + } + } + return nil +} diff --git a/tools/config-gen/parser_test.go b/tools/config-gen/parser_test.go new file mode 100644 index 0000000000..562d3abc49 --- /dev/null +++ b/tools/config-gen/parser_test.go @@ -0,0 +1,435 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "strings" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg/shared" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCheckFlagName_Valid(t *testing.T) { + validNames := []string{"a", "abc", "ab-c", "ab-c-d", "a_b"} + + for _, name := range validNames { + t.Run(name, func(t *testing.T) { + assert.NoError(t, checkFlagName(name)) + }) + } +} + +func TestCheckFlagName_Invalid(t *testing.T) { + invalidNames := []string{"", "a-", "-a", "a--b", "a-b-", "A-b", "a.b", "1-a"} + + for _, name := range invalidNames { + t.Run(name, func(t *testing.T) { + assert.Error(t, checkFlagName(name)) + }) + } +} + +func TestValidateMachineTypeGroups(t *testing.T) { + testCases := []struct { + name string + input map[string][]string + expectErr bool + errContains string + }{ + { + name: "Valid_groups", + input: map[string][]string{ + "another-group": {"gce-vm"}, + "high-performance": {"a2-megagpu-16g", "a3-highgpu-8g"}, + }, + expectErr: false, + }, + { + name: "Empty_groups_map", + input: map[string][]string{}, + expectErr: false, + }, + { + name: "Invalid_group_name_format_(snake_case)", + input: map[string][]string{ + "invalid_group": {"vm"}, + }, + expectErr: true, + errContains: "does not conform", + }, + { + name: "Invalid_group_name_format_(PascalCase)", + input: map[string][]string{ + "InvalidGroup": {"vm"}, + }, + expectErr: true, + errContains: "does not conform", + }, + { + name: "Empty_machine_type_list", + input: map[string][]string{ + "a-valid-group": {}, + }, + expectErr: true, + errContains: "must contain at least one machine type", + }, + { + name: "Unsorted_machine_types_in_a_group", + input: map[string][]string{ + "a-valid-group": {"z-vm", "a-vm"}, + }, + expectErr: true, + errContains: "machine types in group \"a-valid-group\" are not sorted", + }, + { + name: "Duplicate_machine_types_in_a_group", + input: map[string][]string{ + "a-valid-group": {"a-vm", "a-vm", "z-vm"}, + }, + expectErr: true, + errContains: "duplicate machine type found in group \"a-valid-group\"", + }, + { + name: "a_machine_type_in_multiple_groups", + input: map[string][]string{ + "a-valid-group": {"a-vm", "b-vm"}, + "another-valid-group": {"a-vm", "c-vm"}, + }, + expectErr: true, + errContains: "cannot be in multiple groups", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := validateMachineTypeGroups(tc.input) + + if tc.expectErr { + assert.Error(t, err) + if tc.errContains != "" { + assert.Contains(t, err.Error(), tc.errContains) + } + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestValidateForDuplicatesInSortedSlice(t *testing.T) { + testCases := []struct { + name string + input []string + expectErr bool + }{ + { + name: "Slice_with_unique_strings", + input: []string{"a", "b", "c"}, + expectErr: false, + }, + { + name: "Empty_slice", + input: []string{}, + expectErr: false, + }, + { + name: "Slice_with_duplicate_strings", + input: []string{"a", "b", "b"}, + expectErr: true, + }, + { + name: "Slice_with_an_empty_string", + input: []string{"", "c", "c"}, + expectErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := validateForDuplicatesInSortedSlice(tc.input) + + if tc.expectErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestParseParamsYAMLStr_Success(t *testing.T) { + // ARRANGE + yamlContent := ` +machine-type-groups: + high-performance: + - "a2-megagpu-16g" + - "a3-highgpu-8g" + low-latency: + - "c2-standard-4" +params: + - config-path: "app-name" + flag-name: "app-name" + type: "string" + default: "gcsfuse" + "usage": "Application name" + - config-path: "file-system.enable-kernel-reader" + flag-name: "enable-kernel-reader" + type: "bool" + default: false + "usage": "Whether to enable kernel-based reader" + optimizations: + bucket-type-optimization: + - bucket-type: zonal + value: true + - bucket-type: hierarchical + value: false + - bucket-type: flat + value: true + - bucket-type: pirlo + value: true + - config-path: "file-system.max-read-ahead-kb" + flag-name: "max-read-ahead-kb" + type: "int" + default: "128" + "usage": "Maximum read ahead in KB" + optimizations: + bucket-type-optimization: + - bucket-type: zonal + value: 1024 + - bucket-type: hierarchical + value: 2048 + - bucket-type: flat + value: 2048 + - bucket-type: pirlo + value: 2048 + machine-based-optimization: + - group: high-performance + value: 2048 + profiles: + - name: aiml-training + value: 4096 + - config-path: "implicit-dirs" + flag-name: "implicit-dirs" + type: "bool" + default: false + "usage": "Whether or not to enable implicit directories" + optimizations: + machine-based-optimization: + - group: high-performance + value: true + - config-path: "metadata-cache.ttl-secs" + flag-name: "metadata-cache-ttl-secs" + type: "int" + default: "60" + "usage": "Metadata cache TTL in seconds" + optimizations: + machine-based-optimization: + - group: high-performance + value: -1 + profiles: + - name: aiml-training + value: -1 +` + + // ACT + parsedYAML, err := parseParamsYAMLStr(yamlContent) + + // ASSERT + require.NoError(t, err) + + t.Run("TestMachineTypeGroupsParsing", func(t *testing.T) { + expectedGroups := map[string][]string{ + "high-performance": {"a2-megagpu-16g", "a3-highgpu-8g"}, + "low-latency": {"c2-standard-4"}, + } + assert.Equal(t, expectedGroups, parsedYAML.MachineTypeGroups) + }) + + t.Run("TestParamWithOnlyBucketBasedOptimizations", func(t *testing.T) { + param := parsedYAML.Params[1] + require.NotNil(t, param.Optimizations) + expected := &shared.OptimizationRules{ + BucketTypeOptimization: []shared.BucketTypeOptimization{ + {BucketType: "zonal", Value: true}, + {BucketType: "hierarchical", Value: false}, + {BucketType: "flat", Value: true}, + {BucketType: "pirlo", Value: true}, + }, + } + assert.Equal(t, "file-system.enable-kernel-reader", param.ConfigPath) + assert.Equal(t, expected.BucketTypeOptimization, param.Optimizations.BucketTypeOptimization) + assert.Nil(t, param.Optimizations.Profiles) + assert.Nil(t, param.Optimizations.MachineBasedOptimization) + }) + + t.Run("TestParamWithAllOptimizationTypes", func(t *testing.T) { + param := parsedYAML.Params[2] + require.NotNil(t, param.Optimizations) + expected := &shared.OptimizationRules{ + BucketTypeOptimization: []shared.BucketTypeOptimization{ + {BucketType: "zonal", Value: 1024}, + {BucketType: "hierarchical", Value: 2048}, + {BucketType: "flat", Value: 2048}, + {BucketType: "pirlo", Value: 2048}, + }, + MachineBasedOptimization: []shared.MachineBasedOptimization{ + {Group: "high-performance", Value: 2048}, + }, + Profiles: []shared.ProfileOptimization{ + {Name: "aiml-training", Value: 4096}, + }, + } + assert.Equal(t, "file-system.max-read-ahead-kb", param.ConfigPath) + assert.Equal(t, expected.BucketTypeOptimization, param.Optimizations.BucketTypeOptimization) + assert.Equal(t, expected.MachineBasedOptimization, param.Optimizations.MachineBasedOptimization) + assert.Equal(t, expected.Profiles, param.Optimizations.Profiles) + }) + + t.Run("TestParamWithOnlyMachineBasedOptimizations", func(t *testing.T) { + param := parsedYAML.Params[3] + require.NotNil(t, param.Optimizations) + expected := &shared.OptimizationRules{ + MachineBasedOptimization: []shared.MachineBasedOptimization{ + {Group: "high-performance", Value: true}, + }, + } + assert.Equal(t, "implicit-dirs", param.ConfigPath) + assert.Equal(t, expected.MachineBasedOptimization, param.Optimizations.MachineBasedOptimization) + assert.Nil(t, param.Optimizations.Profiles) + }) + + t.Run("TestParamWithMixedOptimizations", func(t *testing.T) { + param := parsedYAML.Params[4] + require.NotNil(t, param.Optimizations) + expected := &shared.OptimizationRules{ + MachineBasedOptimization: []shared.MachineBasedOptimization{ + {Group: "high-performance", Value: -1}, + }, + Profiles: []shared.ProfileOptimization{ + { + Name: "aiml-training", + Value: -1, + }, + }, + } + assert.Equal(t, "metadata-cache.ttl-secs", param.ConfigPath) + assert.Equal(t, expected, param.Optimizations) + }) + + t.Run("TestParamWithNoOptimizations", func(t *testing.T) { + param := parsedYAML.Params[0] + assert.Equal(t, "app-name", param.ConfigPath) + assert.Nil(t, param.Optimizations) + }) +} + +func TestParseParamsYAMLStr_Negative(t *testing.T) { + testCases := []struct { + name string + yamlContent string + expectedErrorSubstring string + }{ + { + name: "MalformedYAML", + yamlContent: ` +params: + - config-path: "a" + - config-path: "b" # Bad indentation +`, + expectedErrorSubstring: "did not find expected '-' indicator", + }, + { + name: "DuplicateFlagName", + yamlContent: ` +params: + - flag-name: "my-flag" + config-path: "a" + - flag-name: "my-flag" + config-path: "b" +`, + expectedErrorSubstring: "duplicate", + }, + { + name: "InvalidGroupName", + yamlContent: ` +machine-type-groups: + Invalid_Group_Name: + - "a-machine" +`, + expectedErrorSubstring: "group name \"Invalid_Group_Name\" does not conform", + }, + { + name: "UnsortedMachineTypesInGroup", + yamlContent: ` +machine-type-groups: + my-group: + - "z-machine" + - "a-machine" +`, + expectedErrorSubstring: "machine types in group \"my-group\" are not sorted alphabetically", + }, + { + name: "DuplicateMachineTypeInGroup", + yamlContent: ` +machine-type-groups: + my-group: + - "a-machine" + - "a-machine" +`, + expectedErrorSubstring: "duplicate machine type found in group \"my-group\"", + }, + { + name: "EmptyMachineTypeList", + yamlContent: ` +machine-type-groups: + my-group: [] +`, + expectedErrorSubstring: "group \"my-group\" must contain at least one machine type", + }, + { + name: "UnsupportedBucketType", + yamlContent: ` +params: + - config-path: "test-param" + flag-name: "test-flag" + type: "bool" + default: false + usage: "Test flag for bucket type validation" + optimizations: + bucket-type-optimization: + - bucket-type: "invalid-bucket-type" + value: true +`, + expectedErrorSubstring: "invalid bucket-type", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // ARRANGE + + // ACT + _, err := parseParamsYAMLStr(tc.yamlContent) + + // ASSERT + require.Error(t, err) + require.True(t, strings.Contains(err.Error(), tc.expectedErrorSubstring), "Expected error to contain %q, but got: %q", tc.expectedErrorSubstring, err.Error()) + }) + } +} diff --git a/tools/config-gen/templates/config.tpl b/tools/config-gen/templates/config.tpl index 92d631ba30..600c84e53e 100644 --- a/tools/config-gen/templates/config.tpl +++ b/tools/config-gen/templates/config.tpl @@ -21,8 +21,95 @@ import ( "github.com/spf13/pflag" "github.com/spf13/viper" + "github.com/googlecloudplatform/gcsfuse/v3/cfg/shared" ) +// AllFlagOptimizationRules is the generated map from a flag's config-path to its specific rules. +var AllFlagOptimizationRules = map[string]shared.OptimizationRules{ +{{- range .FlagTemplateData }} + {{- if .Optimizations }} + {{- $goType := .GoType -}} + "{{ .ConfigPath }}": { + {{- if .Optimizations.MachineBasedOptimization }} + MachineBasedOptimization: []shared.MachineBasedOptimization{ + {{- range .Optimizations.MachineBasedOptimization }} + { + Group: "{{ .Group }}", + Value: {{$goType}}({{ formatValue .Value }}), + }, + {{- end }} + }, + {{- end }} + {{- if .Optimizations.BucketTypeOptimization }} + BucketTypeOptimization: []shared.BucketTypeOptimization{ + {{- range .Optimizations.BucketTypeOptimization }} + { + BucketType: "{{ .BucketType }}", + Value: {{$goType}}({{ formatValue .Value }}), + }, + {{- end }} + }, + {{- end }} + {{- if .Optimizations.Profiles }} + Profiles: []shared.ProfileOptimization{ + {{- range .Optimizations.Profiles }} + { + Name: "{{ .Name }}", + Value: {{$goType}}({{ formatValue .Value }}), + }, + {{- end }} + }, + {{- end }} + }, + {{- end }} +{{- end }} +} + +// machineTypeToGroupMap is the generated map from machine type to the group it belongs to. +var machineTypeToGroupMap = map[string]string{ +{{- range $machineType, $group := .MachineTypeToGroupMap }} + "{{ $machineType }}": "{{ $group }}", +{{- end }} +} + +// ApplyOptimizations modifies the config in-place with optimized values. +// input parameter is optional and provides runtime context for optimizations +// such as bucket type. Pass nil if not available. +func (c *Config) ApplyOptimizations(v *viper.Viper, input *OptimizationInput) map[string]OptimizationResult { + var optimizedFlags = make(map[string]OptimizationResult) + // Skip all optimizations if autoconfig is disabled. + if c.DisableAutoconfig { + return nil + } + + profileName := c.Profile + machineType, err := getMachineType(v) + if err != nil { + // Non-fatal, just means machine-based optimizations won't apply. + machineType = "" + } + c.MachineType = machineType + + // Apply optimizations for each flag that has rules defined. +{{- range .FlagTemplateData }} +{{- if .Optimizations }} + if !v.IsSet("{{ .ConfigPath }}") { + rules := AllFlagOptimizationRules["{{ .ConfigPath }}"] + result := getOptimizedValue(&rules, c.{{ .GoPath }}, profileName, machineType, input, machineTypeToGroupMap) + if result.Optimized { + if val, ok := result.FinalValue.({{ .GoType }}); ok { + if c.{{ .GoPath }} != val { + c.{{ .GoPath }} = val + optimizedFlags["{{ .ConfigPath }}"] = result + } + } + } + } +{{- end }} +{{- end }} + return optimizedFlags +} + {{$bt := .Backticks}} {{range .TypeTemplateData}} type {{ .TypeName}} struct { diff --git a/tools/config-gen/templates/config_test.tpl b/tools/config-gen/templates/config_test.tpl new file mode 100644 index 0000000000..c5e6a75dae --- /dev/null +++ b/tools/config-gen/templates/config_test.tpl @@ -0,0 +1,230 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// GENERATED CODE - DO NOT EDIT MANUALLY. + +package cfg + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/spf13/viper" +) + +func TestApplyOptimizations(t *testing.T) { +{{- range .FlagTemplateData }} +{{- if .Optimizations }} + {{- $flag := . }} + // Tests for {{ $flag.ConfigPath }} + t.Run("{{$flag.ConfigPath}}", func(t *testing.T) { + testCases := []struct { + name string + config Config + userSetFlags map[string]any + input *OptimizationInput + expectOptimized bool + expectedValue any + }{ + { + name: "user_set", + config: Config{ + {{- if .Optimizations.Profiles }} + {{- $profile := index .Optimizations.Profiles 0 }} + Profile: "{{$profile.Name}}", + {{- end }} + }, + userSetFlags: map[string]any{ + "{{$flag.ConfigPath}}": {{- if eq $flag.GoType "int64" }} 98765, + {{- else if eq $flag.GoType "bool" }} {{if eq (printf "%v" $flag.DefaultValue) "true"}}false{{else}}true{{end}}, + {{- else if eq $flag.GoType "string" }} {{$flag.DefaultValue}} + "-non-default", + {{- else if eq $flag.GoType "float64" }} {{$flag.DefaultValue}} + 1.23, + {{- else }} true, + {{- end }} + {{- if .Optimizations.MachineBasedOptimization }} + {{- $mbo := index .Optimizations.MachineBasedOptimization 0 }} + {{- $machineType := index $.MachineTypeGroups $mbo.Group 0 }} + "machine-type": "{{$machineType}}", + {{- else }} + "machine-type": "a2-megagpu-16g", + {{- end }} + }, + {{- if .Optimizations.BucketTypeOptimization }} + {{- $bto := index .Optimizations.BucketTypeOptimization 0 }} + input: &OptimizationInput{BucketType: BucketType{{ $bto.BucketType | title }}}, + {{- else }} + input: nil, + {{- end }} + expectOptimized: false, + expectedValue: + {{- if eq $flag.GoType "int64" }} int64(98765), + {{- else if eq $flag.GoType "bool" }} {{if eq (printf "%v" $flag.DefaultValue) "true"}}false{{else}}true{{end}}, + {{- else if eq $flag.GoType "string" }} {{$flag.DefaultValue}} + "-non-default", + {{- else if eq $flag.GoType "float64" }} {{$flag.DefaultValue}} + 1.23, + {{- else }} // compilation error: unhandled type '{{$flag.GoType}}' in test generation for {{$flag.ConfigPath}} + {{- end }} + }, + { + name: "no_optimization", + config: Config{Profile: "non_existent_profile"}, + userSetFlags: map[string]any{ + "machine-type": "low-end-machine", + }, + input: nil, + expectOptimized: false, + expectedValue: {{$flag.DefaultValue}}, + }, + {{- range .Optimizations.Profiles }} + { + name: "profile_{{.Name}}", + config: Config{Profile: "{{.Name}}"}, + userSetFlags: map[string]any{}, + input: nil, + expectOptimized: {{if ne (printf "%v" .Value) (printf "%v" $flag.DefaultValue)}}true{{else}}false{{end}}, + expectedValue: {{.Value}}, + }, + {{- end }} + {{- range .Optimizations.MachineBasedOptimization }} + {{- $mbo := . }} + {{- $machineType := index $.MachineTypeGroups $mbo.Group 0 }} + { + name: "machine_group_{{$mbo.Group}}", + config: Config{Profile: ""}, + userSetFlags: map[string]any{ + "machine-type": "{{$machineType}}", + }, + input: nil, + expectOptimized: {{if ne (printf "%v" $mbo.Value) (printf "%v" $flag.DefaultValue)}}true{{else}}false{{end}}, + expectedValue: {{$mbo.Value}}, + }, + {{- end }} + {{- range .Optimizations.BucketTypeOptimization }} + {{- $bto := . }} + { + name: "bucket_type_{{$bto.BucketType}}", + config: Config{Profile: ""}, + userSetFlags: map[string]any{}, + input: &OptimizationInput{BucketType: BucketType{{ $bto.BucketType | title }}}, + expectOptimized: {{if ne (printf "%v" $bto.Value) (printf "%v" $flag.DefaultValue)}}true{{else}}false{{end}}, + expectedValue: {{$bto.Value}}, + }, + {{- end }} + {{- if and .Optimizations.Profiles .Optimizations.MachineBasedOptimization }} + {{- $profile := index .Optimizations.Profiles 0 -}} + {{- $mbo := index .Optimizations.MachineBasedOptimization 0 -}} + {{- $machineType := index $.MachineTypeGroups $mbo.Group 0 }} + { + name: "profile_overrides_machine_type", + config: Config{Profile: "{{$profile.Name}}"}, + userSetFlags: map[string]any{ + "machine-type": "{{$machineType}}", + }, + input: nil, + expectOptimized: {{if ne (printf "%v" $profile.Value) (printf "%v" $flag.DefaultValue)}}true{{else}}false{{end}}, + expectedValue: {{$profile.Value}}, + }, + {{- end }} + {{- if and .Optimizations.Profiles .Optimizations.BucketTypeOptimization }} + {{- $profile := index .Optimizations.Profiles 0 -}} + {{- $bto := index .Optimizations.BucketTypeOptimization 0 -}} + { + name: "profile_overrides_bucket_type", + config: Config{Profile: "{{$profile.Name}}"}, + userSetFlags: map[string]any{}, + input: &OptimizationInput{BucketType: BucketType{{ $bto.BucketType | title }}}, + expectOptimized: {{if ne (printf "%v" $profile.Value) (printf "%v" $flag.DefaultValue)}}true{{else}}false{{end}}, + expectedValue: {{$profile.Value}}, + }, + {{- end }} + {{- if and .Optimizations.MachineBasedOptimization .Optimizations.BucketTypeOptimization }} + {{- $mbo := index .Optimizations.MachineBasedOptimization 0 -}} + {{- $bto := index .Optimizations.BucketTypeOptimization 0 -}} + {{- $machineType := index $.MachineTypeGroups $mbo.Group 0 }} + { + name: "machine_type_overrides_bucket_type", + config: Config{Profile: ""}, + userSetFlags: map[string]any{ + "machine-type": "{{$machineType}}", + }, + input: &OptimizationInput{BucketType: BucketType{{ $bto.BucketType | title }}}, + expectOptimized: {{if ne (printf "%v" $mbo.Value) (printf "%v" $flag.DefaultValue)}}true{{else}}false{{end}}, + expectedValue: {{$mbo.Value}}, + }, + {{- end }} + {{- if .Optimizations.MachineBasedOptimization }} + {{- $mbo := index .Optimizations.MachineBasedOptimization 0 -}} + {{- $machineType := index $.MachineTypeGroups $mbo.Group 0 -}} + { + name: "fallback_to_machine_type_with_non_existent_profile", + config: Config{Profile: "non_existent_profile"}, + userSetFlags: map[string]any{ + "machine-type": "{{$machineType}}", + }, + input: nil, + expectOptimized: {{if ne (printf "%v" $mbo.Value) (printf "%v" $flag.DefaultValue)}}true{{else}}false{{end}}, + expectedValue: {{$mbo.Value}}, + }, + {{- $unrelatedProfile := "aiml-training" -}} + {{- $hasRuleForUnrelatedProfile := false -}} + {{- range .Optimizations.Profiles -}} + {{- if eq .Name $unrelatedProfile -}} + {{- $hasRuleForUnrelatedProfile = true -}} + {{- end -}} + {{- end -}} + {{- if not $hasRuleForUnrelatedProfile -}} + { + name: "fallback_to_machine_type_when_aiml-training_is_unrelated", + config: Config{Profile: "{{$unrelatedProfile}}"}, + userSetFlags: map[string]any{ + "machine-type": "{{$machineType}}", + }, + input: nil, + expectOptimized: {{if ne (printf "%v" $mbo.Value) (printf "%v" $flag.DefaultValue)}}true{{else}}false{{end}}, + expectedValue: {{$mbo.Value}}, + }, + {{- end }} + {{- end }} + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // We need a copy of the config for each test case. + c := tc.config + // Set the default or non-default value on the config object. + if tc.name == "user_set" { + c.{{$flag.GoPath}} = tc.expectedValue.({{$flag.GoType}}) + } else { + c.{{$flag.GoPath}} = {{$flag.DefaultValue}} + } + + v := viper.New() + for key, val := range tc.userSetFlags { + v.Set(key, val) + } + + optimizedFlags := c.ApplyOptimizations(v, tc.input) + + if tc.expectOptimized { + assert.Contains(t, optimizedFlags, "{{$flag.ConfigPath}}") + } else { + assert.NotContains(t, optimizedFlags, "{{$flag.ConfigPath}}") + } + // Use EqualValues to handle the int vs int64 type mismatch for default values. + assert.EqualValues(t, tc.expectedValue, c.{{$flag.GoPath}}) + }) + } + }) +{{- end }} +{{- end }} +} diff --git a/tools/config-gen/type_template_data_gen.go b/tools/config-gen/type_template_data_gen.go index cad554f24e..a5e4f3c402 100644 --- a/tools/config-gen/type_template_data_gen.go +++ b/tools/config-gen/type_template_data_gen.go @@ -53,7 +53,7 @@ func capitalizeIdentifier(name string) (string, error) { // For the purposes of capitalization, both "." and "-" are equivalent. name = strings.ReplaceAll(name, ".", "-") var buf strings.Builder - for _, w := range strings.Split(name, "-") { + for w := range strings.SplitSeq(name, "-") { // Capitalize the first letter and concatenate. buf.WriteString(cases.Title(language.English).String(w)) } @@ -76,6 +76,8 @@ func getGoDataType(dt string) string { return "int64" case "[]int": return "[]int64" + case "directPathStrategy": + return "DirectPathStrategy" default: return dt } diff --git a/tools/containerize_gcsfuse_docker/Dockerfile b/tools/containerize_gcsfuse_docker/Dockerfile index 8afd60fe03..e756d7d4d5 100644 --- a/tools/containerize_gcsfuse_docker/Dockerfile +++ b/tools/containerize_gcsfuse_docker/Dockerfile @@ -29,12 +29,12 @@ # (b) using ubuntu/debain image # E.g. # > docker run -it --privileged -v $HOME/key.json:/key.json -v /mnt/gcs:/gcs:rw,rshared -e BUCKET_NAME="my-bucket-name" gcsfuse-ubuntu:v0.41.7 - +ARG GO_VERSION ARG OS_VERSION ARG OS_NAME # Image with gcsfuse installed and its package (.deb) -FROM golang:1.23.3 as gcsfuse-package +FROM golang:${GO_VERSION} as gcsfuse-package RUN apt-get update -qq && apt-get install -y ruby ruby-dev rubygems build-essential rpm fuse && gem install --no-document bundler diff --git a/tools/gcsfuse-scc-gc/README.md b/tools/gcsfuse-scc-gc/README.md new file mode 100644 index 0000000000..c6bee0e4c9 --- /dev/null +++ b/tools/gcsfuse-scc-gc/README.md @@ -0,0 +1,108 @@ +# GCSFuse Shared Chunk Cache (SCC) Garbage Collector + +Removes least recently used files from GCSFuse shared cache to maintain target size. + +## Build + +```bash +cd tools/gcsfuse-scc-gc +go build . +``` + +## Usage + +```bash +./gcsfuse-scc-gc -cache-dir=/mnt/nfs-cache -target-size-mb=10240 +``` + +**Options:** +- `-cache-dir` (required): Path to cache directory +- `-target-size-mb`: Target size in MB (default: 10240) +- `-concurrency`: Maximum concurrent file operations (default: 16) +- `-dry-run`: Show what would be deleted +- `-debug`: Enable debug logging + +## File System Requirements + +This garbage collector tool requires a file system that supports: + +1. **Atomic rename**: The tool uses atomic rename operations to safely expire cache files (renaming `.bin` to `.bak`) without disrupting concurrent reads from multiple GCSFuse instances. + +2. **Access time (atime) tracking**: The LRU eviction algorithm depends on file access times to determine which chunks are least recently used. The file system should track atime, either through: + +**Note:** NFS and most POSIX-compliant file systems meet these requirements. + +## Scheduling + +### Cron (hourly) + +```bash +crontab -e +# Add: +0 * * * * /usr/local/bin/gcsfuse-scc-gc -cache-dir=/mnt/nfs-cache -target-size-mb=10240 2>&1 | logger -t gcsfuse-scc-gc +``` + +### SystemD Timer + +**Service:** `/etc/systemd/system/gcsfuse-scc-gc.service` +```ini +[Unit] +Description=GCSFuse Cache LRU Eviction + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/gcsfuse-scc-gc -cache-dir=/mnt/nfs-cache -target-size-mb=10240 +``` + +**Timer:** `/etc/systemd/system/gcsfuse-scc-gc.timer` +```ini +[Unit] +Description=Run GCSFuse Cache LRU hourly + +[Timer] +OnCalendar=hourly +RandomizedDelaySec=5min + +[Install] +WantedBy=timers.target +``` + +**Enable:** +```bash +sudo systemctl daemon-reload +sudo systemctl enable --now gcsfuse-scc-gc.timer +``` + +## How It Works + +1. Cleans up any leftover `.bak` files from the previous runs. +2. Scans cache directory for `.bin` files with atime and size +3. If total size < target, then exit without expiration or eviction. +4. Sorts by atime and selects oldest files to expire. +5. Renames selected files to `.bak` (kept until next run for ongoing reads). +6. Removes old `.tmp` files (older than 1 hour) +7. Cleans up empty directories. + +**Two-phase eviction:** Files are renamed to `.bak` on the current run and deleted on the next run. This ensure no +concurrent read and eviction of chunked file. + +## Cache Directory Structure + +The tool expects the cache to be organized in a `gcsfuse-shared-chunk-cache` subdirectory within the specified cache directory: +- Format: `<cache-dir>/gcsfuse-shared-chunk-cache/<2-char>/<2-char>/<full-hash>/<start>_<end>.bin` +- Automatically detects and uses this subdirectory structure +- Uses SHA256-based hashing for cache organization + +**Note:** This is separate from the regular GCSFuse file cache which uses `gcsfuse-file-cache` subdirectory. + +## Example + +```bash +./gcsfuse-scc-gc -cache-dir=/mnt/nfs-cache -target-size-mb=10240 -debug +``` +Output: +``` +time=2026-02-12T19:06:36.237Z level=INFO msg="Starting LRU cache eviction" cache_dir=/mnt/nfs-cache target_size_mb=10240 +time=2026-02-12T19:06:36.640Z level=DEBUG msg="Manifest created" files=100 total_size_mb=800 scan_duration=120.731382ms +time=2026-02-12T19:06:36.640Z level=INFO msg="Cache below target, nothing to do" cache_size_mb=800 target_size_mb=10240 +``` diff --git a/tools/gcsfuse-scc-gc/main.go b/tools/gcsfuse-scc-gc/main.go new file mode 100644 index 0000000000..1bd35e1d02 --- /dev/null +++ b/tools/gcsfuse-scc-gc/main.go @@ -0,0 +1,411 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** +* How does gcsfuse-scc-gc (GCSFuse Shared Chunk Cache Garbage Collector) work? +* +* 1. Performs a single walk to scan the cache directory, collecting: +* - `.bin` files (cache chunks) with atime/mtime and size +* - `.bak` files (previously expired files from last run) +* - `.tmp` files (incomplete downloads) +* - Directories (for cleanup) +* 2. Cleans up `.bak` files expired during the previous run. +* 3. Sorts `.bin` files by atime and selects oldest files to expire, only if cache size exceeds target. +* 4. Renames selected files to `.bak` (kept until next run for ongoing reads). +* 5. Removes old `.tmp` files (older than 1 hour). +* 6. Cleans up empty directories (deepest first). + */ + +package main + +import ( + "errors" + "flag" + "log/slog" + "os" + "path/filepath" + "slices" + "sync" + "sync/atomic" + "syscall" + "time" +) + +var ( + cacheDir = flag.String("cache-dir", "", "Path to the cache directory") + targetSize = flag.Int64("target-size-mb", 10240, "Target cache size in MB (default: 10GB)") + concurrency = flag.Int("concurrency", 16, "Maximum concurrent file operations (default: 16)") + dryRun = flag.Bool("dry-run", false, "Dry run mode - don't delete/expire files") + debug = flag.Bool("debug", false, "Enable debug logging") +) + +const ( + MiB = 1024 * 1024 +) + +type FileInfo struct { + Path string + Atime time.Time + Mtime time.Time + Size int64 +} + +type Manifest struct { + Files []FileInfo // .bin files + BakFiles []FileInfo // .bak files + TmpFiles []FileInfo // .tmp files + Dirs []string + TotalSize int64 // Total size of .bin files + ScanDuration time.Duration +} + +func main() { + flag.Parse() + + // Configure logging level + level := slog.LevelInfo + if *debug { + level = slog.LevelDebug + } + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: level}))) + + if *cacheDir == "" { + slog.Error("cache-dir is required") + os.Exit(1) + } + + slog.Info("Starting LRU cache eviction", "cache_dir", *cacheDir, "target_size_mb", *targetSize) + + // Step 1: Single walk to scan cache and collect all file types + manifest, err := scanCache(*cacheDir) + if err != nil { + slog.Error("Failed to scan cache", "error", err) + os.Exit(1) + } + slog.Debug("Cache scanned", + "bin_files", len(manifest.Files), + "total_size_mb", float64(manifest.TotalSize)/MiB, + "bak_files", len(manifest.BakFiles), + "tmp_files", len(manifest.TmpFiles), + "dirs", len(manifest.Dirs), + "scan_duration", manifest.ScanDuration) + + // Step 2: Clean up .bak files from previous run + if !*dryRun { + removeBakFiles(manifest.BakFiles) + } else if len(manifest.BakFiles) > 0 { + slog.Info("DRY RUN: Would remove previously expired files", "file_count", len(manifest.BakFiles)) + printFileInfo(manifest.BakFiles) + } + + targetBytes := *targetSize * MiB + if manifest.TotalSize > targetBytes { + // Step 3: Find LRU files to expire if we are above target size. + filesToExpire := findLRUFiles(manifest, targetBytes) + slog.Info("Expiring files", + "expired_size_mb", float64(manifest.TotalSize-targetBytes)/MiB, + "file_count", len(filesToExpire)) + + // Step 4: Expire files in parallel (rename to .bak) + if !*dryRun { + expireFiles(filesToExpire) + } else if len(filesToExpire) > 0 { + slog.Info("DRY RUN: Would expire files", "file_count", len(filesToExpire)) + printFileInfo(filesToExpire) + } + } else { + slog.Info("Cache below target, nothing to do", + "cache_size_mb", float64(manifest.TotalSize)/MiB, + "target_size_mb", float64(targetBytes)/MiB) + } + + // Step 5: Remove old .tmp files (older than 1 hour) + if !*dryRun { + removeOldTmpFiles(manifest.TmpFiles) + } else if len(manifest.TmpFiles) > 0 { + slog.Info("DRY RUN: Would remove old tmp files", "file_count", len(manifest.TmpFiles)) + printFileInfo(manifest.TmpFiles) + } + + // Step 6: Cleanup empty directories + if !*dryRun { + cleanupEmptyDirs(manifest.Dirs) + } else if len(manifest.Dirs) > 0 { + slog.Info("DRY RUN: Would cleanup empty directories", "dir_count", len(manifest.Dirs)) + for _, dir := range manifest.Dirs { + slog.Debug("Directory", "path", dir) + } + } + + slog.Info("LRU cache eviction completed") +} + +// printFileInfo is a helper to log file info for debugging. +func printFileInfo(info []FileInfo) { + for _, f := range info { + slog.Debug("FileInfo", + "path", f.Path, + "size_mb", float64(f.Size)/MiB, + "atime", f.Atime, + "mtime", f.Mtime) + } +} + +// scanCache performs a single walk to collect all file types +func scanCache(cacheDir string) (*Manifest, error) { + start := time.Now() + manifest := &Manifest{ + Files: make([]FileInfo, 0), + BakFiles: make([]FileInfo, 0), + TmpFiles: make([]FileInfo, 0), + Dirs: make([]string, 0), + } + + // Look for gcsfuse-shared-chunk-cache subdirectory + actualCacheDir := filepath.Join(cacheDir, "gcsfuse-shared-chunk-cache") + if info, err := os.Stat(actualCacheDir); err == nil && info.IsDir() { + cacheDir = actualCacheDir + } + + err := filepath.Walk(cacheDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + slog.Warn("Skipping file due to error", "path", path, "error", err) + return nil // Skip errors + } + + // Collect directories (excluding root) + if info.IsDir() { + if path != cacheDir { + manifest.Dirs = append(manifest.Dirs, path) + } + return nil + } + + ext := filepath.Ext(path) + + // Get file times (needed for .bin and .tmp files) + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return nil + } + atime := time.Unix(stat.Atim.Sec, stat.Atim.Nsec) + mtime := time.Unix(stat.Mtim.Sec, stat.Mtim.Nsec) + + switch ext { + case ".bin": + // Cache chunks - add to manifest + manifest.Files = append(manifest.Files, FileInfo{ + Path: path, + Atime: atime, + Mtime: mtime, + Size: info.Size(), + }) + manifest.TotalSize += info.Size() + + case ".bak": + // Previously expired files - collect for cleanup + manifest.BakFiles = append(manifest.BakFiles, FileInfo{ + Path: path, + Atime: atime, + Mtime: mtime, + Size: info.Size(), + }) + + case ".tmp": + // Temporary files - collect for cleanup + manifest.TmpFiles = append(manifest.TmpFiles, FileInfo{ + Path: path, + Atime: atime, + Mtime: mtime, + Size: info.Size(), + }) + } + + return nil + }) + + manifest.ScanDuration = time.Since(start) + return manifest, err +} + +// removeBakFiles removes .bak files in parallel +func removeBakFiles(files []FileInfo) { + if len(files) == 0 { + return + } + + var totalSize int64 + fileChan := make(chan FileInfo, *concurrency) + var wg sync.WaitGroup + + // Start worker pool + for i := 0; i < *concurrency; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for file := range fileChan { + if err := os.Remove(file.Path); err != nil && !os.IsNotExist(err) { + slog.Warn("Failed to remove old bak file", "path", file.Path, "error", err) + } else { + atomic.AddInt64(&totalSize, file.Size) + } + } + }() + } + + // Feed files to workers + for _, f := range files { + fileChan <- f + } + close(fileChan) + + wg.Wait() + + slog.Info("Removed previously expired (.bak) files", + "file_count", len(files), + "size_mb", float64(totalSize)/MiB) +} + +// removeOldTmpFiles removes .tmp files older than 1 hour in parallel. +// Keeping 1 hour is a safety margin to avoid deleting in-progress chunk download +// over a tmpFile. +func removeOldTmpFiles(files []FileInfo) { + cutoff := time.Now().Add(-1 * time.Hour) + var removedCount int32 + fileChan := make(chan FileInfo, *concurrency) + var wg sync.WaitGroup + + // Start worker pool + for i := 0; i < *concurrency; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for file := range fileChan { + if err := os.Remove(file.Path); err != nil && !os.IsNotExist(err) { + slog.Warn("Failed to remove old tmp file", "path", file.Path, "error", err) + } else { + atomic.AddInt32(&removedCount, 1) + } + } + }() + } + + // Feed files to workers (only those older than cutoff) + for _, f := range files { + if f.Mtime.Before(cutoff) { + fileChan <- f + } + } + close(fileChan) + + wg.Wait() + + if removedCount > 0 { + slog.Info("Removed old temporary files", "file_count", removedCount) + } +} + +// findLRUFiles finds least recently used files until we have enough to expire +func findLRUFiles(manifest *Manifest, targetSize int64) []FileInfo { + if manifest.TotalSize <= targetSize { + return []FileInfo{} + } + + bytesToExpire := manifest.TotalSize - targetSize + + // Sort by atime (oldest first) + slices.SortFunc(manifest.Files, func(a, b FileInfo) int { + recentA := a.Atime + if recentA.Before(a.Mtime) { + recentA = a.Mtime + } + recentB := b.Atime + if recentB.Before(b.Mtime) { + recentB = b.Mtime + } + + if recentA.Before(recentB) { + return -1 + } + if recentA.After(recentB) { + return 1 + } + return 0 + }) + + // Select files until we have enough bytes + var selected []FileInfo + var totalBytes int64 + + for _, f := range manifest.Files { + selected = append(selected, f) + totalBytes += f.Size + if totalBytes >= bytesToExpire { + break + } + } + + return selected +} + +// expireFiles renames files from .bin to .bak in parallel. +// This allows ongoing reads to continue until the next run when .bak files +// are cleaned up. Existing request will continue since rename preserves the +// file handle. +func expireFiles(files []FileInfo) { + fileChan := make(chan FileInfo, *concurrency) + var wg sync.WaitGroup + + // Start worker pool + for i := 0; i < *concurrency; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for file := range fileChan { + bakPath := file.Path + ".bak" + if err := os.Rename(file.Path, bakPath); err != nil && !os.IsNotExist(err) { + slog.Warn("Failed to rename file", "path", file.Path, "error", err) + } + } + }() + } + + // Feed files to workers + for _, f := range files { + fileChan <- f + } + close(fileChan) + + wg.Wait() +} + +// cleanupEmptyDirs removes empty object directories +func cleanupEmptyDirs(dirs []string) { + // Sort by depth (deepest first) + slices.SortFunc(dirs, func(a, b string) int { + aDepth := len(filepath.SplitList(a)) + bDepth := len(filepath.SplitList(b)) + return bDepth - aDepth // Reverse order (deepest first) + }) + + for _, dir := range dirs { + // Try to remove if empty + if err := os.Remove(dir); err != nil { + // Ignore ENOTEMPTY - concurrently files were added back to this directory. + if !errors.Is(err, syscall.ENOTEMPTY) { + slog.Debug("Failed to remove directory", "path", dir, "error", err) + } + } + } +} diff --git a/tools/gcsfuse-scc-gc/main_test.go b/tools/gcsfuse-scc-gc/main_test.go new file mode 100644 index 0000000000..847e4dd580 --- /dev/null +++ b/tools/gcsfuse-scc-gc/main_test.go @@ -0,0 +1,370 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLRUEviction(t *testing.T) { + // Arrange + tmpDir := t.TempDir() + cacheDir := filepath.Join(tmpDir, "cache") + objDir := filepath.Join(cacheDir, "a1", "b2", "hash123") + require.NoError(t, os.MkdirAll(objDir, 0755)) + now := time.Now() + files := []struct { + name string + size int64 + atime time.Time + }{ + {"0_1048576.bin", 1024 * 1024, now.Add(-3 * time.Hour)}, + {"1048576_2097152.bin", 1024 * 1024, now.Add(-2 * time.Hour)}, + {"2097152_3145728.bin", 1024 * 1024, now.Add(-1 * time.Hour)}, + } + for _, f := range files { + path := filepath.Join(objDir, f.name) + require.NoError(t, os.WriteFile(path, make([]byte, f.size), 0644)) + require.NoError(t, os.Chtimes(path, f.atime, f.atime)) + } + + // Act + manifest, err := scanCache(cacheDir) + require.NoError(t, err) + targetSize := int64(2 * 1024 * 1024) + filesToExpire := findLRUFiles(manifest, targetSize) + expireFiles(filesToExpire) + + // Assert + assert.Equal(t, 3, len(manifest.Files)) + assert.Equal(t, 1, len(filesToExpire)) + if len(filesToExpire) > 0 { + assert.Equal(t, "0_1048576.bin", filepath.Base(filesToExpire[0].Path)) + } + bakPath := filesToExpire[0].Path + ".bak" + _, err = os.Stat(bakPath) + assert.NoError(t, err, "Expected .bak file to exist") + _, err = os.Stat(filesToExpire[0].Path) + assert.True(t, os.IsNotExist(err), "Expected original file to be gone") +} + +func TestNoEvictionWhenBelowTarget(t *testing.T) { + // Arrange + tmpDir := t.TempDir() + cacheDir := filepath.Join(tmpDir, "cache") + objDir := filepath.Join(cacheDir, "a1", "b2", "hash123") + require.NoError(t, os.MkdirAll(objDir, 0755)) + path := filepath.Join(objDir, "0_1048576.bin") + require.NoError(t, os.WriteFile(path, make([]byte, 1024), 0644)) + + // Act + manifest, err := scanCache(cacheDir) + require.NoError(t, err) + targetSize := int64(2 * 1024 * 1024) + filesToExpire := findLRUFiles(manifest, targetSize) + + // Assert + assert.Equal(t, 1, len(manifest.Files)) + assert.Equal(t, 0, len(filesToExpire), "Expected no files to expire") +} + +func TestBakFileCleanup(t *testing.T) { + // Arrange + tmpDir := t.TempDir() + cacheDir := filepath.Join(tmpDir, "cache") + objDir := filepath.Join(cacheDir, "a1", "b2", "hash123") + require.NoError(t, os.MkdirAll(objDir, 0755)) + bakPath := filepath.Join(objDir, "old_file.bin.bak") + require.NoError(t, os.WriteFile(bakPath, []byte("test"), 0644)) + + // Act + manifest, err := scanCache(cacheDir) + require.NoError(t, err) + removeBakFiles(manifest.BakFiles) + + // Assert + _, err = os.Stat(bakPath) + assert.True(t, os.IsNotExist(err), "Expected .bak file to be removed") +} + +func TestTmpFileCleanup(t *testing.T) { + // Arrange + tmpDir := t.TempDir() + cacheDir := filepath.Join(tmpDir, "cache") + objDir := filepath.Join(cacheDir, "a1", "b2", "hash123") + require.NoError(t, os.MkdirAll(objDir, 0755)) + now := time.Now() + oldTmpPath := filepath.Join(objDir, "old.tmp") + require.NoError(t, os.WriteFile(oldTmpPath, []byte("test"), 0644)) + oldTime := now.Add(-2 * time.Hour) + require.NoError(t, os.Chtimes(oldTmpPath, oldTime, oldTime)) + recentTmpPath := filepath.Join(objDir, "recent.tmp") + require.NoError(t, os.WriteFile(recentTmpPath, []byte("test"), 0644)) + recentTime := now.Add(-30 * time.Minute) + require.NoError(t, os.Chtimes(recentTmpPath, recentTime, recentTime)) + + // Act + manifest, err := scanCache(cacheDir) + require.NoError(t, err) + removeOldTmpFiles(manifest.TmpFiles) + + // Assert + _, err = os.Stat(oldTmpPath) + assert.True(t, os.IsNotExist(err), "Expected old .tmp file to be removed") + _, err = os.Stat(recentTmpPath) + assert.NoError(t, err, "Expected recent .tmp file to still exist") +} + +func TestOnlyBinFilesProcessed(t *testing.T) { + // Arrange + tmpDir := t.TempDir() + cacheDir := filepath.Join(tmpDir, "cache") + objDir := filepath.Join(cacheDir, "a1", "b2", "hash123") + require.NoError(t, os.MkdirAll(objDir, 0755)) + files := []string{ + "0_1048576.bin", + "test.txt", + "data.json", + "1048576_2097152.bin", + } + for _, f := range files { + path := filepath.Join(objDir, f) + require.NoError(t, os.WriteFile(path, []byte("test"), 0644)) + } + + // Act + manifest, err := scanCache(cacheDir) + require.NoError(t, err) + + // Assert + assert.Equal(t, 2, len(manifest.Files), "Expected 2 .bin files") + for _, f := range manifest.Files { + assert.Equal(t, ".bin", filepath.Ext(f.Path), "All files should be .bin") + } +} + +func TestMultipleFilesExpiredToReachTarget(t *testing.T) { + // Arrange + tmpDir := t.TempDir() + cacheDir := filepath.Join(tmpDir, "cache") + objDir := filepath.Join(cacheDir, "a1", "b2", "hash123") + require.NoError(t, os.MkdirAll(objDir, 0755)) + now := time.Now() + files := []struct { + name string + size int64 + atime time.Time + }{ + {"0_1048576.bin", 1024 * 1024, now.Add(-5 * time.Hour)}, // Oldest + {"1048576_2097152.bin", 1024 * 1024, now.Add(-4 * time.Hour)}, // 2nd oldest + {"2097152_3145728.bin", 1024 * 1024, now.Add(-3 * time.Hour)}, // 3rd oldest + {"3145728_4194304.bin", 1024 * 1024, now.Add(-2 * time.Hour)}, // Newest (kept) + } + for _, f := range files { + path := filepath.Join(objDir, f.name) + require.NoError(t, os.WriteFile(path, make([]byte, f.size), 0644)) + require.NoError(t, os.Chtimes(path, f.atime, f.atime)) + } + + // Act + manifest, err := scanCache(cacheDir) + require.NoError(t, err) + targetSize := int64(1536 * 1024) // 1.5 MB - need to expire ~2.5MB + filesToExpire := findLRUFiles(manifest, targetSize) + + // Assert + assert.Equal(t, 4, len(manifest.Files)) + assert.Equal(t, 3, len(filesToExpire), "Expected 3 oldest files to be expired") + // Verify correct LRU order + assert.Equal(t, "0_1048576.bin", filepath.Base(filesToExpire[0].Path)) + assert.Equal(t, "1048576_2097152.bin", filepath.Base(filesToExpire[1].Path)) + assert.Equal(t, "2097152_3145728.bin", filepath.Base(filesToExpire[2].Path)) +} + +func TestLRUWithIdenticalAtimes(t *testing.T) { + // Arrange + tmpDir := t.TempDir() + cacheDir := filepath.Join(tmpDir, "cache") + objDir := filepath.Join(cacheDir, "a1", "b2", "hash123") + require.NoError(t, os.MkdirAll(objDir, 0755)) + now := time.Now() + sameTime := now.Add(-2 * time.Hour) + files := []struct { + name string + size int64 + atime time.Time + }{ + {"0_1048576.bin", 1024 * 1024, sameTime}, + {"1048576_2097152.bin", 1024 * 1024, sameTime}, + {"2097152_3145728.bin", 1024 * 1024, now.Add(-1 * time.Hour)}, // Newer + } + for _, f := range files { + path := filepath.Join(objDir, f.name) + require.NoError(t, os.WriteFile(path, make([]byte, f.size), 0644)) + require.NoError(t, os.Chtimes(path, f.atime, f.atime)) + } + + // Act + manifest, err := scanCache(cacheDir) + require.NoError(t, err) + targetSize := int64(1024 * 1024) // 1 MB - need to expire 2MB + filesToExpire := findLRUFiles(manifest, targetSize) + + // Assert + assert.Equal(t, 3, len(manifest.Files)) + assert.GreaterOrEqual(t, len(filesToExpire), 2, "Expected at least 2 files to be expired") + // The newest file should not be in the expired list + for _, f := range filesToExpire { + assert.NotEqual(t, "2097152_3145728.bin", filepath.Base(f.Path), + "Newest file should not be expired") + } +} + +func TestAtimeFallbackToMtime(t *testing.T) { + // Arrange + tmpDir := t.TempDir() + cacheDir := filepath.Join(tmpDir, "cache") + objDir := filepath.Join(cacheDir, "a1", "b2", "hash123") + require.NoError(t, os.MkdirAll(objDir, 0755)) + now := time.Now() + oldTime := now.Add(-5 * time.Hour) + recentTime := now.Add(-1 * time.Hour) + path := filepath.Join(objDir, "0_1048576.bin") + require.NoError(t, os.WriteFile(path, make([]byte, 1024), 0644)) + require.NoError(t, os.Chtimes(path, recentTime, recentTime)) + // Create another file with older times + path2 := filepath.Join(objDir, "1048576_2097152.bin") + require.NoError(t, os.WriteFile(path2, make([]byte, 1024), 0644)) + require.NoError(t, os.Chtimes(path2, oldTime, oldTime)) + + // Act + manifest, err := scanCache(cacheDir) + require.NoError(t, err) + targetSize := int64(1024) // Keep only 1KB + filesToExpire := findLRUFiles(manifest, targetSize) + + // Assert + assert.Equal(t, 2, len(manifest.Files)) + require.Equal(t, 1, len(filesToExpire), "Expected 1 file to be expired") + // The older file should be expired first + assert.Equal(t, "1048576_2097152.bin", filepath.Base(filesToExpire[0].Path), + "Oldest file (by mtime fallback) should be expired") +} + +func TestTmpFileAtOneHourBoundary(t *testing.T) { + // Arrange + tmpDir := t.TempDir() + cacheDir := filepath.Join(tmpDir, "cache") + objDir := filepath.Join(cacheDir, "a1", "b2", "hash123") + require.NoError(t, os.MkdirAll(objDir, 0755)) + now := time.Now() + // File well over 1 hour old + oldFile := filepath.Join(objDir, "old.tmp") + require.NoError(t, os.WriteFile(oldFile, []byte("test"), 0644)) + oldTime := now.Add(-2 * time.Hour) + require.NoError(t, os.Chtimes(oldFile, oldTime, oldTime)) + // File just under 1 hour old + recentFile := filepath.Join(objDir, "recent.tmp") + require.NoError(t, os.WriteFile(recentFile, []byte("test"), 0644)) + recentTime := now.Add(-59 * time.Minute) + require.NoError(t, os.Chtimes(recentFile, recentTime, recentTime)) + + // Act + manifest, err := scanCache(cacheDir) + require.NoError(t, err) + removeOldTmpFiles(manifest.TmpFiles) + + // Assert + assert.Equal(t, 2, len(manifest.TmpFiles)) + // File over 1 hour should be removed + _, err = os.Stat(oldFile) + assert.True(t, os.IsNotExist(err), "File over 1 hour should be removed") + // File under 1 hour should still exist + _, err = os.Stat(recentFile) + assert.NoError(t, err, "File under 1 hour should still exist") +} + +func TestEmptyTmpFileList(t *testing.T) { + // Arrange + tmpDir := t.TempDir() + cacheDir := filepath.Join(tmpDir, "cache") + objDir := filepath.Join(cacheDir, "a1", "b2", "hash123") + require.NoError(t, os.MkdirAll(objDir, 0755)) + // Create only .bin files, no .tmp files + binPath := filepath.Join(objDir, "0_1048576.bin") + require.NoError(t, os.WriteFile(binPath, []byte("test"), 0644)) + + // Act + manifest, err := scanCache(cacheDir) + require.NoError(t, err) + // Should not panic with empty list + removeOldTmpFiles(manifest.TmpFiles) + + // Assert + assert.Equal(t, 0, len(manifest.TmpFiles), "Expected no tmp files") + assert.Equal(t, 1, len(manifest.Files), "Expected 1 bin file") +} + +func TestLRUSortingOrderVerification(t *testing.T) { + // Arrange + tmpDir := t.TempDir() + cacheDir := filepath.Join(tmpDir, "cache") + objDir := filepath.Join(cacheDir, "a1", "b2", "hash123") + require.NoError(t, os.MkdirAll(objDir, 0755)) + now := time.Now() + // Create files in non-chronological order + files := []struct { + name string + size int64 + atime time.Time + }{ + {"file3.bin", 512 * 1024, now.Add(-2 * time.Hour)}, // Middle + {"file1.bin", 512 * 1024, now.Add(-5 * time.Hour)}, // Oldest + {"file5.bin", 512 * 1024, now.Add(-30 * time.Minute)}, // Newest + {"file2.bin", 512 * 1024, now.Add(-4 * time.Hour)}, // 2nd oldest + {"file4.bin", 512 * 1024, now.Add(-1 * time.Hour)}, // 2nd newest + } + for _, f := range files { + path := filepath.Join(objDir, f.name) + require.NoError(t, os.WriteFile(path, make([]byte, f.size), 0644)) + require.NoError(t, os.Chtimes(path, f.atime, f.atime)) + } + + // Act + manifest, err := scanCache(cacheDir) + require.NoError(t, err) + targetSize := int64(1024 * 1024) // 1 MB - need to expire ~1.5MB + filesToExpire := findLRUFiles(manifest, targetSize) + + // Assert + assert.Equal(t, 5, len(manifest.Files)) + require.GreaterOrEqual(t, len(filesToExpire), 3, "Expected at least 3 files to be expired") + // Verify files are in strict chronological order (oldest first) + assert.Equal(t, "file1.bin", filepath.Base(filesToExpire[0].Path), "1st expired should be oldest") + assert.Equal(t, "file2.bin", filepath.Base(filesToExpire[1].Path), "2nd expired should be 2nd oldest") + assert.Equal(t, "file3.bin", filepath.Base(filesToExpire[2].Path), "3rd expired should be 3rd oldest") + // Verify chronological ordering + for i := 1; i < len(filesToExpire); i++ { + assert.True(t, filesToExpire[i-1].Atime.Before(filesToExpire[i].Atime) || + filesToExpire[i-1].Atime.Equal(filesToExpire[i].Atime), + "Files should be sorted by atime (oldest first)") + } +} diff --git a/tools/gem_dependency/Gemfile.lock b/tools/gem_dependency/Gemfile.lock index 6c8954e620..5b38a3a870 100644 --- a/tools/gem_dependency/Gemfile.lock +++ b/tools/gem_dependency/Gemfile.lock @@ -23,7 +23,7 @@ GEM insist mustache (= 0.99.8) stud - rexml (3.3.9) + rexml (3.4.2) stud (0.0.23) PLATFORMS diff --git a/tools/integration_tests/benchmarking/benchmark_delete_test.go b/tools/integration_tests/benchmarking/benchmark_delete_test.go index d26016d979..618e3174fc 100644 --- a/tools/integration_tests/benchmarking/benchmark_delete_test.go +++ b/tools/integration_tests/benchmarking/benchmark_delete_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,29 +16,31 @@ package benchmarking import ( "fmt" + "log" "os" "path" "testing" + "time" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/benchmark_setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/benchmark_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) -type benchmarkDeleteTest struct{} +const ( + expectedDeleteLatency time.Duration = 1800 * time.Millisecond +) -func (s *benchmarkDeleteTest) SetupB(b *testing.B) { - testDirPath = setup.SetupTestDirectory(testDirName) +type benchmarkDeleteTest struct { + flags []string } -func (s *benchmarkDeleteTest) TeardownB(b *testing.B) {} +func (s *benchmarkDeleteTest) SetupB(b *testing.B) { + mountGCSFuseAndSetupTestDir(s.flags, testEnv.ctx, testEnv.storageClient) +} -// createFilesToDelete creates the below objects in the bucket. -// benchmarking/a{i}.txt where i is a counter based on the benchtime value. -func createFilesToDelete(b *testing.B) { - for i := 0; i < b.N; i++ { - operations.CreateFileOfSize(5, path.Join(testDirPath, fmt.Sprintf("a%d.txt", i)), b) - } +func (s *benchmarkDeleteTest) TeardownB(b *testing.B) { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) + setup.SaveGCSFuseLogFileInCaseOfFailure(b) } //////////////////////////////////////////////////////////////////////// @@ -46,12 +48,45 @@ func createFilesToDelete(b *testing.B) { //////////////////////////////////////////////////////////////////////// func (s *benchmarkDeleteTest) Benchmark_Delete(b *testing.B) { - createFilesToDelete(b) + createFiles(b) + var maxElapsedDuration time.Duration + maxElapsedIteration := -1 b.ResetTimer() - for i := 0; i < b.N; i++ { - if err := os.Remove(path.Join(testDirPath, fmt.Sprintf("a%d.txt", i))); err != nil { - b.Errorf("testing error: %v", err) + // Don't start the timer yet. + b.StopTimer() + + for i := range b.N { + filePath := path.Join(testEnv.testDirPath, fmt.Sprintf("a%d.txt", i)) + + // Manually time the operation to find the maximum latency with highest accuracy. + // This happens while the benchmark's timer is paused and will not affect the average. + startTime := time.Now() + + // Start the benchmark timer just for the os.Remove call. + b.StartTimer() + err := os.Remove(filePath) + b.StopTimer() // Stop the timer immediately after the operation. + + timeElapsedThisIter := time.Since(startTime) + + // The remaining checks and calculations also happen while the timer is paused. + if err != nil { + b.Errorf("error while deleting %q: %v", filePath, err) } + + if maxElapsedDuration < timeElapsedThisIter { + maxElapsedDuration = timeElapsedThisIter + maxElapsedIteration = i + } + } + + // b.Elapsed() is the sum of the time spent only on os.Remove calls, + // leading to a highly accurate average latency. + averageDeleteLatency := b.Elapsed() / time.Duration(b.N) + + if averageDeleteLatency > expectedDeleteLatency { + b.Errorf("DeleteFile took more time on average (%v) than expected (%v).", averageDeleteLatency, expectedDeleteLatency) + b.Errorf("Maximum time taken by a single iteration = %v, in iteration # %v.", maxElapsedDuration, maxElapsedIteration) } } @@ -60,6 +95,16 @@ func (s *benchmarkDeleteTest) Benchmark_Delete(b *testing.B) { //////////////////////////////////////////////////////////////////////// func Benchmark_Delete(b *testing.B) { + setup.IgnoreTestIfPresubmitFlagIsSet(b) + ts := &benchmarkDeleteTest{} - benchmark_setup.RunBenchmarks(b, ts) + + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, b.Name()) + + // Run tests. + for _, flags := range flagsSet { + ts.flags = flags + log.Printf("Running tests with flags: %s", ts.flags) + benchmark_setup.RunBenchmarks(b, ts) + } } diff --git a/tools/integration_tests/benchmarking/benchmark_rename_test.go b/tools/integration_tests/benchmarking/benchmark_rename_test.go new file mode 100644 index 0000000000..dc3668ec20 --- /dev/null +++ b/tools/integration_tests/benchmarking/benchmark_rename_test.go @@ -0,0 +1,110 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmarking + +import ( + "fmt" + "log" + "os" + "path" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/benchmark_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" +) + +type benchmarkRenameTest struct { + flags []string +} + +const ( + expectedRenameLatency time.Duration = 1900 * time.Millisecond +) + +func (s *benchmarkRenameTest) SetupB(b *testing.B) { + mountGCSFuseAndSetupTestDir(s.flags, testEnv.ctx, testEnv.storageClient) +} + +func (s *benchmarkRenameTest) TeardownB(b *testing.B) { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) + setup.SaveGCSFuseLogFileInCaseOfFailure(b) +} + +//////////////////////////////////////////////////////////////////////// +// Test scenarios +//////////////////////////////////////////////////////////////////////// + +func (s *benchmarkRenameTest) Benchmark_Rename(b *testing.B) { + createFiles(b) + var maxElapsedDuration time.Duration + maxElapsedIteration := -1 + b.ResetTimer() + // Don't start the timer yet. + b.StopTimer() + + for i := range b.N { + sourceFilePath := path.Join(testEnv.testDirPath, fmt.Sprintf("a%d.txt", i)) + dstFilePath := path.Join(testEnv.testDirPath, fmt.Sprintf("b%d.txt", i)) + + // Manually time the operation to find the maximum latency with highest accuracy. + // This happens while the benchmark's timer is paused and will not affect the average. + startTime := time.Now() + + // Start the benchmark timer just for the os.Rename call. + b.StartTimer() + err := os.Rename(sourceFilePath, dstFilePath) + b.StopTimer() // Stop the timer immediately after the operation. + + timeElapsedThisIter := time.Since(startTime) + + // The remaining checks and calculations also happen while the timer is paused. + if err != nil { + b.Errorf("failed to rename %q to %q: %v", sourceFilePath, dstFilePath, err) + } + + if maxElapsedDuration < timeElapsedThisIter { + maxElapsedDuration = timeElapsedThisIter + maxElapsedIteration = i + } + } + + // b.Elapsed() is the sum of the time spent only on os.Rename calls, + // leading to a highly accurate average latency. + averageRenameLatency := b.Elapsed() / time.Duration(b.N) + + if averageRenameLatency > expectedRenameLatency { + b.Errorf("RenameFile took more time on average (%v) than expected %v", averageRenameLatency, expectedRenameLatency) + b.Errorf("Maximum time taken by a single iteration = %v, in iteration # %v.", maxElapsedDuration, maxElapsedIteration) + } +} + +//////////////////////////////////////////////////////////////////////// +// Test Function (Runs once before all tests) +//////////////////////////////////////////////////////////////////////// + +func Benchmark_Rename(b *testing.B) { + setup.IgnoreTestIfPresubmitFlagIsSet(b) + + ts := &benchmarkRenameTest{} + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, b.Name()) + + // Run tests. + for _, flags := range flagsSet { + ts.flags = flags + log.Printf("Running tests with flags: %s", ts.flags) + benchmark_setup.RunBenchmarks(b, ts) + } +} diff --git a/tools/integration_tests/benchmarking/benchmark_stat_test.go b/tools/integration_tests/benchmarking/benchmark_stat_test.go index 4ccc99abd5..1da803c8a8 100644 --- a/tools/integration_tests/benchmarking/benchmark_stat_test.go +++ b/tools/integration_tests/benchmarking/benchmark_stat_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,30 +15,41 @@ package benchmarking import ( + "log" "path" "testing" + "time" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/benchmark_setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/benchmark_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) //////////////////////////////////////////////////////////////////////// // Boilerplate //////////////////////////////////////////////////////////////////////// -type benchmarkStatTest struct{} +const ( + expectedStatLatency time.Duration = 1100 * time.Millisecond +) + +type benchmarkStatTest struct { + flags []string +} func (s *benchmarkStatTest) SetupB(b *testing.B) { - testDirPath = setup.SetupTestDirectory(testDirName) + mountGCSFuseAndSetupTestDir(s.flags, testEnv.ctx, testEnv.storageClient) } -func (s *benchmarkStatTest) TeardownB(b *testing.B) {} +func (s *benchmarkStatTest) TeardownB(b *testing.B) { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) + setup.SaveGCSFuseLogFileInCaseOfFailure(b) +} // createFilesToStat creates the below object in the bucket. // benchmarking/a.txt func createFilesToStat(b *testing.B) { - operations.CreateFileOfSize(5, path.Join(testDirPath, "a.txt"), b) + operations.CreateFileOfSize(1, path.Join(testEnv.testDirPath, "a.txt"), b) } //////////////////////////////////////////////////////////////////////// @@ -47,11 +58,44 @@ func createFilesToStat(b *testing.B) { func (s *benchmarkStatTest) Benchmark_Stat(b *testing.B) { createFilesToStat(b) + var maxElapsedDuration time.Duration + maxElapsedIteration := -1 b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err := operations.StatFile(path.Join(testDirPath, "a.txt")); err != nil { - b.Errorf("testing error: %v", err) + // Don't start the timer yet. + b.StopTimer() + + filePath := path.Join(testEnv.testDirPath, "a.txt") + + for i := range b.N { + // Manually time the operation to find the maximum latency with highest accuracy. + // This happens while the benchmark's timer is paused and will not affect the average. + startTime := time.Now() + + // Start the benchmark timer just for the operations.StatFile call. + b.StartTimer() + _, err := operations.StatFile(filePath) + b.StopTimer() // Stop the timer immediately after the operation. + + timeElapsedThisIter := time.Since(startTime) + + // The remaining checks and calculations also happen while the timer is paused. + if err != nil { + b.Errorf("failed to stat %q: %v", filePath, err) } + + if maxElapsedDuration < timeElapsedThisIter { + maxElapsedDuration = timeElapsedThisIter + maxElapsedIteration = i + } + } + + // b.Elapsed() is the sum of the time spent only on stat calls, + // leading to a highly accurate average latency. + averageStatLatency := b.Elapsed() / time.Duration(b.N) + + if averageStatLatency > expectedStatLatency { + b.Errorf("StatFile took more time on average (%v) than expected (%v)", averageStatLatency, expectedStatLatency) + b.Errorf("Maximum time taken by a single iteration = %v, in iteration # %v.", maxElapsedDuration, maxElapsedIteration) } } @@ -60,6 +104,15 @@ func (s *benchmarkStatTest) Benchmark_Stat(b *testing.B) { //////////////////////////////////////////////////////////////////////// func Benchmark_Stat(b *testing.B) { + setup.IgnoreTestIfPresubmitFlagIsSet(b) + ts := &benchmarkStatTest{} - benchmark_setup.RunBenchmarks(b, ts) + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, b.Name()) + + // Run tests. + for _, flags := range flagsSet { + ts.flags = flags + log.Printf("Running tests with flags: %s", ts.flags) + benchmark_setup.RunBenchmarks(b, ts) + } } diff --git a/tools/integration_tests/benchmarking/setup_test.go b/tools/integration_tests/benchmarking/setup_test.go index 5265ef70f7..e4c7c8276d 100644 --- a/tools/integration_tests/benchmarking/setup_test.go +++ b/tools/integration_tests/benchmarking/setup_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,20 +11,27 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +// +// Note that the expected latency thresholds for the various operations has +// been set to 4 times the observed latency. Any failure of the benchmark tests +// is a direct indicator of anomaly. package benchmarking import ( "context" + "fmt" "log" "os" "path" "testing" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" ) const ( @@ -32,38 +39,92 @@ const ( ) var ( + testEnv env + mountFunc func(*test_suite.TestConfig, []string) error + // mount directory is where our tests run. + mountDir string + // root directory is the directory to be unmounted. + rootDir string +) + +type env struct { storageClient *storage.Client ctx context.Context testDirPath string -) + cfg *test_suite.TestConfig + bucketType string +} + +//////////////////////////////////////////////////////////////////////// +// Helpers +//////////////////////////////////////////////////////////////////////// + +func mountGCSFuseAndSetupTestDir(flags []string, ctx context.Context, storageClient *storage.Client) { + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, flags, mountFunc) + setup.SetMntDir(mountDir) + testEnv.testDirPath = client.SetupTestDirectory(ctx, storageClient, testDirName) +} + +// createFiles creates the below objects in the bucket. +// benchmarking/a{i}.txt where i is a counter based on the benchtime value. +func createFiles(b *testing.B) { + for i := range b.N { + operations.CreateFileOfSize(1, path.Join(testEnv.testDirPath, fmt.Sprintf("a%d.txt", i)), b) + } +} func TestMain(m *testing.M) { setup.ParseSetUpFlags() - setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() - - // Create common storage client to be used in test. - ctx = context.Background() - closeStorageClient := client.CreateStorageClientWithCancel(&ctx, &storageClient) - defer func() { - if err := closeStorageClient(); err != nil { - log.Fatalf("closeStorageClient failed: %v", err) - } - }() - - // If Mounted Directory flag is set, run tests for mounted directory. - setup.RunTestsForMountedDirectoryFlag(m) - - // Else run tests for testBucket. - // Set up test directory. - setup.SetUpTestDirForTestBucketFlag() - - flagsSet := [][]string{ - {"--stat-cache-ttl=0"}, - {"--client-protocol=grpc", "--stat-cache-ttl=0"}, + + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.Benchmarking) == 0 { + log.Println("No configuration found for benchmarking tests in config. Using flags instead.") + // Populate the config manually. + cfg.Benchmarking = make([]test_suite.TestConfig, 1) + cfg.Benchmarking[0].TestBucket = setup.TestBucket() + cfg.Benchmarking[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.Benchmarking[0].LogFile = setup.LogFile() + // Manually add configs for each benchmark test. + cfg.Benchmarking[0].Configs = make([]test_suite.ConfigItem, 3) + cfg.Benchmarking[0].Configs[0].Flags = []string{"--stat-cache-ttl=0", "--stat-cache-ttl=0 --client-protocol=grpc"} + cfg.Benchmarking[0].Configs[0].Run = "Benchmark_Stat" + cfg.Benchmarking[0].Configs[1].Flags = []string{"--stat-cache-ttl=0 --enable-atomic-rename-object=true", "--stat-cache-ttl=0 --enable-atomic-rename-object=true --client-protocol=grpc"} + cfg.Benchmarking[0].Configs[1].Run = "Benchmark_Rename" + cfg.Benchmarking[0].Configs[2].Flags = []string{"--stat-cache-ttl=0", "--client-protocol=grpc --stat-cache-ttl=0"} + cfg.Benchmarking[0].Configs[2].Run = "Benchmark_Delete" + } + + testEnv.ctx = context.Background() + testEnv.cfg = &cfg.Benchmarking[0] + testEnv.bucketType = setup.TestEnvironment(testEnv.ctx, testEnv.cfg) + + // 2. Create storage client before running tests. + var err error + testEnv.storageClient, err = client.CreateStorageClient(testEnv.ctx) + if err != nil { + log.Printf("Error creating storage client: %v\n", err) + os.Exit(1) } - successCode := static_mounting.RunTests(flagsSet, m) + defer testEnv.storageClient.Close() + + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + mountDir, rootDir = testEnv.cfg.GKEMountedDirectory, testEnv.cfg.GKEMountedDirectory + os.Exit(setup.RunTestsForMountedDirectory(testEnv.cfg.GKEMountedDirectory, m)) + } + + // Run tests for testBucket + setup.SetUpTestDirForTestBucket(testEnv.cfg) + + // Save mount and root directory variables. + mountDir, rootDir = testEnv.cfg.GCSFuseMountedDirectory, testEnv.cfg.GCSFuseMountedDirectory + + log.Println("Running static mounting tests...") + mountFunc = static_mounting.MountGcsfuseWithStaticMountingWithConfigFile + successCode := m.Run() // Clean up test directory created. - setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(setup.TestBucket(), testDirName)) + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(testEnv.cfg.TestBucket, testDirName)) os.Exit(successCode) } diff --git a/tools/integration_tests/buffered_read/fallback_test.go b/tools/integration_tests/buffered_read/fallback_test.go new file mode 100644 index 0000000000..9b6c171720 --- /dev/null +++ b/tools/integration_tests/buffered_read/fallback_test.go @@ -0,0 +1,214 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package buffered_read + +import ( + "log" + "os" + "path" + "syscall" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +//////////////////////////////////////////////////////////////////////// +// Test Suite Boilerplate +//////////////////////////////////////////////////////////////////////// + +// fallbackSuiteBase provides shared setup and teardown logic for fallback-related test suites. +type fallbackSuiteBase struct { + suite.Suite + flags []string +} + +func (s *fallbackSuiteBase) SetupSuite() { + setup.SetUpLogFilePath(s.T().Name(), s.flags, GKETempDir, "", testEnv.cfg) + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, s.flags, mountFunc) + setup.SetMntDir(mountDir) +} + +func (s *fallbackSuiteBase) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *fallbackSuiteBase) SetupTest() { + testEnv.testDirPath = setup.SetupTestDirectory(testDirName) +} + +func (s *fallbackSuiteBase) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +// InsufficientPoolCreationSuite tests scenarios where the buffered reader is not +// created due to an insufficient global block pool at the time of file opening. +type InsufficientPoolCreationSuite struct { + fallbackSuiteBase +} + +// RandomReadFallbackSuite tests fallback scenarios related to random reads. +type RandomReadFallbackSuite struct { + fallbackSuiteBase +} + +//////////////////////////////////////////////////////////////////////// +// Test Cases +//////////////////////////////////////////////////////////////////////// + +// TestNewBufferedReader_InsufficientGlobalPool_NoReaderAdded tests that when +// there are not enough blocks in the global pool to satisfy `min-blocks-per-handle`, +// the BufferedReader is not created, and reads fall back to the next reader +// without any buffered reading. +func (s *InsufficientPoolCreationSuite) TestNewBufferedReader_InsufficientGlobalPool_NoReaderAdded() { + err := os.Truncate(setup.LogFile(), 0) + require.NoError(s.T(), err, "Failed to truncate log file") + fileSize := blockSizeInBytes * 3 + chunkSize := int64(1 * util.MiB) + testDir := setup.SetupTestDirectory(testDirName) + fileName := setupFileInTestDir(testEnv.ctx, testEnv.storageClient, testDir, fileSize, s.T()) + filePath := path.Join(testDir, fileName) + + // Open and read the file. Since BufferedReader creation should fail, the read + // will be served by the GCSReader. + content, err := operations.ReadChunkFromFile(filePath, chunkSize, 0, os.O_RDONLY|syscall.O_DIRECT) + + require.NoError(s.T(), err, "Failed to read file") + client.ValidateObjectChunkFromGCS(testEnv.ctx, testEnv.storageClient, path.Base(testDir), fileName, 0, chunkSize, string(content), s.T()) + warningMsg := "Failed to create bufferedReader" + found := operations.CheckLogFileForMessage(s.T(), warningMsg, setup.LogFile()) + assert.True(s.T(), found, "Expected warning message not found in log file") + logEntries := parseBufferedReadLogs(s.T()) + assert.Empty(s.T(), logEntries, "Expected no buffered read log entries") +} + +func (s *RandomReadFallbackSuite) TestRandomRead_Fallback() { + err := os.Truncate(setup.LogFile(), 0) + require.NoError(s.T(), err, "Failed to truncate log file") + const randomReadsThreshold = 3 + // Create a file with 4 blocks. We will read backwards from block 3 to 0 + // to trigger random seek detection. + numBlocks := 4 + fileSize := blockSizeInBytes * int64(numBlocks) + chunkSize := int64(1 * util.KiB) + testDir := setup.SetupTestDirectory(testDirName) + fileName := setupFileInTestDir(testEnv.ctx, testEnv.storageClient, testDir, fileSize, s.T()) + filePath := path.Join(testDir, fileName) + f, err := os.OpenFile(filePath, os.O_RDONLY|syscall.O_DIRECT, 0) + require.NoError(s.T(), err) + defer operations.CloseFileShouldNotThrowError(s.T(), f) + + induceRandomReadFallback(s.T(), f, path.Base(testDir), fileName, chunkSize, blockSizeInBytes, randomReadsThreshold) + + bufferedReadLogEntry := parseAndValidateSingleBufferedReadLog(s.T()) + expected := &Expected{BucketName: setup.TestBucket(), ObjectName: path.Join(path.Base(testDir), fileName)} + validate(expected, bufferedReadLogEntry, true, s.T()) + assert.Equal(s.T(), int64(randomReadsThreshold+1), bufferedReadLogEntry.RandomSeekCount, "RandomSeekCount should be one greater than the threshold.") +} + +func (s *RandomReadFallbackSuite) TestRandomRead_SmallFile_NoFallback() { + err := os.Truncate(setup.LogFile(), 0) + require.NoError(s.T(), err, "Failed to truncate log file") + // File size is small, less than one block. + fileSize := blockSizeInBytes / 2 + chunkSize := int64(1 * util.KiB) + testDir := setup.SetupTestDirectory(testDirName) + fileName := setupFileInTestDir(testEnv.ctx, testEnv.storageClient, testDir, fileSize, s.T()) + filePath := path.Join(testDir, fileName) + f, err := os.OpenFile(filePath, os.O_RDONLY|syscall.O_DIRECT, 0) + require.NoError(s.T(), err) + defer operations.CloseFileShouldNotThrowError(s.T(), f) + // The first read (at offset 0) is sequential. + readAndValidateChunk(f, path.Base(testDir), fileName, 0, chunkSize, s.T()) + + // The second read should be served from the prefetched block and not be a random seek. + readAndValidateChunk(f, path.Base(testDir), fileName, fileSize/2, chunkSize, s.T()) + + bufferedReadLogEntry := parseAndValidateSingleBufferedReadLog(s.T()) + expected := &Expected{BucketName: setup.TestBucket(), ObjectName: path.Join(path.Base(testDir), fileName)} + validate(expected, bufferedReadLogEntry, false, s.T()) + assert.Equal(s.T(), int64(0), bufferedReadLogEntry.RandomSeekCount, "RandomSeekCount should be 0 for small file reads.") +} + +// TestRandomThenSequential_SwitchesBackToBufferedRead verifies that after a fallback +// due to random reads, the buffered reader is re-engaged once the read pattern +// becomes sequential again. +func (s *RandomReadFallbackSuite) TestRandomThenSequential_SwitchesBackToBufferedRead() { + err := os.Truncate(setup.LogFile(), 0) + require.NoError(s.T(), err, "Failed to truncate log file") + fileSize := int64(20 * util.MiB) + testDir := setup.SetupTestDirectory(testDirName) + fileName := setupFileInTestDir(testEnv.ctx, testEnv.storageClient, testDir, fileSize, s.T()) + filePath := path.Join(testDir, fileName) + f, err := os.OpenFile(filePath, os.O_RDONLY|syscall.O_DIRECT, 0) + require.NoError(s.T(), err) + defer operations.CloseFileShouldNotThrowError(s.T(), f) + // We perform 4 backward reads of 5 MiB. This size keeps the average read size + // below the maxReadSize (8 MiB), ensuring the pattern is classified as random. + // The 4th read exceeds the randomReadsThreshold, triggering a fallback. + readAndValidateChunk(f, path.Base(testDir), fileName, 8*util.MiB, 5*util.MiB, s.T()) + readAndValidateChunk(f, path.Base(testDir), fileName, 7*util.MiB, 5*util.MiB, s.T()) + readAndValidateChunk(f, path.Base(testDir), fileName, 6*util.MiB, 5*util.MiB, s.T()) + readAndValidateChunk(f, path.Base(testDir), fileName, 5*util.MiB, 5*util.MiB, s.T()) + + // The initial 1 MiB chunks are handled by the GCSReader due to the fallback. + // The ReadTypeClassifier observes this, re-classifies the pattern as sequential, + // and the BufferedReader restarts to serve the remaining chunks of this read. + readAndValidateChunk(f, path.Base(testDir), fileName, 0, 20*util.MiB, s.T()) + + bufferedReadLogEntry := parseAndValidateSingleBufferedReadLog(s.T()) + require.NotNil(s.T(), bufferedReadLogEntry, "Log entry for the file handle not found.") + assert.True(s.T(), bufferedReadLogEntry.Fallback, "Expected fallback to be true.") + assert.True(s.T(), bufferedReadLogEntry.Restarted, "Expected reader to be restarted.") +} + +//////////////////////////////////////////////////////////////////////// +// Test Function (Runs once before all tests) +//////////////////////////////////////////////////////////////////////// + +func TestInsufficientPoolCreationSuite(t *testing.T) { + ts := &InsufficientPoolCreationSuite{} + + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} + +func TestRandomReadFallbackSuite(t *testing.T) { + ts := &RandomReadFallbackSuite{} + + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/buffered_read/helpers_test.go b/tools/integration_tests/buffered_read/helpers_test.go new file mode 100644 index 0000000000..0882aa05ff --- /dev/null +++ b/tools/integration_tests/buffered_read/helpers_test.go @@ -0,0 +1,134 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package buffered_read + +import ( + "bytes" + "context" + "os" + "path" + "syscall" + "testing" + "time" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Expected is a helper struct that stores list of attributes to be validated from logs. +type Expected struct { + StartTimeStampSeconds int64 + EndTimeStampSeconds int64 + BucketName string + ObjectName string +} + +func readFileAndValidate(ctx context.Context, storageClient *storage.Client, testDir, fileName string, readFullFile bool, offset int64, chunkSizeToRead int64, t *testing.T) *Expected { + expected := &Expected{ + StartTimeStampSeconds: time.Now().Unix(), + BucketName: setup.TestBucket(), + ObjectName: path.Join(path.Base(testDir), fileName), + } + if setup.DynamicBucketMounted() != "" { + expected.BucketName = setup.DynamicBucketMounted() + } + if readFullFile { + file, err := os.OpenFile(path.Join(testDir, fileName), os.O_RDONLY|syscall.O_DIRECT, setup.FilePermission_0600) + require.NoError(t, err) + content, err := operations.ReadFileSequentially(file, chunkSizeToRead) + require.NoError(t, err, "Failed to read file sequentially") + obj := storageClient.Bucket(expected.BucketName).Object(expected.ObjectName) + attrs, err := obj.Attrs(ctx) + require.NoError(t, err, "obj.Attrs") + localCRC32C, err := operations.CalculateCRC32(bytes.NewReader(content)) + require.NoError(t, err, "Error while calculating crc for the content read from mounted file") + assert.Equal(t, attrs.CRC32C, localCRC32C, "CRC32C mismatch. GCS: %d, Local: %d", attrs.CRC32C, localCRC32C) + } else { + content, err := operations.ReadChunkFromFile(path.Join(testDir, fileName), chunkSizeToRead, offset, os.O_RDONLY|syscall.O_DIRECT) + require.NoError(t, err, "Failed to read random file chunk") + client.ValidateObjectChunkFromGCS(ctx, storageClient, path.Base(testDir), fileName, offset, chunkSizeToRead, string(content), t) + } + expected.EndTimeStampSeconds = time.Now().Unix() + return expected +} + +func validate(expected *Expected, logEntry *read_logs.BufferedReadLogEntry, fallback bool, t *testing.T) { + t.Helper() + assert.GreaterOrEqual(t, logEntry.StartTimeSeconds, expected.StartTimeStampSeconds, "start time in logs %d less than actual start time %d.", logEntry.StartTimeSeconds, expected.StartTimeStampSeconds) + + assert.Equal(t, expected.BucketName, logEntry.BucketName, "Bucket names don't match! Expected: %s, Got from logs: %s", + expected.BucketName, logEntry.BucketName) + + assert.Equal(t, expected.ObjectName, logEntry.ObjectName, "Object names don't match! Expected: %s, Got from logs: %s", + expected.ObjectName, logEntry.ObjectName) + + assert.Equal(t, fallback, logEntry.Fallback, "Expected Fallback: %t, Got from logs: %t", fallback, logEntry.Fallback) +} + +func setupFileInTestDir(ctx context.Context, storageClient *storage.Client, testDir string, fileSize int64, t *testing.T) (fileName string) { + fileName = testFileName + setup.GenerateRandomString(4) + client.SetupFileInTestDirectory(ctx, storageClient, path.Base(testDir), fileName, fileSize, t) + return fileName +} + +func parseBufferedReadLogs(t *testing.T) map[int64]*read_logs.BufferedReadLogEntry { + t.Helper() + f := operations.OpenFile(setup.LogFile(), t) + defer operations.CloseFileShouldNotThrowError(t, f) + + logEntries, err := read_logs.ParseBufferedReadLogsFromLogReader(f) + require.NoError(t, err, "Failed to parse log file") + return logEntries +} + +func parseAndValidateSingleBufferedReadLog(t *testing.T) *read_logs.BufferedReadLogEntry { + t.Helper() + logEntries := parseBufferedReadLogs(t) + // The test is expected to generate exactly one buffered read log entry because + // all reads are performed through a single file handle. + require.Len(t, logEntries, 1, "Expected one buffered read log entry for the single file handle.") + + for _, entry := range logEntries { + return entry + } + return nil // Unreachable. +} + +func readAndValidateChunk(f *os.File, testDir, fileName string, offset, chunkSize int64, t *testing.T) { + t.Helper() + readBuffer := make([]byte, chunkSize) + + _, err := f.ReadAt(readBuffer, offset) + + require.NoError(t, err, "ReadAt failed at offset %d", offset) + client.ValidateObjectChunkFromGCS(testEnv.ctx, testEnv.storageClient, testDir, fileName, offset, chunkSize, string(readBuffer), t) +} + +// induceRandomReadFallback performs a sequence of backward reads to trigger the +// random read fallback mechanism. It reads blocks in reverse order, from block +// randomReadsThreshold down to 0, to ensure each is counted as a random seek. +func induceRandomReadFallback(t *testing.T, f *os.File, testDir, fileName string, chunkSize, blockSize int64, randomReadsThreshold int) { + t.Helper() + // Perform randomReadsThreshold + 1 backward reads to trigger the fallback. + for i := 0; i <= randomReadsThreshold; i++ { + offset := (int64(randomReadsThreshold-i) * blockSize) + readAndValidateChunk(f, testDir, fileName, offset, chunkSize, t) + } +} diff --git a/tools/integration_tests/buffered_read/sequential_read_test.go b/tools/integration_tests/buffered_read/sequential_read_test.go new file mode 100644 index 0000000000..f98473f5e2 --- /dev/null +++ b/tools/integration_tests/buffered_read/sequential_read_test.go @@ -0,0 +1,192 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package buffered_read + +import ( + "fmt" + "log" + "os" + "path" + "syscall" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +//////////////////////////////////////////////////////////////////////// +// Test Suite Boilerplate +//////////////////////////////////////////////////////////////////////// + +type SequentialReadSuite struct { + suite.Suite + flags []string +} + +func (s *SequentialReadSuite) SetupSuite() { + setup.SetUpLogFilePath(s.T().Name(), s.flags, GKETempDir, "", testEnv.cfg) + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, s.flags, mountFunc) + setup.SetMntDir(mountDir) +} + +func (s *SequentialReadSuite) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *SequentialReadSuite) SetupTest() { + testEnv.testDirPath = setup.SetupTestDirectory(testDirName) +} + +func (s *SequentialReadSuite) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +// ////////////////////////////////////////////////////////////////////// +// Test Cases +// ////////////////////////////////////////////////////////////////////// +func (s *SequentialReadSuite) TestSequentialRead() { + fileSizeTests := []struct { + name string + fileSize int64 + }{ + { + name: "SmallFile", + fileSize: blockSizeInBytes / 2, + }, + { + name: "LargeFile", + fileSize: blockSizeInBytes * 2, + }, + } + + chunkSizesToRead := []int64{128 * util.KiB, 512 * util.KiB, 1 * util.MiB} + + for _, fsTest := range fileSizeTests { + for _, chunkSize := range chunkSizesToRead { + testName := fmt.Sprintf("%s_%dKiB_Chunk", fsTest.name, chunkSize/util.KiB) + fsTest := fsTest // Capture range variable. + chunkSize := chunkSize // Capture range variable. + + s.T().Run(testName, func(t *testing.T) { + err := os.Truncate(setup.LogFile(), 0) + require.NoError(t, err, "Failed to truncate log file") + testDir := setup.SetupTestDirectory(testDirName) + fileName := setupFileInTestDir(testEnv.ctx, testEnv.storageClient, testDir, fsTest.fileSize, t) + + expected := readFileAndValidate(testEnv.ctx, testEnv.storageClient, testDir, fileName, true, 0, chunkSize, t) + + bufferedReadLogEntry := parseAndValidateSingleBufferedReadLog(t) + validate(expected, bufferedReadLogEntry, false, t) + assert.Equal(t, int64(0), bufferedReadLogEntry.RandomSeekCount, "RandomSeekCount should be 0 for sequential reads.") + }) + } + } +} + +// TestReadHeaderFooterAndBody verifies that a single file handle can correctly handle +// a mix of random reads (header and footer) followed by a large sequential read. +// The key validation is that all these operations should be served from a single +// buffered read log entry, indicating efficient handling. +func (s *SequentialReadSuite) TestReadHeaderFooterAndBody() { + // Header and footer sizes (10KB each) + headerSize := 10 * util.KiB + footerSize := 10 * util.KiB + s.T().Run("Read header footer then body from one file handle", func(t *testing.T) { + err := os.Truncate(setup.LogFile(), 0) + require.NoError(t, err, "Failed to truncate log file") + testDir := setup.SetupTestDirectory(testDirName) + fileSize := blockSizeInBytes * 2 + // Create a file of a given size in the test directory. + fileName := setupFileInTestDir(testEnv.ctx, testEnv.storageClient, testDir, fileSize, t) + filePath := path.Join(testDir, fileName) + // Get the actual file size. + fi, err := os.Stat(filePath) + require.NoError(t, err) + actualFileSize := fi.Size() + // The size of the main content to read sequentially + bodySize := actualFileSize - int64(headerSize) - int64(footerSize) + f, err := os.OpenFile(filePath, os.O_RDONLY|syscall.O_DIRECT, 0) + require.NoError(t, err) + expected := &Expected{ + StartTimeStampSeconds: time.Now().Unix(), + BucketName: setup.TestBucket(), + ObjectName: path.Join(path.Base(testDir), fileName), + } + if setup.DynamicBucketMounted() != "" { + expected.BucketName = setup.DynamicBucketMounted() + } + + // (a) Read first 10KB (header) + readAndValidateChunk(f, path.Base(testDir), fileName, 0, int64(headerSize), t) + // (b) Read last 10KB (footer) + readAndValidateChunk(f, path.Base(testDir), fileName, actualFileSize-int64(footerSize), int64(footerSize), t) + // (c) Read the remaining content sequentially + readAndValidateChunk(f, path.Base(testDir), fileName, int64(headerSize), bodySize, t) + + // Close the file handle to trigger log generation. + operations.CloseFileShouldNotThrowError(t, f) + expected.EndTimeStampSeconds = time.Now().Unix() + // Since all reads were on the same handle, there should be one log entry. + bufferedReadLogEntry := parseAndValidateSingleBufferedReadLog(t) + validate(expected, bufferedReadLogEntry, false, t) + }) +} + +// TestReadSpanningTwoBlocks verifies that a read spanning two buffer blocks is +// handled correctly. +func (s *SequentialReadSuite) TestReadSpanningTwoBlocks() { + // Ensure file is large enough for multi-block reads. + fileSize := 3 * blockSizeInBytes + // We want to read 512KB, with 256KB in the first block and 256KB in the second. + readSize := int64(512 * util.KiB) + // Start the read 256KB before the end of the first block. + readOffset := blockSizeInBytes - (256 * util.KiB) + testDir := setup.SetupTestDirectory(testDirName) + // Truncate the log file before the read operation. + err := os.Truncate(setup.LogFile(), 0) + require.NoError(s.T(), err, "Failed to truncate log file") + fileName := setupFileInTestDir(testEnv.ctx, testEnv.storageClient, testDir, fileSize, s.T()) + + // readFileAndValidate opens, reads, and closes the file in one go. + expected := readFileAndValidate(testEnv.ctx, testEnv.storageClient, testDir, fileName, false, readOffset, readSize, s.T()) + + bufferedReadLogEntry := parseAndValidateSingleBufferedReadLog(s.T()) + validate(expected, bufferedReadLogEntry, false, s.T()) + assert.Equal(s.T(), int64(0), bufferedReadLogEntry.RandomSeekCount) +} + +//////////////////////////////////////////////////////////////////////// +// Test Function (Runs once before all tests) +//////////////////////////////////////////////////////////////////////// + +func TestSequentialReadSuite(t *testing.T) { + ts := &SequentialReadSuite{} + + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/buffered_read/setup_test.go b/tools/integration_tests/buffered_read/setup_test.go new file mode 100644 index 0000000000..a937856fa1 --- /dev/null +++ b/tools/integration_tests/buffered_read/setup_test.go @@ -0,0 +1,131 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package buffered_read + +import ( + "context" + "log" + "os" + "path" + "testing" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" +) + +const ( + testDirName = "BufferedReadTest" + testFileName = "foo" + // Global block size constant for tests + blockSizeInBytes = int64(8 * util.MiB) + GKETempDir = "/gcsfuse-tmp" +) + +var ( + mountFunc func(*test_suite.TestConfig, []string) error + // mount directory is where our tests run. + mountDir string + // root directory is the directory to be unmounted. + rootDir string +) + +type env struct { + storageClient *storage.Client + ctx context.Context + testDirPath string + cfg *test_suite.TestConfig + bucketType string +} + +var testEnv env + +//////////////////////////////////////////////////////////////////////// +// TestMain +//////////////////////////////////////////////////////////////////////// + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.BufferedRead) == 0 { + log.Println("No configuration found for buffered_read tests in config. Using flags instead.") + cfg.BufferedRead = make([]test_suite.TestConfig, 1) + cfg.BufferedRead[0].TestBucket = setup.TestBucket() + cfg.BufferedRead[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.BufferedRead[0].LogFile = setup.LogFile() + cfg.BufferedRead[0].Configs = make([]test_suite.ConfigItem, 3) + + cfg.BufferedRead[0].Configs[0].Flags = []string{ + "--enable-buffered-read --read-block-size-mb=8 --read-max-blocks-per-handle=20 --read-start-blocks-per-handle=1 --read-min-blocks-per-handle=2 --enable-kernel-reader=false --log-file=/gcsfuse-tmp/TestBufferedReadSuite.log --log-severity=TRACE", + "--client-protocol=grpc --enable-buffered-read --read-block-size-mb=8 --read-max-blocks-per-handle=20 --read-start-blocks-per-handle=1 --read-min-blocks-per-handle=2 --enable-kernel-reader=false --log-file=/gcsfuse-tmp/TestBufferedReadSuite.log --log-severity=TRACE", + } + cfg.BufferedRead[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.BufferedRead[0].Configs[0].Run = "TestSequentialReadSuite" + + cfg.BufferedRead[0].Configs[1].Flags = []string{ + "--enable-buffered-read --read-block-size-mb=8 --read-min-blocks-per-handle=2 --read-global-max-blocks=1 --read-max-blocks-per-handle=10 --read-start-blocks-per-handle=2 --enable-kernel-reader=false --log-file=/gcsfuse-tmp/TestInsufficientPoolCreationSuite.log --log-severity=TRACE", + "--client-protocol=grpc --enable-buffered-read --read-block-size-mb=8 --read-min-blocks-per-handle=2 --read-global-max-blocks=1 --read-max-blocks-per-handle=10 --read-start-blocks-per-handle=2 --enable-kernel-reader=false --log-file=/gcsfuse-tmp/TestInsufficientPoolCreationSuite.log --log-severity=TRACE", + } + cfg.BufferedRead[0].Configs[1].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.BufferedRead[0].Configs[1].Run = "TestInsufficientPoolCreationSuite" + + cfg.BufferedRead[0].Configs[2].Flags = []string{ + "--enable-buffered-read --read-block-size-mb=8 --read-max-blocks-per-handle=20 --read-start-blocks-per-handle=2 --read-min-blocks-per-handle=2 --enable-kernel-reader=false --log-file=/gcsfuse-tmp/TestRandomReadFallbackSuite.log --log-severity=TRACE", + "--client-protocol=grpc --enable-buffered-read --read-block-size-mb=8 --read-max-blocks-per-handle=20 --read-start-blocks-per-handle=2 --read-min-blocks-per-handle=2 --enable-kernel-reader=false --log-file=/gcsfuse-tmp/TestRandomReadFallbackSuite.log --log-severity=TRACE", + } + cfg.BufferedRead[0].Configs[2].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.BufferedRead[0].Configs[2].Run = "TestRandomReadFallbackSuite" + } + + testEnv.ctx = context.Background() + testEnv.bucketType = setup.TestEnvironment(testEnv.ctx, &cfg.BufferedRead[0]) + testEnv.cfg = &cfg.BufferedRead[0] + + // 2. Create storage client before running tests. + closeStorageClient := client.CreateStorageClientWithCancel(&testEnv.ctx, &testEnv.storageClient) + defer func() { + err := closeStorageClient() + if err != nil { + log.Printf("closeStorageClient failed: %v\n", err) + } + }() + + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + // Save mount and root directory variables. + mountDir, rootDir = testEnv.cfg.GKEMountedDirectory, testEnv.cfg.GKEMountedDirectory + os.Exit(setup.RunTestsForMountedDirectory(testEnv.cfg.GKEMountedDirectory, m)) + } + + // Run tests for testBucket + // Set up test directory. + setup.SetUpTestDirForTestBucket(testEnv.cfg) + // Override GKE specific paths with GCSFuse paths if running in GCE environment. + setup.OverrideFilePathsInFlagSet(testEnv.cfg, setup.TestDir()) + + // Save mount and root directory variables. + mountDir, rootDir = testEnv.cfg.GCSFuseMountedDirectory, testEnv.cfg.GCSFuseMountedDirectory + + log.Println("Running static mounting tests...") + mountFunc = static_mounting.MountGcsfuseWithStaticMountingWithConfigFile + successCode := m.Run() + + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(testEnv.cfg.TestBucket, testDirName)) + os.Exit(successCode) +} diff --git a/tools/integration_tests/cloud_profiler/cloud_profiler_test.go b/tools/integration_tests/cloud_profiler/cloud_profiler_test.go new file mode 100644 index 0000000000..fe83041aa4 --- /dev/null +++ b/tools/integration_tests/cloud_profiler/cloud_profiler_test.go @@ -0,0 +1,140 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloud_profiler_test + +// Command to run the test from gcsfuse root directory: +// go test ./tools/integration_tests/cloud_profiler/... --integrationTest --testbucket <bucket_name> -testInstalledPackage -v -timeout 20m + +import ( + "context" + "fmt" + "log" + "math" + "os" + "path" + "testing" + "time" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" +) + +const ( + testDirName = "CloudProfilerTest" + testSuffix = "cloud-profiler-test" + retryFrequency = 30 * time.Second + retryDuration = 10 * time.Minute +) + +var ( + storageClient *storage.Client + testVersionName string + testServiceName string + ctx context.Context +) + +// The alphabet defines the sort order: 0 is smallest, z is largest. +const alphabet = "0123456789abcdefghijklmnopqrstuvwxyz" +const fixedLength = 13 // math.MaxInt64 in base 36 fits in 13 characters. + +// getDecreasingString generates a string that decreases lexicographically as time increases, making newer items appear earlier in sorted results. +func getDecreasingString() string { + // Calculate the decreasing value + val := uint64(math.MaxInt64 - time.Now().UnixNano()) + + // Map the value to our 36-character alphabet + res := make([]byte, fixedLength) + for i := fixedLength - 1; i >= 0; i-- { + res[i] = alphabet[val%36] + val /= 36 + } + + return string(res) +} + +//////////////////////////////////////////////////////////////////////// +// TestMain +//////////////////////////////////////////////////////////////////////// + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + prefix := getDecreasingString() + testServiceName = fmt.Sprintf("%s-%s", prefix, testSuffix) + testVersionName = fmt.Sprintf("%s-%s", prefix, testSuffix) + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.CloudProfiler) == 0 { + log.Println("No configuration found for cloud profiler tests in config. Using flags instead.") + + // Populate the config manually. + cfg.CloudProfiler = make([]test_suite.TestConfig, 1) + cfg.CloudProfiler[0].TestBucket = setup.TestBucket() + cfg.CloudProfiler[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.CloudProfiler[0].Configs = make([]test_suite.ConfigItem, 1) + cfg.CloudProfiler[0].Configs[0].Flags = []string{ + "--enable-cloud-profiler --cloud-profiler-cpu --cloud-profiler-heap --cloud-profiler-goroutines --cloud-profiler-mutex --cloud-profiler-allocated-heap", + } + testVersionFlag := fmt.Sprintf(" --cloud-profiler-label=%s", testVersionName) + testServiceNameFlag := fmt.Sprintf(" --cloud-profiler-service-name=%s", testServiceName) + cfg.CloudProfiler[0].Configs[0].Flags[0] = cfg.CloudProfiler[0].Configs[0].Flags[0] + testVersionFlag + testServiceNameFlag + cfg.CloudProfiler[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + } else if cfg.CloudProfiler[0].GKEMountedDirectory == "" { + if len(cfg.CloudProfiler[0].Configs[0].Flags) != 1 { + log.Fatalf("Expected exactly 1 config flag, got %d", len(cfg.CloudProfiler[0].Configs[0].Flags)) + } + // If more flags are added in the future, different testVersionName and testServiceName will need to be generated for each flag set. + flag := cfg.CloudProfiler[0].Configs[0].Flags[0] + flag = setup.ReplaceOrAppendFlag(flag, "${PROFILE_LABEL}", "--cloud-profiler-label=", testVersionName) + cfg.CloudProfiler[0].Configs[0].Flags[0] = setup.ReplaceOrAppendFlag(flag, "${PROFILE_SERVICE_NAME}", "--cloud-profiler-service-name=", testServiceName) + } + + ctx = context.Background() + + bucketType := setup.TestEnvironment(ctx, &cfg.CloudProfiler[0]) + + // 2. Create storage client before running tests. + closeStorageClient := client.CreateStorageClientWithCancel(&ctx, &storageClient) + defer func() { + err := closeStorageClient() + if err != nil { + log.Fatalf("closeStorageClient failed: %v", err) + } + }() + + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + if cfg.CloudProfiler[0].GKEMountedDirectory != "" { + testVersionName = setup.ExtractServiceVersionFromFlags(cfg.CloudProfiler[0].Configs[0].Flags) + testServiceName = setup.CloudProfilerServiceNameFromFlags(cfg.CloudProfiler[0].Configs[0].Flags) + os.Exit(setup.RunTestsForMountedDirectory(cfg.CloudProfiler[0].GKEMountedDirectory, m)) + } + + logger.Infof("Enabling cloud profiler with Service Name: %s and version: %s", testServiceName, testVersionName) + + // Run tests for testBucket + // 4. Build the flag sets dynamically from the config. + flags := setup.BuildFlagSets(cfg.CloudProfiler[0], bucketType, "") + + setup.SetUpTestDirForTestBucket(&cfg.CloudProfiler[0]) + + successCode := static_mounting.RunTestsWithConfigFile(&cfg.CloudProfiler[0], flags, m) + + // Clean up test directory created. + setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(setup.TestBucket(), testDirName)) + os.Exit(successCode) +} diff --git a/tools/integration_tests/cloud_profiler/with_gcp_profiler_service_test.go b/tools/integration_tests/cloud_profiler/with_gcp_profiler_service_test.go new file mode 100644 index 0000000000..c57c8ee059 --- /dev/null +++ b/tools/integration_tests/cloud_profiler/with_gcp_profiler_service_test.go @@ -0,0 +1,145 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloud_profiler_test + +import ( + "context" + "crypto/rand" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/suite" + + "cloud.google.com/go/compute/metadata" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + gcpProfiler "google.golang.org/api/cloudprofiler/v2" + "google.golang.org/api/option" +) + +type CloudProfilerSuite struct { + suite.Suite +} + +func (s *CloudProfilerSuite) writeSingleRandomFile() error { + t := s.T() + data := make([]byte, 100*1024*1024) + if _, err := rand.Read(data); err != nil { + return fmt.Errorf("failed to generate random data: %v", err) + } + + fileName := filepath.Join(setup.MntDir(), fmt.Sprintf("load_file_%d.bin", time.Now().UnixNano())) + f, err := os.Create(fileName) + if err != nil { + return fmt.Errorf("failed to create load file: %v", err) + } + if _, err = f.Write(data); err != nil { + return fmt.Errorf("failed to write to load file %s: %v", fileName, err) + } + + if err = f.Close(); err != nil { + return fmt.Errorf("failed to close load file %s: %v", fileName, err) + } + + t.Logf("Successfully wrote 100MB to %s", fileName) + return nil +} + +func getGCPProjectID(t *testing.T) string { + fetchProjectCtx, fetchProjectCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer fetchProjectCancel() + + projectID, err := metadata.ProjectIDWithContext(fetchProjectCtx) // Reduced timeout, 5s is usually sufficient. + if err != nil { + t.Logf("metadata.ProjectIDWithContext failed: %v, try fetching from environment variable.", err) + projectID := os.Getenv("GOOGLE_CLOUD_PROJECT") + if projectID == "" { + t.Skip("Not able to fetch project ID from metadata server or GOOGLE_CLOUD_PROJECT environment variable. Skipping integration test.") + } + } + return projectID +} + +// checkIfProfileExistForServiceAndVersion queries the Cloud Profiler API for profiles +// returns true just after the first matching profile, false if no matching profile found. +// Ref: https://cloud.google.com/profiler/docs/reference/v2/rest +func checkIfProfileExistForServiceAndVersion( + ctx context.Context, + t *testing.T, // Pass testing.T for logging within the helper + profilerAPIClient *gcpProfiler.Service, + projectID string, +) (bool, error) { + + t.Logf("Querying profiles for service [%s] with version [%s]", testServiceName, testVersionName) + + listCtx, listCancel := context.WithCancel(ctx) + defer listCancel() + + pagesFetched := 0 + profileFound := false + listCall := profilerAPIClient.Projects.Profiles.List(fmt.Sprintf("projects/%s", projectID)) + err := listCall.Pages(listCtx, func(resp *gcpProfiler.ListProfilesResponse) error { + pagesFetched++ + t.Logf("Processing page %d of profiles, number of profiles in page: %d", pagesFetched, len(resp.Profiles)) + for _, p := range resp.Profiles { + if p.Deployment == nil || p.Deployment.Labels == nil { + continue + } + // Since results are sorted on page listing, seeing a greater name means our profile is not yet available. + if p.Deployment.Target > testServiceName { + return fmt.Errorf("Didn't find matching profile after fetching %d pages, profile not yet available.", pagesFetched) + } + // Return if matching profile found. + if p.Deployment.Target == testServiceName && p.Deployment.Labels["version"] == testVersionName { + t.Logf("Found matching profile: Type=%s, ServiceName=%s, Version=%s", p.ProfileType, p.Deployment.Target, p.Deployment.Labels["version"]) + profileFound = true + return fmt.Errorf("Returning error on success to break pagination early") + } + } + return nil + }) + if profileFound { + return true, nil + } + return false, err +} + +func (s *CloudProfilerSuite) TestValidateProfilerWithActualService() { + t := s.T() + // 1. Fetch GCP projectID. + // 2. Create a profiler service api client. + // 3. Make list call to the profiler service api client and fetch the profiles. + // 4. Filter and match if the right profile exists. + projectID := getGCPProjectID(t) + apiCtx := context.Background() + profilerAPIClient, err := gcpProfiler.NewService(apiCtx, option.WithScopes(gcpProfiler.CloudPlatformScope)) + if err != nil { + t.Fatalf("Failed to create Cloud Profiler API client: %v", err) + } + t.Logf("Waiting for cloud profile to eventually appear for service [%s] and version [%s]", testServiceName, testVersionName) + operations.RetryUntil(apiCtx, t, retryFrequency, retryDuration, func() (bool, error) { + if err := s.writeSingleRandomFile(); err != nil { + t.Logf("Failed to write load file: %v. So profile generation may be affected...", err) + } + return checkIfProfileExistForServiceAndVersion(apiCtx, t, profilerAPIClient, projectID) + }) +} + +func TestCloudProfilerSuite(t *testing.T) { + suite.Run(t, new(CloudProfilerSuite)) +} diff --git a/tools/integration_tests/concurrent_operations/concurrent_listing_test.go b/tools/integration_tests/concurrent_operations/concurrent_listing_test.go index 6c01fa58b2..edf8fe35e7 100644 --- a/tools/integration_tests/concurrent_operations/concurrent_listing_test.go +++ b/tools/integration_tests/concurrent_operations/concurrent_listing_test.go @@ -15,6 +15,7 @@ package concurrent_operations import ( + "context" "fmt" "os" "path" @@ -22,11 +23,12 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/test_setup" + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" ) const ( @@ -48,26 +50,30 @@ const ( // This test-suite contains parallelizable test-case. Use "-parallel n" to limit // the degree of parallelism. By default it uses GOMAXPROCS. // Ref: https://stackoverflow.com/questions/24375966/does-go-test-run-unit-tests-concurrently -type concurrentListingTest struct{} - -func (s *concurrentListingTest) Setup(t *testing.T) { - testDirPath = setup.SetupTestDirectory(testDirName) +type concurrentListingTest struct { + flags []string + storageClient *storage.Client + ctx context.Context + baseTestName string + suite.Suite } -func (s *concurrentListingTest) Teardown(t *testing.T) {} +func (s *concurrentListingTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} // createDirectoryStructureForTestCase creates initial directory structure in the // given testCaseDir. // bucket // -// explicitDir/ -// explicitDir/file1.txt -// explicitDir/file2.txt +// explicitDir/ +// explicitDir/file1.txt +// explicitDir/file2.txt func createDirectoryStructureForTestCase(t *testing.T, testCaseDir string) { - operations.CreateDirectory(path.Join(testDirPath, testCaseDir), t) + setup.SetupTestDirectory(path.Join(testDirName, testCaseDir)) // Create explicitDir structure - explicitDir := path.Join(testDirPath, testCaseDir, "explicitDir") + explicitDir := path.Join(testEnv.testDirPath, testCaseDir, "explicitDir") operations.CreateDirectory(explicitDir, t) operations.CreateFileOfSize(5, path.Join(explicitDir, "file1.txt"), t) operations.CreateFileOfSize(10, path.Join(explicitDir, "file2.txt"), t) @@ -79,11 +85,11 @@ func createDirectoryStructureForTestCase(t *testing.T, testCaseDir string) { // Test_OpenDirAndLookUp helps in detecting the deadlock when // OpenDir() and LookUpInode() request for same directory comes in parallel. -func (s *concurrentListingTest) Test_OpenDirAndLookUp(t *testing.T) { - t.Parallel() // Mark the test parallelizable. +func (s *concurrentListingTest) Test_OpenDirAndLookUp() { + s.T().Parallel() // Mark the test parallelizable. testCaseDir := "Test_OpenDirAndLookUp" - createDirectoryStructureForTestCase(t, testCaseDir) - targetDir := path.Join(testDirPath, testCaseDir, "explicitDir") + createDirectoryStructureForTestCase(s.T(), testCaseDir) + targetDir := path.Join(testEnv.testDirPath, testCaseDir, "explicitDir") var wg sync.WaitGroup wg.Add(2) // Fails if the operation takes more than timeout. @@ -92,21 +98,21 @@ func (s *concurrentListingTest) Test_OpenDirAndLookUp(t *testing.T) { // Goroutine 1: Repeatedly calls OpenDir. go func() { defer wg.Done() - for i := 0; i < iterationsForLightOperations; i++ { + for range iterationsForLightOperations { f, err := os.Open(targetDir) - assert.Nil(t, err) + require.Nil(s.T(), err) err = f.Close() - assert.Nil(t, err) + require.Nil(s.T(), err) } }() // Goroutine 1: Repeatedly calls Stat. go func() { defer wg.Done() - for i := 0; i < iterationsForLightOperations; i++ { + for range iterationsForLightOperations { _, err := os.Stat(targetDir) - assert.Nil(t, err) + require.Nil(s.T(), err) } }() @@ -120,17 +126,17 @@ func (s *concurrentListingTest) Test_OpenDirAndLookUp(t *testing.T) { case <-done: // Operation completed successfully before timeout. case <-time.After(timeout): - assert.FailNow(t, "Possible deadlock") + assert.FailNow(s.T(), "Possible deadlock") } } // Test_Parallel_ReadDirAndLookUp tests for potential deadlocks or race conditions when // ReadDir() is called concurrently with LookUp of same dir. -func (s *concurrentListingTest) Test_Parallel_ReadDirAndLookUp(t *testing.T) { - t.Parallel() // Mark the test parallelizable. +func (s *concurrentListingTest) Test_Parallel_ReadDirAndLookUp() { + s.T().Parallel() // Mark the test parallelizable. testCaseDir := "Test_Parallel_ReadDirAndLookUp" - createDirectoryStructureForTestCase(t, testCaseDir) - targetDir := path.Join(testDirPath, testCaseDir, "explicitDir") + createDirectoryStructureForTestCase(s.T(), testCaseDir) + targetDir := path.Join(testEnv.testDirPath, testCaseDir, "explicitDir") var wg sync.WaitGroup wg.Add(2) timeout := 200 * time.Second @@ -138,24 +144,24 @@ func (s *concurrentListingTest) Test_Parallel_ReadDirAndLookUp(t *testing.T) { // Goroutine 1: Repeatedly calls Readdir go func() { defer wg.Done() - for i := 0; i < iterationsForMediumOperations; i++ { + for range iterationsForMediumOperations { f, err := os.Open(targetDir) - assert.Nil(t, err) + require.Nil(s.T(), err) _, err = f.Readdirnames(-1) - assert.Nil(t, err) + require.Nil(s.T(), err) err = f.Close() - assert.Nil(t, err) + require.Nil(s.T(), err) } }() // Goroutine 2: Repeatedly stats go func() { defer wg.Done() - for i := 0; i < iterationsForLightOperations; i++ { + for range iterationsForLightOperations { _, err := os.Stat(targetDir) - assert.Nil(t, err) + require.Nil(s.T(), err) } }() @@ -170,36 +176,36 @@ func (s *concurrentListingTest) Test_Parallel_ReadDirAndLookUp(t *testing.T) { case <-done: // Success: Both operations finished before timeout case <-time.After(timeout): - assert.FailNow(t, "Possible deadlock or race condition detected during Readdir and directory operations") + assert.FailNow(s.T(), "Possible deadlock or race condition detected during Readdir and directory operations") } } // Test_MultipleConcurrentReadDir tests for potential deadlocks or race conditions // when multiple goroutines call Readdir() concurrently on the same directory. -func (s *concurrentListingTest) Test_MultipleConcurrentReadDir(t *testing.T) { - t.Parallel() // Mark the test parallelizable. +func (s *concurrentListingTest) Test_MultipleConcurrentReadDir() { + s.T().Parallel() // Mark the test parallelizable. testCaseDir := "Test_MultipleConcurrentReadDir" - createDirectoryStructureForTestCase(t, testCaseDir) - targetDir := path.Join(testDirPath, testCaseDir, "explicitDir") + createDirectoryStructureForTestCase(s.T(), testCaseDir) + targetDir := path.Join(testEnv.testDirPath, testCaseDir, "explicitDir") var wg sync.WaitGroup goroutineCount := 10 // Number of concurrent goroutines wg.Add(goroutineCount) timeout := 600 * time.Second // More timeout to accommodate the high listing time without kernel-list-cache. // Create multiple go routines to listing concurrently. - for i := 0; i < goroutineCount; i++ { + for range goroutineCount { go func() { defer wg.Done() - for j := 0; j < iterationsForMediumOperations; j++ { + for range iterationsForMediumOperations { f, err := os.Open(targetDir) - assert.Nil(t, err) + require.Nil(s.T(), err) _, err = f.Readdirnames(-1) // Read all directory entries - assert.Nil(t, err) + require.Nil(s.T(), err) err = f.Close() - assert.Nil(t, err) + require.Nil(s.T(), err) } }() } @@ -215,17 +221,17 @@ func (s *concurrentListingTest) Test_MultipleConcurrentReadDir(t *testing.T) { case <-done: // Success: All Readdir operations finished before timeout case <-time.After(timeout): - assert.FailNow(t, "Possible deadlock or race condition detected during concurrent Readdir calls") + assert.FailNow(s.T(), "Possible deadlock or race condition detected during concurrent Readdir calls") } } // Test_Parallel_ReadDirAndFileOperations detects race conditions and deadlocks when one goroutine // performs Readdir() while another concurrently creates and deletes files in the same directory. -func (s *concurrentListingTest) Test_Parallel_ReadDirAndFileOperations(t *testing.T) { - t.Parallel() // Mark the test parallelizable. +func (s *concurrentListingTest) Test_Parallel_ReadDirAndFileOperations() { + s.T().Parallel() // Mark the test parallelizable. testCaseDir := "Test_Parallel_ReadDirAndFileOperations" - createDirectoryStructureForTestCase(t, testCaseDir) - targetDir := path.Join(testDirPath, testCaseDir, "explicitDir") + createDirectoryStructureForTestCase(s.T(), testCaseDir) + targetDir := path.Join(testEnv.testDirPath, testCaseDir, "explicitDir") var wg sync.WaitGroup wg.Add(2) timeout := 400 * time.Second // Adjust timeout as needed @@ -233,39 +239,40 @@ func (s *concurrentListingTest) Test_Parallel_ReadDirAndFileOperations(t *testin // Goroutine 1: Repeatedly calls Readdir go func() { defer wg.Done() - for i := 0; i < iterationsForMediumOperations; i++ { // Adjust iteration count if needed + for range iterationsForMediumOperations { // Adjust iteration count if needed f, err := os.Open(targetDir) - assert.Nil(t, err) + require.Nil(s.T(), err) _, err = f.Readdirnames(-1) - assert.Nil(t, err) + require.Nil(s.T(), err) err = f.Close() - assert.Nil(t, err) + require.Nil(s.T(), err) } }() // Goroutine 2: Creates and deletes files go func() { defer wg.Done() - for i := 0; i < iterationsForHeavyOperations; i++ { // Adjust iteration count if needed - filePath := path.Join(targetDir, "tmp_file.txt") - renamedFilePath := path.Join(targetDir, "renamed_tmp_file.txt") + for range iterationsForHeavyOperations { // Adjust iteration count if needed + filePrefix := setup.GenerateRandomString(5) + filePath := path.Join(targetDir, filePrefix+"tmp_file.txt") + renamedFilePath := path.Join(targetDir, filePrefix+"renamed_tmp_file.txt") // Create f, err := os.Create(filePath) - assert.Nil(t, err) + require.Nil(s.T(), err) err = f.Close() - assert.Nil(t, err) + require.Nil(s.T(), err) // Rename err = os.Rename(filePath, renamedFilePath) - assert.Nil(t, err) + require.Nil(s.T(), err) // Delete err = os.Remove(renamedFilePath) - assert.Nil(t, err) + require.Nil(s.T(), err) } }() @@ -280,17 +287,17 @@ func (s *concurrentListingTest) Test_Parallel_ReadDirAndFileOperations(t *testin case <-done: // Success: Both operations finished before timeout case <-time.After(timeout): - assert.FailNow(t, "Possible deadlock or race condition detected") + assert.FailNow(s.T(), "Possible deadlock or race condition detected") } } // Test_Parallel_ReadDirAndDirOperations tests for potential deadlocks or race conditions when // ReadDir() is called concurrently with directory creation and deletion operations. -func (s *concurrentListingTest) Test_Parallel_ReadDirAndDirOperations(t *testing.T) { - t.Parallel() // Mark the test parallelizable. +func (s *concurrentListingTest) Test_Parallel_ReadDirAndDirOperations() { + s.T().Parallel() // Mark the test parallelizable. testCaseDir := "Test_Parallel_ReadDirAndDirOperations" - createDirectoryStructureForTestCase(t, testCaseDir) - targetDir := path.Join(testDirPath, testCaseDir, "explicitDir") + createDirectoryStructureForTestCase(s.T(), testCaseDir) + targetDir := path.Join(testEnv.testDirPath, testCaseDir, "explicitDir") var wg sync.WaitGroup wg.Add(2) timeout := 200 * time.Second @@ -298,36 +305,37 @@ func (s *concurrentListingTest) Test_Parallel_ReadDirAndDirOperations(t *testing // Goroutine 1: Repeatedly calls Readdir go func() { defer wg.Done() - for i := 0; i < iterationsForMediumOperations; i++ { + for range iterationsForMediumOperations { f, err := os.Open(targetDir) - assert.Nil(t, err) + require.Nil(s.T(), err) _, err = f.Readdirnames(-1) - assert.Nil(t, err) + require.Nil(s.T(), err) err = f.Close() - assert.Nil(t, err) + require.Nil(s.T(), err) } }() // Goroutine 2: Creates and deletes directories go func() { defer wg.Done() - for i := 0; i < iterationsForHeavyOperations; i++ { - dirPath := path.Join(targetDir, "test_dir") - renamedDirPath := path.Join(targetDir, "renamed_test_dir") + for range iterationsForHeavyOperations { + dirPrefix := setup.GenerateRandomString(5) + dirPath := path.Join(targetDir, dirPrefix+"test_dir") + renamedDirPath := path.Join(targetDir, dirPrefix+"renamed_test_dir") // Create err := os.Mkdir(dirPath, 0755) - assert.Nil(t, err) + require.Nil(s.T(), err) // Rename err = os.Rename(dirPath, renamedDirPath) - assert.Nil(t, err) + require.Nil(s.T(), err) // Delete err = os.Remove(renamedDirPath) - assert.Nil(t, err) + require.Nil(s.T(), err) } }() @@ -342,17 +350,17 @@ func (s *concurrentListingTest) Test_Parallel_ReadDirAndDirOperations(t *testing case <-done: // Success: Both operations finished before timeout case <-time.After(timeout): - assert.FailNow(t, "Possible deadlock or race condition detected during Readdir and directory operations") + assert.FailNow(s.T(), "Possible deadlock or race condition detected during Readdir and directory operations") } } // Test_Parallel_ReadDirAndFileEdit tests for potential deadlocks or race conditions when // ReadDir() is called concurrently with modification of underneath file. -func (s *concurrentListingTest) Test_Parallel_ReadDirAndFileEdit(t *testing.T) { - t.Parallel() // Mark the test parallelizable. +func (s *concurrentListingTest) Test_Parallel_ReadDirAndFileEdit() { + s.T().Parallel() // Mark the test parallelizable. testCaseDir := "Test_Parallel_ListDirAndFileEdit" - createDirectoryStructureForTestCase(t, testCaseDir) - targetDir := path.Join(testDirPath, testCaseDir, "explicitDir") + createDirectoryStructureForTestCase(s.T(), testCaseDir) + targetDir := path.Join(testEnv.testDirPath, testCaseDir, "explicitDir") var wg sync.WaitGroup wg.Add(2) timeout := 400 * time.Second @@ -360,35 +368,36 @@ func (s *concurrentListingTest) Test_Parallel_ReadDirAndFileEdit(t *testing.T) { // Goroutine 1: Repeatedly calls Readdir go func() { defer wg.Done() - for i := 0; i < iterationsForMediumOperations; i++ { + for range iterationsForMediumOperations { f, err := os.Open(targetDir) - assert.Nil(t, err) + require.Nil(s.T(), err) _, err = f.Readdirnames(-1) - assert.Nil(t, err) + require.Nil(s.T(), err) err = f.Close() - assert.Nil(t, err) + require.Nil(s.T(), err) } }() // Goroutine 2: Create and edit files go func() { defer wg.Done() - for i := 0; i < iterationsForHeavyOperations; i++ { + for i := range iterationsForHeavyOperations { filePath := path.Join(targetDir, fmt.Sprintf("test_file_%d.txt", i)) // Create file err := os.WriteFile(filePath, []byte("Hello, world!"), setup.FilePermission_0600) - assert.Nil(t, err) + require.Nil(s.T(), err) + time.Sleep(time.Second) // Edit file (append some data) f, err := os.OpenFile(filePath, os.O_APPEND|os.O_WRONLY, setup.FilePermission_0600) - assert.Nil(t, err) + require.Nil(s.T(), err) _, err = f.Write([]byte("This is an edit.")) - assert.Nil(t, err) + require.Nil(s.T(), err) err = f.Close() - assert.Nil(t, err) + require.Nil(s.T(), err) } }() @@ -403,17 +412,17 @@ func (s *concurrentListingTest) Test_Parallel_ReadDirAndFileEdit(t *testing.T) { case <-done: // Success: Both operations finished before timeout case <-time.After(timeout): - assert.FailNow(t, "Possible deadlock or race condition detected during Readdir and directory operations") + assert.FailNow(s.T(), "Possible deadlock or race condition detected during Readdir and directory operations") } } // Test_MultipleConcurrentOperations tests for potential deadlocks or race conditions when // listing, file or folder operations, stat, opendir, file modifications happening concurrently. -func (s *concurrentListingTest) Test_MultipleConcurrentOperations(t *testing.T) { - t.Parallel() // Mark the test parallelizable. +func (s *concurrentListingTest) Test_MultipleConcurrentOperations() { + s.T().Parallel() // Mark the test parallelizable. testCaseDir := "Test_MultipleConcurrentOperations" - createDirectoryStructureForTestCase(t, testCaseDir) - targetDir := path.Join(testDirPath, testCaseDir, "explicitDir") + createDirectoryStructureForTestCase(s.T(), testCaseDir) + targetDir := path.Join(testEnv.testDirPath, testCaseDir, "explicitDir") var wg sync.WaitGroup wg.Add(5) timeout := 400 * time.Second @@ -421,77 +430,79 @@ func (s *concurrentListingTest) Test_MultipleConcurrentOperations(t *testing.T) // Goroutine 1: Repeatedly calls Readdir go func() { defer wg.Done() - for i := 0; i < iterationsForMediumOperations; i++ { // Adjust iteration count if needed + for range iterationsForMediumOperations { // Adjust iteration count if needed f, err := os.Open(targetDir) - assert.Nil(t, err) + require.Nil(s.T(), err) _, err = f.Readdirnames(-1) - assert.Nil(t, err) + require.Nil(s.T(), err) err = f.Close() - assert.Nil(t, err) + require.Nil(s.T(), err) } }() // Goroutine 2: Create and edit files go func() { defer wg.Done() - for i := 0; i < iterationsForHeavyOperations; i++ { + for i := range iterationsForHeavyOperations { filePath := path.Join(targetDir, fmt.Sprintf("test_file_%d.txt", i)) // Create file err := os.WriteFile(filePath, []byte("Hello, world!"), setup.FilePermission_0600) - assert.Nil(t, err) + require.Nil(s.T(), err) + time.Sleep(time.Second) // Edit file (append some data) f, err := os.OpenFile(filePath, os.O_APPEND|os.O_WRONLY, setup.FilePermission_0600) - assert.Nil(t, err) + require.Nil(s.T(), err) _, err = f.Write([]byte("This is an edit.")) - assert.Nil(t, err) + require.Nil(s.T(), err) err = f.Close() - assert.Nil(t, err) + require.Nil(s.T(), err) } }() // Goroutine 3: Creates and deletes directories go func() { defer wg.Done() - for i := 0; i < iterationsForHeavyOperations; i++ { - dirPath := path.Join(targetDir, "test_dir") - renamedDirPath := path.Join(targetDir, "renamed_test_dir") + for range iterationsForHeavyOperations { + dirPrefix := setup.GenerateRandomString(5) + dirPath := path.Join(targetDir, dirPrefix+"test_dir") + renamedDirPath := path.Join(targetDir, dirPrefix+"renamed_test_dir") // Create err := os.Mkdir(dirPath, 0755) - assert.Nil(t, err) + require.Nil(s.T(), err) // Rename err = os.Rename(dirPath, renamedDirPath) - assert.Nil(t, err) + require.Nil(s.T(), err) // Delete err = os.Remove(renamedDirPath) - assert.Nil(t, err) + require.Nil(s.T(), err) } }() // Goroutine 4: Repeatedly stats go func() { defer wg.Done() - for i := 0; i < iterationsForLightOperations; i++ { + for range iterationsForLightOperations { _, err := os.Stat(targetDir) - assert.Nil(t, err) + require.Nil(s.T(), err) } }() // Goroutine 5: Repeatedly calls OpenDir. go func() { defer wg.Done() - for i := 0; i < iterationsForLightOperations; i++ { + for range iterationsForLightOperations { f, err := os.Open(targetDir) - assert.Nil(t, err) + require.Nil(s.T(), err) err = f.Close() - assert.Nil(t, err) + require.Nil(s.T(), err) } }() @@ -506,17 +517,17 @@ func (s *concurrentListingTest) Test_MultipleConcurrentOperations(t *testing.T) case <-done: // Success: Both operations finished before timeout case <-time.After(timeout): - assert.FailNow(t, "Possible deadlock or race condition detected during Readdir and directory operations") + assert.FailNow(s.T(), "Possible deadlock or race condition detected during Readdir and directory operations") } } // Test_ListWithMoveFile tests for potential deadlocks or race conditions when // listing, file or folder operations, move file happening concurrently. -func (s *concurrentListingTest) Test_ListWithMoveFile(t *testing.T) { - t.Parallel() // Mark the test parallelizable. +func (s *concurrentListingTest) Test_ListWithMoveFile() { + s.T().Parallel() // Mark the test parallelizable. testCaseDir := "Test_ListWithMoveFile" - createDirectoryStructureForTestCase(t, testCaseDir) - targetDir := path.Join(testDirPath, testCaseDir, "explicitDir") + createDirectoryStructureForTestCase(s.T(), testCaseDir) + targetDir := path.Join(testEnv.testDirPath, testCaseDir, "explicitDir") var wg sync.WaitGroup wg.Add(2) timeout := 400 * time.Second // Adjust timeout as needed @@ -524,31 +535,32 @@ func (s *concurrentListingTest) Test_ListWithMoveFile(t *testing.T) { // Goroutine 1: Repeatedly calls Readdir go func() { defer wg.Done() - for i := 0; i < iterationsForMediumOperations; i++ { // Adjust iteration count if needed + for range iterationsForMediumOperations { // Adjust iteration count if needed f, err := os.Open(targetDir) - assert.NoError(t, err) + require.NoError(s.T(), err) _, err = f.Readdirnames(-1) - assert.Nil(t, err) + require.Nil(s.T(), err) - assert.NoError(t, f.Close()) + require.NoError(s.T(), f.Close()) } }() - // Create file - err := os.WriteFile(path.Join(testDirPath, "move_file.txt"), []byte("Hello, world!"), setup.FilePermission_0600) - require.NoError(t, err) - // Goroutine 2: Move file go func() { defer wg.Done() - for i := 0; i < iterationsForHeavyOperations; i++ { // Adjust iteration count if needed + for range iterationsForHeavyOperations { // Adjust iteration count if needed + fileName := setup.GenerateRandomString(5) + "move_file.txt" + // Create file + err := os.WriteFile(path.Join(testEnv.testDirPath, fileName), []byte("Hello, world!"), setup.FilePermission_0600) + require.NoError(s.T(), err) + // Move File in the target directory - err = operations.Move(path.Join(testDirPath, "move_file.txt"), path.Join(targetDir, "move_file.txt")) - assert.NoError(t, err) + err = operations.Move(path.Join(testEnv.testDirPath, fileName), path.Join(targetDir, fileName)) + require.NoError(s.T(), err) // Move File out of the target directory - err = operations.Move(path.Join(targetDir, "move_file.txt"), path.Join(testDirPath, "move_file.txt")) - assert.NoError(t, err) + err = operations.Move(path.Join(targetDir, fileName), path.Join(testEnv.testDirPath, fileName)) + require.NoError(s.T(), err) } }() @@ -563,17 +575,17 @@ func (s *concurrentListingTest) Test_ListWithMoveFile(t *testing.T) { case <-done: // Success: Both operations finished before timeout case <-time.After(timeout): - assert.FailNow(t, "Possible deadlock or race condition detected") + assert.FailNow(s.T(), "Possible deadlock or race condition detected") } } // Test_ListWithMoveDir tests for potential deadlocks or race conditions when // listing, file or folder operations, move dir happening concurrently. -func (s *concurrentListingTest) Test_ListWithMoveDir(t *testing.T) { - t.Parallel() // Mark the test parallelizable. +func (s *concurrentListingTest) Test_ListWithMoveDir() { + s.T().Parallel() // Mark the test parallelizable. testCaseDir := "Test_ListWithMoveDir" - createDirectoryStructureForTestCase(t, testCaseDir) - targetDir := path.Join(testDirPath, testCaseDir, "explicitDir") + createDirectoryStructureForTestCase(s.T(), testCaseDir) + targetDir := path.Join(testEnv.testDirPath, testCaseDir, "explicitDir") var wg sync.WaitGroup wg.Add(2) timeout := 400 * time.Second // Adjust timeout as needed @@ -581,30 +593,32 @@ func (s *concurrentListingTest) Test_ListWithMoveDir(t *testing.T) { // Goroutine 1: Repeatedly calls Readdir go func() { defer wg.Done() - for i := 0; i < iterationsForMediumOperations; i++ { // Adjust iteration count if needed + for range iterationsForMediumOperations { // Adjust iteration count if needed f, err := os.Open(targetDir) - require.NoError(t, err) + require.NoError(s.T(), err) _, err = f.Readdirnames(-1) - assert.Nil(t, err) + require.Nil(s.T(), err) - assert.NoError(t, f.Close()) + require.NoError(s.T(), f.Close()) } }() - // Create Dir - err := os.Mkdir(path.Join(testDirPath, "move_dir"), setup.DirPermission_0755) - require.NoError(t, err) // Goroutine 2: Move Dir go func() { defer wg.Done() - for i := 0; i < iterationsForHeavyOperations; i++ { // Adjust iteration count if needed + for range iterationsForHeavyOperations { // Adjust iteration count if needed + dirName := setup.GenerateRandomString(5) + "move_dir" + // Create Dir + err := os.Mkdir(path.Join(testEnv.testDirPath, dirName), setup.DirPermission_0755) + require.NoError(s.T(), err) + // Move Dir in the target dir - err = operations.Move(path.Join(testDirPath, "move_dir"), path.Join(targetDir, "move_dir")) - assert.NoError(t, err) + err = operations.Move(path.Join(testEnv.testDirPath, dirName), path.Join(targetDir, dirName)) + require.NoError(s.T(), err) // Move Dir out of the target dir - err = operations.Move(path.Join(targetDir, "move_dir"), path.Join(testDirPath, "move_dir")) - assert.NoError(t, err) + err = operations.Move(path.Join(targetDir, dirName), path.Join(testEnv.testDirPath, dirName)) + require.NoError(s.T(), err) } }() @@ -619,17 +633,17 @@ func (s *concurrentListingTest) Test_ListWithMoveDir(t *testing.T) { case <-done: // Success: Both operations finished before timeout case <-time.After(timeout): - assert.FailNow(t, "Possible deadlock or race condition detected") + assert.FailNow(s.T(), "Possible deadlock or race condition detected") } } // Test_StatWithNewFileWrite tests for potential deadlocks or race conditions when // statting and creating a new file happen concurrently. -func (s *concurrentListingTest) Test_StatWithNewFileWrite(t *testing.T) { - t.Parallel() +func (s *concurrentListingTest) Test_StatWithNewFileWrite() { + s.T().Parallel() testCaseDir := "Test_StatWithNewFileWrite" - createDirectoryStructureForTestCase(t, testCaseDir) - targetDir := path.Join(testDirPath, testCaseDir, "explicitDir") + createDirectoryStructureForTestCase(s.T(), testCaseDir) + targetDir := path.Join(testEnv.testDirPath, testCaseDir, "explicitDir") var wg sync.WaitGroup wg.Add(2) timeout := 400 * time.Second // Adjust timeout as needed @@ -637,22 +651,22 @@ func (s *concurrentListingTest) Test_StatWithNewFileWrite(t *testing.T) { // Goroutine 1: Repeatedly calls Stat go func() { defer wg.Done() - for i := 0; i < iterationsForMediumOperations; i++ { + for range iterationsForMediumOperations { _, err := os.Stat(targetDir) - assert.NoError(t, err) + require.NoError(s.T(), err) } }() // Goroutine 2: Repeatedly create a file. go func() { defer wg.Done() - for i := 0; i < iterationsForMediumOperations; i++ { + for i := range iterationsForMediumOperations { // Create file filePath := path.Join(targetDir, fmt.Sprintf("tmp_file_%d.txt", i)) err := os.WriteFile(filePath, []byte("Hello, world!"), setup.FilePermission_0600) - assert.NoError(t, err) + require.NoError(s.T(), err) } }() @@ -667,7 +681,7 @@ func (s *concurrentListingTest) Test_StatWithNewFileWrite(t *testing.T) { case <-done: // Success: Both operations finished before timeout case <-time.After(timeout): - assert.FailNow(t, "Possible deadlock or race condition detected") + assert.FailNow(s.T(), "Possible deadlock or race condition detected") } } @@ -676,6 +690,47 @@ func (s *concurrentListingTest) Test_StatWithNewFileWrite(t *testing.T) { //////////////////////////////////////////////////////////////////////// func TestConcurrentListing(t *testing.T) { - ts := &concurrentListingTest{} - test_setup.RunTests(t, ts) + // Run tests for mounted directory if the flag is set. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + ts := &concurrentListingTest{ + ctx: context.Background(), + storageClient: testEnv.storageClient, + baseTestName: t.Name(), + } + setup.SetUpLogFilePath(t.Name(), nil, GKETempDir, "", testEnv.cfg) + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, nil, mountFunc) + setup.SetMntDir(mountDir) + testEnv.testDirPath = path.Join(setup.MntDir(), testDirName) + t.Run("MountedDirectory", func(t *testing.T) { + suite.Run(t, ts) + }) + setup.UnmountGCSFuseWithConfig(testEnv.cfg) + return + } + + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, flags := range flagsSet { + ts := &concurrentListingTest{ + ctx: context.Background(), + storageClient: testEnv.storageClient, + baseTestName: t.Name(), + flags: flags, + } + + // Mount logic must remain OUTSIDE of the test methods so the mount stays alive + setup.SetUpLogFilePath(t.Name(), flags, GKETempDir, "", testEnv.cfg) + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, flags, mountFunc) + setup.SetMntDir(mountDir) + testEnv.testDirPath = path.Join(setup.MntDir(), testDirName) + + // Parallel subtest execution is suspended until its calling test function has + // returned. Hence invoking RunTest inside another test, otherwise unmount will + // happen before the subtest execution starts. + t.Run(fmt.Sprintf("Flags_%v", flags), func(t *testing.T) { + suite.Run(t, ts) + }) + + // Safely unmount AFTER the wrapper t.Run completes (which waits for all parallel subtests) + setup.UnmountGCSFuseWithConfig(testEnv.cfg) + } } diff --git a/tools/integration_tests/concurrent_operations/concurrent_read_test.go b/tools/integration_tests/concurrent_operations/concurrent_read_test.go new file mode 100644 index 0000000000..b63b8fccd3 --- /dev/null +++ b/tools/integration_tests/concurrent_operations/concurrent_read_test.go @@ -0,0 +1,304 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package concurrent_operations + +import ( + "bytes" + "context" + "fmt" + "math/rand" + "os" + "path" + "sync" + "syscall" + "testing" + "time" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +type concurrentReadTest struct { + flags []string + storageClient *storage.Client + ctx context.Context + baseTestName string + suite.Suite +} + +func (s *concurrentReadTest) SetupSuite() { + setup.SetUpLogFilePath(s.T().Name(), s.flags, GKETempDir, "", testEnv.cfg) + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, s.flags, mountFunc) + setup.SetMntDir(mountDir) +} + +func (s *concurrentReadTest) SetupTest() { + testEnv.testDirPath = setup.SetupTestDirectory(testDirName) +} + +func (s *concurrentReadTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *concurrentReadTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +//////////////////////////////////////////////////////////////////////// +// Test scenarios +//////////////////////////////////////////////////////////////////////// + +// Test_ConcurrentSequentialAndRandomReads tests concurrent read operations where +// 5 goroutines read a 100MiB file sequentially and 5 goroutines read randomly. +// This test validates that concurrent sequential and random read patterns work +// correctly without deadlocks or race conditions. It also validates data integrity +// using CRC32 checksums for sequential reads and chunk validation for random reads. +func (s *concurrentReadTest) Test_ConcurrentSequentialAndRandomReads() { + const ( + fileSize = 100 * operations.OneMiB // 100 MiB file + chunkSize = 64 * operations.OneKiB // 64 KiB chunks for reads + sequentialReads = 5 // Number of sequential readers + randomReads = 5 // Number of random readers + ) + // Create a 100MiB test file + testFilePath := path.Join(testEnv.testDirPath, "large_test_file.bin") + operations.CreateFileOfSize(fileSize, testFilePath, s.T()) + var wg sync.WaitGroup + timeout := 300 * time.Second // 5 minutes timeout for 100MiB operations + + // Launch 5 sequential readers + for i := range sequentialReads { + wg.Add(1) + go func(readerID int) { + defer wg.Done() + // Use operations.ReadFileSequentially to read the entire file + file, err := os.OpenFile(testFilePath, os.O_RDONLY|syscall.O_DIRECT, setup.FilePermission_0600) + require.NoError(s.T(), err) + content, err := operations.ReadFileSequentially(file, chunkSize) + require.NoError(s.T(), err, "Sequential reader %d: read failed.", readerID) + require.Equal(s.T(), fileSize, len(content), "Sequential reader %d: expected to read entire file", readerID) + obj := s.storageClient.Bucket(setup.TestBucket()).Object(path.Join(path.Base(testEnv.testDirPath), "large_test_file.bin")) + attrs, err := obj.Attrs(s.ctx) + require.NoError(s.T(), err, "obj.Attrs") + localCRC32C, err := operations.CalculateCRC32(bytes.NewReader(content)) + require.NoError(s.T(), err, "Sequential reader %d: failed to calculate local CRC32C", readerID) + assert.Equal(s.T(), attrs.CRC32C, localCRC32C, "Sequential reader %d: CRC32C mismatch. GCS: %d, Local: %d", readerID, attrs.CRC32C, localCRC32C) + }(i) + } + // Launch 5 random readers + for i := range randomReads { + wg.Add(1) + go func(readerID int) { + defer wg.Done() + numRandomReads := 200 // Number of random read operations per goroutine + rand.New(rand.NewSource(time.Now().UnixNano() + int64(readerID))) + for range numRandomReads { + // Generate random offset within bound. + randomOffset := int64(rand.Intn(fileSize/chunkSize)) * chunkSize + // Use operations.ReadChunkFromFile for reading chunks + chunk, err := operations.ReadChunkFromFile(testFilePath, chunkSize, randomOffset, os.O_RDONLY) + require.NoError(s.T(), err, "Random reader %d: ReadChunkFromFile failed at offset %d", readerID, randomOffset) + client.ValidateObjectChunkFromGCS(s.ctx, s.storageClient, path.Base(testEnv.testDirPath), "large_test_file.bin", randomOffset, int64(len(chunk)), string(chunk), s.T()) + } + }(i) + } + // Wait for all goroutines or timeout + done := make(chan bool, 1) + go func() { + wg.Wait() + done <- true + }() + + select { + case <-done: + s.T().Log("All concurrent read operations completed successfully") + case <-time.After(timeout): + assert.FailNow(s.T(), "Concurrent read operations timed out - possible deadlock or performance issue") + } +} + +// Test_ConcurrentSegmentReadsSharedHandle tests concurrent read operations where +// 5 goroutines read different segments of a file using the same shared file handle. +// This test validates that multiple goroutines can safely read from different +// parts of the same file using a single shared file handle without race conditions, +// with each reader handling a distinct segment of the file for comprehensive coverage. +func (s *concurrentReadTest) Test_ConcurrentSegmentReadsSharedHandle() { + const ( + fileSize = 100 * operations.OneMiB // 100 MiB file + numReaders = 5 // Number of concurrent readers + segmentSize = fileSize / numReaders // Each reader reads 20 MiB segment + ) + // Create a 100MiB test file + testFilePath := path.Join(testEnv.testDirPath, "segment_test_file.bin") + operations.CreateFileOfSize(fileSize, testFilePath, s.T()) + // Open shared file handle that will be used by all goroutines + sharedFile, err := os.Open(testFilePath) + require.NoError(s.T(), err, "Failed to open shared file handle") + defer func() { + err := sharedFile.Close() + require.NoError(s.T(), err, "Failed to close shared file handle") + }() + var wg sync.WaitGroup + segmentData := make([][]byte, numReaders) + timeout := 300 * time.Second // 5 minutes timeout + + // Launch 5 readers, each reading a different segment using the shared file handle + for i := range numReaders { + wg.Add(1) + go func(readerID int) { + defer wg.Done() + // Calculate segment boundaries + segmentStart := int64(readerID) * segmentSize + segmentEnd := segmentStart + segmentSize + if readerID == numReaders-1 { + // Last reader takes any remaining bytes + segmentEnd = fileSize + } + actualSegmentSize := segmentEnd - segmentStart + // Read segment using shared file handle with ReadAt + buffer := make([]byte, actualSegmentSize) + n, err := sharedFile.ReadAt(buffer, segmentStart) + require.NoError(s.T(), err, "Reader %d: ReadAt failed for segment %d-%d", readerID, segmentStart, segmentEnd-1) + require.Equal(s.T(), int(actualSegmentSize), n, "Reader %d: expected to read %d bytes, got %d", readerID, actualSegmentSize, n) + // Store segment data for later validation + segmentData[readerID] = buffer + }(i) + } + // Wait for all goroutines or timeout + done := make(chan bool, 1) + go func() { + wg.Wait() + done <- true + }() + + select { + case <-done: + s.T().Log("All concurrent segment read operations completed successfully") + // Reconstruct the full file from segments and validate checksum + var fullContent bytes.Buffer + for i, segment := range segmentData { + n, err := fullContent.Write(segment) + require.NoError(s.T(), err, "Failed to write segment %d to buffer", i) + require.Equal(s.T(), len(segment), n, "Segment %d: wrote different number of bytes than expected", i) + } + // Validate total size + require.Equal(s.T(), fileSize, fullContent.Len(), "Reconstructed file size mismatch") + // Validate checksum of reconstructed content + reconstructedChecksum, err := operations.CalculateCRC32(bytes.NewReader(fullContent.Bytes())) + require.NoError(s.T(), err, "Failed to calculate reconstructed checksum") + obj := s.storageClient.Bucket(setup.TestBucket()).Object(path.Join(path.Base(testEnv.testDirPath), "segment_test_file.bin")) + attrs, err := obj.Attrs(s.ctx) + require.NoError(s.T(), err, "obj.Attrs") + assert.Equal(s.T(), attrs.CRC32C, reconstructedChecksum, "CRC32C mismatch. GCS: %d, Local: %d", attrs.CRC32C, reconstructedChecksum) + case <-time.After(timeout): + assert.FailNow(s.T(), "Concurrent segment read operations timed out - possible deadlock or performance issue") + } +} + +// Test_MultiThreadedWritePlusRead tests multiple threads doing write followed by read concurrently on different files. +// It creates 10 goroutines, each writing a 32 MiB file and then reading it sequentially. +func (s *concurrentReadTest) Test_MultiThreadedWritePlusRead() { + const ( + fileSize = 32 * operations.OneMiB // 32 MiB file + numGoRoutines = 10 // Number of concurrent readers + chunkSize = 128 * operations.OneKiB // 128 KiB chunks for reads + ) + var wg sync.WaitGroup + wg.Add(numGoRoutines) + timeout := 300 * time.Second + + for i := range numGoRoutines { + go func(workerId int) { + defer wg.Done() + + fileName := fmt.Sprintf("test_%d.bin", workerId) + filePath := path.Join(testEnv.testDirPath, fileName) + f, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC|syscall.O_DIRECT, setup.FilePermission_0600) //nolint:staticcheck + require.NoError(s.T(), err) + randomData, err := operations.GenerateRandomData(fileSize) + require.NoError(s.T(), err) + n, err := f.Write(randomData) + require.NoError(s.T(), err) + require.Equal(s.T(), fileSize, n) + operations.CloseFileShouldNotThrowError(s.T(), f) + + file, err := os.OpenFile(filePath, os.O_RDONLY|syscall.O_DIRECT, setup.FilePermission_0600) + require.NoError(s.T(), err) + content, err := operations.ReadFileSequentially(file, chunkSize) + require.NoError(s.T(), err, "Sequential reader %d: read failed.", workerId) + require.Equal(s.T(), fileSize, len(content), "Sequential reader %d: expected to read entire file", workerId) + obj := s.storageClient.Bucket(setup.TestBucket()).Object(path.Join(path.Base(testEnv.testDirPath), fileName)) + attrs, err := obj.Attrs(s.ctx) + require.NoError(s.T(), err, "obj.Attrs") + localCRC32C, err := operations.CalculateCRC32(bytes.NewReader(content)) + require.NoError(s.T(), err, "Sequential reader %d: failed to calculate local CRC32C", workerId) + assert.Equal(s.T(), attrs.CRC32C, localCRC32C, "Sequential reader %d: CRC32C mismatch. GCS: %d, Local: %d", workerId, attrs.CRC32C, localCRC32C) + }(i) + } + // Wait for all goroutines or timeout + done := make(chan bool, 1) + go func() { + wg.Wait() + done <- true + }() + + select { + case <-done: + s.T().Log("All concurrent goroutines completed successfully") + case <-time.After(timeout): + assert.FailNow(s.T(), "Concurrent go routines timed out.") + } +} + +//////////////////////////////////////////////////////////////////////// +// Test Function (Runs once before all tests) +//////////////////////////////////////////////////////////////////////// + +func TestConcurrentRead(t *testing.T) { + // Run tests for mounted directory if the flag is set. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + ts := &concurrentReadTest{ + ctx: context.Background(), + storageClient: testEnv.storageClient, + baseTestName: t.Name(), + } + t.Run("MountedDirectory", func(t *testing.T) { + suite.Run(t, ts) + }) + return + } + + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, flags := range flagsSet { + ts := &concurrentReadTest{ + ctx: context.Background(), + storageClient: testEnv.storageClient, + baseTestName: t.Name(), + flags: flags, + } + t.Run(fmt.Sprintf("Flags_%v", flags), func(t *testing.T) { + suite.Run(t, ts) + }) + } +} diff --git a/tools/integration_tests/concurrent_operations/setup_test.go b/tools/integration_tests/concurrent_operations/setup_test.go index cdf50e2ef7..06c10c13d9 100644 --- a/tools/integration_tests/concurrent_operations/setup_test.go +++ b/tools/integration_tests/concurrent_operations/setup_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2026 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,31 +22,75 @@ import ( "testing" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/dynamic_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/only_dir_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" ) const ( - testDirName = "ConcurrentOperationsTest" + testDirName = "TestConcurrentOperations" + GKETempDir = "/gcsfuse-tmp" onlyDirMounted = "OnlyDirConcurrentOperationsTest" ) var ( + testEnv env + mountFunc func(*test_suite.TestConfig, []string) error + mountDir string + // root directory is the directory to be unmounted. + rootDir string +) + +type env struct { storageClient *storage.Client ctx context.Context testDirPath string -) + cfg *test_suite.TestConfig + bucketType string +} func TestMain(m *testing.M) { setup.ParseSetUpFlags() - setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.ConcurrentOperations) == 0 { + log.Println("No configuration found for concurrent operations tests in config. Using flags instead.") + // Populate the config manually with the migrated pattern and requested flags. + cfg.ConcurrentOperations = make([]test_suite.TestConfig, 1) + cfg.ConcurrentOperations[0].TestBucket = setup.TestBucket() + cfg.ConcurrentOperations[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.ConcurrentOperations[0].LogFile = setup.LogFile() + cfg.ConcurrentOperations[0].Configs = make([]test_suite.ConfigItem, 3) + + cfg.ConcurrentOperations[0].Configs[0].Flags = []string{ + "", + "--file-cache-cache-file-for-range-read=true --file-cache-enable-parallel-downloads=true --enable-kernel-reader=false --cache-dir=/gcsfuse-tmp/read_large_files", + "--enable-buffered-read --enable-kernel-reader=false --enable-metadata-prefetch", + } + cfg.ConcurrentOperations[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ConcurrentOperations[0].Configs[0].Run = "TestConcurrentRead" + + cfg.ConcurrentOperations[0].Configs[1].Flags = []string{"--enable-kernel-reader=false"} + cfg.ConcurrentOperations[0].Configs[1].Compatible = map[string]bool{"flat": false, "hns": false, "zonal": true} + cfg.ConcurrentOperations[0].Configs[1].Run = "TestConcurrentRead" + + cfg.ConcurrentOperations[0].Configs[2].Flags = []string{ + "--kernel-list-cache-ttl-secs=0 --enable-metadata-prefetch", + "--kernel-list-cache-ttl-secs=0 --enable-metadata-prefetch --client-protocol=grpc", + "--kernel-list-cache-ttl-secs=-1", + "--kernel-list-cache-ttl-secs=-1 --client-protocol=grpc", + } + cfg.ConcurrentOperations[0].Configs[2].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ConcurrentOperations[0].Configs[2].Run = "TestConcurrentListing" + } - // Create common storage client to be used in test. - ctx = context.Background() - closeStorageClient := client.CreateStorageClientWithCancel(&ctx, &storageClient) + testEnv.ctx = context.Background() + testEnv.cfg = &cfg.ConcurrentOperations[0] + testEnv.bucketType = setup.TestEnvironment(testEnv.ctx, testEnv.cfg) + + // 2. Create storage client before running tests. + closeStorageClient := client.CreateStorageClientWithCancel(&testEnv.ctx, &testEnv.storageClient) defer func() { err := closeStorageClient() if err != nil { @@ -54,29 +98,24 @@ func TestMain(m *testing.M) { } }() - // If Mounted Directory flag is set, run tests for mounted directory. - setup.RunTestsForMountedDirectoryFlag(m) - // Else run tests for testBucket. - // Set up test directory. - setup.SetUpTestDirForTestBucketFlag() - - flagsSet := [][]string{ - {"--kernel-list-cache-ttl-secs=-1"}, {"--kernel-list-cache-ttl-secs=0"}, + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + mountDir = testEnv.cfg.GKEMountedDirectory + os.Exit(setup.RunTestsForMountedDirectory(mountDir, m)) } - if !testing.Short() { - setup.AppendFlagsToAllFlagsInTheFlagsSet(&flagsSet, "", "--client-protocol=grpc") - } - successCode := static_mounting.RunTests(flagsSet, m) - if successCode == 0 { - successCode = only_dir_mounting.RunTests(flagsSet, onlyDirMounted, m) - } + // Run tests for testBucket. + setup.SetUpTestDirForTestBucket(testEnv.cfg) + setup.OverrideFilePathsInFlagSet(testEnv.cfg, setup.TestDir()) - if successCode == 0 { - successCode = dynamic_mounting.RunTests(ctx, storageClient, flagsSet, m) - } + // Save mount and root directory variables. + mountDir, rootDir = testEnv.cfg.GCSFuseMountedDirectory, testEnv.cfg.GCSFuseMountedDirectory + + log.Println("Running static mounting tests...") + mountFunc = static_mounting.MountGcsfuseWithStaticMountingWithConfigFile + successCode := m.Run() // Clean up test directory created. - setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(setup.TestBucket(), testDirName)) + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(testEnv.cfg.TestBucket, testDirName)) os.Exit(successCode) } diff --git a/tools/integration_tests/create_package_runtime_table.sh b/tools/integration_tests/create_package_runtime_table.sh new file mode 100755 index 0000000000..5ad82ac404 --- /dev/null +++ b/tools/integration_tests/create_package_runtime_table.sh @@ -0,0 +1,222 @@ +#!/bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ============================================================================== +# DESCRIPTION: +# This script generates a table of test package runtimes using the 'rich' library. +# +# USAGE: +# ./create_package_runtime_table.sh <FILE_PATH> +# +# REQUIREMENTS: +# Python 3.11+. The script automatically creates a temporary virtual +# environment and safely installs the 'rich' library for you. +# +# INPUT FILE FORMAT (<FILE_PATH>): +# Space-separated lines with the following fields: +# <package_name> <bucket_type> <exit_code> <start_time_seconds> <end_time_seconds> +# +# EXAMPLE FILE CONTENT: +# operations hns 0 0 120 +# operations hns 1 0 60 +# local_file hns 0 60 180 +# local_file flat 1 0 120 +# streaming_writes flat 1 120 240 +# ============================================================================== + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 <file>" + echo "Check the script header for input file format requirements." + exit 1 +fi + +PACKAGE_RUNTIME_STATS=$1 + +if [ ! -f "$PACKAGE_RUNTIME_STATS" ]; then + echo "Error: File '$PACKAGE_RUNTIME_STATS' not found." + exit 1 +fi + +VENV_DIR=$(mktemp -d) +trap 'rm -rf "$VENV_DIR"' EXIT + +PYTHON_SCRIPT_FILE="$VENV_DIR/visualize.py" +cat << 'EOF' > "$PYTHON_SCRIPT_FILE" +import sys, os +from collections import defaultdict + +# Column indices for the input file data +IDX_PKG_NAME = 0 +IDX_BUCKET_TYPE = 1 +IDX_EXIT_CODE = 2 +IDX_START_TIME = 3 +IDX_END_TIME = 4 + +MIN_REQUIRED_FIELDS = 5 + +# Minimum widths based on header lengths +MIN_LEN_PKG_NAME_HEADER = 12 +MIN_LEN_BUCKET_TYPE_HEADER = 11 +MIN_LEN_RUNTIME_BAR_HEADER = 31 + +# Estimated padding for table columns +PADDING_TIME_COL = 16 +PADDING_STATUS_COL = 25 +PADDING_BORDERS = 20 + +WIDTH_FALLBACK = 80 +SECONDS_PER_MINUTE = 60 + +if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} <FILE_PATH>") + sys.exit(1) + +path = sys.argv[1] +if not os.path.isfile(path): + print(f"Error: File '{path}' not found.") + sys.exit(1) + +with open(path) as f: + lines = [l.split() for l in f if l.strip()] + +# Group by package and bucket +groups = defaultdict(list) +for p in lines: + if len(p) >= MIN_REQUIRED_FIELDS: + groups[(p[IDX_PKG_NAME], p[IDX_BUCKET_TYPE])].append(p) + +# Sort groups by package name and bucket type +sorted_keys = sorted(groups.keys()) + +try: + from rich.console import Console + from rich.table import Table + import shutil + + # Calculate optimal table width based on content + if groups: + max_pkg = max(MIN_LEN_PKG_NAME_HEADER, max(len(k[0]) for k in groups.keys())) + max_type = max(MIN_LEN_BUCKET_TYPE_HEADER, max(len(k[1]) for k in groups.keys())) + + # For runtime bar, we need to find the max total time + max_total_time = 0 + for key, items in groups.items(): + total_run = 0 + total_wait = 0 + for p in items: + start, end = int(p[IDX_START_TIME]), int(p[IDX_END_TIME]) + total_wait += start // SECONDS_PER_MINUTE + total_run += (end - start + SECONDS_PER_MINUTE) // SECONDS_PER_MINUTE + max_total_time = max(max_total_time, total_wait + total_run) + + max_rt = max(MIN_LEN_RUNTIME_BAR_HEADER, max_total_time) + table_width = max_pkg + max_type + PADDING_TIME_COL + PADDING_STATUS_COL + max_rt + PADDING_BORDERS + else: + table_width = WIDTH_FALLBACK + + term_width = shutil.get_terminal_size().columns + console = Console(width=max(term_width, table_width)) + table = Table(title="e2e Test Packages Runtime", show_header=True, header_style="bold magenta") + for col, kwargs in [("Package Name", {"style": "cyan"}), ("Bucket Type", {"style": "blue"}), + ("Time", {"justify": "right"}), ("Runtime (░=total wait, ▓=total run)", {}), + ("Status", {"justify": "left"})]: table.add_column(col, **kwargs) + + for key in sorted_keys: + items = groups[key] + pkg_name, bucket_type = key + + attempts = len(items) + succeeded = any(int(p[IDX_EXIT_CODE]) == 0 for p in items) + + icon_str = "" + for p in items: + if int(p[IDX_EXIT_CODE]) == 0: + icon_str += "✅" + else: + icon_str += "❌" + + bar_str = "" + last_end = 0 + total_run = 0 + segments = [] + run_times = [] + # Sort items by start time to ensure correct timeline + items.sort(key=lambda x: int(x[IDX_START_TIME])) + for p in items: + start, end = int(p[IDX_START_TIME]), int(p[IDX_END_TIME]) + wait = max(0, (start - last_end) // SECONDS_PER_MINUTE) + run = max(1, (end - start + SECONDS_PER_MINUTE) // SECONDS_PER_MINUTE) + + seg = "" + if wait > 0: + seg += '░' * wait + seg += '▓' * run + segments.append(seg) + + run_times.append(f"{run}m") + last_end = end + total_run += run + + bar_str = '|'.join(segments) + time_str = ', '.join(run_times) + + if succeeded: + if attempts > 1: + status = f"[yellow]FLAKED {icon_str}[/]" + else: + status = "[green]PASSED ✅[/]" + else: + status = f"[red]FAILED {icon_str}[/]" + + table.add_row(pkg_name, bucket_type, time_str, bar_str, status) + + console.print(table) + +except ImportError: + print("Error: The 'rich' library is required to run this script. Please install it (e.g., 'pip install rich').", file=sys.stderr) + sys.exit(1) +except Exception as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) +EOF + +main() { + # Install python3-dev (and python3-venv for debian/ubuntu) globally + local repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + source "${repo_root}/perfmetrics/scripts/os_utils.sh" + + local os_id=$(get_os_id) + if ! install_packages_by_os "$os_id" "python3-dev" "python3-venv"; then + echo "Warning: Failed to install prerequisites. Skipping rich table visualization." + exit 0 + fi + + # Create venv + if ! python3 -m venv "$VENV_DIR"; then + echo "Warning: Failed to create venv. Skipping rich table visualization." + exit 0 + fi + + # Install rich inside the venv + if ! "$VENV_DIR/bin/pip" install --index-url https://pypi.org/simple rich; then + echo "Warning: Failed to install rich in venv. Skipping rich table visualization." + exit 0 + fi + + # Run the Python script using the venv's Python binary + "$VENV_DIR/bin/python3" "$PYTHON_SCRIPT_FILE" "$PACKAGE_RUNTIME_STATS" +} + +main diff --git a/tools/integration_tests/dentry_cache/delete_operation_test.go b/tools/integration_tests/dentry_cache/delete_operation_test.go new file mode 100644 index 0000000000..127b79f7e9 --- /dev/null +++ b/tools/integration_tests/dentry_cache/delete_operation_test.go @@ -0,0 +1,92 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dentry_cache + +import ( + "context" + "log" + "os" + "path" + "testing" + + "cloud.google.com/go/storage" + "github.com/stretchr/testify/suite" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type deleteOperationTest struct { + flags []string + storageClient *storage.Client + ctx context.Context + suite.Suite +} + +func (s *deleteOperationTest) SetupTest() { + testEnv.testDirPath = client.SetupTestDirectory(s.ctx, s.storageClient, testDirName) +} + +func (s *deleteOperationTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +func (s *deleteOperationTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *deleteOperationTest) SetupSuite() { + mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient) +} + +func (s *deleteOperationTest) TestDeleteFileWhenFileIsClobbered() { + testFileName := s.T().Name() + // Create a file with initial content directly in GCS. + filePath := path.Join(testEnv.testDirPath, testFileName) + client.SetupFileInTestDirectory(s.ctx, s.storageClient, testDirName, testFileName, initialContentSize, s.T()) + // Stat file to cache the entry + _, err := os.Stat(filePath) + require.Nil(s.T(), err) + // Modify the object on GCS. + objectName := path.Join(testDirName, testFileName) // This is correct, objectName is relative to bucket. + smallContent, err := operations.GenerateRandomData(updatedContentSize) + require.Nil(s.T(), err) + require.Nil(s.T(), client.WriteToObject(s.ctx, s.storageClient, objectName, string(smallContent), storage.Conditions{})) + + // Deleting the file should not give error + err = os.Remove(filePath) + + assert.Nil(s.T(), err) +} + +func TestDeleteOperationTest(t *testing.T) { + ts := &deleteOperationTest{ctx: context.Background(), storageClient: testEnv.storageClient} + + // Run tests for mounted directory if the flag is set. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/dentry_cache/notifier_test.go b/tools/integration_tests/dentry_cache/notifier_test.go new file mode 100644 index 0000000000..5dcca7abcc --- /dev/null +++ b/tools/integration_tests/dentry_cache/notifier_test.go @@ -0,0 +1,145 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dentry_cache + +import ( + "context" + "log" + "os" + "path" + "testing" + + "cloud.google.com/go/storage" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" +) + +type notifierTest struct { + flags []string + storageClient *storage.Client + ctx context.Context + suite.Suite +} + +func (s *notifierTest) SetupTest() { + testEnv.testDirPath = client.SetupTestDirectory(s.ctx, s.storageClient, testDirName) +} + +func (s *notifierTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +func (s *notifierTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *notifierTest) SetupSuite() { + mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient) +} + +func (s *notifierTest) TestWriteFileWithDentryCacheEnabled() { + testFileName := s.T().Name() + // Create a file with initial content directly in GCS. + filePath := path.Join(testEnv.testDirPath, testFileName) + client.SetupFileInTestDirectory(s.ctx, s.storageClient, testDirName, testFileName, initialContentSize, s.T()) + // Stat file to cache the entry + _, err := os.Stat(filePath) + require.Nil(s.T(), err) + // Modify the object on GCS. + objectName := path.Join(testDirName, testFileName) + smallContent, err := operations.GenerateRandomData(updatedContentSize) + require.Nil(s.T(), err) + require.Nil(s.T(), client.WriteToObject(s.ctx, s.storageClient, objectName, string(smallContent), storage.Conditions{})) + + // First Write File attempt. + err = operations.WriteFile(filePath, "ShouldNotWrite") + + // First Write File attempt should fail because file has been clobbered. + operations.ValidateESTALEError(s.T(), err) + // Second Write File attempt. + err = operations.WriteFile(filePath, "ShouldWrite") + // The notifier is triggered after the first write failure, invalidating the kernel cache entry. + // Therefore, the second write succeeds even before the metadata cache TTL expires. + assert.Nil(s.T(), err) +} + +func (s *notifierTest) TestReadFileWithDentryCacheEnabled() { + testFileName := s.T().Name() + // Create a file with initial content directly in GCS. + filePath := path.Join(testEnv.testDirPath, testFileName) + client.SetupFileInTestDirectory(s.ctx, s.storageClient, testDirName, testFileName, initialContentSize, s.T()) + // Stat file to cache the entry + _, err := os.Stat(filePath) + require.Nil(s.T(), err) + // Modify the object on GCS. + objectName := path.Join(testDirName, testFileName) + smallContent, err := operations.GenerateRandomData(updatedContentSize) + require.Nil(s.T(), err) + require.Nil(s.T(), client.WriteToObject(s.ctx, s.storageClient, objectName, string(smallContent), storage.Conditions{})) + + // First Read File attempt. + _, err = operations.ReadFile(filePath) + // First Read File attempt should fail because file has been clobbered. + operations.ValidateESTALEError(s.T(), err) + // Second Read File attempt. + _, err = operations.ReadFile(filePath) + // The notifier is triggered after the first read failure, invalidating the kernel cache entry. + // Therefore, the second read succeeds even before the metadata cache TTL expires. + assert.Nil(s.T(), err) +} + +func (s *notifierTest) TestDeleteFileWithDentryCacheEnabled() { + testFileName := s.T().Name() + // Create a file with initial content directly in GCS. + filePath := path.Join(testEnv.testDirPath, testFileName) + client.SetupFileInTestDirectory(s.ctx, s.storageClient, testDirName, testFileName, initialContentSize, s.T()) + // Stat file to cache the entry + _, err := os.Stat(filePath) + require.Nil(s.T(), err) + // Delete the object directly from GCS. + objectName := path.Join(testDirName, testFileName) + require.Nil(s.T(), client.DeleteObjectOnGCS(s.ctx, s.storageClient, objectName)) + // Read File to call the notifier to invalidate entry. + _, err = operations.ReadFile(filePath) + // The notifier is triggered after the first read failure, invalidating the kernel cache entry. + operations.ValidateESTALEError(s.T(), err) + + // Stat again, it should give error as entry does not exist. + _, err = os.Stat(filePath) + + assert.NotNil(s.T(), err) +} + +func TestNotifierTest(t *testing.T) { + ts := ¬ifierTest{ctx: context.Background(), storageClient: testEnv.storageClient} + + // Run tests for mounted directory if the flag is set. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/dentry_cache/setup_test.go b/tools/integration_tests/dentry_cache/setup_test.go new file mode 100644 index 0000000000..7480f8d144 --- /dev/null +++ b/tools/integration_tests/dentry_cache/setup_test.go @@ -0,0 +1,122 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Provides integration tests for enabling dentry cache. +package dentry_cache + +import ( + "context" + "log" + "os" + "testing" + + "cloud.google.com/go/storage" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" +) + +const ( + testDirName = "testDirForDentryCache" + initialContentSize = 5 + updatedContentSize = 10 +) + +var ( + testEnv env + mountFunc func(*test_suite.TestConfig, []string) error + // mount directory is where our tests run. + mountDir string + // root directory is the directory to be unmounted. + rootDir string +) + +type env struct { + storageClient *storage.Client + ctx context.Context + testDirPath string + cfg *test_suite.TestConfig + bucketType string +} + +func mountGCSFuseAndSetupTestDir(flags []string, ctx context.Context, storageClient *storage.Client) { + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, flags, mountFunc) + setup.SetMntDir(mountDir) + testEnv.testDirPath = client.SetupTestDirectory(ctx, storageClient, testDirName) +} + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.DentryCache) == 0 { + log.Println("No configuration found for dentry_cache tests in config. Using flags instead.") + // Populate the config manually. + cfg.DentryCache = make([]test_suite.TestConfig, 1) + cfg.DentryCache[0].TestBucket = setup.TestBucket() + cfg.DentryCache[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.DentryCache[0].LogFile = setup.LogFile() + cfg.DentryCache[0].Configs = make([]test_suite.ConfigItem, 3) + cfg.DentryCache[0].Configs[0].Flags = []string{ + "--implicit-dirs --experimental-enable-dentry-cache --metadata-cache-ttl-secs=2", + } + cfg.DentryCache[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.DentryCache[0].Configs[0].Run = "TestStatWithDentryCacheEnabledTest" + cfg.DentryCache[0].Configs[1].Flags = []string{ + "--implicit-dirs --experimental-enable-dentry-cache --metadata-cache-ttl-secs=1000", + } + cfg.DentryCache[0].Configs[1].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.DentryCache[0].Configs[1].Run = "TestDeleteOperationTest" + cfg.DentryCache[0].Configs[2].Flags = []string{ + "--implicit-dirs --experimental-enable-dentry-cache --metadata-cache-ttl-secs=1000", + } + cfg.DentryCache[0].Configs[2].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.DentryCache[0].Configs[2].Run = "TestNotifierTest" + } + + testEnv.ctx = context.Background() + testEnv.cfg = &cfg.DentryCache[0] + testEnv.bucketType = setup.TestEnvironment(testEnv.ctx, testEnv.cfg) + + // 2. Create storage client before running tests. + var err error + testEnv.storageClient, err = client.CreateStorageClient(testEnv.ctx) + if err != nil { + log.Printf("Error creating storage client: %v\n", err) + os.Exit(1) + } + defer testEnv.storageClient.Close() + + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + mountDir, rootDir = testEnv.cfg.GKEMountedDirectory, testEnv.cfg.GKEMountedDirectory + os.Exit(setup.RunTestsForMountedDirectory(testEnv.cfg.GKEMountedDirectory, m)) + } + + // Run tests for testBucket + // Set up test directory. + setup.SetUpTestDirForTestBucket(testEnv.cfg) + + // Save mount and root directory variables. + mountDir, rootDir = testEnv.cfg.GCSFuseMountedDirectory, testEnv.cfg.GCSFuseMountedDirectory + + log.Println("Running static mounting tests...") + mountFunc = static_mounting.MountGcsfuseWithStaticMountingWithConfigFile + successCode := m.Run() + + os.Exit(successCode) +} diff --git a/tools/integration_tests/dentry_cache/stat_test.go b/tools/integration_tests/dentry_cache/stat_test.go new file mode 100644 index 0000000000..b4c7aee8f0 --- /dev/null +++ b/tools/integration_tests/dentry_cache/stat_test.go @@ -0,0 +1,125 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dentry_cache + +import ( + "context" + "log" + "os" + "path" + "testing" + "time" + + "cloud.google.com/go/storage" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" +) + +type statWithDentryCacheEnabledTest struct { + flags []string + storageClient *storage.Client + ctx context.Context + suite.Suite +} + +func (s *statWithDentryCacheEnabledTest) SetupTest() { + testEnv.testDirPath = client.SetupTestDirectory(s.ctx, s.storageClient, testDirName) +} + +func (s *statWithDentryCacheEnabledTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +func (s *statWithDentryCacheEnabledTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *statWithDentryCacheEnabledTest) SetupSuite() { + mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient) +} + +func (s *statWithDentryCacheEnabledTest) TestStatWithDentryCacheEnabled() { + testFileName := s.T().Name() + // Create a file with initial content directly in GCS. + filePath := path.Join(testEnv.testDirPath, testFileName) + client.SetupFileInTestDirectory(s.ctx, s.storageClient, testDirName, testFileName, initialContentSize, s.T()) + + // Stat file to cache the entry + _, err := os.Stat(filePath) + require.Nil(s.T(), err) + // Modify the object on GCS. + objectName := path.Join(testDirName, testFileName) + smallContent, err := operations.GenerateRandomData(updatedContentSize) + require.Nil(s.T(), err) + require.Nil(s.T(), client.WriteToObject(s.ctx, s.storageClient, objectName, string(smallContent), storage.Conditions{})) + + // Stat again, it should give old cached attributes. + fileInfo, err := os.Stat(filePath) + + assert.Nil(s.T(), err) + assert.Equal(s.T(), int64(initialContentSize), fileInfo.Size()) + // Wait for a period more than the timeout (2 second), so that entry expires in cache. + time.Sleep(2100 * time.Millisecond) + // Stat again, it should give updated attributes. + fileInfo, err = os.Stat(filePath) + assert.Nil(s.T(), err) + assert.Equal(s.T(), int64(updatedContentSize), fileInfo.Size()) +} + +func (s *statWithDentryCacheEnabledTest) TestStatWhenFileIsDeletedDirectlyFromGCS() { + testFileName := s.T().Name() + // Create a file with initial content directly in GCS. + filePath := path.Join(testEnv.testDirPath, testFileName) + client.SetupFileInTestDirectory(s.ctx, s.storageClient, testDirName, testFileName, initialContentSize, s.T()) + // Stat file to cache the entry + _, err := os.Stat(filePath) + require.Nil(s.T(), err) + // Delete the object directly from GCS. + objectName := path.Join(testDirName, testFileName) + require.Nil(s.T(), client.DeleteObjectOnGCS(s.ctx, s.storageClient, objectName)) + + // Stat again, it should give old cached attributes rather than giving not found error. + fileInfo, err := os.Stat(filePath) + + assert.Nil(s.T(), err) + assert.Equal(s.T(), int64(initialContentSize), fileInfo.Size()) + // Wait for a period more than the timeout (2 second), so that entry expires in cache. + time.Sleep(2100 * time.Millisecond) + // Stat again, it should give error as file does not exist. + _, err = os.Stat(filePath) + assert.NotNil(s.T(), err) +} + +func TestStatWithDentryCacheEnabledTest(t *testing.T) { + ts := &statWithDentryCacheEnabledTest{ctx: context.Background(), storageClient: testEnv.storageClient} + + // Run tests for mounted directory if the flag is set. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/emulator_tests/README.md b/tools/integration_tests/emulator_tests/README.md new file mode 100644 index 0000000000..638af8cf07 --- /dev/null +++ b/tools/integration_tests/emulator_tests/README.md @@ -0,0 +1,30 @@ +# Emulator based tests +## go-proxy server +Proxy server, which intercepts [storage-testbench](https://github.com/googleapis/storage-testbench) server and perform pre-defined +retry test. + +### Steps to run the test manually +1. Run storage-testbench server by following [this](https://github.com/googleapis/storage-testbench/tree/main?tab=readme-ov-file#initial-set-up) steps. +2. Create test-bucket on server with below command. +``` +cat << EOF > test.json +{"name":"test-bucket"} +EOF + +# Execute the curl command to create bucket on storagetestbench server. +curl -X POST --data-binary @test.json \ + -H "Content-Type: application/json" \ + "$STORAGE_EMULATOR_HOST/storage/v1/b?project=test-project" +rm test.json +``` +3. Run tests with the current directory as emulator_tests. +``` +go test --integrationTest -v --testbucket=test-bucket -timeout 10m +``` + +### Automated emulator test script +1. Run ./emulator_tests.sh + +### Steps to add new tests in the future: +1. Create <feature>_test file [here](https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/tools/integration_tests/emulator_tests). +2. Write tests according to your scenarios. e.g. [write_stall_test](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/tools/integration_tests/emulator_tests/write_stall_test.go) diff --git a/tools/integration_tests/emulator_tests/configs/config.yaml b/tools/integration_tests/emulator_tests/configs/config.yaml new file mode 100644 index 0000000000..7fa3ab1262 --- /dev/null +++ b/tools/integration_tests/emulator_tests/configs/config.yaml @@ -0,0 +1 @@ +targetHost: http://localhost:9000 diff --git a/tools/integration_tests/emulator_tests/configs/empty_gcs_file_2nd_chunk_upload_returns412.yaml b/tools/integration_tests/emulator_tests/configs/empty_gcs_file_2nd_chunk_upload_returns412.yaml new file mode 100644 index 0000000000..5e9259474d --- /dev/null +++ b/tools/integration_tests/emulator_tests/configs/empty_gcs_file_2nd_chunk_upload_returns412.yaml @@ -0,0 +1,9 @@ +targetHost: http://localhost:9000 +retryConfig: +- method: JsonCreate + retryInstruction: "return-412" + retryCount: 1 + # Skip count of four is required as first call is used to create the testDir on the GCS and + # second is used to create the empty GCS file and third, fourth call are used in creating + # resumable upload session uri and first chunk upload. + skipCount: 4 diff --git a/tools/integration_tests/emulator_tests/configs/grpc_header_validation.yaml b/tools/integration_tests/emulator_tests/configs/grpc_header_validation.yaml new file mode 100644 index 0000000000..1f5743010b --- /dev/null +++ b/tools/integration_tests/emulator_tests/configs/grpc_header_validation.yaml @@ -0,0 +1,9 @@ +proxyType: grpc +targetHost: "localhost:8888" +headerValidation: + - headerName: "x-goog-request-params" + expectedPattern: "force_direct_connectivity=ENFORCED" + failOnMismatch: true + - headerName: "x-goog-request-params" + expectedPattern: "direct_connectivity_diagnostic" + failOnMismatch: true diff --git a/tools/integration_tests/emulator_tests/configs/local_file_2nd_chunk_upload_returns412.yaml b/tools/integration_tests/emulator_tests/configs/local_file_2nd_chunk_upload_returns412.yaml new file mode 100644 index 0000000000..8464699da6 --- /dev/null +++ b/tools/integration_tests/emulator_tests/configs/local_file_2nd_chunk_upload_returns412.yaml @@ -0,0 +1,8 @@ +targetHost: http://localhost:9000 +retryConfig: +- method: JsonCreate + retryInstruction: "return-412" + retryCount: 1 + # Skip count of three is required as first call is used to create the testDir on the GCS and then + # second and third call are used in creating resumable upload session uri and first chunk upload. + skipCount: 3 diff --git a/tools/integration_tests/emulator_tests/configs/read_stall_5s.yaml b/tools/integration_tests/emulator_tests/configs/read_stall_5s.yaml new file mode 100644 index 0000000000..16c1445272 --- /dev/null +++ b/tools/integration_tests/emulator_tests/configs/read_stall_5s.yaml @@ -0,0 +1,6 @@ +targetHost: http://localhost:9000 +retryConfig: +- method: XmlRead + retryInstruction: "stall-for-5s-after-0K" + retryCount: 1 + skipCount: 0 diff --git a/tools/integration_tests/emulator_tests/configs/write_stall_40s.yaml b/tools/integration_tests/emulator_tests/configs/write_stall_40s.yaml new file mode 100644 index 0000000000..500ef70028 --- /dev/null +++ b/tools/integration_tests/emulator_tests/configs/write_stall_40s.yaml @@ -0,0 +1,10 @@ +targetHost: http://localhost:9000 +retryConfig: +- method: JsonCreate + retryInstruction: "stall-for-40s-after-15360K" + retryCount: 1 + # To add forced error scenarios for resumable uploads, we need to define skipCount two. + # This is because the first POST request creates the file in our tests, and the second POST request only initiates + # the resumable upload request. Subsequent requests actually upload the data, and it's + # these requests we want to stall for testing. + skipCount: 2 diff --git a/tools/integration_tests/emulator_tests/configs/write_stall_twice_40s.yaml b/tools/integration_tests/emulator_tests/configs/write_stall_twice_40s.yaml new file mode 100644 index 0000000000..0f0f3c19bd --- /dev/null +++ b/tools/integration_tests/emulator_tests/configs/write_stall_twice_40s.yaml @@ -0,0 +1,10 @@ +targetHost: http://localhost:9000 +retryConfig: +- method: JsonCreate + retryInstruction: "stall-for-40s-after-15360K" + retryCount: 2 + # To add forced error scenarios for resumable uploads, we need to define skipCount two. + # This is because the first POST request creates the file in our tests, and the second POST request only initiates + # the resumable upload request. Subsequent requests actually upload the data, and it's + # these requests we want to stall for testing. + skipCount: 2 diff --git a/tools/integration_tests/emulator_tests/configs/write_stalls_four_times_60s.yaml b/tools/integration_tests/emulator_tests/configs/write_stalls_four_times_60s.yaml new file mode 100644 index 0000000000..857858300f --- /dev/null +++ b/tools/integration_tests/emulator_tests/configs/write_stalls_four_times_60s.yaml @@ -0,0 +1,10 @@ +targetHost: http://localhost:9000 +retryConfig: +- method: JsonCreate + retryInstruction: "stall-for-60s-after-15360K" + retryCount: 4 + # To add forced error scenarios for resumable uploads, we need to define skipCount two. + # This is because the first POST request creates the file in our tests, and the second POST request only initiates + # the resumable upload request. Subsequent requests actually upload the data, and it's + # these requests we want to stall for testing. + skipCount: 2 diff --git a/tools/integration_tests/emulator_tests/emulator_tests.sh b/tools/integration_tests/emulator_tests/emulator_tests.sh new file mode 100755 index 0000000000..2d79711d0a --- /dev/null +++ b/tools/integration_tests/emulator_tests/emulator_tests.sh @@ -0,0 +1,195 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Exit on error, treat unset variables as errors, and propagate pipeline errors. +set -euo pipefail +set -x + +# Script uses positional arguments: +# $1: testInstalledPackage (default: false) +# $2: gcsfuse prebuilt directory (default: "") + +# Logging Helpers +log_info() { + echo "[INFO] $(date +"%Y-%m-%d %H:%M:%S"): $1" +} + +log_error() { + echo "[ERROR] $(date +"%Y-%m-%d %H:%M:%S"): $1" +} + + + +TEST_INSTALLED_PACKAGE=${1:-false} +GCSFUSE_PREBUILT_DIR=${2:-""} + +if [[ "$TEST_INSTALLED_PACKAGE" == "true" ]] && [[ -n "$GCSFUSE_PREBUILT_DIR" ]]; then + log_error "test_installed_package=true and gcsfuse_prebuilt_dir are mutually exclusive." + exit 1 +fi + + +uname=$(uname -m) +if [ $uname == "aarch64" ];then + # TODO: Remove this when we have an ARM64 image for the storage test bench.(b/384388821) + log_info "These tests will not run for arm64 machine..." + exit 0 +fi + +# Only run on Go 1.17+ +min_minor_ver=17 + +v=`go version | { read _ _ v _; echo ${v#go}; }` +comps=(${v//./ }) +minor_ver=${comps[1]} + +if [ "$minor_ver" -lt "$min_minor_ver" ]; then + log_info "minor version $minor_ver, skipping" + exit 0 +fi + +# Install dependencies +if sudo docker ps > /dev/null 2>&1; then + log_info "Docker is already installed and usable. Skipping installation steps." +else + # Ubuntu/Debian based machine. + if [ -f /etc/debian_version ]; then + if grep -q "Ubuntu" /etc/os-release; then + os="ubuntu" + elif grep -q "Debian" /etc/os-release; then + os="debian" + fi + + sudo apt-get update + sudo apt-get install -y ca-certificates curl + sudo install -m 0755 -d /etc/apt/keyrings + sudo curl -fsSL https://download.docker.com/linux/${os}/gpg -o /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc + # Add the repository to Apt sources: + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/${os} \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + sudo apt-get install -y lsof + # RHEL/CentOS based machine. + elif [ -f /etc/redhat-release ]; then + sudo dnf -y install dnf-plugins-core + sudo dnf config-manager --add-repo https://download.docker.com/linux/rhel/docker-ce.repo + sudo dnf -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + sudo usermod -aG docker $USER + sudo systemctl start docker + sudo yum -y install lsof + fi +fi + +export STORAGE_EMULATOR_HOST="http://localhost:9000" +export STORAGE_EMULATOR_HOST_GRPC="localhost:8888" + +DEFAULT_IMAGE_NAME='gcr.io/cloud-devrel-public-resources/storage-testbench' +DEFAULT_IMAGE_TAG='latest' +DOCKER_IMAGE=${DEFAULT_IMAGE_NAME}:${DEFAULT_IMAGE_TAG} +CONTAINER_NAME=storage_testbench + +# Note: --net=host makes the container bind directly to the Docker host’s network, +# with no network isolation. If we were to use port-mapping instead, reset connection errors +# would be captured differently and cause unexpected test behaviour. +# The host networking driver works only on Linux hosts. +# See more about using host networking: https://docs.docker.com/network/host/ +DOCKER_NETWORK="--net=host" + +# Get the docker image for the testbench +sudo docker pull $DOCKER_IMAGE + +# Remove the docker container if it's already running. +CONTAINER_ID=$(sudo docker ps -aqf "name=$CONTAINER_NAME") +if [[ -n "$CONTAINER_ID" ]]; then + log_info "Container with ID:[$CONTAINER_ID] is already running with name:[$CONTAINER_NAME]" + log_info "Stopping container...." + sudo docker stop $CONTAINER_ID +fi + +wait_for_emulator() { + local timeout=60 + local count=0 + log_info "Waiting for emulator to be ready..." + while [ $count -lt $timeout ]; do + if curl -s "$STORAGE_EMULATOR_HOST/storage/v1/b?project=test-project" > /dev/null; then + log_info "Emulator is ready!" + return 0 + fi + sleep 1 + count=$((count+1)) + done + log_error "Emulator failed to become ready after $timeout seconds." + return 1 +} + +# Run the emulator container in the background and stream its logs to a file. +# GUNICORN_CMD_ARGS="--timeout 600" is used to increase the Gunicorn timeout from the default 30 seconds +# to 10 minutes, preventing workers from dying during high load which causes emulator test flakiness. +sudo docker run --name $CONTAINER_NAME -e GUNICORN_CMD_ARGS="--timeout 600" --rm -d $DOCKER_NETWORK $DOCKER_IMAGE +log_info "Emulator docker container logs are saved at: $(pwd)/emulator_container.log" +sudo docker logs -f $CONTAINER_NAME > emulator_container.log 2>&1 & + +# Stop the testbench & cleanup environment variables +function cleanup() { + log_info "Cleanup testbench" + sudo docker stop $CONTAINER_NAME || true + unset STORAGE_EMULATOR_HOST; + unset STORAGE_EMULATOR_HOST_GRPC; + log_info "Printing emulator docker container Logs..." + log_info "========================================================================" + cat emulator_container.log || true + log_info "========================================================================" + rm -f test.json || true +} +trap cleanup EXIT + +wait_for_emulator + +# Create the JSON file to create bucket +cat << EOF > test.json +{"name":"test-bucket"} +EOF + +# Execute the curl command to create bucket on storagetestbench server. +if ! curl -X POST --data-binary @test.json \ + -H "Content-Type: application/json" \ + "$STORAGE_EMULATOR_HOST/storage/v1/b?project=test-project"; then + log_error "Failed to create bucket test-bucket" + exit 1 +fi +rm test.json + +# Start the gRPC server on port 8888. +log_info "Starting the gRPC server on port 8888" +response=$(curl -w "%{http_code}\n" --retry 5 --retry-max-time 40 -o /dev/null "$STORAGE_EMULATOR_HOST/start_grpc?port=8888") + +if [[ $response != 200 ]] +then + log_error "Testbench gRPC server did not start correctly" + exit 1 +fi + +args=("--testInstalledPackage=$TEST_INSTALLED_PACKAGE") + +if [[ -n "$GCSFUSE_PREBUILT_DIR" ]]; then + args+=("--gcsfuse_prebuilt_dir=$GCSFUSE_PREBUILT_DIR") +fi + +# Run all emulator test packages in sequence to avoid high cpu usage. +go test -v -p 1 -timeout 10m ./tools/integration_tests/emulator_tests/... --integrationTest --testbucket=test-bucket "${args[@]}" diff --git a/tools/integration_tests/emulator_tests/grpc_header_validation/grpc_header_validation_test.go b/tools/integration_tests/emulator_tests/grpc_header_validation/grpc_header_validation_test.go new file mode 100644 index 0000000000..e0cfaf04fc --- /dev/null +++ b/tools/integration_tests/emulator_tests/grpc_header_validation/grpc_header_validation_test.go @@ -0,0 +1,127 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package grpc_header_validation + +import ( + "fmt" + "log" + "os" + "path" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + emulator_tests "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/emulator_tests/util" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" +) + +type grpcHeaderValidation struct { + port int + proxyProcessId int + proxyServerLogFile string + flags []string + configPath string + suite.Suite +} + +func (g *grpcHeaderValidation) SetupTest() { + g.configPath = "../configs/grpc_header_validation.yaml" + g.proxyServerLogFile = setup.CreateProxyServerLogFile(g.T()) + g.T().Logf("Proxy server log file: %s", g.proxyServerLogFile) + var err error + g.port, g.proxyProcessId, err = emulator_tests.StartProxyServer(g.configPath, g.proxyServerLogFile) + require.NoError(g.T(), err) + g.flags = append(g.flags, fmt.Sprintf("--custom-endpoint=localhost:%d", g.port)) + g.flags = append(g.flags, "--anonymous-access") // Required for gRPC localhost endpoint. + setup.MountGCSFuseWithGivenMountFunc(g.flags, mountFunc) +} + +func (g *grpcHeaderValidation) TearDownTest() { + setup.UnmountGCSFuse(rootDir) + assert.NoError(g.T(), emulator_tests.KillProxyServerProcess(g.proxyProcessId)) + setup.SaveGCSFuseLogFileInCaseOfFailure(g.T()) + setup.SaveProxyServerLogFileInCaseOfFailure(g.proxyServerLogFile, g.T()) +} + +func (g *grpcHeaderValidation) TestGRPCClientSendsExpectedHeaders() { + // GCSFuse mount itself triggers gRPC calls for DirectPath verification, we + // just need to verify the proxy logs contain the expected header. + + // Assert: read and confirm the required headers. + logContent, err := os.ReadFile(g.proxyServerLogFile) + require.NoError(g.T(), err) + logStr := string(logContent) + assert.Contains(g.T(), logStr, "Metadata validation passed") + assert.Contains(g.T(), logStr, "x-goog-request-params") + assert.Contains(g.T(), logStr, "force_direct_connectivity=ENFORCED") + assert.Contains(g.T(), logStr, "direct_connectivity_diagnostic=no_auth") + assert.Contains(g.T(), logStr, "/google.storage.v2.Storage/GetObject") +} + +func (g *grpcHeaderValidation) TestGRPCHeadersInMultipleOperations() { + // Perform multiple file operations to trigger various gRPC calls + testFilePath := path.Join(rootDir, "test_file_for_grpc.txt") + testContent := []byte("This is test content to validate gRPC headers across different operations.") + + // Action + err := os.WriteFile(testFilePath, testContent, 0644) // Trigger (GetObject + BidiWriteObject) + require.NoError(g.T(), err) + readContent, err := os.ReadFile(testFilePath) // Triggers (GetObject + ReadObject) + require.NoError(g.T(), err) + require.Equal(g.T(), testContent, readContent, "File content should match") + _, err = os.Stat(testFilePath) // Triggers (GetObject) + require.NoError(g.T(), err) + entries, err := os.ReadDir(rootDir) // Triggers ListObjects + require.NoError(g.T(), err) + g.T().Logf("Listed %d entries in root directory", len(entries)) + + // Assert: read and verify proxy logs. + logContent, err := os.ReadFile(g.proxyServerLogFile) + require.NoError(g.T(), err) + logStr := string(logContent) + validationCount := strings.Count(logStr, "Metadata validation passed") + assert.Greater(g.T(), validationCount, 0, "Expected at least one successful metadata validation") + assert.Contains(g.T(), logStr, "force_direct_connectivity=ENFORCED") + assert.Contains(g.T(), logStr, "direct_connectivity_diagnostic=no_auth") + assert.Equal(g.T(), 3, strings.Count(logStr, "/google.storage.v2.Storage/GetObject"), "Expected 3 GetObject calls (1 for each operation)") + assert.Equal(g.T(), 1, strings.Count(logStr, "/google.storage.v2.Storage/BidiWriteObject"), "Expected 1 BidiWriteObject call for writing the file") + assert.Equal(g.T(), 1, strings.Count(logStr, "/google.storage.v2.Storage/ReadObject"), "Expected 1 ReadObject call for reading the file") + assert.Equal(g.T(), 1, strings.Count(logStr, "/google.storage.v2.Storage/ListObjects"), "Expected 1 ListObjects call for listing the directory") +} + +func TestGRPCHeaderValidation(t *testing.T) { + ts := &grpcHeaderValidation{} + // Test with gRPC protocol to validate that DirectPath metadata is sent. + // The Go Storage SDK automatically adds force_direct_connectivity=ENFORCED + // to x-goog-request-params when experimental.WithDirectConnectivityEnforced() is used. + // The gRPC proxy intercepts and validates this metadata. + // + // NOTE: This test requires: + // 1. gRPC testbench server running on localhost:8888 (started by emulator_tests.sh) + // 2. Bucket named "test-bucket" created in the testbench + // 3. Run with: go test -v --integrationTest --testbucket=test-bucket -timeout=5m + flagsSet := [][]string{ + {"--client-protocol=grpc"}, + } + + for _, flags := range flagsSet { + ts.flags = flags + log.Printf("Running gRPC header validation tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/emulator_tests/grpc_header_validation/setup_test.go b/tools/integration_tests/emulator_tests/grpc_header_validation/setup_test.go new file mode 100644 index 0000000000..3957404966 --- /dev/null +++ b/tools/integration_tests/emulator_tests/grpc_header_validation/setup_test.go @@ -0,0 +1,49 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package grpc_header_validation + +import ( + "log" + "os" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" +) + +var ( + mountFunc func([]string) error + // root directory is the directory to be unmounted. + rootDir string +) + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + if setup.MountedDirectory() != "" { + log.Printf("These tests will not run with mounted directory..") + return + } + + // Set up test directory. + setup.SetUpTestDirForTestBucketFlag() + + rootDir = setup.MntDir() + + log.Println("Running gRPC header validation tests with static mounting...") + mountFunc = static_mounting.MountGcsfuseWithStaticMounting + successCode := m.Run() + os.Exit(successCode) +} diff --git a/tools/integration_tests/emulator_tests/read_stall/read_stall_test.go b/tools/integration_tests/emulator_tests/read_stall/read_stall_test.go new file mode 100644 index 0000000000..adf77dadd1 --- /dev/null +++ b/tools/integration_tests/emulator_tests/read_stall/read_stall_test.go @@ -0,0 +1,105 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package read_stall + +import ( + "fmt" + "log" + "path" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + emulator_tests "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/emulator_tests/util" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" +) + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +const ( + fileSize = 5 * 1024 * 1024 + forcedStallTime = 5 * time.Second + minReqTimeout = 1500 * time.Millisecond +) + +type readStall struct { + port int + proxyProcessId int + proxyServerLogFile string + flags []string + configPath string + suite.Suite +} + +func (r *readStall) SetupTest() { + r.configPath = "../configs/read_stall_5s.yaml" + r.proxyServerLogFile = setup.CreateProxyServerLogFile(r.T()) + var err error + r.port, r.proxyProcessId, err = emulator_tests.StartProxyServer(r.configPath, r.proxyServerLogFile) + require.NoError(r.T(), err) + setup.AppendProxyEndpointToFlagSet(&r.flags, r.port) + setup.MountGCSFuseWithGivenMountFunc(r.flags, mountFunc) + testDirPath = setup.SetupTestDirectory(r.T().Name()) +} + +func (r *readStall) TearDownTest() { + setup.UnmountGCSFuse(rootDir) + assert.NoError(r.T(), emulator_tests.KillProxyServerProcess(r.proxyProcessId)) + setup.SaveGCSFuseLogFileInCaseOfFailure(r.T()) + setup.SaveProxyServerLogFileInCaseOfFailure(r.proxyServerLogFile, r.T()) +} + +//////////////////////////////////////////////////////////////////////// +// Test scenarios +//////////////////////////////////////////////////////////////////////// + +// TestReadFirstByteStallInducedShouldCompleteInLessThanStallTime verifies that reading the first byte +// of a file completes in less time than the configured stall time, even when a read stall is induced. +// It creates a file, reads the first byte, and asserts that the elapsed time is less than the expected stall duration. +func (r *readStall) TestReadFirstByteStallInducedShouldCompleteInLessThanStallTime() { + filePath := path.Join(testDirPath, "file.txt") + operations.CreateFileOfSize(fileSize, filePath, r.T()) + + elapsedTime, err := emulator_tests.ReadFirstByte(r.T(), filePath) + + assert.NoError(r.T(), err) + assert.Greater(r.T(), elapsedTime, minReqTimeout) + assert.Less(r.T(), elapsedTime, forcedStallTime) +} + +//////////////////////////////////////////////////////////////////////// +// Test Function (Runs once before all tests) +//////////////////////////////////////////////////////////////////////// + +func TestReadStall(t *testing.T) { + ts := &readStall{} + // Define flag set to run the tests. + flagsSet := [][]string{ + {"--enable-read-stall-retry=true", "--read-stall-min-req-timeout=" + fmt.Sprintf("%dms", minReqTimeout.Milliseconds()), "--read-stall-initial-req-timeout=" + fmt.Sprintf("%dms", minReqTimeout.Milliseconds())}, + } + + // Run tests. + for _, flags := range flagsSet { + ts.flags = flags + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/emulator_tests/read_stall/setup_test.go b/tools/integration_tests/emulator_tests/read_stall/setup_test.go new file mode 100644 index 0000000000..68345dfc70 --- /dev/null +++ b/tools/integration_tests/emulator_tests/read_stall/setup_test.go @@ -0,0 +1,50 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package read_stall + +import ( + "log" + "os" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" +) + +var ( + testDirPath string + mountFunc func([]string) error + // root directory is the directory to be unmounted. + rootDir string +) + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + if setup.MountedDirectory() != "" { + log.Printf("These tests will not run with mounted directory..") + return + } + + // Set up test directory. + setup.SetUpTestDirForTestBucketFlag() + + rootDir = setup.MntDir() + + log.Println("Running static mounting tests...") + mountFunc = static_mounting.MountGcsfuseWithStaticMounting + successCode := m.Run() + os.Exit(successCode) +} diff --git a/tools/integration_tests/emulator_tests/streaming_writes_failure/common_failure_test.go b/tools/integration_tests/emulator_tests/streaming_writes_failure/common_failure_test.go new file mode 100644 index 0000000000..673bdeb1a0 --- /dev/null +++ b/tools/integration_tests/emulator_tests/streaming_writes_failure/common_failure_test.go @@ -0,0 +1,281 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streaming_writes_failure + +import ( + "context" + "log" + "os" + + "cloud.google.com/go/storage" + emulator_tests "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/emulator_tests/util" + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +// ////////////////////////////////////////////////////////////////////// +// Boilerplate +// ////////////////////////////////////////////////////////////////////// + +type commonFailureTestSuite struct { + suite.Suite + configPath string + flags []string + testDirPath string + filePath string + fh1 *os.File + storageClient *storage.Client // Storage Client based on proxy server. + closeStorageClient func() error + ctx context.Context + data []byte + port int + proxyProcessId int + proxyServerLogFile string + gcsObjectValidator +} + +type gcsObjectValidator interface { + // Validate file from GCS for empty gcs file and new local files. + validateGcsObject() +} + +// ////////////////////////////////////////////////////////////////////// +// Helpers +// ////////////////////////////////////////////////////////////////////// + +func (t *commonFailureTestSuite) SetupSuite() { + t.flags = []string{"--write-block-size-mb=1", "--write-max-blocks-per-file=1"} + // Generate 5 MB random data. + var err error + t.data, err = operations.GenerateRandomData(5 * operations.MiB) + require.NoError(t.T(), err) + log.Printf("Running tests with flags: %v", t.flags) +} + +func (t *commonFailureTestSuite) setupTest() { + t.T().Helper() + // Start proxy server for each test to ensure the config is initialized per test. + t.proxyServerLogFile = setup.CreateProxyServerLogFile(t.T()) + var err error + t.port, t.proxyProcessId, err = emulator_tests.StartProxyServer(t.configPath, t.proxyServerLogFile) + require.NoError(t.T(), err) + setup.AppendProxyEndpointToFlagSet(&t.flags, t.port) + // Create storage client before running tests. + t.ctx = context.Background() + t.closeStorageClient = CreateStorageClientWithCancel(&t.ctx, &t.storageClient) + setup.MountGCSFuseWithGivenMountFunc(t.flags, mountFunc) + // Setup random testDirName. + testDirName = testDirNamePrefix + setup.GenerateRandomString(5) + // Setup test directory for testing. + t.testDirPath = setup.SetupTestDirectory(testDirName) +} + +func (t *commonFailureTestSuite) TearDownTest() { + setup.UnmountGCSFuse(rootDir) + assert.NoError(t.T(), t.closeStorageClient()) + assert.NoError(t.T(), emulator_tests.KillProxyServerProcess(t.proxyProcessId)) + setup.SaveGCSFuseLogFileInCaseOfFailure(t.T()) + setup.SaveProxyServerLogFileInCaseOfFailure(t.proxyServerLogFile, t.T()) +} + +func (t *commonFailureTestSuite) writingWithNewFileHandleAlsoFails(data []byte, off int64) { + t.T().Helper() + // Opening a new file handle succeeds. + fh := operations.OpenFile(t.filePath, t.T()) + // Writes with this file handle fails. + _, err := fh.WriteAt(data, off) + assert.Error(t.T(), err) + // Closing the file handle returns error. + operations.CloseFileShouldThrowError(t.T(), fh) +} + +func (t *commonFailureTestSuite) writingAfterBwhReinitializationSucceeds() { + t.T().Helper() + // Verify that expectation from GCS matches. + t.validateGcsObject() + // Opening new file handle and writing to file succeeds. + t.fh1 = operations.CreateFile(t.filePath, FilePerms, t.T()) + _, err := t.fh1.WriteAt(t.data, 0) + assert.NoError(t.T(), err) + // Sync succeeds. + err = t.fh1.Sync() + assert.NoError(t.T(), err) +} + +// ////////////////////////////////////////////////////////////////////// +// Tests +// ////////////////////////////////////////////////////////////////////// + +func (t *commonFailureTestSuite) TestStreamingWritesFailsOnSecondChunkUploadFailure() { + // Write first 2 MB (say A,B) block to file succeeds but async upload of block B will result in error. + // Fuse:[B] -> Go-SDK:[A] -> GCS[] + _, err := t.fh1.WriteAt(t.data[:2*operations.MiB], 0) + assert.NoError(t.T(), err) + // Write again 2MB (C, D) will trigger B upload. + // Fuse:[D] -> Go-SDK:[C] -> GCS[A, B -> upload fails] + _, _ = t.fh1.WriteAt(t.data[2*operations.MiB:4*operations.MiB], 2*operations.MiB) + + // Write 5th 1MB results in errors. + _, err = t.fh1.WriteAt(t.data[4*operations.MiB:5*operations.MiB], 4*operations.MiB) + + require.Error(t.T(), err) + t.writingWithNewFileHandleAlsoFails(t.data[4*operations.MiB:5*operations.MiB], 4*operations.MiB) + // Close file handle to reinitialize bwh. + operations.CloseFileShouldThrowError(t.T(), t.fh1) + // Opening new file handle and writing to file succeeds. + t.writingAfterBwhReinitializationSucceeds() + // Close and validate object content found on GCS. + CloseFileAndValidateContentFromGCS(t.ctx, t.storageClient, t.fh1, testDirName, FileName1, string(t.data), t.T()) +} + +func (t *commonFailureTestSuite) TestStreamingWritesTruncateSmallerFailsOnSecondChunkUploadFailure() { + // Write first 2 MB (say A,B) block to file succeeds but async upload of block B will result in error. + // Fuse:[B] -> Go-SDK:[A] -> GCS[] + _, err := t.fh1.WriteAt(t.data[:2*operations.MiB], 0) + assert.NoError(t.T(), err) + // Write again 2MB (C, D) will trigger B upload. + // Fuse:[D] -> Go-SDK:[C] -> GCS[A, B -> upload fails] + _, _ = t.fh1.WriteAt(t.data[2*operations.MiB:4*operations.MiB], 2*operations.MiB) + + // Write 5th 1MB results in errors. + _, err = t.fh1.WriteAt(t.data[4*operations.MiB:5*operations.MiB], 4*operations.MiB) + + require.Error(t.T(), err) + // Truncate to smaller size fails. + err = t.fh1.Truncate(1 * operations.MiB) + assert.Error(t.T(), err) + t.writingWithNewFileHandleAlsoFails(t.data[4*operations.MiB:5*operations.MiB], 4*operations.MiB) + // Close file handle to reinitialize bwh. + operations.CloseFileShouldThrowError(t.T(), t.fh1) + // Opening new file handle and writing to file succeeds. + t.writingAfterBwhReinitializationSucceeds() + // Close and validate object content found on GCS. + CloseFileAndValidateContentFromGCS(t.ctx, t.storageClient, t.fh1, testDirName, FileName1, string(t.data), t.T()) +} + +func (t *commonFailureTestSuite) TestStreamingWritesTruncateBiggerSucceedsOnSecondChunkUploadFailure() { + // Write first 2 MB (say A,B) block to file succeeds but async upload of block B will result in error. + // Fuse:[B] -> Go-SDK:[A] -> GCS[] + _, err := t.fh1.WriteAt(t.data[:2*operations.MiB], 0) + assert.NoError(t.T(), err) + // Write again 2MB (C, D) will trigger B upload. + // Fuse:[D] -> Go-SDK:[C] -> GCS[A, B -> upload fails] + _, _ = t.fh1.WriteAt(t.data[2*operations.MiB:4*operations.MiB], 2*operations.MiB) + + // Write 5th 1MB results in errors. + _, err = t.fh1.WriteAt(t.data[4*operations.MiB:5*operations.MiB], 4*operations.MiB) + + require.Error(t.T(), err) + // Opening new file handle succeeds. + fh2 := operations.OpenFile(t.filePath, t.T()) + // Truncate to bigger size succeeds. + err = fh2.Truncate(5 * operations.MiB) + assert.NoError(t.T(), err) + // Closing all file handles to reinitialize bwh. + operations.CloseFileShouldThrowError(t.T(), fh2) + operations.CloseFileShouldThrowError(t.T(), t.fh1) + // Opening new file handle and writing to file succeeds. + t.writingAfterBwhReinitializationSucceeds() + // Truncate to bigger size succeeds. + err = t.fh1.Truncate(6 * operations.MiB) + assert.NoError(t.T(), err) + // Close and validate object content found on GCS. + emptyBytes := make([]byte, operations.MiB) + CloseFileAndValidateContentFromGCS(t.ctx, t.storageClient, t.fh1, testDirName, FileName1, string(t.data)+string(emptyBytes), t.T()) +} + +func (t *commonFailureTestSuite) TestStreamingWritesSyncFailsOnSecondChunkUploadFailure() { + // Write first 2 MB (say A,B) block to file succeeds but async upload of block B will result in error. + // Fuse:[B] -> Go-SDK:[A] -> GCS[] + _, err := t.fh1.WriteAt(t.data[:2*operations.MiB], 0) + assert.NoError(t.T(), err) + // Sync file succeeds as the block B is only passed to Go-SDK for upload. + // Fuse:[] -> Go-SDK:[B]-> GCS[A] + operations.SyncFile(t.fh1, t.T()) + + // Write next 1 MB block C may succeed based on the status of block B. + // Fuse:[C] -> Go-SDK:[B]-> GCS[A] + _, _ = t.fh1.WriteAt(t.data[2*operations.MiB:3*operations.MiB], 2*operations.MiB) + + // Sync now reports failure from B block upload. + // Fuse:[] -> Go-SDK:[C]-> GCS[A, B -> upload fails] + operations.SyncFileShouldThrowError(t.T(), t.fh1) + // Close file handle to reinitialize bwh. + operations.CloseFileShouldThrowError(t.T(), t.fh1) + // Opening new file handle and writing to file succeeds. + t.writingAfterBwhReinitializationSucceeds() + // Close and validate object content found on GCS. + CloseFileAndValidateContentFromGCS(t.ctx, t.storageClient, t.fh1, testDirName, FileName1, string(t.data), t.T()) +} + +func (t *commonFailureTestSuite) TestStreamingWritesCloseFailsOnSecondChunkUploadFailure() { + // Write first 2 MB (say A,B) block to file succeeds but async upload of block B will result in error. + // Fuse:[B] -> Go-SDK:[A] -> GCS[] + _, err := t.fh1.WriteAt(t.data[:2*operations.MiB], 0) + assert.NoError(t.T(), err) + + // Close fails as it sees error from B block upload. + err = t.fh1.Close() + + require.Error(t.T(), err) + // Opening new file handle and writing to file succeeds. + t.writingAfterBwhReinitializationSucceeds() + // Close and validate object content found on GCS. + CloseFileAndValidateContentFromGCS(t.ctx, t.storageClient, t.fh1, testDirName, FileName1, string(t.data), t.T()) +} + +func (t *commonFailureTestSuite) TestStreamingWritesWhenFinalizeObjectFailure() { + // Write 1 MB data to file succeeds and async upload of block will also succeed. + _, err := t.fh1.WriteAt(t.data[:operations.MiB], 0) + assert.NoError(t.T(), err) + + // Close fails as it sees error on the finalize. + err = t.fh1.Close() + + require.Error(t.T(), err) + // Opening new file handle and writing to file succeeds. + t.writingAfterBwhReinitializationSucceeds() + // Close and validate object content found on GCS. + CloseFileAndValidateContentFromGCS(t.ctx, t.storageClient, t.fh1, testDirName, FileName1, string(t.data), t.T()) +} + +func (t *commonFailureTestSuite) TestStreamingWritesBwhResetsWhenFileHandlesAreOpenInReadMode() { + // Write first 2 MB (say A,B) block to file succeeds but async upload of block B will result in error. + // Fuse:[B] -> Go-SDK:[A] -> GCS[] + _, err := t.fh1.WriteAt(t.data[:2*operations.MiB], 0) + assert.NoError(t.T(), err) + // Write again 2MB (C, D) will trigger B upload. + // Fuse:[D] -> Go-SDK:[C] -> GCS[A, B -> upload fails] + _, _ = t.fh1.WriteAt(t.data[2*operations.MiB:4*operations.MiB], 2*operations.MiB) + + // Write 5th 1MB results in errors. + _, err = t.fh1.WriteAt(t.data[4*operations.MiB:5*operations.MiB], 4*operations.MiB) + + require.Error(t.T(), err) + fh2, err := operations.OpenFileAsReadonly(t.filePath) + assert.NoError(t.T(), err) + // Closing only file handle in write mode. + operations.CloseFileShouldThrowError(t.T(), t.fh1) + // Opening new file handle and writing to file succeeds when file handles in O_RDONLY mode are open. + t.writingAfterBwhReinitializationSucceeds() + operations.CloseFileShouldNotThrowError(t.T(), fh2) + // Close and validate object content found on GCS. + CloseFileAndValidateContentFromGCS(t.ctx, t.storageClient, t.fh1, testDirName, FileName1, string(t.data), t.T()) +} diff --git a/tools/integration_tests/emulator_tests/streaming_writes_failure/empty_gcs_file_failure_test.go b/tools/integration_tests/emulator_tests/streaming_writes_failure/empty_gcs_file_failure_test.go new file mode 100644 index 0000000000..b69a45459c --- /dev/null +++ b/tools/integration_tests/emulator_tests/streaming_writes_failure/empty_gcs_file_failure_test.go @@ -0,0 +1,58 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streaming_writes_failure + +import ( + "path" + "testing" + + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/stretchr/testify/suite" +) + +// ////////////////////////////////////////////////////////////////////// +// Boilerplate +// ////////////////////////////////////////////////////////////////////// + +type emptyGcsFileFailureTestSuite struct { + commonFailureTestSuite +} + +// ////////////////////////////////////////////////////////////////////// +// Helpers +// ////////////////////////////////////////////////////////////////////// + +func (t *emptyGcsFileFailureTestSuite) SetupTest() { + t.configPath = "../configs/empty_gcs_file_2nd_chunk_upload_returns412.yaml" + t.setupTest() + // Create an empty file on GCS. + CreateObjectInGCSTestDir(t.ctx, t.storageClient, testDirName, FileName1, "", t.T()) + ValidateObjectContentsFromGCS(t.ctx, t.storageClient, testDirName, FileName1, "", t.T()) + t.filePath = path.Join(t.testDirPath, FileName1) + t.fh1 = operations.OpenFile(t.filePath, t.T()) +} + +func (t *emptyGcsFileFailureTestSuite) validateGcsObject() { + // Validate empty gcs file is found on GCS. + ValidateObjectContentsFromGCS(t.ctx, t.storageClient, testDirName, FileName1, "", t.T()) +} + +// Executes all failure tests for empty gcs files. +func TestEmptyGcsFileFailureTestSuite(t *testing.T) { + s := new(emptyGcsFileFailureTestSuite) + s.gcsObjectValidator = s + suite.Run(t, s) +} diff --git a/tools/integration_tests/emulator_tests/streaming_writes_failure/new_local_file_failure_test.go b/tools/integration_tests/emulator_tests/streaming_writes_failure/new_local_file_failure_test.go new file mode 100644 index 0000000000..bc8f198a23 --- /dev/null +++ b/tools/integration_tests/emulator_tests/streaming_writes_failure/new_local_file_failure_test.go @@ -0,0 +1,53 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streaming_writes_failure + +import ( + "testing" + + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/stretchr/testify/suite" +) + +// ////////////////////////////////////////////////////////////////////// +// Boilerplate +// ////////////////////////////////////////////////////////////////////// + +type newLocalFileFailureTestSuite struct { + commonFailureTestSuite +} + +// ////////////////////////////////////////////////////////////////////// +// Helpers +// ////////////////////////////////////////////////////////////////////// + +func (t *newLocalFileFailureTestSuite) SetupTest() { + t.configPath = "../configs/local_file_2nd_chunk_upload_returns412.yaml" + t.setupTest() + // Create local file. + t.filePath, t.fh1 = CreateLocalFileInTestDir(t.ctx, t.storageClient, t.testDirPath, FileName1, t.T()) +} + +func (t *newLocalFileFailureTestSuite) validateGcsObject() { + // Validate file not found error from GCS. + ValidateObjectNotFoundErrOnGCS(t.ctx, t.storageClient, testDirName, FileName1, t.T()) +} + +// Executes all failure tests for new local files. +func TestNewLocalFileFailureTestSuite(t *testing.T) { + s := new(newLocalFileFailureTestSuite) + s.gcsObjectValidator = s + suite.Run(t, s) +} diff --git a/tools/integration_tests/emulator_tests/streaming_writes_failure/streaming_writes_failure_test.go b/tools/integration_tests/emulator_tests/streaming_writes_failure/streaming_writes_failure_test.go new file mode 100644 index 0000000000..1dabdb2f98 --- /dev/null +++ b/tools/integration_tests/emulator_tests/streaming_writes_failure/streaming_writes_failure_test.go @@ -0,0 +1,53 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streaming_writes_failure + +import ( + "log" + "os" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" +) + +const ( + testDirNamePrefix = "StreamingWritesFailureTest" +) + +var ( + mountFunc func([]string) error + // root directory is the directory to be unmounted. + rootDir string + testDirName string +) + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + if setup.MountedDirectory() != "" { + log.Printf("These tests will not run with mounted directory.") + return + } + + // Set up test directory. + setup.SetUpTestDirForTestBucketFlag() + rootDir = setup.MntDir() + log.Printf("Test log: %s\n", setup.LogFile()) + log.Println("Running static mounting tests...") + mountFunc = static_mounting.MountGcsfuseWithStaticMounting + successCode := m.Run() + os.Exit(successCode) +} diff --git a/tools/integration_tests/emulator_tests/util/test_helper.go b/tools/integration_tests/emulator_tests/util/test_helper.go new file mode 100644 index 0000000000..c00e94c0de --- /dev/null +++ b/tools/integration_tests/emulator_tests/util/test_helper.go @@ -0,0 +1,254 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package emulator_tests + +import ( + "crypto/rand" + "errors" + "fmt" + "io" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" +) + +const PortAndProxyProcessIdInfoRegex = `Listening Proxy Server On Port \[(\d+)\] with Process ID \[(\d+)\]` + +// StartProxyServer starts the proxy server in the background and returns the port and process ID it is listening on. +// +// It takes the config path and log file path as input. The proxy server is starts in background using +// the go run command. The output of the proxy server is redirected to the log file. It also sets the +// STORAGE_EMULATOR_HOST to `localhost:port` so that any new storage client will use this endpoint by +// default. If any error occurs it fails. +// +// Parameters: +// - configPath: Path for config for Proxy Server. +// - logFilePath: Path for log file for Proxy Server Logs. +// +// Returns: +// - int: Port number that the proxy server is listening on. +// - int: Proxy Server Process ID. +// - error: An error if any error occurs in setting up Proxy Server. +func StartProxyServer(configPath, logFilePath string) (int, int, error) { + cmd := exec.Command("go", "run", "../../../proxy_server/.", "--config-path="+configPath, "--log-file="+logFilePath) + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + err := cmd.Start() + if err != nil { + return -1, -1, fmt.Errorf("error executing proxy server cmd: %v", err) + } + port, proxyProcessId, err := getPortAndProcessInfoFromLogFile(logFilePath) + if err != nil { + return -1, -1, fmt.Errorf("error getting port information from log file: %v", err) + } + // Set STORAGE_EMULATOR_HOST to current proxy server for the test. This ensures + // any storage client created during this test will call this proxy endpoint. + // More details: https://cloud.google.com/go/docs/reference/cloud.google.com/go/storage/latest#hdr-Creating_a_Client + err = os.Setenv("STORAGE_EMULATOR_HOST", fmt.Sprintf("localhost:%d", port)) + if err != nil { + return -1, -1, fmt.Errorf("error setting STORAGE_EMULATOR_HOST environment variable: %v", err) + } + return port, proxyProcessId, nil +} + +// KillProxyServerProcess kills given Proxy Server Process ID. +// It also unsets the STORAGE_EMULATOR_HOST environment variable for proxy server. +// +// Parameters: +// - proxyProcessId: PID of Proxy Server. +// +// Returns: +// - error: if any error occurs in unsetting env or killing proxy Server. +func KillProxyServerProcess(proxyProcessId int) error { + // Unset STORAGE_EMULATOR_HOST environment set in StartProxyServer. + err := os.Unsetenv("STORAGE_EMULATOR_HOST") + if err != nil { + return fmt.Errorf("error unsetting STORAGE_EMULATOR_HOST environment variable: %v", err) + } + // Find Proxy Server Process. + process, err := os.FindProcess(proxyProcessId) + if err != nil { + return fmt.Errorf("error finding the proxy server process with PID %d: %v", proxyProcessId, err) + } + // Send SIGINT to the Process. + err = process.Signal(syscall.SIGINT) + if err != nil { + return fmt.Errorf("error sending SIGINT to the proxy server process with PID %d: %v", proxyProcessId, err) + } + return nil +} + +// WriteFileAndSync creates a file at the given path, writes random data to it, +// and then syncs the file to GCS. It returns the time taken for the sync operation +// and any error encountered. +// +// This function is useful for testing scenarios where file write and sync operations +// might be subject to delays or timeouts. +// +// Parameters: +// - filePath: The path where the file should be created. +// - fileSize: The size of the random data to be written to the file. +// +// Returns: +// - time.Duration: The elapsed time for the file.Sync() operation. +// - error: Any error encountered during file creation, writing, or syncing. +func WriteFileAndSync(filePath string, fileSize int) (time.Duration, error) { + // Create a file for writing + file, err := os.Create(filePath) + if err != nil { + return 0, err + } + defer file.Close() + + // Generate random data + data := make([]byte, fileSize) + if _, err := io.ReadFull(rand.Reader, data); err != nil { + return 0, err + } + + // Write the data to the file + if _, err := file.Write(data); err != nil { + return 0, err + } + + startTime := time.Now() + err = file.Sync() + endTime := time.Now() + + if err != nil { + return 0, err + } + + return endTime.Sub(startTime), nil +} + +// ReadFirstByte reads the first byte of a file and returns the time taken. +func ReadFirstByte(t *testing.T, filePath string) (time.Duration, error) { + t.Helper() + + file, err := operations.OpenFileAsReadonly(filePath) + if err != nil { + return 0, err + } + defer file.Close() + + buffer := make([]byte, 1) + + startTime := time.Now() + _, err = file.Read(buffer) + if err != nil && err != io.EOF { + return 0, err + } + + endTime := time.Now() + + return endTime.Sub(startTime), nil +} + +func GetChunkTransferTimeoutFromFlags(flags []string) (int, error) { + timeout := 10 // Default value + for _, flag := range flags { + if after, ok := strings.CutPrefix(flag, "--chunk-transfer-timeout-secs="); ok { + valueStr := after + var err error + timeout, err = strconv.Atoi(valueStr) + if err != nil { + return 0, err + } + break + } + } + return timeout, nil +} + +// GetPortFromLogFile polls a log file for a specific log message containing the port number. +// It returns the port number if found, or an error if the port is not found within the specified duration. +// +// Parameters: +// - logFilePath: The path to the log file to monitor. +// +// Returns: +// - Port, proxy process ID , or an error if any error occurs. +func getPortAndProcessInfoFromLogFile(logFilePath string) (int, int, error) { + logPollingDuration := 3 * time.Minute // Duration to poll logs. + logPollingFrequency := 3 * time.Second // Frequency to poll logs. + + // Regular expression to extract the port number and Process ID from the log file. + re := regexp.MustCompile(PortAndProxyProcessIdInfoRegex) + + // Calculate the timeout time. + timeout := time.After(logPollingDuration) + + // Create a ticker to poll the log file at the specified frequency. + ticker := time.NewTicker(logPollingFrequency) + defer ticker.Stop() + + // Poll the log file until the timeout or the port is found. + for { + select { + case <-timeout: + // Timeout occurred, return an error. + return -1, -1, errors.New("timeout: port number not found in log file") + case <-ticker.C: + // Read the log file. + content, err := operations.ReadFile(logFilePath) + if err != nil { + // Log file could not be opened, return an error. + return -1, -1, fmt.Errorf("failed to read the log file: %w", err) + } + + // Attempt to extract the port number from the log line. + match := re.FindStringSubmatch(string(content)) + if len(match) == 3 { + // Port number and PID found, parse it and return. + portStr, proxyProcessIdStr := match[1], match[2] + + port, err := strconv.Atoi(portStr) + if err != nil { + // Port number could not be parsed, return an error. + return -1, -1, fmt.Errorf("failed to parse port number: %w", err) + } + proxyProcessId, err := strconv.Atoi(proxyProcessIdStr) + if err != nil { + // Process ID could not be parsed, return an error. + return -1, -1, fmt.Errorf("failed to parse process ID: %w", err) + } + return port, proxyProcessId, nil + } + } + } +} + +func GetChunkRetryDeadlineFromFlags(flags []string) (int, error) { + deadline := 120 // Default value + for _, flag := range flags { + if after, ok := strings.CutPrefix(flag, "--chunk-retry-deadline-secs="); ok { + valueStr := after + var err error + deadline, err = strconv.Atoi(valueStr) + if err != nil { + return 0, err + } + break + } + } + return deadline, nil +} diff --git a/tools/integration_tests/emulator_tests/write_stall/write_stall_test.go b/tools/integration_tests/emulator_tests/write_stall/write_stall_test.go new file mode 100644 index 0000000000..dca243f97a --- /dev/null +++ b/tools/integration_tests/emulator_tests/write_stall/write_stall_test.go @@ -0,0 +1,50 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package write_stall + +import ( + "log" + "os" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" +) + +var ( + testDirPath string + mountFunc func([]string) error + // root directory is the directory to be unmounted. + rootDir string +) + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + if setup.MountedDirectory() != "" { + log.Printf("These tests will not run with mounted directory..") + return + } + + // Set up test directory. + setup.SetUpTestDirForTestBucketFlag() + + rootDir = setup.MntDir() + + log.Println("Running static mounting tests...") + mountFunc = static_mounting.MountGcsfuseWithStaticMounting + successCode := m.Run() + os.Exit(successCode) +} diff --git a/tools/integration_tests/emulator_tests/write_stall/writes_stall_on_sync_test.go b/tools/integration_tests/emulator_tests/write_stall/writes_stall_on_sync_test.go new file mode 100644 index 0000000000..08d5c90750 --- /dev/null +++ b/tools/integration_tests/emulator_tests/write_stall/writes_stall_on_sync_test.go @@ -0,0 +1,223 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package write_stall + +import ( + "fmt" + "log" + "path" + "testing" + "time" + + emulator_tests "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/emulator_tests/util" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +const ( + fileSize = 50 * 1024 * 1024 + stallTime = 40 * time.Second +) + +type chunkTransferTimeoutInfinity struct { + port int + proxyProcessId int + proxyServerLogFile string + flags []string + suite.Suite +} + +func (s *chunkTransferTimeoutInfinity) SetupTest() { + configPath := "../configs/write_stall_40s.yaml" + s.proxyServerLogFile = setup.CreateProxyServerLogFile(s.T()) + var err error + s.port, s.proxyProcessId, err = emulator_tests.StartProxyServer(configPath, s.proxyServerLogFile) + require.NoError(s.T(), err) + setup.AppendProxyEndpointToFlagSet(&s.flags, s.port) + setup.MountGCSFuseWithGivenMountFunc(s.flags, mountFunc) +} + +func (s *chunkTransferTimeoutInfinity) TearDownTest() { + setup.UnmountGCSFuse(rootDir) + assert.NoError(s.T(), emulator_tests.KillProxyServerProcess(s.proxyProcessId)) + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) + setup.SaveProxyServerLogFileInCaseOfFailure(s.proxyServerLogFile, s.T()) +} + +//////////////////////////////////////////////////////////////////////// +// Test scenarios +//////////////////////////////////////////////////////////////////////// + +// This test verifies that write operations stall for the expected duration +// when write stall is induced while uploading first chunk. +// It creates a file, writes data to it, and then calls Sync() to ensure +// the data is written to GCS. The test measures the time taken for the Sync() +// operation and asserts that it is greater than or equal to the configured stall time. +func (s *chunkTransferTimeoutInfinity) TestWriteStallCausesDelay() { + testDir := "TestWriteStallCausesDelay" + testDirPath = setup.SetupTestDirectory(testDir) + filePath := path.Join(testDirPath, "file.txt") + + elapsedTime, err := emulator_tests.WriteFileAndSync(filePath, fileSize) + + assert.NoError(s.T(), err) + assert.GreaterOrEqual(s.T(), elapsedTime, stallTime) +} + +//////////////////////////////////////////////////////////////////////// +// Test Function (Runs once before all tests) +//////////////////////////////////////////////////////////////////////// + +func TestChunkTransferTimeoutInfinity(t *testing.T) { + ts := &chunkTransferTimeoutInfinity{} + // Define flag set to run the tests. + flagsSet := [][]string{ + {"--chunk-transfer-timeout-secs=0"}, + } + + // Run tests. + for _, flags := range flagsSet { + ts.flags = flags + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} + +func TestChunkTransferTimeout(t *testing.T) { + flagSets := [][]string{ + {}, + {"--chunk-transfer-timeout-secs=5"}, + } + + stallScenarios := []struct { + name string + configPath string + expectedTimeout func(int) time.Duration + }{ + { + name: "SingleStall", + configPath: "../configs/write_stall_40s.yaml", + expectedTimeout: func(chunkTransferTimeoutSecs int) time.Duration { + return time.Duration(chunkTransferTimeoutSecs) * time.Second + }, + }, + { + name: "MultipleStalls", + configPath: "../configs/write_stall_twice_40s.yaml", // 2 stalls + // Expect total time to be greater than the timeout multiplied by the number of stalls (2 in this case). + expectedTimeout: func(chunkTransferTimeoutSecs int) time.Duration { + return time.Duration(chunkTransferTimeoutSecs*2) * time.Second + }, + }, + } + + for _, flags := range flagSets { + chunkTransferTimeoutSecs, err := emulator_tests.GetChunkTransferTimeoutFromFlags(flags) + if err != nil { + t.Fatalf("Invalid chunk-transfer-timeout-secs flag: %v", err) + } + + t.Run(fmt.Sprintf("Flags_%v", flags), func(t *testing.T) { + for _, scenario := range stallScenarios { + t.Run(scenario.name, func(t *testing.T) { + proxyServerLogFile := setup.CreateProxyServerLogFile(t) + port, proxyProcessId, err := emulator_tests.StartProxyServer(scenario.configPath, proxyServerLogFile) + require.NoError(t, err) + setup.AppendProxyEndpointToFlagSet(&flags, port) + setup.MountGCSFuseWithGivenMountFunc(flags, mountFunc) + + defer func() { // Defer unmount, killing the proxy server and saving log files. + setup.UnmountGCSFuse(rootDir) + assert.NoError(t, emulator_tests.KillProxyServerProcess(proxyProcessId)) + setup.SaveGCSFuseLogFileInCaseOfFailure(t) + setup.SaveProxyServerLogFileInCaseOfFailure(proxyServerLogFile, t) + }() + + testDir := scenario.name + setup.GenerateRandomString(3) + testDirPath = setup.SetupTestDirectory(testDir) + filePath := path.Join(testDirPath, "file.txt") + + elapsedTime, err := emulator_tests.WriteFileAndSync(filePath, fileSize) + + assert.NoError(t, err, "failed to write file and sync") + expectedTimeout := scenario.expectedTimeout(chunkTransferTimeoutSecs) + assert.GreaterOrEqual(t, elapsedTime, expectedTimeout) + assert.Less(t, elapsedTime, expectedTimeout+5*time.Second) + }) + } + }) + } +} + +func TestChunkRetryDeadline(t *testing.T) { + scenarios := []struct { + name string + flags []string + expectedSuccess bool + expectedStall time.Duration + }{ + { + name: "StallUnderDeadline_Pass", + flags: []string{"--chunk-transfer-timeout-secs=10", "--chunk-retry-deadline-secs=120"}, + expectedStall: 40 * time.Second, + expectedSuccess: true, + }, + { + name: "StallOverDeadline_Fail", + flags: []string{"--chunk-transfer-timeout-secs=10", "--chunk-retry-deadline-secs=32"}, + expectedStall: 40 * time.Second, + expectedSuccess: false, + }, + } + // The proxy is configured to stall the first 4 requests (4 stalls). + // With a 10s chunk-transfer-timeout, the client will timeout after 10s for each of the first 4 attempts. + // This results in 40s of total elapsed time before the 5th attempt (which is the 4th retry) succeeds. + configPath := "../configs/write_stalls_four_times_60s.yaml" + + for _, scenario := range scenarios { + t.Run(scenario.name, func(t *testing.T) { + proxyServerLogFile := setup.CreateProxyServerLogFile(t) + flags := append([]string{}, scenario.flags...) + port, proxyProcessId, err := emulator_tests.StartProxyServer(configPath, proxyServerLogFile) + require.NoError(t, err) + setup.AppendProxyEndpointToFlagSet(&flags, port) + setup.MountGCSFuseWithGivenMountFunc(flags, mountFunc) + defer func() { + setup.UnmountGCSFuse(rootDir) + assert.NoError(t, emulator_tests.KillProxyServerProcess(proxyProcessId)) + setup.SaveGCSFuseLogFileInCaseOfFailure(t) + setup.SaveProxyServerLogFileInCaseOfFailure(proxyServerLogFile, t) + }() + testDir := scenario.name + setup.GenerateRandomString(3) + testDirPath = setup.SetupTestDirectory(testDir) + filePath := path.Join(testDirPath, "file.txt") + + elapsedTime, err := emulator_tests.WriteFileAndSync(filePath, fileSize) + + if scenario.expectedSuccess { + assert.NoError(t, err, "expected success for chunk-retry-deadline-secs") + assert.GreaterOrEqual(t, elapsedTime, scenario.expectedStall) + } else { + assert.Error(t, err, "expected failure due to chunk-retry-deadline-secs") + } + }) + } +} diff --git a/tools/integration_tests/explicit_dir/explicit_dir_test.go b/tools/integration_tests/explicit_dir/explicit_dir_test.go index 85b0b729ac..7bd7eb41e4 100644 --- a/tools/integration_tests/explicit_dir/explicit_dir_test.go +++ b/tools/integration_tests/explicit_dir/explicit_dir_test.go @@ -1,10 +1,10 @@ -// Copyright 2023 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -22,40 +22,56 @@ import ( "testing" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" ) const DirForExplicitDirTests = "dirForExplicitDirTests" +// IMPORTANT: To prevent global variable pollution, enhance code clarity, +// and avoid inadvertent errors. We strongly suggest that, all new package-level +// variables (which would otherwise be declared with `var` at the package root) should +// be added as fields to this 'env' struct instead. +type env struct { + storageClient *storage.Client + ctx context.Context +} + +var testEnv env + func TestMain(m *testing.M) { setup.ParseSetUpFlags() - var storageClient *storage.Client - // Create storage client before running tests. - ctx := context.Background() - storageClient, err := client.CreateStorageClient(ctx) - if err != nil { - log.Printf("Error creating storage client: %v\n", err) - os.Exit(1) - } - defer storageClient.Close() - - // These tests will not run on HNS buckets because the "--implicit-dirs=false" flag does not function similarly to how it does on FLAT buckets. - // Note that HNS buckets do not have the concept of implicit directories. - if setup.IsHierarchicalBucket(ctx, storageClient) { - log.Println("These tests will not run on HNS buckets.") - return + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.ExplicitDir) == 0 { + log.Println("No configuration found for explicit_dir tests in config. Using flags instead.") + // Populate the config manually. + cfg.ExplicitDir = make([]test_suite.TestConfig, 1) + cfg.ExplicitDir[0].TestBucket = setup.TestBucket() + cfg.ExplicitDir[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.ExplicitDir[0].Configs = make([]test_suite.ConfigItem, 1) + cfg.ExplicitDir[0].Configs[0].Flags = []string{"--implicit-dirs=false", "--implicit-dirs=false --client-protocol=grpc"} + cfg.ExplicitDir[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": false, "zonal": false} } - flags := [][]string{{"--implicit-dirs=false"}} - - if !testing.Short() { - flags = append(flags, []string{"--client-protocol=grpc", "--implicit-dirs=false"}) - } + // 2. Create storage client before running tests. + testEnv.ctx = context.Background() + bucketType := setup.TestEnvironment(testEnv.ctx, &cfg.ExplicitDir[0]) + closeStorageClient := client.CreateStorageClientWithCancel(&testEnv.ctx, &testEnv.storageClient) + defer func() { + err := closeStorageClient() + if err != nil { + log.Fatalf("closeStorageClient failed: %v", err) + } + }() - successCode := implicit_and_explicit_dir_setup.RunTestsForImplicitDirAndExplicitDir(flags, m) + // 3. Build the flag sets dynamically from the config. + flags := setup.BuildFlagSets(cfg.ExplicitDir[0], bucketType, "") + // 4. Run tests with the dynamically generated flags. + successCode := implicit_and_explicit_dir_setup.RunTestsForExplicitAndImplicitDir(&cfg.ExplicitDir[0], flags, m) os.Exit(successCode) } diff --git a/tools/integration_tests/explicit_dir/list_test.go b/tools/integration_tests/explicit_dir/list_test.go index 4421ef1ba4..082006c481 100644 --- a/tools/integration_tests/explicit_dir/list_test.go +++ b/tools/integration_tests/explicit_dir/list_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,15 +15,17 @@ package explicit_dir_test import ( + "errors" "fmt" "io/fs" "log" "os" + "path" "path/filepath" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup" ) func TestListOnlyExplicitObjectsFromBucket(t *testing.T) { @@ -38,7 +40,12 @@ func TestListOnlyExplicitObjectsFromBucket(t *testing.T) { // testBucket/dirForExplicitDirTests/explicitDirectory/fileInExplicitDir1 -- File // testBucket/dirForExplicitDirTests/explicitDirectory/fileInExplicitDir2 -- File - implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(DirForExplicitDirTests) + // TODO: Remove the condition and keep the storage-client flow for non-ZB too. + if setup.IsZonalBucketRun() { + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructureUsingStorageClient(testEnv.ctx, t, testEnv.storageClient, DirForExplicitDirTests) + } else { + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(DirForExplicitDirTests) + } implicit_and_explicit_dir_setup.CreateExplicitDirectoryStructure(DirForExplicitDirTests, t) err := filepath.WalkDir(testDir, func(path string, dir fs.DirEntry, err error) error { @@ -100,3 +107,28 @@ func TestListOnlyExplicitObjectsFromBucket(t *testing.T) { return } } + +func TestStatImplicitDirAfterList(t *testing.T) { + testDirPath := setup.SetupTestDirectory(DirForExplicitDirTests) + // TODO: Remove the condition and keep the storage-client flow for non-ZB too. + if setup.IsZonalBucketRun() { + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructureUsingStorageClient(testEnv.ctx, t, testEnv.storageClient, DirForExplicitDirTests) + } else { + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(DirForExplicitDirTests) + } + + // List the directory + _, err := os.ReadDir(testDirPath) + if err != nil { + t.Fatalf("ReadDir failed: %v", err) + } + + // Stat the implicit directory + implicitDirPath := path.Join(testDirPath, implicit_and_explicit_dir_setup.ImplicitDirectory) + _, err = os.Stat(implicitDirPath) + if err == nil { + t.Errorf("expected an error for implicit directory %q, but got none", implicitDirPath) + } else if !errors.Is(err, os.ErrNotExist) { + t.Errorf("expected 'not exist' error, but got: %v", err) + } +} diff --git a/tools/integration_tests/explicit_dir/rename_sym_link_test.go b/tools/integration_tests/explicit_dir/rename_sym_link_test.go new file mode 100644 index 0000000000..d2e1254cd5 --- /dev/null +++ b/tools/integration_tests/explicit_dir/rename_sym_link_test.go @@ -0,0 +1,53 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package explicit_dir_test + +import ( + "os" + "path" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRenameSymlinkToExplicitDir(t *testing.T) { + testDir := setup.SetupTestDirectory(DirForExplicitDirTests) + targetDirName := "target_dir" + targetDirPath := path.Join(testDir, targetDirName) + err := os.Mkdir(targetDirPath, setup.DirPermission_0755) + require.NoError(t, err) + oldSymlinkPath := path.Join(testDir, "symlink_old") + err = os.Symlink(targetDirPath, oldSymlinkPath) + require.NoError(t, err) + newSymlinkPath := path.Join(testDir, "symlink_new") + + err = os.Rename(oldSymlinkPath, newSymlinkPath) + + require.NoError(t, err) + _, err = os.Lstat(oldSymlinkPath) + require.Error(t, err) + assert.True(t, os.IsNotExist(err)) + fi, err := os.Lstat(newSymlinkPath) + require.NoError(t, err) + assert.Equal(t, os.ModeSymlink, fi.Mode()&os.ModeType) + targetRead, err := os.Readlink(newSymlinkPath) + require.NoError(t, err) + assert.Equal(t, targetDirPath, targetRead) + targetFi, err := os.Stat(newSymlinkPath) + require.NoError(t, err) + assert.True(t, targetFi.IsDir()) +} diff --git a/tools/integration_tests/flag_optimizations/mount_test.go b/tools/integration_tests/flag_optimizations/mount_test.go new file mode 100644 index 0000000000..b0aa883b0b --- /dev/null +++ b/tools/integration_tests/flag_optimizations/mount_test.go @@ -0,0 +1,58 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flag_optimizations + +import ( + "strings" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" +) + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +func tearDownMountTest(t *testing.T, err error) { + setup.SaveGCSFuseLogFileInCaseOfFailure(t) + if err == nil { + setup.UnmountGCSFuseWithConfig(&testEnv.cfg) + } +} + +//////////////////////////////////////////////////////////////////////// +// Test Functions +//////////////////////////////////////////////////////////////////////// + +func TestMountFails(t *testing.T) { + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + t.Fatalf("This test is not valid for mounted-directory tests.") + } + + flagsSet := setup.BuildFlagSets(testEnv.cfg, testEnv.bucketType, t.Name()) + + for _, flags := range flagsSet { + tcName := strings.ReplaceAll(strings.Join(flags, ","), "--", "") + t.Run(tcName, func(t *testing.T) { + // Arrange and Act + err := mountGCSFuseAndSetupTestDir(flags, testEnv.ctx, testEnv.storageClient) + defer tearDownMountTest(t, err) + + // Assert + assert.Error(t, err) + }) + } +} diff --git a/tools/integration_tests/flag_optimizations/optimization_test.go b/tools/integration_tests/flag_optimizations/optimization_test.go new file mode 100644 index 0000000000..95412bcb8e --- /dev/null +++ b/tools/integration_tests/flag_optimizations/optimization_test.go @@ -0,0 +1,152 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flag_optimizations + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +func tearDownOptimizationTest(t *testing.T) { + setup.SaveGCSFuseLogFileInCaseOfFailure(t) + setup.UnmountGCSFuseWithConfig(&testEnv.cfg) +} + +//////////////////////////////////////////////////////////////////////// +// Test Functions +//////////////////////////////////////////////////////////////////////// + +func TestImplicitDirsNotEnabled(t *testing.T) { + flagsSet := setup.BuildFlagSets(testEnv.cfg, testEnv.bucketType, t.Name()) + for _, flags := range flagsSet { + t.Run(strings.Join(flags, "_"), func(t *testing.T) { + mustMountGCSFuseAndSetupTestDir(flags, testEnv.ctx, testEnv.storageClient) + defer tearDownOptimizationTest(t) + + // Arrange + implicitDirPath := filepath.Join(testDirName, "implicitDir"+setup.GenerateRandomString(5)) + mountedImplicitDirPath := filepath.Join(setup.MntDir(), implicitDirPath) + client.CreateImplicitDir(testEnv.ctx, testEnv.storageClient, implicitDirPath, t) + defer func() { + err := client.DeleteAllObjectsWithPrefix(testEnv.ctx, testEnv.storageClient, implicitDirPath) + require.NoError(t, err) + }() + + // Act + _, err := os.Stat(mountedImplicitDirPath) + + // Assert + require.Error(t, err, "Found unexpected implicit directory %q", mountedImplicitDirPath) + }) + } +} + +func TestRenameDirLimitNotSet(t *testing.T) { + flagsSet := setup.BuildFlagSets(testEnv.cfg, testEnv.bucketType, t.Name()) + for _, flags := range flagsSet { + t.Run(strings.Join(flags, "_"), func(t *testing.T) { + mustMountGCSFuseAndSetupTestDir(flags, testEnv.ctx, testEnv.storageClient) + defer tearDownOptimizationTest(t) + + // Arrange + srcDirPath := filepath.Join(testDirName, "srcDirContainingFiles"+setup.GenerateRandomString(5)) + mountedSrcDirPath := filepath.Join(setup.MntDir(), srcDirPath) + dstDirPath := filepath.Join(testDirName, "dstDirContainingFiles"+setup.GenerateRandomString(5)) + mountedDstDirPath := filepath.Join(setup.MntDir(), dstDirPath) + require.NoError(t, client.CreateGcsDir(testEnv.ctx, testEnv.storageClient, srcDirPath, setup.TestBucket(), "")) + client.CreateNFilesInDir(testEnv.ctx, testEnv.storageClient, 1, "file", 1024, srcDirPath, t) + defer func() { + err := client.DeleteAllObjectsWithPrefix(testEnv.ctx, testEnv.storageClient, srcDirPath) + require.NoError(t, err) + err = client.DeleteAllObjectsWithPrefix(testEnv.ctx, testEnv.storageClient, dstDirPath) + require.NoError(t, err) + }() + + // Act + err := os.Rename(mountedSrcDirPath, mountedDstDirPath) + + // Assert + require.Error(t, err, "Unexpectedly succeeded in renaming directory %q to %q", mountedSrcDirPath, mountedDstDirPath) + }) + } +} + +func TestImplicitDirsEnabled(t *testing.T) { + flagsSet := setup.BuildFlagSets(testEnv.cfg, testEnv.bucketType, t.Name()) + for _, flags := range flagsSet { + t.Run(strings.Join(flags, "_"), func(t *testing.T) { + mustMountGCSFuseAndSetupTestDir(flags, testEnv.ctx, testEnv.storageClient) + defer tearDownOptimizationTest(t) + + // Arrange + implicitDirPath := filepath.Join(testDirName, "implicitDir"+setup.GenerateRandomString(5)) + mountedImplicitDirPath := filepath.Join(setup.MntDir(), implicitDirPath) + client.CreateImplicitDir(testEnv.ctx, testEnv.storageClient, implicitDirPath, t) + defer func() { + err := client.DeleteAllObjectsWithPrefix(testEnv.ctx, testEnv.storageClient, implicitDirPath) + require.NoError(t, err) + }() + + // Act + fi, err := os.Stat(mountedImplicitDirPath) + + // Assert + require.NoError(t, err, "Got error statting %q: %v", mountedImplicitDirPath, err) + require.NotNil(t, fi, "Expected directory %q", mountedImplicitDirPath) + assert.True(t, fi.IsDir(), "Expected %q to be a directory, but got not-dir", mountedImplicitDirPath) + }) + } +} + +func TestRenameDirLimitSet(t *testing.T) { + flagsSet := setup.BuildFlagSets(testEnv.cfg, testEnv.bucketType, t.Name()) + for _, flags := range flagsSet { + t.Run(strings.Join(flags, "_"), func(t *testing.T) { + mustMountGCSFuseAndSetupTestDir(flags, testEnv.ctx, testEnv.storageClient) + defer tearDownOptimizationTest(t) + + // Arrange + srcDirPath := filepath.Join(testDirName, "srcDirContainingFiles"+setup.GenerateRandomString(5)) + mountedSrcDirPath := filepath.Join(setup.MntDir(), srcDirPath) + dstDirPath := filepath.Join(testDirName, "dstDirContainingFiles"+setup.GenerateRandomString(5)) + mountedDstDirPath := filepath.Join(setup.MntDir(), dstDirPath) + require.NoError(t, client.CreateGcsDir(testEnv.ctx, testEnv.storageClient, srcDirPath, setup.TestBucket(), "")) + client.CreateNFilesInDir(testEnv.ctx, testEnv.storageClient, 1, "file", 1024, srcDirPath, t) + defer func() { + err := client.DeleteAllObjectsWithPrefix(testEnv.ctx, testEnv.storageClient, srcDirPath) + require.NoError(t, err) + err = client.DeleteAllObjectsWithPrefix(testEnv.ctx, testEnv.storageClient, dstDirPath) + require.NoError(t, err) + }() + + // Act + err := os.Rename(mountedSrcDirPath, mountedDstDirPath) + + // Assert + require.NoError(t, err, "Failed to rename directory %q to %q: %v", mountedSrcDirPath, mountedDstDirPath, err) + }) + } +} diff --git a/tools/integration_tests/flag_optimizations/setup_test.go b/tools/integration_tests/flag_optimizations/setup_test.go new file mode 100644 index 0000000000..c1b7ff2b1e --- /dev/null +++ b/tools/integration_tests/flag_optimizations/setup_test.go @@ -0,0 +1,229 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Provide tests for cases where bucket is mounted with flag(s) --machine-type and/or --profile. +package flag_optimizations + +import ( + "context" + "log" + "os" + "path" + "testing" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/dynamic_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/only_dir_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" +) + +const ( + testDirName = "FlagOptimizationsTests" + onlyDirMounted = "OnlyDirMountFlagOptimizations" + GKETempDir = "/gcsfuse-tmp" +) + +// To prevent global variable pollution, enhance code clarity, +// and avoid inadvertent errors. We strongly suggest that, all new package-level +// variables (which would otherwise be declared with `var` at the package root) should +// be added as fields to this 'env' struct instead. +type env struct { + testDirPath string + mountFunc func(*test_suite.TestConfig, []string) error + // mount directory is where our tests run. + mountDir string + // root directory is the directory to be mounted/unmounted. + rootDir string + storageClient *storage.Client + ctx context.Context + bucketType string + cfg test_suite.TestConfig +} + +var ( + testEnv env +) + +//////////////////////////////////////////////////////////////////////// +// Helpers +//////////////////////////////////////////////////////////////////////// + +func mountGCSFuseAndSetupTestDir(flags []string, ctx context.Context, storageClient *storage.Client) error { + err := setup.MayMountGCSFuseWithGivenMountWithConfigFunc(&testEnv.cfg, flags, testEnv.mountFunc) + if err != nil { + return err + } + if testEnv.cfg.GKEMountedDirectory == "" { + setup.SetMntDir(testEnv.mountDir) + } + testEnv.testDirPath = client.SetupTestDirectory(ctx, storageClient, testDirName) + return nil +} + +func mustMountGCSFuseAndSetupTestDir(flags []string, ctx context.Context, storageClient *storage.Client) { + if err := mountGCSFuseAndSetupTestDir(flags, ctx, storageClient); err != nil { + panic(err) + } +} + +//////////////////////////////////////////////////////////////////////// +// TestMain +//////////////////////////////////////////////////////////////////////// + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + // Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.FlagOptimizations) == 0 { + log.Println("No configuration found for flag_optimizations tests in config. Using flags instead.") + // Populate the config manually. + cfg.FlagOptimizations = make([]test_suite.TestConfig, 1) + cfg.FlagOptimizations[0].TestBucket = setup.TestBucket() + cfg.FlagOptimizations[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.FlagOptimizations[0].LogFile = setup.LogFile() + // Initialize the slice to hold 12 specific test configurations + cfg.FlagOptimizations[0].Configs = make([]test_suite.ConfigItem, 12) + cfg.FlagOptimizations[0].Configs[0].Run = "TestMountFails" + cfg.FlagOptimizations[0].Configs[0].Flags = []string{"--profile=unknown-profile"} + cfg.FlagOptimizations[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.FlagOptimizations[0].Configs[0].RunOnGKE = false + cfg.FlagOptimizations[0].Configs[1].Run = "TestImplicitDirsNotEnabled" + cfg.FlagOptimizations[0].Configs[1].Flags = []string{"--machine-type=low-end-machine"} + cfg.FlagOptimizations[0].Configs[1].Compatible = map[string]bool{"flat": true, "hns": false, "zonal": false} + cfg.FlagOptimizations[0].Configs[1].RunOnGKE = true + cfg.FlagOptimizations[0].Configs[2].Run = "TestRenameDirLimitNotSet" + cfg.FlagOptimizations[0].Configs[2].Flags = []string{"--machine-type=low-end-machine", "--profile=aiml-training", "--profile=aiml-serving"} + cfg.FlagOptimizations[0].Configs[2].Compatible = map[string]bool{"flat": true, "hns": false, "zonal": false} + cfg.FlagOptimizations[0].Configs[2].RunOnGKE = true + cfg.FlagOptimizations[0].Configs[3].Run = "TestImplicitDirsEnabled" + cfg.FlagOptimizations[0].Configs[3].Flags = []string{ + "--machine-type=a3-highgpu-8g", + "--profile=aiml-training", + "--profile=aiml-serving", + "--profile=aiml-checkpointing", + "--machine-type=low-end-machine --profile=aiml-training", + "--machine-type=low-end-machine --profile=aiml-serving", + "--machine-type=low-end-machine --profile=aiml-checkpointing", + } + cfg.FlagOptimizations[0].Configs[3].Compatible = map[string]bool{"flat": true, "hns": false, "zonal": false} + cfg.FlagOptimizations[0].Configs[3].RunOnGKE = true + cfg.FlagOptimizations[0].Configs[4].Run = "TestRenameDirLimitSet" + cfg.FlagOptimizations[0].Configs[4].Flags = []string{ + "--machine-type=a3-highgpu-8g", + "--profile=aiml-checkpointing", + "--machine-type=low-end-machine --profile=aiml-checkpointing", + } + cfg.FlagOptimizations[0].Configs[4].Compatible = map[string]bool{"flat": true, "hns": false, "zonal": false} + cfg.FlagOptimizations[0].Configs[4].RunOnGKE = true + + cfg.FlagOptimizations[0].Configs[5].Run = "TestZonalBucketOptimizations" + cfg.FlagOptimizations[0].Configs[5].Flags = []string{"--log-severity=trace"} + cfg.FlagOptimizations[0].Configs[5].Compatible = map[string]bool{"flat": false, "hns": false, "zonal": true} + cfg.FlagOptimizations[0].Configs[5].RunOnGKE = false + + cfg.FlagOptimizations[0].Configs[6].Run = "TestZonalBucketOptimizations_ExplicitOverrides" + cfg.FlagOptimizations[0].Configs[6].Flags = []string{"--implicit-dirs --max-read-ahead-kb=2048 --max-background=50 --congestion-threshold=30 --log-severity=trace"} + cfg.FlagOptimizations[0].Configs[6].Compatible = map[string]bool{"flat": false, "hns": false, "zonal": true} + cfg.FlagOptimizations[0].Configs[6].RunOnGKE = false + + cfg.FlagOptimizations[0].Configs[7].Run = "TestZonalBucketOptimizations_Dynamic" + cfg.FlagOptimizations[0].Configs[7].Flags = []string{"--log-severity=trace"} + cfg.FlagOptimizations[0].Configs[7].Compatible = map[string]bool{"flat": false, "hns": false, "zonal": true} + cfg.FlagOptimizations[0].Configs[7].RunOnGKE = false + + cfg.FlagOptimizations[0].Configs[8].Run = "TestKernelReader_DefaultAndPrecedence" + cfg.FlagOptimizations[0].Configs[8].Flags = []string{ + "--implicit-dirs --log-severity=trace", + "--implicit-dirs --log-severity=trace --file-cache-max-size-mb=-1 --cache-dir=/gcsfuse-tmp/TestKernelReader_DefaultAndPrecedence_FileCache", + "--implicit-dirs --log-severity=trace --enable-buffered-read=true", + "--implicit-dirs --log-severity=trace --enable-buffered-read=true --file-cache-max-size-mb=-1 --cache-dir=/gcsfuse-tmp/TestKernelReader_DefaultAndPrecedence_Both", + } + cfg.FlagOptimizations[0].Configs[8].Compatible = map[string]bool{"flat": false, "hns": false, "zonal": true} + cfg.FlagOptimizations[0].Configs[8].RunOnGKE = false + + cfg.FlagOptimizations[0].Configs[9].Run = "TestFileCache_KernelReaderDisabled" + cfg.FlagOptimizations[0].Configs[9].Flags = []string{"--implicit-dirs --log-severity=trace --enable-kernel-reader=false --file-cache-max-size-mb=-1 --cache-dir=/gcsfuse-tmp/TestFileCache_KernelReaderDisabled"} + cfg.FlagOptimizations[0].Configs[9].Compatible = map[string]bool{"flat": false, "hns": false, "zonal": true} + cfg.FlagOptimizations[0].Configs[9].RunOnGKE = false + + cfg.FlagOptimizations[0].Configs[10].Run = "TestBufferedReader_KernelReaderDisabled" + cfg.FlagOptimizations[0].Configs[10].Flags = []string{"--implicit-dirs --log-severity=trace --enable-kernel-reader=false --enable-buffered-read"} + cfg.FlagOptimizations[0].Configs[10].Compatible = map[string]bool{"flat": false, "hns": false, "zonal": true} + cfg.FlagOptimizations[0].Configs[10].RunOnGKE = false + + cfg.FlagOptimizations[0].Configs[11].Run = "TestKernelReader_Dynamic" + cfg.FlagOptimizations[0].Configs[11].Flags = []string{"--implicit-dirs --log-severity=trace"} + cfg.FlagOptimizations[0].Configs[11].Compatible = map[string]bool{"flat": false, "hns": false, "zonal": true} + cfg.FlagOptimizations[0].Configs[11].RunOnGKE = false + } + + testEnv.ctx = context.Background() + testEnv.bucketType = setup.TestEnvironment(testEnv.ctx, &cfg.FlagOptimizations[0]) + testEnv.cfg = cfg.FlagOptimizations[0] + closeStorageClient := client.CreateStorageClientWithCancel(&testEnv.ctx, &testEnv.storageClient) + defer func() { + err := closeStorageClient() + if err != nil { + log.Fatalf("closeStorageClient failed: %v", err) + } + }() + + setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() + + // To run mountedDirectory tests, we need both testBucket and mountedDirectory + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + os.Exit(setup.RunTestsForMountedDirectory(testEnv.cfg.GKEMountedDirectory, m)) + } + + // Run tests for testBucket + // Set up test directory. + setup.SetUpTestDirForTestBucket(&testEnv.cfg) + // Override GKE specific paths with GCSFuse paths if running in GCE environment. + setup.OverrideFilePathsInFlagSet(&testEnv.cfg, setup.TestDir()) + + // Save mount and root directory variables. + testEnv.mountDir, testEnv.rootDir = setup.MntDir(), setup.MntDir() + + log.Println("Running static mounting tests...") + testEnv.mountFunc = static_mounting.MountGcsfuseWithStaticMountingWithConfigFile + successCode := m.Run() + + if successCode == 0 { + log.Println("Running dynamic mounting tests...") + // Save mount directory variable to have path of bucket to run tests. + testEnv.mountDir = path.Join(setup.MntDir(), setup.TestBucket()) + testEnv.mountFunc = dynamic_mounting.MountGcsfuseWithDynamicMountingWithConfig + successCode = m.Run() + } + + if successCode == 0 { + log.Println("Running only dir mounting tests...") + setup.SetOnlyDirMounted(onlyDirMounted + "/") + testEnv.mountDir = testEnv.rootDir + testEnv.mountFunc = only_dir_mounting.MountGcsfuseWithOnlyDirWithConfigFile + successCode = m.Run() + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(setup.TestBucket(), setup.OnlyDirMounted(), testDirName)) + } + + // If failed, then save the gcsfuse log file(s). + setup.SaveLogFileInCaseOfFailure(successCode) + + // Clean up test directory created. + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(setup.TestBucket(), testDirName)) + os.Exit(successCode) +} diff --git a/tools/integration_tests/flag_optimizations/zonal_bucket_optimization_test.go b/tools/integration_tests/flag_optimizations/zonal_bucket_optimization_test.go new file mode 100644 index 0000000000..2d27258964 --- /dev/null +++ b/tools/integration_tests/flag_optimizations/zonal_bucket_optimization_test.go @@ -0,0 +1,322 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flag_optimizations + +import ( + "fmt" + "log" + "os" + "strings" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/kernelparams" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "golang.org/x/sys/unix" +) + +const ( + // kernelReaderInitMsg indicates the MRD pool initialization. + // MRD pool is used only by kernel reader. This confirms that the kernel reader is enabled and initializing. + kernelReaderInitMsg = "Initializing MRD Pool with size:" + + // fileCacheMsg indicates that the file cache is being used. + fileCacheMsg = "FileCache(" + + // bufferedReaderSchedMsg indicates the buffered reader is scheduling a block download. + bufferedReaderSchedMsg = "Scheduling block:" + + // readFileStartMsg indicates the start of a ReadFile operation. + readFileStartMsg = "<- ReadFile" + + // readFileEndMsg indicates the completion of a ReadFile operation. + readFileEndMsg = "-> ReadFile" +) + +//////////////////////////////////////////////////////////////////////// +// Helpers +//////////////////////////////////////////////////////////////////////// + +// verifyKernelParam checks if the kernel parameter at path matches expectedVal (if set) +// or optimizedVal (considering mount type). +func (s *KernelReaderParamsSuite) verifyKernelParam(path string, expectedVal string, optimizedVal string) { + s.T().Helper() + content, err := os.ReadFile(path) + require.NoError(s.T(), err) + val := strings.TrimSpace(string(content)) + + if expectedVal != "" { + assert.Equal(s.T(), expectedVal, val, "Param %s mismatch", path) + } else if setup.IsDynamicMount(testEnv.mountDir, testEnv.rootDir) { + assert.NotEqual(s.T(), optimizedVal, val, "Param %s should NOT match optimized default for dynamic mount", path) + } else { + assert.Equal(s.T(), optimizedVal, val, "Param %s mismatch with optimized default", path) + } +} + +// validateParallelReads parses the log content to verify that parallel reads occurred. +// It tracks the number of concurrent "ReadFile" operations and asserts that +// the maximum parallelism observed is greater than 1. +func (s *ReadStrategySuite) validateParallelReads(logContent string) { + s.T().Helper() + lines := strings.Split(logContent, "\n") + currentParallelism := 0 + maxParallelism := 0 + for _, line := range lines { + if strings.Contains(line, readFileStartMsg) { + currentParallelism++ + } + if strings.Contains(line, readFileEndMsg) { + currentParallelism-- + } + if currentParallelism > maxParallelism { + maxParallelism = currentParallelism + } + if maxParallelism >= 2 { + break + } + } + assert.Greater(s.T(), maxParallelism, 1, "Expected parallel reads (max parallelism > 1)") +} + +// createAndReadFile creates a 10MB file using O_DIRECT (avoiding write cache) +// and reads it back using os.ReadFile (triggering readahead). +func createAndReadFile(t *testing.T, testName string) { + t.Helper() + testName = strings.ReplaceAll(testName, "/", "_") + fileName := testEnv.testDirPath + "/" + testName + "_test_file.txt" + // Use operations.CreateFileOfSize which uses O_DIRECT to avoid polluting page cache during write. + operations.CreateFileOfSize(10*1024*1024, fileName, t) + t.Cleanup(func() { + if err := os.Remove(fileName); err != nil { + t.Logf("Failed to remove file %s: %v", fileName, err) + } + }) + require.NoError(t, os.Truncate(setup.LogFile(), 0), "Failed to truncate log file") + + // Read the file using os.ReadFile which uses page cache to trigger kernel readahead. + _, err := os.ReadFile(fileName) + + require.NoError(t, err, "Failed to read file") +} + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +// KernelParamsSuite tests the behavior of zonal bucket optimizations, +// specifically verifying kernel parameters. +type KernelReaderParamsSuite struct { + suite.Suite + flags []string + expectedReadAhead string + expectedMaxBackground string + expectedCongestionThreshold string +} + +func (s *KernelReaderParamsSuite) SetupSuite() { + mustMountGCSFuseAndSetupTestDir(s.flags, testEnv.ctx, testEnv.storageClient) +} + +func (s *KernelReaderParamsSuite) TearDownSuite() { + tearDownOptimizationTest(s.T()) +} + +// TestKernelParamVerification verifies the values of max_read_ahead_kb, +// max_background, and congestion_threshold for Zonal Buckets. +// For non dynamic ZB mounts, they should be updated to the optimized values +// (unless explicitly changed via config or CLI). +func (s *KernelReaderParamsSuite) TestKernelParamVerification() { + // Verify kernel parameters in /sys + var stat unix.Stat_t + err := unix.Stat(setup.MntDir(), &stat) + require.NoError(s.T(), err) + devMajor := unix.Major(stat.Dev) + devMinor := unix.Minor(stat.Dev) + readAheadPath, err := kernelparams.PathForParam(kernelparams.MaxReadAheadKb, devMajor, devMinor) + require.NoError(s.T(), err) + maxBackgroundPath, err := kernelparams.PathForParam(kernelparams.MaxBackgroundRequests, devMajor, devMinor) + require.NoError(s.T(), err) + congestionThresholdPath, err := kernelparams.PathForParam(kernelparams.CongestionWindowThreshold, devMajor, devMinor) + require.NoError(s.T(), err) + + optimizedReadAhead := "16384" + optimizedMaxBackground := fmt.Sprintf("%d", cfg.DefaultMaxBackground()) + optimizedCongestion := fmt.Sprintf("%d", cfg.DefaultCongestionThreshold()) + + s.verifyKernelParam(readAheadPath, s.expectedReadAhead, optimizedReadAhead) + s.verifyKernelParam(maxBackgroundPath, s.expectedMaxBackground, optimizedMaxBackground) + s.verifyKernelParam(congestionThresholdPath, s.expectedCongestionThreshold, optimizedCongestion) +} + +func TestZonalBucketOptimizations(t *testing.T) { + if setup.IsDynamicMount(testEnv.mountDir, testEnv.rootDir) { + t.Skip("Skipping test for dynamic mounting") + } + flagsSet := setup.BuildFlagSets(testEnv.cfg, testEnv.bucketType, t.Name()) + for _, flags := range flagsSet { + t.Run("", func(t *testing.T) { + log.Printf("Running tests with flags: %s", flags) + s := &KernelReaderParamsSuite{ + flags: flags, + } + suite.Run(t, s) + }) + } +} + +func TestZonalBucketOptimizations_ExplicitOverrides(t *testing.T) { + if setup.IsDynamicMount(testEnv.mountDir, testEnv.rootDir) { + t.Skip("Skipping test for dynamic mounting") + } + flagsSet := setup.BuildFlagSets(testEnv.cfg, testEnv.bucketType, t.Name()) + for _, flags := range flagsSet { + t.Run("", func(t *testing.T) { + log.Printf("Running tests with flags: %s", flags) + s := &KernelReaderParamsSuite{ + flags: flags, + expectedReadAhead: "2048", + expectedMaxBackground: "50", + expectedCongestionThreshold: "30", + } + suite.Run(t, s) + }) + } +} + +func TestZonalBucketOptimizations_Dynamic(t *testing.T) { + if !setup.IsDynamicMount(testEnv.mountDir, testEnv.rootDir) { + t.Skip("Skipping test for non dynamic mounting") + } + flags := []string{"--log-severity=trace"} + log.Printf("Running tests with flags: %s", flags) + s := &KernelReaderParamsSuite{ + flags: flags, + } + suite.Run(t, s) +} + +// ReadStrategySuite tests the behavior of the kernel reader under different configurations, +// verifying log output and read parallelism. +type ReadStrategySuite struct { + suite.Suite + flags []string + expectedLog string + unexpectedLog string + validateParallelism bool +} + +func (s *ReadStrategySuite) SetupSuite() { + err := mountGCSFuseAndSetupTestDir(s.flags, testEnv.ctx, testEnv.storageClient) + require.NoError(s.T(), err) +} + +func (s *ReadStrategySuite) TearDownSuite() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) + setup.UnmountGCSFuseAndDeleteLogFile(testEnv.rootDir) +} + +// TestKernelReaderBehavior verifies the read strategy behavior based on flags. +// Specifically for Zonal Buckets, it checks that if not explicitly disabled, +// Kernel Reader is used (taking precedence) even if Buffered Read or File Cache are enabled. +// It uses log assertions and optionally validates parallel reads. +func (s *ReadStrategySuite) TestKernelReaderBehavior() { + createAndReadFile(s.T(), s.T().Name()) + + logContent, err := os.ReadFile(setup.LogFile()) + + require.NoError(s.T(), err, "Failed to read log file") + if s.expectedLog != "" { + assert.Contains(s.T(), string(logContent), s.expectedLog, "Expected log '%s' not found in logs", s.expectedLog) + } + if s.unexpectedLog != "" { + assert.NotContains(s.T(), string(logContent), s.unexpectedLog, "Unexpected log '%s' found in logs", s.unexpectedLog) + } + if s.validateParallelism { + s.validateParallelReads(string(logContent)) + } +} + +func TestKernelReader(t *testing.T) { + if setup.IsDynamicMount(testEnv.mountDir, testEnv.rootDir) { + t.Skip("Skipping test for dynamic mounting") + } + testCases := []struct { + testName string + expectedLog string + unexpectedLog string + validateParallelism bool + }{ + // Tests that kernel reader is used by default and takes precedence over buffered reader and file cache. + { + testName: "TestKernelReader_IsDefaultAndTakesPrecedence", + expectedLog: kernelReaderInitMsg, + validateParallelism: true, + }, + // Tests that file cache is used when kernel reader is explicitly disabled. + { + testName: "TestFileCache_KernelReaderDisabled", + expectedLog: fileCacheMsg, + unexpectedLog: kernelReaderInitMsg, + validateParallelism: false, + }, + // Tests that buffered reader is used when kernel reader is explicitly disabled. + { + testName: "TestBufferedReader_KernelReaderDisabled", + expectedLog: bufferedReaderSchedMsg, + unexpectedLog: kernelReaderInitMsg, + validateParallelism: false, + }, + } + + for _, tc := range testCases { + flagsSet := setup.BuildFlagSets(testEnv.cfg, testEnv.bucketType, tc.testName) + for _, flags := range flagsSet { + t.Run(tc.testName, func(t *testing.T) { + log.Printf("Running tests with flags: %s", flags) + s := &ReadStrategySuite{ + flags: flags, + expectedLog: tc.expectedLog, + unexpectedLog: tc.unexpectedLog, + validateParallelism: tc.validateParallelism, + } + suite.Run(t, s) + }) + } + } +} + +func TestKernelReader_Dynamic(t *testing.T) { + if !setup.IsDynamicMount(testEnv.mountDir, testEnv.rootDir) { + t.Skip("Skipping test for non dynamic mounting") + } + configName := "TestKernelReader_Dynamic" + flagsSet := setup.BuildFlagSets(testEnv.cfg, testEnv.bucketType, configName) + for _, flags := range flagsSet { + t.Run(configName, func(t *testing.T) { + log.Printf("Running tests with flags: %s", flags) + s := &ReadStrategySuite{ + flags: flags, + unexpectedLog: kernelReaderInitMsg, + validateParallelism: false, + } + suite.Run(t, s) + }) + } +} diff --git a/tools/integration_tests/grpc_validation/grpc_validation_test.go b/tools/integration_tests/grpc_validation/grpc_validation_test.go new file mode 100644 index 0000000000..a40ce727c1 --- /dev/null +++ b/tools/integration_tests/grpc_validation/grpc_validation_test.go @@ -0,0 +1,162 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package grpc_validation + +import ( + "fmt" + "os" + "testing" + "time" + + client_util "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type gRPCValidation struct { + suite.Suite + singleRegionBucketForGRPCSuccess string + singleRegionBucketForGRPCFailure string + multiRegionBucketForGRPCSuccess string + multiRegionBucketForGRPCFailure string +} + +// Setup involves: +// Finding out test regions. +// Creating unique test bucket names. +// Creating the test buckets. +// Test Cases: +// Test Case 1: single region test: success case , when test VM and bucket are colocated +// Test Case 2: single region test: failure case , when test VM and bucket are out-of-region +// Test Case 3: multi region test: success case , when test VM and bucket are colocated +// Test Case 4: multi region test: failure case , when test VM and bucket are out-of-region +func (g *gRPCValidation) SetupSuite() { + + // Based on the test region which is initialized in the TestMain() function, + // we will pick out the test region for: + singleRegionForGRPCSuccess := findSingleRegionForGRPCDirectPathSuccessCase(testRegion) + singleRegionForGRPCFailure := pickFailureRegionFromListOfRegions(singleRegionForGRPCSuccess, singleRegions) + multiRegionForGRPCSuccess := findMultiRegionForGRPCDirectPathSuccessCase(testRegion) + multiRegionForGRPCFailure := pickFailureRegionFromListOfRegions(multiRegionForGRPCSuccess, multiRegions) + + // Set the test bucket names with unique suffix + g.singleRegionBucketForGRPCSuccess = createTestBucketName(singleRegionForGRPCSuccess) + g.singleRegionBucketForGRPCFailure = createTestBucketName(singleRegionForGRPCFailure) + g.multiRegionBucketForGRPCSuccess = createTestBucketName(multiRegionForGRPCSuccess) + g.multiRegionBucketForGRPCFailure = createTestBucketName(multiRegionForGRPCFailure) + + // Create the test buckets + if err := createTestBucket(singleRegionForGRPCSuccess, g.singleRegionBucketForGRPCSuccess); err != nil { + g.T().Fatalf("Could not create bucket in the required region, err : %v", err) + } + if err := createTestBucket(singleRegionForGRPCFailure, g.singleRegionBucketForGRPCFailure); err != nil { + g.T().Fatalf("Could not create bucket in the required region, err : %v", err) + } + if err := createTestBucket(multiRegionForGRPCSuccess, g.multiRegionBucketForGRPCSuccess); err != nil { + g.T().Fatalf("Could not create bucket in the required region, err : %v", err) + } + if err := createTestBucket(multiRegionForGRPCFailure, g.multiRegionBucketForGRPCFailure); err != nil { + g.T().Fatalf("Could not create bucket in the required region, err : %v", err) + } +} + +// Delete the test buckets created. +func (g *gRPCValidation) TearDownSuite() { + bucketsToDelete := []string{ + g.singleRegionBucketForGRPCSuccess, + g.singleRegionBucketForGRPCFailure, + g.multiRegionBucketForGRPCSuccess, + g.multiRegionBucketForGRPCFailure, + } + for _, bucket := range bucketsToDelete { + if err := client_util.DeleteBucket(ctx, client, bucket); err != nil { + g.T().Logf("Failed to delete bucket %s: %v", bucket, err) + } + } +} + +func TestGRPCValidationSuite(t *testing.T) { + suite.Run(t, new(gRPCValidation)) +} + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func (g *gRPCValidation) TestGRPCDirectPathConnections() { + testCases := []struct { + name string + bucketName string + expectedSuccess bool + expectedLogSubstring string + }{ + { + name: "SingleRegion_Success", + bucketName: g.singleRegionBucketForGRPCSuccess, + expectedLogSubstring: fmt.Sprintf("Successfully connected over gRPC DirectPath for %s", g.singleRegionBucketForGRPCSuccess), + }, + { + name: "SingleRegion_Failure", + bucketName: g.singleRegionBucketForGRPCFailure, + expectedLogSubstring: fmt.Sprintf("Direct path connectivity unavailable for %s, reason:", g.singleRegionBucketForGRPCFailure), + }, + { + name: "MultiRegion_Success", + bucketName: g.multiRegionBucketForGRPCSuccess, + expectedLogSubstring: fmt.Sprintf("Successfully connected over gRPC DirectPath for %s", g.multiRegionBucketForGRPCSuccess), + }, + { + name: "MultiRegion_Failure", + bucketName: g.multiRegionBucketForGRPCFailure, + expectedLogSubstring: fmt.Sprintf("Direct path connectivity unavailable for %s, reason: ", g.multiRegionBucketForGRPCFailure), + }, + } + + for _, tc := range testCases { + g.T().Run(tc.name, func(t *testing.T) { + mountPoint, err := os.MkdirTemp("", "grpc_validation_test") + assert.NoError(t, err) + logFile := fmt.Sprintf("/tmp/grpc_%s_%d.txt", tc.name, time.Now().UnixNano()) + args := []string{"--client-protocol=grpc", "--log-severity=TRACE", fmt.Sprintf("--log-file=%s", logFile), tc.bucketName, mountPoint} + err = mounting.MountGcsfuse(setup.BinFile(), args) + if err != nil { + if tc.expectedSuccess { + t.Errorf("Unexpected mount failure: %v", err) + } + return + } + + defer func() { + if err := util.Unmount(mountPoint); err != nil { + t.Logf("Warning: unmount failed: %v", err) + } + os.Remove(mountPoint) + // Only remove the log file if the test succeeded + if t.Failed() { + t.Logf("Test failed, log file '%s' will not be deleted for inspection.", logFile) + } else { + os.Remove(logFile) + } + }() + success := operations.CheckLogFileForMessage(g.T(), tc.expectedLogSubstring, logFile) + require.Equal(t, true, success) + }) + } +} diff --git a/tools/integration_tests/grpc_validation/setup_test.go b/tools/integration_tests/grpc_validation/setup_test.go new file mode 100644 index 0000000000..5d1c4f4c2a --- /dev/null +++ b/tools/integration_tests/grpc_validation/setup_test.go @@ -0,0 +1,170 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package grpc_validation + +import ( + "context" + "fmt" + "log" + "os" + "strings" + "testing" + "time" + + "cloud.google.com/go/storage" + client_util "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "go.opentelemetry.io/contrib/detectors/gcp" + "go.opentelemetry.io/otel/sdk/resource" +) + +// gRPC directPath can be established in a scenario like VM in us-central1-a and +// the bucket in us-central1. Since we are creating the buckets dynamically, we need +// to select regions and not zones. +// Note for single region bucket testing, +// We can work with 2 regions only since for testing success case, we are actually finding out +// the test region using go library. Just for failure case, we need to find out a region which +// is bound to be one of these two since continent differs. +// Example case: Test Region : us-central1 , Failure Test Region : europe-west4 +// +// Test Region : us-west1, Failure test Region : us-central1 +var singleRegions = []string{ + "us-central1", + "europe-west4", +} + +// gRPC is now supported for multi region buckets as well. +// Same logic, if the test VM is in us, then failure region = eu +// If the test VM is in non-US, then failure region = us +var multiRegions = []string{ + "us", + "eu", +} +var gcpProject = "gcs-fuse-test" +var ( + ctx context.Context + client *storage.Client + testRegion string +) + +// Since gRPC directpath does not work over cloudtop, so these validation tests will be skipped +// when run on cloudtop. +var cloudtopProd = "cloudtop-prod" + +//////////////////////////////////////////////////////////////////////// +// Helper Functions +//////////////////////////////////////////////////////////////////////// + +// For both multi region buckets and single region buckets test, we need to decide failure case +// region based on the region of the VM on which the test is running. +// Reference for the GCP resource attribute: https://opentelemetry.io/docs/specs/semconv/attributes-registry/cloud/#cloud-availability-zone +func findTestExecutionEnvironment(ctx context.Context) (string, error) { + detectedAttrs, err := resource.New(ctx, resource.WithDetectors(gcp.NewDetector())) + if err != nil { + log.Printf("Error fetching the test environment.All tests will be skipped. Error : %v", err) + return "", err + } + attrs := detectedAttrs.Set() + if v, exists := attrs.Value("gcp.gce.instance.hostname"); exists && strings.Contains(strings.ToLower(v.AsString()), cloudtopProd) { + return cloudtopProd, nil + } + if v, exists := attrs.Value("cloud.availability_zone"); exists { + return v.AsString(), nil + } + return "", nil +} + +// For testing with single region buckets, we need to find the region for success case. +// If the input is 'us-west1-a' which is the test VM Zone, then this function returns 'us-west1' used +// creating the test buckets. +func findSingleRegionForGRPCDirectPathSuccessCase(testRegion string) string { + parts := strings.Split(testRegion, "-") + if len(parts) >= 2 { + return strings.Join(parts[:len(parts)-1], "-") //rejoin the first parts of the zone, excluding the last part. + } + return "" +} + +// For testing with multi region buckets, we need to find the region for success case. +// If the input is 'us-west1-a' which is the test VM Zone, then this function returns 'us' used +// creating the test buckets. +func findMultiRegionForGRPCDirectPathSuccessCase(testRegion string) string { + parts := strings.Split(testRegion, "-") + if len(parts) > 0 { + return parts[0] // Return the first part of the string i.e. if us-central1 then us + } + return "" +} + +// Generic function to pick a region other than the passed value to validate for failure scenario. +func pickFailureRegionFromListOfRegions(successRegion string, regions []string) string { + for _, otherRegion := range regions { + if otherRegion != successRegion { + return otherRegion + } + } + return "" +} + +// Creating test bucket name with unique suffix. +func createTestBucketName(region string) string { + epoch := time.Now().UnixNano() // Get the current Unix epoch time + return fmt.Sprintf("grpc_validation_%s_%d", region, epoch) +} + +// Based on the test case, we need to create the bucket in/out of region with the test VM. +// This has to be done dynamically at the time of test setup. +// Based on what region we pass, the test bucket will be multi region or single region. +func createTestBucket(testBucketRegion, testBucketName string) (err error) { + bucket := client.Bucket(testBucketName) + if err = bucket.Create(ctx, gcpProject, &storage.BucketAttrs{Location: testBucketRegion}); err != nil { + log.Printf("Error while creating bucket, error: %v", err) + return err + } + return nil +} + +func TestMain(m *testing.M) { + // Parse flags from the setup. + var err error + setup.ParseSetUpFlags() + if setup.IsPresubmitRun() { + log.Println("Skipping test package : grpc_validation since this is a presubmit test run") + os.Exit(0) + } + setup.SetUpTestDirForTestBucketFlag() + + // Creating a common storage client for the test + ctx = context.Background() + if client, err = client_util.CreateStorageClient(ctx); err != nil { + log.Fatalf("Creation of storage client failed with error : %v", err) + } + defer client.Close() + + testRegion, err = findTestExecutionEnvironment(ctx) + if err != nil { + log.Fatalf("Failed to retrieve test VM region: %v", err) + } + + if testRegion == cloudtopProd { + log.Println("Skipping tests due to cloudtop environment.") + os.Exit(0) + } + + // Run tests. + code := m.Run() + // Exit. + os.Exit(code) +} diff --git a/tools/integration_tests/gzip/gzip_test.go b/tools/integration_tests/gzip/gzip_test.go index 93e13e2ec1..a217256f3b 100644 --- a/tools/integration_tests/gzip/gzip_test.go +++ b/tools/integration_tests/gzip/gzip_test.go @@ -25,10 +25,11 @@ import ( "testing" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" ) const ( @@ -60,7 +61,7 @@ var ( ctx context.Context ) -func setup_testdata(m *testing.M) error { +func setup_testdata() error { fmds := []struct { filename string filesize int @@ -168,7 +169,7 @@ func setup_testdata(m *testing.M) error { return nil } -func destroy_testdata(m *testing.M, storageClient *storage.Client) error { +func destroy_testdata(storageClient *storage.Client) error { for _, gcsObjectPath := range gcsObjectsToBeDeletedEventually { err := client.DeleteObjectOnGCS(ctx, storageClient, gcsObjectPath) if err != nil { @@ -193,50 +194,58 @@ func createContentOfSize(contentSize int) (string, error) { func TestMain(m *testing.M) { setup.ParseSetUpFlags() - var err error - ctx = context.Background() - if storageClient, err = client.CreateStorageClient(ctx); err != nil { - log.Fatalf("Error creating storage client: %v\n", err) + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.Gzip) == 0 { + log.Println("No configuration found for gzip tests in config. Using flags instead.") + // Populate the config manually. + cfg.Gzip = make([]test_suite.TestConfig, 1) + cfg.Gzip[0].TestBucket = setup.TestBucket() + cfg.Gzip[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.Gzip[0].Configs = make([]test_suite.ConfigItem, 1) + cfg.Gzip[0].Configs[0].Flags = []string{ + "--sequential-read-size-mb=1 --implicit-dirs", + "--sequential-read-size-mb=1 --implicit-dirs --client-protocol=grpc", + } + cfg.Gzip[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} } + + // 2. Create storage client before running tests. + ctx = context.Background() + bucketType := setup.TestEnvironment(ctx, &cfg.Gzip[0]) + closeStorageClient := client.CreateStorageClientWithCancel(&ctx, &storageClient) defer func() { - if err := storageClient.Close(); err != nil { - log.Printf("failed to close storage client: %v", err) + err := closeStorageClient() + if err != nil { + log.Fatalf("closeStorageClient failed: %v", err) } }() - commonFlags := []string{"--sequential-read-size-mb=" + fmt.Sprint(SeqReadSizeMb), "--implicit-dirs"} - flags := [][]string{commonFlags} - - if !testing.Short() { - gRPCFlags := append(commonFlags, "--client-protocol=grpc") - flags = append(flags, gRPCFlags) - } - - setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() - - if setup.TestBucket() == "" && setup.MountedDirectory() != "" { - log.Fatal("Please pass the name of bucket mounted at mountedDirectory to --testBucket flag.") - } - - err = setup_testdata(m) + err := setup_testdata() if err != nil { log.Fatalf("Failed to setup test data: %v", err) } defer func() { - err := destroy_testdata(m, storageClient) + err := destroy_testdata(storageClient) if err != nil { log.Printf("Failed to destoy gzip test data: %v", err) } }() - // Run tests for mountedDirectory only if --mountedDirectory flag is set. - setup.RunTestsForMountedDirectoryFlag(m) + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + // flags to be set, as Gzip tests validates content from the bucket. + if cfg.Gzip[0].GKEMountedDirectory != "" && cfg.Gzip[0].TestBucket != "" { + os.Exit(setup.RunTestsForMountedDirectory(cfg.Gzip[0].GKEMountedDirectory, m)) + } + + // Run tests for testBucket. + // 4. Build the flag sets dynamically from the config. + flags := setup.BuildFlagSets(cfg.Gzip[0], bucketType, "") - // Run tests for testBucket - setup.SetUpTestDirForTestBucketFlag() + setup.SetUpTestDirForTestBucket(&cfg.Gzip[0]) - successCode := static_mounting.RunTests(flags, m) + successCode := static_mounting.RunTestsWithConfigFile(&cfg.Gzip[0], flags, m) os.Exit(successCode) } diff --git a/tools/integration_tests/gzip/read_gzip_test.go b/tools/integration_tests/gzip/read_gzip_test.go index fee1a9cf71..a43bfdeab8 100644 --- a/tools/integration_tests/gzip/read_gzip_test.go +++ b/tools/integration_tests/gzip/read_gzip_test.go @@ -24,9 +24,9 @@ import ( "path" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) // Verify that the passed file exists on the GCS test-bucket and in the mounted bucket @@ -52,7 +52,7 @@ func verifyFileSizeAndFullFileRead(t *testing.T, filename string) { mountedFilePath, (*fi).Size(), gcsObjectPath, gcsObjectSize) } - localCopy, err := downloadGzipGcsObjectAsCompressed(t, setup.TestBucket(), path.Join(TestBucketPrefixPath, filename)) + localCopy, err := downloadGzipGcsObjectAsCompressed(t, path.Join(TestBucketPrefixPath, filename)) if err != nil { t.Fatalf("failed to download gcs object (gs:/%s) to local-disk: %v", gcsObjectPath, err) } @@ -83,7 +83,7 @@ func verifyRangedRead(t *testing.T, filename string) { t.Fatalf("Failed to open local mounted file %s: %v", mountedFilePath, err) } - localCopy, err := downloadGzipGcsObjectAsCompressed(t, setup.TestBucket(), path.Join(TestBucketPrefixPath, filename)) + localCopy, err := downloadGzipGcsObjectAsCompressed(t, path.Join(TestBucketPrefixPath, filename)) if err != nil { t.Fatalf("failed to download gcs object (gs:/%s) to local-disk: %v", gcsObjectPath, err) } @@ -117,7 +117,7 @@ func verifyRangedRead(t *testing.T, filename string) { // Uses go storage client library to download object. Use of gsutil/gcloud is not // possible as they both always read back objects with content-encoding: gzip as // uncompressed/decompressed irrespective of any argument passed. -func downloadGzipGcsObjectAsCompressed(t *testing.T, bucketName, objPathInBucket string) (tempfile string, err error) { +func downloadGzipGcsObjectAsCompressed(t *testing.T, objPathInBucket string) (tempfile string, err error) { gcsObjectSize, err := client.GetGcsObjectSize(ctx, storageClient, objPathInBucket) if err != nil { diff --git a/tools/integration_tests/gzip/write_gzip_test.go b/tools/integration_tests/gzip/write_gzip_test.go index 8c14ad7e5c..14362d1fbc 100644 --- a/tools/integration_tests/gzip/write_gzip_test.go +++ b/tools/integration_tests/gzip/write_gzip_test.go @@ -19,9 +19,9 @@ import ( "path" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) // Size of the overwritten content created in bytes. @@ -45,7 +45,7 @@ func verifyFullFileOverwrite(t *testing.T, filename string) { t.Fatalf("Failed to get stat info of mounted file %s: %v\n", mountedFilePath, err) } - if (*fi).Size() != int64(gcsObjectSize) { + if (*fi).Size() != gcsObjectSize { t.Fatalf("Size of file mounted through gcsfuse (%s, %d) doesn't match the size of the file on GCS (%s, %d)", mountedFilePath, (*fi).Size(), gcsObjectPath, gcsObjectSize) } diff --git a/tools/integration_tests/implicit_dir/delete_test.go b/tools/integration_tests/implicit_dir/delete_test.go index d403fdfedf..43816a46b1 100644 --- a/tools/integration_tests/implicit_dir/delete_test.go +++ b/tools/integration_tests/implicit_dir/delete_test.go @@ -19,8 +19,9 @@ import ( "path" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup" ) // Directory Structure @@ -31,7 +32,12 @@ import ( func TestDeleteNonEmptyImplicitDir(t *testing.T) { testDirName := "testDeleteNonEmptyImplicitDir" testDirPath := setupTestDir(testDirName) - implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName)) + // TODO: Remove the condition and keep the storage-client flow for non-ZB too. + if setup.IsZonalBucketRun() { + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructureUsingStorageClient(testEnv.ctx, t, testEnv.storageClient, path.Join(DirForImplicitDirTests, testDirName)) + } else { + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName)) + } dirPath := path.Join(testDirPath, implicit_and_explicit_dir_setup.ImplicitDirectory) @@ -46,7 +52,12 @@ func TestDeleteNonEmptyImplicitDir(t *testing.T) { func TestDeleteNonEmptyImplicitSubDir(t *testing.T) { testDirName := "testDeleteNonEmptyImplicitSubDir" testDirPath := setupTestDir(testDirName) - implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName)) + // TODO: Remove the condition and keep the storage-client flow for non-ZB too. + if setup.IsZonalBucketRun() { + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructureUsingStorageClient(testEnv.ctx, t, testEnv.storageClient, path.Join(DirForImplicitDirTests, testDirName)) + } else { + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName)) + } subDirPath := path.Join(testDirPath, implicit_and_explicit_dir_setup.ImplicitDirectory, implicit_and_explicit_dir_setup.ImplicitSubDirectory) @@ -63,7 +74,12 @@ func TestDeleteNonEmptyImplicitSubDir(t *testing.T) { func TestDeleteImplicitDirWithExplicitSubDir(t *testing.T) { testDirName := "testDeleteImplicitDirWithExplicitSubDir" testDirPath := setupTestDir(testDirName) - implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName)) + // TODO: Remove the condition and keep the storage-client flow for non-ZB too. + if setup.IsZonalBucketRun() { + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructureUsingStorageClient(testEnv.ctx, t, testEnv.storageClient, path.Join(DirForImplicitDirTests, testDirName)) + } else { + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName)) + } explicitDirPath := path.Join(testDirPath, implicit_and_explicit_dir_setup.ImplicitDirectory, ExplicitDirInImplicitDir) @@ -84,7 +100,12 @@ func TestDeleteImplicitDirWithExplicitSubDir(t *testing.T) { func TestDeleteImplicitDirWithImplicitSubDirContainingExplicitDir(t *testing.T) { testDirName := "testDeleteImplicitDirWithImplicitSubDirContainingExplicitDir" testDirPath := setupTestDir(testDirName) - implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName)) + // TODO: Remove the condition and keep the storage-client flow for non-ZB too. + if setup.IsZonalBucketRun() { + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructureUsingStorageClient(testEnv.ctx, t, testEnv.storageClient, path.Join(DirForImplicitDirTests, testDirName)) + } else { + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName)) + } explicitDirPath := path.Join(testDirPath, implicit_and_explicit_dir_setup.ImplicitDirectory, implicit_and_explicit_dir_setup.ImplicitSubDirectory, ExplicitDirInImplicitSubDir) operations.CreateDirectoryWithNFiles(NumberOfFilesInExplicitDirInImplicitSubDir, explicitDirPath, PrefixFileInExplicitDirInImplicitSubDir, t) @@ -106,7 +127,12 @@ func TestDeleteImplicitDirWithImplicitSubDirContainingExplicitDir(t *testing.T) func TestDeleteImplicitDirInExplicitDir(t *testing.T) { testDirName := "testDeleteImplicitDirInExplicitDir" testDirPath := setupTestDir(testDirName) - implicit_and_explicit_dir_setup.CreateImplicitDirectoryInExplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName), t) + // TODO: Remove the condition and keep the storage-client flow for non-ZB too. + if setup.IsZonalBucketRun() { + implicit_and_explicit_dir_setup.CreateImplicitDirectoryInExplicitDirectoryStructureUsingStorageClient(testEnv.ctx, t, testEnv.storageClient, path.Join(DirForImplicitDirTests, testDirName)) + } else { + implicit_and_explicit_dir_setup.CreateImplicitDirectoryInExplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName), t) + } dirPath := path.Join(testDirPath, implicit_and_explicit_dir_setup.ExplicitDirectory, implicit_and_explicit_dir_setup.ImplicitDirectory) @@ -125,7 +151,12 @@ func TestDeleteImplicitDirInExplicitDir(t *testing.T) { func TestDeleteExplicitDirContainingImplicitSubDir(t *testing.T) { testDirName := "testDeleteExplicitDirContainingImplicitSubDir" testDirPath := setupTestDir(testDirName) - implicit_and_explicit_dir_setup.CreateImplicitDirectoryInExplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName), t) + // TODO: Remove the condition and keep the storage-client flow for non-ZB too. + if setup.IsZonalBucketRun() { + implicit_and_explicit_dir_setup.CreateImplicitDirectoryInExplicitDirectoryStructureUsingStorageClient(testEnv.ctx, t, testEnv.storageClient, path.Join(DirForImplicitDirTests, testDirName)) + } else { + implicit_and_explicit_dir_setup.CreateImplicitDirectoryInExplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName), t) + } dirPath := path.Join(testDirPath, implicit_and_explicit_dir_setup.ExplicitDirectory) diff --git a/tools/integration_tests/implicit_dir/implicit_dir_test.go b/tools/integration_tests/implicit_dir/implicit_dir_test.go index da6490f851..d7002af1e4 100644 --- a/tools/integration_tests/implicit_dir/implicit_dir_test.go +++ b/tools/integration_tests/implicit_dir/implicit_dir_test.go @@ -23,9 +23,10 @@ import ( "testing" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" ) const ExplicitDirInImplicitDir = "explicitDirInImplicitDir" @@ -36,10 +37,17 @@ const NumberOfFilesInExplicitDirInImplicitSubDir = 1 const NumberOfFilesInExplicitDirInImplicitDir = 1 const DirForImplicitDirTests = "dirForImplicitDirTests" -var ( +// IMPORTANT: To prevent global variable pollution, enhance code clarity, +// and avoid inadvertent errors. We strongly suggest that, all new package-level +// variables (which would otherwise be declared with `var` at the package root) should +// be added as fields to this 'env' struct instead. +type env struct { storageClient *storage.Client ctx context.Context -) + testDirPath string +} + +var testEnv env func setupTestDir(dirName string) string { dir := setup.SetupTestDirectory(DirForImplicitDirTests) @@ -51,12 +59,29 @@ func setupTestDir(dirName string) string { return dirPath } + func TestMain(m *testing.M) { setup.ParseSetUpFlags() - // Create storage client before running tests. - ctx = context.Background() - closeStorageClient := client.CreateStorageClientWithCancel(&ctx, &storageClient) + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.ImplicitDir) == 0 { + log.Println("No configuration found for implicit_dir tests in config. Using flags instead.") + // Populate the config manually. + cfg.ImplicitDir = make([]test_suite.TestConfig, 1) + cfg.ImplicitDir[0].TestBucket = setup.TestBucket() + cfg.ImplicitDir[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.ImplicitDir[0].Configs = make([]test_suite.ConfigItem, 2) + cfg.ImplicitDir[0].Configs[0].Flags = []string{"--implicit-dirs"} + cfg.ImplicitDir[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ImplicitDir[0].Configs[1].Flags = []string{"--implicit-dirs --client-protocol=grpc"} + cfg.ImplicitDir[0].Configs[1].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": false} + } + + // 2. Create storage client before running tests. + testEnv.ctx = context.Background() + bucketType := setup.TestEnvironment(testEnv.ctx, &cfg.ImplicitDir[0]) + closeStorageClient := client.CreateStorageClientWithCancel(&testEnv.ctx, &testEnv.storageClient) defer func() { err := closeStorageClient() if err != nil { @@ -64,19 +89,14 @@ func TestMain(m *testing.M) { } }() - flagsSet := [][]string{{"--implicit-dirs"}} - - if hnsFlagSet, err := setup.AddHNSFlagForHierarchicalBucket(ctx, storageClient); err == nil { - flagsSet = append(flagsSet, hnsFlagSet) - } - - if !testing.Short() { - flagsSet = append(flagsSet, []string{"--client-protocol=grpc", "--implicit-dirs=true"}) - } + // 3. Build the flag sets dynamically from the config. + flags := setup.BuildFlagSets(cfg.ImplicitDir[0], bucketType, "") - successCode := implicit_and_explicit_dir_setup.RunTestsForImplicitDirAndExplicitDir(flagsSet, m) + // 4. Run tests with the dynamically generated flags. + successCode := implicit_and_explicit_dir_setup.RunTestsForExplicitAndImplicitDir(&cfg.ImplicitDir[0], flags, m) + setup.SaveLogFileInCaseOfFailure(successCode) - // Clean up test directory created. - setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(setup.TestBucket(), testDirName)) + // 5. Clean up test directory created. + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(setup.TestBucket(), testDirName)) os.Exit(successCode) } diff --git a/tools/integration_tests/implicit_dir/list_test.go b/tools/integration_tests/implicit_dir/list_test.go index a9addd94fe..5168870087 100644 --- a/tools/integration_tests/implicit_dir/list_test.go +++ b/tools/integration_tests/implicit_dir/list_test.go @@ -23,7 +23,9 @@ import ( "path/filepath" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup" + "github.com/stretchr/testify/assert" ) func TestListImplicitObjectsFromBucket(t *testing.T) { @@ -39,7 +41,12 @@ func TestListImplicitObjectsFromBucket(t *testing.T) { // testBucket/dirForImplicitDirTests/testDir/explicitDirectory/fileInExplicitDir1 -- File // testBucket/dirForImplicitDirTests/testDir/explicitDirectory/fileInExplicitDir2 -- File - implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName)) + // TODO: Remove the condition and keep the storage-client flow for non-ZB too. + if setup.IsZonalBucketRun() { + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructureUsingStorageClient(testEnv.ctx, t, testEnv.storageClient, path.Join(DirForImplicitDirTests, testDirName)) + } else { + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName)) + } implicit_and_explicit_dir_setup.CreateExplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName), t) err := filepath.WalkDir(testDirPath, func(path string, dir fs.DirEntry, err error) error { @@ -137,3 +144,26 @@ func TestListImplicitObjectsFromBucket(t *testing.T) { return } } + +func TestStatImplicitDirAfterList(t *testing.T) { + testDirPath := setup.SetupTestDirectory(DirForImplicitDirTests) + // TODO: Remove the condition and keep the storage-client flow for non-ZB too. + if setup.IsZonalBucketRun() { + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructureUsingStorageClient(testEnv.ctx, t, testEnv.storageClient, DirForImplicitDirTests) + } else { + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(DirForImplicitDirTests) + } + + // List the directory + _, err := os.ReadDir(testDirPath) + if err != nil { + t.Fatalf("ReadDir failed: %v", err) + } + + // Stat the implicit directory + implicitDirPath := path.Join(testDirPath, implicit_and_explicit_dir_setup.ImplicitDirectory) + f, err := os.Stat(implicitDirPath) + if assert.NoError(t, err) { + assert.True(t, f.IsDir()) + } +} diff --git a/tools/integration_tests/implicit_dir/local_file_test.go b/tools/integration_tests/implicit_dir/local_file_test.go index 728221fad0..119597b8d6 100644 --- a/tools/integration_tests/implicit_dir/local_file_test.go +++ b/tools/integration_tests/implicit_dir/local_file_test.go @@ -19,46 +19,57 @@ import ( "path/filepath" "testing" - . "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" ) const ( testDirName = "ImplicitDirTest" ) -var ( - testDirPath string -) - // ////////////////////////////////////////////////////////////////////// // Tests // ////////////////////////////////////////////////////////////////////// func TestNewFileUnderImplicitDirectoryShouldNotGetSyncedToGCSTillClose(t *testing.T) { - testDirPath = setup.SetupTestDirectory(testDirName) - CreateImplicitDir(ctx, storageClient, testDirName, t) + testBaseDirName := path.Join(testDirName, operations.GetRandomName(t)) + testEnv.testDirPath = setup.SetupTestDirectoryRecursive(testBaseDirName) + CreateImplicitDir(testEnv.ctx, testEnv.storageClient, testBaseDirName, t) fileName := path.Join(ImplicitDirName, FileName1) - _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t) + _, fh := CreateLocalFileInTestDir(testEnv.ctx, testEnv.storageClient, testEnv.testDirPath, fileName, t) operations.WriteWithoutClose(fh, FileContents, t) - ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t) + if !setup.IsZonalBucketRun() { + // For non-zonal buckets, the object is not visible until the file is closed. + ValidateObjectNotFoundErrOnGCS(testEnv.ctx, testEnv.storageClient, testBaseDirName, fileName, t) + } else { + // For zonal buckets, the object is unfinalized, but visible. + // A zonal bucket object written without sync would be recognized as having zero-size. + ValidateObjectContentsFromGCS(testEnv.ctx, testEnv.storageClient, testBaseDirName, fileName, "", t) + + // A zonal bucket object written with sync can be fully read. + err := fh.Sync() + require.NoError(t, err) + ValidateObjectContentsFromGCS(testEnv.ctx, testEnv.storageClient, testBaseDirName, fileName, FileContents, t) + } // Validate. - CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, fileName, FileContents, t) + CloseFileAndValidateContentFromGCS(testEnv.ctx, testEnv.storageClient, fh, testBaseDirName, fileName, FileContents, t) } func TestReadDirForImplicitDirWithLocalFile(t *testing.T) { - testDirPath = setup.SetupTestDirectory(testDirName) - CreateImplicitDir(ctx, storageClient, testDirName, t) + testBaseDirName := path.Join(testDirName, operations.GetRandomName(t)) + testEnv.testDirPath = setup.SetupTestDirectoryRecursive(testBaseDirName) + CreateImplicitDir(testEnv.ctx, testEnv.storageClient, testBaseDirName, t) fileName1 := path.Join(ImplicitDirName, FileName1) fileName2 := path.Join(ImplicitDirName, FileName2) - _, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName1, t) - _, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName2, t) + _, fh1 := CreateLocalFileInTestDir(testEnv.ctx, testEnv.storageClient, testEnv.testDirPath, fileName1, t) + _, fh2 := CreateLocalFileInTestDir(testEnv.ctx, testEnv.storageClient, testEnv.testDirPath, fileName2, t) // Attempt to list implicit directory. - entries := operations.ReadDirectory(path.Join(testDirPath, ImplicitDirName), t) + entries := operations.ReadDirectory(path.Join(testEnv.testDirPath, ImplicitDirName), t) // Verify entries received successfully. operations.VerifyCountOfDirectoryEntries(3, len(entries), t) @@ -66,8 +77,8 @@ func TestReadDirForImplicitDirWithLocalFile(t *testing.T) { operations.VerifyFileEntry(entries[1], FileName2, 0, t) operations.VerifyFileEntry(entries[2], ImplicitFileName1, GCSFileSize, t) // Close the local files. - CloseFileAndValidateContentFromGCS(ctx, storageClient, fh1, testDirName, fileName1, "", t) - CloseFileAndValidateContentFromGCS(ctx, storageClient, fh2, testDirName, fileName2, "", t) + CloseFileAndValidateContentFromGCS(testEnv.ctx, testEnv.storageClient, fh1, testBaseDirName, fileName1, "", t) + CloseFileAndValidateContentFromGCS(testEnv.ctx, testEnv.storageClient, fh2, testBaseDirName, fileName2, "", t) } func TestRecursiveListingWithLocalFiles(t *testing.T) { @@ -80,20 +91,21 @@ func TestRecursiveListingWithLocalFiles(t *testing.T) { // mntDir/implicit/foo2 --- file // mntDir/implicit/implicitFile1 --- file - testDirPath = setup.SetupTestDirectory(testDirName) + testBaseDirName := path.Join(testDirName, operations.GetRandomName(t)) + testEnv.testDirPath = setup.SetupTestDirectoryRecursive(testBaseDirName) fileName2 := path.Join(ExplicitDirName, ExplicitFileName1) fileName3 := path.Join(ImplicitDirName, FileName2) // Create local file in mnt/ dir. - _, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + _, fh1 := CreateLocalFileInTestDir(testEnv.ctx, testEnv.storageClient, testEnv.testDirPath, FileName1, t) // Create explicit dir with 1 local file. - operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t) - _, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName2, t) + operations.CreateDirectory(path.Join(testEnv.testDirPath, ExplicitDirName), t) + _, fh2 := CreateLocalFileInTestDir(testEnv.ctx, testEnv.storageClient, testEnv.testDirPath, fileName2, t) // Create implicit dir with 1 local file1 and 1 synced file. - CreateImplicitDir(ctx, storageClient, testDirName, t) - _, fh3 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName3, t) + CreateImplicitDir(testEnv.ctx, testEnv.storageClient, testBaseDirName, t) + _, fh3 := CreateLocalFileInTestDir(testEnv.ctx, testEnv.storageClient, testEnv.testDirPath, fileName3, t) // Recursively list mntDir/ directory. - err := filepath.WalkDir(testDirPath, + err := filepath.WalkDir(testEnv.testDirPath, func(walkPath string, dir fs.DirEntry, err error) error { if err != nil { return err @@ -115,14 +127,14 @@ func TestRecursiveListingWithLocalFiles(t *testing.T) { } // Check if mntDir/explicitFoo/ has correct objects. - if walkPath == path.Join(testDirPath, ExplicitDirName) { + if walkPath == path.Join(testEnv.testDirPath, ExplicitDirName) { // numberOfObjects = 1 operations.VerifyCountOfDirectoryEntries(1, len(objs), t) operations.VerifyFileEntry(objs[0], ExplicitFileName1, 0, t) } // Check if mntDir/implicitFoo/ has correct objects. - if walkPath == path.Join(testDirPath, ImplicitDirName) { + if walkPath == path.Join(testEnv.testDirPath, ImplicitDirName) { // numberOfObjects = 2 operations.VerifyCountOfDirectoryEntries(2, len(objs), t) operations.VerifyFileEntry(objs[0], FileName2, 0, t) @@ -135,7 +147,7 @@ func TestRecursiveListingWithLocalFiles(t *testing.T) { if err != nil { t.Errorf("filepath.WalkDir() err: %v", err) } - CloseFileAndValidateContentFromGCS(ctx, storageClient, fh1, testDirName, FileName1, "", t) - CloseFileAndValidateContentFromGCS(ctx, storageClient, fh2, testDirName, fileName2, "", t) - CloseFileAndValidateContentFromGCS(ctx, storageClient, fh3, testDirName, fileName3, "", t) + CloseFileAndValidateContentFromGCS(testEnv.ctx, testEnv.storageClient, fh1, testBaseDirName, FileName1, "", t) + CloseFileAndValidateContentFromGCS(testEnv.ctx, testEnv.storageClient, fh2, testBaseDirName, fileName2, "", t) + CloseFileAndValidateContentFromGCS(testEnv.ctx, testEnv.storageClient, fh3, testBaseDirName, fileName3, "", t) } diff --git a/tools/integration_tests/implicit_dir/rename_sym_link_test.go b/tools/integration_tests/implicit_dir/rename_sym_link_test.go new file mode 100644 index 0000000000..3ee372707c --- /dev/null +++ b/tools/integration_tests/implicit_dir/rename_sym_link_test.go @@ -0,0 +1,56 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package implicit_dir_test + +import ( + "os" + "path" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRenameSymlinkToImplicitDir(t *testing.T) { + testDir := setup.SetupTestDirectory(DirForImplicitDirTests) + implicitDirName := "implicit_dir" + // Create an object that defines an implicit directory. This creates `implicit_dir/`. + objectNameInGCS := path.Join(DirForImplicitDirTests, implicitDirName, "placeholder") + err := client.CreateObjectOnGCS(testEnv.ctx, testEnv.storageClient, objectNameInGCS, "") + require.NoError(t, err) + implicitDirPath := path.Join(testDir, implicitDirName) + oldSymlinkPath := path.Join(testDir, "symlink_old") + err = os.Symlink(implicitDirPath, oldSymlinkPath) + require.NoError(t, err) + newSymlinkPath := path.Join(testDir, "symlink_new") + + err = os.Rename(oldSymlinkPath, newSymlinkPath) + + require.NoError(t, err) + _, err = os.Lstat(oldSymlinkPath) + require.Error(t, err) + assert.True(t, os.IsNotExist(err)) + fi, err := os.Lstat(newSymlinkPath) + require.NoError(t, err) + assert.Equal(t, os.ModeSymlink, fi.Mode()&os.ModeType) + targetRead, err := os.Readlink(newSymlinkPath) + require.NoError(t, err) + assert.Equal(t, implicitDirPath, targetRead) + targetFi, err := os.Stat(newSymlinkPath) + require.NoError(t, err) + assert.True(t, targetFi.IsDir()) +} diff --git a/tools/integration_tests/improved_run_e2e_tests.sh b/tools/integration_tests/improved_run_e2e_tests.sh new file mode 100755 index 0000000000..2fac101833 --- /dev/null +++ b/tools/integration_tests/improved_run_e2e_tests.sh @@ -0,0 +1,1079 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script Usage Documentation +usage() { + echo "Usage: $0 [options]" + echo "Options:" + echo " --bucket-location <location> Google Cloud Storage bucket location (e.g, 'us-central1') (Defaults to VM region if not provided)." + echo " --project-id <project-id> Google Cloud Project ID in which the bucket should be created. (e.g., 'gcs-fuse-test')" + echo " (Defaults to VM project if not provided or gcs-fuse-test if running on cloudtop)." + echo " --test-installed-package Test installed gcsfuse package. (Default: false)" + echo " --install-package-from-path <path> Google Cloud Storage bucket path for GCSFuse package for testing (e.g. gs://<bucket-name>/my-gcsfuse-package.rpm)" + echo " This option is mutually exclusive with --test-installed-package. (Default: "")" + echo " --skip-non-essential-tests Skip non-essential tests inside packages. (Default: false)" + echo " --test-on-tpc-endpoint Run tests on TPC endpoint. (Default: false)" + echo " --presubmit Run tests with presubmit flag. (Default: false)" + echo " --zonal Run tests with zonal bucket in --bucket-location region." + echo " The placement for Zonal buckets by deafault is Zone A of --bucket-location. (Default: false)" + echo " --no-build-binary-in-script To disable building gcsfuse binary in script. (Default: false)" + echo " --package-level-parallelism <parallelism> To adjust the number of packages to execute in parallel. (Default: 10)" + echo " --track-resource-usage To track resource(cpu/mem/disk) usage during e2e run. (Default: false)" + echo " --output-dir <output-dir> Directory in which all of log files generated by this script will be stored. (Default: /tmp)" + echo " --run-package <regex> Regex for packages to run. Supports '!' prefix for exclusion." + echo " Example: 'cloud_profiler|operations' to run only cloud_profiler and operations test packages." + echo " Example: '!cloud_profiler|operations' to run all test packages except cloud_profiler and operations." + echo " --skip-emulator Skip running emulator tests. (Default: false)" + echo " --flake-attempts <number> Number of attempts to run a package if it fails. (Default: 1)" + echo " --help Display this help and exit." + exit "$1" +} + +# Logging Helpers +log_info() { + echo "[INFO] $(date +"%Y-%m-%d %H:%M:%S"): $1" +} + +log_error() { + echo "[ERROR] $(date +"%Y-%m-%d %H:%M:%S"): $1" +} + +# Check or install bash version before continuing script. +readonly REQUIRED_BASH_MAJOR=5 +readonly REQUIRED_BASH_MINOR=1 +readonly BASH_INSTALL_VERSION="5.3" +readonly BASH_INSTALLATION_PATH="/usr/local/bin/bash" # Using 5.3 for installation as bash 5.1 has an installation bug. + +if (( BASH_VERSINFO[0] < REQUIRED_BASH_MAJOR || ( BASH_VERSINFO[0] == REQUIRED_BASH_MAJOR && BASH_VERSINFO[1] < REQUIRED_BASH_MINOR ) )); then + log_info "Current Bash version (${BASH_VERSINFO[0]}.${BASH_VERSINFO[1]}) is older than required (${REQUIRED_BASH_MAJOR}.${REQUIRED_BASH_MINOR})." + log_info "Installing Bash ${BASH_INSTALL_VERSION}..." + + # Dynamically find the repo root so we can locate the install script safely + SCRIPT_DIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") + REPO_ROOT=$(realpath "${SCRIPT_DIR}/../..") + + # Run the installation script + "${REPO_ROOT}/perfmetrics/scripts/install_bash.sh" "${BASH_INSTALL_VERSION}" + if [[ ! -x "${BASH_INSTALLATION_PATH}" ]]; then + log_error "Failed to locate the newly installed bash at ${BASH_INSTALLATION_PATH}" + exit 1 + fi + + log_info "Re-executing the e2e script using the newly installed Bash ${BASH_INSTALL_VERSION}..." + # The 'exec' command completely replaces the current old-bash process + # with the new bash process, passing along the script name ($0) and all arguments ($@). + exec "${BASH_INSTALLATION_PATH}" "$0" "$@" +fi +log_info "Bash version: ${BASH_VERSINFO[0]}.${BASH_VERSINFO[1]}" + +# Constants +readonly GO_VERSION=$(cat .go-version) +readonly TPCZERO_PROJECT_ID="tpczero-system:gcsfuse-test-project" +readonly TPC_BUCKET_LOCATION="u-us-prp1" +readonly BUCKET_PREFIX="gcsfuse-e2e" +readonly INTEGRATION_TEST_PACKAGE_DIR="./tools/integration_tests" +readonly INTEGRATION_TEST_PACKAGE_TIMEOUT_IN_MINS=90 +readonly ZONAL_BUCKET_SUPPORTED_LOCATIONS=("us-central1" "us-west4") +# 6 second delay between creating buckets as both hns and flat runs create buckets in parallel. +# Ref: https://cloud.google.com/storage/quotas#buckets +readonly DELAY_BETWEEN_BUCKET_CREATION=6 +readonly ZONAL="zonal" +readonly FLAT="flat" +readonly HNS="hns" +readonly SUCCESS_DIR_NAME="success_package_logs" +readonly FAILED_DIR_NAME="failed_package_logs" + +# Extract GCE VM Project and Location. +ZONE=$(curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/zone) +ZONE_NAME=$(basename "$ZONE") +GCE_VM_LOCATION="${ZONE_NAME%-*}" +GCE_VM_PROJECT_ID=$(curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/project/project-id) +log_info "Project ID from GCE VM: '$GCE_VM_PROJECT_ID'" +log_info "Location from GCE VM: '$GCE_VM_LOCATION'" +log_info "Running e2e script as '$(whoami)'" +log_info "Current directory is '$(pwd)'" +# If HOME is not set, find it dynamically and export it +if [ -z "$HOME" ]; then + export HOME=$(getent passwd "$(whoami)" | cut -d: -f6) +fi +log_info "HOME is set to '$HOME'" + +# This variable will store the path if the script builds GCSFuse binaries (gcsfuse, mount.gcsfuse) +BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR="" + +# Output directory where all artifacts generated by this script will be stored. +OUTPUT_DIR="" + +KOKORO_DIR_AVAILABLE=false +if [[ -n "$KOKORO_ARTIFACTS_DIR" ]]; then + KOKORO_DIR_AVAILABLE=true +fi + +# Argument Parsing and Assignments +# Set default values for optional arguments +SKIP_NON_ESSENTIAL_TESTS_ON_PACKAGE=false +TEST_INSTALLED_PACKAGE=false +BUCKET_LOCATION="" +PROJECT_ID="" +INSTALL_PACKAGE_FROM_PATH="" +RUN_TEST_ON_TPC_ENDPOINT=false +RUN_TESTS_WITH_PRESUBMIT_FLAG=false +RUN_TESTS_WITH_ZONAL_BUCKET=false +BUILD_BINARY_IN_SCRIPT=true +TRACK_RESOURCE_USAGE=false +PACKAGE_LEVEL_PARALLELISM=10 # Controls how many test packages are run in parallel for hns, flat or zonal buckets. +RUN_PACKAGE_REGEX="" +SKIP_EMULATOR=false +FLAKE_ATTEMPTS=1 + +# Define options for getopt +# A long option name followed by a colon indicates it requires an argument. +LONG=bucket-location:,project-id:,test-installed-package,install-package-from-path:,skip-non-essential-tests,no-build-binary-in-script,test-on-tpc-endpoint,presubmit,zonal,package-level-parallelism:,track-resource-usage,output-dir:,help,run-package:,skip-emulator,flake-attempts: + +# Parse the options using getopt +# --options "" specifies that there are no short options. +PARSED=$(getopt --options "" --longoptions "$LONG" --name "$0" -- "$@") +if [[ $? -ne 0 ]]; then + # getopt will have already printed an error message + usage 1 +fi + +# Read the parsed options back into the positional parameters. +eval set -- "$PARSED" + +# Loop through the options and assign values to our variables +while (( $# >= 1 )); do + case "$1" in + --bucket-location) + BUCKET_LOCATION="$2" + shift 2 + ;; + --project-id) + PROJECT_ID="$2" + shift 2 + ;; + --package-level-parallelism) + PACKAGE_LEVEL_PARALLELISM="$2" + shift 2 + ;; + --test-installed-package) + TEST_INSTALLED_PACKAGE=true + shift + ;; + --install-package-from-path) + INSTALL_PACKAGE_FROM_PATH="$2" + shift 2 + ;; + --skip-non-essential-tests) + SKIP_NON_ESSENTIAL_TESTS_ON_PACKAGE=true + shift + ;; + --no-build-binary-in-script) + BUILD_BINARY_IN_SCRIPT=false + shift + ;; + --test-on-tpc-endpoint) + RUN_TEST_ON_TPC_ENDPOINT=true + shift + ;; + --presubmit) + RUN_TESTS_WITH_PRESUBMIT_FLAG=true + shift + ;; + --zonal) + RUN_TESTS_WITH_ZONAL_BUCKET=true + shift + ;; + --track-resource-usage) + TRACK_RESOURCE_USAGE=true + shift + ;; + --output-dir) + OUTPUT_DIR="$2" + shift 2 + ;; + --run-package) + RUN_PACKAGE_REGEX="$2" + shift 2 + ;; + --skip-emulator) + SKIP_EMULATOR=true + shift + ;; + --flake-attempts) + FLAKE_ATTEMPTS="$2" + shift 2 + ;; + --help) + usage 0 + ;; + --) + shift + break + ;; + *) + log_error "Unrecognized arguments [$*]." + usage 1 + ;; + esac +done + +# Validates option value to be non-empty and should not be another option name. +validate_option_value() { + local option=$1 + local value=$2 + if [[ -z "$value" || "$value" == -* ]]; then + log_error "Invalid or empty value [$value] for option $option." + usage 1 + fi +} + +# Fallback to /tmp if OUTPUT_DIR is unset +BASE_PATH="${OUTPUT_DIR:-/tmp}" +mkdir -p "$BASE_PATH" || { + log_error "Failed to create or access output directory '$BASE_PATH'"; + exit 1 +} +OUTPUT_DIR=$(mktemp -d "${BASE_PATH%/}/gcsfuse-e2e-run-XXXXXXXX") || { + log_error "Failed to create unique output directory in '$BASE_PATH'"; + exit 1 +} +log_info "Output directory for the e2e run is set to '$OUTPUT_DIR'" + +if [[ -z "$BUCKET_LOCATION" ]]; then + log_info "Bucket Location is not provided, using GCE VM Location '$GCE_VM_LOCATION' as bucket location." + BUCKET_LOCATION="$GCE_VM_LOCATION" +fi + +if [[ -z "$PROJECT_ID" ]]; then + log_info "Project ID is not provided, using GCE VM Project ID '$GCE_VM_PROJECT_ID' as Project ID." + PROJECT_ID="$GCE_VM_PROJECT_ID" +fi + +# Check if it contains "cloudtop" +if [[ "$PROJECT_ID" == *"cloudtop"* ]]; then + log_info "You are running this script on cloudtop. Manually overriding the Project ID to 'gcs-fuse-test'." + PROJECT_ID="gcs-fuse-test" +fi + +# Validate long options which need values(default or user provided). +validate_option_value "--bucket-location" "$BUCKET_LOCATION" +validate_option_value "--project-id" "$PROJECT_ID" +validate_option_value "--package-level-parallelism" "$PACKAGE_LEVEL_PARALLELISM" +validate_option_value "--flake-attempts" "$FLAKE_ATTEMPTS" + +# Validate test install package from path +if ${TEST_INSTALLED_PACKAGE} && [[ -n "$INSTALL_PACKAGE_FROM_PATH" ]]; then + log_error "Option --test-installed-package and --install-package-from-path are mutually exclusive. Please set only one" + usage 1 +fi + +# Zonal Bucket location validation. +if ${RUN_TESTS_WITH_ZONAL_BUCKET}; then + supported_bucket=false + for location in "${ZONAL_BUCKET_SUPPORTED_LOCATIONS[@]}"; do + if [[ "$BUCKET_LOCATION" == "$location" ]]; then + supported_bucket=true + break + fi + done + if ! ${supported_bucket}; then + log_error "Unsupported Bucket Location ${BUCKET_LOCATION} for Zonal Run. Supported Locations are: ${ZONAL_BUCKET_SUPPORTED_LOCATIONS[*]}" + exit 1 + fi +fi + +# Create file helper creates a file in the output directory. +create_file_helper() { + local relative_path="$1" + + if [[ -z "$relative_path" ]]; then + log_error "Usage: create_file_helper <relative_path>" + exit 1 + fi + + local full_path="${OUTPUT_DIR%/}/$relative_path" + local target_dir + target_dir=$(dirname "$full_path") + + # Create parent directories and then the empty file + { + mkdir -p "$target_dir" && touch "$full_path" + } &> /dev/null || { + log_error "Failed to create file at: $full_path" + exit 1 + } + echo "$full_path" +} + +LOG_LOCK_FILE=$(create_file_helper "logging.lock") +BUCKET_CREATION_LOCK_FILE=$(create_file_helper "bucket_creation.lock") +PACKAGE_RUNTIME_STATS=$(create_file_helper "package_runtime_stats.txt") +RESOURCE_USAGE_FILE=$(create_file_helper "system_resource_usage.txt") +CREATED_BUCKETS_LIST_FILE=$(create_file_helper "created_buckets_list.txt") + +# Test packages which can be run for both Zonal and Regional buckets. +# Sorted list descending run times. (Longest Processing Time first strategy) +TEST_PACKAGES_COMMON=( + "managed_folders" + "operations" + "read_large_files" + "concurrent_operations" + "read_cache" + "list_large_dir" + "mount_timeout" + "write_large_files" + "implicit_dir" + "interrupt" + "local_file" + "readonly" + "readonly_creds" + "rename_dir_limit" + "kernel_list_cache" + "streaming_writes" + "benchmarking" + "explicit_dir" + "gzip" + "log_rotation" + "monitoring" + "mounting" + "unsupported_path" + # "grpc_validation" + "negative_stat_cache" + "stale_handle" + "release_version" + "readdirplus" + "dentry_cache" + "buffered_read" + "flag_optimizations" + "symlink_handling" +) + +# filter_array: Filters an array in place keeping only elements matching the regex. +# Supports '!' prefix to invert the match (exclude). +# Args: $1 = name of the array variable, $2 = regex pattern. +filter_array() { + local -n arr=$1 + local regex=$2 + + # Return early if regex or array is empty + [[ -z "$regex" || ${#arr[@]} -eq 0 ]] && return + + # Check if regex starts with '!', meaning we want to exclude matches + local invert="" + if [[ "$regex" == !* ]]; then + invert="-v" + regex="${regex#!}" # Strip the '!' prefix + fi + + # Filter the array using grep and map the results back to the array + mapfile -t arr < <(printf '%s\n' "${arr[@]}" | grep $invert -E "$regex") +} + +# Test packages for regional buckets. +TEST_PACKAGES_FOR_RB=("${TEST_PACKAGES_COMMON[@]}" "inactive_stream_timeout" "cloud_profiler" "requester_pays_bucket") +# Test packages for zonal buckets. +TEST_PACKAGES_FOR_ZB=("${TEST_PACKAGES_COMMON[@]}" "rapid_appends" "unfinalized_object") +# Test packages for TPC buckets. +TEST_PACKAGES_FOR_TPC=("operations") + +# Parse and apply --run-package filters if provided +if [[ -n "$RUN_PACKAGE_REGEX" ]]; then + filter_array TEST_PACKAGES_FOR_RB "$RUN_PACKAGE_REGEX" + filter_array TEST_PACKAGES_FOR_ZB "$RUN_PACKAGE_REGEX" +fi + +# acquire_lock: Acquires exclusive lock or exits script on failure. +# Args: $1 = path to lock file. +acquire_lock() { + if [[ -z "$1" ]]; then + log_error "acquire_lock: Lock file path is required." + exit 1 + fi + local lock_file="$1" + local timeout_seconds=600 # 10 minutes + exec 200>"$lock_file" || { + log_error "Could not open lock file $lock_file." + exit 1 + } + # Attempt to acquire the lock with a timeout + if ! flock -x -w "$timeout_seconds" 200; then + log_error "Failed to acquire lock on $lock_file within $timeout_seconds seconds." + # Close the file descriptor if the lock was not acquired + exec 200>&- + exit 1 + fi + return 0 +} + +# release_lock: Releases lock or exits script on failure. +# Args: $1 = path to lock file +release_lock() { + if [[ -z "$1" ]]; then + log_error "release_lock: Lock file path is required." + exit 1 + fi + local lock_file="$1" + [[ -e "/proc/self/fd/200" || -L "/proc/self/fd/200" ]] && exec 200>&- || { + log_error "Lock file descriptor (FD 200) not open for $lock_file. Possible previous error or double release." + exit 1 + } # FD not open or close failed + return 0 +} + +# logs info to stdout exclusively. used in background commands to ensure logs aren't interleaved. +log_info_locked() { + acquire_lock "$LOG_LOCK_FILE" + log_info "$1" + release_lock "$LOG_LOCK_FILE" +} + +# logs error to stdout exclusively. Used in background commands to ensure logs aren't interleaved. +log_error_locked() { + acquire_lock "$LOG_LOCK_FILE" + log_error "$1" + release_lock "$LOG_LOCK_FILE" +} + +# Helper method to organize the test log file based on exit code and bucket type. +# It organizes the log files in the following directory. +# ${OUTPUT_DIR}/failed_package_logs/${BUCKET_TYPE}/... +# ${OUTPUT_DIR}/success_package_logs/${BUCKET_TYPE}/... +organize_test_logfile() { + if [[ $# -ne 4 ]]; then + log_error "organize_test_logfile() called with incorrect number of arguments." + return 1 + fi + local exit_code="$1" + local log_file="$2" + local base_filename="$3" + local bucket_type="$4" + + local status_dir + if [[ "$exit_code" -eq 0 ]]; then + status_dir="${SUCCESS_DIR_NAME}" + else + status_dir="${FAILED_DIR_NAME}" + fi + + local dest_dir="${OUTPUT_DIR}/${status_dir}" + if [[ -n "$bucket_type" ]]; then + dest_dir="${dest_dir}/${bucket_type}" + fi + + mkdir -p "$dest_dir" + cp "$log_file" "$dest_dir/${base_filename}.txt" + rm -f "$log_file" +} + +# Helper method to create "flat", "hns" or "zonal" bucket. +create_bucket() { + if [[ $# -ne 2 ]]; then + log_error "create_bucket() called with incorrect number of arguments." + return 1 + fi + local package="$1" + local bucket_type="$2" + local bucket_name="${BUCKET_PREFIX}-${package}-${bucket_type}-$(date +%s%N)" + local bucket_cmd_parts=("gcloud" "alpha" "storage" "buckets" "create" "gs://${bucket_name}" "--project=${PROJECT_ID}" "--location=${BUCKET_LOCATION}" "--uniform-bucket-level-access") + if [[ "$bucket_type" == "$HNS" ]]; then + bucket_cmd_parts+=("--enable-hierarchical-namespace") + elif [[ "$bucket_type" == "$ZONAL" ]]; then + bucket_cmd_parts+=("--enable-hierarchical-namespace" "--placement=${BUCKET_LOCATION}-a" "--default-storage-class=RAPID") + elif [[ "$bucket_type" != "$FLAT" ]]; then + log_error "Invalid bucket type: $bucket_type." + return 1 + fi + local bucket_cmd bucket_cmd_log attempt=5 + bucket_cmd=$(printf "%q " "${bucket_cmd_parts[@]}") + bucket_cmd_log=$(create_file_helper "bucket_creation_logs/${bucket_name}.txt") + while : ; do + attempt=$((attempt - 1)) + if [ $attempt -lt 0 ]; then + log_error "Unable to create bucket [${bucket_name}] after 5 attempts." + cat "$bucket_cmd_log" + return 1 + fi + acquire_lock "$BUCKET_CREATION_LOCK_FILE" + eval "$bucket_cmd" > "$bucket_cmd_log" 2>&1 + local status=$? + sleep "$DELAY_BETWEEN_BUCKET_CREATION" # have 6 seconds gap between creating buckets. + release_lock "$BUCKET_CREATION_LOCK_FILE" + if [ $status -eq 0 ]; then + break + fi + done + echo "$bucket_name" + # Append to created buckets list file for cleanup + acquire_lock "$BUCKET_CREATION_LOCK_FILE" + echo "$bucket_name" >> "$CREATED_BUCKETS_LIST_FILE" + release_lock "$BUCKET_CREATION_LOCK_FILE" + rm -rf "$bucket_cmd_log" + return 0 +} + +# Helper method to cleanup buckets created during this run. +cleanup_created_buckets() { + if [ ! -f "$CREATED_BUCKETS_LIST_FILE" ]; then + return 0 + fi + + local -a bucket_uris + mapfile -t bucket_uris < "$CREATED_BUCKETS_LIST_FILE" + + if [ ${#bucket_uris[@]} -eq 0 ]; then + log_info "No buckets were created to cleanup." + return 0 + fi + + log_info "Found ${#bucket_uris[@]} buckets created during run to cleanup." + + local batch_count=0 + local start_index=0 + local total_buckets=${#bucket_uris[@]} + # Number of buckets to delete in a single batch. + local bucket_deletion_batch_size=10 + + while [ "$start_index" -lt "$total_buckets" ]; do + # Calculate end index for the current batch + local end_index=$((start_index + bucket_deletion_batch_size)) + if [ "$end_index" -gt "$total_buckets" ]; then + end_index=$total_buckets + fi + + # Extract batch slice + local length=$((end_index - start_index)) + local batch=("${bucket_uris[@]:$start_index:$length}") + + # Add gs:// prefix if missing (bucket names are stored as name only) + local batch_uris=() + local batch_names_str="" + for b in "${batch[@]}"; do + batch_uris+=("gs://$b") + if [ -z "$batch_names_str" ]; then + batch_names_str="$b" + else + batch_names_str="$batch_names_str, $b" + fi + done + + batch_count=$((batch_count + 1)) + log_info "Deleting bucket batch: $batch_count ..." + + # Delete batch + if ! gcloud storage rm -r "${batch_uris[@]}" --no-user-output-enabled --verbosity=error; then + log_error "Failed to delete one or more buckets in batch: $batch_count. These buckets would be cleaned up by the periodic cleanup job." + # We continue here to try deleting other batches if one fails + fi + + start_index=$end_index + done + + rm -f "$CREATED_BUCKETS_LIST_FILE" + log_info "Bucket cleanup complete." +} + +# Get command of the PID and check if it contains the string. Kill if it does. +safe_kill() { + local pid=$1 + local str=$2 + local cmd + + if [[ -n "$pid" && -n "$str" ]] && cmd=$(ps -p "$pid" -o cmd=) && [[ "$cmd" == *"$str"* ]]; then + kill "$pid" + else + return 1 + fi +} + +# Cleanup ensures each of the buckets created is destroyed and the temp files are cleaned up. +clean_up() { + if ${TRACK_RESOURCE_USAGE}; then + if ! safe_kill "$RESOURCE_USAGE_PID" "resource_usage.sh"; then + log_error "Failed to stop resource usage collection process (or it's already stopped)" + else + log_info "Resource usage collection process stopped." + fi + fi + if [ -n "${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}" ] && [ -d "${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}" ]; then + log_info "Cleaning up GCSFuse build directory created by script: ${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}" + rm -rf "${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}" + fi + cleanup_created_buckets +} + +# run_package_parallel: Executes test packages in parallel. +# The function returns a non-zero exit status if any of the packages fail all attempts. +# +# Usage: run_package_parallel "parallelism" "bucket_type" "max_retries" "package1" "package2" ... +# First argument is extent of parallelism for this command. +# Second argument is the bucket type ("flat", "hns", "zonal"). +# Third argument is the maximum number of retries for failed commands. +# Rest of the arguments are package names to run. +# +# Example: +# run_package_parallel 2 "flat" 2 "managed_folders" "operations" "read_large_files" +# This command will run at max 2 packages in parallel, with up to 2 retries on failure. +run_package_parallel() { + if [[ $# -lt 3 ]]; then + log_error_locked "run_package_parallel() called with incorrect number of arguments." + return 1 + fi + local parallelism="$1" bucket_type="$2" flake_attempts="$3" + shift 3 + + local -a package_list=("$@") + local -A package_status=() + local -A package_attempt=() + + for pkg in "${package_list[@]}"; do + package_status["$pkg"]=1 + package_attempt["$pkg"]=1 + done + + local -A package_name_by_pid=() + + while :; do + # Launch packages up to parallelism limit + for pkg in "${package_list[@]}"; do + # Skip if we hit parallelism limit + [[ ${#package_name_by_pid[@]} -ge $parallelism ]] && continue + + # Skip if package already succeeded + [[ "${package_status["$pkg"]}" -eq 0 ]] && continue + + # Skip if max retries exceeded + [[ "${package_attempt["$pkg"]}" -gt "$flake_attempts" ]] && continue + + # Skip if already running + [[ " ${package_name_by_pid[@]} " =~ " $pkg " ]] && continue + + create_bucket_and_run_package "${bucket_type}" "$pkg" "${package_attempt["$pkg"]}" & + local pid=$! + package_name_by_pid["$pid"]="$pkg" + done + + # Break if no commands are running + [[ ${#package_name_by_pid[@]} -eq 0 ]] && break + + # Wait for any background process to finish + local waited_pid + wait -n -p waited_pid + local exit_status=$? + + local pkg="${package_name_by_pid[$waited_pid]}" + unset "package_name_by_pid[$waited_pid]" + + package_status["$pkg"]=$exit_status + + if [[ "$exit_status" -ne 0 ]]; then + package_attempt["$pkg"]=$((package_attempt[$pkg] + 1)) + fi + done + + # Return non-zero if any package failed all attempts + for s in "${package_status[@]}"; do + if [[ "$s" -ne 0 ]]; then + return 1 + fi + done + + return 0 +} + +# Helper method that creates a bucket and then runs the test package. +create_bucket_and_run_package() { + if [[ $# -ne 3 ]]; then + log_error_locked "create_bucket_and_run_package() called with incorrect number of arguments." + return 1 + fi + local bucket_type="$1" + local package_name="$2" + local attempt_number="$3" + + if ! bucket_name=$(create_bucket "$package_name" "$bucket_type"); then + log_error_locked "Failed to create bucket of type ${bucket_type} for package ${package_name}. Bucket creation output: ${bucket_name}" + return 1 + fi + test_package "$package_name" "$bucket_name" "$bucket_type" "$attempt_number" +} + +# Helper method to execute an E2E test package. +test_package() { + if [[ $# -ne 4 ]]; then + log_error_locked "test_package() called with incorrect number of arguments." + return 1 + fi + local package_name="$1" + local bucket_name="$2" + local bucket_type="$3" + local attempt_number="$4" + + # Build go package test command. + local go_test_cmd_parts=("GODEBUG=asyncpreemptoff=1" "go" "test" "-v" "-timeout=${INTEGRATION_TEST_PACKAGE_TIMEOUT_IN_MINS}m" "${INTEGRATION_TEST_PACKAGE_DIR}/${package_name}") + if ${SKIP_NON_ESSENTIAL_TESTS_ON_PACKAGE}; then + go_test_cmd_parts+=("-short") + fi + if [[ "$package_name" == "benchmarking" ]]; then + go_test_cmd_parts+=("-bench=." "-benchtime=100x") + fi + # Test Binary flags after this. + go_test_cmd_parts+=("-args" "--integrationTest" "--testbucket=${bucket_name}") + if ${TEST_INSTALLED_PACKAGE}; then + go_test_cmd_parts+=("--testInstalledPackage") + fi + if ${RUN_TESTS_WITH_PRESUBMIT_FLAG}; then + go_test_cmd_parts+=("--presubmit") + fi + if [[ "$bucket_type" == "$ZONAL" ]]; then + go_test_cmd_parts+=("--zonal") + fi + if ${RUN_TEST_ON_TPC_ENDPOINT}; then + go_test_cmd_parts+=("--testOnTPCEndPoint") + fi + if [[ -n "$BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR" ]]; then + go_test_cmd_parts+=("--gcsfuse_prebuilt_dir=${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}") + fi + + local go_test_cmd test_package_log_file start=$SECONDS exit_code=0 + # Use printf %q to quote each argument safely for eval + # This ensures spaces and special characters within arguments are handled correctly. + go_test_cmd=$(printf "%q " "${go_test_cmd_parts[@]}") + test_package_log_file=$(create_file_helper "running_package_logs/${bucket_type}/${package_name}_attempt_${attempt_number}.txt") + # Run the package test command and capture log output with runtime stats. + log_info "Started running test package [$package_name] for bucket type [$bucket_type] with bucket name [$bucket_name] (Attempt: $attempt_number)" + + if ! eval "$go_test_cmd" > "$test_package_log_file" 2>&1; then + exit_code=1 + if [[ "$attempt_number" -lt "$FLAKE_ATTEMPTS" ]]; then + log_info "Failed test package [$package_name] for bucket type [$bucket_type] (Attempt: $attempt_number). Will retry." + else + log_info "Failed test package [$package_name] for bucket type [$bucket_type] (Attempt: $attempt_number). No more retries." + fi + else + log_info "Passed test package [$package_name] for bucket type [$bucket_type] (Attempt: $attempt_number)" + fi + + local end=$SECONDS + + # Add the package stats to the file. + echo "${package_name} ${bucket_type} ${exit_code} ${start} ${end}" >> "$PACKAGE_RUNTIME_STATS" + # Generate Kokoro artifacts(log) files only on terminal attempt. + if [[ "$exit_code" -eq 0 || "$attempt_number" -ge "$FLAKE_ATTEMPTS" ]]; then + generate_test_log_artifacts "$test_package_log_file" "$package_name" "$bucket_type" + fi + # Call the helper to organize logs and cleanup the original file + organize_test_logfile "$exit_code" "$test_package_log_file" "${package_name}_attempt_${attempt_number}" "$bucket_type" + return "$exit_code" +} + +# Helper method to generate Kokoro artifacts(log) files when building in Kokoro environment. +generate_test_log_artifacts() { + # If KOKORO_ARTIFACTS_DIR is not set, skip artifact generation. + if ! $KOKORO_DIR_AVAILABLE; then + return 0 + fi + + if [[ $# -ne 3 ]]; then + log_error_locked "generate_test_log_artifacts() called with incorrect number of arguments." + return 1 + fi + + local log_file="$1" + local package_name="$2" + local bucket_type="$3" + + if [ ! -f "$log_file" ]; then + return 0 + fi + + local output_dir="${KOKORO_ARTIFACTS_DIR}/${bucket_type}/${package_name}" + mkdir -p "$output_dir" + local sponge_log_file="${output_dir}/sponge_log.log" + local sponge_xml_file="${output_dir}/sponge_log.xml" + + cp "$log_file" "$sponge_log_file" + + echo '<?xml version="1.0" encoding="UTF-8"?>' > "${sponge_xml_file}" + echo '<testsuites>' >> "${sponge_xml_file}" + + # Remove first 2 lines and last line from log. + local report_log=$(cat "$log_file") + # For benchmarking package, filter out benchmark results to avoid incorrect XML results. + if [[ "$package_name" == "benchmarking" ]]; then + report_log=$(echo "$report_log" | grep -v '^Benchmark_[^[:space:]]*$') + fi + + echo "$report_log" | go-junit-report | sed '1,2d;$d' >> "${sponge_xml_file}" + echo '</testsuites>' >> "${sponge_xml_file}" + + return 0 +} + +install_package_from_path() { + if [[ $# -ne 1 ]]; then + log_error_locked "install_package_from_path() called with incorrect number of arguments." + return 1 + fi + local package_path="$1" + log_info "Downloading $(basename "${package_path}")..." + gcloud storage cp "${package_path}" /tmp/ --quiet || return 1 + if [ -f /etc/os-release ]; then + # We source in a subshell to prevent variable pollution, + # then capture only the ID and ID_LIKE fields. + DISTRO_DATA=$( (source /etc/os-release; echo "${ID:-} ${ID_LIKE:-}") ) + # Check for debian or ubuntu in the ID or the ID_LIKE chain + if [[ "$DISTRO_DATA" == *"debian"* ]] || [[ "$DISTRO_DATA" == *"ubuntu"* ]]; then + sudo dpkg -i "/tmp/$(basename "${package_path}")" + elif [[ "$DISTRO_DATA" == *"rhel"* ]] || [[ "$DISTRO_DATA" == *"centos"* ]]; then + sudo yum -y localinstall "/tmp/$(basename "${package_path}")" + else + log_error "This script only supports Debian/Ubuntu/rhel/centos based distributions." + log_info "Your distribution is:" + cat /etc/os-release + exit 1 + fi + else + log_error "/etc/os-release not found. Unable to determine distribution" + exit 1 + fi +} + +build_gcsfuse_once() { + local build_output_dir # For the final gcsfuse binaries + build_output_dir=$(mktemp -d -t gcsfuse_e2e_run_build_XXXXXX) + log_info "GCSFuse binaries will be built in ${build_output_dir}/" + + local gcsfuse_src_dir + # Determine GCSFuse source directory + # If this script is in tools/integration_tests, project root is ../../ + SCRIPT_DIR_REALPATH=$(realpath "$(dirname "${BASH_SOURCE[0]}")") + gcsfuse_src_dir=$(realpath "${SCRIPT_DIR_REALPATH}/../../") + + if [[ ! -f "${gcsfuse_src_dir}/go.mod" ]]; then + log_error "Could not reliably determine GCSFuse project root from ${SCRIPT_DIR_REALPATH}. Expected go.mod at ${gcsfuse_src_dir}" >&2 + rm -rf "${build_output_dir}" + exit 1 + fi + log_info "Using GCSFuse source directory: ${gcsfuse_src_dir}" + + log_info "Building GCSFuse using 'go run ./tools/build_gcsfuse/main.go'..." + (cd "${gcsfuse_src_dir}" && go run ./tools/build_gcsfuse/main.go . "${build_output_dir}" "0.0.0") + if [ $? -ne 0 ]; then + log_error "Building GCSFuse binaries using 'go run ./tools/build_gcsfuse/main.go' failed." + rm -rf "${build_output_dir}" # Clean up created temp dir + return 1 + fi + + # Set the directory path for use by the script (to form the go test flag) + BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR="${build_output_dir}" + log_info "GCSFuse binaries built by script in: ${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}" + log_info "GCSFuse executable: ${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}/bin/gcsfuse" + return 0 +} + +install_packages() { + local os_id + + # Determine the absolute location of THIS script + SCRIPT_DIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") + + # Calculate the Repo Root + REPO_ROOT="${SCRIPT_DIR}/../.." + + source "${REPO_ROOT}/perfmetrics/scripts/os_utils.sh" + + if ! os_id=$(get_os_id); then + log_error "Failed to detect OS ID." + exit 1 + fi + log_info "Detected OS ID: $os_id" + + install_packages_by_os "$os_id" "python3" "gcc" "python3-dev" "python3-setuptools" "python3-crcmod" "fuse3" || { + log_error "Failed to install required packages." + exit 1 + } + + # Execute install_go.sh using the absolute path + bash "${REPO_ROOT}/perfmetrics/scripts/install_go.sh" "$GO_VERSION" + export PATH="/usr/local/go/bin:$PATH" + + # Install latest gcloud version. + bash "${REPO_ROOT}/perfmetrics/scripts/install_latest_gcloud.sh" + export PATH="/usr/local/google-cloud-sdk/bin:$PATH" + export CLOUDSDK_PYTHON="$HOME/.local/python-3.11.9/bin/python3.11" + export PATH="$HOME/.local/python-3.11.9/bin:$PATH" + if ${KOKORO_DIR_AVAILABLE} ; then + # Install go-junit-report to generate XML test reports from go logs. + go install github.com/jstemmer/go-junit-report/v2@latest + export PATH="$(go env GOPATH)/bin:$PATH" + fi +} + +# Generic function to run a group of E2E tests for a given bucket type. +# Args: +# $1: Descriptive group name (e.g., "REGIONAL", "ZONAL", "TPC") +# $2: Bucket type ("flat", "hns", "zonal") +# $@: A list of test package names to run. +run_test_group() { + local group_name="$1" + local bucket_type="$2" + shift 2 + local -a test_packages=("$@") + local group_exit_code=0 + log_info_locked "Started running e2e tests for ${group_name} group (bucket type: ${bucket_type})." + + run_package_parallel "$PACKAGE_LEVEL_PARALLELISM" "$bucket_type" "$FLAKE_ATTEMPTS" "${test_packages[@]}" + group_exit_code=$? + + if [ "$group_exit_code" -ne 0 ]; then + log_error_locked "The e2e tests for ${group_name} group (bucket type: ${bucket_type}) FAILED." + return 1 + fi + log_info_locked "The e2e tests for ${group_name} group (bucket type: ${bucket_type}) successful." + return 0 +} + +run_e2e_tests_for_emulator() { + local package_name="emulator_tests" + local bucket_type="emulator" + + local exit_code=0 + local attempt=1 + + while :; do + local start=$SECONDS + emulator_test_log=$(create_file_helper "running_package_logs/${bucket_type}/${package_name}_attempt_${attempt}.txt") + + log_info_locked "Started running test package [$package_name] for bucket type [$bucket_type] (Attempt: $attempt)" + + if ! ./tools/integration_tests/emulator_tests/emulator_tests.sh "$TEST_INSTALLED_PACKAGE" "$BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR" > "$emulator_test_log" 2>&1; then + exit_code=1 + if [[ "$attempt" -lt "$FLAKE_ATTEMPTS" ]]; then + log_info_locked "Failed test package [$package_name] for bucket type [$bucket_type] (Attempt: $attempt). Will retry." + else + log_info_locked "Failed test package [$package_name] for bucket type [$bucket_type] (Attempt: $attempt). No more retries." + fi + else + log_info_locked "Passed test package [$package_name] for bucket type [$bucket_type] (Attempt: $attempt)" + exit_code=0 + fi + + # Call the helper to organize logs and cleanup the original file + organize_test_logfile "$exit_code" "$emulator_test_log" "${package_name}_attempt_${attempt}" "$bucket_type" + + local end=$SECONDS + echo "${package_name} ${bucket_type} ${exit_code} ${start} ${end}" >> "$PACKAGE_RUNTIME_STATS" + + if [[ "$exit_code" -eq 0 || "$attempt" -ge "$FLAKE_ATTEMPTS" ]]; then + break + fi + + attempt=$((attempt + 1)) + done + + return "$exit_code" +} + +main() { + # Clean up everything on exit. + trap clean_up EXIT + log_info "" + log_info "------ Upgrading gcloud and installing packages ------" + log_info "" + set -e + install_packages + set +e + log_info "------ Upgrading gcloud and installing packages took $SECONDS seconds ------" + + log_info "" + log_info "------ Started running E2E test packages ------" + log_info "" + + # Decide whether to install a package from a path or build GCSFuse based on RUN_E2E_TESTS_ON_PACKAGE + if [[ -n "$INSTALL_PACKAGE_FROM_PATH" ]]; then + log_info "Installing package from the path '${INSTALL_PACKAGE_FROM_PATH}'" + if ! install_package_from_path "$INSTALL_PACKAGE_FROM_PATH"; then + log_error "Unable to install the package from path '${INSTALL_PACKAGE_FROM_PATH}'. Exiting." + exit 1 + fi + # Setting test installed package to true + TEST_INSTALLED_PACKAGE=true + elif (! ${TEST_INSTALLED_PACKAGE} ) && ${BUILD_BINARY_IN_SCRIPT}; then + log_info "TEST_INSTALLED_PACKAGE is not 'true' (value: '${TEST_INSTALLED_PACKAGE}') and BUILD_BINARY_IN_SCRIPT is 'true'." + log_info "Building GCSFuse inside script..." + if ! build_gcsfuse_once; then + log_error "build_gcsfuse_once failed. Exiting." + # The trap will handle cleanup + exit 1 + fi + log_info "Script built GCSFuse at: ${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}" + fi + + # Reset SECONDS to 0 + SECONDS=0 + + if ${TRACK_RESOURCE_USAGE}; then + # Start collecting system resource usage in background. + log_info "Starting resource usage collection process." + ./tools/integration_tests/resource_usage.sh "COLLECT" "$RESOURCE_USAGE_FILE" & + RESOURCE_USAGE_PID=$! + log_info "Resource usage collection process started at PID: $RESOURCE_USAGE_PID" + fi + + local pids=() + local overall_exit_code=0 + if ${RUN_TESTS_WITH_ZONAL_BUCKET}; then + run_test_group "ZONAL" "$ZONAL" "${TEST_PACKAGES_FOR_ZB[@]}" & pids+=($!) + elif ${RUN_TEST_ON_TPC_ENDPOINT}; then + # Override PROJECT_ID and BUCKET_LOCATION for TPC tests + PROJECT_ID="$TPCZERO_PROJECT_ID" + BUCKET_LOCATION="$TPC_BUCKET_LOCATION" + run_test_group "TPC" "$HNS" "${TEST_PACKAGES_FOR_TPC[@]}" & pids+=($!) + run_test_group "TPC" "$FLAT" "${TEST_PACKAGES_FOR_TPC[@]}" & pids+=($!) + else + run_test_group "REGIONAL" "$HNS" "${TEST_PACKAGES_FOR_RB[@]}" & pids+=($!) + run_test_group "REGIONAL" "$FLAT" "${TEST_PACKAGES_FOR_RB[@]}" & pids+=($!) + if ! ${SKIP_EMULATOR}; then + run_e2e_tests_for_emulator & pids+=($!) # Emulator tests are a separate group + fi + fi + # Wait for all background processes to complete and aggregate their exit codes + for pid in "${pids[@]}"; do + wait "$pid" + overall_exit_code=$((overall_exit_code || $?)) + done + elapsed_min=$(((SECONDS + 60) / 60)) + log_info "------ E2E test packages complete run took ${elapsed_min} minutes ------" + log_info "" + + # Print package runtime stats table. + ./tools/integration_tests/create_package_runtime_table.sh "$PACKAGE_RUNTIME_STATS" + + if ${TRACK_RESOURCE_USAGE}; then + # Kill resource usage background PID and print resource usage. + log_info "Stopping resource usage collection process: $RESOURCE_USAGE_PID" + if safe_kill "$RESOURCE_USAGE_PID" "resource_usage.sh"; then + log_info "Resource usage collection process stopped." + ./tools/integration_tests/resource_usage.sh "PRINT" "$RESOURCE_USAGE_FILE" + else + log_error "Failed to stop resource usage collection process (or it's already stopped)" + fi + fi + exit $overall_exit_code +} + +#Main method to run script +main diff --git a/tools/integration_tests/inactive_stream_timeout/setup_test.go b/tools/integration_tests/inactive_stream_timeout/setup_test.go new file mode 100644 index 0000000000..bc4b5790e7 --- /dev/null +++ b/tools/integration_tests/inactive_stream_timeout/setup_test.go @@ -0,0 +1,204 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inactive_stream_timeout + +import ( + "bufio" + "context" + "fmt" + "log" + "os" + "path" + "strings" + "testing" + "time" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/dynamic_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/only_dir_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" + "github.com/stretchr/testify/require" +) + +const ( + kTestDirName = "inactiveReadTimeout" + kOnlyDirMounted = "onlyDirInactiveReadTimeout" + kFileSize = 10 * 1024 * 1024 // 10 MiB + kChunkSizeToRead = 128 * 1024 // 128 KiB + kDefaultInactiveReadTimeoutInSeconds = 1 // A short timeout for testing + GKETempDir = "/gcsfuse-tmp" + OldGKElogFilePath = "/tmp/inactive_stream_timeout_logs/log.json" + retryFrequency = 1 * time.Second + retryDuration = 30 * time.Second +) + +type env struct { + storageClient *storage.Client + ctx context.Context + testDirPath string + cfg *test_suite.TestConfig + bucketType string +} + +var ( + testEnv env + mountFunc func(*test_suite.TestConfig, []string) error + // mount directory is where our tests run. + mountDir string + // root directory is the directory to be unmounted. + rootDir string +) + +func mountGCSFuseAndSetupTestDir(flags []string, ctx context.Context, storageClient *storage.Client) { + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, flags, mountFunc) + // In case of GKE, the mount directory is not created by the test. + if testEnv.cfg.GKEMountedDirectory != "" { + setup.SetMntDir(testEnv.cfg.GKEMountedDirectory) + } + testEnv.testDirPath = client.SetupTestDirectory(ctx, storageClient, kTestDirName) +} + +// doesNotHaveInactiveReaderClosedLogLineInLogFile checks if the "Closing reader for object ... due to inactivity" +// log message is absent for the given objectName, b/w the [startTime, endTime] interval. +// It sleeps for 5 seconds before checking to allow logs to be flushed. +func doesNotHaveInactiveReaderClosedLogLineInLogFile(t *testing.T, objectName, logFile string, startTime, endTime time.Time) { + t.Helper() + time.Sleep(5 * time.Second) + + _, err := hasInactiveReaderClosedLogLineInLogFile(t, objectName, logFile, startTime, endTime) + if err == nil { + t.Fatalf("Unexpected 'Inactive Reader Closed' log message found in log file %s for object %s", logFile, objectName) + } +} + +// hasInactiveReaderClosedLogInLogFile checks if the "Closing reader for object ... due to inactivity" +// log message is present for the given objectName, b/w the [startTime, endTime] interval. +func hasInactiveReaderClosedLogLineInLogFile(t *testing.T, objectName, logFile string, startTime, endTime time.Time) (string, error) { + t.Helper() + expectedMsgSubstring := fmt.Sprintf("Closing reader for object %q due to inactivity.", objectName) + + file, err := os.Open(logFile) + require.NoError(t, err, "failed to open log file") + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + + // Pre-filter: skip lines that do not contain the core parts of the expected message. + // We cannot use `expectedMsgSubstring` directly for the pre-filter because the double + // quotes around the object name will be escaped as `\"` in the raw JSON string. + if !strings.Contains(line, "Closing reader for object") || !strings.Contains(line, objectName) { + continue + } + + logEntry, err := read_logs.ParseJsonLogLineIntoLogEntryStruct(line) + if err == nil && logEntry != nil { + if (logEntry.Timestamp.After(startTime) || logEntry.Timestamp.Equal(startTime)) && + (logEntry.Timestamp.Before(endTime) || logEntry.Timestamp.Equal(endTime)) { + if strings.Contains(logEntry.Message, expectedMsgSubstring) { + return line, nil + } + } + } + } + + if err := scanner.Err(); err != nil { + return "", fmt.Errorf("failed to read log file: %w", err) + } + + return "", fmt.Errorf("expected log message substring %q not found between %s and %s", expectedMsgSubstring, startTime, endTime) +} + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + // 1. read config file + configFile := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(configFile.InactiveStreamTimeout) == 0 { + log.Println("No configuration found for inactive_stream_timeout tests in config. Using default flags.") + configFile.InactiveStreamTimeout = make([]test_suite.TestConfig, 1) + testEnv.cfg = &configFile.InactiveStreamTimeout[0] + testEnv.cfg.TestBucket = setup.TestBucket() + testEnv.cfg.LogFile = setup.LogFile() + testEnv.cfg.GKEMountedDirectory = setup.MountedDirectory() + + testEnv.cfg.Configs = make([]test_suite.ConfigItem, 2) + testEnv.cfg.Configs[0].Flags = []string{ + "--read-inactive-stream-timeout=1s --client-protocol=http1 --log-format=json --log-file=/gcsfuse-tmp/TestTimeoutEnabledSuite.log", + "--read-inactive-stream-timeout=1s --client-protocol=grpc --log-format=json --log-file=/gcsfuse-tmp/TestTimeoutEnabledSuite.log", + } + testEnv.cfg.Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + testEnv.cfg.Configs[0].Run = "TestTimeoutEnabledSuite" + + testEnv.cfg.Configs[1].Flags = []string{ + "--read-inactive-stream-timeout=0s --client-protocol=http1 --log-format=json --log-file=/gcsfuse-tmp/TestTimeoutDisabledSuite.log", + } + testEnv.cfg.Configs[1].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + testEnv.cfg.Configs[1].Run = "TestTimeoutDisabledSuite" + } + testEnv.cfg = &configFile.InactiveStreamTimeout[0] + testEnv.ctx = context.Background() + testEnv.bucketType = setup.TestEnvironment(testEnv.ctx, testEnv.cfg) + + // 2. Create common storage client to be used in test. + var err error + testEnv.storageClient, err = client.CreateStorageClient(testEnv.ctx) + if err != nil { + log.Fatalf("client.CreateStorageClient: %v", err) + } + defer testEnv.storageClient.Close() + + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + mountDir = testEnv.cfg.GKEMountedDirectory + os.Exit(setup.RunTestsForMountedDirectory(testEnv.cfg.GKEMountedDirectory, m)) + } + + // Run tests for testBucket + // Set up test directory. + setup.SetUpTestDirForTestBucket(testEnv.cfg) + // Override GKE specific paths with GCSFuse paths if running in GCE environment. + setup.OverrideFilePathsInFlagSet(testEnv.cfg, setup.TestDir()) + // Save mount and root directory variables. + mountDir, rootDir = setup.MntDir(), setup.MntDir() + log.Println("Running static mounting tests...") + mountFunc = static_mounting.MountGcsfuseWithStaticMountingWithConfigFile + successCode := m.Run() + + if successCode == 0 { + log.Println("Running dynamic mounting tests...") + // Save mount directory variable to have path of bucket to run tests. + mountDir = path.Join(setup.MntDir(), setup.TestBucket()) + mountFunc = dynamic_mounting.MountGcsfuseWithDynamicMountingWithConfig + successCode = m.Run() + } + + if successCode == 0 { + log.Println("Running only dir mounting tests...") + setup.SetOnlyDirMounted(kOnlyDirMounted + "/") + mountDir = rootDir + mountFunc = only_dir_mounting.MountGcsfuseWithOnlyDirWithConfigFile + successCode = m.Run() + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(setup.TestBucket(), setup.OnlyDirMounted())) + } + + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(setup.TestBucket(), kTestDirName)) + os.Exit(successCode) +} diff --git a/tools/integration_tests/inactive_stream_timeout/with_timeout_test.go b/tools/integration_tests/inactive_stream_timeout/with_timeout_test.go new file mode 100644 index 0000000000..a29bee7fad --- /dev/null +++ b/tools/integration_tests/inactive_stream_timeout/with_timeout_test.go @@ -0,0 +1,140 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inactive_stream_timeout + +import ( + "context" + "log" + "path" + "testing" + "time" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type timeoutEnabledSuite struct { + flags []string + storageClient *storage.Client + ctx context.Context + baseTestName string + suite.Suite +} + +func (s *timeoutEnabledSuite) SetupSuite() { + setup.SetUpLogFilePath(s.T().Name(), s.flags, GKETempDir, OldGKElogFilePath, testEnv.cfg) + mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient) +} + +func (s *timeoutEnabledSuite) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *timeoutEnabledSuite) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +//////////////////////////////////////////////////////////////////////// +// Test scenarios +//////////////////////////////////////////////////////////////////////// + +func (s *timeoutEnabledSuite) TestReaderCloses() { + timeoutDuration := kDefaultInactiveReadTimeoutInSeconds * time.Second + testDir := path.Join(mountDir, kTestDirName) + fileName := "foo" + setup.GenerateRandomString(5) + client.SetupFileInTestDirectory(s.ctx, s.storageClient, kTestDirName, fileName, kFileSize, s.T()) + mountFilePath := path.Join(testDir, fileName) + + // 1. Open file. + fileHandle, err := operations.OpenFileAsReadonly(mountFilePath) + require.NoError(s.T(), err) + defer fileHandle.Close() + + // 2. Read small chunk from 0 offset. + buff := make([]byte, kChunkSizeToRead) + _, err = fileHandle.ReadAt(buff, 0) + require.NoError(s.T(), err) + endTimeRead := time.Now() + + // 3. Wait for timeout + time.Sleep(2*timeoutDuration + 1*time.Second) // Add buffer + endTimeWait := time.Now() + + s.T().Log("Waiting for 'Closing reader' log message...") + operations.RetryUntil(testEnv.ctx, s.T(), retryFrequency, retryDuration, func() (string, error) { + return hasInactiveReaderClosedLogLineInLogFile(s.T(), path.Join(kTestDirName, fileName), testEnv.cfg.LogFile, endTimeRead, endTimeWait) + }) + + // 4. Further reads should work as it is, yeah it will create a new reader. + _, err = fileHandle.ReadAt(buff, 8) + require.NoError(s.T(), err) +} + +func (s *timeoutEnabledSuite) TestReaderStaysOpenWithinTimeout() { + timeoutDuration := kDefaultInactiveReadTimeoutInSeconds * time.Second + testDir := path.Join(mountDir, kTestDirName) + fileName := "foo" + setup.GenerateRandomString(5) + client.SetupFileInTestDirectory(s.ctx, s.storageClient, kTestDirName, fileName, kFileSize, s.T()) + mountFilePath := path.Join(testDir, fileName) + + fileHandle, err := operations.OpenFileAsReadonly(mountFilePath) + require.NoError(s.T(), err) + defer fileHandle.Close() + + // 1. First read. + buff := make([]byte, kChunkSizeToRead) + _, err = fileHandle.ReadAt(buff, 0) + require.NoError(s.T(), err) + endTimeRead1 := time.Now() + + // 2. Wait for a period SHORTER than the timeout. + time.Sleep(timeoutDuration / 2) + startTimeRead2 := time.Now() + + // 3. Second read. + _, err = fileHandle.ReadAt(buff, int64(kChunkSizeToRead)) // Read the next chunk + require.NoError(s.T(), err, "Second read within timeout failed") + + // 4. Check log: "Closing reader for object..." should NOT be present for this object + // between the first read's end and the second read's start. + doesNotHaveInactiveReaderClosedLogLineInLogFile(s.T(), path.Join(kTestDirName, fileName), testEnv.cfg.LogFile, endTimeRead1, startTimeRead2) +} + +//////////////////////////////////////////////////////////////////////// +// Test Function (Runs once before all tests) +//////////////////////////////////////////////////////////////////////// + +func TestTimeoutEnabledSuite(t *testing.T) { + ts := &timeoutEnabledSuite{ + ctx: context.Background(), + storageClient: testEnv.storageClient, + baseTestName: t.Name(), + } + // Run tests for mounted directory if the flag is set. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { + log.Printf("Running inactive_read_timeout tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/inactive_stream_timeout/without_timeout_test.go b/tools/integration_tests/inactive_stream_timeout/without_timeout_test.go new file mode 100644 index 0000000000..54386324bf --- /dev/null +++ b/tools/integration_tests/inactive_stream_timeout/without_timeout_test.go @@ -0,0 +1,104 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inactive_stream_timeout + +import ( + "context" + "log" + "path" + "testing" + "time" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type timeoutDisabledSuite struct { + flags []string + storageClient *storage.Client + ctx context.Context + baseTestName string + suite.Suite +} + +func (s *timeoutDisabledSuite) SetupSuite() { + setup.SetUpLogFilePath(s.T().Name(), s.flags, GKETempDir, OldGKElogFilePath, testEnv.cfg) + mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient) +} + +func (s *timeoutDisabledSuite) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *timeoutDisabledSuite) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +//////////////////////////////////////////////////////////////////////// +// Test scenarios +//////////////////////////////////////////////////////////////////////// + +func (s *timeoutDisabledSuite) TestNoReaderCloser() { + timeoutDuration := kDefaultInactiveReadTimeoutInSeconds * time.Second + testDir := path.Join(mountDir, kTestDirName) + fileName := "foo" + setup.GenerateRandomString(5) + client.SetupFileInTestDirectory(s.ctx, s.storageClient, kTestDirName, fileName, kFileSize, s.T()) + mountFilePath := path.Join(testDir, fileName) + + // 1. Open file. + fileHandle, err := operations.OpenFileAsReadonly(mountFilePath) + require.NoError(s.T(), err) + defer fileHandle.Close() + + // 2. Read small chunk from 0 offset. + buff := make([]byte, kChunkSizeToRead) + _, err = fileHandle.ReadAt(buff, 0) + require.NoError(s.T(), err) + endTimeRead := time.Now() + + // 3. Wait for timeout + time.Sleep(2*timeoutDuration + 1*time.Second) // Add buffer + endTimeWait := time.Now() + + // 4. Shouldn't be any `Close reader logs...`. + doesNotHaveInactiveReaderClosedLogLineInLogFile(s.T(), path.Join(kTestDirName, fileName), testEnv.cfg.LogFile, endTimeRead, endTimeWait) +} + +//////////////////////////////////////////////////////////////////////// +// Test Function (Runs once before all tests) +//////////////////////////////////////////////////////////////////////// + +func TestTimeoutDisabledSuite(t *testing.T) { + ts := &timeoutDisabledSuite{ + ctx: context.Background(), + storageClient: testEnv.storageClient, + baseTestName: t.Name(), + } + // Run tests for mounted directory if the flag is set. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { + log.Printf("Running inactive_read_timeout tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/interrupt/git_clone_test.go b/tools/integration_tests/interrupt/git_clone_test.go index 8531f70107..273326614b 100644 --- a/tools/integration_tests/interrupt/git_clone_test.go +++ b/tools/integration_tests/interrupt/git_clone_test.go @@ -19,13 +19,16 @@ package interrupt import ( "log" + "math/rand" "path" + "strings" "testing" + "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/util" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/test_setup" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/suite" ) const ( @@ -44,11 +47,13 @@ var ( // Boilerplate //////////////////////////////////////////////////////////////////////// -type ignoreInterruptsTest struct{} +type ignoreInterruptsTest struct { + suite.Suite +} -func (s *ignoreInterruptsTest) Teardown(t *testing.T) {} +func (s *ignoreInterruptsTest) TearDownTest() {} -func (s *ignoreInterruptsTest) Setup(t *testing.T) { +func (s *ignoreInterruptsTest) SetupTest() { testDirPath = setup.SetupTestDirectory(testDirName) } @@ -56,8 +61,26 @@ func (s *ignoreInterruptsTest) Setup(t *testing.T) { // Helpers //////////////////////////////////////////////////////////////////////// -func cloneRepository() ([]byte, error) { - return operations.ExecuteToolCommandfInDirectory(testDirPath, tool, "clone %s", repoURL) +func (s *ignoreInterruptsTest) cloneRepository() (output []byte, err error) { + maxAttempts := 5 + isRetryableError := func(err error) bool { + lowerErr := strings.ToLower(err.Error()) + return strings.Contains(lowerErr, "could not resolve host") || strings.Contains(lowerErr, "could not read from remote repository") || strings.Contains(lowerErr, "failed to connect to github.com") + } + for i := range maxAttempts { + output, err = operations.ExecuteToolCommandfInDirectory(testDirPath, tool, "clone %s", repoURL) + + if err == nil || !isRetryableError(err) { + return + } + s.T().Logf("failed to clone %q with stdout = %q and retryable error = %v", repoURL, string(output), err) + if i < maxAttempts-1 { + // Wait for [1ms, 2000ms] before trying again. + time.Sleep(time.Millisecond * time.Duration(1+rand.Intn(2000))) + } + } + // All retries failed + return } func checkoutBranch(branchName string) ([]byte, error) { @@ -96,58 +119,58 @@ func setGithubUserConfig() { // Test scenarios //////////////////////////////////////////////////////////////////////// -func (s *ignoreInterruptsTest) TestGitClone(t *testing.T) { - output, err := cloneRepository() +func (s *ignoreInterruptsTest) TestGitClone() { + output, err := s.cloneRepository() if err != nil { - t.Errorf("Git clone failed: %s: %v", string(output), err) + s.T().Errorf("Git clone failed: %s: %v", string(output), err) } } -func (s *ignoreInterruptsTest) TestGitCheckout(t *testing.T) { - _, err := cloneRepository() +func (s *ignoreInterruptsTest) TestGitCheckout() { + _, err := s.cloneRepository() if err != nil { - t.Errorf("cloneRepository() failed: %v", err) + s.T().Errorf("cloneRepository() failed: %v", err) } output, err := checkoutBranch(branchName) if err != nil { - t.Errorf("Git checkout failed: %s: %v", string(output), err) + s.T().Errorf("Git checkout failed: %s: %v", string(output), err) } } -func (s *ignoreInterruptsTest) TestGitEmptyCommit(t *testing.T) { - _, err := cloneRepository() +func (s *ignoreInterruptsTest) TestGitEmptyCommit() { + _, err := s.cloneRepository() if err != nil { - t.Errorf("cloneRepository() failed: %v", err) + s.T().Errorf("cloneRepository() failed: %v", err) } setGithubUserConfig() output, err := emptyCommit() if err != nil { - t.Errorf("Git empty commit failed: %s: %v", string(output), err) + s.T().Errorf("Git empty commit failed: %s: %v", string(output), err) } } -func (s *ignoreInterruptsTest) TestGitCommitWithChanges(t *testing.T) { - _, err := cloneRepository() +func (s *ignoreInterruptsTest) TestGitCommitWithChanges() { + _, err := s.cloneRepository() if err != nil { - t.Errorf("cloneRepository() failed: %v", err) + s.T().Errorf("cloneRepository() failed: %v", err) } setGithubUserConfig() filePath := path.Join(testDirPath, repoName, testFileName) - operations.CreateFileOfSize(util.MiB, filePath, t) + operations.CreateFileOfSize(util.MiB, filePath, s.T()) output, err := gitAdd(filePath) if err != nil { - t.Errorf("Git add failed: %s: %v", string(output), err) + s.T().Errorf("Git add failed: %s: %v", string(output), err) } output, err = nonEmptyCommit() if err != nil { - t.Errorf("Git commit failed: %s: %v", string(output), err) + s.T().Errorf("Git commit failed: %s: %v", string(output), err) } } @@ -157,5 +180,5 @@ func (s *ignoreInterruptsTest) TestGitCommitWithChanges(t *testing.T) { func TestIgnoreInterrupts(t *testing.T) { ts := &ignoreInterruptsTest{} - test_setup.RunTests(t, ts) + suite.Run(t, ts) } diff --git a/tools/integration_tests/interrupt/interrupt_test.go b/tools/integration_tests/interrupt/interrupt_test.go index 8158ac4f79..62bbbe482b 100644 --- a/tools/integration_tests/interrupt/interrupt_test.go +++ b/tools/integration_tests/interrupt/interrupt_test.go @@ -22,15 +22,21 @@ import ( "testing" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" ) const ( testDirName = "InterruptTest" ) +var ( + storageClient *storage.Client + ctx context.Context +) + //////////////////////////////////////////////////////////////////////// // TestMain //////////////////////////////////////////////////////////////////////// @@ -38,8 +44,30 @@ const ( func TestMain(m *testing.M) { setup.ParseSetUpFlags() - var storageClient *storage.Client - ctx := context.Background() + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.Interrupt) == 0 { + log.Println("No configuration found for interrupt tests in config. Using flags instead.") + // Populate the config manually. + cfg.Interrupt = make([]test_suite.TestConfig, 1) + cfg.Interrupt[0].TestBucket = setup.TestBucket() + cfg.Interrupt[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.Interrupt[0].Configs = make([]test_suite.ConfigItem, 2) + cfg.Interrupt[0].Configs[0].Flags = []string{ + "--implicit-dirs=true --enable-streaming-writes=false", + "--ignore-interrupts=true --enable-streaming-writes=false", + "--ignore-interrupts=false --enable-streaming-writes=false", + } + cfg.Interrupt[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.Interrupt[0].Configs[1].Flags = []string{ + "--enable-streaming-writes=true", + } + cfg.Interrupt[0].Configs[1].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": false} + } + + // 2. Create storage client before running tests. + ctx = context.Background() + bucketType := setup.TestEnvironment(ctx, &cfg.Interrupt[0]) closeStorageClient := client.CreateStorageClientWithCancel(&ctx, &storageClient) defer func() { err := closeStorageClient() @@ -48,27 +76,21 @@ func TestMain(m *testing.M) { } }() - setup.RunTestsForMountedDirectoryFlag(m) + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + // flags to be set, as Interrupt tests validates content from the bucket. + if cfg.Interrupt[0].GKEMountedDirectory != "" && cfg.Interrupt[0].TestBucket != "" { + os.Exit(setup.RunTestsForMountedDirectory(cfg.Interrupt[0].GKEMountedDirectory, m)) + } - // Else run tests for testBucket. - // Set up test directory. - setup.SetUpTestDirForTestBucketFlag() + // Run tests for testBucket + // 4. Build the flag sets dynamically from the config. + flags := setup.BuildFlagSets(cfg.Interrupt[0], bucketType, "") - // Set up flags to run tests on. - yamlContent1 := map[string]interface{}{ - "file-system": map[string]interface{}{ - "ignore-interrupts": true, - }, - } - yamlContent2 := map[string]interface{}{} // test default - flags := [][]string{ - {"--implicit-dirs=true"}, - {"--config-file=" + setup.YAMLConfigFile(yamlContent1, "ignore_interrupts.yaml")}, - {"--config-file=" + setup.YAMLConfigFile(yamlContent2, "default_ignore_interrupts.yaml")}} + setup.SetUpTestDirForTestBucket(&cfg.Interrupt[0]) - successCode := static_mounting.RunTests(flags, m) + successCode := static_mounting.RunTestsWithConfigFile(&cfg.Interrupt[0], flags, m) - // Clean up test directory created. setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(setup.TestBucket(), testDirName)) os.Exit(successCode) + } diff --git a/tools/integration_tests/kernel_list_cache/disabled_kernel_list_cache_test.go b/tools/integration_tests/kernel_list_cache/disabled_kernel_list_cache_test.go index f3756293b1..51079db468 100644 --- a/tools/integration_tests/kernel_list_cache/disabled_kernel_list_cache_test.go +++ b/tools/integration_tests/kernel_list_cache/disabled_kernel_list_cache_test.go @@ -20,12 +20,12 @@ import ( "path" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/test_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" ) //////////////////////////////////////////////////////////////////////// @@ -34,55 +34,65 @@ import ( type disabledKernelListCacheTest struct { flags []string + suite.Suite } -func (s *disabledKernelListCacheTest) Setup(t *testing.T) { - mountGCSFuseAndSetupTestDir(s.flags, ctx, storageClient, testDirName) +func (s *disabledKernelListCacheTest) SetupSuite() { + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, s.flags, mountFunc) + setup.SetMntDir(mountDir) } -func (s *disabledKernelListCacheTest) Teardown(t *testing.T) { - setup.UnmountGCSFuse(rootDir) +func (s *disabledKernelListCacheTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *disabledKernelListCacheTest) SetupTest() { + testEnv.testDirPath = setup.SetupTestDirectory(testDirName) +} + +func (s *disabledKernelListCacheTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) } //////////////////////////////////////////////////////////////////////// // Test scenarios //////////////////////////////////////////////////////////////////////// -func (s *disabledKernelListCacheTest) TestKernelListCache_AlwaysCacheMiss(t *testing.T) { - targetDir := path.Join(testDirPath, "explicit_dir") - operations.CreateDirectory(targetDir, t) +func (s *disabledKernelListCacheTest) TestKernelListCache_AlwaysCacheMiss() { + targetDir := path.Join(testEnv.testDirPath, "explicit_dir") + operations.CreateDirectory(targetDir, s.T()) // Create test data - f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f1) - f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f2) + f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f1) + f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f2) // First read, kernel will cache the dir response. f, err := os.Open(targetDir) - require.NoError(t, err) + require.NoError(s.T(), err) defer func() { - assert.Nil(t, f.Close()) + assert.Nil(s.T(), f.Close()) }() names1, err := f.Readdirnames(-1) - assert.Nil(t, err) - require.Equal(t, 2, len(names1)) - require.Equal(t, "file1.txt", names1[0]) - require.Equal(t, "file2.txt", names1[1]) + assert.Nil(s.T(), err) + require.Equal(s.T(), 2, len(names1)) + require.Equal(s.T(), "file1.txt", names1[0]) + require.Equal(s.T(), "file2.txt", names1[1]) err = f.Close() - require.NoError(t, err) + require.NoError(s.T(), err) // Adding one object to make sure to change the ReadDir() response. - client.CreateObjectInGCSTestDir(ctx, storageClient, testDirName, path.Join("explicit_dir", "file3.txt"), "", t) + client.CreateObjectInGCSTestDir(testEnv.ctx, testEnv.storageClient, testDirName, path.Join("explicit_dir", "file3.txt"), "", s.T()) // Zero ttl, means readdir will always be served from gcsfuse. f, err = os.Open(targetDir) - assert.NoError(t, err) + assert.NoError(s.T(), err) names2, err := f.Readdirnames(-1) - assert.NoError(t, err) + assert.NoError(s.T(), err) - require.Equal(t, 3, len(names2)) - assert.Equal(t, "file1.txt", names2[0]) - assert.Equal(t, "file2.txt", names2[1]) - assert.Equal(t, "file3.txt", names2[2]) + require.Equal(s.T(), 3, len(names2)) + assert.Equal(s.T(), "file1.txt", names2[0]) + assert.Equal(s.T(), "file2.txt", names2[1]) + assert.Equal(s.T(), "file3.txt", names2[2]) } //////////////////////////////////////////////////////////////////////// @@ -92,21 +102,16 @@ func (s *disabledKernelListCacheTest) TestKernelListCache_AlwaysCacheMiss(t *tes func TestDisabledKernelListCacheTest(t *testing.T) { ts := &disabledKernelListCacheTest{} - // Run tests for mounted directory if the flag is set. - if setup.AreBothMountedDirectoryAndTestBucketFlagsSet() { - test_setup.RunTests(t, ts) + // Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) return } - // Define flag set to run the tests. - flagsSet := [][]string{ - {"--kernel-list-cache-ttl-secs=0", "--stat-cache-ttl=0", "--rename-dir-limit=10"}, - } - - // Run tests. - for _, flags := range flagsSet { - ts.flags = flags + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { log.Printf("Running tests with flags: %s", ts.flags) - test_setup.RunTests(t, ts) + suite.Run(t, ts) } } diff --git a/tools/integration_tests/kernel_list_cache/finite_kernel_list_cache_test.go b/tools/integration_tests/kernel_list_cache/finite_kernel_list_cache_test.go index 65e223dcfe..3b34580489 100644 --- a/tools/integration_tests/kernel_list_cache/finite_kernel_list_cache_test.go +++ b/tools/integration_tests/kernel_list_cache/finite_kernel_list_cache_test.go @@ -21,12 +21,12 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/test_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" ) //////////////////////////////////////////////////////////////////////// @@ -35,68 +35,80 @@ import ( type finiteKernelListCacheTest struct { flags []string + suite.Suite } -func (s *finiteKernelListCacheTest) Setup(t *testing.T) { - mountGCSFuseAndSetupTestDir(s.flags, ctx, storageClient, testDirName) +func (s *finiteKernelListCacheTest) SetupSuite() { + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, s.flags, mountFunc) + setup.SetMntDir(mountDir) } -func (s *finiteKernelListCacheTest) Teardown(t *testing.T) { - setup.UnmountGCSFuse(rootDir) +func (s *finiteKernelListCacheTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *finiteKernelListCacheTest) SetupTest() { + testEnv.testDirPath = setup.SetupTestDirectory(testDirName) +} + +func (s *finiteKernelListCacheTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) } //////////////////////////////////////////////////////////////////////// // Test scenarios //////////////////////////////////////////////////////////////////////// -func (s *finiteKernelListCacheTest) TestKernelListCache_CacheHitWithinLimit_CacheMissAfterLimit(t *testing.T) { - targetDir := path.Join(testDirPath, "explicit_dir") - operations.CreateDirectory(targetDir, t) +func (s *finiteKernelListCacheTest) TestKernelListCache_CacheHitWithinLimit_CacheMissAfterLimit() { + operations.SkipKLCTestForUnsupportedKernelVersion(s.T()) + + targetDir := path.Join(testEnv.testDirPath, "explicit_dir") + operations.CreateDirectory(targetDir, s.T()) // Create test data - f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f1) - f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f2) + f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f1) + f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f2) // First read, kernel will cache the dir response. f, err := os.Open(targetDir) - require.NoError(t, err) + require.NoError(s.T(), err) defer func() { - assert.Nil(t, f.Close()) + assert.Nil(s.T(), f.Close()) }() names1, err := f.Readdirnames(-1) - require.NoError(t, err) - require.Equal(t, 2, len(names1)) - require.Equal(t, "file1.txt", names1[0]) - require.Equal(t, "file2.txt", names1[1]) + require.NoError(s.T(), err) + require.Equal(s.T(), 2, len(names1)) + require.Equal(s.T(), "file1.txt", names1[0]) + require.Equal(s.T(), "file2.txt", names1[1]) err = f.Close() - require.NoError(t, err) + require.NoError(s.T(), err) // Adding one object to make sure to change the ReadDir() response. - client.CreateObjectInGCSTestDir(ctx, storageClient, testDirName, path.Join("explicit_dir", "file3.txt"), "", t) + client.CreateObjectInGCSTestDir(testEnv.ctx, testEnv.storageClient, testDirName, path.Join("explicit_dir", "file3.txt"), "", s.T()) time.Sleep(2 * time.Second) // Kernel cache will not invalidate within ttl. f, err = os.Open(targetDir) - assert.NoError(t, err) + assert.NoError(s.T(), err) names2, err := f.Readdirnames(-1) - assert.NoError(t, err) - require.Equal(t, 2, len(names2)) - assert.Equal(t, "file1.txt", names2[0]) - assert.Equal(t, "file2.txt", names2[1]) + assert.NoError(s.T(), err) + require.Equal(s.T(), 2, len(names2)) + assert.Equal(s.T(), "file1.txt", names2[0]) + assert.Equal(s.T(), "file2.txt", names2[1]) // Waiting 3 more seconds to exceed the 5-second TTL for invalidating the kernel cache. time.Sleep(3 * time.Second) // The response will be served from GCSFuse after the TTL expires. f, err = os.Open(targetDir) - assert.NoError(t, err) + assert.NoError(s.T(), err) names3, err := f.Readdirnames(-1) - assert.NoError(t, err) + assert.NoError(s.T(), err) - require.Equal(t, 3, len(names3)) - assert.Equal(t, "file1.txt", names3[0]) - assert.Equal(t, "file2.txt", names3[1]) - assert.Equal(t, "file3.txt", names3[2]) + require.Equal(s.T(), 3, len(names3)) + assert.Equal(s.T(), "file1.txt", names3[0]) + assert.Equal(s.T(), "file2.txt", names3[1]) + assert.Equal(s.T(), "file3.txt", names3[2]) } //////////////////////////////////////////////////////////////////////// @@ -106,21 +118,16 @@ func (s *finiteKernelListCacheTest) TestKernelListCache_CacheHitWithinLimit_Cach func TestFiniteKernelListCacheTest(t *testing.T) { ts := &finiteKernelListCacheTest{} - // Run tests for mounted directory if the flag is set. - if setup.AreBothMountedDirectoryAndTestBucketFlagsSet() { - test_setup.RunTests(t, ts) + // Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) return } - // Define flag set to run the tests. - flagsSet := [][]string{ - {"--kernel-list-cache-ttl-secs=5", "--rename-dir-limit=10"}, - } - - // Run tests. - for _, flags := range flagsSet { - ts.flags = flags + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { log.Printf("Running tests with flags: %s", ts.flags) - test_setup.RunTests(t, ts) + suite.Run(t, ts) } } diff --git a/tools/integration_tests/kernel_list_cache/infinite_kernel_list_cache_delete_dir_test.go b/tools/integration_tests/kernel_list_cache/infinite_kernel_list_cache_delete_dir_test.go new file mode 100644 index 0000000000..a124ed3140 --- /dev/null +++ b/tools/integration_tests/kernel_list_cache/infinite_kernel_list_cache_delete_dir_test.go @@ -0,0 +1,141 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kernel_list_cache + +import ( + "log" + "os" + "path" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +type infiniteKernelListCacheDeleteDirTest struct { + flags []string + suite.Suite +} + +func (s *infiniteKernelListCacheDeleteDirTest) SetupSuite() { + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, s.flags, mountFunc) + setup.SetMntDir(mountDir) +} + +func (s *infiniteKernelListCacheDeleteDirTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *infiniteKernelListCacheDeleteDirTest) SetupTest() { + testEnv.testDirPath = setup.SetupTestDirectory(testDirName) +} + +func (s *infiniteKernelListCacheDeleteDirTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +//////////////////////////////////////////////////////////////////////// +// Test scenarios +//////////////////////////////////////////////////////////////////////// + +func (s *infiniteKernelListCacheDeleteDirTest) TestKernelListCache_ListAndDeleteDirectory() { + targetDir := path.Join(testEnv.testDirPath, "explicit_dir") + operations.CreateDirectory(targetDir, s.T()) + // Create test data + f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f1) + f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f2) + + // (a) First read served from GCS, kernel will cache the dir response. + f, err := os.Open(targetDir) + assert.NoError(s.T(), err) + names1, err := f.Readdirnames(-1) + assert.NoError(s.T(), err) + require.Equal(s.T(), 2, len(names1)) + assert.Equal(s.T(), "file1.txt", names1[0]) + assert.Equal(s.T(), "file2.txt", names1[1]) + err = f.Close() + assert.NoError(s.T(), err) + // Adding one object to make sure to change the ReadDir() response. + // All files including file3.txt will be deleted by os.RemoveAll + client.CreateObjectInGCSTestDir(testEnv.ctx, testEnv.storageClient, testDirName, path.Join("explicit_dir", "file3.txt"), "", s.T()) + + err = os.RemoveAll(targetDir) + + assert.NoError(s.T(), err) +} + +func (s *infiniteKernelListCacheDeleteDirTest) TestKernelListCache_DeleteAndListDirectory() { + targetDir := path.Join(testEnv.testDirPath, "explicit_dir") + operations.CreateDirectory(targetDir, s.T()) + // Create test data + f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f1) + f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f2) + + err := os.RemoveAll(targetDir) + assert.NoError(s.T(), err) + + // Adding object to GCS to make sure to change the ReadDir() response. + err = client.CreateObjectOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(testDirName, "explicit_dir")+"/", "") + require.NoError(s.T(), err) + client.CreateObjectInGCSTestDir(testEnv.ctx, testEnv.storageClient, testDirName, path.Join("explicit_dir", "file3.txt"), "", s.T()) + // Read will be served from GCS as removing the directory also deletes the cache. + f, err := os.Open(targetDir) + assert.NoError(s.T(), err) + names1, err := f.Readdirnames(-1) + assert.NoError(s.T(), err) + require.Equal(s.T(), 1, len(names1)) + assert.Equal(s.T(), "file3.txt", names1[0]) + err = f.Close() + assert.NoError(s.T(), err) + + // 2nd RemoveAll call will also succeed. + err = os.RemoveAll(targetDir) + assert.NoError(s.T(), err) +} + +//////////////////////////////////////////////////////////////////////// +// Test Function (Runs once before all tests) +//////////////////////////////////////////////////////////////////////// + +func TestInfiniteKernelListCacheDeleteDirTest(t *testing.T) { + operations.SkipKLCTestForUnsupportedKernelVersion(t) + + ts := &infiniteKernelListCacheDeleteDirTest{} + + // Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/kernel_list_cache/infinite_kernel_list_cache_test.go b/tools/integration_tests/kernel_list_cache/infinite_kernel_list_cache_test.go index 87bcafa8b9..aaacafc3af 100644 --- a/tools/integration_tests/kernel_list_cache/infinite_kernel_list_cache_test.go +++ b/tools/integration_tests/kernel_list_cache/infinite_kernel_list_cache_test.go @@ -21,12 +21,12 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/test_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" ) //////////////////////////////////////////////////////////////////////// @@ -35,197 +35,211 @@ import ( type infiniteKernelListCacheTest struct { flags []string + suite.Suite } -func (s *infiniteKernelListCacheTest) Setup(t *testing.T) { - mountGCSFuseAndSetupTestDir(s.flags, ctx, storageClient, testDirName) +func (s *infiniteKernelListCacheTest) SetupSuite() { + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, s.flags, mountFunc) + setup.SetMntDir(mountDir) } -func (s *infiniteKernelListCacheTest) Teardown(t *testing.T) { - setup.UnmountGCSFuse(rootDir) +func (s *infiniteKernelListCacheTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *infiniteKernelListCacheTest) SetupTest() { + testEnv.testDirPath = setup.SetupTestDirectory(testDirName) +} + +func (s *infiniteKernelListCacheTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) } //////////////////////////////////////////////////////////////////////// // Test scenarios //////////////////////////////////////////////////////////////////////// -func (s *infiniteKernelListCacheTest) TestKernelListCache_AlwaysCacheHit(t *testing.T) { - targetDir := path.Join(testDirPath, "explicit_dir") - operations.CreateDirectory(targetDir, t) +func (s *infiniteKernelListCacheTest) TestKernelListCache_AlwaysCacheHit() { + explicit_dir := "explicit_dir" + setup.GenerateRandomString(5) + targetDir := path.Join(testEnv.testDirPath, explicit_dir) + operations.CreateDirectory(targetDir, s.T()) // Create test data - f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f1) - f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f2) + f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f1) + f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f2) // First read, kernel will cache the dir response. f, err := os.Open(targetDir) - require.NoError(t, err) + require.NoError(s.T(), err) defer func() { - assert.Nil(t, f.Close()) + assert.Nil(s.T(), f.Close()) }() names1, err := f.Readdirnames(-1) - require.NoError(t, err) - require.Equal(t, 2, len(names1)) - require.Equal(t, "file1.txt", names1[0]) - require.Equal(t, "file2.txt", names1[1]) + require.NoError(s.T(), err) + require.Equal(s.T(), 2, len(names1)) + require.Equal(s.T(), "file1.txt", names1[0]) + require.Equal(s.T(), "file2.txt", names1[1]) err = f.Close() - require.NoError(t, err) + require.NoError(s.T(), err) // Adding one object to make sure to change the ReadDir() response. - client.CreateObjectInGCSTestDir(ctx, storageClient, testDirName, path.Join("explicit_dir", "file3.txt"), "", t) + client.CreateObjectInGCSTestDir(testEnv.ctx, testEnv.storageClient, testDirName, path.Join(explicit_dir, "file3.txt"), "", s.T()) // Waiting for 5 seconds to see if the kernel cache expires. time.Sleep(5 * time.Second) // Kernel cache will not invalidate since infinite ttl. f, err = os.Open(targetDir) - assert.NoError(t, err) + assert.NoError(s.T(), err) names2, err := f.Readdirnames(-1) - assert.NoError(t, err) + assert.NoError(s.T(), err) - require.Equal(t, 2, len(names2)) - assert.Equal(t, "file1.txt", names2[0]) - assert.Equal(t, "file2.txt", names2[1]) + require.Equal(s.T(), 2, len(names2)) + assert.Equal(s.T(), "file1.txt", names2[0]) + assert.Equal(s.T(), "file2.txt", names2[1]) } // (a) First ReadDir() will be served from GCSFuse filesystem. // (b) Second ReadDir() will also be served from GCSFuse filesystem, because of // addition of new file. -func (s *infiniteKernelListCacheTest) TestKernelListCache_CacheMissOnAdditionOfFile(t *testing.T) { - targetDir := path.Join(testDirPath, "explicit_dir") - operations.CreateDirectory(targetDir, t) +func (s *infiniteKernelListCacheTest) TestKernelListCache_CacheMissOnAdditionOfFile() { + explicit_dir := "explicit_dir" + setup.GenerateRandomString(5) + targetDir := path.Join(testEnv.testDirPath, explicit_dir) + operations.CreateDirectory(targetDir, s.T()) // Create test data - f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f1) - f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f2) + f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f1) + f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f2) // First read, kernel will cache the dir response. f, err := os.Open(targetDir) - require.NoError(t, err) + require.NoError(s.T(), err) defer func() { - assert.NoError(t, f.Close()) + assert.NoError(s.T(), f.Close()) }() names1, err := f.Readdirnames(-1) - require.NoError(t, err) - require.Equal(t, 2, len(names1)) - require.Equal(t, "file1.txt", names1[0]) - require.Equal(t, "file2.txt", names1[1]) + require.NoError(s.T(), err) + require.Equal(s.T(), 2, len(names1)) + require.Equal(s.T(), "file1.txt", names1[0]) + require.Equal(s.T(), "file2.txt", names1[1]) err = f.Close() - require.NoError(t, err) + require.NoError(s.T(), err) // Adding one object to make sure to change the ReadDir() response. - client.CreateObjectInGCSTestDir(ctx, storageClient, testDirName, path.Join("explicit_dir", "file3.txt"), "", t) + client.CreateObjectInGCSTestDir(testEnv.ctx, testEnv.storageClient, testDirName, path.Join(explicit_dir, "file3.txt"), "", s.T()) // Ideally no invalidation since infinite ttl, but creation of a new file inside // directory evicts the list cache for that directory. fNew, err := os.Create(path.Join(targetDir, "file4.txt")) - require.NoError(t, err) - assert.NotNil(t, fNew) + require.NoError(s.T(), err) + assert.NotNil(s.T(), fNew) defer func() { - assert.NoError(t, fNew.Close()) - assert.NoError(t, os.Remove(path.Join(targetDir, "file4.txt"))) + assert.NoError(s.T(), fNew.Close()) + assert.NoError(s.T(), os.Remove(path.Join(targetDir, "file4.txt"))) }() - f, err = os.Open(path.Join(testDirPath, "explicit_dir")) - assert.NoError(t, err) + f, err = os.Open(path.Join(testEnv.testDirPath, explicit_dir)) + assert.NoError(s.T(), err) names2, err := f.Readdirnames(-1) - assert.NoError(t, err) - require.Equal(t, 4, len(names2)) - assert.Equal(t, "file1.txt", names2[0]) - assert.Equal(t, "file2.txt", names2[1]) - assert.Equal(t, "file3.txt", names2[2]) - assert.Equal(t, "file4.txt", names2[3]) + assert.NoError(s.T(), err) + require.Equal(s.T(), 4, len(names2)) + assert.Equal(s.T(), "file1.txt", names2[0]) + assert.Equal(s.T(), "file2.txt", names2[1]) + assert.Equal(s.T(), "file3.txt", names2[2]) + assert.Equal(s.T(), "file4.txt", names2[3]) } // (a) First ReadDir() will be served from GCSFuse filesystem. // (b) Second ReadDir() will also be served from GCSFuse filesystem, because of // deletion of new file. -func (s *infiniteKernelListCacheTest) TestKernelListCache_CacheMissOnDeletionOfFile(t *testing.T) { - targetDir := path.Join(testDirPath, "explicit_dir") - operations.CreateDirectory(targetDir, t) +func (s *infiniteKernelListCacheTest) TestKernelListCache_CacheMissOnDeletionOfFile() { + explicit_dir := "explicit_dir" + setup.GenerateRandomString(5) + targetDir := path.Join(testEnv.testDirPath, explicit_dir) + operations.CreateDirectory(targetDir, s.T()) // Create test data - f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f1) - f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f2) + f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f1) + f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f2) // First read, kernel will cache the dir response. f, err := os.Open(targetDir) - assert.NoError(t, err) + assert.NoError(s.T(), err) defer func() { - assert.NoError(t, f.Close()) + assert.NoError(s.T(), f.Close()) }() names1, err := f.Readdirnames(-1) - assert.NoError(t, err) - require.Equal(t, 2, len(names1)) - require.Equal(t, "file1.txt", names1[0]) - require.Equal(t, "file2.txt", names1[1]) + assert.NoError(s.T(), err) + require.Equal(s.T(), 2, len(names1)) + require.Equal(s.T(), "file1.txt", names1[0]) + require.Equal(s.T(), "file2.txt", names1[1]) err = f.Close() - assert.NoError(t, err) + assert.NoError(s.T(), err) // Adding one object to make sure to change the ReadDir() response. - client.CreateObjectInGCSTestDir(ctx, storageClient, testDirName, path.Join("explicit_dir", "file3.txt"), "", t) + client.CreateObjectInGCSTestDir(testEnv.ctx, testEnv.storageClient, testDirName, path.Join(explicit_dir, "file3.txt"), "", s.T()) // Ideally no invalidation since infinite ttl, but deletion of file inside // directory evicts the list cache for that directory. err = os.Remove(path.Join(targetDir, "file2.txt")) - require.NoError(t, err) + require.NoError(s.T(), err) f, err = os.Open(targetDir) - assert.NoError(t, err) + assert.NoError(s.T(), err) names2, err := f.Readdirnames(-1) - assert.NoError(t, err) - require.Equal(t, 2, len(names2)) - assert.Equal(t, "file1.txt", names2[0]) - assert.Equal(t, "file3.txt", names2[1]) + assert.NoError(s.T(), err) + require.Equal(s.T(), 2, len(names2)) + assert.Equal(s.T(), "file1.txt", names2[0]) + assert.Equal(s.T(), "file3.txt", names2[1]) } // (a) First ReadDir() will be served from GCSFuse filesystem. // (b) Second ReadDir() will also be served from GCSFuse filesystem, because of // file rename. -func (s *infiniteKernelListCacheTest) TestKernelListCache_CacheMissOnFileRename(t *testing.T) { - targetDir := path.Join(testDirPath, "explicit_dir") - operations.CreateDirectory(targetDir, t) +func (s *infiniteKernelListCacheTest) TestKernelListCache_CacheMissOnFileRename() { + explicit_dir := "explicit_dir" + setup.GenerateRandomString(5) + targetDir := path.Join(testEnv.testDirPath, explicit_dir) + operations.CreateDirectory(targetDir, s.T()) // Create test data - f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f1) - f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f2) + f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f1) + f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f2) // First read, kernel will cache the dir response. f, err := os.Open(targetDir) - assert.NoError(t, err) + assert.NoError(s.T(), err) defer func() { - assert.NoError(t, f.Close()) + assert.NoError(s.T(), f.Close()) }() names1, err := f.Readdirnames(-1) - assert.NoError(t, err) - require.Equal(t, 2, len(names1)) - require.Equal(t, "file1.txt", names1[0]) - require.Equal(t, "file2.txt", names1[1]) + assert.NoError(s.T(), err) + require.Equal(s.T(), 2, len(names1)) + require.Equal(s.T(), "file1.txt", names1[0]) + require.Equal(s.T(), "file2.txt", names1[1]) err = f.Close() - assert.NoError(t, err) + assert.NoError(s.T(), err) // Adding one object to make sure to change the ReadDir() response. - client.CreateObjectInGCSTestDir(ctx, storageClient, testDirName, path.Join("explicit_dir", "file3.txt"), "", t) + client.CreateObjectInGCSTestDir(testEnv.ctx, testEnv.storageClient, testDirName, path.Join(explicit_dir, "file3.txt"), "", s.T()) // Ideally no invalidation since infinite ttl, but rename of a file inside // directory evicts the list cache for that directory. err = os.Rename(path.Join(targetDir, "file2.txt"), path.Join(targetDir, "renamed_file2.txt")) - require.NoError(t, err) + require.NoError(s.T(), err) f, err = os.Open(targetDir) - require.NoError(t, err) + require.NoError(s.T(), err) names2, err := f.Readdirnames(-1) - assert.NoError(t, err) - require.Equal(t, 3, len(names2)) - assert.Equal(t, "file1.txt", names2[0]) - assert.Equal(t, "file3.txt", names2[1]) - assert.Equal(t, "renamed_file2.txt", names2[2]) + assert.NoError(s.T(), err) + require.Equal(s.T(), 3, len(names2)) + assert.Equal(s.T(), "file1.txt", names2[0]) + assert.Equal(s.T(), "file3.txt", names2[1]) + assert.Equal(s.T(), "renamed_file2.txt", names2[2]) } // explicit_dir/file1.txt @@ -250,266 +264,211 @@ func (s *infiniteKernelListCacheTest) TestKernelListCache_CacheMissOnFileRename( // // ls explicit_dir/sub_dir // file2.txt, file3.txt, file4.txt, file5.txt -func (s *infiniteKernelListCacheTest) TestKernelListCache_EvictCacheEntryOfOnlyDirectParent(t *testing.T) { - targetDir := path.Join(testDirPath, "explicit_dir") - operations.CreateDirectory(targetDir, t) +func (s *infiniteKernelListCacheTest) TestKernelListCache_EvictCacheEntryOfOnlyDirectParent() { + explicit_dir := "explicit_dir" + setup.GenerateRandomString(5) + targetDir := path.Join(testEnv.testDirPath, explicit_dir) + operations.CreateDirectory(targetDir, s.T()) subDir := path.Join(targetDir, "sub_dir") - operations.CreateDirectory(subDir, t) + operations.CreateDirectory(subDir, s.T()) // Create test files - f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f1) - f2 := operations.CreateFile(path.Join(subDir, "file2.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f2) - f3 := operations.CreateFile(path.Join(subDir, "file3.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f3) + f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f1) + f2 := operations.CreateFile(path.Join(subDir, "file2.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f2) + f3 := operations.CreateFile(path.Join(subDir, "file3.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f3) // Initial read of parent directory (caches results) f, err := os.Open(targetDir) - require.NoError(t, err) + require.NoError(s.T(), err) names1, err := f.Readdirnames(-1) // Read all filenames - require.NoError(t, err) - require.NoError(t, f.Close()) - require.Equal(t, 2, len(names1)) - require.Equal(t, "file1.txt", names1[0]) - require.Equal(t, "sub_dir", names1[1]) - // Initial read of sub-directory (caches results) + require.NoError(s.T(), err) + require.NoError(s.T(), f.Close()) + require.Equal(s.T(), 2, len(names1)) + require.Equal(s.T(), "file1.txt", names1[0]) + require.Equal(s.T(), "sub_dir", names1[1]) + // Initial read of subdirectory (caches results) f, err = os.Open(subDir) - require.NoError(t, err) + require.NoError(s.T(), err) names2, err := f.Readdirnames(-1) - require.NoError(t, err) - require.NoError(t, f.Close()) - require.Equal(t, 2, len(names2)) - assert.Equal(t, "file2.txt", names2[0]) - assert.Equal(t, "file3.txt", names2[1]) - // Add a new file to the sub-directory to trigger a cache invalidation scenario + require.NoError(s.T(), err) + require.NoError(s.T(), f.Close()) + require.Equal(s.T(), 2, len(names2)) + assert.Equal(s.T(), "file2.txt", names2[0]) + assert.Equal(s.T(), "file3.txt", names2[1]) + // Add a new file to the subdirectory to trigger a cache invalidation scenario fNew, err := os.Create(path.Join(subDir, "file4.txt")) - require.NoError(t, err) - require.NoError(t, fNew.Close()) - client.CreateObjectInGCSTestDir(ctx, storageClient, testDirName, - path.Join("explicit_dir", "sub_dir", "file5.txt"), "", t) + require.NoError(s.T(), err) + require.NoError(s.T(), fNew.Close()) + client.CreateObjectInGCSTestDir(testEnv.ctx, testEnv.storageClient, testDirName, + path.Join(explicit_dir, "sub_dir", "file5.txt"), "", s.T()) // Add a new file to the parent directory through the client to verify that the // cache is not invalidated in the case of the parent. - client.CreateObjectInGCSTestDir(ctx, storageClient, testDirName, - path.Join("explicit_dir", "file6.txt"), "", t) + client.CreateObjectInGCSTestDir(testEnv.ctx, testEnv.storageClient, testDirName, + path.Join(explicit_dir, "file6.txt"), "", s.T()) // Re-read parent directory (should still use the cache and NOT show the change in the sub-dir) f1, err = os.Open(targetDir) - require.NoError(t, err) + require.NoError(s.T(), err) names1, err = f1.Readdirnames(-1) - require.NoError(t, err) - require.NoError(t, f1.Close()) - // Re-read sub-directory (cache should be invalidated and show the new file) + require.NoError(s.T(), err) + require.NoError(s.T(), f1.Close()) + // Re-read subdirectory (cache should be invalidated and show the new file) f2, err = os.Open(subDir) - require.NoError(t, err) + require.NoError(s.T(), err) names2, err = f2.Readdirnames(-1) - require.NoError(t, f2.Close()) - require.NoError(t, err) + require.NoError(s.T(), f2.Close()) + require.NoError(s.T(), err) // This is expected to be 2 as it is reading from cache for parent directory - require.Equal(t, 2, len(names1)) - assert.Equal(t, "file1.txt", names1[0]) - assert.Equal(t, "sub_dir", names1[1]) + require.Equal(s.T(), 2, len(names1)) + assert.Equal(s.T(), "file1.txt", names1[0]) + assert.Equal(s.T(), "sub_dir", names1[1]) // Cache invalidated, expect 4 items now as call went to GCS - require.Equal(t, 4, len(names2)) - assert.Equal(t, "file2.txt", names2[0]) - assert.Equal(t, "file3.txt", names2[1]) - assert.Equal(t, "file4.txt", names2[2]) - assert.Equal(t, "file5.txt", names2[3]) + require.Equal(s.T(), 4, len(names2)) + assert.Equal(s.T(), "file2.txt", names2[0]) + assert.Equal(s.T(), "file3.txt", names2[1]) + assert.Equal(s.T(), "file4.txt", names2[2]) + assert.Equal(s.T(), "file5.txt", names2[3]) } // (a) First ReadDir() will be served from GCSFuse filesystem. // (b) Second ReadDir() will also be served from GCSFuse filesystem, because of // addition of new directory. -func (s *infiniteKernelListCacheTest) TestKernelListCache_CacheMissOnAdditionOfDirectory(t *testing.T) { - targetDir := path.Join(testDirPath, "explicit_dir") - operations.CreateDirectory(targetDir, t) +func (s *infiniteKernelListCacheTest) TestKernelListCache_CacheMissOnAdditionOfDirectory() { + explicit_dir := "explicit_dir" + setup.GenerateRandomString(5) + targetDir := path.Join(testEnv.testDirPath, explicit_dir) + operations.CreateDirectory(targetDir, s.T()) // Create test data - f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f1) - f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f2) + f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f1) + f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f2) // First read, kernel will cache the dir response. f, err := os.Open(targetDir) - require.NoError(t, err) + require.NoError(s.T(), err) defer func() { - assert.NoError(t, f.Close()) + assert.NoError(s.T(), f.Close()) }() names1, err := f.Readdirnames(-1) - require.NoError(t, err) - require.Equal(t, 2, len(names1)) - require.Equal(t, "file1.txt", names1[0]) - require.Equal(t, "file2.txt", names1[1]) + require.NoError(s.T(), err) + require.Equal(s.T(), 2, len(names1)) + require.Equal(s.T(), "file1.txt", names1[0]) + require.Equal(s.T(), "file2.txt", names1[1]) err = f.Close() - require.NoError(t, err) + require.NoError(s.T(), err) // Adding one object to make sure to change the ReadDir() response. - client.CreateObjectInGCSTestDir(ctx, storageClient, testDirName, path.Join("explicit_dir", "file3.txt"), "", t) + client.CreateObjectInGCSTestDir(testEnv.ctx, testEnv.storageClient, testDirName, path.Join(explicit_dir, "file3.txt"), "", s.T()) // Ideally no invalidation since infinite ttl, but creation of a new directory inside // directory evicts the list cache for that directory. err = os.Mkdir(path.Join(targetDir, "sub_dir"), setup.DirPermission_0755) - require.NoError(t, err) + require.NoError(s.T(), err) f, err = os.Open(targetDir) - assert.Nil(t, err) + assert.Nil(s.T(), err) names2, err := f.Readdirnames(-1) - assert.Nil(t, err) - require.Equal(t, 4, len(names2)) - assert.Equal(t, "file1.txt", names2[0]) - assert.Equal(t, "file2.txt", names2[1]) - assert.Equal(t, "file3.txt", names2[2]) - assert.Equal(t, "sub_dir", names2[3]) + assert.Nil(s.T(), err) + require.Equal(s.T(), 4, len(names2)) + assert.Equal(s.T(), "file1.txt", names2[0]) + assert.Equal(s.T(), "file2.txt", names2[1]) + assert.Equal(s.T(), "file3.txt", names2[2]) + assert.Equal(s.T(), "sub_dir", names2[3]) } // (a) First ReadDir() will be served from GCSFuse filesystem. // (b) Second ReadDir() will also be served from GCSFuse filesystem, because of // deletion of directory. -func (s *infiniteKernelListCacheTest) TestKernelListCache_CacheMissOnDeletionOfDirectory(t *testing.T) { - targetDir := path.Join(testDirPath, "explicit_dir") - operations.CreateDirectory(targetDir, t) +func (s *infiniteKernelListCacheTest) TestKernelListCache_CacheMissOnDeletionOfDirectory() { + explicit_dir := "explicit_dir" + setup.GenerateRandomString(5) + targetDir := path.Join(testEnv.testDirPath, explicit_dir) + operations.CreateDirectory(targetDir, s.T()) // Create test data - f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f1) - f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f2) + f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f1) + f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f2) err := os.Mkdir(path.Join(targetDir, "sub_dir"), setup.DirPermission_0755) - require.NoError(t, err) + require.NoError(s.T(), err) // First read, kernel will cache the dir response. f, err := os.Open(targetDir) - require.NoError(t, err) + require.NoError(s.T(), err) defer func() { - assert.NoError(t, f.Close()) + assert.NoError(s.T(), f.Close()) }() names1, err := f.Readdirnames(-1) - require.NoError(t, err) - require.Equal(t, 3, len(names1)) - require.Equal(t, "file1.txt", names1[0]) - require.Equal(t, "file2.txt", names1[1]) - require.Equal(t, "sub_dir", names1[2]) + require.NoError(s.T(), err) + require.Equal(s.T(), 3, len(names1)) + require.Equal(s.T(), "file1.txt", names1[0]) + require.Equal(s.T(), "file2.txt", names1[1]) + require.Equal(s.T(), "sub_dir", names1[2]) err = f.Close() - require.NoError(t, err) + require.NoError(s.T(), err) // Adding one object to make sure to change the ReadDir() response. - client.CreateObjectInGCSTestDir(ctx, storageClient, testDirName, path.Join("explicit_dir", "file3.txt"), "", t) + client.CreateObjectInGCSTestDir(testEnv.ctx, testEnv.storageClient, testDirName, path.Join(explicit_dir, "file3.txt"), "", s.T()) // Ideally no invalidation since infinite ttl, but creation of a new file inside // directory evicts the list cache for that directory. err = os.Remove(path.Join(targetDir, "sub_dir")) - require.Nil(t, err) + require.Nil(s.T(), err) f, err = os.Open(targetDir) - assert.Nil(t, err) + assert.Nil(s.T(), err) names2, err := f.Readdirnames(-1) - assert.Nil(t, err) - require.Equal(t, 3, len(names2)) - assert.Equal(t, "file1.txt", names2[0]) - assert.Equal(t, "file2.txt", names2[1]) - assert.Equal(t, "file3.txt", names2[2]) + assert.Nil(s.T(), err) + require.Equal(s.T(), 3, len(names2)) + assert.Equal(s.T(), "file1.txt", names2[0]) + assert.Equal(s.T(), "file2.txt", names2[1]) + assert.Equal(s.T(), "file3.txt", names2[2]) } // (a) First ReadDir() will be served from GCSFuse filesystem. // (b) Second ReadDir() will also be served from GCSFuse filesystem, because of // directory rename. -func (s *infiniteKernelListCacheTest) TestKernelListCache_CacheMissOnDirectoryRename(t *testing.T) { - targetDir := path.Join(testDirPath, "explicit_dir") - operations.CreateDirectory(targetDir, t) +func (s *infiniteKernelListCacheTest) TestKernelListCache_CacheMissOnDirectoryRename() { + explicit_dir := "explicit_dir" + setup.GenerateRandomString(5) + targetDir := path.Join(testEnv.testDirPath, explicit_dir) + operations.CreateDirectory(targetDir, s.T()) // Create test data - f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f1) - f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f2) + f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f1) + f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f2) err := os.Mkdir(path.Join(targetDir, "sub_dir"), setup.DirPermission_0755) - require.NoError(t, err) + require.NoError(s.T(), err) // First read, kernel will cache the dir response. f, err := os.Open(targetDir) - require.NoError(t, err) + require.NoError(s.T(), err) defer func() { - assert.NoError(t, f.Close()) + assert.NoError(s.T(), f.Close()) }() names1, err := f.Readdirnames(-1) - require.NoError(t, err) - require.Equal(t, 3, len(names1)) - require.Equal(t, "file1.txt", names1[0]) - require.Equal(t, "file2.txt", names1[1]) - require.Equal(t, "sub_dir", names1[2]) + require.NoError(s.T(), err) + require.Equal(s.T(), 3, len(names1)) + require.Equal(s.T(), "file1.txt", names1[0]) + require.Equal(s.T(), "file2.txt", names1[1]) + require.Equal(s.T(), "sub_dir", names1[2]) err = f.Close() - require.NoError(t, err) + require.NoError(s.T(), err) // Adding one object to make sure to change the ReadDir() response. - client.CreateObjectInGCSTestDir(ctx, storageClient, testDirName, path.Join("explicit_dir", "file3.txt"), "", t) + client.CreateObjectInGCSTestDir(testEnv.ctx, testEnv.storageClient, testDirName, path.Join(explicit_dir, "file3.txt"), "", s.T()) // Ideally no invalidation since infinite ttl, but creation of a new file inside // directory evicts the list cache for that directory. err = os.Rename(path.Join(targetDir, "sub_dir"), path.Join(targetDir, "renamed_sub_dir")) - require.Nil(t, err) + require.Nil(s.T(), err) defer func() { - assert.Nil(t, os.Remove(path.Join(targetDir, "renamed_sub_dir"))) + assert.Nil(s.T(), os.Remove(path.Join(targetDir, "renamed_sub_dir"))) }() f, err = os.Open(targetDir) - assert.Nil(t, err) + assert.Nil(s.T(), err) names2, err := f.Readdirnames(-1) - assert.Nil(t, err) - require.Equal(t, 4, len(names2)) - assert.Equal(t, "file1.txt", names2[0]) - assert.Equal(t, "file2.txt", names2[1]) - assert.Equal(t, "file3.txt", names2[2]) - assert.Equal(t, "renamed_sub_dir", names2[3]) -} - -func (s *infiniteKernelListCacheTest) TestKernelListCache_ListAndDeleteDirectory(t *testing.T) { - targetDir := path.Join(testDirPath, "explicit_dir") - operations.CreateDirectory(targetDir, t) - // Create test data - f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f1) - f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f2) - - // (a) First read served from GCS, kernel will cache the dir response. - f, err := os.Open(targetDir) - assert.NoError(t, err) - names1, err := f.Readdirnames(-1) - assert.NoError(t, err) - require.Equal(t, 2, len(names1)) - assert.Equal(t, "file1.txt", names1[0]) - assert.Equal(t, "file2.txt", names1[1]) - err = f.Close() - assert.NoError(t, err) - // Adding one object to make sure to change the ReadDir() response. - // All files including file3.txt will be deleted by os.RemoveAll - client.CreateObjectInGCSTestDir(ctx, storageClient, testDirName, path.Join("explicit_dir", "file3.txt"), "", t) - - err = os.RemoveAll(targetDir) - - assert.NoError(t, err) -} - -func (s *infiniteKernelListCacheTest) TestKernelListCache_DeleteAndListDirectory(t *testing.T) { - targetDir := path.Join(testDirPath, "explicit_dir") - operations.CreateDirectory(targetDir, t) - // Create test data - f1 := operations.CreateFile(path.Join(targetDir, "file1.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f1) - f2 := operations.CreateFile(path.Join(targetDir, "file2.txt"), setup.FilePermission_0600, t) - operations.CloseFile(f2) - - err := os.RemoveAll(targetDir) - assert.NoError(t, err) - - // Adding object to GCS to make sure to change the ReadDir() response. - err = client.CreateObjectOnGCS(ctx, storageClient, path.Join(testDirName, "explicit_dir")+"/", "") - require.NoError(t, err) - client.CreateObjectInGCSTestDir(ctx, storageClient, testDirName, path.Join("explicit_dir", "file3.txt"), "", t) - // Read will be served from GCS as removing the directory also deletes the cache. - f, err := os.Open(targetDir) - assert.NoError(t, err) - names1, err := f.Readdirnames(-1) - assert.NoError(t, err) - require.Equal(t, 1, len(names1)) - assert.Equal(t, "file3.txt", names1[0]) - err = f.Close() - assert.NoError(t, err) - - // 2nd RemoveAll call will also succeed. - err = os.RemoveAll(targetDir) - assert.NoError(t, err) + assert.Nil(s.T(), err) + require.Equal(s.T(), 4, len(names2)) + assert.Equal(s.T(), "file1.txt", names2[0]) + assert.Equal(s.T(), "file2.txt", names2[1]) + assert.Equal(s.T(), "file3.txt", names2[2]) + assert.Equal(s.T(), "renamed_sub_dir", names2[3]) } //////////////////////////////////////////////////////////////////////// @@ -519,21 +478,16 @@ func (s *infiniteKernelListCacheTest) TestKernelListCache_DeleteAndListDirectory func TestInfiniteKernelListCacheTest(t *testing.T) { ts := &infiniteKernelListCacheTest{} - // Run tests for mounted directory if the flag is set. - if setup.AreBothMountedDirectoryAndTestBucketFlagsSet() { - test_setup.RunTests(t, ts) + // Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) return } - // Define flag set to run the tests. - flagsSet := [][]string{ - {"--kernel-list-cache-ttl-secs=-1"}, - } - - // Run tests. - for _, flags := range flagsSet { - ts.flags = flags + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { log.Printf("Running tests with flags: %s", ts.flags) - test_setup.RunTests(t, ts) + suite.Run(t, ts) } } diff --git a/tools/integration_tests/kernel_list_cache/setup_test.go b/tools/integration_tests/kernel_list_cache/setup_test.go index 10dc3e0c1f..4ff64f0332 100644 --- a/tools/integration_tests/kernel_list_cache/setup_test.go +++ b/tools/integration_tests/kernel_list_cache/setup_test.go @@ -19,82 +19,115 @@ import ( "log" "os" "path" + "strings" "testing" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/dynamic_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/only_dir_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/dynamic_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/only_dir_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" ) const ( testDirName = "KernelListCacheTest" onlyDirMounted = "OnlyDirMountKernelListCache" + GKETempDir = "/gcsfuse-tmp" ) var ( - testDirPath string - mountFunc func([]string) error + mountFunc func(*test_suite.TestConfig, []string) error // mount directory is where our tests run. mountDir string // root directory is the directory to be unmounted. - rootDir string - storageClient *storage.Client - ctx context.Context + rootDir string ) -//////////////////////////////////////////////////////////////////////// -// Helpers -//////////////////////////////////////////////////////////////////////// - -func mountGCSFuseAndSetupTestDir(flags []string, ctx context.Context, storageClient *storage.Client, testDirName string) { - // When tests are running in GKE environment, use the mounted directory provided as test flag. - if setup.MountedDirectory() != "" { - mountDir = setup.MountedDirectory() - } - setup.MountGCSFuseWithGivenMountFunc(flags, mountFunc) - setup.SetMntDir(mountDir) - testDirPath = setup.SetupTestDirectory(testDirName) +type env struct { + storageClient *storage.Client + ctx context.Context + testDirPath string + cfg *test_suite.TestConfig + bucketType string } +var testEnv env + //////////////////////////////////////////////////////////////////////// // TestMain //////////////////////////////////////////////////////////////////////// func TestMain(m *testing.M) { setup.ParseSetUpFlags() - setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() - // Create common storage client to be used in test. - ctx = context.Background() - closeStorageClient := client.CreateStorageClientWithCancel(&ctx, &storageClient) + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.KernelListCache) == 0 { + log.Println("No configuration found for kernel_list_cache tests in config. Using flags instead.") + // Populate the config manually. + cfg.KernelListCache = make([]test_suite.TestConfig, 1) + cfg.KernelListCache[0].TestBucket = setup.TestBucket() + cfg.KernelListCache[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.KernelListCache[0].LogFile = setup.LogFile() + // Initialize the slice to hold 15 specific test configurations + cfg.KernelListCache[0].Configs = make([]test_suite.ConfigItem, 4) + cfg.KernelListCache[0].Configs[0].Flags = []string{"--kernel-list-cache-ttl-secs=-1"} + cfg.KernelListCache[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.KernelListCache[0].Configs[0].Run = "TestInfiniteKernelListCacheTest" + // Note: metadata cache is disabled to avoid cache consistency issue between + // gcsfuse cache and kernel cache. As gcsfuse cache might hold the entry which + // already became stale due to delete operation. + cfg.KernelListCache[0].Configs[1].Flags = []string{"--kernel-list-cache-ttl-secs=-1 --metadata-cache-ttl-secs=0 --metadata-cache-negative-ttl-secs=0"} + cfg.KernelListCache[0].Configs[1].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.KernelListCache[0].Configs[1].Run = "TestInfiniteKernelListCacheDeleteDirTest" + cfg.KernelListCache[0].Configs[2].Flags = []string{"--kernel-list-cache-ttl-secs=5 --rename-dir-limit=10"} + cfg.KernelListCache[0].Configs[2].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.KernelListCache[0].Configs[2].Run = "TestFiniteKernelListCacheTest" + cfg.KernelListCache[0].Configs[3].Flags = []string{"--kernel-list-cache-ttl-secs=0 --stat-cache-ttl=0 --rename-dir-limit=10"} + cfg.KernelListCache[0].Configs[3].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.KernelListCache[0].Configs[3].Run = "TestDisabledKernelListCacheTest" + } + + testEnv.ctx = context.Background() + testEnv.bucketType = setup.TestEnvironment(testEnv.ctx, &cfg.KernelListCache[0]) + testEnv.cfg = &cfg.KernelListCache[0] + + // 2. Create storage client before running tests. + closeStorageClient := client.CreateStorageClientWithCancel(&testEnv.ctx, &testEnv.storageClient) defer func() { err := closeStorageClient() if err != nil { - log.Fatalf("closeStorageClient failed: %v", err) + log.Printf("closeStorageClient failed: %v\n", err) } }() - // If Mounted Directory flag is set, run tests for mounted directory. - setup.RunTestsForMountedDirectoryFlag(m) - // Else run tests for testBucket. + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + // Save mount and root directory variables. + mountDir, rootDir = testEnv.cfg.GKEMountedDirectory, testEnv.cfg.GKEMountedDirectory + os.Exit(setup.RunTestsForMountedDirectory(testEnv.cfg.GKEMountedDirectory, m)) + } + + // Run tests for testBucket // Set up test directory. - setup.SetUpTestDirForTestBucketFlag() + setup.SetUpTestDirForTestBucket(testEnv.cfg) + // Override GKE specific paths with GCSFuse paths if running in GCE environment. + overrideFilePathsInFlagSet(testEnv.cfg, setup.TestDir()) // Save mount and root directory variables. - mountDir, rootDir = setup.MntDir(), setup.MntDir() + mountDir, rootDir = testEnv.cfg.GCSFuseMountedDirectory, testEnv.cfg.GCSFuseMountedDirectory log.Println("Running static mounting tests...") - mountFunc = static_mounting.MountGcsfuseWithStaticMounting + mountFunc = static_mounting.MountGcsfuseWithStaticMountingWithConfigFile successCode := m.Run() if successCode == 0 { log.Println("Running dynamic mounting tests...") // Save mount directory variable to have path of bucket to run tests. - mountDir = path.Join(setup.MntDir(), setup.TestBucket()) - mountFunc = dynamic_mounting.MountGcsfuseWithDynamicMounting + mountDir = path.Join(testEnv.cfg.GCSFuseMountedDirectory, testEnv.cfg.TestBucket) + mountFunc = dynamic_mounting.MountGcsfuseWithDynamicMountingWithConfig successCode = m.Run() } @@ -102,12 +135,21 @@ func TestMain(m *testing.M) { log.Println("Running only dir mounting tests...") setup.SetOnlyDirMounted(onlyDirMounted + "/") mountDir = rootDir - mountFunc = only_dir_mounting.MountGcsfuseWithOnlyDir + mountFunc = only_dir_mounting.MountGcsfuseWithOnlyDirWithConfigFile successCode = m.Run() - setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(setup.TestBucket(), setup.OnlyDirMounted(), testDirName)) + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(testEnv.cfg.TestBucket, setup.OnlyDirMounted(), testDirName)) } // Clean up test directory created. - setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(setup.TestBucket(), testDirName)) + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(testEnv.cfg.TestBucket, testDirName)) os.Exit(successCode) } + +func overrideFilePathsInFlagSet(t *test_suite.TestConfig, GCSFuseTempDirPath string) { + for _, flags := range t.Configs { + for i := range flags.Flags { + // Iterate over the indices of the flags slice + flags.Flags[i] = strings.ReplaceAll(flags.Flags[i], "/gcsfuse-tmp", path.Join(GCSFuseTempDirPath, "gcsfuse-tmp")) + } + } +} diff --git a/tools/integration_tests/list_large_dir/list_dir_with_twelve_thousand_files_test.go b/tools/integration_tests/list_large_dir/list_dir_with_twelve_thousand_files_test.go index 1a3fe1625e..4ab6612041 100644 --- a/tools/integration_tests/list_large_dir/list_dir_with_twelve_thousand_files_test.go +++ b/tools/integration_tests/list_large_dir/list_dir_with_twelve_thousand_files_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +//     http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -12,163 +12,158 @@ // See the License for the specific language governing permissions and // limitations under the License. -package list_large_dir_test +package list_large_dir import ( + "context" + "fmt" + "log" "math" "os" "path" + "runtime" "strconv" "strings" + "sync" "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "golang.org/x/sync/errgroup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" ) // ////////////////////////////////////////////////////////////////////// -// Helpers +// Boilerplate // ////////////////////////////////////////////////////////////////////// -func validateDirectoryWithTwelveThousandFiles(objs []os.DirEntry, t *testing.T) { - // number of objs - 12000 - if len(objs) != NumberOfFilesInDirectoryWithTwelveThousandFiles { - t.Errorf("Listed incorrect number of files from directory: %v, expected 12000", len(objs)) - } - - // Checking if all the object is File type. - for i := 0; i < len(objs); i++ { - if objs[i].IsDir() { - t.Errorf("Listed object is incorrect.") - } - } - for i := 0; i < len(objs); i++ { - checkIfObjNameIsCorrect(objs[i].Name(), PrefixFileInDirectoryWithTwelveThousandFiles, NumberOfFilesInDirectoryWithTwelveThousandFiles, t) - } +type listLargeDir struct { + flags []string + isKernelListCacheEnabled bool + suite.Suite } -func validateDirectoryWithTwelveThousandFilesHundredExplicitDirAndHundredImplicitDir(objs []os.DirEntry, t *testing.T) { - var numberOfFiles = 0 - var numberOfDirs = 0 - - // Checking if correct objects present in bucket. - for i := 0; i < len(objs); i++ { - if !objs[i].IsDir() { - numberOfFiles++ - - // Checking if Prefix1 to Prefix12000 present in the bucket - checkIfObjNameIsCorrect(objs[i].Name(), PrefixFileInDirectoryWithTwelveThousandFiles, NumberOfFilesInDirectoryWithTwelveThousandFiles, t) - } +func (t *listLargeDir) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} - if objs[i].IsDir() { - numberOfDirs++ +func (t *listLargeDir) SetupSuite() { + err := DeleteAllObjectsWithPrefix(testEnv.ctx, testEnv.storageClient, t.T().Name()) + assert.NoError(t.T(), err) + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, t.flags, mountFunc) +} - if strings.Contains(objs[i].Name(), PrefixExplicitDirInLargeDirListTest) { - // Checking if explicitDir1 to explicitDir100 present in the bucket. - checkIfObjNameIsCorrect(objs[i].Name(), PrefixExplicitDirInLargeDirListTest, NumberOfExplicitDirsInDirectoryWithTwelveThousandFiles, t) - } else { - // Checking if implicitDir1 to implicitDir100 present in the bucket. - checkIfObjNameIsCorrect(objs[i].Name(), PrefixImplicitDirInLargeDirListTest, NumberOfImplicitDirsInDirectoryWithTwelveThousandFiles, t) - } - } - } +func (t *listLargeDir) SetupTest() { +} - // number of dirs = 200(Number of implicit + Number of explicit directories) - if numberOfDirs != NumberOfImplicitDirsInDirectoryWithTwelveThousandFiles+NumberOfExplicitDirsInDirectoryWithTwelveThousandFiles { - t.Errorf("Listed incorrect number of directories from directory: %v, expected 200", numberOfDirs) - } - // number of files = 12000 - if numberOfFiles != NumberOfFilesInDirectoryWithTwelveThousandFiles { - t.Errorf("Listed incorrect number of files from directory: %v, expected 12000", numberOfFiles) - } +func (t *listLargeDir) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(t.T()) } -func validateDirectoryWithTwelveThousandFilesAndHundredExplicitDirectory(objs []os.DirEntry, t *testing.T) { - var numberOfFiles = 0 - var numberOfDirs = 0 +// ////////////////////////////////////////////////////////////////////// +// Helpers +// ////////////////////////////////////////////////////////////////////// - // Checking if correct objects present in bucket. - for i := 0; i < len(objs); i++ { - if !objs[i].IsDir() { +// validateDirectory checks if the directory listing matches expectations. +func validateDirectory(t *testing.T, objs []os.DirEntry, expectExplicitDirs, expectImplicitDirs bool) { + t.Helper() + + var ( + numberOfFiles int + numberOfExplicitDirs int + numberOfImplicitDirs int + ) + + for _, obj := range objs { + + if strings.Contains(obj.Name(), prefixExplicitDirInLargeDirListTest) { + numberOfExplicitDirs++ + checkIfObjNameIsCorrect(t, obj.Name(), prefixExplicitDirInLargeDirListTest, numberOfExplicitDirsInDirectoryWithTwelveThousandFiles) + } else if strings.Contains(obj.Name(), prefixImplicitDirInLargeDirListTest) { + numberOfImplicitDirs++ + checkIfObjNameIsCorrect(t, obj.Name(), prefixImplicitDirInLargeDirListTest, numberOfImplicitDirsInDirectoryWithTwelveThousandFiles) + } else { numberOfFiles++ - // Checking if Prefix1 to Prefix12000 present in the bucket - checkIfObjNameIsCorrect(objs[i].Name(), PrefixFileInDirectoryWithTwelveThousandFiles, NumberOfFilesInDirectoryWithTwelveThousandFiles, t) + checkIfObjNameIsCorrect(t, obj.Name(), prefixFileInDirectoryWithTwelveThousandFiles, numberOfFilesInDirectoryWithTwelveThousandFiles) } + } - if objs[i].IsDir() { - numberOfDirs++ - // Checking if Prefix1 to Prefix100 present in the bucket - checkIfObjNameIsCorrect(objs[i].Name(), PrefixExplicitDirInLargeDirListTest, NumberOfExplicitDirsInDirectoryWithTwelveThousandFiles, t) - } + if numberOfFiles != numberOfFilesInDirectoryWithTwelveThousandFiles { + t.Errorf("Incorrect number of files: got %d, want %d", numberOfFiles, numberOfFilesInDirectoryWithTwelveThousandFiles) } - // number of explicit dirs = 100 - if numberOfDirs != NumberOfExplicitDirsInDirectoryWithTwelveThousandFiles { - t.Errorf("Listed incorrect number of directories from directory: %v, expected 100", numberOfDirs) + if expectExplicitDirs && numberOfExplicitDirs != numberOfExplicitDirsInDirectoryWithTwelveThousandFiles { + t.Errorf("Incorrect number of explicit directories: got %d, want %d", numberOfExplicitDirs, numberOfExplicitDirsInDirectoryWithTwelveThousandFiles) } - // number of files = 12000 - if numberOfFiles != NumberOfFilesInDirectoryWithTwelveThousandFiles { - t.Errorf("Listed incorrect number of files from directory: %v, expected 12000", numberOfFiles) + + if expectImplicitDirs && numberOfImplicitDirs != numberOfImplicitDirsInDirectoryWithTwelveThousandFiles { + t.Errorf("Incorrect number of implicit directories: got %d, want %d", numberOfImplicitDirs, numberOfImplicitDirsInDirectoryWithTwelveThousandFiles) } } -func checkIfObjNameIsCorrect(objName string, prefix string, maxNumber int, t *testing.T) { - // Extracting object number. - objNumberStr := strings.ReplaceAll(objName, prefix, "") +// checkIfObjNameIsCorrect validates the object name against a prefix and expected range. +func checkIfObjNameIsCorrect(t *testing.T, objName string, prefix string, maxNumber int) { + t.Helper() + + objNumberStr := strings.TrimPrefix(objName, prefix) objNumber, err := strconv.Atoi(objNumberStr) if err != nil { - t.Errorf("Error in extracting file number: %v", err) + t.Errorf("Error extracting object number from %q: %v", objName, err) } if objNumber < 1 || objNumber > maxNumber { - t.Errorf("Correct object does not exist.") + t.Errorf("Invalid object number in %q: %d (should be between 1 and %d)", objName, objNumber, maxNumber) } } -func createTwelveThousandFilesAndUploadOnTestBucket(t *testing.T) { - // Creating twelve thousand files in DirectoryWithTwelveThousandFiles directory to upload them on a bucket for testing. - localDirPath := path.Join(os.Getenv("HOME"), DirectoryWithTwelveThousandFiles) - operations.CreateDirectoryWithNFiles(NumberOfFilesInDirectoryWithTwelveThousandFiles, localDirPath, PrefixFileInDirectoryWithTwelveThousandFiles, t) - - // Uploading twelve thousand files to directoryWithTwelveThousandFiles in testBucket. - dirPath := path.Join(setup.TestBucket(), DirectoryForListLargeFileTests, DirectoryWithTwelveThousandFiles) - setup.RunScriptForTestData("testdata/upload_files_to_bucket.sh", dirPath, DirectoryWithTwelveThousandFiles, PrefixFileInDirectoryWithTwelveThousandFiles) +// testdataUploadFilesToBucket uploads matching files from a local directory to a specified path in a GCS bucket. +func testdataUploadFilesToBucket(ctx context.Context, t *testing.T, storageClient *storage.Client, bucketNameWithDirPath, dirWith12KFiles, filesPrefix string) { + t.Helper() + bucketName, dirPathInBucket := operations.SplitBucketNameAndDirPath(t, bucketNameWithDirPath) + err := client.BatchUploadFilesWithoutIntermediateDelays(ctx, storageClient, bucketName, dirPathInBucket, dirWith12KFiles, filesPrefix) + assert.NoError(t, err) } -// Create a hundred explicit directories. -func createHundredExplicitDir(dirPath string, t *testing.T) { - // Create hundred explicit directories. - for i := 1; i <= NumberOfExplicitDirsInDirectoryWithTwelveThousandFiles; i++ { - subDirPath := path.Join(dirPath, PrefixExplicitDirInLargeDirListTest+strconv.Itoa(i)) - operations.CreateDirectoryWithNFiles(0, subDirPath, "", t) - } +// createFilesAndUpload generates files and uploads them to the specified directory. +func createFilesAndUpload(t *testing.T, dirPath string) { + t.Helper() + + localDirPath := path.Join(os.Getenv("HOME"), directoryWithTwelveThousandFiles) + operations.CreateDirectoryWithNFiles(numberOfFilesInDirectoryWithTwelveThousandFiles, localDirPath, prefixFileInDirectoryWithTwelveThousandFiles, t) + defer os.RemoveAll(localDirPath) + + testdataUploadFilesToBucket(testEnv.ctx, t, testEnv.storageClient, dirPath, localDirPath, prefixFileInDirectoryWithTwelveThousandFiles) } -func listDirectoryTime(dirPath string, validateDirectory func([]os.DirEntry, *testing.T), t *testing.T) (time.Duration, time.Duration) { - // List Directory first time +// listDirTime measures the time taken to list a directory with and without cache. +func listDirTime(t *testing.T, dirPath string, expectExplicitDirs bool, expectImplicitDirs bool) (time.Duration, time.Duration) { + t.Helper() + startTime := time.Now() objs, err := os.ReadDir(dirPath) if err != nil { t.Fatalf("Error in listing directory: %v", err) } endTime := time.Now() - validateDirectory(objs, t) + + validateDirectory(t, objs, expectExplicitDirs, expectImplicitDirs) firstListTime := endTime.Sub(startTime) - // Listing the directory a second time should retrieve the response from the kernel cache. minSecondListTime := time.Duration(math.MaxInt64) - for i := 0; i < 5; i++ { + for range 5 { startTime = time.Now() objs, err = os.ReadDir(dirPath) if err != nil { t.Fatalf("Error in listing directory: %v", err) } endTime = time.Now() - validateDirectory(objs, t) + validateDirectory(t, objs, expectExplicitDirs, expectImplicitDirs) secondListTime := endTime.Sub(startTime) - - // Update the minimum listing time for the second listing if secondListTime < minSecondListTime { minSecondListTime = secondListTime } @@ -176,66 +171,147 @@ func listDirectoryTime(dirPath string, validateDirectory func([]os.DirEntry, *te return firstListTime, minSecondListTime } -//////////////////////////////////////////////////////////////////////// +// testdataCreateImplicitDir creates implicit directories by uploading files with nested paths. +func testdataCreateImplicitDir(t *testing.T, ctx context.Context, storageClient *storage.Client, bucketNameWithDirPath string) { + t.Helper() + + bucketName, dirPathInBucket := operations.SplitBucketNameAndDirPath(t, bucketNameWithDirPath) + + testFile, err := operations.CreateLocalTempFile("", false) + if err != nil { + t.Fatalf("Failed to create local file for creating copies ...") + } + + var wg sync.WaitGroup + sem := make(chan struct{}, runtime.NumCPU()/2) // Concurrency limiter + + for suffix := 1; suffix <= numberOfImplicitDirsInDirectoryWithTwelveThousandFiles; suffix++ { + objectPath := path.Join(dirPathInBucket, fmt.Sprintf("%s%d", prefixImplicitDirInLargeDirListTest, suffix), testFile) + + wg.Add(1) + go func(destinationPath string) { + defer wg.Done() + sem <- struct{}{} // acquire semaphore + defer func() { <-sem }() // release semaphore + + client.CopyFileInBucketWithPreconditions(ctx, storageClient, testFile, destinationPath, bucketName, &storage.Conditions{DoesNotExist: true}) + }(objectPath) + } + + wg.Wait() +} + +// testdataCreateExplicitDir creates explicit directories (trailing slash objects) in the bucket. +func testdataCreateExplicitDir(t *testing.T, ctx context.Context, storageClient *storage.Client, bucketNameWithDirPath string) { + t.Helper() + + bucketName, dirPathInBucket := operations.SplitBucketNameAndDirPath(t, bucketNameWithDirPath) + + g, ctx := errgroup.WithContext(ctx) + g.SetLimit(runtime.NumCPU() / 2) // Concurrency limiter + + for dirIndex := 1; dirIndex <= numberOfExplicitDirsInDirectoryWithTwelveThousandFiles; dirIndex++ { + capturedIndex := dirIndex + g.Go(func() error { + dirName := fmt.Sprintf("%s%d", prefixExplicitDirInLargeDirListTest, capturedIndex) + return client.CreateGcsDir(ctx, storageClient, dirName, bucketName, dirPathInBucket) + }) + } + + if err := g.Wait(); err != nil { + t.Fatalf("Failed to create explicit dirs: %v", err) + } +} + +// prepareTestDirectory sets up a test directory with files and required explicit and implicit directories. +func prepareTestDirectory(t *testing.T, withExplicitDirs bool, withImplicitDirs bool) string { + t.Helper() + + testDirPathOnBucket := path.Join(testEnv.cfg.TestBucket, t.Name()) + testDirPath := path.Join(setup.MntDir(), t.Name()) + + err := os.MkdirAll(testDirPath, 0755) + if err != nil { + t.Fatalf("Failed to create directory: %v", err) + } + + createFilesAndUpload(t, testDirPathOnBucket) + + if withExplicitDirs { + testdataCreateExplicitDir(t, testEnv.ctx, testEnv.storageClient, testDirPathOnBucket) + } + + if withImplicitDirs { + testdataCreateImplicitDir(t, testEnv.ctx, testEnv.storageClient, testDirPathOnBucket) + } + + return testDirPath +} + +// ////////////////////////////////////////////////////////////////////// // Tests +// ////////////////////////////////////////////////////////////////////// + +func (t *listLargeDir) TestListDirectoryWithTwelveThousandFiles() { + dirPath := prepareTestDirectory(t.T(), false, false) + + firstListTime, secondListTime := listDirTime(t.T(), dirPath, false, false) + + if t.isKernelListCacheEnabled { + assert.Less(t.T(), secondListTime, firstListTime) + assert.Less(t.T(), 2*secondListTime, firstListTime) + } +} + +func (t *listLargeDir) TestListDirectoryWithTwelveThousandFilesAndHundredExplicitDir() { + dirPath := prepareTestDirectory(t.T(), true, false) + + firstListTime, secondListTime := listDirTime(t.T(), dirPath, true, false) + + if t.isKernelListCacheEnabled { + assert.Less(t.T(), secondListTime, firstListTime) + assert.Less(t.T(), 2*secondListTime, firstListTime) + } +} + +func (t *listLargeDir) TestListDirectoryWithTwelveThousandFilesAndHundredExplicitDirAndHundredImplicitDir() { + if setup.IsZonalBucketRun() { + t.T().Skipf("Redundant test for ZB as implicit-dir is a non-HNS concept, hence not applicable here. ") + } + dirPath := prepareTestDirectory(t.T(), true, true) + + firstListTime, secondListTime := listDirTime(t.T(), dirPath, true, true) + + if t.isKernelListCacheEnabled { + assert.Less(t.T(), secondListTime, firstListTime) + assert.Less(t.T(), 2*secondListTime, firstListTime) + } +} + +//////////////////////////////////////////////////////////////////////// +// Test Suite Function //////////////////////////////////////////////////////////////////////// -// Test with a bucket with twelve thousand files. -func TestListDirectoryWithTwelveThousandFiles(t *testing.T) { - createTwelveThousandFilesAndUploadOnTestBucket(t) - testDirPath := path.Join(setup.MntDir(), DirectoryForListLargeFileTests) - testDirPathOnBucket := path.Join(setup.TestBucket(), DirectoryForListLargeFileTests) - dirPath := path.Join(testDirPath, DirectoryWithTwelveThousandFiles) - - firstListTime, secondListTime := listDirectoryTime(dirPath, validateDirectoryWithTwelveThousandFiles, t) - - // Fetching data from the kernel for the second list will be faster. - assert.Less(t, secondListTime, firstListTime) - // The second directory listing should be 2 times better performant since it - // will be retrieved from the kernel cache. - assert.Less(t, 2*secondListTime, firstListTime) - - // Clear the data after testing. - setup.RunScriptForTestData("testdata/delete_objects.sh", testDirPathOnBucket) -} - -// Test with a bucket with twelve thousand files and hundred explicit directories. -func TestListDirectoryWithTwelveThousandFilesAndHundredExplicitDir(t *testing.T) { - createTwelveThousandFilesAndUploadOnTestBucket(t) - testDirPath := path.Join(setup.MntDir(), DirectoryForListLargeFileTests) - testDirPathOnBucket := path.Join(setup.TestBucket(), DirectoryForListLargeFileTests) - dirPath := path.Join(testDirPath, DirectoryWithTwelveThousandFiles) - createHundredExplicitDir(dirPath, t) - - firstListTime, secondListTime := listDirectoryTime(dirPath, validateDirectoryWithTwelveThousandFilesAndHundredExplicitDirectory, t) - - // Fetching data from the kernel for the second list will be faster. - assert.Less(t, secondListTime, firstListTime) - // The second directory listing should be 2 times better performant since it - // will be retrieved from the kernel cache. - assert.Less(t, 2*secondListTime, firstListTime) - - // Clear the bucket after testing. - setup.RunScriptForTestData("testdata/delete_objects.sh", testDirPathOnBucket) -} - -// Test with a bucket with twelve thousand files, hundred explicit directories, and hundred implicit directories. -func TestListDirectoryWithTwelveThousandFilesAndHundredExplicitDirAndHundredImplicitDir(t *testing.T) { - createTwelveThousandFilesAndUploadOnTestBucket(t) - testDirPath := path.Join(setup.MntDir(), DirectoryForListLargeFileTests) - testDirPathOnBucket := path.Join(setup.TestBucket(), DirectoryForListLargeFileTests) - dirPath := path.Join(testDirPath, DirectoryWithTwelveThousandFiles) - createHundredExplicitDir(dirPath, t) - subDirPath := path.Join(testDirPathOnBucket, DirectoryWithTwelveThousandFiles) - setup.RunScriptForTestData("testdata/create_implicit_dir.sh", subDirPath, PrefixImplicitDirInLargeDirListTest, strconv.Itoa(NumberOfImplicitDirsInDirectoryWithTwelveThousandFiles)) - - firstListTime, secondListTime := listDirectoryTime(dirPath, validateDirectoryWithTwelveThousandFilesHundredExplicitDirAndHundredImplicitDir, t) - - // Fetching data from the kernel for the second list will be faster. - assert.Less(t, secondListTime, firstListTime) - // The second directory listing should be 2 times better performant since it - // will be retrieved from the kernel cache. - assert.Less(t, 2*secondListTime, firstListTime) - // Clear the bucket after testing. - setup.RunScriptForTestData("testdata/delete_objects.sh", testDirPathOnBucket) +func (ts *listLargeDir) runTests(t *testing.T) { + // Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} +func TestListLargeDirWithKernelListCache(t *testing.T) { + ts := &listLargeDir{isKernelListCacheEnabled: true} + ts.runTests(t) +} + +func TestListLargeDirWithoutKernelListCache(t *testing.T) { + ts := &listLargeDir{isKernelListCacheEnabled: false} + ts.runTests(t) } diff --git a/tools/integration_tests/list_large_dir/list_large_dir_test.go b/tools/integration_tests/list_large_dir/list_large_dir_test.go index 5d81f76389..4fc88dfeef 100644 --- a/tools/integration_tests/list_large_dir/list_large_dir_test.go +++ b/tools/integration_tests/list_large_dir/list_large_dir_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -13,46 +13,94 @@ // limitations under the License. // Provide test for listing large directory -package list_large_dir_test +package list_large_dir import ( + "context" "log" "os" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" ) -const DirectoryForListLargeFileTests = "directoryForListLargeFileTests" -const PrefixFileInDirectoryWithTwelveThousandFiles = "fileInDirectoryWithTwelveThousandFiles" -const PrefixExplicitDirInLargeDirListTest = "explicitDirInLargeDirListTest" -const PrefixImplicitDirInLargeDirListTest = "implicitDirInLargeDirListTest" -const NumberOfFilesInDirectoryWithTwelveThousandFiles = 12000 -const NumberOfImplicitDirsInDirectoryWithTwelveThousandFiles = 100 -const NumberOfExplicitDirsInDirectoryWithTwelveThousandFiles = 100 +const ( + prefixFileInDirectoryWithTwelveThousandFiles = "fileInDirectoryWithTwelveThousandFiles" + prefixExplicitDirInLargeDirListTest = "explicitDirInLargeDirListTest" + prefixImplicitDirInLargeDirListTest = "implicitDirInLargeDirListTest" + numberOfFilesInDirectoryWithTwelveThousandFiles = 12000 + numberOfImplicitDirsInDirectoryWithTwelveThousandFiles = 100 + numberOfExplicitDirsInDirectoryWithTwelveThousandFiles = 100 +) + +var ( + directoryWithTwelveThousandFiles = "directoryWithTwelveThousandFiles" + setup.GenerateRandomString(5) + mountFunc func(*test_suite.TestConfig, []string) error +) + +type env struct { + storageClient *storage.Client + ctx context.Context + bucketType string + cfg *test_suite.TestConfig +} -var DirectoryWithTwelveThousandFiles = "directoryWithTwelveThousandFiles" + setup.GenerateRandomString(5) +var testEnv env func TestMain(m *testing.M) { setup.ParseSetUpFlags() - flags := [][]string{{"--implicit-dirs", "--stat-cache-ttl=0", "--kernel-list-cache-ttl-secs=-1"}} - if !testing.Short() { - flags = append(flags, []string{"--client-protocol=grpc", "--implicit-dirs=true", "--stat-cache-ttl=0", "--kernel-list-cache-ttl-secs=-1"}) - } - - if setup.TestBucket() == "" && setup.MountedDirectory() != "" { - log.Print("Please pass the name of bucket mounted at mountedDirectory to --testBucket flag.") - os.Exit(1) + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.ListLargeDir) == 0 { + log.Println("No configuration found for list large dir tests in config. Using flags instead.") + // Populate the config manually. + cfg.ListLargeDir = make([]test_suite.TestConfig, 1) + cfg.ListLargeDir[0].TestBucket = setup.TestBucket() + cfg.ListLargeDir[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.ListLargeDir[0].Configs = make([]test_suite.ConfigItem, 2) + cfg.ListLargeDir[0].Configs[0].Flags = []string{ + "--implicit-dirs=true,--stat-cache-ttl=0,--kernel-list-cache-ttl-secs=-1", + "--client-protocol=grpc,--implicit-dirs=true,--stat-cache-ttl=0,--kernel-list-cache-ttl-secs=-1", + } + cfg.ListLargeDir[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ListLargeDir[0].Configs[0].Run = "TestListLargeDirWithKernelListCache" + cfg.ListLargeDir[0].Configs[1].Flags = []string{ + "--enable-metadata-prefetch --implicit-dirs=true", + "--client-protocol=grpc --enable-metadata-prefetch --implicit-dirs=true", + } + cfg.ListLargeDir[0].Configs[1].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ListLargeDir[0].Configs[1].Run = "TestListLargeDirWithoutKernelListCache" } - // Run tests for mountedDirectory only if --mountedDirectory flag is set. - setup.RunTestsForMountedDirectoryFlag(m) + // 2. Create storage client before running tests. + testEnv.ctx = context.Background() + testEnv.cfg = &cfg.ListLargeDir[0] + testEnv.bucketType = setup.TestEnvironment(testEnv.ctx, &cfg.ListLargeDir[0]) + closeStorageClient := client.CreateStorageClientWithCancel(&testEnv.ctx, &testEnv.storageClient) + defer func() { + err := closeStorageClient() + if err != nil { + log.Fatalf("closeStorageClient failed: %v", err) + } + }() - setup.SetUpTestDirForTestBucketFlag() + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + // flags to be set, as ListLargeDir tests validates content from the bucket. + if cfg.ListLargeDir[0].GKEMountedDirectory != "" && cfg.ListLargeDir[0].TestBucket != "" { + os.Exit(setup.RunTestsForMountedDirectory(cfg.ListLargeDir[0].GKEMountedDirectory, m)) + } - successCode := static_mounting.RunTests(flags, m) + // Run tests for testBucket + // 4. Build the flag sets dynamically from the config. + setup.SetUpTestDirForTestBucket(&cfg.ListLargeDir[0]) + log.Println("Running static mounting tests...") + mountFunc = static_mounting.MountGcsfuseWithStaticMountingWithConfigFile + successCode := m.Run() os.Exit(successCode) } diff --git a/tools/integration_tests/list_large_dir/testdata/create_implicit_dir.sh b/tools/integration_tests/list_large_dir/testdata/create_implicit_dir.sh deleted file mode 100755 index d8d5828c07..0000000000 --- a/tools/integration_tests/list_large_dir/testdata/create_implicit_dir.sh +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -# $1 testbucket -# $2 PrefixImplicitDirInLargeDirListTest -# $3 NumberOfImplicitDirsInDirectoryWithTwelveThousandFiles - 100 -TEST_BUCKET=$1 -IMPLICIT_DIR=$2 -NUMBER_OF_FILES=$3 - -a=1 -#Iterate the loop until a greater than 100 -touch testFile.txt -while [ $a -le $NUMBER_OF_FILES ] -do - dir=$IMPLICIT_DIR$a - a=`expr $a + 1` - gcloud storage cp testFile.txt gs://$TEST_BUCKET/$dir/ -done diff --git a/tools/integration_tests/list_large_dir/testdata/delete_objects.sh b/tools/integration_tests/list_large_dir/testdata/delete_objects.sh deleted file mode 100755 index 4d79e99cae..0000000000 --- a/tools/integration_tests/list_large_dir/testdata/delete_objects.sh +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -# Here $1 refers to the testBucket argument -gcloud storage rm -r gs://$1/** - -# If bucket is empty it will throw an CommandException. -if [ $? -eq 1 ]; then - echo "Bucket is already empty." - exit 0 -fi diff --git a/tools/integration_tests/list_large_dir/testdata/upload_files_to_bucket.sh b/tools/integration_tests/list_large_dir/testdata/upload_files_to_bucket.sh deleted file mode 100755 index 7bf9597393..0000000000 --- a/tools/integration_tests/list_large_dir/testdata/upload_files_to_bucket.sh +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -# $1 testbucket -# $2 DirectoryWithTwelveThousandFiles -# $3 PrefixFileInDirectoryWithTwelveThousandFiles -TEST_BUCKET=$1 -DIR_WITH_TWELVE_THOUSAND_FILES=$2 -FILES=$3 - -cd ~/$DIR_WITH_TWELVE_THOUSAND_FILES -gcloud storage mv $FILES* gs://$TEST_BUCKET/ -cd ../ -rm -r ~/$DIR_WITH_TWELVE_THOUSAND_FILES diff --git a/tools/integration_tests/local_file/create_file_test.go b/tools/integration_tests/local_file/create_file_test.go index 9a18af5475..d21a357952 100644 --- a/tools/integration_tests/local_file/create_file_test.go +++ b/tools/integration_tests/local_file/create_file_test.go @@ -13,45 +13,62 @@ // limitations under the License. // Provides integration tests for create local file. -package local_file_test +package local_file import ( "path" - "testing" - . "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) -func TestNewFileShouldNotGetSyncedToGCSTillClose(t *testing.T) { +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func (t *LocalFileTestSuite) TestNewFileShouldNotGetSyncedToGCSTillClose() { + fileName := path.Base(t.T().Name()) testDirPath = setup.SetupTestDirectory(testDirName) // Validate. - NewFileShouldGetSyncedToGCSAtClose(ctx, storageClient, testDirPath, FileName1, t) + NewFileShouldGetSyncedToGCSAtClose(ctx, storageClient, testDirPath, fileName, t.T()) } -func TestNewFileUnderExplicitDirectoryShouldNotGetSyncedToGCSTillClose(t *testing.T) { +func (t *LocalFileTestSuite) TestNewFileUnderExplicitDirectoryShouldNotGetSyncedToGCSTillClose() { testDirPath = setup.SetupTestDirectory(testDirName) // Make explicit directory. - operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t) + operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t.T()) + fileName := path.Base(t.T().Name()) // Validate. - NewFileShouldGetSyncedToGCSAtClose(ctx, storageClient, testDirPath, path.Join(ExplicitDirName, ExplicitFileName1), t) + NewFileShouldGetSyncedToGCSAtClose(ctx, storageClient, testDirPath, path.Join(ExplicitDirName, fileName), t.T()) } -func TestCreateNewFileWhenSameFileExistsOnGCS(t *testing.T) { +func (t *LocalFileTestSuite) TestCreateNewFileWhenSameFileExistsOnGCS() { + fileName := path.Base(t.T().Name()) testDirPath = setup.SetupTestDirectory(testDirName) // Create a local file. - _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) // Create a file on GCS with the same name. - CreateObjectInGCSTestDir(ctx, storageClient, testDirName, FileName1, GCSFileContent, t) + CreateObjectInGCSTestDir(ctx, storageClient, testDirName, fileName, GCSFileContent, t.T()) // Write to local file. - operations.WriteWithoutClose(fh, FileContents, t) - // Close the local file. - operations.CloseFileShouldNotThrowError(fh, t) + operations.WriteWithoutClose(fh, FileContents, t.T()) + // Validate closing local file throws error. + err := fh.Close() + operations.ValidateESTALEError(t.T(), err) // Ensure that the content on GCS is not overwritten. - ValidateObjectContentsFromGCS(ctx, storageClient, testDirName, FileName1, GCSFileContent, t) + ValidateObjectContentsFromGCS(ctx, storageClient, testDirName, fileName, GCSFileContent, t.T()) +} + +func (t *LocalFileTestSuite) TestEmptyFileCreation() { + fileName := path.Base(t.T().Name()) + testDirPath = setup.SetupTestDirectory(testDirName) + // Create a local file. + _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) + + // Close the file and validate that the file is created on GCS. + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, fileName, "", t.T()) } diff --git a/tools/integration_tests/local_file/edit_file_test.go b/tools/integration_tests/local_file/edit_file_test.go new file mode 100644 index 0000000000..3e139a9b46 --- /dev/null +++ b/tools/integration_tests/local_file/edit_file_test.go @@ -0,0 +1,70 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package local_file + +import ( + "os" + "path" + + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" +) + +func (t *LocalFileTestSuite) TestEditsToNewlyCreatedFile() { + fileName := path.Base(t.T().Name()) + testDirPath = setup.SetupTestDirectory(testDirName) + // Create a local file. + _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) + // Write some contents to file sequentially. + for range 3 { + operations.WriteWithoutClose(fh, FileContents, t.T()) + } + // Close the file and validate that the file is created on GCS. + expectedContent := FileContents + FileContents + FileContents + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, fileName, expectedContent, t.T()) + + // Perform edit + fhNew := operations.OpenFile(path.Join(testDirPath, fileName), t.T()) + newContent := "newContent" + _, err := fhNew.WriteAt([]byte(newContent), 0) + + require.Nil(t.T(), err) + CloseFileAndValidateContentFromGCS(ctx, storageClient, fhNew, testDirName, fileName, newContent+FileContents+FileContents, t.T()) +} + +func (t *LocalFileTestSuite) TestAppendsToNewlyCreatedFile() { + fileName := path.Base(t.T().Name()) + testDirPath = setup.SetupTestDirectory(testDirName) + // Create a local file. + _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) + // Write some contents to file sequentially. + for range 3 { + operations.WriteWithoutClose(fh, FileContents, t.T()) + } + // Close the file and validate that the file is created on GCS. + expectedContent := FileContents + FileContents + FileContents + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, fileName, expectedContent, t.T()) + + // Append to the file. + fhNew, err := os.OpenFile(path.Join(testDirPath, fileName), os.O_RDWR|os.O_APPEND, operations.FilePermission_0777) + require.Nil(t.T(), err) + appendedContent := "appendedContent" + _, err = fhNew.Write([]byte(appendedContent)) + + require.Nil(t.T(), err) + CloseFileAndValidateContentFromGCS(ctx, storageClient, fhNew, testDirName, fileName, expectedContent+appendedContent, t.T()) +} diff --git a/tools/integration_tests/local_file/local_file_helper.go b/tools/integration_tests/local_file/local_file_helper.go new file mode 100644 index 0000000000..221140c356 --- /dev/null +++ b/tools/integration_tests/local_file/local_file_helper.go @@ -0,0 +1,60 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package local_file + +import ( + "context" + "os" + "testing" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" +) + +const ( + onlyDirMounted = "OnlyDirMountLocalFiles" + testDirLocalFileTest = "LocalFileTest" +) + +var ( + testDirName string + testDirPath string + storageClient *storage.Client + ctx context.Context +) + +//////////////////////////////////////////////////////////////////////// +// Helpers +//////////////////////////////////////////////////////////////////////// + +func WritingToLocalFileShouldNotWriteToGCS(ctx context.Context, storageClient *storage.Client, + fh *os.File, testDirName, fileName string, t *testing.T) { + operations.WriteWithoutClose(fh, client.FileContents, t) + client.ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t) +} + +func NewFileShouldGetSyncedToGCSAtClose(ctx context.Context, storageClient *storage.Client, + testDirPath, fileName string, t *testing.T) { + // Create a local file. + _, fh := client.CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t) + + // Writing contents to local file shouldn't create file on GCS. + testDirName := client.GetDirName(testDirPath) + WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, fileName, t) + + // Close the file and validate if the file is created on GCS. + client.CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, fileName, client.FileContents, t) +} diff --git a/tools/integration_tests/local_file/local_file_test.go b/tools/integration_tests/local_file/local_file_test.go deleted file mode 100644 index 15393c8a34..0000000000 --- a/tools/integration_tests/local_file/local_file_test.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Provides integration tests for file and directory operations. - -package local_file_test - -import ( - "context" - "log" - "os" - "path" - "testing" - - "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/dynamic_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/only_dir_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" -) - -const ( - testDirName = "LocalFileTest" - onlyDirMounted = "OnlyDirMountLocalFiles" -) - -var ( - testDirPath string - storageClient *storage.Client - ctx context.Context -) - -//////////////////////////////////////////////////////////////////////// -// Helpers -//////////////////////////////////////////////////////////////////////// - -func WritingToLocalFileShouldNotWriteToGCS(ctx context.Context, storageClient *storage.Client, - fh *os.File, testDirName, fileName string, t *testing.T) { - operations.WriteWithoutClose(fh, client.FileContents, t) - client.ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t) -} - -func NewFileShouldGetSyncedToGCSAtClose(ctx context.Context, storageClient *storage.Client, - testDirPath, fileName string, t *testing.T) { - // Create a local file. - _, fh := client.CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t) - - // Writing contents to local file shouldn't create file on GCS. - testDirName := client.GetDirName(testDirPath) - WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, fileName, t) - - // Close the file and validate if the file is created on GCS. - client.CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, fileName, client.FileContents, t) -} - -//////////////////////////////////////////////////////////////////////// -// TestMain -//////////////////////////////////////////////////////////////////////// - -func TestMain(m *testing.M) { - setup.ParseSetUpFlags() - - setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() - - // Create storage client before running tests. - ctx = context.Background() - closeStorageClient := client.CreateStorageClientWithCancel(&ctx, &storageClient) - defer func() { - err := closeStorageClient() - if err != nil { - log.Fatalf("closeStorageClient failed: %v", err) - } - }() - // To run mountedDirectory tests, we need both testBucket and mountedDirectory - // flags to be set, as local_file tests validates content from the bucket. - if setup.AreBothMountedDirectoryAndTestBucketFlagsSet() { - setup.RunTestsForMountedDirectoryFlag(m) - } - - // Else run tests for testBucket. - // Set up test directory. - setup.SetUpTestDirForTestBucketFlag() - - // Set up flags to run tests on. - // Not setting config file explicitly with 'create-empty-file: false' as it is default. - flagsSet := [][]string{ - {"--implicit-dirs=true", "--rename-dir-limit=3"}, - {"--implicit-dirs=false", "--rename-dir-limit=3"}} - - if hnsFlagSet, err := setup.AddHNSFlagForHierarchicalBucket(ctx, storageClient); err == nil { - flagsSet = append(flagsSet, hnsFlagSet) - } - - if !testing.Short() { - setup.AppendFlagsToAllFlagsInTheFlagsSet(&flagsSet, "--client-protocol=grpc") - } - - successCode := static_mounting.RunTests(flagsSet, m) - - if successCode == 0 { - successCode = only_dir_mounting.RunTests(flagsSet, onlyDirMounted, m) - } - - // Dynamic mounting tests create a bucket and perform tests on that bucket, - // which is not a hierarchical bucket. So we are not running those tests with - // hierarchical bucket. - if successCode == 0 && !setup.IsHierarchicalBucket(ctx, storageClient) { - successCode = dynamic_mounting.RunTests(ctx, storageClient, flagsSet, m) - } - - // Clean up test directory created. - setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(setup.TestBucket(), testDirName)) - os.Exit(successCode) -} diff --git a/tools/integration_tests/local_file/read_dir_test.go b/tools/integration_tests/local_file/read_dir_test.go index 0622f43951..4e8f5903e0 100644 --- a/tools/integration_tests/local_file/read_dir_test.go +++ b/tools/integration_tests/local_file/read_dir_test.go @@ -13,7 +13,7 @@ // limitations under the License. // Provides integration tests for readDir call containing local files. -package local_file_test +package local_file import ( "io/fs" @@ -25,9 +25,9 @@ import ( "testing" "time" - . "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -38,7 +38,7 @@ import ( func creatingNLocalFilesShouldNotThrowError(n int, wg *sync.WaitGroup, t *testing.T) { defer wg.Done() operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t) - for i := 0; i < n; i++ { + for i := range n { filePath := path.Join(testDirPath, ExplicitDirName, FileName1+strconv.FormatInt(int64(i), 10)) operations.CreateFile(filePath, FilePerms, t) } @@ -46,7 +46,7 @@ func creatingNLocalFilesShouldNotThrowError(n int, wg *sync.WaitGroup, t *testin func readingDirNTimesShouldNotThrowError(n int, wg *sync.WaitGroup, t *testing.T) { defer wg.Done() - for i := 0; i < n; i++ { + for i := range n { _, err := os.ReadDir(setup.MntDir()) if err != nil { t.Errorf("Error while reading directory %dth time: %v", i, err) @@ -58,7 +58,7 @@ func readingDirNTimesShouldNotThrowError(n int, wg *sync.WaitGroup, t *testing.T // Tests // ////////////////////////////////////////////////////////////////////// -func TestReadDir(t *testing.T) { +func (t *LocalFileTestSuite) TestReadDir() { // Structure // mntDir/ // mntDir/explicit/ --- directory @@ -67,57 +67,57 @@ func TestReadDir(t *testing.T) { // mntDir/foo2 --- non empty local file // mntDir/foo3 --- gcs synced file + fileName1 := path.Base(t.T().Name()) + "1" + fileName2 := path.Base(t.T().Name()) + "2" + fileName3 := path.Base(t.T().Name()) + "3" + explicitFileName := path.Base(t.T().Name()) + "-explicitFile" testDirPath = setup.SetupTestDirectory(testDirName) // Create explicit dir with 1 local file. - operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t) - _, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, - path.Join(ExplicitDirName, ExplicitFileName1), t) + operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t.T()) + _, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, path.Join(ExplicitDirName, explicitFileName), t.T()) // Create empty local file. - _, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + _, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName1, t.T()) // Create non-empty local file. - _, fh3 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName2, t) - WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh3, testDirName, FileName2, t) + _, fh3 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName2, t.T()) + WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh3, testDirName, fileName2, t.T()) // Create GCS synced file. - CreateObjectInGCSTestDir(ctx, storageClient, testDirName, FileName3, GCSFileContent, t) + CreateObjectInGCSTestDir(ctx, storageClient, testDirName, fileName3, GCSFileContent, t.T()) // Attempt to list mnt and explicit directory. - entriesMnt := operations.ReadDirectory(testDirPath, t) - entriesDir := operations.ReadDirectory(path.Join(testDirPath, ExplicitDirName), t) + entriesMnt := operations.ReadDirectory(testDirPath, t.T()) + entriesDir := operations.ReadDirectory(path.Join(testDirPath, ExplicitDirName), t.T()) // Verify entriesMnt received successfully. - operations.VerifyCountOfDirectoryEntries(4, len(entriesMnt), t) - operations.VerifyDirectoryEntry(entriesMnt[0], ExplicitDirName, t) - operations.VerifyFileEntry(entriesMnt[1], FileName1, 0, t) - operations.VerifyFileEntry(entriesMnt[2], FileName2, SizeOfFileContents, t) - operations.VerifyFileEntry(entriesMnt[3], FileName3, GCSFileSize, t) + operations.VerifyCountOfDirectoryEntries(4, len(entriesMnt), t.T()) + operations.VerifyFileEntry(entriesMnt[0], fileName1, 0, t.T()) + operations.VerifyFileEntry(entriesMnt[1], fileName2, SizeOfFileContents, t.T()) + operations.VerifyFileEntry(entriesMnt[2], fileName3, GCSFileSize, t.T()) + operations.VerifyDirectoryEntry(entriesMnt[3], ExplicitDirName, t.T()) // Verify entriesDir received successfully. - operations.VerifyCountOfDirectoryEntries(1, len(entriesDir), t) - operations.VerifyFileEntry(entriesDir[0], ExplicitFileName1, 0, t) + operations.VerifyCountOfDirectoryEntries(1, len(entriesDir), t.T()) + operations.VerifyFileEntry(entriesDir[0], explicitFileName, 0, t.T()) // Close the local files. - CloseFileAndValidateContentFromGCS(ctx, storageClient, fh1, testDirName, - path.Join(ExplicitDirName, ExplicitFileName1), "", t) - CloseFileAndValidateContentFromGCS(ctx, storageClient, fh2, testDirName, - FileName1, "", t) - CloseFileAndValidateContentFromGCS(ctx, storageClient, fh3, testDirName, - FileName2, FileContents, t) - ValidateObjectContentsFromGCS(ctx, storageClient, testDirName, FileName3, - GCSFileContent, t) + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh1, testDirName, path.Join(ExplicitDirName, explicitFileName), "", t.T()) + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh2, testDirName, fileName1, "", t.T()) + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh3, testDirName, fileName2, FileContents, t.T()) + ValidateObjectContentsFromGCS(ctx, storageClient, testDirName, fileName3, GCSFileContent, t.T()) } -func TestRecursiveListingWithLocalFiles(t *testing.T) { +func (t *LocalFileTestSuite) TestRecursiveListingWithLocalFiles() { // Structure // mntDir/ // mntDir/foo1 --- file // mntDir/explicit/ --- directory // mntDir/explicit/explicitFile1 --- file + fileName := path.Base(t.T().Name()) + explicitFileName := path.Base(t.T().Name()) + "-explicitFile" testDirPath = setup.SetupTestDirectory(testDirName) // Create local file in mnt/ dir. - _, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + _, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) // Create explicit dir with 1 local file. - operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t) - _, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, - path.Join(ExplicitDirName, ExplicitFileName1), t) + operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t.T()) + _, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, path.Join(ExplicitDirName, explicitFileName), t.T()) // Recursively list mntDir/ directory. err := filepath.WalkDir(testDirPath, func(walkPath string, dir fs.DirEntry, err error) error { @@ -129,21 +129,21 @@ func TestRecursiveListingWithLocalFiles(t *testing.T) { return nil } - objs := operations.ReadDirectory(walkPath, t) + objs := operations.ReadDirectory(walkPath, t.T()) // Check if mntDir has correct objects. if walkPath == testDirPath { // numberOfObjects = 2 - operations.VerifyCountOfDirectoryEntries(2, len(objs), t) - operations.VerifyDirectoryEntry(objs[0], ExplicitDirName, t) - operations.VerifyFileEntry(objs[1], FileName1, 0, t) + operations.VerifyCountOfDirectoryEntries(2, len(objs), t.T()) + operations.VerifyFileEntry(objs[0], fileName, 0, t.T()) + operations.VerifyDirectoryEntry(objs[1], ExplicitDirName, t.T()) } // Check if mntDir/explicit/ has correct objects. if walkPath == path.Join(setup.MntDir(), ExplicitDirName) { // numberOfObjects = 1 - operations.VerifyCountOfDirectoryEntries(1, len(objs), t) - operations.VerifyFileEntry(objs[0], ExplicitFileName1, 0, t) + operations.VerifyCountOfDirectoryEntries(1, len(objs), t.T()) + operations.VerifyFileEntry(objs[0], explicitFileName, 0, t.T()) } return nil @@ -151,57 +151,60 @@ func TestRecursiveListingWithLocalFiles(t *testing.T) { // Validate and close the files. if err != nil { - t.Fatalf("filepath.WalkDir() err: %v", err) + t.T().Fatalf("filepath.WalkDir() err: %v", err) } CloseFileAndValidateContentFromGCS(ctx, storageClient, fh1, testDirName, - FileName1, "", t) + fileName, "", t.T()) CloseFileAndValidateContentFromGCS(ctx, storageClient, fh2, testDirName, - path.Join(ExplicitDirName, ExplicitFileName1), "", t) + path.Join(ExplicitDirName, explicitFileName), "", t.T()) } -func TestReadDirWithSameNameLocalAndGCSFile(t *testing.T) { +func (t *LocalFileTestSuite) TestReadDirWithSameNameLocalAndGCSFile() { + fileName := path.Base(t.T().Name()) testDirPath = setup.SetupTestDirectory(testDirName) // Create local file. - _, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + _, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) // Create same name gcs file. time.Sleep(2 * time.Second) - CreateObjectInGCSTestDir(ctx, storageClient, testDirName, FileName1, GCSFileContent, t) + CreateObjectInGCSTestDir(ctx, storageClient, testDirName, fileName, GCSFileContent, t.T()) // Attempt to list testDir. _, err := os.ReadDir(testDirPath) if err != nil { - t.Fatalf("ReadDir err: %v", err) + t.T().Fatalf("ReadDir err: %v", err) } - // Close the local file. - operations.CloseFileShouldNotThrowError(fh1, t) + // Validate closing local file throws error. + err = fh1.Close() + operations.ValidateESTALEError(t.T(), err) } -func TestConcurrentReadDirAndCreationOfLocalFiles_DoesNotThrowError(t *testing.T) { +func (t *LocalFileTestSuite) TestConcurrentReadDirAndCreationOfLocalFiles_DoesNotThrowError() { testDirPath = setup.SetupTestDirectory(testDirName) var wg sync.WaitGroup wg.Add(2) // Concurrently create 100 local files and read directory 200 times. - go creatingNLocalFilesShouldNotThrowError(100, &wg, t) - go readingDirNTimesShouldNotThrowError(200, &wg, t) + go creatingNLocalFilesShouldNotThrowError(100, &wg, t.T()) + go readingDirNTimesShouldNotThrowError(200, &wg, t.T()) wg.Wait() } -func TestStatLocalFileAfterRecreatingItWithSameName(t *testing.T) { +func (t *LocalFileTestSuite) TestStatLocalFileAfterRecreatingItWithSameName() { + fileName := path.Base(t.T().Name()) testDirPath = setup.SetupTestDirectory(testDirName) - filePath := path.Join(testDirPath, FileName1) - operations.CreateFile(filePath, FilePerms, t) + filePath := path.Join(testDirPath, fileName) + operations.CreateFile(filePath, FilePerms, t.T()) _, err := os.Stat(filePath) - require.NoError(t, err) + require.NoError(t.T(), err) err = os.Remove(filePath) - require.NoError(t, err) - operations.CreateFile(filePath, FilePerms, t) + require.NoError(t.T(), err) + operations.CreateFile(filePath, FilePerms, t.T()) f, err := os.Stat(filePath) - assert.NoError(t, err) - assert.Equal(t, FileName1, f.Name()) - assert.False(t, f.IsDir()) + assert.NoError(t.T(), err) + assert.Equal(t.T(), fileName, f.Name()) + assert.False(t.T(), f.IsDir()) } diff --git a/tools/integration_tests/local_file/read_file_test.go b/tools/integration_tests/local_file/read_file_test.go index f4e113f9df..6d1576c74f 100644 --- a/tools/integration_tests/local_file/read_file_test.go +++ b/tools/integration_tests/local_file/read_file_test.go @@ -13,33 +13,38 @@ // limitations under the License. // Provides integration tests for read operation on local files. -package local_file_test +package local_file import ( - "testing" + "path" - . "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) -func TestReadLocalFile(t *testing.T) { +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func (t *LocalFileTestSuite) TestReadLocalFile() { + fileName := path.Base(t.T().Name()) testDirPath = setup.SetupTestDirectory(testDirName) // Create a local file. - _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) // Write FileContents twice to local file. content := FileContents + FileContents - WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, FileName1, t) - WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, FileName1, t) + WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, fileName, t.T()) + WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, fileName, t.T()) // Read the local file contents. buf := make([]byte, len(content)) n, err := fh.ReadAt(buf, 0) if err != nil || len(content) != n || content != string(buf) { - t.Fatalf("Read file operation failed on local file: %v "+ + t.T().Fatalf("Read file operation failed on local file: %v "+ "Expected content: %s, Got Content: %s", err, content, string(buf)) } // Close the file and validate that the file is created on GCS. - CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, FileName1, content, t) + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, fileName, content, t.T()) } diff --git a/tools/integration_tests/local_file/remove_dir_test.go b/tools/integration_tests/local_file/remove_dir_test.go index b93c6e074c..d6ee2718e6 100644 --- a/tools/integration_tests/local_file/remove_dir_test.go +++ b/tools/integration_tests/local_file/remove_dir_test.go @@ -13,61 +13,68 @@ // limitations under the License. // // Provides integration tests for removeDir operation on directories containing local files. -package local_file_test +package local_file import ( "path" - "testing" - . "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) -func TestRmDirOfDirectoryContainingGCSAndLocalFiles(t *testing.T) { +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func (t *LocalFileTestSuite) TestRmDirOfDirectoryContainingGCSAndLocalFiles() { + syncedFileName := path.Base(t.T().Name()) + "synced" + localFileName := path.Base(t.T().Name()) + "local" testDirPath = setup.SetupTestDirectory(testDirName) // Create explicit directory with one synced and one local file. - operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t) - syncedFile := path.Join(ExplicitDirName, FileName1) - localFile := path.Join(ExplicitDirName, FileName2) - _, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, syncedFile, t) - CloseFileAndValidateContentFromGCS(ctx, storageClient, fh1, testDirName, syncedFile, "", t) - _, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, localFile, t) + operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t.T()) + syncedFile := path.Join(ExplicitDirName, syncedFileName) + localFile := path.Join(ExplicitDirName, localFileName) + _, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, syncedFile, t.T()) + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh1, testDirName, syncedFile, "", t.T()) + _, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, localFile, t.T()) // Attempt to remove explicit directory. operations.RemoveDir(path.Join(testDirPath, ExplicitDirName)) // Verify that directory is removed. - operations.ValidateNoFileOrDirError(path.Join(testDirPath, ExplicitDirName), t) + operations.ValidateNoFileOrDirError(t.T(), path.Join(testDirPath, ExplicitDirName)) // Validate writing content to unlinked local file does not throw error. - operations.WriteWithoutClose(fh2, FileContents, t) + operations.WriteWithoutClose(fh2, FileContents, t.T()) // Validate flush file does not throw error and does not create object on GCS. - operations.CloseFileShouldNotThrowError(fh2, t) - ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, localFile, t) + operations.CloseFileShouldNotThrowError(t.T(), fh2) + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, localFile, t.T()) // Validate synced files are also deleted. - ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, syncedFile, t) - ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, ExplicitDirName, t) + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, syncedFile, t.T()) + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, ExplicitDirName, t.T()) } -func TestRmDirOfDirectoryContainingOnlyLocalFiles(t *testing.T) { +func (t *LocalFileTestSuite) TestRmDirOfDirectoryContainingOnlyLocalFiles() { + localFile1Name := path.Base(t.T().Name()) + "1" + localFile2Name := path.Base(t.T().Name()) + "2" testDirPath = setup.SetupTestDirectory(testDirName) // Create a directory with two local files. - operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t) - localFile1 := path.Join(ExplicitDirName, FileName1) - localFile2 := path.Join(ExplicitDirName, FileName2) - _, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, localFile1, t) - _, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, localFile2, t) + operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t.T()) + localFile1 := path.Join(ExplicitDirName, localFile1Name) + localFile2 := path.Join(ExplicitDirName, localFile2Name) + _, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, localFile1, t.T()) + _, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, localFile2, t.T()) // Attempt to remove explicit directory. operations.RemoveDir(path.Join(testDirPath, ExplicitDirName)) // Verify rmDir operation succeeds. - operations.ValidateNoFileOrDirError(path.Join(testDirPath, ExplicitDirName), t) + operations.ValidateNoFileOrDirError(t.T(), path.Join(testDirPath, ExplicitDirName)) // Close the local files and validate they are not present on GCS. - operations.CloseFileShouldNotThrowError(fh1, t) - ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, localFile1, t) - operations.CloseFileShouldNotThrowError(fh2, t) - ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, localFile2, t) + operations.CloseFileShouldNotThrowError(t.T(), fh1) + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, localFile1, t.T()) + operations.CloseFileShouldNotThrowError(t.T(), fh2) + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, localFile2, t.T()) // Validate directory is also deleted. - ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, ExplicitDirName, t) + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, ExplicitDirName, t.T()) } diff --git a/tools/integration_tests/local_file/rename_test.go b/tools/integration_tests/local_file/rename_test.go index ef058138eb..8e66f05c29 100644 --- a/tools/integration_tests/local_file/rename_test.go +++ b/tools/integration_tests/local_file/rename_test.go @@ -13,7 +13,7 @@ // limitations under the License. // Provides integration tests for rename operation on local files. -package local_file_test +package local_file import ( "os" @@ -21,9 +21,10 @@ import ( "strings" "testing" - . "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" ) //////////////////////////////////////////////////////////////////////// @@ -41,38 +42,39 @@ func verifyRenameOperationNotSupported(err error, t *testing.T) { // Tests //////////////////////////////////////////////////////////////////////// -func TestRenameOfLocalFileFails(t *testing.T) { +func (t *LocalFileTestSuite) TestRenameOfLocalFile() { + fileName := path.Base(t.T().Name()) + newFileName := fileName + "new" testDirPath = setup.SetupTestDirectory(testDirName) // Create local file with some content. - _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) - WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, FileName1, t) + _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) + defer operations.CloseFileShouldNotThrowError(t.T(), fh) + WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, fileName, t.T()) // Attempt to rename local file. err := os.Rename( - path.Join(testDirPath, FileName1), - path.Join(testDirPath, NewFileName)) - - // Verify rename operation fails. - verifyRenameOperationNotSupported(err, t) - // write more content to local file. - WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, FileName1, t) - // Close the local file. - CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, - FileName1, FileContents+FileContents, t) + path.Join(testDirPath, fileName), + path.Join(testDirPath, newFileName)) + + // Validate that move didn't throw any error. + require.NoError(t.T(), err) + // Verify the new object contents. + ValidateObjectContentsFromGCS(ctx, storageClient, testDirName, newFileName, FileContents, t.T()) + // Validate old object is deleted. + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t.T()) } -func TestRenameOfDirectoryWithLocalFileFails(t *testing.T) { +func (t *LocalFileTestSuite) TestRenameOfDirectoryWithLocalFileFails() { + fileName1 := path.Base(t.T().Name()) + "1" + fileName2 := path.Base(t.T().Name()) + "2" testDirPath = setup.SetupTestDirectory(testDirName) //Create directory with 1 synced and 1 local file. - operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t) + operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t.T()) // Create synced file. - CreateObjectInGCSTestDir(ctx, storageClient, testDirName, - path.Join(ExplicitDirName, FileName1), GCSFileContent, t) + CreateObjectInGCSTestDir(ctx, storageClient, testDirName, path.Join(ExplicitDirName, fileName1), GCSFileContent, t.T()) // Create local file with some content. - _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, - path.Join(ExplicitDirName, FileName2), t) - WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, - path.Join(ExplicitDirName, FileName2), t) + _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, path.Join(ExplicitDirName, fileName2), t.T()) + WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, path.Join(ExplicitDirName, fileName2), t.T()) // Attempt to rename directory containing local file. err := os.Rename( @@ -80,33 +82,37 @@ func TestRenameOfDirectoryWithLocalFileFails(t *testing.T) { path.Join(testDirPath, NewDirName)) // Verify rename operation fails. - verifyRenameOperationNotSupported(err, t) + verifyRenameOperationNotSupported(err, t.T()) // Write more content to local file. - WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, FileName2, t) + WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, fileName2, t.T()) // Close the local file. - CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, - path.Join(ExplicitDirName, FileName2), FileContents+FileContents, t) + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, path.Join(ExplicitDirName, fileName2), FileContents+FileContents, t.T()) } -func TestRenameOfLocalFileSucceedsAfterSync(t *testing.T) { - TestRenameOfLocalFileFails(t) +func (t *LocalFileTestSuite) TestRenameOfLocalFileSucceedsAfterSync() { + fileName := path.Base(t.T().Name()) + newFileName := fileName + "new" + testDirPath = setup.SetupTestDirectory(testDirName) + // Create local file with some content. + _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) + WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, fileName, t.T()) + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, fileName, FileContents, t.T()) // Attempt to Rename synced file. err := os.Rename( - path.Join(testDirPath, FileName1), - path.Join(testDirPath, NewFileName)) + path.Join(testDirPath, fileName), + path.Join(testDirPath, newFileName)) // Validate. if err != nil { - t.Fatalf("os.Rename() failed on synced file: %v", err) + t.T().Fatalf("os.Rename() failed on synced file: %v", err) } - ValidateObjectContentsFromGCS(ctx, storageClient, testDirName, NewFileName, - FileContents+FileContents, t) - ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, FileName1, t) + ValidateObjectContentsFromGCS(ctx, storageClient, testDirName, newFileName, FileContents, t.T()) + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t.T()) } -func TestRenameOfDirectoryWithLocalFileSucceedsAfterSync(t *testing.T) { - TestRenameOfDirectoryWithLocalFileFails(t) +func (t *LocalFileTestSuite) TestRenameOfDirectoryWithLocalFileSucceedsAfterSync() { + t.TestRenameOfDirectoryWithLocalFileFails() // Attempt to rename directory again after sync. err := os.Rename( @@ -115,14 +121,12 @@ func TestRenameOfDirectoryWithLocalFileSucceedsAfterSync(t *testing.T) { // Validate. if err != nil { - t.Fatalf("os.Rename() failed on directory containing synced files: %v", err) + t.T().Fatalf("os.Rename() failed on directory containing synced files: %v", err) } - ValidateObjectContentsFromGCS(ctx, storageClient, testDirName, - path.Join(NewDirName, FileName1), GCSFileContent, t) - ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, - path.Join(ExplicitDirName, FileName1), t) - ValidateObjectContentsFromGCS(ctx, storageClient, testDirName, - path.Join(NewDirName, FileName2), FileContents+FileContents, t) - ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, - path.Join(ExplicitDirName, FileName2), t) + fileName1 := path.Base(t.T().Name()) + "1" + fileName2 := path.Base(t.T().Name()) + "2" + ValidateObjectContentsFromGCS(ctx, storageClient, testDirName, path.Join(NewDirName, fileName1), GCSFileContent, t.T()) + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, path.Join(ExplicitDirName, fileName1), t.T()) + ValidateObjectContentsFromGCS(ctx, storageClient, testDirName, path.Join(NewDirName, fileName2), FileContents+FileContents, t.T()) + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, path.Join(ExplicitDirName, fileName2), t.T()) } diff --git a/tools/integration_tests/local_file/setup_test.go b/tools/integration_tests/local_file/setup_test.go new file mode 100644 index 0000000000..b8d1d80327 --- /dev/null +++ b/tools/integration_tests/local_file/setup_test.go @@ -0,0 +1,110 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Provides integration tests for file and directory operations. + +package local_file + +import ( + "context" + "log" + "os" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/dynamic_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/only_dir_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" + "github.com/stretchr/testify/suite" +) + +//////////////////////////////////////////////////////////////////////// +// TestMain +//////////////////////////////////////////////////////////////////////// + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + // set the test dir to local file test + testDirName = testDirLocalFileTest + + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.LocalFile) == 0 { + log.Println("No configuration found for LocalFile tests in config. Using flags instead.") + // Populate the config manually. + cfg.LocalFile = make([]test_suite.TestConfig, 1) + cfg.LocalFile[0].TestBucket = setup.TestBucket() + cfg.LocalFile[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.LocalFile[0].Configs = make([]test_suite.ConfigItem, 2) + cfg.LocalFile[0].Configs[0].Flags = []string{ + "--implicit-dirs=true --rename-dir-limit=3 --enable-streaming-writes=false", + "--implicit-dirs=false --rename-dir-limit=3 --enable-streaming-writes=false --client-protocol=grpc", + "--rename-dir-limit=3 --write-block-size-mb=1 --write-max-blocks-per-file=2 --write-global-max-blocks=0", + } + cfg.LocalFile[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.LocalFile[0].Configs[1].Flags = []string{ + "--rename-dir-limit=3 --write-block-size-mb=1 --write-max-blocks-per-file=2 --write-global-max-blocks=-1 --client-protocol=grpc", + "--rename-dir-limit=3 --write-block-size-mb=1 --write-max-blocks-per-file=2 --write-global-max-blocks=-1", + } + cfg.LocalFile[0].Configs[1].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": false} + } + + // 2. Create storage client before running tests. + ctx = context.Background() + bucketType := setup.TestEnvironment(ctx, &cfg.LocalFile[0]) + closeStorageClient := client.CreateStorageClientWithCancel(&ctx, &storageClient) + defer func() { + err := closeStorageClient() + if err != nil { + log.Fatalf("closeStorageClient failed: %v", err) + } + }() + + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + // flags to be set, as LocalFile tests validates content from the bucket. + if cfg.LocalFile[0].GKEMountedDirectory != "" && cfg.LocalFile[0].TestBucket != "" { + os.Exit(setup.RunTestsForMountedDirectory(cfg.LocalFile[0].GKEMountedDirectory, m)) + } + + // Run tests for testBucket. + // 4. Build the flag sets dynamically from the config. + flags := setup.BuildFlagSets(cfg.LocalFile[0], bucketType, "") + + setup.SetUpTestDirForTestBucket(&cfg.LocalFile[0]) + + successCode := static_mounting.RunTestsWithConfigFile(&cfg.LocalFile[0], flags, m) + + if successCode == 0 { + successCode = only_dir_mounting.RunTestsWithConfigFile(&cfg.LocalFile[0], flags, onlyDirMounted, m) + } + + // Dynamic mounting tests. + if successCode == 0 { + successCode = dynamic_mounting.RunTestsWithConfigFile(&cfg.LocalFile[0], flags, m) + } + + os.Exit(successCode) +} + +type LocalFileTestSuite struct { + suite.Suite +} + +func TestLocalFileTestSuite(t *testing.T) { + s := new(LocalFileTestSuite) + suite.Run(t, s) +} diff --git a/tools/integration_tests/local_file/stat_file_test.go b/tools/integration_tests/local_file/stat_file_test.go index d84d765371..bccf7f58b3 100644 --- a/tools/integration_tests/local_file/stat_file_test.go +++ b/tools/integration_tests/local_file/stat_file_test.go @@ -13,71 +13,73 @@ // limitations under the License. // Provides integration tests for stat operation on local files. -package local_file_test +package local_file import ( "os" - "testing" + "path" - "github.com/googlecloudplatform/gcsfuse/v2/internal/fs/inode" - . "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/internal/fs/inode" + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) -func TestStatOnLocalFile(t *testing.T) { +func (t *LocalFileTestSuite) TestStatOnLocalFile() { + fileName := path.Base(t.T().Name()) testDirPath = setup.SetupTestDirectory(testDirName) // Create a local file. - filePath, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + filePath, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) // Stat the local file. - operations.VerifyStatFile(filePath, 0, FilePerms, t) + operations.VerifyStatFile(filePath, 0, FilePerms, t.T()) // Writing contents to local file shouldn't create file on GCS. - WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, FileName1, t) + WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, fileName, t.T()) // Stat the local file again to check if new content is written. - operations.VerifyStatFile(filePath, SizeOfFileContents, FilePerms, t) + operations.VerifyStatFile(filePath, SizeOfFileContents, FilePerms, t.T()) // Close the file and validate that the file is created on GCS. CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, - FileName1, FileContents, t) + fileName, FileContents, t.T()) } -func TestStatOnLocalFileWithConflictingFileNameSuffix(t *testing.T) { +func (t *LocalFileTestSuite) TestStatOnLocalFileWithConflictingFileNameSuffix() { + fileName := path.Base(t.T().Name()) testDirPath = setup.SetupTestDirectory(testDirName) // Create a local file. - filePath, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + filePath, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) // Stat the local file. - operations.VerifyStatFile(filePath+inode.ConflictingFileNameSuffix, 0, FilePerms, t) + operations.VerifyStatFile(filePath+inode.ConflictingFileNameSuffix, 0, FilePerms, t.T()) // Close the file and validate that the file is created on GCS. CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, - FileName1, "", t) + fileName, "", t.T()) } -func TestTruncateLocalFile(t *testing.T) { +func (t *LocalFileTestSuite) TestTruncateLocalFileToSmallerSize() { testDirPath = setup.SetupTestDirectory(testDirName) // Create a local file. - filePath, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + fileName := path.Base(t.T().Name()) + setup.GenerateRandomString(5) + filePath, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) // Writing contents to local file . - WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, FileName1, t) + operations.WriteWithoutClose(fh, FileContents, t.T()) // Stat the file to validate if new contents are written. - operations.VerifyStatFile(filePath, SizeOfFileContents, FilePerms, t) + operations.VerifyStatFile(filePath, SizeOfFileContents, FilePerms, t.T()) - // Truncate the file to update the file size. - err := os.Truncate(filePath, SizeTruncate) + // Truncate the file to update file size to smaller file size. + err := os.Truncate(filePath, SmallerSizeTruncate) if err != nil { - t.Fatalf("os.Truncate err: %v", err) + t.T().Fatalf("os.Truncate err: %v", err) } - ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, FileName1, t) // Stat the file to validate if file is truncated correctly. - operations.VerifyStatFile(filePath, SizeTruncate, FilePerms, t) + operations.VerifyStatFile(filePath, SmallerSizeTruncate, FilePerms, t.T()) // Close the file and validate that the file is created on GCS. CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, - FileName1, "testS", t) + fileName, FileContents[:SmallerSizeTruncate], t.T()) } diff --git a/tools/integration_tests/local_file/sym_link_test.go b/tools/integration_tests/local_file/sym_link_test.go index 0dc0178945..c44f13da11 100644 --- a/tools/integration_tests/local_file/sym_link_test.go +++ b/tools/integration_tests/local_file/sym_link_test.go @@ -13,24 +13,26 @@ // limitations under the License. // // Provides integration tests for symlink operation on local files. -package local_file_test +package local_file import ( "os" "path" - "strings" "testing" - . "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func createAndVerifySymLink(t *testing.T) (filePath, symlink string, fh *os.File) { + fileName := path.Base(t.Name()) testDirPath = setup.SetupTestDirectory(testDirName) // Create a local file. - filePath, fh = CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) - WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, FileName1, t) + filePath, fh = CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t) + WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, fileName, t) // Create the symlink. symlink = path.Join(testDirPath, "bar") @@ -42,22 +44,39 @@ func createAndVerifySymLink(t *testing.T) (filePath, symlink string, fh *os.File return } -func TestCreateSymlinkForLocalFile(t *testing.T) { - _, _, fh := createAndVerifySymLink(t) - CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, - FileName1, FileContents, t) +func (t *LocalFileTestSuite) TestCreateSymlinkForLocalFile() { + fileName := path.Base(t.T().Name()) + _, _, fh := createAndVerifySymLink(t.T()) + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, fileName, FileContents, t.T()) } -func TestReadSymlinkForDeletedLocalFile(t *testing.T) { - filePath, symlink, fh := createAndVerifySymLink(t) +func (t *LocalFileTestSuite) TestReadSymlinkForDeletedLocalFile() { + fileName := path.Base(t.T().Name()) + filePath, symlink, fh := createAndVerifySymLink(t.T()) // Remove filePath and then close the fileHandle to avoid syncing to GCS. operations.RemoveFile(filePath) - operations.CloseFileShouldNotThrowError(fh, t) - ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, FileName1, t) + operations.CloseFileShouldNotThrowError(t.T(), fh) + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t.T()) // Reading symlink should fail. _, err := os.Stat(symlink) - if err == nil || !strings.Contains(err.Error(), "no such file or directory") { - t.Fatalf("Reading symlink for deleted local file did not fail.") - } + + require.Error(t.T(), err) + assert.True(t.T(), os.IsNotExist(err), "Reading symlink for deleted local file should have failed with 'no such file or directory'. Got: %v", err) +} + +func (t *LocalFileTestSuite) TestRenameSymlinkForLocalFile() { + fileName := path.Base(t.T().Name()) + filePath, symlinkPath, fh := createAndVerifySymLink(t.T()) + newSymlinkPath := path.Join(testDirPath, "newSymlink") + + err := os.Rename(symlinkPath, newSymlinkPath) + + require.NoError(t.T(), err, "os.Rename failed for symlink") + _, err = os.Lstat(symlinkPath) + require.Error(t.T(), err) + assert.True(t.T(), os.IsNotExist(err), "Old symlink should not exist after rename. err: %v", err) + operations.VerifyReadLink(filePath, newSymlinkPath, t.T()) + operations.VerifyReadFile(newSymlinkPath, FileContents, t.T()) + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, fileName, FileContents, t.T()) } diff --git a/tools/integration_tests/local_file/unlinked_file_test.go b/tools/integration_tests/local_file/unlinked_file_test.go index d1db7ae3c4..a4d2f6bf21 100644 --- a/tools/integration_tests/local_file/unlinked_file_test.go +++ b/tools/integration_tests/local_file/unlinked_file_test.go @@ -13,89 +13,142 @@ // limitations under the License. // Provides integration tests for operation on unlinked local files. -package local_file_test +package local_file import ( "path" - "testing" - . "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) -func TestStatOnUnlinkedLocalFile(t *testing.T) { +func (t *LocalFileTestSuite) TestStatOnUnlinkedLocalFile() { + fileName := path.Base(t.T().Name()) testDirPath = setup.SetupTestDirectory(testDirName) // Create a local file. - filePath, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + filePath, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) // Unlink the local file. operations.RemoveFile(filePath) // Stat the local file and validate error. - operations.ValidateNoFileOrDirError(path.Join(testDirPath, FileName1), t) + operations.ValidateNoFileOrDirError(t.T(), path.Join(testDirPath, fileName)) // Close the file and validate that file is not created on GCS. - operations.CloseFileShouldNotThrowError(fh, t) - ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, FileName1, t) + operations.CloseFileShouldNotThrowError(t.T(), fh) + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t.T()) } -func TestReadDirContainingUnlinkedLocalFiles(t *testing.T) { +func (t *LocalFileTestSuite) TestReadDirContainingUnlinkedLocalFiles() { + fileName1 := path.Base(t.T().Name()) + "1" + fileName2 := path.Base(t.T().Name()) + "2" + fileName3 := path.Base(t.T().Name()) + "3" testDirPath = setup.SetupTestDirectory(testDirName) // Create local files. - _, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) - _, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName2, t) - filepath3, fh3 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName3, t) + _, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName1, t.T()) + _, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName2, t.T()) + filepath3, fh3 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName3, t.T()) // Unlink local file 3. operations.RemoveFile(filepath3) // Attempt to list testDir. - entries := operations.ReadDirectory(testDirPath, t) + entries := operations.ReadDirectory(testDirPath, t.T()) // Verify unlinked entries are not listed. - operations.VerifyCountOfDirectoryEntries(2, len(entries), t) - operations.VerifyFileEntry(entries[0], FileName1, 0, t) - operations.VerifyFileEntry(entries[1], FileName2, 0, t) + operations.VerifyCountOfDirectoryEntries(2, len(entries), t.T()) + operations.VerifyFileEntry(entries[0], fileName1, 0, t.T()) + operations.VerifyFileEntry(entries[1], fileName2, 0, t.T()) // Close the local files and validate they are written to GCS. CloseFileAndValidateContentFromGCS(ctx, storageClient, fh1, testDirName, - FileName1, "", t) + fileName1, "", t.T()) CloseFileAndValidateContentFromGCS(ctx, storageClient, fh2, testDirName, - FileName2, "", t) + fileName2, "", t.T()) // Verify unlinked file is not written to GCS. - operations.CloseFileShouldNotThrowError(fh3, t) - ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, FileName3, t) + operations.CloseFileShouldNotThrowError(t.T(), fh3) + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName3, t.T()) } -func TestWriteOnUnlinkedLocalFileSucceeds(t *testing.T) { +func (t *LocalFileTestSuite) TestWriteOnUnlinkedLocalFileSucceeds() { + fileName := path.Base(t.T().Name()) testDirPath = setup.SetupTestDirectory(testDirName) // Create local file. - filepath, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + filepath, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) // Verify unlink operation succeeds. operations.RemoveFile(filepath) - operations.ValidateNoFileOrDirError(path.Join(testDirPath, FileName1), t) + operations.ValidateNoFileOrDirError(t.T(), path.Join(testDirPath, fileName)) // Write to unlinked local file. - operations.WriteWithoutClose(fh, FileContents, t) + operations.WriteWithoutClose(fh, FileContents, t.T()) // Validate flush file does not throw error. - operations.CloseFileShouldNotThrowError(fh, t) + operations.CloseFileShouldNotThrowError(t.T(), fh) // Validate unlinked file is not written to GCS. - ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, FileName1, t) + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t.T()) } -func TestSyncOnUnlinkedLocalFile(t *testing.T) { +func (t *LocalFileTestSuite) TestSyncOnUnlinkedLocalFile() { + fileName := path.Base(t.T().Name()) testDirPath = setup.SetupTestDirectory(testDirName) // Create local file. - filepath, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + filepath, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) // Attempt to unlink local file. operations.RemoveFile(filepath) // Verify unlink operation succeeds. - operations.ValidateNoFileOrDirError(path.Join(testDirPath, FileName1), t) + operations.ValidateNoFileOrDirError(t.T(), path.Join(testDirPath, fileName)) // Validate sync operation does not write to GCS after unlink. - operations.SyncFile(fh, t) - ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, FileName1, t) + operations.SyncFile(fh, t.T()) + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t.T()) // Close the local file and validate it is not present on GCS. - operations.CloseFileShouldNotThrowError(fh, t) - ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, FileName1, t) + operations.CloseFileShouldNotThrowError(t.T(), fh) + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t.T()) +} + +func (t *LocalFileTestSuite) TestFileWithSameNameCanBeCreatedWhenDeletedBeforeSync() { + fileName := path.Base(t.T().Name()) + testDirPath = setup.SetupTestDirectory(testDirName) + // Create a local file. + filePath, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) + // Write some content. + operations.WriteWithoutClose(fh, FileContents, t.T()) + // Remove and close the file. + operations.RemoveFile(filePath) + // Currently flush calls returns error if unlinked. Ignoring that error here. + _ = fh.Close() + // Validate that file is not created on GCS + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t.T()) + // Verify unlink operation succeeds. + operations.ValidateNoFileOrDirError(t.T(), path.Join(testDirPath, fileName)) + + // Create a local file. + _, fh = CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) + + newContents := "newContents" + operations.WriteWithoutClose(fh, newContents, t.T()) + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, fileName, newContents, t.T()) +} + +func (t *LocalFileTestSuite) TestFileWithSameNameCanBeCreatedAfterDelete() { + fileName := path.Base(t.T().Name()) + testDirPath = setup.SetupTestDirectory(testDirName) + // Create a local file. + filePath, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) + // Write some content. + operations.WriteWithoutClose(fh, FileContents, t.T()) + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, + fileName, FileContents, t.T()) + // Remove the file. + operations.RemoveFile(filePath) + // Validate that file id deleted from GCS + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t.T()) + // Verify unlink operation succeeds. + operations.ValidateNoFileOrDirError(t.T(), path.Join(testDirPath, fileName)) + + // Create a local file. + _, fh = CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) + + newContents := "newContents" + operations.WriteWithoutClose(fh, newContents, t.T()) + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, fileName, newContents, t.T()) } diff --git a/tools/integration_tests/local_file/write_file_test.go b/tools/integration_tests/local_file/write_file_test.go index a98aaaa5b2..9aadb5441b 100644 --- a/tools/integration_tests/local_file/write_file_test.go +++ b/tools/integration_tests/local_file/write_file_test.go @@ -13,44 +13,106 @@ // limitations under the License. // Provides integration tests for write on local files. -package local_file_test +package local_file import ( - "testing" + "path" - . "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) -func TestMultipleWritesToLocalFile(t *testing.T) { +func (t *LocalFileTestSuite) TestMultipleWritesToLocalFile() { + fileName := path.Base(t.T().Name()) testDirPath = setup.SetupTestDirectory(testDirName) // Create a local file. - _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) // Write some contents to file sequentially. - for i := 0; i < 3; i++ { - operations.WriteWithoutClose(fh, FileContents, t) + for range 3 { + operations.WriteWithoutClose(fh, FileContents, t.T()) } - ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, FileName1, t) + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t.T()) // Close the file and validate that the file is created on GCS. CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, - FileName1, FileContents+FileContents+FileContents, t) + fileName, FileContents+FileContents+FileContents, t.T()) } -func TestRandomWritesToLocalFile(t *testing.T) { +func (t *LocalFileTestSuite) TestRandomWritesToLocalFile() { + fileName := path.Base(t.T().Name()) testDirPath = setup.SetupTestDirectory(testDirName) // Create a local file. - _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) // Write some contents to file randomly. - operations.WriteAt("string1", 0, fh, t) - operations.WriteAt("string2", 2, fh, t) - operations.WriteAt("string3", 3, fh, t) - ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, FileName1, t) + operations.WriteAt("string1", 0, fh, t.T()) + operations.WriteAt("string2", 2, fh, t.T()) + operations.WriteAt("string3", 3, fh, t.T()) + // Close the file and validate that the file is created on GCS. + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, fileName, "stsstring3", t.T()) +} + +func (t *LocalFileTestSuite) TestOutOfOrderWritesToNewFile() { + fileName := path.Base(t.T().Name()) + testDirPath = setup.SetupTestDirectory(testDirName) + // Create a local file. + _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) + + // Write some contents to file sequentially. + for range 2 { + operations.WriteWithoutClose(fh, FileContents, t.T()) + } + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t.T()) + + // Write at previous offset. + operations.WriteAt("hello", 0, fh, t.T()) + + expectedString := "hellotringtestString" + // Close the file and validate that the file is created on GCS. + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, + fileName, expectedString, t.T()) +} + +func (t *LocalFileTestSuite) TestMultipleOutOfOrderWritesToNewFile() { + fileName := path.Base(t.T().Name()) + testDirPath = setup.SetupTestDirectory(testDirName) + // Create a local file. + _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) + + // Write some contents to file sequentially. + for range 2 { + operations.WriteWithoutClose(fh, FileContents, t.T()) + } + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t.T()) + + // Write at previous offset. + operations.WriteAt("hello", 15, fh, t.T()) + // Write at new offset. + operations.WriteAt("hey", 30, fh, t.T()) + + emptyBytes := [10]byte{} + expectedString := "testStringtestShello" + string(emptyBytes[:]) + "hey" + // Close the file and validate that the file is created on GCS. + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, + fileName, expectedString, t.T()) +} + +func (t *LocalFileTestSuite) TestWritesToNewFileStartingAtNonZeroOffset() { + fileName := path.Base(t.T().Name()) + testDirPath = setup.SetupTestDirectory(testDirName) + // Create a local file. + _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t.T()) + // Write at future offset. + operations.WriteAt("hello", 15, fh, t.T()) + // Write at zero offset now. + operations.WriteAt("hey", 0, fh, t.T()) + + emptyBytes := [12]byte{} + expectedString := "hey" + string(emptyBytes[:]) + "hello" // Close the file and validate that the file is created on GCS. CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, - FileName1, "stsstring3", t) + fileName, expectedString, t.T()) } diff --git a/tools/integration_tests/log_content/file_upload_log_test.go b/tools/integration_tests/log_content/file_upload_log_test.go deleted file mode 100644 index c70f870c70..0000000000 --- a/tools/integration_tests/log_content/file_upload_log_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package log_content - -import ( - "fmt" - "math" - "os" - "path" - "testing" - - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" -) - -const ( - BigFileSize int64 = 50 * operations.MiB - SmallFileSize int64 = operations.MiB - DirForBigFileUploadLogTest = "dirForBigFileUploadLogTest" - DirForSmallFileUploadLogTest = "dirForSmallFileUploadLogTest" - FileName = "fileName.txt" -) - -func uploadFile(t *testing.T, dirNamePrefix string, fileSize int64) { - testDir, err := os.MkdirTemp(setup.MntDir(), dirNamePrefix+"-*") - if err != nil || testDir == "" { - t.Fatalf("Error in creating test-directory:%v", err) - } - // Clean up. - defer operations.RemoveDir(testDir) - - filePath := path.Join(testDir, FileName) - - // Sequentially write the data in file. - err = operations.WriteFileSequentially(filePath, fileSize, fileSize) - if err != nil { - t.Fatalf("Error in writing file: %v", err) - } -} - -func extractRelevantLogsFromLogFile(t *testing.T, logFile string, logFileOffset int64) (logString string) { - // Read the entire log file at once. This can be optimized by reading - // a bunch of lines at once, then eliminating the found - // expected substrings one by one. - bytes, err := os.ReadFile(logFile) - if err != nil { - t.Errorf("Failed in reading logfile %q: %v", logFile, err) - } - completeLogString := string(bytes) - - logString = completeLogString[logFileOffset:] - return -} - -func uploadFileAndReturnLogs(t *testing.T, dirName string, fileSize int64) string { - var err error - var logFileOffset int64 - if logFileOffset, err = operations.SizeOfFile(setup.LogFile()); err != nil { - t.Fatal(err) - } - - uploadFile(t, dirName, fileSize) - return extractRelevantLogsFromLogFile(t, setup.LogFile(), logFileOffset) -} - -func TestBigFileUploadLog(t *testing.T) { - logString := uploadFileAndReturnLogs(t, DirForBigFileUploadLogTest, BigFileSize) - - // Big files (> 16 MiB) are uploaded sequentially in chunks of size - // 16 MiB each and each chunk's successful upload generates a log. - gcsWriteChunkSize := 16 * operations.MiB - numTotalChunksToBeCompleted := int(math.Floor(float64(BigFileSize) / float64(gcsWriteChunkSize))) - var expectedSubstrings []string - for numChunksCompletedSoFar := 1; numChunksCompletedSoFar <= numTotalChunksToBeCompleted; numChunksCompletedSoFar++ { - expectedSubstrings = append(expectedSubstrings, fmt.Sprintf("%d bytes uploaded so far", numChunksCompletedSoFar*gcsWriteChunkSize)) - } - - operations.VerifyExpectedSubstrings(t, logString, expectedSubstrings) -} - -func TestSmallFileUploadLog(t *testing.T) { - logString := uploadFileAndReturnLogs(t, DirForSmallFileUploadLogTest, SmallFileSize) - - // The file being uploaded is too small (<16 MB) for progress logs - // to be printed. - unexpectedLogSubstrings := []string{"bytes uploaded so far"} - operations.VerifyUnexpectedSubstrings(t, logString, unexpectedLogSubstrings) -} diff --git a/tools/integration_tests/log_content/log_content_test.go b/tools/integration_tests/log_content/log_content_test.go deleted file mode 100644 index 6b3984887b..0000000000 --- a/tools/integration_tests/log_content/log_content_test.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Provides integration tests for write large files sequentially and randomly. -package log_content - -import ( - "log" - "os" - "testing" - - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" -) - -func TestMain(m *testing.M) { - setup.ParseSetUpFlags() - - // This test supports the scenario where only a testBucket has been passed. - // If a user passes a mountedDirectory, then the - // test cannot ensure that logs are generated for it, - // and thus does not support that scenario. - setup.ExitWithFailureIfMountedDirectoryIsSetOrTestBucketIsNotSet() - - // Enable tests for testBucket - setup.SetUpTestDirForTestBucketFlag() - - // Set up a log file. - logFile, err := os.CreateTemp(setup.TestDir(), "log_content_test-*.log") - if err != nil || logFile == nil { - log.Fatalf("Failed to create temp-file for logging: %v", err) - } - defer logFile.Close() - setup.SetLogFile(logFile.Name()) - - // No explicit flags need to be set. Only debugs log are to be enabled, - // which are enabled by default by static_mounting.RunTests - // and by the above call to set log-file. - flagsSet := [][]string{{}, {"--client-protocol=grpc"}} - - successCode := 0 - if successCode == 0 { - successCode = static_mounting.RunTests(flagsSet, m) - } - - os.Exit(successCode) -} diff --git a/tools/integration_tests/log_rotation/log_rotation_test.go b/tools/integration_tests/log_rotation/log_rotation_test.go index bbcdfbc10f..f395dffaa9 100644 --- a/tools/integration_tests/log_rotation/log_rotation_test.go +++ b/tools/integration_tests/log_rotation/log_rotation_test.go @@ -22,53 +22,63 @@ import ( "os" "path" "testing" + "time" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" ) const ( - testDirName = "LogRotationTest" - logFileName = "log.txt" - logDirName = "gcsfuse_integration_test_logs" + testDirName = "TestLogRotation" maxFileSizeMB = 2 activeLogFileCount = 1 stderrLogFileCount = 1 backupLogFileCount = 2 logFileCount = activeLogFileCount + backupLogFileCount + stderrLogFileCount // Adding 1 for stderr logs file + GKETempDir = "/gcsfuse-tmp" + retryFrequency = 3 * time.Second + retryTimeout = 2 * time.Minute ) var ( - logDirPath string - logFilePath string + storageClient *storage.Client + ctx context.Context + cfg *test_suite.TestConfig ) -func getMountConfigForLogRotation(maxFileSizeMB, backupFileCount int, compress bool, logFilePath string) map[string]interface{} { - yamlContent := map[string]interface{}{ - "logging": map[string]interface{}{ - "severity": "TRACE", - "file-path": logFilePath, - "log-rotate": map[string]interface{}{ - "max-file-size-mb": maxFileSizeMB, - "backup-file-count": backupFileCount, - "compress": compress, - }, - }, - } - return yamlContent +func setupLogFilePath(testName string) { + var logFilePath = path.Join(setup.TestDir(), GKETempDir, testName) + ".log" + cfg.LogFile = logFilePath } -//////////////////////////////////////////////////////////////////////// -// TestMain -//////////////////////////////////////////////////////////////////////// - func TestMain(m *testing.M) { setup.ParseSetUpFlags() - var storageClient *storage.Client - ctx := context.Background() + // 1. Load and parse the common configuration. + config := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(config.LogRotation) == 0 { + log.Println("No configuration found for log rotation tests in config. Using flags instead.") + // Populate the config manually. + config.LogRotation = make([]test_suite.TestConfig, 1) + config.LogRotation[0].TestBucket = setup.TestBucket() + config.LogRotation[0].GKEMountedDirectory = setup.MountedDirectory() + config.LogRotation[0].LogFile = setup.LogFile() + config.LogRotation[0].Configs = make([]test_suite.ConfigItem, 1) + config.LogRotation[0].Configs[0].Flags = []string{ + "--log-file=/gcsfuse-tmp/TestLogRotation.log --log-rotate-max-file-size-mb=2 --log-rotate-backup-file-count=2 --log-rotate-compress=false --log-severity=trace", + "--log-file=/gcsfuse-tmp/TestLogRotation.log --log-rotate-max-file-size-mb=2 --log-rotate-backup-file-count=2 --log-rotate-compress=true --log-severity=trace", + } + config.LogRotation[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + } + + cfg = &config.LogRotation[0] + ctx = context.Background() + bucketType := setup.TestEnvironment(ctx, cfg) + + // 2. Create storage client before running tests. closeStorageClient := client.CreateStorageClientWithCancel(&ctx, &storageClient) defer func() { err := closeStorageClient() @@ -77,40 +87,27 @@ func TestMain(m *testing.M) { } }() - setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() - - // Run tests for mountedDirectory only if --mountedDirectory flag is set. - - logDirPath = setup.ValidateLogDirForMountedDirTests(logDirName) - logFilePath = path.Join(logDirPath, logFileName) - setup.RunTestsForMountedDirectoryFlag(m) - - // Else run tests for testBucket. - // Set up test directory. - setup.SetUpTestDirForTestBucketFlag() - - // Set up directory for logs. - logDirPath = setup.SetUpLogDirForTestDirTests(logDirName) - logFilePath = path.Join(logDirPath, logFileName) - setup.SetLogFile(logFilePath) - - // Set up config files. - // TODO: add tests for backupLogFileCount = 0. - configFile1 := setup.YAMLConfigFile( - getMountConfigForLogRotation(maxFileSizeMB, backupLogFileCount, true, logFilePath), - "config1.yaml") - configFile2 := setup.YAMLConfigFile( - getMountConfigForLogRotation(maxFileSizeMB, backupLogFileCount, false, logFilePath), - "config2.yaml") - - // Set up flags to run tests on. - // Not setting config file explicitly with 'create-empty-file: false' as it is default. - flags := [][]string{ - {"--config-file=" + configFile1}, - {"--config-file=" + configFile2}, + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + if cfg.GKEMountedDirectory != "" && cfg.TestBucket != "" { + log.Println("These tests will not run with mounted directory..") + return } - successCode := static_mounting.RunTests(flags, m) + // 4. Build the flag sets dynamically from the config. + setup.SetUpTestDirForTestBucket(cfg) + + // 5. Create the temporary directory for log rotation logs for GCE environment. + if err := os.MkdirAll(path.Join(setup.TestDir(), GKETempDir), 0755); err != nil { + log.Fatalf("Failed to create log directory: %v", err) + } + + // 6. Override GKE specific paths with GCSFuse paths if running in GCE environment. + setup.OverrideFilePathsInFlagSet(cfg, setup.TestDir()) + + flags := setup.BuildFlagSets(*cfg, bucketType, "") + setupLogFilePath(testDirName) + + successCode := static_mounting.RunTestsWithConfigFile(cfg, flags, m) // Clean up test directory created. setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(setup.TestBucket(), testDirName)) diff --git a/tools/integration_tests/log_rotation/logrotate_logfile_test.go b/tools/integration_tests/log_rotation/logrotate_logfile_test.go index 530ec9f5cd..d8bf755942 100644 --- a/tools/integration_tests/log_rotation/logrotate_logfile_test.go +++ b/tools/integration_tests/log_rotation/logrotate_logfile_test.go @@ -21,10 +21,9 @@ import ( "strings" "sync" "testing" - "time" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) const ( @@ -49,23 +48,26 @@ func runOperationsOnFileTillLogRotation(t *testing.T, wg *sync.WaitGroup, fileNa testDirPath := path.Join(setup.MntDir(), testDirName) filePath := path.Join(testDirPath, fileName) operations.CreateFileWithContent(filePath, filePerms, string(randomData), t) + currentLogFile := cfg.LogFile // Keep performing operations in mounted directory until log file is rotated. var lastLogFileSize int64 = 0 var retryStatLogFile = true for { - // Perform Read operation to generate logs. + // 1. Perform Read operation to generate logs _, err = operations.ReadFile(filePath) if err != nil { t.Errorf("ReadFile failed: %v", err) } // Break the loop when log file is rotated. - fi, err := operations.StatFile(logFilePath) + fi, err := operations.StatFile(currentLogFile) if err != nil { - t.Logf("stat operation on file %s failed: %v", logFilePath, err) + // --- StatFile Error Handling with Retry Limit --- + t.Logf("Stat operation on file %s failed: %v.", + currentLogFile, err) if !retryStatLogFile { - t.Errorf("Stat retry exhausted on log file: %s", logFilePath) + t.Errorf("Stat retry exhausted on log file") } retryStatLogFile = false continue @@ -82,7 +84,7 @@ func runParallelOperationsInMountedDirectoryTillLogRotation(t *testing.T) { // Parallelly performs operations on 5 files in-order to generate logs. var wg sync.WaitGroup wg.Add(5) - for i := 0; i < 5; i++ { + for i := range 5 { go runOperationsOnFileTillLogRotation(t, &wg, fmt.Sprintf(testFileName+"-%d", i)) } wg.Wait() @@ -104,42 +106,55 @@ func validateLogFileSize(t *testing.T, dirEntry os.DirEntry) { func TestLogRotation(t *testing.T) { setup.SetupTestDirectory(testDirName) - // Perform log rotation 4 times. - for i := 0; i < 4; i++ { + for range 4 { runParallelOperationsInMountedDirectoryTillLogRotation(t) } - // Adding 1-second sleep here because there is slight delay in compression - // of log files. - time.Sleep(1 * time.Second) - - // Validate log files generated. - dirEntries := operations.ReadDirectory(logDirPath, t) - if len(dirEntries) != logFileCount { - t.Errorf("Expected log files in dirEntries folder: %d, got: %d", - logFileCount, len(dirEntries)) - } - rotatedCompressedFileCtr := 0 - logFileCtr := 0 - rotatedUncompressedFileCtr := 0 - for i := 0; i < logFileCount; i++ { - if dirEntries[i].Name() == logFileName { - logFileCtr++ - validateLogFileSize(t, dirEntries[i]) - } else if strings.Contains(dirEntries[i].Name(), "txt.gz") { - rotatedCompressedFileCtr++ - } else if !strings.Contains(dirEntries[i].Name(), ".stderr") { - rotatedUncompressedFileCtr++ - validateLogFileSize(t, dirEntries[i]) + logFilesDirectory := path.Dir(cfg.LogFile) + activeLogFileName := t.Name() + ".log" + + t.Logf("Validating log files are eventually generated as expected. Expected total files: %d, active files: %d, backup files: %d.", logFileCount, activeLogFileCount, backupLogFileCount) + dirEntries := operations.RetryUntil(ctx, t, retryFrequency, retryTimeout, func() ([]os.DirEntry, error) { + entries := operations.ReadDirectory(logFilesDirectory, t) + + var rotatedCompressedFileCtr, logFileCtr, rotatedUncompressedFileCtr int + for _, entry := range entries { + if entry.IsDir() { + continue + } + if entry.Name() == activeLogFileName { + logFileCtr++ + } else if strings.HasSuffix(entry.Name(), ".log.gz") { + rotatedCompressedFileCtr++ + } else if !strings.HasSuffix(entry.Name(), ".stderr") { + rotatedUncompressedFileCtr++ + } } - } - if logFileCtr != activeLogFileCount { - t.Errorf("expected countOfLogFile: %d, got: %d", activeLogFileCount, logFileCtr) - } + rotatedLogFiles := rotatedCompressedFileCtr + rotatedUncompressedFileCtr + if len(entries) != logFileCount { + return nil, fmt.Errorf("Expected log files in dirEntries folder: %d, got: %d", logFileCount, len(entries)) + } + if logFileCtr != activeLogFileCount { + return nil, fmt.Errorf("expected countOfLogFile: %d, got: %d", activeLogFileCount, logFileCtr) + } + if rotatedLogFiles != backupLogFileCount { + return nil, fmt.Errorf("expected rotated files: %d, got: %d", backupLogFileCount, rotatedLogFiles) + } + + return entries, nil + }) - rotatedLogFiles := rotatedCompressedFileCtr + rotatedUncompressedFileCtr - if rotatedLogFiles != backupLogFileCount { - t.Errorf("expected rotated files: %d, got: %d", backupLogFileCount, rotatedLogFiles) + // Validate sizes for individual files + for _, entry := range dirEntries { + if entry.IsDir() { + continue + } + if entry.Name() == activeLogFileName { + validateLogFileSize(t, entry) + } else if !strings.HasSuffix(entry.Name(), ".log.gz") && !strings.HasSuffix(entry.Name(), ".stderr") { + validateLogFileSize(t, entry) + } } + } diff --git a/tools/integration_tests/managed_folders/admin_permissions_test.go b/tools/integration_tests/managed_folders/admin_permissions_test.go index 8f20bb96c8..12c95e71df 100644 --- a/tools/integration_tests/managed_folders/admin_permissions_test.go +++ b/tools/integration_tests/managed_folders/admin_permissions_test.go @@ -26,11 +26,11 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/creds_tests" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/test_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/creds_tests" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/suite" ) // ////////////////////////////////////////////////////////////////////// @@ -41,153 +41,163 @@ const ( CreateTestFile = "createTestFile" ) -var ( - bucket string - testDir string - serviceAccount string - localKeyFilePath string -) - // The permission granted by roles at project, bucket, and managed folder // levels apply additively (union) throughout the resource hierarchy. -// Hence here managed folder will have admin permission throughout all the tests. +// Hence, here managed folder will have admin permission throughout all the tests. type managedFoldersAdminPermission struct { bucketPermission string managedFoldersPermission string + flags []string + suite.Suite +} + +func (s *managedFoldersAdminPermission) SetupSuite() { + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, s.flags, testEnv.mountFunc) + setup.SetMntDir(testEnv.mountDir) } -func (s *managedFoldersAdminPermission) Setup(t *testing.T) { - createDirectoryStructureForNonEmptyManagedFolders(ctx, storageClient, controlClient, t) +func (s *managedFoldersAdminPermission) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *managedFoldersAdminPermission) SetupTest() { + testEnv.testDirPath = setup.SetupTestDirectory(TestDirForManagedFolderTest) + createDirectoryStructureForNonEmptyManagedFolders(testEnv.ctx, testEnv.storageClient, testEnv.controlClient, s.T()) if s.managedFoldersPermission != "nil" { - providePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder1), serviceAccount, s.managedFoldersPermission, t) - providePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder2), serviceAccount, s.managedFoldersPermission, t) + providePermissionToManagedFolder(testEnv.bucket, path.Join(testEnv.testDir, ManagedFolder1), testEnv.serviceAccount, s.managedFoldersPermission, s.T()) + providePermissionToManagedFolder(testEnv.bucket, path.Join(testEnv.testDir, ManagedFolder2), testEnv.serviceAccount, s.managedFoldersPermission, s.T()) // Waiting for 60 seconds for policy changes to propagate. This values we kept based on our experiments. time.Sleep(60 * time.Second) } } -func (s *managedFoldersAdminPermission) Teardown(t *testing.T) { - // Due to bucket view permissions, it prevents cleaning resources outside of managed folders. So we are cleaning managed folders resources only. +func (s *managedFoldersAdminPermission) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) + // Due to bucket view permissions, it prevents cleaning resources outside managed folders. So we are cleaning managed folders resources only. if s.bucketPermission == ViewPermission { - revokePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder1), serviceAccount, s.managedFoldersPermission, t) + revokePermissionToManagedFolder(testEnv.bucket, path.Join(testEnv.testDir, ManagedFolder1), testEnv.serviceAccount, s.managedFoldersPermission, s.T()) setup.CleanUpDir(path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder1)) - revokePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder2), serviceAccount, s.managedFoldersPermission, t) + revokePermissionToManagedFolder(testEnv.bucket, path.Join(testEnv.testDir, ManagedFolder2), testEnv.serviceAccount, s.managedFoldersPermission, s.T()) setup.CleanUpDir(path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder2)) return } setup.CleanUpDir(path.Join(setup.MntDir(), TestDirForManagedFolderTest)) } -func (s *managedFoldersAdminPermission) TestCreateObjectInManagedFolder(t *testing.T) { - testDirPath := path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder1) +//////////////////////////////////////////////////////////////////////// +// Test scenarios +//////////////////////////////////////////////////////////////////////// + +func (s *managedFoldersAdminPermission) TestCreateObjectInManagedFolder() { + testDirPath := path.Join(testEnv.testDirPath, ManagedFolder1) file := path.Join(testDirPath, CreateTestFile) - createFileForTest(file, t) + createFileForTest(file, s.T()) } -func (s *managedFoldersAdminPermission) TestDeleteObjectInManagedFolder(t *testing.T) { - filePath := path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder1, FileInNonEmptyManagedFoldersTest) +func (s *managedFoldersAdminPermission) TestDeleteObjectInManagedFolder() { + filePath := path.Join(testEnv.testDirPath, ManagedFolder1, FileInNonEmptyManagedFoldersTest) err := os.Remove(filePath) if err != nil { - t.Errorf("Error in removing file from managed folder: %v", err) + s.T().Errorf("Error in removing file from managed folder: %v", err) } _, err = operations.StatFile(filePath) if err == nil { - t.Errorf("file is not removed.") + s.T().Errorf("file is not removed.") } } // Managed folders will not be deleted, but they will become empty. Default empty managed folders will be hidden. -func (s *managedFoldersAdminPermission) TestDeleteManagedFolder(t *testing.T) { - dirPath := path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder1) +func (s *managedFoldersAdminPermission) TestDeleteManagedFolder() { + dirPath := path.Join(testEnv.testDirPath, ManagedFolder1) err := os.RemoveAll(dirPath) if err != nil { - t.Errorf("Error in removing managed folder: %v", err) + s.T().Errorf("Error in removing managed folder: %v", err) } _, err = os.Stat(dirPath) if err == nil { - t.Errorf("Directory is not removed.") + s.T().Errorf("Directory is not removed.") } } -func (s *managedFoldersAdminPermission) TestCopyObjectWithInManagedFolder(t *testing.T) { - testDirPath := path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder1) +func (s *managedFoldersAdminPermission) TestCopyObjectWithInManagedFolder() { + testDirPath := path.Join(testEnv.testDirPath, ManagedFolder1) srcCopyFile := path.Join(testDirPath, FileInNonEmptyManagedFoldersTest) destCopyFile := path.Join(testDirPath, DestFile) err := operations.CopyFile(srcCopyFile, destCopyFile) if err != nil { - t.Errorf("Error in copying file managed folder from src: %s to dest %s: %v", srcCopyFile, destCopyFile, err) + s.T().Errorf("Error in copying file managed folder from src: %s to dest %s: %v", srcCopyFile, destCopyFile, err) } _, err = operations.StatFile(destCopyFile) if err != nil { - t.Errorf("Error in stating destination file: %v", err) + s.T().Errorf("Error in stating destination file: %v", err) } } -func (s *managedFoldersAdminPermission) TestCopyManagedFolder(t *testing.T) { - srcDirPath := path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder1) - destDirPath := path.Join(setup.MntDir(), TestDirForManagedFolderTest, DestFolder) +func (s *managedFoldersAdminPermission) TestCopyManagedFolder() { + srcDirPath := path.Join(testEnv.testDirPath, ManagedFolder1) + destDirPath := path.Join(testEnv.testDirPath, DestFolder) err := operations.CopyDir(srcDirPath, destDirPath) if s.bucketPermission == ViewPermission { - operations.CheckErrorForReadOnlyFileSystem(err, t) + operations.CheckErrorForReadOnlyFileSystem(s.T(), err) } else { _, err = os.Stat(destDirPath) if err != nil { - t.Errorf("Error in stating destination dir: %v", err) + s.T().Errorf("Error in stating destination dir: %v", err) } } } -func (s *managedFoldersAdminPermission) TestMoveObjectWithInManagedFolder(t *testing.T) { - testDirPath := path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder1) +func (s *managedFoldersAdminPermission) TestMoveObjectWithInManagedFolder() { + testDirPath := path.Join(testEnv.testDirPath, ManagedFolder1) srcMoveFile := path.Join(testDirPath, FileInNonEmptyManagedFoldersTest) destMoveFile := path.Join(testDirPath, DestFile) err := operations.Move(srcMoveFile, destMoveFile) if err != nil { - t.Errorf("Error in moving file managed folder from src: %s to dest %s: %v", srcMoveFile, destMoveFile, err) + s.T().Errorf("Error in moving file managed folder from src: %s to dest %s: %v", srcMoveFile, destMoveFile, err) } _, err = operations.StatFile(destMoveFile) if err != nil { - t.Errorf("Error in stating destination file: %v", err) + s.T().Errorf("Error in stating destination file: %v", err) } _, err = operations.StatFile(srcMoveFile) if err == nil { - t.Errorf("SrcFile is not removed after move.") + s.T().Errorf("SrcFile is not removed after move.") } } -func (s *managedFoldersAdminPermission) TestMoveManagedFolder(t *testing.T) { - srcDirPath := path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder1) - destDirPath := path.Join(setup.MntDir(), TestDirForManagedFolderTest, DestFolder) +func (s *managedFoldersAdminPermission) TestMoveManagedFolder() { + srcDirPath := path.Join(testEnv.testDirPath, ManagedFolder1) + destDirPath := path.Join(testEnv.testDirPath, DestFolder) err := operations.Move(srcDirPath, destDirPath) if s.bucketPermission == ViewPermission { - operations.CheckErrorForReadOnlyFileSystem(err, t) + operations.CheckErrorForReadOnlyFileSystem(s.T(), err) } else { _, err = os.Stat(destDirPath) if err != nil { - t.Errorf("Error in stating destination dir: %v", err) + s.T().Errorf("Error in stating destination dir: %v", err) } _, err = os.Stat(srcDirPath) if err == nil { - t.Errorf("SrcDir is not removed after move.") + s.T().Errorf("SrcDir is not removed after move.") } } } -func (s *managedFoldersAdminPermission) TestListNonEmptyManagedFoldersWithAdminPermission(t *testing.T) { - listNonEmptyManagedFolders(t) +func (s *managedFoldersAdminPermission) TestListNonEmptyManagedFoldersWithAdminPermission() { + listNonEmptyManagedFolders(s.T()) } //////////////////////////////////////////////////////////////////////// @@ -196,41 +206,31 @@ func (s *managedFoldersAdminPermission) TestListNonEmptyManagedFoldersWithAdminP func TestManagedFolders_FolderAdminPermission(t *testing.T) { ts := &managedFoldersAdminPermission{} - - setup.RunTestsOnlyForStaticMount(mountDir, t) - - // Fetch credentials and apply permission on bucket. - serviceAccount, localKeyFilePath = creds_tests.CreateCredentials(ctx) - creds_tests.ApplyPermissionToServiceAccount(ctx, storageClient, serviceAccount, AdminPermission, setup.TestBucket()) - - flags := []string{"--implicit-dirs", "--key-file=" + localKeyFilePath, "--rename-dir-limit=5", "--stat-cache-ttl=0"} - if hnsFlagSet, err := setup.AddHNSFlagForHierarchicalBucket(ctx, storageClient); err == nil { - flags = hnsFlagSet - flags = append(flags, "--key-file="+localKeyFilePath, "--stat-cache-ttl=0") - } - - setup.MountGCSFuseWithGivenMountFunc(flags, mountFunc) - defer setup.UnmountGCSFuseAndDeleteLogFile(rootDir) - setup.SetMntDir(mountDir) - - // Run tests on given {Bucket permission, Managed folder permission}. - permissions := [][]string{{AdminPermission, "nil"}, {AdminPermission, IAMRoleForViewPermission}, {AdminPermission, IAMRoleForAdminPermission}, {ViewPermission, IAMRoleForAdminPermission}} - - for i := 0; i < len(permissions); i++ { - log.Printf("Running tests with flags, bucket have %s permission and managed folder have %s permissions: %s", permissions[i][0], permissions[i][1], flags) - bucket, testDir = setup.GetBucketAndObjectBasedOnTypeOfMount(TestDirForManagedFolderTest) - ts.bucketPermission = permissions[i][0] - if ts.bucketPermission == ViewPermission { - creds_tests.RevokePermission(ctx, storageClient, serviceAccount, AdminPermission, setup.TestBucket()) - creds_tests.ApplyPermissionToServiceAccount(ctx, storageClient, serviceAccount, ViewPermission, setup.TestBucket()) - defer creds_tests.RevokePermission(ctx, storageClient, serviceAccount, ViewPermission, setup.TestBucket()) + setup.RunTestsOnlyForStaticMount(testEnv.mountDir, t) + + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { + creds_tests.ApplyPermissionToServiceAccount(testEnv.ctx, testEnv.storageClient, testEnv.serviceAccount, AdminPermission, setup.TestBucket()) + // Run tests on given {Bucket permission, Managed folder permission}. + permissions := [][]string{{AdminPermission, "nil"}, {AdminPermission, IAMRoleForViewPermission}, {AdminPermission, IAMRoleForAdminPermission}, {ViewPermission, IAMRoleForAdminPermission}} + for i := range permissions { + log.Printf("Running tests with flags, bucket have %s permission and managed folder have %s permissions: %s", permissions[i][0], permissions[i][1], ts.flags) + testEnv.bucket, testEnv.testDir = setup.GetBucketAndObjectBasedOnTypeOfMount(TestDirForManagedFolderTest) + ts.bucketPermission = permissions[i][0] + if ts.bucketPermission == ViewPermission { + creds_tests.RevokePermission(testEnv.ctx, testEnv.storageClient, testEnv.serviceAccount, AdminPermission, setup.TestBucket()) + creds_tests.ApplyPermissionToServiceAccount(testEnv.ctx, testEnv.storageClient, testEnv.serviceAccount, ViewPermission, setup.TestBucket()) + } + ts.managedFoldersPermission = permissions[i][1] + suite.Run(t, ts) + if ts.bucketPermission == ViewPermission { + creds_tests.RevokePermission(testEnv.ctx, testEnv.storageClient, testEnv.serviceAccount, ViewPermission, setup.TestBucket()) + } } - ts.managedFoldersPermission = permissions[i][1] - - test_setup.RunTests(t, ts) + t.Cleanup(func() { + client.DeleteManagedFoldersInBucket(testEnv.ctx, testEnv.controlClient, path.Join(testEnv.testDir, ManagedFolder1), setup.TestBucket()) + client.DeleteManagedFoldersInBucket(testEnv.ctx, testEnv.controlClient, path.Join(testEnv.testDir, ManagedFolder2), setup.TestBucket()) + }) } - t.Cleanup(func() { - client.DeleteManagedFoldersInBucket(ctx, controlClient, path.Join(testDir, ManagedFolder1), setup.TestBucket()) - client.DeleteManagedFoldersInBucket(ctx, controlClient, path.Join(testDir, ManagedFolder2), setup.TestBucket()) - }) } diff --git a/tools/integration_tests/managed_folders/list_empty_managed_folders_test.go b/tools/integration_tests/managed_folders/list_empty_managed_folders_test.go index 557b110ead..b6b50b27f2 100644 --- a/tools/integration_tests/managed_folders/list_empty_managed_folders_test.go +++ b/tools/integration_tests/managed_folders/list_empty_managed_folders_test.go @@ -24,11 +24,11 @@ import ( "path/filepath" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/test_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/stretchr/testify/suite" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) const ( @@ -45,18 +45,30 @@ const ( //////////////////////////////////////////////////////////////////////// type enableEmptyManagedFoldersTrue struct { + suite.Suite + flags []string } -func (s *enableEmptyManagedFoldersTrue) Setup(t *testing.T) { - setup.SetupTestDirectory(TestDirForEmptyManagedFoldersTest) +func (s *enableEmptyManagedFoldersTrue) SetupSuite() { + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, s.flags, testEnv.mountFunc) + setup.SetMntDir(testEnv.mountDir) } -func (s *enableEmptyManagedFoldersTrue) Teardown(t *testing.T) { +func (s *enableEmptyManagedFoldersTrue) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *enableEmptyManagedFoldersTrue) SetupTest() { + testEnv.testDirPath = setup.SetupTestDirectory(TestDirForEmptyManagedFoldersTest) +} + +func (s *enableEmptyManagedFoldersTrue) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) // Clean up test directory. bucket, testDir := setup.GetBucketAndObjectBasedOnTypeOfMount(TestDirForEmptyManagedFoldersTest) - client.DeleteManagedFoldersInBucket(ctx, controlClient, path.Join(testDir, EmptyManagedFolder1), setup.TestBucket()) - client.DeleteManagedFoldersInBucket(ctx, controlClient, path.Join(testDir, EmptyManagedFolder2), setup.TestBucket()) - setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(bucket, testDir)) + client.DeleteManagedFoldersInBucket(testEnv.ctx, testEnv.controlClient, path.Join(testDir, EmptyManagedFolder1), setup.TestBucket()) + client.DeleteManagedFoldersInBucket(testEnv.ctx, testEnv.controlClient, path.Join(testDir, EmptyManagedFolder2), setup.TestBucket()) + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(bucket, testDir)) } //////////////////////////////////////////////////////////////////////// @@ -69,20 +81,20 @@ func createDirectoryStructureForEmptyManagedFoldersTest(t *testing.T) { // testBucket/EmptyManagedFoldersTest/simulatedFolder // testBucket/EmptyManagedFoldersTest/testFile bucket, testDir := setup.GetBucketAndObjectBasedOnTypeOfMount(TestDirForEmptyManagedFoldersTest) - client.CreateManagedFoldersInBucket(ctx, controlClient, path.Join(testDir, EmptyManagedFolder1), bucket) - client.CreateManagedFoldersInBucket(ctx, controlClient, path.Join(testDir, EmptyManagedFolder2), bucket) + client.CreateManagedFoldersInBucket(testEnv.ctx, testEnv.controlClient, path.Join(testDir, EmptyManagedFolder1), bucket) + client.CreateManagedFoldersInBucket(testEnv.ctx, testEnv.controlClient, path.Join(testDir, EmptyManagedFolder2), bucket) operations.CreateDirectory(path.Join(setup.MntDir(), TestDirForEmptyManagedFoldersTest, SimulatedFolder), t) f := operations.CreateFile(path.Join(setup.MntDir(), TestDirForEmptyManagedFoldersTest, File), setup.FilePermission_0600, t) - operations.CloseFile(f) + operations.CloseFileShouldNotThrowError(t, f) } //////////////////////////////////////////////////////////////////////// // Test scenarios //////////////////////////////////////////////////////////////////////// -func (s *enableEmptyManagedFoldersTrue) TestListDirectoryForEmptyManagedFolders(t *testing.T) { +func (s *enableEmptyManagedFoldersTrue) TestListDirectoryForEmptyManagedFolders() { // Create directory structure for testing. - createDirectoryStructureForEmptyManagedFoldersTest(t) + createDirectoryStructureForEmptyManagedFoldersTest(s.T()) // Recursively walk into directory and test. err := filepath.WalkDir(path.Join(setup.MntDir(), TestDirForEmptyManagedFoldersTest), func(path string, dir fs.DirEntry, err error) error { @@ -104,27 +116,27 @@ func (s *enableEmptyManagedFoldersTrue) TestListDirectoryForEmptyManagedFolders( if dir.Name() == TestDirForEmptyManagedFoldersTest { // numberOfObjects - 4 if len(objs) != NumberOfObjectsInDirForListTest { - t.Errorf("Incorrect number of objects in the directory %s expected %d: got %d: ", dir.Name(), NumberOfObjectsInDirForListTest, len(objs)) + s.T().Errorf("Incorrect number of objects in the directory %s expected %d: got %d: ", dir.Name(), NumberOfObjectsInDirForListTest, len(objs)) } // testBucket/managedFolderTest/emptyManagedFolder1 -- ManagedFolder1 if objs[0].Name() != EmptyManagedFolder1 || objs[0].IsDir() != true { - t.Errorf("Listed incorrect object expected %s: got %s: ", EmptyManagedFolder1, objs[0].Name()) + s.T().Errorf("Listed incorrect object expected %s: got %s: ", EmptyManagedFolder1, objs[0].Name()) } // testBucket/managedFolderTest/emptyManagedFolder2 -- ManagedFolder2 if objs[1].Name() != EmptyManagedFolder2 || objs[1].IsDir() != true { - t.Errorf("Listed incorrect object expected %s: got %s: ", EmptyManagedFolder2, objs[1].Name()) + s.T().Errorf("Listed incorrect object expected %s: got %s: ", EmptyManagedFolder2, objs[1].Name()) } // testBucket/managedFolderTest/simulatedFolder -- SimulatedFolder if objs[2].Name() != SimulatedFolder || objs[2].IsDir() != true { - t.Errorf("Listed incorrect object expected %s: got %s: ", SimulatedFolder, objs[2].Name()) + s.T().Errorf("Listed incorrect object expected %s: got %s: ", SimulatedFolder, objs[2].Name()) } // testBucket/managedFolderTest/testFile -- File if objs[3].Name() != File || objs[3].IsDir() != false { - t.Errorf("Listed incorrect object expected %s: got %s: ", File, objs[3].Name()) + s.T().Errorf("Listed incorrect object expected %s: got %s: ", File, objs[3].Name()) } return nil } @@ -132,49 +144,27 @@ func (s *enableEmptyManagedFoldersTrue) TestListDirectoryForEmptyManagedFolders( if dir.Name() == EmptyManagedFolder1 || dir.Name() == EmptyManagedFolder2 || dir.Name() == SimulatedFolder { // numberOfObjects - 0 if len(objs) != 0 { - t.Errorf("Incorrect number of objects in the directory %s expected %d: got %d: ", dir.Name(), 0, len(objs)) + s.T().Errorf("Incorrect number of objects in the directory %s expected %d: got %d: ", dir.Name(), 0, len(objs)) } } return nil }) if err != nil { - t.Errorf("error walking the path : %v\n", err) + s.T().Errorf("error walking the path : %v\n", err) return } } -func getMountConfigForEmptyManagedFolders() map[string]interface{} { - mountConfig := map[string]interface{}{ - "list": map[string]interface{}{ - "enable-empty-managed-folders": true, - }, - } - return mountConfig -} - // ////////////////////////////////////////////////////////////////////// // Test Function (Runs once before all tests) // ////////////////////////////////////////////////////////////////////// func TestEnableEmptyManagedFoldersTrue(t *testing.T) { ts := &enableEmptyManagedFoldersTrue{} - // Run tests for mountedDirectory only if --mountedDirectory and --testBucket flag is set. - if setup.AreBothMountedDirectoryAndTestBucketFlagsSet() { - test_setup.RunTests(t, ts) - return + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) } - - configFile := setup.YAMLConfigFile(getMountConfigForEmptyManagedFolders(), "config.yaml") - flags := []string{"--implicit-dirs", "--config-file=" + configFile} - - setup.MountGCSFuseWithGivenMountFunc(flags, mountFunc) - defer func() { - setup.SetMntDir(rootDir) - setup.UnMountBucket() - }() - setup.SetMntDir(mountDir) - - // Run tests. - log.Printf("Running tests with flags: %s", flags) - test_setup.RunTests(t, ts) } diff --git a/tools/integration_tests/managed_folders/managed_folders_test.go b/tools/integration_tests/managed_folders/managed_folders_test.go index b335daf581..0508fcfdce 100644 --- a/tools/integration_tests/managed_folders/managed_folders_test.go +++ b/tools/integration_tests/managed_folders/managed_folders_test.go @@ -24,29 +24,40 @@ import ( "cloud.google.com/go/storage" control "cloud.google.com/go/storage/control/apiv2" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/only_dir_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" - - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/dynamic_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" -) - -const ( - onlyDirMounted = "TestManagedFolderOnlyDir" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/creds_tests" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/only_dir_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/dynamic_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) -var ( - mountFunc func([]string) error +type env struct { + ctx context.Context + storageClient *storage.Client + controlClient *control.StorageControlClient + bucketType string + cfg *test_suite.TestConfig + mountFunc func(*test_suite.TestConfig, []string) error // Mount directory is where our tests run. mountDir string // Root directory is the directory to be unmounted. - rootDir string - storageClient *storage.Client - controlClient *control.StorageControlClient - ctx context.Context + rootDir string + serviceAccount string + localKeyFilePath string + bucket string + testDir string + testDirPath string +} + +const ( + onlyDirMounted = "TestManagedFolderOnlyDir" ) +var testEnv env + //////////////////////////////////////////////////////////////////////// // TestMain //////////////////////////////////////////////////////////////////////// @@ -54,58 +65,95 @@ var ( func TestMain(m *testing.M) { setup.ParseSetUpFlags() - ctx = context.Background() - closeStorageClient := client.CreateStorageClientWithCancel(&ctx, &storageClient) - closeControlClient := client.CreateControlClientWithCancel(&ctx, &controlClient) + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.ManagedFolders) == 0 { + log.Println("No configuration found for managed_folders tests in config. Using flags instead.") + // Populate the config manually. + cfg.ManagedFolders = make([]test_suite.TestConfig, 1) + cfg.ManagedFolders[0].TestBucket = setup.TestBucket() + cfg.ManagedFolders[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.ManagedFolders[0].LogFile = setup.LogFile() + // Initialize the slice to hold 15 specific test configurations + cfg.ManagedFolders[0].Configs = make([]test_suite.ConfigItem, 3) + cfg.ManagedFolders[0].Configs[0].Flags = []string{"--implicit-dirs --key-file=${KEY_FILE} --rename-dir-limit=3"} + cfg.ManagedFolders[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ManagedFolders[0].Configs[0].Run = "TestManagedFolders_FolderViewPermission" + cfg.ManagedFolders[0].Configs[1].Flags = []string{"--implicit-dirs --enable-empty-managed-folders"} + cfg.ManagedFolders[0].Configs[1].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ManagedFolders[0].Configs[1].Run = "TestEnableEmptyManagedFoldersTrue" + cfg.ManagedFolders[0].Configs[2].Flags = []string{"--implicit-dirs --key-file=${KEY_FILE} --rename-dir-limit=5 --stat-cache-ttl=0"} + cfg.ManagedFolders[0].Configs[2].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ManagedFolders[0].Configs[2].Run = "TestManagedFolders_FolderAdminPermission" + } + + testEnv.ctx = context.Background() + testEnv.bucketType = setup.TestEnvironment(testEnv.ctx, &cfg.ManagedFolders[0]) + testEnv.cfg = &cfg.ManagedFolders[0] + + // 2. Create storage client before running tests. + closeStorageClient := client.CreateStorageClientWithCancel(&testEnv.ctx, &testEnv.storageClient) + closeControlClient := client.CreateControlClientWithCancel(&testEnv.ctx, &testEnv.controlClient) defer func() { err := closeStorageClient() if err != nil { - log.Fatalf("closeStorageClient failed: %v", err) + log.Printf("closeStorageClient failed: %v\n", err) } err = closeControlClient() if err != nil { - log.Fatalf("closeControlClient failed: %v", err) + log.Printf("closeControlClient failed: %v\n", err) + } + }() + + // Fetch credentials and apply permission on bucket. + testEnv.serviceAccount, testEnv.localKeyFilePath = creds_tests.CreateCredentials(testEnv.ctx) + defer func() { + if err := os.Remove(testEnv.localKeyFilePath); err != nil { + log.Printf("Failed to delete temp credentials file %s: %v", testEnv.localKeyFilePath, err) } }() - setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() + for i, testCase := range cfg.ManagedFolders[0].Configs { + for j, flag := range testCase.Flags { + cfg.ManagedFolders[0].Configs[i].Flags[j] = setup.ReplaceOrAppendFlag(flag, "${KEY_FILE}", "--key-file=", testEnv.localKeyFilePath) + } + } - if setup.MountedDirectory() != "" { + // 3. these tests won't run with mountedDirectory. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { log.Printf("These tests will not run with mounted directory..") return } - // Else run tests for testBucket. + // Run tests for testBucket // Set up test directory. - setup.SetUpTestDirForTestBucketFlag() + setup.SetUpTestDirForTestBucket(testEnv.cfg) // Save mount and root directory variables. - mountDir, rootDir = setup.MntDir(), setup.MntDir() + testEnv.mountDir, testEnv.rootDir = testEnv.cfg.GCSFuseMountedDirectory, testEnv.cfg.GCSFuseMountedDirectory log.Println("Running static mounting tests...") - mountFunc = static_mounting.MountGcsfuseWithStaticMounting + testEnv.mountFunc = static_mounting.MountGcsfuseWithStaticMountingWithConfigFile successCode := m.Run() - setup.SaveLogFileInCaseOfFailure(successCode) if successCode == 0 { - log.Println("Running only dir mounting tests...") - setup.SetOnlyDirMounted(onlyDirMounted + "/") - client.CreateManagedFoldersInBucket(ctx, controlClient, onlyDirMounted, setup.TestBucket()) - defer client.DeleteManagedFoldersInBucket(ctx, controlClient, onlyDirMounted, setup.TestBucket()) - mountFunc = only_dir_mounting.MountGcsfuseWithOnlyDir + log.Println("Running dynamic mounting tests...") + // Save mount directory variable to have path of bucket to run tests. + testEnv.mountDir = path.Join(testEnv.cfg.GCSFuseMountedDirectory, testEnv.cfg.TestBucket) + testEnv.mountFunc = dynamic_mounting.MountGcsfuseWithDynamicMountingWithConfig successCode = m.Run() - setup.SaveLogFileInCaseOfFailure(successCode) - setup.SetOnlyDirMounted("") } if successCode == 0 { - log.Println("Running dynamic mounting tests...") - // Save mount directory variable to have path of bucket to run tests. - mountDir = path.Join(setup.MntDir(), setup.TestBucket()) - mountFunc = dynamic_mounting.MountGcsfuseWithDynamicMounting + log.Println("Running only dir mounting tests...") + setup.SetOnlyDirMounted(onlyDirMounted + "/") + testEnv.mountDir = testEnv.rootDir + testEnv.mountFunc = only_dir_mounting.MountGcsfuseWithOnlyDirWithConfigFile successCode = m.Run() - setup.SaveLogFileInCaseOfFailure(successCode) + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(testEnv.cfg.TestBucket, setup.OnlyDirMounted(), TestDirForManagedFolderTest)) } + // Clean up test directory created. + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(testEnv.cfg.TestBucket, TestDirForManagedFolderTest)) os.Exit(successCode) } diff --git a/tools/integration_tests/managed_folders/test_helper.go b/tools/integration_tests/managed_folders/test_helper.go index 9f23fee1c3..ebf01c4139 100644 --- a/tools/integration_tests/managed_folders/test_helper.go +++ b/tools/integration_tests/managed_folders/test_helper.go @@ -28,9 +28,9 @@ import ( "cloud.google.com/go/storage" control "cloud.google.com/go/storage/control/apiv2" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) const ( @@ -73,7 +73,7 @@ func providePermissionToManagedFolder(bucket, managedFolderPath, serviceAccount, // Indent for readability jsonData, err := json.MarshalIndent(policy, "", " ") if err != nil { - t.Fatalf(fmt.Sprintf("Error in marshal the data into JSON format: %v", err)) + t.Fatalf("Error in marshal the data into JSON format: %v", err) } f, err := os.CreateTemp(os.TempDir(), "iam-policy-*.json") @@ -84,20 +84,20 @@ func providePermissionToManagedFolder(bucket, managedFolderPath, serviceAccount, // Write the JSON to a FileInNonEmptyManagedFoldersTest _, err = f.Write(jsonData) if err != nil { - t.Fatalf(fmt.Sprintf("Error in writing iam policy in json FileInNonEmptyManagedFoldersTest : %v", err)) + t.Fatalf("Error in writing iam policy in json FileInNonEmptyManagedFoldersTest : %v", err) } - gcloudProvidePermissionCmd := fmt.Sprintf("alpha storage managed-folders set-iam-policy gs://%s/%s %s", bucket, managedFolderPath, f.Name()) - _, err = operations.ExecuteGcloudCommandf(gcloudProvidePermissionCmd) + gcloudProvidePermissionCmd := fmt.Sprintf("storage managed-folders set-iam-policy gs://%s/%s %s", bucket, managedFolderPath, f.Name()) + _, err = operations.ExecuteGcloudCommand(gcloudProvidePermissionCmd) if err != nil { t.Fatalf("Error in providing permission to managed folder: %v", err) } } func revokePermissionToManagedFolder(bucket, managedFolderPath, serviceAccount, iamRole string, t *testing.T) { - gcloudRevokePermissionCmd := fmt.Sprintf("alpha storage managed-folders remove-iam-policy-binding gs://%s/%s --member=%s --role=%s", bucket, managedFolderPath, serviceAccount, iamRole) + gcloudRevokePermissionCmd := fmt.Sprintf("storage managed-folders remove-iam-policy-binding gs://%s/%s --member=%s --role=%s", bucket, managedFolderPath, serviceAccount, iamRole) - _, err := operations.ExecuteGcloudCommandf(gcloudRevokePermissionCmd) + _, err := operations.ExecuteGcloudCommand(gcloudRevokePermissionCmd) if err != nil && !strings.Contains(err.Error(), "Policy binding with the specified principal, role, and condition not found!") && !strings.Contains(err.Error(), "The specified managed folder does not exist.") { t.Fatalf("Error in removing permission to managed folder: %v", err) } @@ -117,7 +117,7 @@ func createDirectoryStructureForNonEmptyManagedFolders(ctx context.Context, stor log.Fatalf("Failed to clean up test directory: %v", err) } f := operations.CreateFile(path.Join("/tmp", FileInNonEmptyManagedFoldersTest), setup.FilePermission_0600, t) - defer operations.CloseFile(f) + defer operations.CloseFileShouldNotThrowError(t, f) managedFolder1 := path.Join(testDir, ManagedFolder1) managedFolder2 := path.Join(testDir, ManagedFolder2) simulatedFolderNonEmptyManagedFoldersTest := path.Join(testDir, SimulatedFolderNonEmptyManagedFoldersTest) @@ -231,7 +231,7 @@ func copyDirAndCheckErrForViewPermission(src, dest string, t *testing.T) { t.Errorf(" Managed folder unexpectedly got copied with view only permission.") } - operations.CheckErrorForReadOnlyFileSystem(err, t) + operations.CheckErrorForReadOnlyFileSystem(t, err) } func copyObjectAndCheckErrForViewPermission(src, dest string, t *testing.T) { @@ -240,7 +240,7 @@ func copyObjectAndCheckErrForViewPermission(src, dest string, t *testing.T) { t.Errorf("Objects in managed folder unexpectedly got copied with view only permission.") } - operations.CheckErrorForReadOnlyFileSystem(err, t) + operations.CheckErrorForReadOnlyFileSystem(t, err) } func moveAndCheckErrForViewPermission(src, dest string, t *testing.T) { @@ -249,12 +249,12 @@ func moveAndCheckErrForViewPermission(src, dest string, t *testing.T) { t.Errorf("Objects in managed folder unexpectedly got moved with view only permission.") } - operations.CheckErrorForReadOnlyFileSystem(err, t) + operations.CheckErrorForReadOnlyFileSystem(t, err) } func createFileForTest(filePath string, t *testing.T) { file, err := os.Create(filePath) - defer operations.CloseFile(file) + defer operations.CloseFileShouldNotThrowError(t, file) if err != nil { t.Errorf("Error in creating local file, %v", err) } diff --git a/tools/integration_tests/managed_folders/view_permissions_test.go b/tools/integration_tests/managed_folders/view_permissions_test.go index 52d81db849..0f6651eddf 100644 --- a/tools/integration_tests/managed_folders/view_permissions_test.go +++ b/tools/integration_tests/managed_folders/view_permissions_test.go @@ -25,10 +25,10 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/creds_tests" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/test_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/creds_tests" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/suite" ) const ( @@ -42,92 +42,109 @@ const ( // The permission granted by roles at project, bucket, and managed folder // levels apply additively (union) throughout the resource hierarchy. -// Hence here managed folder will have view permission throughout all the tests. +// Hence, here managed folder will have view permission throughout all the tests. type managedFoldersViewPermission struct { + flags []string + suite.Suite } -func (s *managedFoldersViewPermission) Setup(t *testing.T) { +func (s *managedFoldersViewPermission) SetupSuite() { + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, s.flags, testEnv.mountFunc) + setup.SetMntDir(testEnv.mountDir) + testEnv.testDirPath = setup.SetupTestDirectory(TestDirForManagedFolderTest) } -func (s *managedFoldersViewPermission) Teardown(t *testing.T) { +func (s *managedFoldersViewPermission) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) } -func (s *managedFoldersViewPermission) TestListNonEmptyManagedFolders(t *testing.T) { - listNonEmptyManagedFolders(t) +func (s *managedFoldersViewPermission) SetupTest() { } -func (s *managedFoldersViewPermission) TestCreateObjectInManagedFolder(t *testing.T) { - filePath := path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder2, DestFile) +func (s *managedFoldersViewPermission) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +//////////////////////////////////////////////////////////////////////// +// Test scenarios +//////////////////////////////////////////////////////////////////////// + +func (s *managedFoldersViewPermission) TestListNonEmptyManagedFolders() { + listNonEmptyManagedFolders(s.T()) +} + +func (s *managedFoldersViewPermission) TestCreateObjectInManagedFolder() { + filePath := path.Join(testEnv.testDirPath, ManagedFolder2, DestFile) + + // The error must happen either at file creation or file handle close. file, err := os.Create(filePath) - if err != nil { - t.Errorf("Error in creating file locally.") - } - t.Cleanup(func() { + if file != nil { err = file.Close() - operations.CheckErrorForReadOnlyFileSystem(err, t) - }) + } + + operations.CheckErrorForReadOnlyFileSystem(s.T(), err) } -func (s *managedFoldersViewPermission) TestDeleteObjectFromManagedFolder(t *testing.T) { - err := os.Remove(path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder1, FileInNonEmptyManagedFoldersTest)) +func (s *managedFoldersViewPermission) TestDeleteObjectFromManagedFolder() { + err := os.Remove(path.Join(testEnv.testDirPath, ManagedFolder1, FileInNonEmptyManagedFoldersTest)) if err == nil { - t.Errorf("File from managed folder gets deleted with view only permission.") + s.T().Errorf("File from managed folder gets deleted with view only permission.") } - operations.CheckErrorForReadOnlyFileSystem(err, t) + operations.CheckErrorForReadOnlyFileSystem(s.T(), err) } -func (s *managedFoldersViewPermission) TestDeleteNonEmptyManagedFolder(t *testing.T) { - err := os.RemoveAll(path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder1)) +func (s *managedFoldersViewPermission) TestDeleteNonEmptyManagedFolder() { + err := os.RemoveAll(path.Join(testEnv.testDirPath, ManagedFolder1)) if err == nil { - t.Errorf("Managed folder deleted with view only permission.") + s.T().Errorf("Managed folder deleted with view only permission.") } - operations.CheckErrorForReadOnlyFileSystem(err, t) + operations.CheckErrorForReadOnlyFileSystem(s.T(), err) } -func (s *managedFoldersViewPermission) TestMoveManagedFolder(t *testing.T) { - srcDir := path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder1) - destDir := path.Join(setup.MntDir(), TestDirForManagedFolderTest, DestFolder) +func (s *managedFoldersViewPermission) TestMoveManagedFolder() { + srcDir := path.Join(testEnv.testDirPath, ManagedFolder1) + destDir := path.Join(testEnv.testDirPath, DestFolder) - moveAndCheckErrForViewPermission(srcDir, destDir, t) + moveAndCheckErrForViewPermission(srcDir, destDir, s.T()) } -func (s *managedFoldersViewPermission) TestMoveObjectWithInManagedFolder(t *testing.T) { - srcFile := path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder1, FileInNonEmptyManagedFoldersTest) - destFile := path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder1, DestFile) +func (s *managedFoldersViewPermission) TestMoveObjectWithInManagedFolder() { + srcFile := path.Join(testEnv.testDirPath, ManagedFolder1, FileInNonEmptyManagedFoldersTest) + destFile := path.Join(testEnv.testDirPath, ManagedFolder1, DestFile) - moveAndCheckErrForViewPermission(srcFile, destFile, t) + moveAndCheckErrForViewPermission(srcFile, destFile, s.T()) } -func (s *managedFoldersViewPermission) TestMoveObjectOutOfManagedFolder(t *testing.T) { - srcFile := path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder1, FileInNonEmptyManagedFoldersTest) - destFile := path.Join(setup.MntDir(), TestDirForManagedFolderTest, DestFile) +func (s *managedFoldersViewPermission) TestMoveObjectOutOfManagedFolder() { + srcFile := path.Join(testEnv.testDirPath, ManagedFolder1, FileInNonEmptyManagedFoldersTest) + destFile := path.Join(testEnv.testDirPath, DestFile) - moveAndCheckErrForViewPermission(srcFile, destFile, t) + moveAndCheckErrForViewPermission(srcFile, destFile, s.T()) } -func (s *managedFoldersViewPermission) TestCopyNonEmptyManagedFolder(t *testing.T) { - srcDir := path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder1) - destDir := path.Join(setup.MntDir(), TestDirForManagedFolderTest, DestFolder) +func (s *managedFoldersViewPermission) TestCopyNonEmptyManagedFolder() { + srcDir := path.Join(testEnv.testDirPath, ManagedFolder1) + destDir := path.Join(testEnv.testDirPath, DestFolder) - copyDirAndCheckErrForViewPermission(srcDir, destDir, t) + copyDirAndCheckErrForViewPermission(srcDir, destDir, s.T()) } -func (s *managedFoldersViewPermission) TestCopyObjectWithInManagedFolder(t *testing.T) { - srcFile := path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder1, FileInNonEmptyManagedFoldersTest) - destFile := path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder1, DestFile) +func (s *managedFoldersViewPermission) TestCopyObjectWithInManagedFolder() { + srcFile := path.Join(testEnv.testDirPath, ManagedFolder1, FileInNonEmptyManagedFoldersTest) + destFile := path.Join(testEnv.testDirPath, ManagedFolder1, DestFile) - copyObjectAndCheckErrForViewPermission(srcFile, destFile, t) + copyObjectAndCheckErrForViewPermission(srcFile, destFile, s.T()) } -func (s *managedFoldersViewPermission) TestCopyObjectOutOfManagedFolder(t *testing.T) { - srcFile := path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder1, FileInNonEmptyManagedFoldersTest) - destFile := path.Join(setup.MntDir(), TestDirForManagedFolderTest, DestFile) +func (s *managedFoldersViewPermission) TestCopyObjectOutOfManagedFolder() { + srcFile := path.Join(testEnv.testDirPath, ManagedFolder1, FileInNonEmptyManagedFoldersTest) + destFile := path.Join(testEnv.testDirPath, DestFile) - copyObjectAndCheckErrForViewPermission(srcFile, destFile, t) + copyObjectAndCheckErrForViewPermission(srcFile, destFile, s.T()) } // ////////////////////////////////////////////////////////////////////// @@ -136,35 +153,28 @@ func (s *managedFoldersViewPermission) TestCopyObjectOutOfManagedFolder(t *testi func TestManagedFolders_FolderViewPermission(t *testing.T) { ts := &managedFoldersViewPermission{} - // Fetch credentials and apply permission on bucket. - serviceAccount, localKeyFilePath := creds_tests.CreateCredentials(ctx) - creds_tests.ApplyPermissionToServiceAccount(ctx, storageClient, serviceAccount, ViewPermission, setup.TestBucket()) - defer creds_tests.RevokePermission(ctx, storageClient, serviceAccount, ViewPermission, setup.TestBucket()) - - flags := []string{"--implicit-dirs", "--key-file=" + localKeyFilePath, "--rename-dir-limit=3"} - if hnsFlagSet, err := setup.AddHNSFlagForHierarchicalBucket(ctx, storageClient); err == nil { - flags = hnsFlagSet - flags = append(flags, "--key-file="+localKeyFilePath) + creds_tests.ApplyPermissionToServiceAccount(testEnv.ctx, testEnv.storageClient, testEnv.serviceAccount, ViewPermission, setup.TestBucket()) + defer creds_tests.RevokePermission(testEnv.ctx, testEnv.storageClient, testEnv.serviceAccount, ViewPermission, setup.TestBucket()) + + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { + testEnv.bucket, testEnv.testDir = setup.GetBucketAndObjectBasedOnTypeOfMount(TestDirForManagedFolderTest) + // Create directory structure for testing. + createDirectoryStructureForNonEmptyManagedFolders(testEnv.ctx, testEnv.storageClient, testEnv.controlClient, t) + defer cleanup(testEnv.ctx, testEnv.storageClient, testEnv.controlClient, testEnv.bucket, testEnv.testDir, testEnv.serviceAccount, IAMRoleForViewPermission, t) + + // Run tests. + log.Printf("Running tests with flags and managed folder have nil permissions: %s", ts.flags) + suite.Run(t, ts) + + // Provide storage.objectViewer role to managed folders. + providePermissionToManagedFolder(testEnv.bucket, path.Join(testEnv.testDir, ManagedFolder1), testEnv.serviceAccount, IAMRoleForViewPermission, t) + providePermissionToManagedFolder(testEnv.bucket, path.Join(testEnv.testDir, ManagedFolder2), testEnv.serviceAccount, IAMRoleForViewPermission, t) + // Waiting for 60 seconds for policy changes to propagate. This values we kept based on our experiments. + time.Sleep(60 * time.Second) + + log.Printf("Running tests with flags and managed folder have view permissions: %s", ts.flags) + suite.Run(t, ts) } - setup.MountGCSFuseWithGivenMountFunc(flags, mountFunc) - defer setup.UnmountGCSFuseAndDeleteLogFile(rootDir) - setup.SetMntDir(mountDir) - - bucket, testDir = setup.GetBucketAndObjectBasedOnTypeOfMount(TestDirForManagedFolderTest) - // Create directory structure for testing. - createDirectoryStructureForNonEmptyManagedFolders(ctx, storageClient, controlClient, t) - defer cleanup(ctx, storageClient, controlClient, bucket, testDir, serviceAccount, IAMRoleForViewPermission, t) - - // Run tests. - log.Printf("Running tests with flags and managed folder have nil permissions: %s", flags) - test_setup.RunTests(t, ts) - - // Provide storage.objectViewer role to managed folders. - providePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder1), serviceAccount, IAMRoleForViewPermission, t) - providePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder2), serviceAccount, IAMRoleForViewPermission, t) - // Waiting for 60 seconds for policy changes to propagate. This values we kept based on our experiments. - time.Sleep(60 * time.Second) - - log.Printf("Running tests with flags and managed folder have view permissions: %s", flags) - test_setup.RunTests(t, ts) } diff --git a/tools/integration_tests/monitoring/buffered_read_prom_test.go b/tools/integration_tests/monitoring/buffered_read_prom_test.go new file mode 100644 index 0000000000..aa3b60f263 --- /dev/null +++ b/tools/integration_tests/monitoring/buffered_read_prom_test.go @@ -0,0 +1,103 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package monitoring + +import ( + "io" + "log" + "path" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type PromBufferedReadTest struct { + PromTestBase +} + +func (p *PromBufferedReadTest) TestBufferedReadMetrics() { + _, err := operations.ReadFile(path.Join(testEnv.testDirPath, "hello.txt")) + + require.NoError(p.T(), err) + assertNonZeroCountMetric(p.T(), "gcs_read_bytes_count", "", "", p.prometheusPort) + assertNonZeroCountMetric(p.T(), "gcs_download_bytes_count", "read_type", "Buffered", p.prometheusPort) + assertNonZeroHistogramMetric(p.T(), "buffered_read/read_latency", "", "", p.prometheusPort) +} + +func (p *PromBufferedReadTest) TestRandomReadFallback() { + const blockSize = 4 * 1024 * 1024 + const fileSize = 4 * blockSize + const fileName = "random_read_fallback.txt" + filePath := path.Join(testEnv.testDirPath, fileName) + operations.CreateFileOfSize(fileSize, filePath, p.T()) + f, err := operations.OpenFileAsReadonly(filePath) + require.NoError(p.T(), err) + defer operations.CloseFileShouldNotThrowError(p.T(), f) + buf := make([]byte, 10) + // With random-seek-threshold: 2, the 3rd random read should trigger a fallback. + // First random read. + _, err = f.ReadAt(buf, 3*blockSize+100) + require.NoError(p.T(), err, "ReadAt in block 3 failed") + // Second random read. + _, err = f.ReadAt(buf, 2*blockSize+100) + require.NoError(p.T(), err, "ReadAt in block 2 failed") + + // Third random read, which exceeds the threshold and triggers fallback. + _, err = f.ReadAt(buf, 1*blockSize+100) + + require.NoError(p.T(), err, "ReadAt in block 1 failed") + assertNonZeroCountMetric(p.T(), "buffered_read_fallback_trigger_count", "reason", "random_read_detected", p.prometheusPort) +} + +func (p *PromBufferedReadTest) TestInsufficientMemoryFallback() { + const blockSize = 4 * 1024 * 1024 + const fileSize = 10 * blockSize // 40 MiB file + filePath := path.Join(testEnv.testDirPath, "insufficient_mem_test.txt") + operations.CreateFileOfSize(fileSize, filePath, p.T()) + f1, err := operations.OpenFileAsReadonly(filePath) + require.NoError(p.T(), err) + defer operations.CloseFileShouldNotThrowError(p.T(), f1) + f2, err := operations.OpenFileAsReadonly(filePath) + require.NoError(p.T(), err) + defer operations.CloseFileShouldNotThrowError(p.T(), f2) + // Read the entire file from the first handle. This will trigger prefetching + // that allocates blocks up to the global limit, exhausting the pool. + _, err = io.ReadAll(f1) + require.NoError(p.T(), err) + + // Attempt to read from the second handle. This should fail to create a + // BufferedReader due to no available blocks, triggering the metric. + smallBuf := make([]byte, 10) + _, err = f2.Read(smallBuf) + + require.NoError(p.T(), err) + assertNonZeroCountMetric(p.T(), "buffered_read_fallback_trigger_count", "reason", "insufficient_memory", p.prometheusPort) +} + +func TestPromBufferedReadSuite(t *testing.T) { + t.SkipNow() + ts := &PromBufferedReadTest{} + ts.suiteName = "TestPromBufferedReadSuite" + flagSets := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, flags := range flagSets { + ts.flags = flags + ts.prometheusPort = parsePortFromFlags(flags) + log.Printf("Running prom buffered read tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/monitoring/kernel_reader_prom_test.go b/tools/integration_tests/monitoring/kernel_reader_prom_test.go new file mode 100644 index 0000000000..f9d2979a10 --- /dev/null +++ b/tools/integration_tests/monitoring/kernel_reader_prom_test.go @@ -0,0 +1,61 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package monitoring + +import ( + "log" + "os" + "path" + "strings" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type PromKernelReaderTest struct { + PromTestBase +} + +func (p *PromKernelReaderTest) TestKernelReaderMetrics() { + testName := strings.ReplaceAll(p.T().Name(), "/", "_") + gcsDir := path.Join(testDirName, testName) + fileName := "mrd_test_file.txt" + client.SetupFileInTestDirectory(testEnv.ctx, testEnv.storageClient, gcsDir, fileName, 10*1024*1024, p.T()) + + // Read file to trigger metrics + _, err := os.ReadFile(path.Join(testEnv.testDirPath, fileName)) + + require.NoError(p.T(), err) + assertNonZeroCountMetric(p.T(), "fs_ops_count", "fs_op", "ReadFile", p.prometheusPort) + assertNonZeroCountMetric(p.T(), "gcs_download_bytes_count", "read_type", "Parallel", p.prometheusPort) + assertNonZeroCountMetric(p.T(), "gcs_read_bytes_count", "", "", p.prometheusPort) + assertNonZeroCountMetric(p.T(), "gcs_read_count", "read_type", "Parallel", p.prometheusPort) + assertNonZeroCountMetric(p.T(), "gcs_request_count", "gcs_method", "MultiRangeDownloader::Add", p.prometheusPort) + assertNonZeroHistogramMetric(p.T(), "gcs_request_latencies", "gcs_method", "MultiRangeDownloader::Add", p.prometheusPort) +} + +func TestPromKernelReaderSuite(t *testing.T) { + ts := &PromKernelReaderTest{} + flagSets := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, flags := range flagSets { + ts.flags = flags + ts.prometheusPort = parsePortFromFlags(flags) + log.Printf("Running prom kernel reader tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/monitoring/prom_test.go b/tools/integration_tests/monitoring/prom_test.go new file mode 100644 index 0000000000..6ab1632abf --- /dev/null +++ b/tools/integration_tests/monitoring/prom_test.go @@ -0,0 +1,100 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package monitoring + +import ( + "log" + "os" + "path" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/pkg/xattr" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type PromTest struct { + PromTestBase +} + +func (p *PromTest) TestStatMetrics() { + prometheusPort := p.prometheusPort + _, err := os.Stat(path.Join(testEnv.testDirPath, "hello.txt")) + + require.NoError(p.T(), err) + assertNonZeroCountMetric(p.T(), "fs_ops_count", "fs_op", "LookUpInode", prometheusPort) + assertNonZeroHistogramMetric(p.T(), "fs_ops_latency", "fs_op", "LookUpInode", prometheusPort) + assertNonZeroCountMetric(p.T(), "gcs_request_count", "gcs_method", "StatObject", prometheusPort) + assertNonZeroHistogramMetric(p.T(), "gcs_request_latencies", "gcs_method", "StatObject", prometheusPort) +} + +func (p *PromTest) TestFsOpsErrorMetrics() { + prometheusPort := p.prometheusPort + _, err := os.Stat(path.Join(testEnv.testDirPath, "non_existent_path.txt")) + require.Error(p.T(), err) + + assertNonZeroCountMetric(p.T(), "fs_ops_error_count", "fs_op", "LookUpInode", prometheusPort) + assertNonZeroHistogramMetric(p.T(), "fs_ops_latency", "fs_op", "LookUpInode", prometheusPort) +} + +func (p *PromTest) TestListMetrics() { + prometheusPort := p.prometheusPort + _, err := os.ReadDir(testEnv.testDirPath) + + require.NoError(p.T(), err) + assertNonZeroCountMetric(p.T(), "fs_ops_count", "fs_op", "ReadDir", prometheusPort) + assertNonZeroCountMetric(p.T(), "fs_ops_count", "fs_op", "OpenDir", prometheusPort) + assertNonZeroCountMetric(p.T(), "gcs_request_count", "gcs_method", "ListObjects", prometheusPort) + assertNonZeroHistogramMetric(p.T(), "gcs_request_latencies", "gcs_method", "ListObjects", prometheusPort) +} + +func (p *PromTest) TestSetXAttrMetrics() { + prometheusPort := p.prometheusPort + err := xattr.Set(path.Join(testEnv.testDirPath, "hello.txt"), "alpha", []byte("beta")) + + require.Error(p.T(), err) + assertNonZeroCountMetric(p.T(), "fs_ops_error_count", "fs_op", "Others", prometheusPort) +} + +func (p *PromTest) TestReadMetrics() { + prometheusPort := p.prometheusPort + _, err := os.ReadFile(path.Join(testEnv.testDirPath, "hello.txt")) + + require.NoError(p.T(), err) + assertNonZeroCountMetric(p.T(), "file_cache_read_bytes_count", "read_type", "Sequential", prometheusPort) + assertNonZeroCountMetric(p.T(), "file_cache_read_count", "cache_hit", "false", prometheusPort) + assertNonZeroCountMetric(p.T(), "file_cache_read_count", "read_type", "Sequential", prometheusPort) + assertNonZeroHistogramMetric(p.T(), "file_cache_read_latencies", "cache_hit", "false", prometheusPort) + assertNonZeroCountMetric(p.T(), "fs_ops_count", "fs_op", "OpenFile", prometheusPort) + assertNonZeroCountMetric(p.T(), "fs_ops_count", "fs_op", "ReadFile", prometheusPort) + assertNonZeroCountMetric(p.T(), "gcs_request_count", "gcs_method", "NewReader", prometheusPort) + assertNonZeroCountMetric(p.T(), "gcs_reader_count", "io_method", "opened", prometheusPort) + assertNonZeroCountMetric(p.T(), "gcs_reader_count", "io_method", "closed", prometheusPort) + assertNonZeroCountMetric(p.T(), "gcs_download_bytes_count", "", "", prometheusPort) + assertNonZeroHistogramMetric(p.T(), "gcs_request_latencies", "gcs_method", "NewReader", prometheusPort) +} + +func TestPromOTELSuite(t *testing.T) { + ts := &PromTest{} + ts.suiteName = "TestPromOTELSuite" + flagSets := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, flags := range flagSets { + ts.flags = flags + ts.prometheusPort = parsePortFromFlags(flags) + log.Printf("Running monitoring tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/monitoring/prom_w_grpc_metrics_test.go b/tools/integration_tests/monitoring/prom_w_grpc_metrics_test.go new file mode 100644 index 0000000000..fc6e5380b3 --- /dev/null +++ b/tools/integration_tests/monitoring/prom_w_grpc_metrics_test.go @@ -0,0 +1,64 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package monitoring + +import ( + "log" + "os" + "path" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +// PromGrpcMetricsTest is the test suite for gRPC metrics. +type PromGrpcMetricsTest struct { + PromTestBase +} + +func (p *PromGrpcMetricsTest) TestStorageClientGrpcMetrics() { + _, err := os.ReadFile(path.Join(testEnv.testDirPath, "hello.txt")) + require.NoError(p.T(), err) + + // Assert that gRPC metrics are present. + if testEnv.bucketType == "zonal" { + assertNonZeroCountMetric(p.T(), "grpc_client_attempt_started", "grpc_method", "google.storage.v2.Storage/BidiReadObject", p.prometheusPort) + } else { + assertNonZeroCountMetric(p.T(), "grpc_client_attempt_started", "grpc_method", "google.storage.v2.Storage/ReadObject", p.prometheusPort) + } + assertNonZeroCountMetric(p.T(), "grpc_client_attempt_started", "", "", p.prometheusPort) + assertNonZeroHistogramMetric(p.T(), "grpc_client_attempt_duration_seconds", "", "", p.prometheusPort) + assertNonZeroHistogramMetric(p.T(), "grpc_client_call_duration_seconds", "", "", p.prometheusPort) + assertNonZeroHistogramMetric(p.T(), "grpc_client_attempt_rcvd_total_compressed_message_size_bytes", "", "", p.prometheusPort) + assertNonZeroHistogramMetric(p.T(), "grpc_client_attempt_sent_total_compressed_message_size_bytes", "", "", p.prometheusPort) +} + +func TestPromGrpcMetricsSuite(t *testing.T) { + ts := &PromGrpcMetricsTest{} + ts.suiteName = "TestPromGrpcMetricsSuite" + if testEnv.cfg.GKEMountedDirectory == "" { + // Skip the test if the testing environment is GCE VM. + t.SkipNow() + } + flagSets := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, flags := range flagSets { + ts.flags = flags + ts.prometheusPort = parsePortFromFlags(flags) + log.Printf("Running prom grpc metrics tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/monitoring/setup_test.go b/tools/integration_tests/monitoring/setup_test.go new file mode 100644 index 0000000000..0f45353ab0 --- /dev/null +++ b/tools/integration_tests/monitoring/setup_test.go @@ -0,0 +1,229 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package monitoring + +import ( + "context" + "fmt" + "log" + "net/http" + "os" + "path" + "strconv" + "strings" + "testing" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" + promclient "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +const ( + testDirName = "monitoring" + gkeTempDir = "/gcsfuse-tmp" +) + +type env struct { + storageClient *storage.Client + ctx context.Context + testDirPath string + cfg *test_suite.TestConfig + bucketType string +} + +var ( + testEnv env + mountFunc func(*test_suite.TestConfig, []string) error +) + +// PromTestBase preserves the base struct and common methods. +type PromTestBase struct { + suite.Suite + flags []string + prometheusPort int + suiteName string +} + +func (p *PromTestBase) SetupSuite() { + setup.SetUpLogFilePath(p.T().Name(), p.flags, gkeTempDir, "", testEnv.cfg) + mountGCSFuseAndSetupTestDir(p.flags, testEnv.ctx, testEnv.storageClient) +} + +func (p *PromTestBase) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (p *PromTestBase) SetupTest() { + testName := strings.ReplaceAll(p.T().Name(), "/", "_") + gcsDir := path.Join(testDirName, testName) + // Use the setup helper to prepare the test directory. + testEnv.testDirPath = client.SetupTestDirectory(testEnv.ctx, testEnv.storageClient, gcsDir) + // Setup a standard hello.txt file for metrics collection. + client.SetupFileInTestDirectory(testEnv.ctx, testEnv.storageClient, gcsDir, "hello.txt", 10, p.T()) +} + +func (p *PromTestBase) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(p.T()) +} + +func mountGCSFuseAndSetupTestDir(flags []string, ctx context.Context, storageClient *storage.Client) { + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, flags, mountFunc) + if testEnv.cfg.GKEMountedDirectory != "" { + setup.SetMntDir(testEnv.cfg.GKEMountedDirectory) + } + testEnv.testDirPath = client.SetupTestDirectory(ctx, storageClient, testDirName) +} + +func parsePortFromFlags(flags []string) int { + for _, flagStr := range flags { + parts := strings.Split(flagStr, " ") + for _, part := range parts { + if strings.HasPrefix(part, "--prometheus-port=") { + portStr := strings.TrimPrefix(part, "--prometheus-port=") + port, _ := strconv.Atoi(portStr) + return port + } + } + } + return 0 +} + +func parsePromFormat(t *testing.T, prometheusPort int) (map[string]*promclient.MetricFamily, error) { + t.Helper() + resp, err := http.Get(fmt.Sprintf("http://localhost:%d/metrics", prometheusPort)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + parser := expfmt.NewTextParser(model.UTF8Validation) + return parser.TextToMetricFamilies(resp.Body) +} + +func assertNonZeroCountMetric(t *testing.T, metricName, labelName, labelValue string, prometheusPort int) { + t.Helper() + mf, err := parsePromFormat(t, prometheusPort) + require.NoError(t, err) + for k, v := range mf { + if k != metricName || *v.Type != promclient.MetricType_COUNTER { + continue + } + for _, m := range v.Metric { + if labelName != "" { + for _, l := range m.Label { + if *l.Name == labelName && *l.Value == labelValue && *m.Counter.Value > 0 { + return + } + } + } else if *m.Counter.Value > 0 { + return + } + } + } + require.Fail(t, fmt.Sprintf("Metric %s with label %s=%s not found or zero", metricName, labelName, labelValue)) +} + +func assertNonZeroHistogramMetric(t *testing.T, metricName, labelName, labelValue string, prometheusPort int) { + t.Helper() + mf, err := parsePromFormat(t, prometheusPort) + require.NoError(t, err) + for k, v := range mf { + if k != metricName || *v.Type != promclient.MetricType_HISTOGRAM { + continue + } + for _, m := range v.Metric { + if labelName != "" { + for _, l := range m.Label { + if *l.Name == labelName && *l.Value == labelValue && *m.Histogram.SampleCount > 0 { + return + } + } + } else if *m.Histogram.SampleCount > 0 { + return + } + } + } + require.Fail(t, fmt.Sprintf("Metric %s with label %s=%s not found or zero", metricName, labelName, labelValue)) +} + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + configFile := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(configFile.Monitoring) == 0 { + log.Println("No configuration found for monitoring tests in config. Using default flags.") + configFile.Monitoring = make([]test_suite.TestConfig, 1) + testEnv.cfg = &configFile.Monitoring[0] + testEnv.cfg.TestBucket = setup.TestBucket() + testEnv.cfg.LogFile = setup.LogFile() + testEnv.cfg.GKEMountedDirectory = setup.MountedDirectory() + + testEnv.cfg.Configs = make([]test_suite.ConfigItem, 7) + testEnv.cfg.Configs[0].Flags = []string{"--prometheus-port=9190 --file-cache-max-size-mb=-1 --cache-dir=/gcsfuse-tmp/PromOTELSuite --log-file=/gcsfuse-tmp/TestPromOTELSuite.log --enable-kernel-reader=false"} + testEnv.cfg.Configs[0].Compatible = map[string]bool{"flat": true, "hns": false, "zonal": false} + testEnv.cfg.Configs[0].Run = "TestPromOTELSuite" + testEnv.cfg.Configs[1].Flags = []string{"--prometheus-port=10190 --file-cache-max-size-mb=-1 --cache-dir=/gcsfuse-tmp/PromOTELSuite --log-file=/gcsfuse-tmp/TestPromOTELSuite.log --enable-kernel-reader=false"} + testEnv.cfg.Configs[1].Compatible = map[string]bool{"flat": false, "hns": true, "zonal": true} + testEnv.cfg.Configs[1].Run = "TestPromOTELSuite" + + testEnv.cfg.Configs[2].Flags = []string{"--prometheus-port=9191 --enable-buffered-read --read-block-size-mb=4 --read-random-seek-threshold=2 --read-global-max-blocks=5 --read-min-blocks-per-handle=2 --read-start-blocks-per-handle=2 --log-file=/gcsfuse-tmp/TestPromBufferedReadSuite.log --enable-kernel-reader=false"} + testEnv.cfg.Configs[2].Compatible = map[string]bool{"flat": true, "hns": false, "zonal": false} + testEnv.cfg.Configs[2].Run = "TestPromBufferedReadSuite" + testEnv.cfg.Configs[3].Flags = []string{"--prometheus-port=10191 --enable-buffered-read --read-block-size-mb=4 --read-random-seek-threshold=2 --read-global-max-blocks=5 --read-min-blocks-per-handle=2 --read-start-blocks-per-handle=2 --log-file=/gcsfuse-tmp/TestPromBufferedReadSuite.log --enable-kernel-reader=false"} + testEnv.cfg.Configs[3].Compatible = map[string]bool{"flat": false, "hns": true, "zonal": true} + testEnv.cfg.Configs[3].Run = "TestPromBufferedReadSuite" + + testEnv.cfg.Configs[4].Flags = []string{"--client-protocol=grpc --experimental-enable-grpc-metrics=true --prometheus-port=9192 --file-cache-max-size-mb=-1 --cache-dir=/gcsfuse-tmp/TestPromGrpcMetricsSuite --log-file=/gcsfuse-tmp/TestPromGrpcMetricsSuite.log --enable-kernel-reader=false"} + testEnv.cfg.Configs[4].Compatible = map[string]bool{"flat": true, "hns": false, "zonal": false} + testEnv.cfg.Configs[4].Run = "TestPromGrpcMetricsSuite" + testEnv.cfg.Configs[5].Flags = []string{"--client-protocol=grpc --experimental-enable-grpc-metrics=true --prometheus-port=10192 --file-cache-max-size-mb=-1 --cache-dir=/gcsfuse-tmp/TestPromGrpcMetricsSuite --log-file=/gcsfuse-tmp/TestPromGrpcMetricsSuite.log --enable-kernel-reader=false"} + testEnv.cfg.Configs[5].Compatible = map[string]bool{"flat": false, "hns": true, "zonal": true} + testEnv.cfg.Configs[5].Run = "TestPromGrpcMetricsSuite" + + testEnv.cfg.Configs[6].Flags = []string{"--prometheus-port=9193 --log-file=/gcsfuse-tmp/TestPromKernelReaderSuite.log"} + testEnv.cfg.Configs[6].Compatible = map[string]bool{"flat": false, "hns": false, "zonal": true} + testEnv.cfg.Configs[6].Run = "TestPromKernelReaderSuite" + } + testEnv.cfg = &configFile.Monitoring[0] + testEnv.ctx = context.Background() + testEnv.bucketType = setup.TestEnvironment(testEnv.ctx, testEnv.cfg) + + var err error + testEnv.storageClient, err = client.CreateStorageClient(testEnv.ctx) + if err != nil { + log.Fatalf("client.CreateStorageClient: %v", err) + } + defer testEnv.storageClient.Close() + + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + os.Exit(setup.RunTestsForMountedDirectory(testEnv.cfg.GKEMountedDirectory, m)) + } + + setup.SetUpTestDirForTestBucket(testEnv.cfg) + setup.OverrideFilePathsInFlagSet(testEnv.cfg, setup.TestDir()) + + log.Println("Running static mounting tests...") + mountFunc = static_mounting.MountGcsfuseWithStaticMountingWithConfigFile + successCode := m.Run() + + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(setup.TestBucket(), testDirName)) + os.Exit(successCode) +} diff --git a/tools/integration_tests/mount_timeout/gcsfuse_mount_timeout_test.go b/tools/integration_tests/mount_timeout/gcsfuse_mount_timeout_test.go new file mode 100644 index 0000000000..5e1bbcf976 --- /dev/null +++ b/tools/integration_tests/mount_timeout/gcsfuse_mount_timeout_test.go @@ -0,0 +1,408 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mount_timeout + +import ( + "bufio" + "fmt" + "math" + "os" + "path" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +const ( + iterations int = 10 +) + +func TestMountTimeout(t *testing.T) { + if setup.IsZonalBucketRun() { + zone := os.Getenv("TEST_ENV") + switch zone { + case testEnvZoneGCEUSCentral1A: + // Set strict zone-based config values. + config := ZBMountTimeoutTestCaseConfig{ + sameZoneZonalBucket: zonalUSCentral1ABucket, + crossZoneZonalBucket: zonalUSWest4ABucket, + sameZoneMountTimeout: zonalSameZoneExpectedMountTime, + crossZoneMountTimeout: zonalCrossZoneExpectedMountTime, + } + t.Log("Running tests with region based timeout values since the GCE VM is located in us-central...\n") + suite.Run(t, &ZBMountTimeoutTest{config: config}) + case testEnvZoneGCEUSWEST4A: + // Set strict zone-based config values. + config := ZBMountTimeoutTestCaseConfig{ + sameZoneZonalBucket: zonalUSWest4ABucket, + crossZoneZonalBucket: zonalUSCentral1ABucket, + sameZoneMountTimeout: relaxedExpectedMountTime, + crossZoneMountTimeout: relaxedExpectedMountTime, + } + t.Logf("Running tests with relaxed timeout of %f sec for all scenarios since the GCE VM is not located in us-central...\n", relaxedExpectedMountTime.Seconds()) + suite.Run(t, &ZBMountTimeoutTest{config: config}) + default: + // Skip the tests if the testing environment is not GCE VM. + t.Logf("Skipping tests since the testing environment (%q) is not a ZB supported region...\n", zone) + t.Skip() + } + } else { + if os.Getenv("TEST_ENV") == testEnvGCEUSCentral { + // Set strict region based timeout values if testing environment is GCE VM in us-central. + timeout := RegionWiseTimeouts{ + multiRegionUSTimeout: multiRegionUSExpectedMountTime, + multiRegionAsiaTimeout: multiRegionAsiaExpectedMountTime, + dualRegionUSTimeout: dualRegionUSExpectedMountTime, + dualRegionAsiaTimeout: dualRegionAsiaExpectedMountTime, + singleRegionUSCentralTimeout: singleRegionUSCentralExpectedMountTime, + singleRegionAsiaEastTimeout: singleRegionAsiaEastExpectedMountTime, + } + t.Log("Running tests with region based timeout values since the GCE VM is located in us-central...\n") + suite.Run(t, &NonZBMountTimeoutTest{timeouts: timeout}) + } else if os.Getenv("TEST_ENV") == testEnvGCENonUSCentral { + // Set common relaxed timeout values if testing environment is GCE VM not in us-central. + timeout := RegionWiseTimeouts{ + multiRegionUSTimeout: relaxedExpectedMountTime, + multiRegionAsiaTimeout: relaxedExpectedMountTime, + dualRegionUSTimeout: relaxedExpectedMountTime, + dualRegionAsiaTimeout: relaxedExpectedMountTime, + singleRegionUSCentralTimeout: relaxedExpectedMountTime, + singleRegionAsiaEastTimeout: relaxedExpectedMountTime, + } + t.Logf("Running tests with relaxed timeout of %f sec for all scenarios since the GCE VM is not located in us-central...\n", relaxedExpectedMountTime.Seconds()) + suite.Run(t, &NonZBMountTimeoutTest{timeouts: timeout}) + } else { + // Skip the tests if the testing environment is not GCE VM. + t.Log("Skipping tests since the testing environment is not GCE VM...\n") + t.Skip() + } + } +} + +type RegionWiseTimeouts struct { + multiRegionUSTimeout time.Duration + multiRegionAsiaTimeout time.Duration + dualRegionUSTimeout time.Duration + dualRegionAsiaTimeout time.Duration + singleRegionUSCentralTimeout time.Duration + singleRegionAsiaEastTimeout time.Duration +} + +type ZBMountTimeoutTestCaseConfig struct { + sameZoneZonalBucket string + crossZoneZonalBucket string + sameZoneMountTimeout time.Duration + crossZoneMountTimeout time.Duration +} + +type MountTimeoutTest struct { + suite.Suite + // Path to the gcsfuse binary. + gcsfusePath string + + // A temporary directory into which a file system may be mounted. Removed in + // TearDown. + dir string +} + +type NonZBMountTimeoutTest struct { + MountTimeoutTest + timeouts RegionWiseTimeouts +} + +type ZBMountTimeoutTest struct { + MountTimeoutTest + config ZBMountTimeoutTestCaseConfig +} + +func (testSuite *MountTimeoutTest) SetupTest() { + var err error + testSuite.gcsfusePath = path.Join(gBuildDir, "bin/gcsfuse") + // Set up the temporary directory. + testSuite.dir, err = os.MkdirTemp("", "mount_timeout_test") + assert.NoError(testSuite.T(), err) +} + +func (testSuite *MountTimeoutTest) TearDownTest() { + err := os.Remove(testSuite.dir) + assert.NoError(testSuite.T(), err) +} + +// mountOrTimeout mounts the bucket with the given client protocol. If the time taken +// exceeds the expected for the particular test case , an error is thrown and test will fail. +func (testSuite *MountTimeoutTest) mountOrTimeout(bucketName, clientProtocol string, expectedMountTime time.Duration) (err error) { + minMountTime := time.Duration(math.MaxInt64) + logFile := setup.LogFile() + defer func() { + if err != nil { + setup.SaveLogFileAsArtifact(logFile, setup.GCSFuseLogFilePrefix+filepath.Base(logFile)) + } + }() + + // Iterating 10 times to account for randomness in time taken to mount. + args := []string{"--client-protocol", clientProtocol, "--log-severity=trace", "--log-file=" + logFile, bucketName, testSuite.dir} + for i := range iterations { + start := time.Now() + if err = mounting.MountGcsfuse(testSuite.gcsfusePath, args); err != nil { + err = fmt.Errorf("mount failed for bucket %q (client protocol: %s) on attempt#%v: %w", bucketName, clientProtocol, i, err) + return err + } + mountTime := time.Since(start) + + minMountTime = time.Duration(math.Min(float64(minMountTime), float64(mountTime))) + + if err = unmountAndWait(testSuite.dir); err != nil { + err = fmt.Errorf("unmountAndWait failed for bucket %q on attempt#%v: %w", bucketName, i, err) + return err + } + } + + if minMountTime > expectedMountTime { + err = fmt.Errorf("mount took too long for bucket %q (client protocol: %s). expected: %v, actual minimum mount time: %v", bucketName, clientProtocol, expectedMountTime, minMountTime) + return err + } + return nil +} + +// unmountAndWait unmounts the given directory and waits for the associated +// resources to be released by polling, with a timeout. +func unmountAndWait(mountDir string) error { + // isMounted checks if a directory is currently a mount point by reading /proc/mounts. + isMounted := func(mountPoint string) (bool, error) { + file, err := os.Open("/proc/mounts") + if err != nil { + // On non-Linux systems, /proc/mounts doesn't exist, so we can't poll. + if os.IsNotExist(err) { + return false, nil + } + return false, fmt.Errorf("could not open /proc/mounts: %w", err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + fields := strings.Fields(line) + // The second field in /proc/mounts is the mount point. + if len(fields) >= 2 && fields[1] == mountPoint { + return true, nil // Found the mount point + } + } + return false, scanner.Err() + } + + if err := util.Unmount(mountDir); err != nil { + // It might already be unmounted or in a weird state. If the error + // indicates it's not mounted, we can proceed to verify. + if !strings.Contains(err.Error(), "not mounted") && !strings.Contains(err.Error(), "no such file or directory") { + return fmt.Errorf("unmount call failed: %w", err) + } + } + + // Poll for up to 5 seconds for the unmount to complete. + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + timeout := time.After(5 * time.Second) + + for { + select { + case <-timeout: + return fmt.Errorf("timed out waiting for %q to unmount", mountDir) + case <-ticker.C: + mounted, err := isMounted(mountDir) + if err != nil || !mounted { + return err // Success or error checking mount status. + } + } + } +} + +func (testSuite *NonZBMountTimeoutTest) TestMountMultiRegionUSBucketWithTimeout() { + testCases := []struct { + name string + clientProtocol cfg.Protocol + }{ + { + name: "multiRegionUSClientProtocolGRPC", + clientProtocol: cfg.GRPC, + }, + { + name: "multiRegionUSClientProtocolHttp1", + clientProtocol: cfg.HTTP1, + }, + { + name: "multiRegionUSClientProtocolHttp2", + clientProtocol: cfg.HTTP2, + }, + } + for _, tc := range testCases { + setup.SetLogFile(fmt.Sprintf("%s%s.txt", logfilePathPrefix, tc.name)) + + err := testSuite.mountOrTimeout(multiRegionUSBucket, string(tc.clientProtocol), testSuite.timeouts.multiRegionUSTimeout) + assert.NoError(testSuite.T(), err) + } +} + +func (testSuite *NonZBMountTimeoutTest) TestMountMultiRegionAsiaBucketWithTimeout() { + testCases := []struct { + name string + clientProtocol cfg.Protocol + }{ + { + name: "multiRegionAsiaClientProtocolGRPC", + clientProtocol: cfg.GRPC, + }, + { + name: "multiRegionAsiaClientProtocolHttp1", + clientProtocol: cfg.HTTP1, + }, + { + name: "multiRegionAsiaClientProtocolHttp2", + clientProtocol: cfg.HTTP2, + }, + } + for _, tc := range testCases { + setup.SetLogFile(fmt.Sprintf("%s%s.txt", logfilePathPrefix, tc.name)) + + err := testSuite.mountOrTimeout(multiRegionAsiaBucket, string(tc.clientProtocol), testSuite.timeouts.multiRegionAsiaTimeout) + assert.NoError(testSuite.T(), err) + } +} + +func (testSuite *NonZBMountTimeoutTest) TestMountDualRegionUSBucketWithTimeout() { + testCases := []struct { + name string + clientProtocol cfg.Protocol + }{ + { + name: "dualRegionUSClientProtocolGRPC", + clientProtocol: cfg.GRPC, + }, + { + name: "dualRegionUSClientProtocolHttp1", + clientProtocol: cfg.HTTP1, + }, + { + name: "dualRegionUSClientProtocolHttp2", + clientProtocol: cfg.HTTP2, + }, + } + for _, tc := range testCases { + setup.SetLogFile(fmt.Sprintf("%s%s.txt", logfilePathPrefix, tc.name)) + + err := testSuite.mountOrTimeout(dualRegionUSBucket, string(tc.clientProtocol), testSuite.timeouts.dualRegionUSTimeout) + assert.NoError(testSuite.T(), err) + } +} + +func (testSuite *NonZBMountTimeoutTest) TestMountDualRegionAsiaBucketWithTimeout() { + testCases := []struct { + name string + clientProtocol cfg.Protocol + }{ + { + name: "dualRegionAsiaClientProtocolGRPC", + clientProtocol: cfg.GRPC, + }, + { + name: "dualRegionAsiaClientProtocolHttp1", + clientProtocol: cfg.HTTP1, + }, + { + name: "dualRegionAsiaClientProtocolHttp2", + clientProtocol: cfg.HTTP2, + }, + } + for _, tc := range testCases { + setup.SetLogFile(fmt.Sprintf("%s%s.txt", logfilePathPrefix, tc.name)) + + err := testSuite.mountOrTimeout(dualRegionAsiaBucket, string(tc.clientProtocol), testSuite.timeouts.dualRegionAsiaTimeout) + assert.NoError(testSuite.T(), err) + } +} + +func (testSuite *NonZBMountTimeoutTest) TestMountSingleRegionUSBucketWithTimeout() { + testCases := []struct { + name string + clientProtocol cfg.Protocol + }{ + { + name: "singleRegionUSClientProtocolGRPC", + clientProtocol: cfg.GRPC, + }, + { + name: "singleRegionUSClientProtocolHttp1", + clientProtocol: cfg.HTTP1, + }, + { + name: "singleRegionUSClientProtocolHttp2", + clientProtocol: cfg.HTTP2, + }, + } + for _, tc := range testCases { + setup.SetLogFile(fmt.Sprintf("%s%s.txt", logfilePathPrefix, tc.name)) + + err := testSuite.mountOrTimeout(singleRegionUSCentralBucket, string(tc.clientProtocol), testSuite.timeouts.singleRegionUSCentralTimeout) + assert.NoError(testSuite.T(), err) + } +} + +func (testSuite *NonZBMountTimeoutTest) TestMountSingleRegionAsiaBucketWithTimeout() { + testCases := []struct { + name string + clientProtocol cfg.Protocol + }{ + { + name: "singleRegionAsiaClientProtocolGRPC", + clientProtocol: cfg.GRPC, + }, + { + name: "singleRegionAsiaClientProtocolHttp1", + clientProtocol: cfg.HTTP1, + }, + { + name: "singleRegionAsiaClientProtocolHttp2", + clientProtocol: cfg.HTTP2, + }, + } + for _, tc := range testCases { + setup.SetLogFile(fmt.Sprintf("%s%s.txt", logfilePathPrefix, tc.name)) + + err := testSuite.mountOrTimeout(singleRegionAsiaEastBucket, string(tc.clientProtocol), testSuite.timeouts.singleRegionAsiaEastTimeout) + assert.NoError(testSuite.T(), err) + } +} + +func (testSuite *ZBMountTimeoutTest) TestMountSameZoneZonalBucketWithTimeout() { + setup.SetLogFile(fmt.Sprintf("%s%s.txt", logfilePathPrefix, "SameZoneZonalBucket")) + + err := testSuite.mountOrTimeout(testSuite.config.sameZoneZonalBucket, string(cfg.GRPC), testSuite.config.sameZoneMountTimeout) + assert.NoError(testSuite.T(), err) +} + +func (testSuite *ZBMountTimeoutTest) TestMountCrossZoneZonalBucketWithTimeout() { + setup.SetLogFile(fmt.Sprintf("%s%s.txt", logfilePathPrefix, "CrossZoneZonalBucket")) + + err := testSuite.mountOrTimeout(testSuite.config.crossZoneZonalBucket, string(cfg.GRPC), testSuite.config.crossZoneMountTimeout) + assert.NoError(testSuite.T(), err) +} diff --git a/tools/integration_tests/mount_timeout/mount_access_test.go b/tools/integration_tests/mount_timeout/mount_access_test.go new file mode 100644 index 0000000000..77c9b5a44d --- /dev/null +++ b/tools/integration_tests/mount_timeout/mount_access_test.go @@ -0,0 +1,103 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mount_timeout + +import ( + "fmt" + "os" + "path" + "path/filepath" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/creds_tests" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +const ( + // This role is a custom role and granting this role to any service account grants only storage.objects.list permission. + // Custom roles follow the naming pattern projects/<project-id>/roles/<custom-role-name> + listPermCustomRoleName = "storage.objects.list" +) + +type MountAccessTest struct { + suite.Suite + // Path to the gcsfuse binary. + gcsfusePath string + + // A temporary directory into which a file system may be mounted. Removed in + // TearDown. + dir string +} + +func (testSuite *MountAccessTest) SetupTest() { + var err error + testSuite.gcsfusePath = path.Join(gBuildDir, "bin/gcsfuse") + // Set up the temporary directory. + testSuite.dir, err = os.MkdirTemp("", "mount_timeout_test") + assert.NoError(testSuite.T(), err) +} + +func (testSuite *MountAccessTest) TearDownTest() { + err := os.Remove(testSuite.dir) + assert.NoError(testSuite.T(), err) +} + +// mountWithKeyFile mounts the bucket with the given key file. +// It returns any error during mounting or unmounting. +func (testSuite *MountAccessTest) mountWithKeyFile(bucketName, keyFile string) (err error) { + logFile := setup.LogFile() + defer func() { + if err != nil { + setup.SaveLogFileAsArtifact(logFile, setup.GCSFuseLogFilePrefix+filepath.Base(logFile)) + } + }() + + args := []string{"--key-file=" + keyFile, "--log-severity=trace", "--log-file=" + logFile, bucketName, testSuite.dir} + + if err = mounting.MountGcsfuse(testSuite.gcsfusePath, args); err != nil { + err = fmt.Errorf("mount failed for bucket %q with key-file having minimal access: %w", bucketName, err) + return err + } + if err = unmountAndWait(testSuite.dir); err != nil { + err = fmt.Errorf("unmountAndWait failed for bucket %q on mount point %q with err: %w", bucketName, testSuite.dir, err) + return err + } + return nil +} + +func (testSuite *MountAccessTest) TestMountingWithMinimalAccessSucceeds() { + serviceAccount, localKeyFilePath := creds_tests.CreateCredentials(gCtx) + defer func() { + if err := os.Remove(localKeyFilePath); err != nil { + testSuite.T().Logf("Failed to delete temp credentials file %s: %v", localKeyFilePath, err) + } + }() + creds_tests.ApplyCustomRoleToServiceAccountOnBucket(gCtx, gStorageClient, serviceAccount, listPermCustomRoleName, testBucket) + defer creds_tests.RevokeCustomRoleFromServiceAccountOnBucket(gCtx, gStorageClient, serviceAccount, listPermCustomRoleName, testBucket) + + err := testSuite.mountWithKeyFile(testBucket, localKeyFilePath) + + assert.NoError(testSuite.T(), err) +} + +func TestMountAccess(t *testing.T) { + // Set log file. + setup.SetLogFile(fmt.Sprintf("%s%s.txt", logfilePathPrefix, t.Name())) + + suite.Run(t, &MountAccessTest{}) +} diff --git a/tools/integration_tests/mount_timeout/mount_timeout_test.go b/tools/integration_tests/mount_timeout/mount_timeout_test.go new file mode 100644 index 0000000000..75a861f966 --- /dev/null +++ b/tools/integration_tests/mount_timeout/mount_timeout_test.go @@ -0,0 +1,208 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mount_timeout + +import ( + "context" + "fmt" + "log" + "os" + "os/exec" + "runtime" + "strings" + "testing" + "time" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" + "github.com/googlecloudplatform/gcsfuse/v3/tools/util" + "go.opentelemetry.io/contrib/detectors/gcp" + "go.opentelemetry.io/otel/sdk/resource" +) + +var ( + // A directory containing outputs created by build_gcsfuse, set up and deleted + // in TestMain. + gBuildDir string + + // On Linux, the path to fusermount, whose directory must be in gcsfuse's PATH + // variable in order to successfully mount. Set by TestMain. + gFusermountPath string + + // storage client for permission setup on the bucket for mount access. + gStorageClient *storage.Client + + gCtx context.Context + + testBucket string +) + +const ( + // Constants specific to non-ZB E2E runs. + testEnvGCEUSCentral string = "gce-us-central" + testEnvGCENonUSCentral string = "gce-non-us-central" + testEnvNonGCE string = "non-gce" + multiRegionUSBucket string = "mount_timeout_test_bucket_us" + multiRegionAsiaBucket string = "mount_timeout_test_bucket_asia" + dualRegionUSBucket string = "mount_timeout_test_bucket_nam4" + dualRegionAsiaBucket string = "mount_timeout_test_bucket_asia1" + singleRegionUSCentralBucket string = "mount_timeout_test_bucket_us-central1" + singleRegionAsiaEastBucket string = "mount_timeout_test_bucket_asia-east1" + singleRegionAsiaEastExpectedMountTime time.Duration = 5500 * time.Millisecond + multiRegionUSExpectedMountTime time.Duration = 4500 * time.Millisecond + multiRegionAsiaExpectedMountTime time.Duration = 7500 * time.Millisecond + dualRegionUSExpectedMountTime time.Duration = 4500 * time.Millisecond + dualRegionAsiaExpectedMountTime time.Duration = 6250 * time.Millisecond + singleRegionUSCentralExpectedMountTime time.Duration = 2500 * time.Millisecond + // Constants specific to ZB E2E runs. + testEnvZoneGCEUSCentral1A string = "gce-zone-us-central1-a" + testEnvZoneGCEUSWEST4A string = "gce-zone-us-west4a-a" + testEnvZoneGCEOther string = "gce-zone-other" + zonalUSCentral1ABucket string = "mount_timeout_test_bucket_zb_usc1a" + zonalUSWest4ABucket string = "mount_timeout_test_bucket_zb_usw4a" + zonalSameZoneExpectedMountTime time.Duration = 2500 * time.Millisecond + zonalCrossZoneExpectedMountTime time.Duration = 5000 * time.Millisecond + // Commont constants. + relaxedExpectedMountTime time.Duration = 8000 * time.Millisecond + logfilePathPrefix string = "/tmp/gcsfuse_mount_timeout_" +) + +// findTestExecutionEnvironment determines the environment in which the tests are running. +// It uses the GCP resource detector to identify the environment. +// +// If the tests are running on a GCE instance with a hostname containing non-gce. +// it returns testEnvNonGCE since it implies that the tests are being run on cloudtop. +// +// If the tests are running on a VM in the "us-central" region, it returns gce-us-central . +// Otherwise, if running in any other region, it returns gce-non-us-central. +// +// For all other cases, it returns non-gce. +func findTestExecutionEnvironment(ctx context.Context) string { + detectedAttrs, err := resource.New(ctx, resource.WithDetectors(gcp.NewDetector())) + if err != nil { + log.Printf("Error fetching the test environment.All tests will be skipped.") + } + attrs := detectedAttrs.Set() + if v, exists := attrs.Value("gcp.gce.instance.hostname"); exists && strings.Contains(strings.ToLower(v.AsString()), "cloudtop-prod") { + if !setup.IsZonalBucketRun() { + return testEnvNonGCE + } else { + return testEnvZoneGCEOther + } + } + if !setup.IsZonalBucketRun() { + if v, exists := attrs.Value("cloud.region"); exists { + if strings.Contains(strings.ToLower(v.AsString()), "us-central") { + return testEnvGCEUSCentral + } else { + return testEnvGCENonUSCentral + } + } + return testEnvNonGCE + } else { + if v, exists := attrs.Value("cloud.availability_zone"); exists { + switch strings.ToLower(v.AsString()) { + case "us-central1-a": + return testEnvZoneGCEUSCentral1A + case "us-west4-a": + return testEnvZoneGCEUSWEST4A + default: + return testEnvZoneGCEOther + } + } + return testEnvZoneGCEOther + } +} + +func TestMain(m *testing.M) { + // Parse flags from the setup. + setup.ParseSetUpFlags() + + var err error + + // Find fusermount if we're running on Linux. + if runtime.GOOS == "linux" { + gFusermountPath, err = exec.LookPath("fusermount") + if err != nil { + log.Fatalf("LookPath(fusermount): %p", err) + } + } + + // Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.MountTimeout) == 0 { + log.Println("No configuration found for mount_timeout tests in config. Using flags instead.") + cfg.MountTimeout = make([]test_suite.TestConfig, 1) + cfg.MountTimeout[0].TestBucket = setup.TestBucket() + cfg.MountTimeout[0].GKEMountedDirectory = setup.MountedDirectory() + } + + // Skip for GKE or mounted directory tests. + if cfg.MountTimeout[0].GKEMountedDirectory != "" { + log.Print("These tests will not run for mountedDirectory flag.") + os.Exit(0) + } + + testBucket = cfg.MountTimeout[0].TestBucket + + gCtx = context.Background() + + // Create storage client for direct bucket access. + gStorageClient, err = client.CreateStorageClient(gCtx) + if err != nil { + log.Printf("Error creating storage client: %v\n", err) + os.Exit(1) + } + defer gStorageClient.Close() + + testEnv := findTestExecutionEnvironment(gCtx) + err = os.Setenv("TEST_ENV", testEnv) + if err != nil { + fmt.Println("Error setting environment variable:", err) + return + } + + if setup.TestInstalledPackage() { + // when testInstalledPackage flag is set, gcsfuse is preinstalled on the + // machine. Hence, here we are overwriting gBuildDir to /. + gBuildDir = "/" + code := m.Run() + os.Exit(code) + } + + // To test locally built package + // Set up a directory into which we will build. + gBuildDir, err = os.MkdirTemp("", "gcsfuse_integration_tests") + if err != nil { + log.Fatalf("TempDir: %p", err) + return + } + + // Build into that directory. + err = util.BuildGcsfuse(gBuildDir) + if err != nil { + log.Fatalf("buildGcsfuse: %p", err) + return + } + + // Run tests. + code := m.Run() + + // Clean up and exit. + os.RemoveAll(gBuildDir) + os.Exit(code) +} diff --git a/tools/integration_tests/mounting/gcsfuse_darwin_test.go b/tools/integration_tests/mounting/gcsfuse_darwin_test.go index e237b4932f..a41925d39d 100644 --- a/tools/integration_tests/mounting/gcsfuse_darwin_test.go +++ b/tools/integration_tests/mounting/gcsfuse_darwin_test.go @@ -22,8 +22,8 @@ import ( "math" "syscall" - "github.com/googlecloudplatform/gcsfuse/v2/internal/canned" - "github.com/googlecloudplatform/gcsfuse/v2/tools/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/canned" + "github.com/googlecloudplatform/gcsfuse/v3/tools/util" . "github.com/jacobsa/ogletest" ) diff --git a/tools/integration_tests/mounting/gcsfuse_linux_test.go b/tools/integration_tests/mounting/gcsfuse_linux_test.go index e4b0254b04..3bf0c78bc9 100644 --- a/tools/integration_tests/mounting/gcsfuse_linux_test.go +++ b/tools/integration_tests/mounting/gcsfuse_linux_test.go @@ -22,8 +22,8 @@ import ( "math" "syscall" - "github.com/googlecloudplatform/gcsfuse/v2/internal/canned" - "github.com/googlecloudplatform/gcsfuse/v2/tools/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/canned" + "github.com/googlecloudplatform/gcsfuse/v3/tools/util" . "github.com/jacobsa/ogletest" ) diff --git a/tools/integration_tests/mounting/gcsfuse_test.go b/tools/integration_tests/mounting/gcsfuse_test.go index 505c99b4ca..539be7e74a 100644 --- a/tools/integration_tests/mounting/gcsfuse_test.go +++ b/tools/integration_tests/mounting/gcsfuse_test.go @@ -21,13 +21,12 @@ import ( "os/exec" "path" "path/filepath" - "strings" "syscall" "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/canned" - "github.com/googlecloudplatform/gcsfuse/v2/tools/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/canned" + "github.com/googlecloudplatform/gcsfuse/v3/tools/util" "github.com/jacobsa/fuse/fusetesting" . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/ogletest" @@ -223,17 +222,6 @@ func (t *GcsfuseTest) KeyFile() { } } -func (t *GcsfuseTest) Version() { - for _, arg := range []string{"-v", "--v", "--version", "-version"} { - cmd := t.gcsfuseCommand([]string{arg}, nil) - - output, err := cmd.CombinedOutput() - - AssertEq(nil, err) - AssertTrue(strings.Contains(string(output), "fake_version")) - } -} - func (t *GcsfuseTest) CannedContents() { var err error var fi os.FileInfo diff --git a/tools/integration_tests/mounting/main_test.go b/tools/integration_tests/mounting/main_test.go index 7f0b746d40..27fdde30bf 100644 --- a/tools/integration_tests/mounting/main_test.go +++ b/tools/integration_tests/mounting/main_test.go @@ -21,8 +21,9 @@ import ( "runtime" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/util" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" + "github.com/googlecloudplatform/gcsfuse/v3/tools/util" ) // A directory containing outputs created by build_gcsfuse, set up and deleted @@ -47,6 +48,21 @@ func TestMain(m *testing.M) { } } + // Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.Mounting) == 0 { + log.Println("No configuration found for mounting tests in config. Using flags instead.") + cfg.Mounting = make([]test_suite.TestConfig, 1) + cfg.Mounting[0].TestBucket = setup.TestBucket() + cfg.Mounting[0].GKEMountedDirectory = setup.MountedDirectory() + } + + // Skip for GKE or mounted directory tests. + if cfg.Mounting[0].GKEMountedDirectory != "" { + log.Print("These tests will not run for mountedDirectory flag.") + os.Exit(0) + } + if setup.TestInstalledPackage() { // when testInstalledPackage flag is set, gcsfuse is preinstalled on the // machine. Hence, here we are overwriting gBuildDir to /. diff --git a/tools/integration_tests/mounting/mount_helper_test.go b/tools/integration_tests/mounting/mount_helper_test.go index 68a4ad0e53..984e2a0379 100644 --- a/tools/integration_tests/mounting/mount_helper_test.go +++ b/tools/integration_tests/mounting/mount_helper_test.go @@ -22,8 +22,8 @@ import ( "runtime" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/internal/canned" - "github.com/googlecloudplatform/gcsfuse/v2/tools/util" + "github.com/googlecloudplatform/gcsfuse/v3/internal/canned" + "github.com/googlecloudplatform/gcsfuse/v3/tools/util" . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/ogletest" ) diff --git a/tools/integration_tests/negative_stat_cache/disabled_negative_stat_cache_test.go b/tools/integration_tests/negative_stat_cache/disabled_negative_stat_cache_test.go new file mode 100644 index 0000000000..a36373c87a --- /dev/null +++ b/tools/integration_tests/negative_stat_cache/disabled_negative_stat_cache_test.go @@ -0,0 +1,104 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package negative_stat_cache + +import ( + "log" + "os" + "path" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type disabledNegativeStatCacheTest struct { + flags []string + testDir string + suite.Suite +} + +func (s *disabledNegativeStatCacheTest) SetupSuite() { + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, s.flags, testEnv.mountFunc) + setup.SetMntDir(testEnv.mountDir) +} + +func (s *disabledNegativeStatCacheTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *disabledNegativeStatCacheTest) SetupTest() { + s.testDir = testDirName + setup.GenerateRandomString(5) + testEnv.testDirPath = setup.SetupTestDirectory(s.testDir) +} + +func (s *disabledNegativeStatCacheTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +//////////////////////////////////////////////////////////////////////// +// Test scenarios +//////////////////////////////////////////////////////////////////////// + +func (s *disabledNegativeStatCacheTest) TestNegativeStatCacheDisabled() { + targetDir := path.Join(testEnv.testDirPath, "explicit_dir") + // Create test directory + operations.CreateDirectory(targetDir, s.T()) + targetFile := path.Join(targetDir, "file1.txt") + + // Error should be returned as file does not exist + _, err := os.OpenFile(targetFile, os.O_RDONLY, os.FileMode(0600)) + + assert.NotNil(s.T(), err) + // Assert the underlying error is File Not Exist + assert.ErrorContains(s.T(), err, "explicit_dir/file1.txt: no such file or directory") + + // Adding the object with same name + client.CreateObjectInGCSTestDir(testEnv.ctx, testEnv.storageClient, s.testDir, "explicit_dir/file1.txt", "some-content", s.T()) + + // File should be returned, as call will be served from GCS and gcsfuse should not return from cache + f, err := os.OpenFile(targetFile, os.O_RDONLY, os.FileMode(0600)) + + //Assert File is found + assert.NoError(s.T(), err) + assert.Contains(s.T(), f.Name(), "explicit_dir/file1.txt") + assert.Nil(s.T(), f.Close()) +} + +//////////////////////////////////////////////////////////////////////// +// Test Function (Runs once before all tests) +//////////////////////////////////////////////////////////////////////// + +func TestDisabledNegativeStatCacheTest(t *testing.T) { + ts := &disabledNegativeStatCacheTest{} + + // Run tests for mounted directory if the flag is set. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + // Define flag set to run the tests. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + + // Run tests. + for _, ts.flags = range flagsSet { + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/negative_stat_cache/finite_int_negative_stat_cache_test.go b/tools/integration_tests/negative_stat_cache/finite_int_negative_stat_cache_test.go new file mode 100644 index 0000000000..e74e00cf3c --- /dev/null +++ b/tools/integration_tests/negative_stat_cache/finite_int_negative_stat_cache_test.go @@ -0,0 +1,115 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package negative_stat_cache + +import ( + "log" + "os" + "path" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type finiteNegativeStatCacheTest struct { + flags []string + testDir string + suite.Suite +} + +func (s *finiteNegativeStatCacheTest) SetupSuite() { + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, s.flags, testEnv.mountFunc) + setup.SetMntDir(testEnv.mountDir) +} + +func (s *finiteNegativeStatCacheTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *finiteNegativeStatCacheTest) SetupTest() { + s.testDir = testDirName + setup.GenerateRandomString(5) + testEnv.testDirPath = setup.SetupTestDirectory(s.testDir) +} + +func (s *finiteNegativeStatCacheTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +//////////////////////////////////////////////////////////////////////// +// Test scenarios +//////////////////////////////////////////////////////////////////////// + +func (s *finiteNegativeStatCacheTest) TestFiniteNegativeStatCache() { + targetDir := path.Join(testEnv.testDirPath, "explicit_dir") + // Create test directory + operations.CreateDirectory(targetDir, s.T()) + targetFile := path.Join(targetDir, "file1.txt") + + // Error should be returned as file does not exist + _, err := os.OpenFile(targetFile, os.O_RDONLY, os.FileMode(0600)) + + assert.NotNil(s.T(), err) + // Assert the underlying error is File Not Exist + assert.ErrorContains(s.T(), err, "explicit_dir/file1.txt: no such file or directory") + + // Adding the object with same name + client.CreateObjectInGCSTestDir(testEnv.ctx, testEnv.storageClient, s.testDir, path.Join("explicit_dir", "file1.txt"), "some-content", s.T()) + + // Error should be returned again, as call will not be served from GCS due to finite gcsfuse stat cache + _, err = os.OpenFile(targetFile, os.O_RDONLY, os.FileMode(0600)) + + assert.NotNil(s.T(), err) + // Assert the underlying error is File Not Exist + assert.ErrorContains(s.T(), err, "explicit_dir/file1.txt: no such file or directory") + + //Wait for Cache to expire + time.Sleep(5 * time.Second) + + // File should be returned, as call will be served from GCS and gcsfuse should not return from cache + f, err := os.OpenFile(targetFile, os.O_RDONLY, os.FileMode(0600)) + + //Assert File is found + assert.NoError(s.T(), err) + assert.Contains(s.T(), f.Name(), "explicit_dir/file1.txt") + assert.Nil(s.T(), f.Close()) +} + +//////////////////////////////////////////////////////////////////////// +// Test Function (Runs once before all tests) +//////////////////////////////////////////////////////////////////////// + +func TestFiniteNegativeStatCacheTest(t *testing.T) { + ts := &finiteNegativeStatCacheTest{} + + // Run tests for mounted directory if the flag is set. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + // Define flag set to run the tests. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + + // Run tests. + for _, ts.flags = range flagsSet { + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/negative_stat_cache/infinite_negative_stat_cache_test.go b/tools/integration_tests/negative_stat_cache/infinite_negative_stat_cache_test.go new file mode 100644 index 0000000000..4b586357a7 --- /dev/null +++ b/tools/integration_tests/negative_stat_cache/infinite_negative_stat_cache_test.go @@ -0,0 +1,136 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package negative_stat_cache + +import ( + "log" + "os" + "path" + "syscall" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type infiniteNegativeStatCacheTest struct { + flags []string + testDir string + suite.Suite +} + +func (s *infiniteNegativeStatCacheTest) SetupSuite() { + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, s.flags, testEnv.mountFunc) + setup.SetMntDir(testEnv.mountDir) +} + +func (s *infiniteNegativeStatCacheTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *infiniteNegativeStatCacheTest) SetupTest() { + s.testDir = testDirName + setup.GenerateRandomString(5) + testEnv.testDirPath = setup.SetupTestDirectory(s.testDir) +} + +func (s *infiniteNegativeStatCacheTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +//////////////////////////////////////////////////////////////////////// +// Test scenarios +//////////////////////////////////////////////////////////////////////// + +func (s *infiniteNegativeStatCacheTest) TestInfiniteNegativeStatCache() { + targetDir := path.Join(testEnv.testDirPath, "explicit_dir") + // Create test directory + operations.CreateDirectory(targetDir, s.T()) + targetFile := path.Join(targetDir, "file1.txt") + + // Error should be returned as file does not exist + _, err := os.OpenFile(targetFile, os.O_RDONLY, os.FileMode(0600)) + + assert.NotNil(s.T(), err) + // Assert the underlying error is File Not Exist + assert.ErrorContains(s.T(), err, "explicit_dir/file1.txt: no such file or directory") + + // Adding the object with same name + client.CreateObjectInGCSTestDir(testEnv.ctx, testEnv.storageClient, s.testDir, "explicit_dir/file1.txt", "some-content", s.T()) + + // Error should be returned again, as call will not be served from GCS due to infinite gcsfuse stat cache + _, err = os.OpenFile(targetFile, os.O_RDONLY, os.FileMode(0600)) + + assert.NotNil(s.T(), err) + // Assert the underlying error is File Not Exist + assert.ErrorContains(s.T(), err, "explicit_dir/file1.txt: no such file or directory") +} + +// TestAlreadyExistFolder tests the scenario where a folder creation attempt fails +// with EEXIST. The infinite negative cache is essential because LookUpInode must +// return a "not found" error to trigger the subsequent create operation. This occurs +// when a folder is created externally after gcsfuse has cached a negative stat entry for that path. +// The negative cache prevents gcsfuse from seeing the externally created folder, +// leading to an EEXIST error when attempting to create the same folder again. +func (s *infiniteNegativeStatCacheTest) TestAlreadyExistFolder() { + dirName := "testAlreadyExistFolder" + dirPath := path.Join(testEnv.testDirPath, dirName) + dirPathOnBucket := path.Join(s.testDir, dirName) + // Stat should return an error because the directory doesn't exist yet, + // populating the negative metadata cache. + _, err := os.Stat(dirPath) + require.Error(s.T(), err) + require.True(s.T(), os.IsNotExist(err)) + // Create the directory in the bucket using a different client outside of gcsfuse. + if setup.IsHierarchicalBucket(testEnv.ctx, testEnv.storageClient) { + _, err = client.CreateFolderInBucket(testEnv.ctx, testEnv.storageControlClient, dirPathOnBucket) + } else { + err = client.CreateObjectOnGCS(testEnv.ctx, testEnv.storageClient, dirPathOnBucket+"/", "") + } + require.NoError(s.T(), err) + + // Attempting to create the directory again should fail with EEXIST because the + // negative stat cache entry persists, causing LookUpInode to return a "not found" error + // and triggering a directory creation attempt despite the directory already existing in GCS. + err = os.Mkdir(dirPath, setup.DirPermission_0755) + + assert.ErrorIs(s.T(), err, syscall.EEXIST) +} + +//////////////////////////////////////////////////////////////////////// +// Test Function (Runs once before all tests) +//////////////////////////////////////////////////////////////////////// + +func TestInfiniteNegativeStatCacheTest(t *testing.T) { + ts := &infiniteNegativeStatCacheTest{} + + // Run tests for mounted directory if the flag is set. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + // Define flag set to run the tests. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + + // Run tests. + for _, ts.flags = range flagsSet { + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/negative_stat_cache/setup_test.go b/tools/integration_tests/negative_stat_cache/setup_test.go new file mode 100644 index 0000000000..5774000c81 --- /dev/null +++ b/tools/integration_tests/negative_stat_cache/setup_test.go @@ -0,0 +1,146 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package negative_stat_cache + +import ( + "context" + "log" + "os" + "path" + "testing" + + "cloud.google.com/go/storage" + control "cloud.google.com/go/storage/control/apiv2" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/dynamic_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/only_dir_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" +) + +const ( + testDirName = "NegativeStatCacheTest" + onlyDirMounted = "OnlyDirMountNegativeStatCache" +) + +// IMPORTANT: To prevent global variable pollution, enhance code clarity, +// and avoid inadvertent errors. We strongly suggest that, all new package-level +// variables (which would otherwise be declared with `var` at the package root) should +// be added as fields to this 'env' struct instead. +type env struct { + mountFunc func(*test_suite.TestConfig, []string) error + // mount directory is where our tests run. + mountDir string + // root directory is the directory to be unmounted. + rootDir string + storageClient *storage.Client + storageControlClient *control.StorageControlClient + ctx context.Context + testDirPath string + cfg *test_suite.TestConfig + bucketType string +} + +var testEnv env + +//////////////////////////////////////////////////////////////////////// +// TestMain +//////////////////////////////////////////////////////////////////////// + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.NegativeStatCache) == 0 { + log.Println("No configuration found for negative_stat_cache tests in config. Using flags instead.") + // Populate the config manually. + cfg.NegativeStatCache = make([]test_suite.TestConfig, 1) + cfg.NegativeStatCache[0].TestBucket = setup.TestBucket() + cfg.NegativeStatCache[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.NegativeStatCache[0].LogFile = setup.LogFile() + // Initialize the slice to hold specific test configurations + cfg.NegativeStatCache[0].Configs = make([]test_suite.ConfigItem, 3) + cfg.NegativeStatCache[0].Configs[0].Flags = []string{"--metadata-cache-negative-ttl-secs=0"} + cfg.NegativeStatCache[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.NegativeStatCache[0].Configs[0].Run = "TestDisabledNegativeStatCacheTest" + cfg.NegativeStatCache[0].Configs[1].Flags = []string{"--metadata-cache-negative-ttl-secs=5"} + cfg.NegativeStatCache[0].Configs[1].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.NegativeStatCache[0].Configs[1].Run = "TestFiniteNegativeStatCacheTest" + cfg.NegativeStatCache[0].Configs[2].Flags = []string{"--metadata-cache-negative-ttl-secs=-1"} + cfg.NegativeStatCache[0].Configs[2].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.NegativeStatCache[0].Configs[2].Run = "TestInfiniteNegativeStatCacheTest" + } + + testEnv.ctx = context.Background() + testEnv.bucketType = setup.TestEnvironment(testEnv.ctx, &cfg.NegativeStatCache[0]) + testEnv.cfg = &cfg.NegativeStatCache[0] + + // Create common storage client to be used in test. + closeStorageClient := client.CreateStorageClientWithCancel(&testEnv.ctx, &testEnv.storageClient) + defer func() { + err := closeStorageClient() + if err != nil { + log.Printf("closeStorageClient failed: %v\n", err) + } + }() + closeStorageControlClient := client.CreateControlClientWithCancel(&testEnv.ctx, &testEnv.storageControlClient) + defer func() { + err := closeStorageControlClient() + if err != nil { + log.Printf("closeStorageControlClient failed: %v\n", err) + } + }() + + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + // Save mount and root directory variables. + testEnv.mountDir, testEnv.rootDir = testEnv.cfg.GKEMountedDirectory, testEnv.cfg.GKEMountedDirectory + os.Exit(setup.RunTestsForMountedDirectory(testEnv.cfg.GKEMountedDirectory, m)) + } + + // Run tests for testBucket + // Set up test directory. + setup.SetUpTestDirForTestBucket(testEnv.cfg) + + // Save mount and root directory variables. + testEnv.mountDir, testEnv.rootDir = testEnv.cfg.GCSFuseMountedDirectory, testEnv.cfg.GCSFuseMountedDirectory + + log.Println("Running static mounting tests...") + testEnv.mountFunc = static_mounting.MountGcsfuseWithStaticMountingWithConfigFile + successCode := m.Run() + + if successCode == 0 { + log.Println("Running dynamic mounting tests...") + // Save mount directory variable to have path of bucket to run tests. + testEnv.mountDir = path.Join(testEnv.cfg.GCSFuseMountedDirectory, testEnv.cfg.TestBucket) + testEnv.mountFunc = dynamic_mounting.MountGcsfuseWithDynamicMountingWithConfig + successCode = m.Run() + } + + if successCode == 0 { + log.Println("Running only dir mounting tests...") + setup.SetOnlyDirMounted(onlyDirMounted + "/") + testEnv.mountDir = testEnv.rootDir + testEnv.mountFunc = only_dir_mounting.MountGcsfuseWithOnlyDirWithConfigFile + successCode = m.Run() + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(testEnv.cfg.TestBucket, setup.OnlyDirMounted(), testDirName)) + } + + // Clean up test directory created. + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(testEnv.cfg.TestBucket, testDirName)) + os.Exit(successCode) +} diff --git a/tools/integration_tests/operations/copy_dir_test.go b/tools/integration_tests/operations/copy_dir_test.go index 0728a59d92..4ac4421d59 100644 --- a/tools/integration_tests/operations/copy_dir_test.go +++ b/tools/integration_tests/operations/copy_dir_test.go @@ -21,8 +21,8 @@ import ( "path" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) // Create below directory structure. @@ -59,7 +59,7 @@ func createSrcDirectoryWithObjects(dirPath string, t *testing.T) string { } // Closing file at the end - defer operations.CloseFile(file) + defer operations.CloseFileShouldNotThrowError(t, file) return dirPath } diff --git a/tools/integration_tests/operations/copy_file_test.go b/tools/integration_tests/operations/copy_file_test.go index 2d6ceb57c0..b1d7c0cced 100644 --- a/tools/integration_tests/operations/copy_file_test.go +++ b/tools/integration_tests/operations/copy_file_test.go @@ -20,15 +20,16 @@ import ( "path" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) func TestCopyFile(t *testing.T) { testDir := setup.SetupTestDirectory(DirForOperationTests) - fileName := path.Join(testDir, tempFileName) + fileName := path.Join(testDir, tempFileName+setup.GenerateRandomString(5)) operations.CreateFileWithContent(fileName, setup.FilePermission_0600, Content, t) + defer os.Remove(fileName) content, err := operations.ReadFile(fileName) if err != nil { @@ -44,6 +45,7 @@ func TestCopyFile(t *testing.T) { if err != nil { t.Errorf("Error : %v", err) } + defer os.Remove(newFileName) // Check if the data in the copied file matches the original file, // and the data in original file is unchanged. diff --git a/tools/integration_tests/operations/create_three_level_dir_test.go b/tools/integration_tests/operations/create_three_level_dir_test.go index a264009d20..2d87115b82 100644 --- a/tools/integration_tests/operations/create_three_level_dir_test.go +++ b/tools/integration_tests/operations/create_three_level_dir_test.go @@ -24,8 +24,8 @@ import ( "path/filepath" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) func TestCreateThreeLevelDirectories(t *testing.T) { diff --git a/tools/integration_tests/operations/delete_dir_test.go b/tools/integration_tests/operations/delete_dir_test.go index e89fd88cb5..8eab7fd244 100644 --- a/tools/integration_tests/operations/delete_dir_test.go +++ b/tools/integration_tests/operations/delete_dir_test.go @@ -20,8 +20,8 @@ import ( "path" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) func TestDeleteEmptyExplicitDir(t *testing.T) { diff --git a/tools/integration_tests/operations/delete_file_test.go b/tools/integration_tests/operations/delete_file_test.go index 5cf56ba3f6..ac6fad0474 100644 --- a/tools/integration_tests/operations/delete_file_test.go +++ b/tools/integration_tests/operations/delete_file_test.go @@ -20,8 +20,8 @@ import ( "path" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) const DirNameInTestBucket = "A" // testBucket/A @@ -48,7 +48,7 @@ func createFile(filePath string, t *testing.T) { } // Closing file at the end - operations.CloseFile(file) + operations.CloseFileShouldNotThrowError(t, file) } // Remove testBucket/A.txt diff --git a/tools/integration_tests/operations/file_and_dir_attributes_test.go b/tools/integration_tests/operations/file_and_dir_attributes_test.go index 23704d8458..24eb08f078 100644 --- a/tools/integration_tests/operations/file_and_dir_attributes_test.go +++ b/tools/integration_tests/operations/file_and_dir_attributes_test.go @@ -16,81 +16,101 @@ package operations_test import ( + "context" + "fmt" "os" "path" "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) -const DirAttrTest = "dirAttrTest" -const PrefixFileInDirAttrTest = "fileInDirAttrTest" -const NumberOfFilesInDirAttrTest = 2 -const BytesWrittenInFile = 14 +const ( + DirAttrTest = "dirAttrTest" + PrefixFileInDirAttrTest = "fileInDirAttrTest" + NumberOfFilesInDirAttrTest = 2 + BytesWrittenInFile = 14 + retryFrequency = 10 * time.Second + retryDuration = 3 * time.Minute +) -func checkIfObjectAttrIsCorrect(objName string, preCreateTime time.Time, postCreateTime time.Time, byteSize int64, t *testing.T) { +func checkIfObjectAttrIsCorrect(objName string, preCreateTime time.Time, postCreateTime time.Time, byteSize int64, t *testing.T) error { oStat, err := os.Stat(objName) if err != nil { - t.Errorf("os.Stat error: %s, %v", objName, err) + return fmt.Errorf("stat object %q failed: %w", objName, err) } statObjName := path.Join(setup.MntDir(), DirForOperationTests, oStat.Name()) if objName != statObjName { - t.Errorf("File name not matched in os.Stat, found: %s, expected: %s", statObjName, objName) + return fmt.Errorf("object name mismatch: got %q, want %q", statObjName, objName) } statModTime := oStat.ModTime() - if (preCreateTime.After(statModTime)) || (postCreateTime.Before(statModTime)) { - t.Errorf("File modification time not in the expected time-range") + if preCreateTime.After(statModTime) || postCreateTime.Before(statModTime) { + return fmt.Errorf("object modification time %v is not within expected range [%v, %v]", statModTime, preCreateTime, postCreateTime) } if oStat.Size() != byteSize { - t.Errorf("File size is not %v bytes, found size: %d bytes", BytesWrittenInFile, oStat.Size()) + return fmt.Errorf("object size mismatch: got %d bytes, want %d bytes", oStat.Size(), byteSize) } + t.Logf("Attributes for %q are correct. ModTime: %v (expected range: [%v, %v])", objName, statModTime, preCreateTime, postCreateTime) + return nil } func TestFileAttributes(t *testing.T) { testDir := setup.SetupTestDirectory(DirForOperationTests) - // kernel time can be slightly out of sync of time.Now(), so using - // operations.TimeSlop to adjust pre and post create time. - // Ref: https://github.com/golang/go/issues/33510 - preCreateTime := time.Now().Add(-operations.TimeSlop) - fileName := path.Join(testDir, tempFileName) - operations.CreateFileWithContent(fileName, setup.FilePermission_0600, Content, t) - postCreateTime := time.Now().Add(+operations.TimeSlop) - - // The file size in createTempFile() is BytesWrittenInFile bytes - // https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/tools/integration_tests/util/setup/setup.go#L124 - checkIfObjectAttrIsCorrect(fileName, preCreateTime, postCreateTime, BytesWrittenInFile, t) + t.Logf("Verifying file attributes. Expecting: correct name, size = %d bytes, and mod time within window.", BytesWrittenInFile) + operations.RetryUntil(context.Background(), t, retryFrequency, retryDuration, func() (bool, error) { + fileName := path.Join(testDir, operations.GetRandomName(t)) + // kernel time can be slightly out of sync of time.Now(), so using + // operations.TimeSlop to adjust pre and post create time. + // Ref: https://github.com/golang/go/issues/33510 + preCreateTime := time.Now().Add(-operations.TimeSlop) + operations.CreateFileWithContent(fileName, setup.FilePermission_0600, Content, t) + postCreateTime := time.Now().Add(+operations.TimeSlop) + + // The file size in createTempFile() is BytesWrittenInFile bytes + // https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/tools/integration_tests/util/setup/setup.go#L124 + err := checkIfObjectAttrIsCorrect(fileName, preCreateTime, postCreateTime, BytesWrittenInFile, t) + return err == nil, err + }) } func TestEmptyDirAttributes(t *testing.T) { testDir := setup.SetupTestDirectory(DirForOperationTests) - // kernel time can be slightly out of sync of time.Now(), so using - // operations.TimeSlop to adjust pre and post create time. - // Ref: https://github.com/golang/go/issues/33510 - preCreateTime := time.Now().Add(-operations.TimeSlop) - dirName := path.Join(testDir, DirAttrTest) - operations.CreateDirectoryWithNFiles(0, dirName, "", t) - postCreateTime := time.Now().Add(operations.TimeSlop) - - checkIfObjectAttrIsCorrect(path.Join(testDir, DirAttrTest), preCreateTime, postCreateTime, 0, t) + t.Log("Verifying empty directory attributes. Expecting: correct name, size = 0 bytes, and mod time within window.") + operations.RetryUntil(context.Background(), t, retryFrequency, retryDuration, func() (bool, error) { + dirName := path.Join(testDir, operations.GetRandomName(t)) + // kernel time can be slightly out of sync of time.Now(), so using + // operations.TimeSlop to adjust pre and post create time. + // Ref: https://github.com/golang/go/issues/33510 + preCreateTime := time.Now().Add(-operations.TimeSlop) + operations.CreateDirectoryWithNFiles(0, dirName, "", t) + postCreateTime := time.Now().Add(operations.TimeSlop) + + err := checkIfObjectAttrIsCorrect(dirName, preCreateTime, postCreateTime, 0, t) + return err == nil, err + }) } func TestNonEmptyDirAttributes(t *testing.T) { testDir := setup.SetupTestDirectory(DirForOperationTests) - // kernel time can be slightly out of sync of time.Now(), so using - // operations.TimeSlop to adjust pre and post create time. - // Ref: https://github.com/golang/go/issues/33510 - preCreateTime := time.Now().Add(-operations.TimeSlop) - dirName := path.Join(testDir, DirAttrTest) - operations.CreateDirectoryWithNFiles(NumberOfFilesInDirAttrTest, dirName, PrefixFileInDirAttrTest, t) - postCreateTime := time.Now().Add(operations.TimeSlop) - - checkIfObjectAttrIsCorrect(dirName, preCreateTime, postCreateTime, 0, t) + t.Log("Verifying non-empty directory attributes. Expecting: correct name, size = 0 bytes, and mod time within window.") + operations.RetryUntil(context.Background(), t, retryFrequency, retryDuration, func() (bool, error) { + dirName := path.Join(testDir, operations.GetRandomName(t)) + // kernel time can be slightly out of sync of time.Now(), so using + // operations.TimeSlop to adjust pre and post create time. + // Ref: https://github.com/golang/go/issues/33510 + preCreateTime := time.Now().Add(-operations.TimeSlop) + operations.CreateDirectoryWithNFiles(NumberOfFilesInDirAttrTest, dirName, PrefixFileInDirAttrTest, t) + postCreateTime := time.Now().Add(operations.TimeSlop) + + err := checkIfObjectAttrIsCorrect(dirName, preCreateTime, postCreateTime, 0, t) + return err == nil, err + }) } diff --git a/tools/integration_tests/operations/list_dir_test.go b/tools/integration_tests/operations/list_dir_test.go index 7e02e90c37..8a4bf2a643 100644 --- a/tools/integration_tests/operations/list_dir_test.go +++ b/tools/integration_tests/operations/list_dir_test.go @@ -22,10 +22,14 @@ import ( "os" "path" "path/filepath" + "syscall" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" ) func createDirectoryStructureForTest(t *testing.T) { @@ -180,3 +184,18 @@ func TestListDirectoryRecursively(t *testing.T) { return } } + +func TestReadFileWorksAfterListDir(t *testing.T) { + testDir := setup.SetupTestDirectory(DirForOperationTests) + obj1 := t.Name() + "-1" + var objSize int64 = 5 + client.SetupFileInTestDirectory(ctx, storageClient, DirForOperationTests, obj1, objSize, t) + + _ = operations.ReadDirectory(testDir, t) + fh, err := os.OpenFile(path.Join(testDir, obj1), syscall.O_DIRECT, operations.FilePermission_0600) + require.NoError(t, err) + content, err := operations.ReadFileSequentially(fh, util.MiB) + + require.NoError(t, err) + require.EqualValues(t, objSize, len(content)) +} diff --git a/tools/integration_tests/operations/move_file_test.go b/tools/integration_tests/operations/move_file_test.go index a2fa38eddf..eb2eb4a81d 100644 --- a/tools/integration_tests/operations/move_file_test.go +++ b/tools/integration_tests/operations/move_file_test.go @@ -18,10 +18,12 @@ package operations_test import ( "os" "path" + "strings" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" ) // Create below directory and file. @@ -40,7 +42,7 @@ func createSrcDirectoryAndFile(dirPath string, filePath string, t *testing.T) { } // Closing file at the end. - defer operations.CloseFile(file) + defer operations.CloseFileShouldNotThrowError(t, file) err = operations.WriteFile(file.Name(), MoveFileContent) if err != nil { @@ -100,3 +102,26 @@ func TestMoveFileWithinDifferentDirectory(t *testing.T) { checkIfFileMoveOperationSucceeded(filePath, destDirPath, t) } + +// Rename file from Test/move1.txt to Test/move2.txt +func TestMoveFileWithDestFileExist(t *testing.T) { + // Set up the test directory. + testDir := setup.SetupTestDirectory(DirForOperationTests) + // Define source and destination file names. + srcFilePath := path.Join(testDir, "move1.txt") + destFilePath := path.Join(testDir, "move2.txt") + // Create the source and dest file with some content. + operations.CreateFileWithContent(srcFilePath, setup.FilePermission_0600, Content, t) + operations.CreateFileWithContent(destFilePath, setup.FilePermission_0600, "Hello from dest file", t) + + // Move the file. + err := operations.Move(srcFilePath, destFilePath) + + assert.NoError(t, err, "error in file moving") + // Verify the file was renamed and content is preserved. + setup.CompareFileContents(t, destFilePath, Content) + // Verify the old file is removed. + _, err = os.Stat(srcFilePath) + assert.Error(t, err) + assert.True(t, strings.Contains(err.Error(), "no such file or directory")) +} diff --git a/tools/integration_tests/operations/operations_test.go b/tools/integration_tests/operations/operations_test.go index ba7242e1d7..ce51e73eba 100644 --- a/tools/integration_tests/operations/operations_test.go +++ b/tools/integration_tests/operations/operations_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -20,17 +20,18 @@ import ( "log" "os" "path" - "strconv" + "strings" "testing" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/creds_tests" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/dynamic_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/only_dir_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/persistent_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/creds_tests" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/dynamic_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/only_dir_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/persistent_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" ) const DirForOperationTests = "dirForOperationsTest" @@ -90,62 +91,82 @@ const Content = "line 1\nline 2\n" const onlyDirMounted = "OnlyDirMountOperations" var ( - cacheDir string storageClient *storage.Client ctx context.Context ) -func createMountConfigsAndEquivalentFlags() (flags [][]string) { - cacheDirPath := path.Join(os.TempDir(), cacheDir) - - // Set up config file with create-empty-file: true. - mountConfig1 := map[string]interface{}{ - "write": map[string]interface{}{ - "create-empty-file": true, - }, +func overrideFilePathsInFlagSet(t *test_suite.TestConfig, GCSFuseTempDirPath string) { + for _, flags := range t.Configs { + for i := range flags.Flags { + // Iterate over the indices of the flags slice + flags.Flags[i] = strings.ReplaceAll(flags.Flags[i], "/gcsfuse-tmp", path.Join(GCSFuseTempDirPath, "gcsfuse-tmp")) + } } +} - filePath1 := setup.YAMLConfigFile(mountConfig1, "config1.yaml") - flags = append(flags, []string{"--config-file=" + filePath1}) - - // Set up config file for file cache. - mountConfig2 := map[string]interface{}{ - "file-cache": map[string]interface{}{ - // Keeping the size as low because the operations are performed on small - // files - "max-size-mb": 2, - }, - "cache-dir": cacheDirPath, +func RunTestOnTPCEndPoint(cfg test_suite.Config, m *testing.M) int { + ctx = context.Background() + var err error + if storageClient, err = client.CreateStorageClient(ctx); err != nil { + log.Fatalf("Error creating storage client: %v\n", err) } - filePath2 := setup.YAMLConfigFile(mountConfig2, "config2.yaml") - flags = append(flags, []string{"--config-file=" + filePath2}) - - mountConfig3 := map[string]interface{}{ - "metadata-cache": map[string]interface{}{ - "ttl-secs": 0, - }, + cfg.Operations = make([]test_suite.TestConfig, 1) + cfg.Operations[0].TestBucket = setup.TestBucket() + cfg.Operations[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.Operations[0].Configs = make([]test_suite.ConfigItem, 1) + cfg.Operations[0].Configs[0].Flags = []string{ + "--enable-atomic-rename-object=true", + "--experimental-enable-json-read=true", + "--metadata-cache-ttl-secs=0 --enable-streaming-writes=false", + "--kernel-list-cache-ttl-secs=-1 --implicit-dirs=true", } - filePath3 := setup.YAMLConfigFile(mountConfig3, "config3.yaml") - flags = append(flags, []string{"--config-file=" + filePath3}) + cfg.Operations[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + var flags [][]string - mountConfig4 := map[string]interface{}{ - "file-system": map[string]interface{}{ - "kernel-list-cache-ttl-secs": -1, - }, + // Iterate over the original flags and split each string by spaces + for _, flagSet := range cfg.Operations[0].Configs[0].Flags { + splitFlags := strings.Fields(flagSet) + flags = append(flags, splitFlags) } - filePath4 := setup.YAMLConfigFile(mountConfig4, "config4.yaml") - flags = append(flags, []string{"--config-file=" + filePath4, "--implicit-dirs=true"}) - return flags + setup.SetUpTestDirForTestBucket(&cfg.Operations[0]) + successCodeTPC := static_mounting.RunTestsWithConfigFile(&cfg.Operations[0], flags, m) + return successCodeTPC } func TestMain(m *testing.M) { setup.ParseSetUpFlags() - setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + + // TODO: b/469970353 : Update tpc_build.sh to run using test_config.yaml file. + if setup.TestOnTPCEndPoint() { + log.Println("Running TPC tests without config file.") + successCodeTPC := RunTestOnTPCEndPoint(cfg, m) + os.Exit(successCodeTPC) + } + + if len(cfg.Operations) == 0 { + log.Println("No configuration found for operations tests in config. Using flags instead.") + // Populate the config manually. + cfg.Operations = make([]test_suite.TestConfig, 1) + cfg.Operations[0].TestBucket = setup.TestBucket() + cfg.Operations[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.Operations[0].Configs = make([]test_suite.ConfigItem, 1) + cfg.Operations[0].Configs[0].Flags = []string{ + "--metadata-cache-ttl-secs=0 --enable-streaming-writes=false", + "--kernel-list-cache-ttl-secs=-1 --implicit-dirs=true --enable-metadata-prefetch", + "--experimental-enable-json-read=true --enable-atomic-rename-object=true", + "--client-protocol=grpc --implicit-dirs=true --enable-atomic-rename-object=true --enable-metadata-prefetch", + } + cfg.Operations[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + } - // Create storage client before running tests. - var err error ctx = context.Background() + bucketType := setup.TestEnvironment(ctx, &cfg.Operations[0]) + + // 2. Create storage client before running tests. + var err error storageClient, err = client.CreateStorageClient(ctx) if err != nil { log.Printf("Error creating storage client: %v\n", err) @@ -153,67 +174,37 @@ func TestMain(m *testing.M) { } defer storageClient.Close() - cacheDir = "cache-dir-operations-hns-" + strconv.FormatBool(setup.IsHierarchicalBucket(ctx, storageClient)) - - // To run mountedDirectory tests, we need both testBucket and mountedDirectory - // flags to be set, as operations tests validates content from the bucket. - if setup.AreBothMountedDirectoryAndTestBucketFlagsSet() { - setup.RunTestsForMountedDirectoryFlag(m) - } - - // Run tests for testBucket - // Set up test directory. - setup.SetUpTestDirForTestBucketFlag() - // Set up flags to run tests on. - // Note: GRPC related tests will work only if you have allow-list bucket. - // Note: We are not testing specifically for implicit-dirs because they are covered as part of the other flags. - flagsSet := [][]string{} - - // Enable experimental-enable-json-read=true case, but for non-presubmit runs only. - if !setup.IsPresubmitRun() { - flagsSet = append(flagsSet, []string{ - // By default, creating emptyFile is disabled. - "--experimental-enable-json-read=true"}) - } - - // gRPC tests will not run in TPC environment - if !testing.Short() && !setup.TestOnTPCEndPoint() { - flagsSet = append(flagsSet, []string{"--client-protocol=grpc", "--implicit-dirs=true"}) + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + if cfg.Operations[0].GKEMountedDirectory != "" && cfg.Operations[0].TestBucket != "" { + os.Exit(setup.RunTestsForMountedDirectory(cfg.Operations[0].GKEMountedDirectory, m)) } - // HNS tests utilize the gRPC protocol, which is not supported by TPC. - if !setup.TestOnTPCEndPoint() { - if setup.IsHierarchicalBucket(ctx, storageClient) { - flagsSet = [][]string{{"--experimental-enable-json-read=true"}} - } - } + // 4. Override GKE specific paths with GCSFuse paths if running in GCE environment. + overrideFilePathsInFlagSet(&cfg.Operations[0], setup.TestDir()) - mountConfigFlags := createMountConfigsAndEquivalentFlags() - flagsSet = append(flagsSet, mountConfigFlags...) - - // Only running static_mounting test for TPC. - if setup.TestOnTPCEndPoint() { - successCodeTPC := static_mounting.RunTests(flagsSet, m) - os.Exit(successCodeTPC) - } + // Run tests for testBucket + // 5. Build the flag sets dynamically from the config. + flags := setup.BuildFlagSets(cfg.Operations[0], bucketType, "") + setup.SetUpTestDirForTestBucket(&cfg.Operations[0]) - successCode := static_mounting.RunTests(flagsSet, m) + successCode := static_mounting.RunTestsWithConfigFile(&cfg.Operations[0], flags, m) if successCode == 0 { - successCode = only_dir_mounting.RunTests(flagsSet, onlyDirMounted, m) + successCode = only_dir_mounting.RunTestsWithConfigFile(&cfg.Operations[0], flags, onlyDirMounted, m) } if successCode == 0 { - successCode = persistent_mounting.RunTests(flagsSet, m) + successCode = persistent_mounting.RunTestsWithConfigFile(&cfg.Operations[0], flags, m) } if successCode == 0 { - successCode = dynamic_mounting.RunTests(ctx, storageClient, flagsSet, m) + successCode = dynamic_mounting.RunTestsWithConfigFile(&cfg.Operations[0], flags, m) } if successCode == 0 { // Test for admin permission on test bucket. - successCode = creds_tests.RunTestsForKeyFileAndGoogleApplicationCredentialsEnvVarSet(ctx, storageClient, flagsSet, "objectAdmin", m) + log.Printf("Running cred tests...") + successCode = creds_tests.RunTestsForDifferentAuthMethods(ctx, &cfg.Operations[0], storageClient, flags, "objectAdmin", m) } os.Exit(successCode) diff --git a/tools/integration_tests/operations/parallel_dirops_test.go b/tools/integration_tests/operations/parallel_dirops_test.go index 7d27b7cf75..c5f68a82ec 100644 --- a/tools/integration_tests/operations/parallel_dirops_test.go +++ b/tools/integration_tests/operations/parallel_dirops_test.go @@ -25,69 +25,102 @@ import ( "sync" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" "github.com/stretchr/testify/assert" ) -// createDirectoryStructureForParallelDiropsTest creates the following files and +type testDirStrucure struct { + testDir string + explicitDir1Name string + file1InExplicitDir1Name string + file2InExplicitDir1Name string + explicitDir2Name string + file1InExplicitDir2Name string + file1Name string + file2Name string +} + +// createDirStructure creates the following files and // directory structure. // bucket // -// file1.txt -// file2.txt -// explicitDir1/file1.txt -// explicitDir1/file2.txt -// explicitDir2/file1.txt +// file1Name +// file2Name +// explicitDir1Name/file1InExplicitDir1Name +// explicitDir1Name/file2InExplicitDir1Name +// explicitDir2Name/file1InExplicitDir2Name // // Also returns the path to test directory. -func createDirectoryStructureForParallelDiropsTest(t *testing.T) string { - testDir := setup.SetupTestDirectory(DirForOperationTests) - setup.CleanUpDir(testDir) +func createDirStructure(t *testing.T) testDirStrucure { + var tds testDirStrucure + tds.testDir = setup.SetupTestDirectory(DirForOperationTests + "-" + setup.GenerateRandomString(5)) // Create explicitDir1 structure - explicitDir1 := path.Join(testDir, "explicitDir1") + tds.explicitDir1Name = "explicitDir1-" + setup.GenerateRandomString(5) + explicitDir1 := path.Join(tds.testDir, tds.explicitDir1Name) operations.CreateDirectory(explicitDir1, t) - filePath1 := path.Join(explicitDir1, "file1.txt") + tds.file1InExplicitDir1Name = "file1-" + setup.GenerateRandomString(5) + ".txt" + filePath1 := path.Join(explicitDir1, tds.file1InExplicitDir1Name) operations.CreateFileOfSize(5, filePath1, t) - filePath2 := path.Join(explicitDir1, "file2.txt") + tds.file2InExplicitDir1Name = "file2-" + setup.GenerateRandomString(5) + ".txt" + filePath2 := path.Join(explicitDir1, tds.file2InExplicitDir1Name) operations.CreateFileOfSize(10, filePath2, t) // Create explicitDir2 structure - explicitDir2 := path.Join(testDir, "explicitDir2") + tds.explicitDir2Name = "explicitDir2-" + setup.GenerateRandomString(5) + explicitDir2 := path.Join(tds.testDir, tds.explicitDir2Name) operations.CreateDirectory(explicitDir2, t) - filePath1 = path.Join(explicitDir2, "file1.txt") + tds.file1InExplicitDir2Name = "file1-" + setup.GenerateRandomString(5) + ".txt" + filePath1 = path.Join(explicitDir2, tds.file1InExplicitDir2Name) operations.CreateFileOfSize(11, filePath1, t) - filePath1 = path.Join(testDir, "file1.txt") + tds.file1Name = "file1-" + setup.GenerateRandomString(5) + ".txt" + filePath1 = path.Join(tds.testDir, tds.file1Name) operations.CreateFileOfSize(5, filePath1, t) - filePath2 = path.Join(testDir, "file2.txt") + tds.file2Name = "file2-" + setup.GenerateRandomString(5) + ".txt" + filePath2 = path.Join(tds.testDir, tds.file2Name) operations.CreateFileOfSize(3, filePath2, t) - return testDir + return tds +} + +// deleteDirStructure deletes the following files and +// directory structure. +// bucket +// +// file1Name +// file2Name +// explicitDir1Name/file1InExplicitDir1Name +// explicitDir1Name/file2InExplicitDir1Name +// explicitDir2Name/file1InExplicitDir2Name +// +// Also returns the path to test directory. +func deleteDirStructure(tds testDirStrucure) { + setup.CleanUpDir(tds.testDir) +} + +// lookUpFileStat performs a lookup for the given file path and returns the FileInfo and error. +func lookUpFileStat(wg *sync.WaitGroup, filePath string, result *os.FileInfo, err *error) { + defer wg.Done() + fileInfo, lookupErr := os.Stat(filePath) + *result = fileInfo + *err = lookupErr } func TestParallelLookUpsForSameFile(t *testing.T) { // Create directory structure for testing. - testDir := createDirectoryStructureForParallelDiropsTest(t) - lookUpFunc := func(wg *sync.WaitGroup, filePath string) (os.FileInfo, error) { - defer wg.Done() - fileInfo, err := os.Stat(filePath) - return fileInfo, err - } + tds := createDirStructure(t) + defer deleteDirStructure(tds) var stat1, stat2 os.FileInfo var err1, err2 error // Parallel lookups of file just under mount. - filePath := path.Join(testDir, "file1.txt") + filePath := path.Join(tds.testDir, tds.file1Name) wg := sync.WaitGroup{} wg.Add(2) - go func() { - stat1, err1 = lookUpFunc(&wg, filePath) - }() - go func() { - stat2, err2 = lookUpFunc(&wg, filePath) - }() + go lookUpFileStat(&wg, filePath, &stat1, &err1) + go lookUpFileStat(&wg, filePath, &stat2, &err2) wg.Wait() // Assert both stats passed and give correct information @@ -99,14 +132,10 @@ func TestParallelLookUpsForSameFile(t *testing.T) { assert.Contains(t, filePath, stat2.Name()) // Parallel lookups of file under a directory in mount. - filePath = path.Join(testDir, "explicitDir1/file2.txt") + filePath = path.Join(tds.testDir, tds.explicitDir1Name, tds.file2InExplicitDir1Name) wg.Add(2) - go func() { - stat1, err1 = lookUpFunc(&wg, filePath) - }() - go func() { - stat2, err2 = lookUpFunc(&wg, filePath) - }() + go lookUpFileStat(&wg, filePath, &stat1, &err1) + go lookUpFileStat(&wg, filePath, &stat2, &err2) wg.Wait() // Assert both stats passed and give correct information @@ -120,25 +149,22 @@ func TestParallelLookUpsForSameFile(t *testing.T) { func TestParallelReadDirs(t *testing.T) { // Create directory structure for testing. - testDir := createDirectoryStructureForParallelDiropsTest(t) - readDirFunc := func(wg *sync.WaitGroup, dirPath string) ([]os.DirEntry, error) { + tds := createDirStructure(t) + defer deleteDirStructure(tds) + readDirFunc := func(wg *sync.WaitGroup, dirPath string, dirEntries *[]os.DirEntry, err *error) { defer wg.Done() - dirEntries, err := os.ReadDir(dirPath) - return dirEntries, err + *dirEntries, *err = os.ReadDir(dirPath) } var dirEntries1, dirEntries2 []os.DirEntry var err1, err2 error // Parallel readDirs of explicit dir under mount. - dirPath := path.Join(testDir, "explicitDir1") + dirPath := path.Join(tds.testDir, tds.explicitDir1Name) wg := sync.WaitGroup{} wg.Add(2) - go func() { - dirEntries1, err1 = readDirFunc(&wg, dirPath) - }() - go func() { - dirEntries2, err2 = readDirFunc(&wg, dirPath) - }() + go readDirFunc(&wg, dirPath, &dirEntries1, &err1) + go readDirFunc(&wg, dirPath, &dirEntries2, &err2) + wg.Wait() // Assert both readDirs passed and give correct information @@ -146,22 +172,18 @@ func TestParallelReadDirs(t *testing.T) { assert.NoError(t, err2) assert.Equal(t, 2, len(dirEntries1)) assert.Equal(t, 2, len(dirEntries2)) - assert.Contains(t, "file1.txt", dirEntries1[0].Name()) - assert.Contains(t, "file2.txt", dirEntries1[1].Name()) - assert.Contains(t, "file1.txt", dirEntries2[0].Name()) - assert.Contains(t, "file2.txt", dirEntries2[1].Name()) + assert.Contains(t, tds.file1InExplicitDir1Name, dirEntries1[0].Name()) + assert.Contains(t, tds.file2InExplicitDir1Name, dirEntries1[1].Name()) + assert.Contains(t, tds.file1InExplicitDir1Name, dirEntries2[0].Name()) + assert.Contains(t, tds.file2InExplicitDir1Name, dirEntries2[1].Name()) // Parallel readDirs of a directory and its parent directory. - dirPath = path.Join(testDir, "explicitDir1") - parentDirPath := testDir + dirPath = path.Join(tds.testDir, tds.explicitDir1Name) + parentDirPath := tds.testDir wg = sync.WaitGroup{} wg.Add(2) - go func() { - dirEntries1, err1 = readDirFunc(&wg, dirPath) - }() - go func() { - dirEntries2, err2 = readDirFunc(&wg, parentDirPath) - }() + go readDirFunc(&wg, dirPath, &dirEntries1, &err1) + go readDirFunc(&wg, parentDirPath, &dirEntries2, &err2) wg.Wait() // Assert both readDirs passed and give correct information @@ -169,40 +191,31 @@ func TestParallelReadDirs(t *testing.T) { assert.NoError(t, err2) assert.Equal(t, 2, len(dirEntries1)) assert.Equal(t, 4, len(dirEntries2)) - assert.Contains(t, "file1.txt", dirEntries1[0].Name()) - assert.Contains(t, "file2.txt", dirEntries1[1].Name()) - assert.Contains(t, "explicitDir1", dirEntries2[0].Name()) - assert.Contains(t, "explicitDir2", dirEntries2[1].Name()) - assert.Contains(t, "file1.txt", dirEntries2[2].Name()) - assert.Contains(t, "file2.txt", dirEntries2[3].Name()) + assert.Contains(t, tds.file1InExplicitDir1Name, dirEntries1[0].Name()) + assert.Contains(t, tds.file2InExplicitDir1Name, dirEntries1[1].Name()) + assert.Contains(t, tds.explicitDir1Name, dirEntries2[0].Name()) + assert.Contains(t, tds.explicitDir2Name, dirEntries2[1].Name()) + assert.Contains(t, tds.file1Name, dirEntries2[2].Name()) + assert.Contains(t, tds.file2Name, dirEntries2[3].Name()) } func TestParallelLookUpAndDeleteSameDir(t *testing.T) { // Create directory structure for testing. - testDir := createDirectoryStructureForParallelDiropsTest(t) - lookUpFunc := func(wg *sync.WaitGroup, dirPath string) (os.FileInfo, error) { + tds := createDirStructure(t) + defer deleteDirStructure(tds) + deleteFunc := func(wg *sync.WaitGroup, dirPath string, err *error) { defer wg.Done() - fileInfo, err := os.Stat(dirPath) - return fileInfo, err - } - deleteFunc := func(wg *sync.WaitGroup, dirPath string) error { - defer wg.Done() - err := os.RemoveAll(dirPath) - return err + *err = os.RemoveAll(dirPath) } var statInfo os.FileInfo var lookUpErr, deleteErr error // Parallel lookup and deletion of explicit dir under mount. - dirPath := path.Join(testDir, "explicitDir1") + dirPath := path.Join(tds.testDir, tds.explicitDir1Name) wg := sync.WaitGroup{} wg.Add(2) - go func() { - statInfo, lookUpErr = lookUpFunc(&wg, dirPath) - }() - go func() { - deleteErr = deleteFunc(&wg, dirPath) - }() + go lookUpFileStat(&wg, dirPath, &statInfo, &lookUpErr) + go deleteFunc(&wg, dirPath, &deleteErr) wg.Wait() assert.NoError(t, deleteErr) @@ -210,8 +223,9 @@ func TestParallelLookUpAndDeleteSameDir(t *testing.T) { assert.True(t, os.IsNotExist(err)) // Assert either dir is looked up first or deleted first if lookUpErr == nil { - assert.Contains(t, statInfo.Name(), "explicitDir1") - assert.True(t, statInfo.IsDir()) + assert.NotNil(t, statInfo, "statInfo should not be nil when lookUpErr is nil") + assert.Contains(t, statInfo.Name(), tds.explicitDir1Name) + assert.True(t, statInfo.IsDir(), "The created path should be a directory") } else { assert.True(t, os.IsNotExist(lookUpErr)) } @@ -219,26 +233,19 @@ func TestParallelLookUpAndDeleteSameDir(t *testing.T) { func TestParallelLookUpsForDifferentFiles(t *testing.T) { // Create directory structure for testing. - testDir := createDirectoryStructureForParallelDiropsTest(t) - lookUpFunc := func(wg *sync.WaitGroup, filePath string) (os.FileInfo, error) { - defer wg.Done() - fileInfo, err := os.Stat(filePath) - return fileInfo, err - } + tds := createDirStructure(t) + defer deleteDirStructure(tds) var stat1, stat2 os.FileInfo var err1, err2 error // Parallel lookups of two files just under mount. - filePath1 := path.Join(testDir, "file1.txt") - filePath2 := path.Join(testDir, "file2.txt") + filePath1 := path.Join(tds.testDir, tds.file1Name) + filePath2 := path.Join(tds.testDir, tds.file2Name) wg := sync.WaitGroup{} wg.Add(2) - go func() { - stat1, err1 = lookUpFunc(&wg, filePath1) - }() - go func() { - stat2, err2 = lookUpFunc(&wg, filePath2) - }() + go lookUpFileStat(&wg, filePath1, &stat1, &err1) + go lookUpFileStat(&wg, filePath2, &stat2, &err2) + wg.Wait() // Assert both stats passed and give correct information @@ -250,16 +257,12 @@ func TestParallelLookUpsForDifferentFiles(t *testing.T) { assert.Contains(t, filePath2, stat2.Name()) // Parallel lookups of two files under a directory in mount. - filePath1 = path.Join(testDir, "explicitDir1", "file1.txt") - filePath2 = path.Join(testDir, "explicitDir1", "file2.txt") + filePath1 = path.Join(tds.testDir, tds.explicitDir1Name, tds.file1InExplicitDir1Name) + filePath2 = path.Join(tds.testDir, tds.explicitDir1Name, tds.file2InExplicitDir1Name) wg = sync.WaitGroup{} wg.Add(2) - go func() { - stat1, err1 = lookUpFunc(&wg, filePath1) - }() - go func() { - stat2, err2 = lookUpFunc(&wg, filePath2) - }() + go lookUpFileStat(&wg, filePath1, &stat1, &err1) + go lookUpFileStat(&wg, filePath2, &stat2, &err2) wg.Wait() // Assert both stats passed and give correct information @@ -273,34 +276,28 @@ func TestParallelLookUpsForDifferentFiles(t *testing.T) { func TestParallelReadDirAndMkdirInsideSameDir(t *testing.T) { // Create directory structure for testing. - testDir := createDirectoryStructureForParallelDiropsTest(t) - readDirFunc := func(wg *sync.WaitGroup, dirPath string) ([]os.DirEntry, error) { + tds := createDirStructure(t) + defer deleteDirStructure(tds) + readDirFunc := func(wg *sync.WaitGroup, dirPath string, dirEntries *[]os.DirEntry, err *error) { defer wg.Done() - var dirEntries []os.DirEntry - err := filepath.WalkDir(dirPath, func(path string, d fs.DirEntry, err error) error { - dirEntries = append(dirEntries, d) + *err = filepath.WalkDir(dirPath, func(path string, d fs.DirEntry, err error) error { + *dirEntries = append(*dirEntries, d) return nil }) - return dirEntries, err } - mkdirFunc := func(wg *sync.WaitGroup, dirPath string) error { + mkdirFunc := func(wg *sync.WaitGroup, dirPath string, err *error) { defer wg.Done() - err := os.Mkdir(dirPath, setup.DirPermission_0755) - return err + *err = os.Mkdir(dirPath, setup.DirPermission_0755) } var dirEntries []os.DirEntry var readDirErr, mkdirErr error // Parallel readDirs and mkdir inside the same directory. - newDirPath := path.Join(testDir, "newDir") + newDirPath := path.Join(tds.testDir, "newDir") wg := sync.WaitGroup{} wg.Add(2) - go func() { - dirEntries, readDirErr = readDirFunc(&wg, testDir) - }() - go func() { - mkdirErr = mkdirFunc(&wg, newDirPath) - }() + go readDirFunc(&wg, tds.testDir, &dirEntries, &readDirErr) + go mkdirFunc(&wg, newDirPath, &mkdirErr) wg.Wait() // Assert both listing and mkdir succeeded @@ -308,7 +305,7 @@ func TestParallelReadDirAndMkdirInsideSameDir(t *testing.T) { assert.NoError(t, mkdirErr) dirStatInfo, err := os.Stat(newDirPath) assert.NoError(t, err) - assert.True(t, dirStatInfo.IsDir()) + assert.True(t, dirStatInfo.IsDir(), "The created path should be a directory") // List should happen either before or after creation of newDir. assert.GreaterOrEqual(t, len(dirEntries), 8) assert.LessOrEqual(t, len(dirEntries), 9) @@ -319,30 +316,23 @@ func TestParallelReadDirAndMkdirInsideSameDir(t *testing.T) { func TestParallelLookUpAndDeleteSameFile(t *testing.T) { // Create directory structure for testing. - testDir := createDirectoryStructureForParallelDiropsTest(t) - lookUpFunc := func(wg *sync.WaitGroup, filePath string) (os.FileInfo, error) { + tds := createDirStructure(t) + defer deleteDirStructure(tds) + deleteFileFunc := func(wg *sync.WaitGroup, filePath string, err *error) { defer wg.Done() - fileInfo, err := os.Stat(filePath) - return fileInfo, err - } - deleteFileFunc := func(wg *sync.WaitGroup, filePath string) error { - defer wg.Done() - err := os.Remove(filePath) - return err + *err = os.Remove(filePath) } var fileInfo os.FileInfo var lookUpErr, deleteErr error // Parallel lookup and deletion of a file. - filePath := path.Join(testDir, "explicitDir1", "file1.txt") + filePath := path.Join(tds.testDir, tds.explicitDir1Name, tds.file1InExplicitDir1Name) wg := sync.WaitGroup{} wg.Add(2) - go func() { - fileInfo, lookUpErr = lookUpFunc(&wg, filePath) - }() - go func() { - deleteErr = deleteFileFunc(&wg, filePath) - }() + + go lookUpFileStat(&wg, filePath, &fileInfo, &lookUpErr) + go deleteFileFunc(&wg, filePath, &deleteErr) + wg.Wait() assert.NoError(t, deleteErr) @@ -350,9 +340,10 @@ func TestParallelLookUpAndDeleteSameFile(t *testing.T) { assert.True(t, os.IsNotExist(err)) // Assert either file is looked up first or deleted first if lookUpErr == nil { + assert.NotNil(t, fileInfo, "fileInfo should not be nil when lookUpErr is nil") assert.Equal(t, int64(5), fileInfo.Size()) - assert.Contains(t, fileInfo.Name(), "file1.txt") - assert.False(t, fileInfo.IsDir()) + assert.Contains(t, fileInfo.Name(), tds.file1InExplicitDir1Name) + assert.False(t, fileInfo.IsDir(), "The created path should not be a directory") } else { assert.True(t, os.IsNotExist(lookUpErr)) } @@ -360,31 +351,23 @@ func TestParallelLookUpAndDeleteSameFile(t *testing.T) { func TestParallelLookUpAndRenameSameFile(t *testing.T) { // Create directory structure for testing. - testDir := createDirectoryStructureForParallelDiropsTest(t) - lookUpFunc := func(wg *sync.WaitGroup, filePath string) (os.FileInfo, error) { - defer wg.Done() - fileInfo, err := os.Stat(filePath) - return fileInfo, err - } - renameFunc := func(wg *sync.WaitGroup, oldFilePath string, newFilePath string) error { + tds := createDirStructure(t) + defer deleteDirStructure(tds) + renameFunc := func(wg *sync.WaitGroup, oldFilePath string, newFilePath string, err *error) { defer wg.Done() - err := os.Rename(oldFilePath, newFilePath) - return err + *err = os.Rename(oldFilePath, newFilePath) } var fileInfo os.FileInfo var lookUpErr, renameErr error // Parallel lookup and rename of a file. - filePath := path.Join(testDir, "explicitDir1", "file1.txt") - newFilePath := path.Join(testDir, "newFile.txt") + filePath := path.Join(tds.testDir, tds.explicitDir1Name, tds.file1InExplicitDir1Name) + newFilePath := path.Join(tds.testDir, "newFile.txt") wg := sync.WaitGroup{} wg.Add(2) - go func() { - fileInfo, lookUpErr = lookUpFunc(&wg, filePath) - }() - go func() { - renameErr = renameFunc(&wg, filePath, newFilePath) - }() + go lookUpFileStat(&wg, filePath, &fileInfo, &lookUpErr) + go renameFunc(&wg, filePath, newFilePath, &renameErr) + wg.Wait() assert.NoError(t, renameErr) @@ -395,9 +378,10 @@ func TestParallelLookUpAndRenameSameFile(t *testing.T) { assert.Equal(t, int64(5), newFileInfo.Size()) // Assert either file is renamed first or looked up first if lookUpErr == nil { + assert.NotNil(t, fileInfo, "fileInfo should not be nil when lookUpErr is nil") assert.Equal(t, int64(5), fileInfo.Size()) - assert.Contains(t, fileInfo.Name(), "file1.txt") - assert.False(t, fileInfo.IsDir()) + assert.Contains(t, fileInfo.Name(), tds.file1InExplicitDir1Name) + assert.False(t, fileInfo.IsDir(), "The created path should not be a directory") } else { assert.True(t, os.IsNotExist(lookUpErr)) } @@ -405,41 +389,35 @@ func TestParallelLookUpAndRenameSameFile(t *testing.T) { func TestParallelLookUpAndMkdirSameDir(t *testing.T) { // Create directory structure for testing. - testDir := createDirectoryStructureForParallelDiropsTest(t) - lookUpFunc := func(wg *sync.WaitGroup, dirPath string) (os.FileInfo, error) { + tds := createDirStructure(t) + defer deleteDirStructure(tds) + mkdirFunc := func(wg *sync.WaitGroup, dirPath string, err *error) { defer wg.Done() - fileInfo, err := os.Stat(dirPath) - return fileInfo, err - } - mkdirFunc := func(wg *sync.WaitGroup, dirPath string) error { - defer wg.Done() - err := os.Mkdir(dirPath, setup.DirPermission_0755) - return err + *err = os.Mkdir(dirPath, setup.DirPermission_0755) } + var statInfo os.FileInfo var lookUpErr, mkdirErr error - // Parallel lookup and mkdir of a new directory. - dirPath := path.Join(testDir, "newDir") - wg := sync.WaitGroup{} + dirPath := path.Join(tds.testDir, "newDir") + var wg sync.WaitGroup wg.Add(2) - go func() { - statInfo, lookUpErr = lookUpFunc(&wg, dirPath) - }() - go func() { - mkdirErr = mkdirFunc(&wg, dirPath) - }() + + go lookUpFileStat(&wg, dirPath, &statInfo, &lookUpErr) + go mkdirFunc(&wg, dirPath, &mkdirErr) wg.Wait() - assert.NoError(t, mkdirErr) // Assert either directory is created first or looked up first + assert.NoError(t, mkdirErr, "mkdirFunc should not fail") + if lookUpErr == nil { + assert.NotNil(t, statInfo, "statInfo should not be nil when lookUpErr is nil") assert.Contains(t, statInfo.Name(), "newDir") assert.True(t, statInfo.IsDir()) } else { - assert.True(t, os.IsNotExist(lookUpErr)) + assert.True(t, os.IsNotExist(lookUpErr), "lookUpErr should indicate directory does not exist") dirStatInfo, err := os.Stat(dirPath) - assert.NoError(t, err) - assert.True(t, dirStatInfo.IsDir()) + assert.NoError(t, err, "os.Stat should succeed after directory creation") + assert.True(t, dirStatInfo.IsDir(), "The created path should be a directory") } } diff --git a/tools/integration_tests/operations/read_test.go b/tools/integration_tests/operations/read_test.go index 9e22f64c39..cd923ff188 100644 --- a/tools/integration_tests/operations/read_test.go +++ b/tools/integration_tests/operations/read_test.go @@ -19,8 +19,8 @@ import ( "os" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) func TestReadAfterWrite(t *testing.T) { @@ -31,7 +31,7 @@ func TestReadAfterWrite(t *testing.T) { t.Errorf("Mkdir at %q: %v", testDir, err) return } - for i := 0; i < 10; i++ { + for range 10 { tmpFile, err := os.CreateTemp(tmpDir, tempFileName) if err != nil { t.Errorf("Create file at %q: %v", tmpDir, err) @@ -39,7 +39,7 @@ func TestReadAfterWrite(t *testing.T) { } // Closing file at the end - operations.CloseFile(tmpFile) + operations.CloseFileShouldNotThrowError(t, tmpFile) fileName := tmpFile.Name() diff --git a/tools/integration_tests/operations/rename_dir_test.go b/tools/integration_tests/operations/rename_dir_test.go new file mode 100644 index 0000000000..1e16f5603a --- /dev/null +++ b/tools/integration_tests/operations/rename_dir_test.go @@ -0,0 +1,62 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Provides integration tests for rename dir. +package operations_test + +import ( + "os" + "path" + "strings" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" +) + +func TestRenameDirToNonEmptyDestDirectory(t *testing.T) { + // Set up the test directory. + testDir := setup.SetupTestDirectory(DirForOperationTests) + // Create source directory. + srcDirPath := path.Join(testDir, "srcDir") + err := os.Mkdir(srcDirPath, 0700) + assert.NoError(t, err) + // Create destination directory and put a file in it. + destDirPath := path.Join(testDir, "destDir") + err = os.Mkdir(destDirPath, 0700) + assert.NoError(t, err) + destFilePath := path.Join(destDirPath, "file.txt") + operations.CreateFileWithContent(destFilePath, setup.FilePermission_0600, Content, t) + // Attempt to rename the source directory to the destination directory. + err = os.Rename(srcDirPath, destDirPath) + // Assert that an error occurred (because destination is not empty). + assert.Error(t, err) + assert.True(t, strings.Contains(err.Error(), "file exists") || strings.Contains(err.Error(), "directory not empty") || strings.Contains(err.Error(), "not empty"), "Error message should mention file exists or not empty, but got: %v", err) + + // Perform operations on source and destination to ensure they are healthy. + // Context: https://b.corp.google.com/issues/504921217 + _, err = os.Stat(srcDirPath) + assert.NoError(t, err) + _, err = os.Stat(destDirPath) + assert.NoError(t, err) + + // Delete both directories successfully + err = os.Remove(destFilePath) + assert.NoError(t, err) + err = os.Remove(destDirPath) + assert.NoError(t, err) + err = os.Remove(srcDirPath) + assert.NoError(t, err) +} diff --git a/tools/integration_tests/operations/rename_file_test.go b/tools/integration_tests/operations/rename_file_test.go index 32d09a3595..51c6fc8b0f 100644 --- a/tools/integration_tests/operations/rename_file_test.go +++ b/tools/integration_tests/operations/rename_file_test.go @@ -16,11 +16,15 @@ package operations_test import ( + "os" "path" + "strings" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestRenameFile(t *testing.T) { @@ -38,8 +42,51 @@ func TestRenameFile(t *testing.T) { err = operations.RenameFile(fileName, newFileName) if err != nil { - t.Errorf("Error in file copying: %v", err) + t.Errorf("Error in file renaming: %v", err) } // Check if the data in the file is the same after renaming. setup.CompareFileContents(t, newFileName, string(content)) } + +func TestRenameFileWithSrcFileDoesNoExist(t *testing.T) { + // Set up the test directory. + testDir := setup.SetupTestDirectory(DirForOperationTests) + // Define source and destination file names. + srcFilePath := path.Join(testDir, "move1.txt") // This file does not exist. + destFilePath := path.Join(testDir, "move2.txt") + + // Attempt to rename the non-existent file. + err := operations.RenameFile(srcFilePath, destFilePath) + + // Assert that an error occurred. + assert.Error(t, err) + assert.True(t, strings.Contains(err.Error(), "no such file or directory")) +} + +func TestRenameSymlinkToFile(t *testing.T) { + testDir := setup.SetupTestDirectory(DirForOperationTests) + targetName := "target.txt" + targetPath := path.Join(testDir, targetName) + err := os.WriteFile(targetPath, []byte("taco"), setup.FilePermission_0600) + require.NoError(t, err) + oldSymlinkPath := path.Join(testDir, "symlink_old") + err = os.Symlink(targetPath, oldSymlinkPath) + require.NoError(t, err) + newSymlinkPath := path.Join(testDir, "symlink_new") + + err = os.Rename(oldSymlinkPath, newSymlinkPath) + + require.NoError(t, err) + _, err = os.Lstat(oldSymlinkPath) + require.Error(t, err) + assert.True(t, os.IsNotExist(err)) + fi, err := os.Lstat(newSymlinkPath) + require.NoError(t, err) + assert.Equal(t, os.ModeSymlink, fi.Mode()&os.ModeType) + targetRead, err := os.Readlink(newSymlinkPath) + require.NoError(t, err) + assert.Equal(t, targetPath, targetRead) + content, err := operations.ReadFile(newSymlinkPath) + require.NoError(t, err) + assert.Equal(t, "taco", string(content)) +} diff --git a/tools/integration_tests/operations/stat_file_test.go b/tools/integration_tests/operations/stat_file_test.go new file mode 100644 index 0000000000..a706f97416 --- /dev/null +++ b/tools/integration_tests/operations/stat_file_test.go @@ -0,0 +1,34 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operations_test + +import ( + "os" + "syscall" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestStatWithTrailingNewline(t *testing.T) { + testDir := setup.SetupTestDirectory(DirForOperationTests) + + _, err := os.Stat(testDir + "/\n") + + require.Error(t, err) + assert.Equal(t, err.(*os.PathError).Err, syscall.ENOENT) +} diff --git a/tools/integration_tests/operations/write_test.go b/tools/integration_tests/operations/write_test.go index 84a9f39e16..f32411fb0d 100644 --- a/tools/integration_tests/operations/write_test.go +++ b/tools/integration_tests/operations/write_test.go @@ -25,12 +25,11 @@ import ( "time" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) const tempFileName = "tmpFile" @@ -69,7 +68,14 @@ func validateObjectAttributes(attr1, attr2 *storage.ObjectAttrs, t *testing.T) { const componentCount = 0 const sizeBeforeOperation = int64(len(tempFileContent)) const sizeAfterOperation = sizeBeforeOperation + int64(len(appendContent)) - const storageClass = "STANDARD" + storageClass := "STANDARD" + if attr1 == nil || attr2 == nil { + t.Fatalf("attr1 or attr2 is nil. attr1: %v, attr2: %v", attr1, attr2) + } + + if setup.IsZonalBucketRun() { + storageClass = "RAPID" + } if attr1.ContentType != contentType || attr2.ContentType != contentType { t.Errorf("Expected content type: %s, Got: %s, %s", contentType, attr1.ContentType, attr2.ContentType) @@ -99,13 +105,17 @@ func validateObjectAttributes(attr1, attr2 *storage.ObjectAttrs, t *testing.T) { t.Error("Expected CRC32 attributes to be non 0") } if attr1.MediaLink == "" || attr2.MediaLink == "" { - t.Errorf("Expected media link to be non empty") + if setup.IsZonalBucketRun() { + t.Logf("media link is empty, but it is a known limitation in RAPID/zonal buckets.") + } else { + t.Errorf("Expected media link to be non empty") + } } if attr1.StorageClass != storageClass || attr2.StorageClass != storageClass { - t.Errorf("Expected storage class ") + t.Errorf("Expected storage class to be %q, but found attr1.StorageClass = %q (bucketName = %q), attr2.StorageClass = %q (bucketName = %q)", storageClass, attr1.StorageClass, attr1.Bucket, attr2.StorageClass, attr2.Bucket) } - attr1MTime, _ := time.Parse(time.RFC3339Nano, attr1.Metadata[gcsx.MtimeMetadataKey]) - attr2MTime, _ := time.Parse(time.RFC3339Nano, attr2.Metadata[gcsx.MtimeMetadataKey]) + attr1MTime, _ := time.Parse(time.RFC3339Nano, attr1.Metadata[gcs.MtimeMetadataKey]) + attr2MTime, _ := time.Parse(time.RFC3339Nano, attr2.Metadata[gcs.MtimeMetadataKey]) if attr2MTime.Before(attr1MTime) { t.Errorf("Unexpected MTime received. After operation1: %v, After operation2: %v", attr1MTime, attr2MTime) } @@ -163,7 +173,7 @@ func TestWriteAtRandom(t *testing.T) { t.Errorf("WriteString-Random: %v", err) } // Closing file at the end - operations.CloseFile(f) + operations.CloseFileShouldNotThrowError(t, f) setup.CompareFileContents(t, fileName, "line 1\nline 5\n") // Validate that extended object attributes are non nil/ non-empty. @@ -214,10 +224,10 @@ func TestWriteAtFileOperationsDoesNotChangeObjectAttributes(t *testing.T) { // Over-write the file. fh, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC|syscall.O_DIRECT, operations.FilePermission_0600) if err != nil { - t.Errorf("Could not open file %s after creation.", fileName) + t.Fatalf("Could not open file %s after creation.", fileName) } operations.WriteAt(tempFileContent+appendContent, 0, fh, t) - operations.CloseFile(fh) + operations.CloseFileShouldNotThrowError(t, fh) attr2 := validateExtendedObjectAttributesNonEmpty(path.Join(DirForOperationTests, tempFileName), t) // Validate object attributes are as expected. diff --git a/tools/integration_tests/rapid_appends/appends_test.go b/tools/integration_tests/rapid_appends/appends_test.go new file mode 100644 index 0000000000..6652b3a773 --- /dev/null +++ b/tools/integration_tests/rapid_appends/appends_test.go @@ -0,0 +1,332 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rapid_appends + +import ( + "os" + "path" + "syscall" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +//////////////////////////////////////////////////////////////////////// +// Tests for the DualMountAppendsTestSuite +//////////////////////////////////////////////////////////////////////// + +func (t *DualMountAppendsTestSuite) TestAppendSessionInvalidatedByAnotherClientUponTakeover() { + const initialContent = "dummy content" + const appendContent = "appended content" + + t.createUnfinalizedObject() + defer t.deleteUnfinalizedObject() + + // Initiate an append session using the primary file handle. + appendFileHandle := operations.OpenFileInMode(t.T(), path.Join(t.primaryMount.testDirPath, t.fileName), fileOpenModeAppend|syscall.O_DIRECT) + n, err := appendFileHandle.WriteString(initialContent) + require.NoError(t.T(), err) + require.Equal(t.T(), len(initialContent), n) + + // Open a new file handle from the secondary mount to the same file. + newAppendFileHandle := operations.OpenFileInMode(t.T(), path.Join(t.secondaryMount.testDirPath, t.fileName), fileOpenModeAppend|syscall.O_DIRECT) + defer operations.CloseFileShouldNotThrowError(t.T(), newAppendFileHandle) + + // This append should succeed, confirming the takeover. + n, err = newAppendFileHandle.WriteString(appendContent) + require.NoError(t.T(), err) + require.Equal(t.T(), len(appendContent), n) + + // This should now fail, as its append session has been invalidated. + _, _ = appendFileHandle.WriteString(appendContent) + err = appendFileHandle.Sync() + operations.ValidateESTALEError(t.T(), err) + + // Syncing from the new handle must succeed. + err = newAppendFileHandle.Sync() + require.NoError(t.T(), err) + + // Read directly using storage client to validate the contents which has persisted in + // GCS after takeover from the secondary mount. + // Close the open append handle before issuing read on the file as Sync() triggered on + // ReadFile() due to BWH still being initialized, is expected to error out with stale NFS file handle. + operations.CloseFileShouldThrowError(t.T(), appendFileHandle) + expectedContent := t.fileContent + appendContent + content, err := client.ReadObjectFromGCS(testEnv.ctx, testEnv.storageClient, path.Join(testDirName, t.fileName)) + require.NoError(t.T(), err) + assert.Equal(t.T(), expectedContent, string(content)) +} + +//////////////////////////////////////////////////////////////////////// +// Tests for the SingleMountAppendsTestSuite +//////////////////////////////////////////////////////////////////////// + +func (t *SingleMountAppendsTestSuite) TestContentAppendedInNonAppendModeNotVisibleTillClose() { + // Initially create an unfinalized object. + t.createUnfinalizedObject() + defer t.deleteUnfinalizedObject() + + initialContent := t.fileContent + wh, err := os.OpenFile(path.Join(t.primaryMount.testDirPath, t.fileName), fileOpenModeRPlus|syscall.O_DIRECT, operations.FilePermission_0600) + require.NoError(t.T(), err) + + // Write sufficient data to the end of file. + data := setup.GenerateRandomString(contentSizeForBW * operations.OneMiB) + n, err := wh.WriteAt([]byte(data), int64(len(initialContent))) + require.NoError(t.T(), err) + require.Equal(t.T(), len(data), n) + + // Read from GCS to validate that appended content is not yet visible. + contentBeforeClose, err := client.ReadObjectFromGCS(testEnv.ctx, testEnv.storageClient, path.Join(testDirName, t.fileName)) + require.NoError(t.T(), err) + assert.Equal(t.T(), initialContent, string(contentBeforeClose)) + + // Close the file handle to persist the data. + err = wh.Close() + require.NoError(t.T(), err) + + // Validate that the appended content is now visible in GCS. + expectedContent := initialContent + data + contentAfterClose, err := client.ReadObjectFromGCS(testEnv.ctx, testEnv.storageClient, path.Join(testDirName, t.fileName)) + require.NoError(t.T(), err) + assert.Equal(t.T(), expectedContent, string(contentAfterClose)) +} + +func (t *SingleMountAppendsTestSuite) TestAppendsToFinalizedObjectNotVisibleUntilClose() { + const initialContent = "dummy content" + + t.fileName = fileNamePrefix + setup.GenerateRandomString(5) + // Create Finalized Object in the GCS bucket. + client.CreateFinalizedObjectInGCSTestDir( + testEnv.ctx, testEnv.storageClient, testDirName, t.fileName, initialContent, t.T()) + + // Append to the finalized object from the primary mount. + data := setup.GenerateRandomString(contentSizeForBW * operations.OneMiB) + filePath := path.Join(t.primaryMount.testDirPath, t.fileName) + fh, err := os.OpenFile(filePath, fileOpenModeAppend|syscall.O_DIRECT, operations.FilePermission_0600) + require.NoError(t.T(), err) + n, err := fh.Write([]byte(data)) + require.NoError(t.T(), err) + require.Equal(t.T(), len(data), n) + + // Read from GCS to validate appended content is not yet visible. + contentBeforeClose, err := client.ReadObjectFromGCS(testEnv.ctx, testEnv.storageClient, path.Join(testDirName, t.fileName)) + require.NoError(t.T(), err) + assert.Equal(t.T(), initialContent, string(contentBeforeClose)) + + // Close the file handle and verify appended content is now visible. + require.NoError(t.T(), fh.Close()) + expectedContent := initialContent + data + contentAfterClose, err := client.ReadObjectFromGCS(testEnv.ctx, testEnv.storageClient, path.Join(testDirName, t.fileName)) + require.NoError(t.T(), err) + assert.Equal(t.T(), expectedContent, string(contentAfterClose)) +} + +func (t *SingleMountAppendsTestSuite) TestAppendsVisibleInRealTimeWithConcurrentRPlusHandle() { + const initialContent = "dummy content" + + // Initially create an unfinalized object. + t.createUnfinalizedObject() + defer t.deleteUnfinalizedObject() + + primaryPath := path.Join(t.primaryMount.testDirPath, t.fileName) + // Open first handle in append mode. + appendFileHandle := operations.OpenFileInMode(t.T(), primaryPath, fileOpenModeAppend|syscall.O_DIRECT) + defer appendFileHandle.Close() + readHandle := operations.OpenFileInMode(t.T(), primaryPath, fileOpenModeRPlus|syscall.O_DIRECT) + defer readHandle.Close() + + // Write initial content with append handle to trigger buffered write workflow. + n, err := appendFileHandle.Write([]byte(initialContent)) + require.NoError(t.T(), err) + require.Equal(t.T(), len(initialContent), n) + + // Append additional content with the "r+" handle. + data := setup.GenerateRandomString(contentSizeForBW * blockSize) + appendOffset := int64(unfinalizedObjectSize + len(initialContent)) + n, err = readHandle.WriteAt([]byte(data), appendOffset) + require.NoError(t.T(), err) + require.Equal(t.T(), len(data), n) + + // Read from back-door to validate visibility on GCS. + // The first 1MiB block is guaranteed to be flushed due to implicit behavior. + // That block includes both the initial content (written via "a" file handle ) + // and some part of data written by the "r+" file handle. + dataInBlockOffset := blockSize - len(initialContent) + expectedContent := t.fileContent + initialContent + data[0:dataInBlockOffset] + contentRead, err := client.ReadObjectFromGCS(testEnv.ctx, testEnv.storageClient, path.Join(testDirName, t.fileName)) + require.NoError(t.T(), err) + require.GreaterOrEqual(t.T(), len(contentRead), len(expectedContent)) + assert.Equal(t.T(), expectedContent, string(contentRead[0:len(expectedContent)])) +} + +func (t *SingleMountAppendsTestSuite) TestRandomWritesVisibleAfterCloseWithConcurrentRPlusHandle() { + const initialContent = "dummy content" + t.createUnfinalizedObject() + defer t.deleteUnfinalizedObject() + + primaryPath := path.Join(t.primaryMount.testDirPath, t.fileName) + appendFileHandle := operations.OpenFileInMode(t.T(), primaryPath, fileOpenModeAppend|syscall.O_DIRECT) + defer appendFileHandle.Close() + readHandle := operations.OpenFileInMode(t.T(), primaryPath, fileOpenModeRPlus|syscall.O_DIRECT) + + n, err := appendFileHandle.Write([]byte(initialContent)) + require.NoError(t.T(), err) + require.Equal(t.T(), len(initialContent), n) + t.fileContent = t.fileContent + initialContent + + // Random write at an incorrect offset. + data := setup.GenerateRandomString(contentSizeForBW * blockSize) + n, err = readHandle.WriteAt([]byte(data), int64(len(t.fileContent))+1) + require.NoError(t.T(), err) + require.Equal(t.T(), len(data), n) + + // Validate content is not yet visible. + contentBeforeClose, err := client.ReadObjectFromGCS(testEnv.ctx, testEnv.storageClient, path.Join(testDirName, t.fileName)) + require.NoError(t.T(), err) + assert.Equal(t.T(), t.fileContent, string(contentBeforeClose)) + + // Close handle and validate final content (with null byte for the gap). + readHandle.Close() + expectedContent := t.fileContent + "\x00" + data + contentAfterClose, err := client.ReadObjectFromGCS(testEnv.ctx, testEnv.storageClient, path.Join(testDirName, t.fileName)) + require.NoError(t.T(), err) + assert.Equal(t.T(), expectedContent, string(contentAfterClose)) +} + +func (t *SingleMountAppendsTestSuite) TestFallbackHappensWhenNonAppendHandleDoesFirstWrite() { + // Initially create an unfinalized object. + t.createUnfinalizedObject() + defer t.deleteUnfinalizedObject() + + primaryPath := path.Join(t.primaryMount.testDirPath, t.fileName) + appendFileHandle := operations.OpenFileInMode(t.T(), primaryPath, fileOpenModeAppend|syscall.O_DIRECT) + defer appendFileHandle.Close() + readHandle := operations.OpenFileInMode(t.T(), primaryPath, fileOpenModeRPlus|syscall.O_DIRECT) + + // Append content using the "r+" handle first. + data := setup.GenerateRandomString(contentSizeForBW * blockSize) + n, err := readHandle.WriteAt([]byte(data), int64(len(t.fileContent))) + require.NoError(t.T(), err) + require.Equal(t.T(), len(data), n) + + // Validate content is not yet visible. + contentBeforeClose, err := client.ReadObjectFromGCS(testEnv.ctx, testEnv.storageClient, path.Join(testDirName, t.fileName)) + require.NoError(t.T(), err) + assert.Equal(t.T(), t.fileContent, string(contentBeforeClose)) + + // Close handle and validate final content. + readHandle.Close() + expectedContent := t.fileContent + data + contentAfterClose, err := client.ReadObjectFromGCS(testEnv.ctx, testEnv.storageClient, path.Join(testDirName, t.fileName)) + require.NoError(t.T(), err) + assert.Equal(t.T(), expectedContent, string(contentAfterClose)) +} + +func (t *SingleMountAppendsTestSuite) TestKernelShouldSeeUpdatedSizeOnAppends_ValidStatCache() { + const initialContent = "dummy content" + + t.createUnfinalizedObject() + defer t.deleteUnfinalizedObject() + filePath := path.Join(t.primaryMount.testDirPath, t.fileName) + + // Append to the object and close the file handle. + appendFileHandle := operations.OpenFileInMode(t.T(), filePath, fileOpenModeAppend|syscall.O_DIRECT) + n, err := appendFileHandle.Write([]byte(initialContent)) + require.NoError(t.T(), err) + require.Equal(t.T(), len(initialContent), n) + appendFileHandle.Close() + + // Since we don't wait for the cache to expire, the stat should reflect the new size immediately. + expectedFileSize := int64(unfinalizedObjectSize + len(initialContent)) + fileInfo, err := operations.StatFile(filePath) + require.NoError(t.T(), err) + assert.Equal(t.T(), expectedFileSize, (*fileInfo).Size()) +} + +func (t *SingleMountAppendsTestSuite) TestKernelShouldSeeUpdatedSizeOnAppends_ExpiredStatCache() { + const initialContent = "dummy content" + + t.createUnfinalizedObject() + defer t.deleteUnfinalizedObject() + filePath := path.Join(t.primaryMount.testDirPath, t.fileName) + + // Append to the object and close the file handle. + appendFileHandle := operations.OpenFileInMode(t.T(), filePath, fileOpenModeAppend|syscall.O_DIRECT) + n, err := appendFileHandle.Write([]byte(initialContent)) + require.NoError(t.T(), err) + require.Equal(t.T(), len(initialContent), n) + appendFileHandle.Close() + + // Expire stat cache. By default, stat cache ttl is 60 seconds. + time.Sleep(defaultMetadataCacheTTL) + + // The stat should now fetch the latest size from the source, reflecting the new size. + expectedFileSize := int64(unfinalizedObjectSize + len(initialContent)) + fileInfo, err := operations.StatFile(filePath) + require.NoError(t.T(), err) + assert.Equal(t.T(), expectedFileSize, (*fileInfo).Size()) +} + +func (t *SingleMountAppendsTestSuite) TestOpenAppendCloseReopenFromSingleMount() { + // Initially create an unfinalized object. + t.createUnfinalizedObject() + defer t.deleteUnfinalizedObject() + filePath := path.Join(t.primaryMount.testDirPath, t.fileName) + + for i := 0; i < 3; i++ { + // Open file in append mode. + f := operations.OpenFileInMode(t.T(), filePath, fileOpenModeAppend|syscall.O_DIRECT) + + // Append content. + appendContent := setup.GenerateRandomString(appendSize) + t.appendToFile(f, appendContent) + + // Close the handle. + require.NoError(t.T(), f.Close()) + + // Validate file size observed by kernel. + fi, err := os.Stat(filePath) + require.NoError(t.T(), err) + assert.Equal(t.T(), int64(len(t.fileContent)), fi.Size()) + + // Validate that writes went through. + content, err := client.ReadObjectFromGCS(testEnv.ctx, testEnv.storageClient, path.Join(testDirName, t.fileName)) + require.NoError(t.T(), err) + assert.Equal(t.T(), t.fileContent, string(content)) + } +} + +//////////////////////////////////////////////////////////////////////// +// Test Runner +//////////////////////////////////////////////////////////////////////// + +func TestSingleMountAppendsTestSuite(t *testing.T) { + RunTests(t, "TestSingleMountAppendsTestSuite", func(primaryFlags, secondaryFlags []string) suite.TestingSuite { + return &SingleMountAppendsTestSuite{BaseSuite{primaryFlags: primaryFlags, secondaryFlags: secondaryFlags}} + }) +} + +func TestDualMountAppendsTestSuite(t *testing.T) { + RunTests(t, "TestDualMountAppendsTestSuite", func(primaryFlags, secondaryFlags []string) suite.TestingSuite { + return &DualMountAppendsTestSuite{BaseSuite{primaryFlags: primaryFlags, secondaryFlags: secondaryFlags}} + }) +} diff --git a/tools/integration_tests/rapid_appends/reads_after_appends_test.go b/tools/integration_tests/rapid_appends/reads_after_appends_test.go new file mode 100644 index 0000000000..ee12148300 --- /dev/null +++ b/tools/integration_tests/rapid_appends/reads_after_appends_test.go @@ -0,0 +1,178 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rapid_appends + +import ( + "math/rand/v2" + "os" + "path" + "syscall" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +//////////////////////////////////////////////////////////////////////// +// Helpers +//////////////////////////////////////////////////////////////////////// + +// declare a function type for read and verify +type readAndVerifyFunc func(t *testing.T, filePath string, expectedContent []byte) + +func readSequentiallyAndVerify(t *testing.T, filePath string, expectedContent []byte) { + file, err := os.OpenFile(filePath, os.O_RDONLY, setup.FilePermission_0600) + require.NoError(t, err) + readContent, err := operations.ReadFileSequentially(file, 1024*1024) + + // For sequential reads, we expect the content to be exactly as expected. + require.NoErrorf(t, err, "failed to read file %q sequentially: %v", filePath, err) + require.Equal(t, expectedContent, readContent) +} + +func readRandomlyAndVerify(t *testing.T, filePath string, expectedContent []byte) { + file, err := os.OpenFile(filePath, os.O_RDONLY, setup.FilePermission_0600) + require.NoErrorf(t, err, "failed to open file %q: %v", filePath, err) + defer operations.CloseFileShouldNotThrowError(t, file) // This line is already correct. + if len(expectedContent) == 0 { + t.SkipNow() + } + fileInfo, err := file.Stat() + require.NoError(t, err) + fileSize := fileInfo.Size() + require.GreaterOrEqualf(t, fileSize, int64(len(expectedContent)), "file %q is too small to read %d bytes", filePath, len(expectedContent)) + + // Content to be read from [0, maxOffset) . + maxOffset := len(expectedContent) + // Limit number of reads if the content to read is too small. + numReads := min(maxOffset, 10) + for i := range numReads { + // Ensure offset <= maxOffset-1 . + offset := rand.IntN(maxOffset) + // Ensure (offset+readSize) <= maxOffset and readSize >= 1. + readSize := rand.IntN(maxOffset-offset) + 1 + buffer := make([]byte, readSize) + + n, err := file.ReadAt(buffer, int64(offset)) + + require.NoErrorf(t, err, "Random-read failed at iter#%d to read file %q at [%d, %d): %v", i, filePath, offset, offset+readSize, err) + require.Equalf(t, readSize, n, "failed to read %v bytes from %q at offset %v. Read bytes = %v.", readSize, filePath, offset, n) + require.Equalf(t, expectedContent[offset:offset+n], buffer[:n], "content mismatch in random read at iter#%d at offset [%d, %d): expected %q, got %q", i, offset, offset+readSize, expectedContent[offset:offset+n], buffer[:n]) + } +} + +//////////////////////////////////////////////////////////////////////// +// Tests for the SingleMountReadsTestSuite +//////////////////////////////////////////////////////////////////////// + +// runAppendAndReadTest contains the core test logic for the SingleMountReadsTestSuite. +func (t *SingleMountReadsTestSuite) runAppendAndReadTest(verifyFunc readAndVerifyFunc) { + t.createUnfinalizedObject() + defer t.deleteUnfinalizedObject() + + appendFileHandle := operations.OpenFileInMode(t.T(), path.Join(t.primaryMount.testDirPath, t.fileName), fileOpenModeAppend|syscall.O_DIRECT) + defer operations.CloseFileShouldNotThrowError(t.T(), appendFileHandle) + + readPath := path.Join(t.primaryMount.testDirPath, t.fileName) + for i := range numAppends { + // Wait for a minute for stat to return the correct file size, which is needed by appendToFile. + if i > 0 { + time.Sleep(operations.WaitDurationAfterFlushZB) + } + + t.appendToFile(appendFileHandle, setup.GenerateRandomString(appendSize)) + sizeAfterAppend := len(t.fileContent) + + // For same-mount appends/reads, file size is always current. + verifyFunc(t.T(), readPath, []byte(t.fileContent[:sizeAfterAppend])) + } +} + +func (t *SingleMountReadsTestSuite) TestSequentialRead() { + t.runAppendAndReadTest(readSequentiallyAndVerify) +} + +func (t *SingleMountReadsTestSuite) TestRandomRead() { + t.runAppendAndReadTest(readRandomlyAndVerify) +} + +//////////////////////////////////////////////////////////////////////// +// Tests for the DualMountReadsTestSuite +//////////////////////////////////////////////////////////////////////// + +// runAppendAndReadTest contains the core test logic for the DualMountReadsTestSuite. +func (t *DualMountReadsTestSuite) runAppendAndReadTest(verifyFunc readAndVerifyFunc) { + t.createUnfinalizedObject() + defer t.deleteUnfinalizedObject() + + appendFileHandle := operations.OpenFileInMode(t.T(), path.Join(t.getAppendPath(), t.fileName), fileOpenModeAppend|syscall.O_DIRECT) + defer operations.CloseFileShouldNotThrowError(t.T(), appendFileHandle) + + readPath := path.Join(t.primaryMount.testDirPath, t.fileName) + for i := range numAppends { + sizeBeforeAppend := len(t.fileContent) + t.appendToFile(appendFileHandle, setup.GenerateRandomString(appendSize)) + sizeAfterAppend := len(t.fileContent) + + // If metadata cache is enabled, gcsfuse reads up to the cached file size. + // The initial read (i=0) bypasses cache, seeing the latest file size. + if !t.isMetadataCacheEnabled() || (i == 0) { + verifyFunc(t.T(), readPath, []byte(t.fileContent[:sizeAfterAppend])) + } else { + // Read only up to the cached file size (before append). + verifyFunc(t.T(), readPath, []byte(t.fileContent[:sizeBeforeAppend])) + + // Wait for metadata cache to expire to fetch the latest size for the next read. + // Metadata update for appends in current iteration itself takes a minute, so the + // cached size will expire in ttl-60 secs from now, so wait accordingly. + time.Sleep(time.Duration(metadataCacheTTLSecs*time.Second - operations.WaitDurationAfterFlushZB)) + // Expect read up to the latest file size which is the size after the append. + verifyFunc(t.T(), readPath, []byte(t.fileContent[:sizeAfterAppend])) + } + } +} + +func (t *DualMountReadsTestSuite) TestSequentialRead() { + t.runAppendAndReadTest(readSequentiallyAndVerify) +} + +func (t *DualMountReadsTestSuite) TestRandomRead() { + t.runAppendAndReadTest(readRandomlyAndVerify) +} + +//////////////////////////////////////////////////////////////////////// +// Test Runner +//////////////////////////////////////////////////////////////////////// + +func TestSingleMountReadsTestSuite(t *testing.T) { + RunTests(t, "TestSingleMountReadsTestSuite", func(primaryFlags, secondaryFlags []string) suite.TestingSuite { + return &SingleMountReadsTestSuite{BaseSuite{primaryFlags: primaryFlags, secondaryFlags: secondaryFlags}} + }) +} + +func TestDualMountReadsTestSuiteWithMetadataCache(t *testing.T) { + RunTests(t, "TestDualMountReadsTestSuiteWithMetadataCache", func(primaryFlags, secondaryFlags []string) suite.TestingSuite { + return &DualMountReadsTestSuite{BaseSuite{primaryFlags: primaryFlags, secondaryFlags: secondaryFlags, metadataCacheEnabled: true}} + }) +} + +func TestDualMountReadsTestSuiteWithoutMetadataCache(t *testing.T) { + RunTests(t, "TestDualMountReadsTestSuiteWithoutMetadataCache", func(primaryFlags, secondaryFlags []string) suite.TestingSuite { + return &DualMountReadsTestSuite{BaseSuite{primaryFlags: primaryFlags, secondaryFlags: secondaryFlags, metadataCacheEnabled: false}} + }) +} diff --git a/tools/integration_tests/rapid_appends/setup_test.go b/tools/integration_tests/rapid_appends/setup_test.go new file mode 100644 index 0000000000..b064916169 --- /dev/null +++ b/tools/integration_tests/rapid_appends/setup_test.go @@ -0,0 +1,169 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rapid_appends + +import ( + "context" + "log" + "os" + "path" + "testing" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" +) + +const ( + testDirName = "RapidAppendsTest" + fileNamePrefix = "rapid-append-file-" + contentSizeForBW = 3 + blockSize = operations.OneMiB + numAppends = 2 + appendSize = 10 + unfinalizedObjectSize = 10 + metadataCacheTTLSecs = 70 + fileOpenModeRPlus = os.O_RDWR + fileOpenModeAppend = os.O_APPEND | os.O_WRONLY +) + +var ( + testEnv env +) + +type env struct { + storageClient *storage.Client + ctx context.Context + cfg *test_suite.TestConfig + bucketType string +} + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.RapidAppends) == 0 { + log.Println("No configuration found for rapid_appends tests in config. Using flags instead.") + // Populate the config manually. + cfg.RapidAppends = make([]test_suite.TestConfig, 1) + cfg.RapidAppends[0].TestBucket = setup.TestBucket() + cfg.RapidAppends[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.RapidAppends[0].LogFile = setup.LogFile() + cfg.RapidAppends[0].Configs = make([]test_suite.ConfigItem, 5) + + // 1. TestSingleMountAppendsTestSuite + cfg.RapidAppends[0].Configs[0].Flags = []string{"--write-block-size-mb=1"} + cfg.RapidAppends[0].Configs[0].Compatible = map[string]bool{"flat": false, "hns": false, "zonal": true} + cfg.RapidAppends[0].Configs[0].Run = "TestSingleMountAppendsTestSuite" + + // 2. TestDualMountAppendsTestSuite + cfg.RapidAppends[0].Configs[1].Flags = []string{"--write-block-size-mb=1"} + cfg.RapidAppends[0].Configs[1].SecondaryFlags = []string{"--write-block-size-mb=1"} + cfg.RapidAppends[0].Configs[1].Compatible = map[string]bool{"flat": false, "hns": false, "zonal": true} + cfg.RapidAppends[0].Configs[1].Run = "TestDualMountAppendsTestSuite" + + // 3. TestSingleMountReadsTestSuite + cfg.RapidAppends[0].Configs[2].Flags = []string{ + "--metadata-cache-ttl-secs=0", // NoCache + "--metadata-cache-ttl-secs=70", // MetadataCache + "--file-cache-max-size-mb=-1 --cache-dir=/gcsfuse-tmp/cache --metadata-cache-ttl-secs=0", // FileCache + "--metadata-cache-ttl-secs=70 --file-cache-max-size-mb=-1 --cache-dir=/gcsfuse-tmp/cache", // MetadataAndFileCache + "--metadata-cache-ttl-secs=0 --enable-kernel-reader=false", // NoCacheWithoutKernelReader + "--metadata-cache-ttl-secs=70 --enable-kernel-reader=false", // MetadataCacheWithMRDWrapperWithoutKernelReader + "--file-cache-max-size-mb=-1 --cache-dir=/gcsfuse-tmp/cache --metadata-cache-ttl-secs=0 --enable-kernel-reader=false", // FileCacheWithoutKernelReader + "--metadata-cache-ttl-secs=70 --file-cache-max-size-mb=-1 --cache-dir=/gcsfuse-tmp/cache --enable-kernel-reader=false", // MetadataAndFileCacheWithoutKernelReader + } + cfg.RapidAppends[0].Configs[2].Compatible = map[string]bool{"flat": false, "hns": false, "zonal": true} + cfg.RapidAppends[0].Configs[2].Run = "TestSingleMountReadsTestSuite" + + // 4. TestDualMountReadsTestSuiteWithMetadataCache + cfg.RapidAppends[0].Configs[3].Flags = []string{ + "--metadata-cache-ttl-secs=70", + "--metadata-cache-ttl-secs=70 --file-cache-max-size-mb=-1 --cache-dir=/gcsfuse-tmp/cache-primary", + "--metadata-cache-ttl-secs=70 --enable-kernel-reader=false", + "--metadata-cache-ttl-secs=70 --file-cache-max-size-mb=-1 --cache-dir=/gcsfuse-tmp/cache-primary --enable-kernel-reader=false", + } + cfg.RapidAppends[0].Configs[3].SecondaryFlags = []string{ + "--write-block-size-mb=1", + "--write-block-size-mb=1", + "--write-block-size-mb=1", + "--write-block-size-mb=1", + } + cfg.RapidAppends[0].Configs[3].Compatible = map[string]bool{"flat": false, "hns": false, "zonal": true} + cfg.RapidAppends[0].Configs[3].Run = "TestDualMountReadsTestSuiteWithMetadataCache" + + // 5. TestDualMountReadsTestSuiteWithoutMetadataCache + cfg.RapidAppends[0].Configs[4].Flags = []string{ + "--metadata-cache-ttl-secs=0", + "--file-cache-max-size-mb=-1 --cache-dir=/gcsfuse-tmp/cache-primary --metadata-cache-ttl-secs=0", + "--metadata-cache-ttl-secs=0 --enable-kernel-reader=false", + "--file-cache-max-size-mb=-1 --cache-dir=/gcsfuse-tmp/cache-primary --metadata-cache-ttl-secs=0 --enable-kernel-reader=false", + } + cfg.RapidAppends[0].Configs[4].SecondaryFlags = []string{ + "--write-block-size-mb=1", + "--write-block-size-mb=1", + "--write-block-size-mb=1", + "--write-block-size-mb=1", + } + cfg.RapidAppends[0].Configs[4].Compatible = map[string]bool{"flat": false, "hns": false, "zonal": true} + cfg.RapidAppends[0].Configs[4].Run = "TestDualMountReadsTestSuiteWithoutMetadataCache" + } + + testEnv.ctx = context.Background() + testEnv.cfg = &cfg.RapidAppends[0] + testEnv.bucketType = setup.TestEnvironment(testEnv.ctx, testEnv.cfg) + if !setup.IsZonalBucketRun() { + log.Fatalf("This test package is only compatible for zonal bucket runs") + } + + // 2. Create storage client before running tests. + var err error + testEnv.storageClient, err = client.CreateStorageClient(testEnv.ctx) + if err != nil { + log.Printf("Error creating storage client: %v\n", err) + os.Exit(1) + } + defer testEnv.storageClient.Close() + + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + // For GKE, we expect both directories to be mounted if it's a dual mount test. + // If using config, GKEMountedDirectorySecondary should be set. + testEnv.cfg.GCSFuseMountedDirectory = testEnv.cfg.GKEMountedDirectory + testEnv.cfg.GCSFuseMountedDirectorySecondary = testEnv.cfg.GKEMountedDirectorySecondary + os.Exit(m.Run()) + } + + // For GCE environment + setup.SetUpTestDirForTestBucket(testEnv.cfg) + // Override GKE specific paths with GCSFuse paths if running in GCE environment. + setup.OverrideFilePathsInFlagSet(testEnv.cfg, setup.TestDir()) + + // For dual mount, we create another directory. + secondaryDir, err := os.MkdirTemp(setup.TestDir(), "gcsfuse-secondary-mount") + if err != nil { + log.Fatalf("Failed to create secondary mount directory: %v", err) + } + testEnv.cfg.GCSFuseMountedDirectorySecondary = secondaryDir + + log.Println("Running static mounting tests for rapid appends...") + successCode := m.Run() + + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(setup.TestBucket(), testDirName)) + os.Exit(successCode) +} diff --git a/tools/integration_tests/rapid_appends/suites_test.go b/tools/integration_tests/rapid_appends/suites_test.go new file mode 100644 index 0000000000..4cb36695fd --- /dev/null +++ b/tools/integration_tests/rapid_appends/suites_test.go @@ -0,0 +1,212 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rapid_appends + +import ( + "log" + "os" + "path" + "testing" + "time" + + "strings" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +//////////////////////////////////////////////////////////////////////// +// Suite Definitions +//////////////////////////////////////////////////////////////////////// + +const ( + defaultMetadataCacheTTL = time.Minute +) + +// Struct to store the details of a mount point +type mountPoint struct { + rootDir string // Root directory of the test folder, which contains mnt and gcsfuse.log. + mntDir string // Directory where the GCS bucket is mounted. This is 'mnt' inside rootDir. + testDirPath string // Path to the 'RapidAppendsTest' directory inside mntDir. + logFilePath string // Path to the GCSFuse log file. This is gcsfuse.log inside rootDir. +} + +// BaseSuite provides the common structure and configuration-driven setup logic. +type BaseSuite struct { + suite.Suite + primaryFlags []string + secondaryFlags []string + primaryMount mountPoint + secondaryMount mountPoint + fileName string + fileContent string + metadataCacheEnabled bool +} + +// SingleMountReadsTestSuite groups all single-mount tests related to reading after appends. +type SingleMountReadsTestSuite struct{ BaseSuite } + +// DualMountReadsTestSuite groups all dual-mount tests related to reading after appends. +type DualMountReadsTestSuite struct{ BaseSuite } + +// SingleMountAppendsTestSuite groups general single-mount tests for append behavior. +type SingleMountAppendsTestSuite struct{ BaseSuite } + +// DualMountAppendsTestSuite groups general dual-mount tests for append behavior. +type DualMountAppendsTestSuite struct{ BaseSuite } + +//////////////////////////////////////////////////////////////////////// +// Common Suite Logic +//////////////////////////////////////////////////////////////////////// + +func (t *BaseSuite) SetupTest() { + if testEnv.cfg.GKEMountedDirectory != "" { + // GKE Mode: Already mounted + t.primaryMount.mntDir = testEnv.cfg.GKEMountedDirectory + t.primaryMount.testDirPath = path.Join(t.primaryMount.mntDir, testDirName) + t.primaryMount.logFilePath = testEnv.cfg.LogFile // Might be empty, but that's fine for GKE + + if len(t.secondaryFlags) > 0 { + t.secondaryMount.mntDir = testEnv.cfg.GKEMountedDirectorySecondary + t.secondaryMount.testDirPath = path.Join(t.secondaryMount.mntDir, testDirName) + } + } else { + // GCE Mode: Mount it + t.primaryMount.setupTestDir(testEnv.cfg.GCSFuseMountedDirectory, testEnv.cfg.LogFile) + t.mountGcsfuse(t.primaryMount, "primary", t.primaryFlags) + + if len(t.secondaryFlags) > 0 { + secondaryLog := path.Join(path.Dir(testEnv.cfg.LogFile), "gcsfuse_secondary.log") + t.secondaryMount.setupTestDir(testEnv.cfg.GCSFuseMountedDirectorySecondary, secondaryLog) + t.mountGcsfuse(t.secondaryMount, "secondary", t.secondaryFlags) + } + } +} + +func (t *BaseSuite) TearDownTest() { + if t.T().Failed() { + // Save logs for both mounts on failure to aid debugging. + testName := strings.ReplaceAll(t.T().Name(), "/", "_") + if t.primaryMount.logFilePath != "" { + setup.SaveLogFileAsArtifact(t.primaryMount.logFilePath, "gcsfuse-primary-log-"+testName) + } + if len(t.secondaryFlags) > 0 && t.secondaryMount.logFilePath != "" { + setup.SaveLogFileAsArtifact(t.secondaryMount.logFilePath, "gcsfuse-secondary-log-"+testName) + } + } + + if testEnv.cfg.GKEMountedDirectory != "" { + // GKE Mode: Just cleanup files + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(setup.TestBucket(), testDirName)) + } else { + // GCE Mode: Unmount and clean up + t.unmountAndCleanupMount(t.primaryMount, "primary") + if len(t.secondaryFlags) > 0 { + t.unmountAndCleanupMount(t.secondaryMount, "secondary") + } + } +} + +//////////////////////////////////////////////////////////////////////// +// Helpers +//////////////////////////////////////////////////////////////////////// + +func (mnt *mountPoint) setupTestDir(mountDir, logFile string) { + mnt.rootDir = setup.TestDir() + mnt.mntDir = mountDir + mnt.logFilePath = logFile + mnt.testDirPath = path.Join(mountDir, testDirName) +} + +func (t *BaseSuite) mountGcsfuse(mnt mountPoint, mountType string, flags []string) { + setup.SetMntDir(mnt.mntDir) + setup.SetLogFile(mnt.logFilePath) + err := static_mounting.MountGcsfuseWithStaticMounting(flags) + require.NoError(t.T(), err, "Unable to mount %s: %v", mountType, err) + mnt.testDirPath = setup.SetupTestDirectory(testDirName) + log.Printf("Running tests with %s mount flags %v", mountType, flags) +} + +func (t *BaseSuite) unmountAndCleanupMount(m mountPoint, name string) { + setup.UnmountGCSFuse(m.mntDir) + // Cleaning up the intermediate generated test files. + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(setup.TestBucket(), testDirName)) +} + +func (t *BaseSuite) createUnfinalizedObject() { + t.fileName = fileNamePrefix + setup.GenerateRandomString(5) + // Create unfinalized object. + t.fileContent = setup.GenerateRandomString(unfinalizedObjectSize) + client.CreateUnfinalizedObject(testEnv.ctx, t.T(), testEnv.storageClient, path.Join(testDirName, t.fileName), t.fileContent) +} + +func (t *BaseSuite) deleteUnfinalizedObject() { + if t.fileName != "" { + err := os.Remove(path.Join(t.primaryMount.testDirPath, t.fileName)) + require.NoError(t.T(), err) + t.fileName = "" + } +} + +func (t *BaseSuite) getAppendPath() string { + if len(t.secondaryFlags) > 0 { + return t.secondaryMount.testDirPath + } + return t.primaryMount.testDirPath +} + +func (t *BaseSuite) appendToFile(file *os.File, appendContent string) { + t.T().Helper() + n, err := file.WriteString(appendContent) + require.NoError(t.T(), err) + require.Equal(t.T(), len(appendContent), n) + t.fileContent += appendContent + if len(t.secondaryFlags) > 0 { + operations.SyncFile(file, t.T()) + } +} + +func getNewEmptyCacheDir(rootDir string) string { + cacheDirPath, err := os.MkdirTemp(rootDir, "cache_dir_*") + if err != nil { + log.Fatalf("Failed to create temporary directory for cache dir for tests: %v", err) + } + return cacheDirPath +} + +func (t *BaseSuite) isMetadataCacheEnabled() bool { + return t.metadataCacheEnabled +} + +func RunTests(t *testing.T, runName string, factory func(primaryFlags, secondaryFlags []string) suite.TestingSuite) { + for _, cfg := range testEnv.cfg.Configs { + if cfg.Run == runName { + for i, flagStr := range cfg.Flags { + flagStr = strings.ReplaceAll(flagStr, ",", " ") + primaryFlags := strings.Fields(flagStr) + var secondaryFlags []string + if len(cfg.SecondaryFlags) > i { + secFlagStr := strings.ReplaceAll(cfg.SecondaryFlags[i], ",", " ") + secondaryFlags = strings.Fields(secFlagStr) + } + suite.Run(t, factory(primaryFlags, secondaryFlags)) + } + } + } +} diff --git a/tools/integration_tests/read_cache/cache_file_for_exclude_regex_test.go b/tools/integration_tests/read_cache/cache_file_for_exclude_regex_test.go new file mode 100644 index 0000000000..98a2e7be95 --- /dev/null +++ b/tools/integration_tests/read_cache/cache_file_for_exclude_regex_test.go @@ -0,0 +1,110 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package read_cache + +import ( + "context" + "fmt" + "log" + "os" + "testing" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +type cacheFileForExcludeRegexTest struct { + flags []string + storageClient *storage.Client + ctx context.Context + baseTestName string + suite.Suite +} + +func (s *cacheFileForExcludeRegexTest) SetupSuite() { + setupLogFileAndCacheDir(s.baseTestName) + mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient) +} + +func (s *cacheFileForExcludeRegexTest) SetupTest() { + //Truncate log file created. + err := os.Truncate(testEnv.cfg.LogFile, 0) + require.NoError(s.T(), err) + // Clean up the cache directory path as gcsfuse don't clean up on mounting. + operations.RemoveDir(testEnv.cacheDirPath) + testEnv.testDirPath = client.SetupUniqueTestDirectory(s.ctx, s.storageClient, testDirPrefix) +} + +func (s *cacheFileForExcludeRegexTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +func (s *cacheFileForExcludeRegexTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +//////////////////////////////////////////////////////////////////////// +// Test scenarios +//////////////////////////////////////////////////////////////////////// + +func (s *cacheFileForExcludeRegexTest) TestReadsForExcludedFile() { + testFileName := setupFileInTestDir(s.ctx, s.storageClient, fileSizeForRangeRead, s.T()) + + expectedOutcome1 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, zeroOffset, s.T()) + expectedOutcome2 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, offset1000, s.T()) + + structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + validate(expectedOutcome1, structuredReadLogs[0], true, false, 1, s.T()) + validate(expectedOutcome2, structuredReadLogs[1], false, false, 1, s.T()) + validateFileIsNotCached(testFileName, s.T()) +} + +//////////////////////////////////////////////////////////////////////// +// Test Function (Runs once before all tests) +//////////////////////////////////////////////////////////////////////// + +func TestCacheFileForExcludeRegexTest(t *testing.T) { + ts := &cacheFileForExcludeRegexTest{ + ctx: context.Background(), + storageClient: testEnv.storageClient, + baseTestName: t.Name(), + } + // Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + if setup.OnlyDirMounted() != "" { + flagsSet = append(flagsSet, + []string{fmt.Sprintf("--file-cache-exclude-regex=^%s/%s/", setup.TestBucket(), onlyDirMounted), "--file-cache-max-size-mb=50", "--file-cache-cache-file-for-range-read=true", "--file-cache-enable-parallel-downloads=false", "--file-cache-enable-o-direct=false", fmt.Sprintf("--cache-dir=%s/gcsfuse-tmp/TestCacheFileForExcludeRegexTest", setup.TestDir()), fmt.Sprintf("--log-file=%s/gcsfuse-tmp/TestCacheFileForExcludeRegexTest.log", setup.TestDir()), "--log-severity=TRACE", "--enable-kernel-reader=false"}, + []string{fmt.Sprintf("--file-cache-exclude-regex=^%s/%s/", setup.TestBucket(), onlyDirMounted), "--file-cache-max-size-mb=50", "--file-cache-cache-file-for-range-read=true", "--file-cache-enable-parallel-downloads=false", "--file-cache-enable-o-direct=false", fmt.Sprintf("--cache-dir=%s/gcsfuse-tmp/TestCacheFileForExcludeRegexTest", setup.TestDir()), fmt.Sprintf("--log-file=%s/gcsfuse-tmp/TestCacheFileForExcludeRegexTest.log", setup.TestDir()), "--log-severity=TRACE", "--client-protocol=grpc", "--enable-kernel-reader=false"}, + ) + } + for _, ts.flags = range flagsSet { + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/read_cache/cache_file_for_include_regex_test.go b/tools/integration_tests/read_cache/cache_file_for_include_regex_test.go new file mode 100644 index 0000000000..17e13ade62 --- /dev/null +++ b/tools/integration_tests/read_cache/cache_file_for_include_regex_test.go @@ -0,0 +1,140 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package read_cache + +import ( + "context" + "log" + "os" + "path" + "testing" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +type cacheFileForIncludeRegexTest struct { + flags []string + storageClient *storage.Client + ctx context.Context + baseTestName string + suite.Suite +} + +func (s *cacheFileForIncludeRegexTest) SetupSuite() { + setupLogFileAndCacheDir(s.baseTestName) + mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient) +} + +func (s *cacheFileForIncludeRegexTest) SetupTest() { + //Truncate log file created. + err := os.Truncate(testEnv.cfg.LogFile, 0) + require.NoError(s.T(), err) + // Clean up the cache directory path as gcsfuse don't clean up on mounting. + operations.RemoveDir(testEnv.cacheDirPath) + testEnv.testDirPath = client.SetupUniqueTestDirectory(s.ctx, s.storageClient, testDirPrefix) +} + +func (s *cacheFileForIncludeRegexTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +func (s *cacheFileForIncludeRegexTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +//////////////////////////////////////////////////////////////////////// +// Test scenarios +//////////////////////////////////////////////////////////////////////// + +func (s *cacheFileForIncludeRegexTest) TestCacheFileForIncludeRegexForIncludedFile() { + testFileName := setupFileInTestDir(s.ctx, s.storageClient, fileSize, s.T()) + + // Read the file and validate that it is cached. + expectedOutcome1 := readFileAndGetExpectedOutcome(testEnv.testDirPath, testFileName, true, 0, s.T()) + expectedOutcome2 := readFileAndGetExpectedOutcome(testEnv.testDirPath, testFileName, true, 0, s.T()) + + structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + validate(expectedOutcome1, structuredReadLogs[0], true, false, chunksRead, s.T()) + validate(expectedOutcome2, structuredReadLogs[1], true, true, chunksRead, s.T()) + validateFileIsCached(testFileName, s.T()) +} + +func (s *cacheFileForIncludeRegexTest) TestCacheFileForIncludeRegexForNonIncludedFile() { + testFileName := "non-matching-regex" + setup.GenerateRandomString(testFileNameSuffixLength) + client.SetupFileInTestDirectory(s.ctx, s.storageClient, path.Base(testEnv.testDirPath), testFileName, fileSize, s.T()) + + // Read the file and validate that it is not cached. + expectedOutcome1 := readFileAndGetExpectedOutcome(testEnv.testDirPath, testFileName, true, 0, s.T()) + expectedOutcome2 := readFileAndGetExpectedOutcome(testEnv.testDirPath, testFileName, true, 0, s.T()) + + structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + validate(expectedOutcome1, structuredReadLogs[0], true, false, chunksRead, s.T()) + validate(expectedOutcome2, structuredReadLogs[1], true, false, chunksRead, s.T()) + validateFileIsNotCached(testFileName, s.T()) +} + +func (s *cacheFileForIncludeRegexTest) TestCacheFileForIncludeRegexForIncludedAndExcludeNoOverlap() { + includedFileName := setupFileInTestDir(s.ctx, s.storageClient, fileSize, s.T()) + excludedFileName := "non-matching-regex" + setup.GenerateRandomString(testFileNameSuffixLength) + client.SetupFileInTestDirectory(s.ctx, s.storageClient, path.Base(testEnv.testDirPath), excludedFileName, fileSize, s.T()) + + // Read the included file and validate that it is cached. + expectedOutcome1 := readFileAndGetExpectedOutcome(testEnv.testDirPath, includedFileName, true, 0, s.T()) + expectedOutcome2 := readFileAndGetExpectedOutcome(testEnv.testDirPath, includedFileName, true, 0, s.T()) + // Read the excluded file and validate that it is not cached. + expectedOutcome3 := readFileAndGetExpectedOutcome(testEnv.testDirPath, excludedFileName, true, 0, s.T()) + expectedOutcome4 := readFileAndGetExpectedOutcome(testEnv.testDirPath, excludedFileName, true, 0, s.T()) + + structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + validate(expectedOutcome1, structuredReadLogs[0], true, false, chunksRead, s.T()) + validate(expectedOutcome2, structuredReadLogs[1], true, true, chunksRead, s.T()) + validateFileIsCached(includedFileName, s.T()) + validate(expectedOutcome3, structuredReadLogs[2], true, false, chunksRead, s.T()) + validate(expectedOutcome4, structuredReadLogs[3], true, false, chunksRead, s.T()) + validateFileIsNotCached(excludedFileName, s.T()) +} + +//////////////////////////////////////////////////////////////////////// +// Test Function (Runs once before all tests) +//////////////////////////////////////////////////////////////////////// + +func TestCacheFileForIncludeRegexTest(t *testing.T) { + ts := &cacheFileForIncludeRegexTest{ + ctx: context.Background(), + storageClient: testEnv.storageClient, + baseTestName: t.Name(), + } + // Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/read_cache/cache_file_for_range_read_false_test.go b/tools/integration_tests/read_cache/cache_file_for_range_read_false_test.go index a003a7d5b1..b4e4dbd7b1 100644 --- a/tools/integration_tests/read_cache/cache_file_for_range_read_false_test.go +++ b/tools/integration_tests/read_cache/cache_file_for_range_read_false_test.go @@ -4,31 +4,31 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package read_cache import ( "context" "log" + "os" "path" - "sync" "testing" + "time" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/log_parser/json_parser/read_logs" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/test_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" ) //////////////////////////////////////////////////////////////////////// @@ -40,191 +40,181 @@ type cacheFileForRangeReadFalseTest struct { storageClient *storage.Client ctx context.Context isParallelDownloadsEnabled bool + isCacheOnRAM bool + baseTestName string + suite.Suite +} + +func (s *cacheFileForRangeReadFalseTest) SetupSuite() { + setupLogFileAndCacheDir(s.baseTestName) + if s.isCacheOnRAM { + testEnv.cacheDirPath = "/dev/shm/" + s.baseTestName + } + mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient) } -func (s *cacheFileForRangeReadFalseTest) Setup(t *testing.T) { - setupForMountedDirectoryTests() +func (s *cacheFileForRangeReadFalseTest) SetupTest() { + //Truncate log file created. + err := os.Truncate(testEnv.cfg.LogFile, 0) + require.NoError(s.T(), err) // Clean up the cache directory path as gcsfuse don't clean up on mounting. - operations.RemoveDir(cacheDirPath) - mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient, testDirName) + operations.RemoveDir(testEnv.cacheDirPath) + testEnv.testDirPath = client.SetupUniqueTestDirectory(s.ctx, s.storageClient, testDirPrefix) +} + +func (s *cacheFileForRangeReadFalseTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) } -func (s *cacheFileForRangeReadFalseTest) Teardown(t *testing.T) { - setup.UnmountGCSFuseAndDeleteLogFile(rootDir) +func (s *cacheFileForRangeReadFalseTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) } //////////////////////////////////////////////////////////////////////// // Helpers //////////////////////////////////////////////////////////////////////// -func readFileAsync(t *testing.T, wg *sync.WaitGroup, testFileName string, expectedOutcome **Expected) { - go func() { - defer wg.Done() - *expectedOutcome = readFileAndGetExpectedOutcome(testDirPath, testFileName, true, zeroOffset, t) - }() +func readFileBetweenOffset(t *testing.T, file *os.File, startOffset, endOffSet int64) *Expected { + t.Helper() + expected := &Expected{ + StartTimeStampSeconds: time.Now().Unix(), + BucketName: setup.TestBucket(), + ObjectName: path.Join(path.Base(testEnv.testDirPath), path.Base(file.Name())), + } + if setup.DynamicBucketMounted() != "" { + expected.BucketName = setup.DynamicBucketMounted() + } + + expected.content = operations.ReadFileBetweenOffset(t, file, startOffset, endOffSet, chunkSizeToRead) + expected.EndTimeStampSeconds = time.Now().Unix() + return expected } //////////////////////////////////////////////////////////////////////// // Test scenarios //////////////////////////////////////////////////////////////////////// -func (s *cacheFileForRangeReadFalseTest) TestRangeReadsWithCacheMiss(t *testing.T) { - testFileName := setupFileInTestDir(s.ctx, s.storageClient, testDirName, fileSizeForRangeRead, t) +func (s *cacheFileForRangeReadFalseTest) TestRangeReadsWithCacheMiss() { + testFileName := setupFileInTestDir(s.ctx, s.storageClient, fileSizeForRangeRead, s.T()) // Do a random read on file and validate from gcs. - expectedOutcome1 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, offset5000, t) + expectedOutcome1 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, offset5000, s.T()) // Read file again from offset 1000 and validate from gcs. - expectedOutcome2 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, offset1000, t) + expectedOutcome2 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, offset1000, s.T()) - structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t) - validate(expectedOutcome1, structuredReadLogs[0], false, false, 1, t) - validate(expectedOutcome2, structuredReadLogs[1], false, false, 1, t) - validateFileIsNotCached(testFileName, t) + structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + validate(expectedOutcome1, structuredReadLogs[0], false, false, 1, s.T()) + validate(expectedOutcome2, structuredReadLogs[1], false, false, 1, s.T()) + validateFileIsNotCached(testFileName, s.T()) } -func (s *cacheFileForRangeReadFalseTest) TestConcurrentReads_ReadIsTreatedNonSequentialAfterFileIsRemovedFromCache(t *testing.T) { +func (s *cacheFileForRangeReadFalseTest) TestReadIsTreatedNonSequentialAfterFileIsRemovedFromCache() { var testFileNames [2]string - var expectedOutcome [2]*Expected - testFileNames[0] = setupFileInTestDir(s.ctx, s.storageClient, testDirName, fileSizeSameAsCacheCapacity, t) - testFileNames[1] = setupFileInTestDir(s.ctx, s.storageClient, testDirName, fileSizeSameAsCacheCapacity, t) + var expectedOutcome [4]*Expected + testFileNames[0] = setupFileInTestDir(s.ctx, s.storageClient, fileSizeSameAsCacheCapacity, s.T()) + testFileNames[1] = setupFileInTestDir(s.ctx, s.storageClient, fileSizeSameAsCacheCapacity, s.T()) randomReadChunkCount := fileSizeSameAsCacheCapacity / chunkSizeToRead - - var wg sync.WaitGroup - for i := 0; i < 2; i++ { - wg.Add(1) - readFileAsync(t, &wg, testFileNames[i], &expectedOutcome[i]) - } - wg.Wait() - - structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t) - require.Equal(t, 2, len(structuredReadLogs)) - // Goroutine execution order isn't guaranteed. - // If the object name in expected outcome doesn't align with the logs, swap - // the expected outcome objects and file names at positions 0 and 1. - if expectedOutcome[0].ObjectName != structuredReadLogs[0].ObjectName { - expectedOutcome[0], expectedOutcome[1] = expectedOutcome[1], expectedOutcome[0] - testFileNames[0], testFileNames[1] = testFileNames[1], testFileNames[0] - } - validate(expectedOutcome[0], structuredReadLogs[0], true, false, randomReadChunkCount, t) - validate(expectedOutcome[1], structuredReadLogs[1], true, false, randomReadChunkCount, t) - // Validate last chunk was considered non-sequential and cache hit false for first read. - assert.False(t, structuredReadLogs[0].Chunks[randomReadChunkCount-1].IsSequential) - assert.False(t, structuredReadLogs[0].Chunks[randomReadChunkCount-1].CacheHit) - // Validate last chunk was considered sequential and cache hit true for second read. - assert.True(t, structuredReadLogs[1].Chunks[randomReadChunkCount-1].IsSequential) + readTillChunk := randomReadChunkCount / 2 + fh1 := operations.OpenFile(path.Join(testEnv.testDirPath, testFileNames[0]), s.T()) + defer operations.CloseFileShouldNotThrowError(s.T(), fh1) + fh2 := operations.OpenFile(path.Join(testEnv.testDirPath, testFileNames[1]), s.T()) + defer operations.CloseFileShouldNotThrowError(s.T(), fh2) + + // Use file handle 1 to read file 1 partially. + expectedOutcome[0] = readFileBetweenOffset(s.T(), fh1, 0, int64(readTillChunk*chunkSizeToRead)) + // Use file handle 2 to read file 2 partially. This will evict file 1 from + // cache due to cache capacity constraints. + expectedOutcome[1] = readFileBetweenOffset(s.T(), fh2, 0, int64(readTillChunk*chunkSizeToRead)) + // Read remaining file 1. File 2 remains cached. Cache eviction happens on + // cache handler creation, which is tied to the file handle. Since the handle + // isn't recreated, eviction doesn't occur. + expectedOutcome[2] = readFileBetweenOffset(s.T(), fh1, int64(readTillChunk*chunkSizeToRead)+1, fileSizeSameAsCacheCapacity) + // Read remaining file 2. + expectedOutcome[3] = readFileBetweenOffset(s.T(), fh2, int64(readTillChunk*chunkSizeToRead)+1, fileSizeSameAsCacheCapacity) + + // Merge the expected outcomes. + expectedOutcome[0].EndTimeStampSeconds = expectedOutcome[2].EndTimeStampSeconds + expectedOutcome[0].content = expectedOutcome[0].content + expectedOutcome[2].content + expectedOutcome[1].EndTimeStampSeconds = expectedOutcome[3].EndTimeStampSeconds + expectedOutcome[1].content = expectedOutcome[1].content + expectedOutcome[3].content + // Parse the logs and validate with expected outcome. + structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + require.Equal(s.T(), 2, len(structuredReadLogs)) + validate(expectedOutcome[0], structuredReadLogs[0], true, false, randomReadChunkCount, s.T()) + validate(expectedOutcome[1], structuredReadLogs[1], true, false, randomReadChunkCount, s.T()) + // Validate after cache eviction, read was considered non-sequential and cache + // hit false for first file. + // Checking for the last chunk, not readTillChunk+1, due to potential kernel + // over-reads on some architectures. + assert.False(s.T(), structuredReadLogs[0].Chunks[randomReadChunkCount-1].IsSequential) + assert.False(s.T(), structuredReadLogs[0].Chunks[randomReadChunkCount-1].CacheHit) + // Validate for 2nd file read was considered sequential because of no cache eviction. + assert.True(s.T(), structuredReadLogs[1].Chunks[randomReadChunkCount-1].IsSequential) if !s.isParallelDownloadsEnabled { // When parallel downloads are enabled, we can't concretely say that the read will be cache Hit. - assert.True(t, structuredReadLogs[1].Chunks[randomReadChunkCount-1].CacheHit) + assert.True(s.T(), structuredReadLogs[1].Chunks[randomReadChunkCount-1].CacheHit) } - validateFileIsNotCached(testFileNames[0], t) - validateFileInCacheDirectory(testFileNames[1], fileSizeSameAsCacheCapacity, s.ctx, s.storageClient, t) + validateFileIsNotCached(testFileNames[0], s.T()) + validateFileInCacheDirectory(testFileNames[1], fileSizeSameAsCacheCapacity, s.ctx, s.storageClient, s.T()) } //////////////////////////////////////////////////////////////////////// // Test Function (Runs once before all tests) //////////////////////////////////////////////////////////////////////// -func TestCacheFileForRangeReadFalseTest(t *testing.T) { - ts := &cacheFileForRangeReadFalseTest{ctx: context.Background()} - // Create storage client before running tests. - closeStorageClient := client.CreateStorageClientWithCancel(&ts.ctx, &ts.storageClient) - defer func() { - err := closeStorageClient() - if err != nil { - t.Errorf("closeStorageClient failed: %v", err) - } - }() - - // Run tests for mounted directory if the flag is set. - if setup.AreBothMountedDirectoryAndTestBucketFlagsSet() { - test_setup.RunTests(t, ts) +func (s *cacheFileForRangeReadFalseTest) runTests(t *testing.T) { + t.Helper() + // Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, s) return } - // Run with cache directory pointing to RAM based dir - ramCacheDir := path.Join("/dev/shm", cacheDirName) - - // Run tests with parallel downloads disabled. - flagsSet := []gcsfuseTestFlags{ - { - cliFlags: []string{"--implicit-dirs"}, - cacheSize: cacheCapacityForRangeReadTestInMiB, - cacheFileForRangeRead: false, - fileName: configFileName, - enableParallelDownloads: false, - enableODirect: false, - cacheDirPath: getDefaultCacheDirPathForTests(), - }, - { - cliFlags: nil, - cacheSize: cacheCapacityForRangeReadTestInMiB, - cacheFileForRangeRead: false, - fileName: configFileName, - enableParallelDownloads: false, - enableODirect: false, - cacheDirPath: ramCacheDir, - }, + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, s.flags = range flagsSet { + log.Printf("Running tests with flags: %s", s.flags) + suite.Run(t, s) } - flagsSet = appendClientProtocolConfigToFlagSet(flagsSet) - for _, flags := range flagsSet { - configFilePath := createConfigFile(&flags) - ts.flags = []string{"--config-file=" + configFilePath} - if flags.cliFlags != nil { - ts.flags = append(ts.flags, flags.cliFlags...) - } - log.Printf("Running tests with flags: %s", ts.flags) - test_setup.RunTests(t, ts) +} + +func TestCacheFileForRangeReadFalseTest(t *testing.T) { + ts := &cacheFileForRangeReadFalseTest{ + ctx: context.Background(), + storageClient: testEnv.storageClient, + baseTestName: t.Name(), } + ts.runTests(t) +} - // Run tests with parallel downloads enabled. - flagsSet = []gcsfuseTestFlags{ - { - cliFlags: nil, - cacheSize: cacheCapacityForRangeReadTestInMiB, - cacheFileForRangeRead: false, - fileName: configFileNameForParallelDownloadTests, - enableParallelDownloads: true, - enableODirect: false, - cacheDirPath: getDefaultCacheDirPathForTests(), - }, - { - cliFlags: nil, - cacheSize: cacheCapacityForRangeReadTestInMiB, - cacheFileForRangeRead: false, - fileName: configFileNameForParallelDownloadTests, - enableParallelDownloads: true, - enableODirect: false, - cacheDirPath: ramCacheDir, - }, - { - cliFlags: nil, - cacheSize: cacheCapacityForRangeReadTestInMiB, - cacheFileForRangeRead: false, - fileName: configFileNameForParallelDownloadTests, - enableParallelDownloads: true, - enableODirect: true, - cacheDirPath: getDefaultCacheDirPathForTests(), - }, - { - cliFlags: nil, - cacheSize: cacheCapacityForRangeReadTestInMiB, - cacheFileForRangeRead: false, - fileName: configFileNameForParallelDownloadTests, - enableParallelDownloads: true, - enableODirect: true, - cacheDirPath: ramCacheDir, - }, +func TestCacheFileForRangeReadFalseWithRamCache(t *testing.T) { + ts := &cacheFileForRangeReadFalseTest{ + ctx: context.Background(), + storageClient: testEnv.storageClient, + baseTestName: t.Name(), + isCacheOnRAM: true, } - flagsSet = appendClientProtocolConfigToFlagSet(flagsSet) - for _, flags := range flagsSet { - configFilePath := createConfigFile(&flags) - ts.flags = []string{"--config-file=" + configFilePath} - if flags.cliFlags != nil { - ts.flags = append(ts.flags, flags.cliFlags...) - } - ts.isParallelDownloadsEnabled = true - log.Printf("Running tests with flags: %s", ts.flags) - test_setup.RunTests(t, ts) + ts.runTests(t) +} + +func TestCacheFileForRangeReadFalseWithParallelDownloads(t *testing.T) { + ts := &cacheFileForRangeReadFalseTest{ + ctx: context.Background(), + storageClient: testEnv.storageClient, + baseTestName: t.Name(), + isParallelDownloadsEnabled: true, + } + ts.runTests(t) +} + +func TestCacheFileForRangeReadFalseWithParallelDownloadsAndRamCache(t *testing.T) { + ts := &cacheFileForRangeReadFalseTest{ + ctx: context.Background(), + storageClient: testEnv.storageClient, + baseTestName: t.Name(), + isParallelDownloadsEnabled: true, + isCacheOnRAM: true, } + ts.runTests(t) } diff --git a/tools/integration_tests/read_cache/cache_file_for_range_read_true_test.go b/tools/integration_tests/read_cache/cache_file_for_range_read_true_test.go index 9f217d8eb3..95b561dd4c 100644 --- a/tools/integration_tests/read_cache/cache_file_for_range_read_true_test.go +++ b/tools/integration_tests/read_cache/cache_file_for_range_read_true_test.go @@ -4,29 +4,30 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package read_cache import ( "context" + "fmt" "log" - "path" + "os" "testing" - "time" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/log_parser/json_parser/read_logs" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/test_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" ) //////////////////////////////////////////////////////////////////////// @@ -37,132 +38,113 @@ type cacheFileForRangeReadTrueTest struct { flags []string storageClient *storage.Client ctx context.Context + baseTestName string + isCacheOnRAM bool + suite.Suite } -func (s *cacheFileForRangeReadTrueTest) Setup(t *testing.T) { - setupForMountedDirectoryTests() +func (s *cacheFileForRangeReadTrueTest) SetupSuite() { + setupLogFileAndCacheDir(s.baseTestName) + if s.isCacheOnRAM { + testEnv.cacheDirPath = "/dev/shm/" + s.baseTestName + } + mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient) +} + +func (s *cacheFileForRangeReadTrueTest) SetupTest() { + //Truncate log file created. + err := os.Truncate(testEnv.cfg.LogFile, 0) + require.NoError(s.T(), err) + s.T().Logf("GCSFuse Log File: %s", testEnv.cfg.LogFile) // Clean up the cache directory path as gcsfuse don't clean up on mounting. - operations.RemoveDir(cacheDirPath) - mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient, testDirName) + operations.RemoveDir(testEnv.cacheDirPath) + testEnv.testDirPath = client.SetupUniqueTestDirectory(s.ctx, s.storageClient, testDirPrefix) +} + +func (s *cacheFileForRangeReadTrueTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) } -func (s *cacheFileForRangeReadTrueTest) Teardown(t *testing.T) { - setup.UnmountGCSFuseAndDeleteLogFile(rootDir) +func (s *cacheFileForRangeReadTrueTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) } //////////////////////////////////////////////////////////////////////// // Test scenarios //////////////////////////////////////////////////////////////////////// -func (s *cacheFileForRangeReadTrueTest) TestRangeReadsWithCacheHit(t *testing.T) { - testFileName := setupFileInTestDir(s.ctx, s.storageClient, testDirName, fileSizeForRangeRead, t) - - // Do a random read on file and validate from gcs. - expectedOutcome1 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, offset5000, t) - // Wait for the cache to propagate the updates before proceeding to get cache hit. - time.Sleep(2 * time.Second) - // Read file again from zeroOffset 1000 and validate from gcs. - expectedOutcome2 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, offset1000, t) - - structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t) - validate(expectedOutcome1, structuredReadLogs[0], false, false, 1, t) - validate(expectedOutcome2, structuredReadLogs[1], false, true, 1, t) +func (s *cacheFileForRangeReadTrueTest) TestRangeReadsWithCacheHit() { + testFileName := setupFileInTestDir(s.ctx, s.storageClient, fileSizeForRangeRead, s.T()) + + // Do a first random read on file and validate from gcs. + firstReadOutcome := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, offset5000, s.T()) + // Validate a single read log has cache hit 'false' recorded. + structuredReadFalseCacheHit := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + assert.Len(s.T(), structuredReadFalseCacheHit, 1, "Should have exactly 1 false cache hit read record in logs") + validate(firstReadOutcome, structuredReadFalseCacheHit[0], + /*isSeq=*/ false, + /*cacheHit=*/ false, + /*chunkCount=*/ 1, + s.T()) + + // RetryUntil we have exactly 1 Download Job logs (downloaded till <offset>) + s.T().Logf("Waiting for file cache Job with data reaching %d bytes", fileSizeForRangeRead) + jobLog := operations.RetryUntil(s.ctx, s.T(), retryFrequency, retryDuration, func() ([]*read_logs.Job, error) { + logs := read_logs.GetJobLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + if len(logs) == 1 { + for _, entry := range logs[0].JobEntries { + if entry.Offset >= fileSizeForRangeRead { + s.T().Logf("Found file cache Job with sufficient data (offset %d): %v", entry.Offset, logs[0]) + return logs, nil + } + } + } + return nil, fmt.Errorf("expected 1 Job with an entry >= %d bytes, found %d jobs", fileSizeForRangeRead, len(logs)) + }) + assert.Equal(s.T(), structuredReadFalseCacheHit[0].ObjectName, jobLog[0].ObjectName) + // Read file second time from Offset 1000 and validate from gcs. + secondReadOutcome := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, offset1000, s.T()) + + // Validate two read Logs and the second log must have cache hit 'true' recorded. + structuredReadLogsCacheHitTrueOnSecondLog := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + assert.Len(s.T(), structuredReadLogsCacheHitTrueOnSecondLog, 2, "Should have exactly 2 read records in logs") + validate(secondReadOutcome, structuredReadLogsCacheHitTrueOnSecondLog[1], + /*isSeq=*/ false, + /*cacheHit=*/ true, + /*chunkCount=*/ 1, + s.T()) // Validate cached content with gcs. - validateFileInCacheDirectory(testFileName, fileSizeForRangeRead, s.ctx, s.storageClient, t) + validateFileInCacheDirectory(testFileName, fileSizeForRangeRead, s.ctx, s.storageClient, s.T()) // Validate cache size within limit. - validateCacheSizeWithinLimit(cacheCapacityForRangeReadTestInMiB, t) + validateCacheSizeWithinLimit(cacheCapacityForRangeReadTestInMiB, s.T()) } //////////////////////////////////////////////////////////////////////// // Test Function (Runs once before all tests) //////////////////////////////////////////////////////////////////////// -func TestCacheFileForRangeReadTrueTest(t *testing.T) { - ts := &cacheFileForRangeReadTrueTest{ctx: context.Background()} - // Create storage client before running tests. - closeStorageClient := client.CreateStorageClientWithCancel(&ts.ctx, &ts.storageClient) - defer func() { - err := closeStorageClient() - if err != nil { - t.Errorf("closeStorageClient failed: %v", err) - } - }() - - // Run tests for mounted directory if the flag is set. - if setup.AreBothMountedDirectoryAndTestBucketFlagsSet() { - test_setup.RunTests(t, ts) +func (s *cacheFileForRangeReadTrueTest) runTests(t *testing.T) { + t.Helper() + // Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, s) return } - // Run with cache directory pointing to RAM based dir - ramCacheDir := path.Join("/dev/shm", cacheDirName) - - // Define flag set to run the tests. - flagsSet := []gcsfuseTestFlags{ - { - cliFlags: []string{"--implicit-dirs"}, - cacheSize: cacheCapacityForRangeReadTestInMiB, - cacheFileForRangeRead: true, - fileName: configFileName, - enableParallelDownloads: false, - enableODirect: false, - cacheDirPath: getDefaultCacheDirPathForTests(), - }, - { - cliFlags: nil, - cacheSize: cacheCapacityForRangeReadTestInMiB, - cacheFileForRangeRead: true, - fileName: configFileNameForParallelDownloadTests, - enableParallelDownloads: true, - enableODirect: false, - cacheDirPath: getDefaultCacheDirPathForTests(), - }, - { - cliFlags: nil, - cacheSize: cacheCapacityForRangeReadTestInMiB, - cacheFileForRangeRead: true, - fileName: configFileNameForParallelDownloadTests, - enableParallelDownloads: true, - enableODirect: false, - cacheDirPath: ramCacheDir, - }, - { - cliFlags: nil, - cacheSize: cacheCapacityForRangeReadTestInMiB, - cacheFileForRangeRead: true, - fileName: configFileNameForParallelDownloadTests, - enableParallelDownloads: true, - enableODirect: true, - cacheDirPath: getDefaultCacheDirPathForTests(), - }, - { - cliFlags: nil, - cacheSize: cacheCapacityForRangeReadTestInMiB, - cacheFileForRangeRead: true, - fileName: configFileNameForParallelDownloadTests, - enableParallelDownloads: true, - enableODirect: true, - cacheDirPath: ramCacheDir, - }, - { - cliFlags: nil, - cacheSize: cacheCapacityForRangeReadTestInMiB, - cacheFileForRangeRead: true, - fileName: configFileName, - enableParallelDownloads: false, - enableODirect: false, - cacheDirPath: ramCacheDir, - }, - } - flagsSet = appendClientProtocolConfigToFlagSet(flagsSet) - // Run tests. - for _, flags := range flagsSet { - configFilePath := createConfigFile(&flags) - ts.flags = []string{"--config-file=" + configFilePath} - if flags.cliFlags != nil { - ts.flags = append(ts.flags, flags.cliFlags...) - } - log.Printf("Running tests with flags: %s", ts.flags) - test_setup.RunTests(t, ts) + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, s.flags = range flagsSet { + log.Printf("Running tests with flags: %s", s.flags) + suite.Run(t, s) } } + +func TestCacheFileForRangeReadTrueTest(t *testing.T) { + ts := &cacheFileForRangeReadTrueTest{ctx: context.Background(), storageClient: testEnv.storageClient, baseTestName: t.Name()} + ts.runTests(t) +} + +func TestCacheFileForRangeReadTrueWithRamCache(t *testing.T) { + ts := &cacheFileForRangeReadTrueTest{ctx: context.Background(), storageClient: testEnv.storageClient, baseTestName: t.Name(), isCacheOnRAM: true} + ts.runTests(t) +} diff --git a/tools/integration_tests/read_cache/chunk_cache_disabled_test.go b/tools/integration_tests/read_cache/chunk_cache_disabled_test.go new file mode 100644 index 0000000000..c445ca701d --- /dev/null +++ b/tools/integration_tests/read_cache/chunk_cache_disabled_test.go @@ -0,0 +1,98 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package read_cache + +import ( + "context" + "log" + "os" + "testing" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type chunkCacheDisabledTest struct { + flags []string + storageClient *storage.Client + ctx context.Context + baseTestName string + suite.Suite +} + +func (s *chunkCacheDisabledTest) SetupSuite() { + setupLogFileAndCacheDir(s.baseTestName) + mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient) +} + +func (s *chunkCacheDisabledTest) SetupTest() { + //Truncate log file created. + err := os.Truncate(testEnv.cfg.LogFile, 0) + require.NoError(s.T(), err) + // Clean up the cache directory path as gcsfuse don't clean up on mounting. + operations.RemoveDir(testEnv.cacheDirPath) + testEnv.testDirPath = client.SetupUniqueTestDirectory(s.ctx, s.storageClient, testDirPrefix) +} + +func (s *chunkCacheDisabledTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +func (s *chunkCacheDisabledTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *chunkCacheDisabledTest) TestNormalFileCacheWithChunkCacheDisabled() { + testFileName := setupFileInTestDir(s.ctx, s.storageClient, 10*util.MiB, s.T()) + + expectedOutcome := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, 0, s.T()) + + // Verify cache miss for normal file cache + structuredLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + require.Len(s.T(), structuredLogs, 1) + validate(expectedOutcome, structuredLogs[0], true, false, 1, s.T()) + + jobLogs := read_logs.GetJobLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + require.Len(s.T(), jobLogs, 1, "Job logs should have exactly 1 entry") + assert.Empty(s.T(), jobLogs[0].ChunkCacheDownloads, "Should not have chunk downloads") + assert.NotEmpty(s.T(), jobLogs[0].JobEntries, "Should have normal file cache downloads") +} + +func TestChunkCacheDisabledTest(t *testing.T) { + ts := &chunkCacheDisabledTest{ + ctx: context.Background(), + storageClient: testEnv.storageClient, + baseTestName: t.Name(), + } + // Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, flags := range flagsSet { + ts.flags = flags + log.Printf("running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/read_cache/chunk_cache_eviction_test.go b/tools/integration_tests/read_cache/chunk_cache_eviction_test.go new file mode 100644 index 0000000000..e4ee9c0b8f --- /dev/null +++ b/tools/integration_tests/read_cache/chunk_cache_eviction_test.go @@ -0,0 +1,117 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package read_cache + +import ( + "context" + "log" + "os" + "testing" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/data" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type chunkCacheEvictionTest struct { + flags []string + storageClient *storage.Client + ctx context.Context + baseTestName string + suite.Suite +} + +func (s *chunkCacheEvictionTest) SetupSuite() { + setupLogFileAndCacheDir(s.baseTestName) + mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient) +} + +func (s *chunkCacheEvictionTest) SetupTest() { + //Truncate log file created. + err := os.Truncate(testEnv.cfg.LogFile, 0) + require.NoError(s.T(), err) + // Clean up the cache directory path as gcsfuse don't clean up on mounting. + operations.RemoveDir(testEnv.cacheDirPath) + testEnv.testDirPath = client.SetupUniqueTestDirectory(s.ctx, s.storageClient, testDirPrefix) +} + +func (s *chunkCacheEvictionTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +func (s *chunkCacheEvictionTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *chunkCacheEvictionTest) TestEviction() { + testFileName1 := setupFileInTestDir(s.ctx, s.storageClient, 20*util.MiB, s.T()) + testFileName2 := setupFileInTestDir(s.ctx, s.storageClient, 20*util.MiB, s.T()) + + expectedOutcome1 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName1, 20*util.MiB, false, s.T()) + expectedOutcome2 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName2, 20*util.MiB, false, s.T()) + expectedOutcome3 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName1, 0, s.T()) + + structuredLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + require.Len(s.T(), structuredLogs, 3) + validate(expectedOutcome1, structuredLogs[0], true, true, int(20*util.MiB/chunkSizeToRead), s.T()) + validate(expectedOutcome2, structuredLogs[1], true, true, int(20*util.MiB/chunkSizeToRead), s.T()) + validate(expectedOutcome3, structuredLogs[2], true, true, 1, s.T()) + jobLogs := read_logs.GetJobLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + require.NotEmpty(s.T(), jobLogs) + var allDownloads []read_logs.ChunkDownloadLogEntry + for _, jobLog := range jobLogs { + allDownloads = append(allDownloads, jobLog.ChunkCacheDownloads...) + } + // Expected download sequence: + // 1. File 1 is read (20MB). Cache grows to 20MB (exceeding 15MB limit, but eviction is deferred). + // 2. File 2 is read. Insertion of File 2 triggers eviction of File 1 (LRU) to enforce limit. + // 3. File 2 is downloaded (20MB). Cache grows to 20MB. + // 4. File 1 is read again. Insertion of File 1 triggers eviction of File 2. File 1 [0, 10) is re-downloaded. + expectedDownloads := []data.ObjectRange{ + {Start: 0, End: 10 * util.MiB}, // File 1 chunk 1 + {Start: 10 * util.MiB, End: 20 * util.MiB}, // File 1 chunk 2 + {Start: 0, End: 10 * util.MiB}, // File 2 chunk 1 + {Start: 10 * util.MiB, End: 20 * util.MiB}, // File 2 chunk 2 + {Start: 0, End: 10 * util.MiB}, // File 1 chunk 1 (again) + } + // Skip content validation as downloads span multiple files. + validateDownloads(s.T(), allDownloads, expectedDownloads, "") +} + +func TestChunkCacheEviction(t *testing.T) { + ts := &chunkCacheEvictionTest{ + ctx: context.Background(), + storageClient: testEnv.storageClient, + baseTestName: t.Name(), + } + // Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, flags := range flagsSet { + ts.flags = flags + log.Printf("running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/read_cache/chunk_cache_test.go b/tools/integration_tests/read_cache/chunk_cache_test.go new file mode 100644 index 0000000000..f33db364eb --- /dev/null +++ b/tools/integration_tests/read_cache/chunk_cache_test.go @@ -0,0 +1,260 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package read_cache + +import ( + "context" + "log" + "os" + "path" + "sync" + "syscall" + "testing" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/data" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +type chunkCacheTest struct { + flags []string + storageClient *storage.Client + ctx context.Context + baseTestName string + suite.Suite +} + +func (s *chunkCacheTest) SetupSuite() { + setupLogFileAndCacheDir(s.baseTestName) + mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient) +} + +func (s *chunkCacheTest) SetupTest() { + //Truncate log file created. + err := os.Truncate(testEnv.cfg.LogFile, 0) + require.NoError(s.T(), err) + // Clean up the cache directory path as gcsfuse don't clean up on mounting. + operations.RemoveDir(testEnv.cacheDirPath) + testEnv.testDirPath = client.SetupUniqueTestDirectory(s.ctx, s.storageClient, testDirPrefix) +} + +func (s *chunkCacheTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +func (s *chunkCacheTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *chunkCacheTest) TestRandomRead() { + testFileName := setupFileInTestDir(s.ctx, s.storageClient, 20*util.MiB, s.T()) + + expectedOutcome1 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, 15*util.MiB, s.T()) + expectedOutcome2 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, 5*util.MiB, s.T()) + expectedOutcome3 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, 10*util.MiB, s.T()) + + structuredLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + require.Len(s.T(), structuredLogs, 3) + validate(expectedOutcome1, structuredLogs[0], false, true, 1, s.T()) + validate(expectedOutcome2, structuredLogs[1], false, true, 1, s.T()) + validate(expectedOutcome3, structuredLogs[2], false, true, 1, s.T()) + jobLogs := read_logs.GetJobLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + require.NotEmpty(s.T(), jobLogs) + expectedDownloads := []data.ObjectRange{ + {Start: 10 * util.MiB, End: 20 * util.MiB}, + {Start: 0, End: 10 * util.MiB}, + } + validateDownloads(s.T(), jobLogs[0].ChunkCacheDownloads, expectedDownloads, testFileName) +} + +func (s *chunkCacheTest) TestFullSequentialRead() { + testFileName := setupFileInTestDir(s.ctx, s.storageClient, 30*util.MiB, s.T()) + + expectedOutcome := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, 30*util.MiB, false, s.T()) + + structuredLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + require.Len(s.T(), structuredLogs, 1) + validate(expectedOutcome, structuredLogs[0], true, true, int(30*util.MiB/chunkSizeToRead), s.T()) + jobLogs := read_logs.GetJobLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + require.NotEmpty(s.T(), jobLogs) + expectedDownloads := []data.ObjectRange{ + {Start: 0, End: 10 * util.MiB}, + {Start: 10 * util.MiB, End: 20 * util.MiB}, + {Start: 20 * util.MiB, End: 30 * util.MiB}, + } + validateDownloads(s.T(), jobLogs[0].ChunkCacheDownloads, expectedDownloads, testFileName) +} + +func (s *chunkCacheTest) TestSequentialReadWithCachedChunk() { + testFileName := setupFileInTestDir(s.ctx, s.storageClient, 30*util.MiB, s.T()) + + expectedOutcome1 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, 15*util.MiB, s.T()) + expectedOutcome2 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, 30*util.MiB, false, s.T()) + + structuredLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + require.Len(s.T(), structuredLogs, 2) + validate(expectedOutcome1, structuredLogs[0], false, true, 1, s.T()) + validate(expectedOutcome2, structuredLogs[1], true, true, int(30*util.MiB/chunkSizeToRead), s.T()) + jobLogs := read_logs.GetJobLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + require.NotEmpty(s.T(), jobLogs) + // Since [10, 20) is already downloaded in the first read, we expect that we don't download it again. + expectedDownloads := []data.ObjectRange{ + {Start: 10 * util.MiB, End: 20 * util.MiB}, + {Start: 0, End: 10 * util.MiB}, + {Start: 20 * util.MiB, End: 30 * util.MiB}, + } + validateDownloads(s.T(), jobLogs[0].ChunkCacheDownloads, expectedDownloads, testFileName) +} + +func (s *chunkCacheTest) TestReadSpanningTwoBlocks() { + testFileName := setupFileInTestDir(s.ctx, s.storageClient, 30*util.MiB, s.T()) + + content, err := operations.ReadChunkFromFile(path.Join(testEnv.testDirPath, testFileName), 1*util.MiB, 19.5*util.MiB, os.O_RDONLY|syscall.O_DIRECT) + + require.NoError(s.T(), err) + client.ValidateObjectChunkFromGCS(s.ctx, s.storageClient, path.Base(testEnv.testDirPath), testFileName, 19.5*util.MiB, 1*util.MiB, string(content), s.T()) + jobLogs := read_logs.GetJobLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + require.NotEmpty(s.T(), jobLogs) + expectedDownloads := []data.ObjectRange{ + {Start: 10 * util.MiB, End: 20 * util.MiB}, + {Start: 20 * util.MiB, End: 30 * util.MiB}, + } + validateDownloads(s.T(), jobLogs[0].ChunkCacheDownloads, expectedDownloads, testFileName) +} + +func (s *chunkCacheTest) TestConcurrentDeduplication() { + testFileName := setupFileInTestDir(s.ctx, s.storageClient, 10*util.MiB, s.T()) + outcomes := make([]*Expected, 8) + var wg sync.WaitGroup + wg.Add(8) + + for i := 0; i < 8; i++ { + go func(i int) { + defer wg.Done() + outcomes[i] = readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, 5*util.MiB, s.T()) + }(i) + } + wg.Wait() + + structuredLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + require.Len(s.T(), structuredLogs, 8) + for i := 0; i < 8; i++ { + validate(outcomes[i], structuredLogs[i], false, true, 1, s.T()) + } + jobLogs := read_logs.GetJobLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + require.NotEmpty(s.T(), jobLogs) + // Despite 8 concurrent reads from the same chunk, it should be downloaded only once. + expectedDownloads := []data.ObjectRange{ + {Start: 0, End: 10 * util.MiB}, + } + validateDownloads(s.T(), jobLogs[0].ChunkCacheDownloads, expectedDownloads, testFileName) +} + +func (s *chunkCacheTest) TestReadOfDeletedfile() { + testFileName := setupFileInTestDir(s.ctx, s.storageClient, 20*util.MiB, s.T()) + // Read first chunk to ensure cache file is created and populated. + readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, 0, s.T()) + // Delete the file from GCS to trigger a failure during the next download attempt. + _, objectName := setup.GetBucketAndObjectBasedOnTypeOfMount(path.Join(path.Base(testEnv.testDirPath), testFileName)) + err := client.DeleteObjectOnGCS(s.ctx, s.storageClient, objectName) + require.NoError(s.T(), err) + + // Read second chunk. This should fail because the source object is gone. + _, err = operations.ReadChunkFromFile(path.Join(testEnv.testDirPath, testFileName), chunkSizeToRead, 10*util.MiB, os.O_RDONLY|syscall.O_DIRECT) + + require.Error(s.T(), err, "Read should fail after GCS object is deleted") + operations.ValidateESTALEError(s.T(), err) + logContent, err := os.ReadFile(testEnv.cfg.LogFile) + require.NoError(s.T(), err) + assert.Contains(s.T(), string(logContent), "Sparse file read failed") + assert.Contains(s.T(), string(logContent), "Falling back to GCS") +} + +func (s *chunkCacheTest) TestCacheFileAllocatedSize() { + testFileName := setupFileInTestDir(s.ctx, s.storageClient, 21*util.MiB, s.T()) + + tests := []struct { + name string + offset int64 + expectedAllocatedSize int64 + isSeq bool + expectedDownloads []data.ObjectRange + }{ + { + name: "ReadLastChunk", + offset: 20 * util.MiB, + expectedAllocatedSize: 1 * util.MiB, + isSeq: false, + expectedDownloads: []data.ObjectRange{ + {Start: 20 * util.MiB, End: 21 * util.MiB}, + }, + }, + { + name: "ReadFirstChunk", + offset: 0, + expectedAllocatedSize: 11 * util.MiB, // 1MB (from previous) + 10MB (current) + isSeq: true, + expectedDownloads: []data.ObjectRange{ + {Start: 20 * util.MiB, End: 21 * util.MiB}, + {Start: 0, End: 10 * util.MiB}, + }, + }, + } + for i, tc := range tests { + s.T().Run(tc.name, func(t *testing.T) { + expectedOutcome := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, tc.offset, t) + + validateAllocatedFileSize(t, testFileName, tc.expectedAllocatedSize) + structuredLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, t) + require.Len(t, structuredLogs, i+1) + validate(expectedOutcome, structuredLogs[i], tc.isSeq, true, 1, t) + jobLogs := read_logs.GetJobLogsSortedByTimestamp(testEnv.cfg.LogFile, t) + require.NotEmpty(t, jobLogs) + validateDownloads(t, jobLogs[0].ChunkCacheDownloads, tc.expectedDownloads, testFileName) + }) + } +} + +func TestChunkCacheTest(t *testing.T) { + ts := &chunkCacheTest{ + ctx: context.Background(), + storageClient: testEnv.storageClient, + baseTestName: t.Name(), + } + // Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, flags := range flagsSet { + ts.flags = flags + log.Printf("running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/read_cache/disabled_cache_ttl_test.go b/tools/integration_tests/read_cache/disabled_cache_ttl_test.go index 10c1f36391..870fe0a235 100644 --- a/tools/integration_tests/read_cache/disabled_cache_ttl_test.go +++ b/tools/integration_tests/read_cache/disabled_cache_ttl_test.go @@ -4,28 +4,29 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package read_cache import ( "context" "log" + "os" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/log_parser/json_parser/read_logs" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/test_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) //////////////////////////////////////////////////////////////////////// @@ -36,40 +37,53 @@ type disabledCacheTTLTest struct { flags []string storageClient *storage.Client ctx context.Context + baseTestName string + suite.Suite +} + +func (s *disabledCacheTTLTest) SetupSuite() { + setupLogFileAndCacheDir(s.baseTestName) + mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient) } -func (s *disabledCacheTTLTest) Setup(t *testing.T) { - setupForMountedDirectoryTests() +func (s *disabledCacheTTLTest) SetupTest() { + //Truncate log file created. + err := os.Truncate(testEnv.cfg.LogFile, 0) + require.NoError(s.T(), err) // Clean up the cache directory path as gcsfuse don't clean up on mounting. - operations.RemoveDir(cacheDirPath) - mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient, testDirName) + operations.RemoveDir(testEnv.cacheDirPath) + testEnv.testDirPath = client.SetupUniqueTestDirectory(s.ctx, s.storageClient, testDirPrefix) } -func (s *disabledCacheTTLTest) Teardown(t *testing.T) { - setup.UnmountGCSFuseAndDeleteLogFile(rootDir) +func (s *disabledCacheTTLTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +func (s *disabledCacheTTLTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) } //////////////////////////////////////////////////////////////////////// // Test scenarios //////////////////////////////////////////////////////////////////////// -func (s *disabledCacheTTLTest) TestReadAfterObjectUpdateIsCacheMiss(t *testing.T) { - testFileName := setupFileInTestDir(s.ctx, s.storageClient, testDirName, fileSize, t) +func (s *disabledCacheTTLTest) TestReadAfterObjectUpdateIsCacheMiss() { + testFileName := setupFileInTestDir(s.ctx, s.storageClient, fileSize, s.T()) // Read file 1st time. - expectedOutcome1 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, true, t) + expectedOutcome1 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, true, s.T()) // Modify the file. - modifyFile(s.ctx, s.storageClient, testFileName, t) + modifyFile(s.ctx, s.storageClient, testFileName, s.T()) // Read same file again immediately. New content should be served as cache ttl is 0. - expectedOutcome2 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, smallContentSize, true, t) + expectedOutcome2 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, smallContentSize, true, s.T()) // Read the same file again. The data should be served from cache. - expectedOutcome3 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, smallContentSize, true, t) + expectedOutcome3 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, smallContentSize, true, s.T()) // Parse the log file and validate cache hit or miss from the structured logs. - structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t) - validate(expectedOutcome1, structuredReadLogs[0], true, false, chunksRead, t) - validate(expectedOutcome2, structuredReadLogs[1], true, false, chunksReadAfterUpdate, t) - validate(expectedOutcome3, structuredReadLogs[2], true, true, chunksReadAfterUpdate, t) + structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + validate(expectedOutcome1, structuredReadLogs[0], true, false, chunksRead, s.T()) + validate(expectedOutcome2, structuredReadLogs[1], true, false, chunksReadAfterUpdate, s.T()) + validate(expectedOutcome3, structuredReadLogs[2], true, true, chunksReadAfterUpdate, s.T()) } //////////////////////////////////////////////////////////////////////// @@ -77,52 +91,18 @@ func (s *disabledCacheTTLTest) TestReadAfterObjectUpdateIsCacheMiss(t *testing.T //////////////////////////////////////////////////////////////////////// func TestDisabledCacheTTLTest(t *testing.T) { - ts := &disabledCacheTTLTest{ctx: context.Background()} - // Create storage client before running tests. - closeStorageClient := client.CreateStorageClientWithCancel(&ts.ctx, &ts.storageClient) - defer func() { - err := closeStorageClient() - if err != nil { - t.Errorf("closeStorageClient failed: %v", err) - } - }() - - // Run tests for mounted directory if the flag is set. - if setup.AreBothMountedDirectoryAndTestBucketFlagsSet() { - test_setup.RunTests(t, ts) + ts := &disabledCacheTTLTest{ctx: context.Background(), storageClient: testEnv.storageClient, baseTestName: t.Name()} + + // Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) return } - // Define flag set to run the tests. - flagsSet := []gcsfuseTestFlags{ - { - cliFlags: []string{"--implicit-dirs", "--stat-cache-ttl=0s"}, - cacheSize: cacheCapacityInMB, - cacheFileForRangeRead: false, - fileName: configFileName, - enableParallelDownloads: false, - enableODirect: false, - cacheDirPath: getDefaultCacheDirPathForTests(), - }, - { - cliFlags: []string{"--stat-cache-ttl=0s"}, - cacheSize: cacheCapacityInMB, - cacheFileForRangeRead: false, - fileName: configFileNameForParallelDownloadTests, - enableParallelDownloads: true, - enableODirect: false, - cacheDirPath: getDefaultCacheDirPathForTests(), - }, - } - flagsSet = appendClientProtocolConfigToFlagSet(flagsSet) - // Run tests. - for _, flags := range flagsSet { - configFilePath := createConfigFile(&flags) - ts.flags = []string{"--config-file=" + configFilePath} - if flags.cliFlags != nil { - ts.flags = append(ts.flags, flags.cliFlags...) - } + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { log.Printf("Running tests with flags: %s", ts.flags) - test_setup.RunTests(t, ts) + suite.Run(t, ts) } } diff --git a/tools/integration_tests/read_cache/helpers_test.go b/tools/integration_tests/read_cache/helpers_test.go index de875adc3e..eeb9788b78 100644 --- a/tools/integration_tests/read_cache/helpers_test.go +++ b/tools/integration_tests/read_cache/helpers_test.go @@ -17,6 +17,7 @@ package read_cache import ( "context" "fmt" + "io" "io/fs" "log" "os" @@ -27,10 +28,12 @@ import ( "time" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/log_parser/json_parser/read_logs" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/data" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -47,7 +50,7 @@ func readFileAndGetExpectedOutcome(testDirPath, fileName string, readFullFile bo expected := &Expected{ StartTimeStampSeconds: time.Now().Unix(), BucketName: setup.TestBucket(), - ObjectName: path.Join(testDirName, fileName), + ObjectName: path.Join(path.Base(testEnv.testDirPath), fileName), } if setup.DynamicBucketMounted() != "" { expected.BucketName = setup.DynamicBucketMounted() @@ -57,7 +60,9 @@ func readFileAndGetExpectedOutcome(testDirPath, fileName string, readFullFile bo var err error if readFullFile { - content, err = operations.ReadFileSequentially(path.Join(testDirPath, fileName), chunkSizeToRead) + file, err := os.OpenFile(path.Join(testDirPath, fileName), os.O_RDONLY|syscall.O_DIRECT, setup.FilePermission_0600) + require.NoError(t, err) + content, err = operations.ReadFileSequentially(file, chunkSizeToRead) if err != nil { t.Errorf("Failed to read file sequentially: %v", err) } @@ -73,8 +78,8 @@ func readFileAndGetExpectedOutcome(testDirPath, fileName string, readFullFile bo return expected } -func validate(expected *Expected, logEntry *read_logs.StructuredReadLogEntry, - isSeq, cacheHit bool, chunkCount int, t *testing.T) { +func validate(expected *Expected, logEntry *read_logs.StructuredReadLogEntry, isSeq, cacheHit bool, chunkCount int, t *testing.T) { + t.Helper() if logEntry.StartTimeSeconds < expected.StartTimeStampSeconds { t.Errorf("start time in logs %d less than actual start time %d.", logEntry.StartTimeSeconds, expected.StartTimeStampSeconds) } @@ -90,6 +95,10 @@ func validate(expected *Expected, logEntry *read_logs.StructuredReadLogEntry, t.Errorf("chunks read don't match! Expected: %d, Got from logs: %d", chunkCount, len(logEntry.Chunks)) } + // If no chunks are found in logs, we can't validate further. + if len(logEntry.Chunks) == 0 { + return + } if logEntry.Chunks[len(logEntry.Chunks)-1].StartTimeSeconds > expected.EndTimeStampSeconds { t.Errorf("end time in logs more than actual end time.") } @@ -102,11 +111,11 @@ func validate(expected *Expected, logEntry *read_logs.StructuredReadLogEntry, } func getCachedFilePath(fileName string) string { - bucketName := setup.TestBucket() + bucketName := testEnv.cfg.TestBucket if setup.DynamicBucketMounted() != "" { bucketName = setup.DynamicBucketMounted() } - return path.Join(cacheDirPath, cacheSubDirectoryName, bucketName, testDirName, fileName) + return path.Join(testEnv.cacheDirPath, cacheSubDirectoryName, bucketName, path.Base(testEnv.testDirPath), fileName) } func validateFileSizeInCacheDirectory(fileName string, filesize int64, t *testing.T) { @@ -136,7 +145,7 @@ func validateFileSizeInCacheDirectory(fileName string, filesize int64, t *testin func validateFileInCacheDirectory(fileName string, filesize int64, ctx context.Context, storageClient *storage.Client, t *testing.T) { validateFileSizeInCacheDirectory(fileName, filesize, t) - gcsCRC, err := client.GetCRCFromGCS(path.Join(testDirName, fileName), ctx, storageClient) + gcsCRC, err := client.GetCRCFromGCS(path.Join(path.Base(testEnv.testDirPath), fileName), ctx, storageClient) require.NoError(t, err) maxRetries := 20 retryDelay := 500 * time.Millisecond @@ -166,24 +175,32 @@ func validateFileIsNotCached(fileName string, t *testing.T) { } } -func remountGCSFuse(flags []string, t *testing.T) { +func validateFileIsCached(fileName string, t *testing.T) { + // Validate that the file is present in cache location. + expectedPathOfCachedFile := getCachedFilePath(fileName) + _, err := operations.StatFile(expectedPathOfCachedFile) + if err != nil { + t.Errorf("File %s not found in cache directory", expectedPathOfCachedFile) + } +} + +func remountGCSFuse(flags []string) { setup.SetMntDir(rootDir) setup.UnmountGCSFuseAndDeleteLogFile(rootDir) - setup.MountGCSFuseWithGivenMountFunc(flags, mountFunc) + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, flags, mountFunc) setup.SetMntDir(mountDir) } -func readFileAndValidateCacheWithGCS(ctx context.Context, storageClient *storage.Client, - filename string, fileSize int64, checkCacheSize bool, t *testing.T) (expectedOutcome *Expected) { +func readFileAndValidateCacheWithGCS(ctx context.Context, storageClient *storage.Client, filename string, fileSize int64, checkCacheSize bool, t *testing.T) (expectedOutcome *Expected) { // Read file via gcsfuse mount. - expectedOutcome = readFileAndGetExpectedOutcome(testDirPath, filename, true, zeroOffset, t) + expectedOutcome = readFileAndGetExpectedOutcome(testEnv.testDirPath, filename, true, zeroOffset, t) // Validate CRC32 of content read via gcsfuse with CRC32 value on gcs. gotCRC32Value, err := operations.CalculateCRC32(strings.NewReader(expectedOutcome.content)) if err != nil { t.Errorf("CalculateCRC32 Failed: %v", err) } - gcsCRC, err := client.GetCRCFromGCS(path.Join(testDirName, filename), ctx, storageClient) + gcsCRC, err := client.GetCRCFromGCS(path.Join(path.Base(testEnv.testDirPath), filename), ctx, storageClient) if err != nil || gcsCRC != gotCRC32Value { t.Errorf("Content served CRC mismatch: %v", err) } @@ -200,33 +217,30 @@ func readFileAndValidateCacheWithGCS(ctx context.Context, storageClient *storage func readChunkAndValidateObjectContentsFromGCS(ctx context.Context, storageClient *storage.Client, filename string, offset int64, t *testing.T) (expectedOutcome *Expected) { // Read file via gcsfuse mount. - expectedOutcome = readFileAndGetExpectedOutcome(testDirPath, filename, false, offset, t) + expectedOutcome = readFileAndGetExpectedOutcome(testEnv.testDirPath, filename, false, offset, t) // Validate content read via gcsfuse with gcs. - client.ValidateObjectChunkFromGCS(ctx, storageClient, testDirName, filename, offset, chunkSizeToRead, + client.ValidateObjectChunkFromGCS(ctx, storageClient, path.Base(testEnv.testDirPath), filename, offset, chunkSizeToRead, expectedOutcome.content, t) return expectedOutcome } -func readFileAndValidateFileIsNotCached(ctx context.Context, storageClient *storage.Client, - filename string, readFullFile bool, offset int64, t *testing.T) (expectedOutcome *Expected) { +func readFileAndValidateFileIsNotCached(ctx context.Context, storageClient *storage.Client, fileName string, readFullFile bool, offset int64, t *testing.T) (expectedOutcome *Expected) { // Read file via gcsfuse mount. - expectedOutcome = readFileAndGetExpectedOutcome(testDirPath, filename, readFullFile, offset, t) + expectedOutcome = readFileAndGetExpectedOutcome(testEnv.testDirPath, fileName, readFullFile, offset, t) // Validate that the file is not cached. - validateFileIsNotCached(filename, t) + validateFileIsNotCached(fileName, t) // validate the content read matches the content on GCS. if readFullFile { - client.ValidateObjectContentsFromGCS(ctx, storageClient, testDirName, filename, - expectedOutcome.content, t) + client.ValidateObjectContentsFromGCS(ctx, storageClient, path.Base(testEnv.testDirPath), fileName, expectedOutcome.content, t) } else { - client.ValidateObjectChunkFromGCS(ctx, storageClient, testDirName, filename, - offset, chunkSizeToRead, expectedOutcome.content, t) + client.ValidateObjectChunkFromGCS(ctx, storageClient, path.Base(testEnv.testDirPath), fileName, offset, chunkSizeToRead, expectedOutcome.content, t) } return expectedOutcome } func modifyFile(ctx context.Context, storageClient *storage.Client, testFileName string, t *testing.T) { - objectName := path.Join(testDirName, testFileName) + objectName := path.Join(path.Base(testEnv.testDirPath), testFileName) smallContent, err := operations.GenerateRandomData(smallContentSize) if err != nil { t.Errorf("Could not generate random data to modify file: %v", err) @@ -238,7 +252,7 @@ func modifyFile(ctx context.Context, storageClient *storage.Client, testFileName } func validateCacheSizeWithinLimit(cacheCapacity int64, t *testing.T) { - cacheSize, err := operations.DirSizeMiB(cacheDirPath) + cacheSize, err := operations.DirSizeMiB(testEnv.cacheDirPath) if err != nil { t.Errorf("Error in getting cache size: %v", cacheSize) } @@ -247,9 +261,9 @@ func validateCacheSizeWithinLimit(cacheCapacity int64, t *testing.T) { } } -func setupFileInTestDir(ctx context.Context, storageClient *storage.Client, testDirName string, fileSize int64, t *testing.T) (fileName string) { +func setupFileInTestDir(ctx context.Context, storageClient *storage.Client, fileSize int64, t *testing.T) (fileName string) { testFileName := testFileName + setup.GenerateRandomString(testFileNameSuffixLength) - client.SetupFileInTestDirectory(ctx, storageClient, testDirName, testFileName, fileSize, t) + client.SetupFileInTestDirectory(ctx, storageClient, path.Base(testEnv.testDirPath), testFileName, fileSize, t) return testFileName } @@ -260,3 +274,60 @@ func runTestsOnlyForDynamicMount(t *testing.T) { t.SkipNow() } } + +func validateAllocatedFileSize(t *testing.T, fileName string, expectedSize int64) { + cachedFilePath := getCachedFilePath(fileName) + fi, err := os.Stat(cachedFilePath) + require.NoError(t, err) + stat := fi.Sys().(*syscall.Stat_t) + allocatedSize := stat.Blocks * 512 + + // Allow small overhead (1KB) for metadata stored for this file. + overhead := int64(1024) + assert.LessOrEqual(t, allocatedSize, expectedSize+overhead, "Allocated size should be close to expected size") + assert.GreaterOrEqual(t, allocatedSize, expectedSize, "Allocated size should be at least expected size") +} + +func validateDownloads(t *testing.T, downloads []read_logs.ChunkDownloadLogEntry, expectedRanges []data.ObjectRange, fileName string) { + require.Len(t, downloads, len(expectedRanges), "Expected exactly %d downloads", len(expectedRanges)) + + // Count expected + expectedCounts := make(map[data.ObjectRange]int) + for _, r := range expectedRanges { + expectedCounts[r]++ + } + + // Count actual + actualCounts := make(map[data.ObjectRange]int) + for _, d := range downloads { + r := data.ObjectRange{Start: d.StartOffset, End: d.EndOffset} + actualCounts[r]++ + if fileName != "" { + validateContent(t, fileName, d.StartOffset, d.EndOffset) + } + } + assert.Equal(t, expectedCounts, actualCounts, "Download ranges mismatch") +} + +func validateContent(t *testing.T, fileName string, start, end int64) { + // Read from cache + cacheFilePath := getCachedFilePath(fileName) + file, err := os.Open(cacheFilePath) + require.NoError(t, err) + defer file.Close() + size := end - start + cacheContent := make([]byte, size) + _, err = file.ReadAt(cacheContent, start) + require.NoError(t, err) + + // Read from GCS + objectName := path.Join(path.Base(testEnv.testDirPath), fileName) + bucketName, objectName := setup.GetBucketAndObjectBasedOnTypeOfMount(objectName) + rc, err := testEnv.storageClient.Bucket(bucketName).Object(objectName).NewRangeReader(testEnv.ctx, start, size) + require.NoError(t, err) + defer rc.Close() + + gcsContent, err := io.ReadAll(rc) + require.NoError(t, err) + assert.Equal(t, gcsContent, cacheContent, "Content mismatch for range [%d, %d)", start, end) +} diff --git a/tools/integration_tests/read_cache/job_chunk_test.go b/tools/integration_tests/read_cache/job_chunk_test.go index 9ad7f45bfd..e6fc87956e 100644 --- a/tools/integration_tests/read_cache/job_chunk_test.go +++ b/tools/integration_tests/read_cache/job_chunk_test.go @@ -4,130 +4,121 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package read_cache import ( "context" "log" - "path" + "os" "sync" "testing" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/util" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/log_parser/json_parser/read_logs" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/test_setup" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" ) //////////////////////////////////////////////////////////////////////// // Boilerplate //////////////////////////////////////////////////////////////////////// +const cacheSizeMB int64 = 48 + type jobChunkTest struct { flags []string storageClient *storage.Client ctx context.Context + baseTestName string chunkSize int64 + suite.Suite +} + +func (s *jobChunkTest) SetupSuite() { + setupLogFileAndCacheDir(s.baseTestName) + mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient) } -func (s *jobChunkTest) Setup(t *testing.T) { - setupForMountedDirectoryTests() +func (s *jobChunkTest) SetupTest() { + //Truncate log file created. + err := os.Truncate(testEnv.cfg.LogFile, 0) + require.NoError(s.T(), err) // Clean up the cache directory path as gcsfuse don't clean up on mounting. - operations.RemoveDir(cacheDirPath) - mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient, testDirName) + operations.RemoveDir(testEnv.cacheDirPath) + testEnv.testDirPath = client.SetupUniqueTestDirectory(s.ctx, s.storageClient, testDirPrefix) } -func (s *jobChunkTest) Teardown(t *testing.T) { - setup.UnmountGCSFuseAndDeleteLogFile(rootDir) +func (s *jobChunkTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) } -func createConfigFileForJobChunkTest(cacheSize int64, cacheFileForRangeRead bool, fileName string, parallelDownloadsPerFile, maxParallelDownloads, downloadChunkSizeMB int, clientProtocol string) string { - cacheDirPath = path.Join(setup.TestDir(), cacheDirName) - - // Set up config file for file cache. - mountConfig := map[string]interface{}{ - "file-cache": map[string]interface{}{ - "max-size-mb": cacheSize, - "cache-file-for-range-read": cacheFileForRangeRead, - "enable-parallel-downloads": true, - "parallel-downloads-per-file": parallelDownloadsPerFile, - "max-parallel-downloads": maxParallelDownloads, - "download-chunk-size-mb": downloadChunkSizeMB, - "enable-crc": enableCrcCheck, - }, - "cache-dir": cacheDirPath, - "gcs-connection": map[string]interface{}{ - "client-protocol": clientProtocol, - }, - } - filePath := setup.YAMLConfigFile(mountConfig, fileName) - return filePath +func (s *jobChunkTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) } //////////////////////////////////////////////////////////////////////// // Test scenarios //////////////////////////////////////////////////////////////////////// -func (s *jobChunkTest) TestJobChunkSizeForSingleFileReads(t *testing.T) { +func (s *jobChunkTest) TestJobChunkSizeForSingleFileReads() { var fileSize int64 = 16 * util.MiB - testFileName := setupFileInTestDir(s.ctx, s.storageClient, testDirName, fileSize, t) + testFileName := setupFileInTestDir(s.ctx, s.storageClient, fileSize, s.T()) - expectedOutcome := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, false, t) + expectedOutcome := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, false, s.T()) // Parse the log file and validate cache hit or miss from the structured logs. - structuredJobLogs := read_logs.GetJobLogsSortedByTimestamp(setup.LogFile(), t) - assert.Equal(t, expectedOutcome.BucketName, structuredJobLogs[0].BucketName) - assert.Equal(t, expectedOutcome.ObjectName, structuredJobLogs[0].ObjectName) + structuredJobLogs := read_logs.GetJobLogsSortedByTimestamp(setup.LogFile(), s.T()) + assert.Equal(s.T(), expectedOutcome.BucketName, structuredJobLogs[0].BucketName) + assert.Equal(s.T(), expectedOutcome.ObjectName, structuredJobLogs[0].ObjectName) // We need to check that downloadedOffset is always greater than the previous downloadedOffset // and is in multiples of chunkSize. for i := 1; i < len(structuredJobLogs[0].JobEntries); i++ { offsetDiff := structuredJobLogs[0].JobEntries[i].Offset - structuredJobLogs[0].JobEntries[i-1].Offset - assert.Greater(t, offsetDiff, int64(0)) + assert.Greater(s.T(), offsetDiff, int64(0)) // This is true for all entries except last one. // Will be true for last entry only if the fileSize is multiple of chunkSize. - assert.Equal(t, int64(0), offsetDiff%s.chunkSize) + assert.Equal(s.T(), int64(0), offsetDiff%s.chunkSize) } // Validate that last downloadedOffset is same as fileSize. - assert.Equal(t, fileSize, structuredJobLogs[0].JobEntries[len(structuredJobLogs[0].JobEntries)-1].Offset) + assert.Equal(s.T(), fileSize, structuredJobLogs[0].JobEntries[len(structuredJobLogs[0].JobEntries)-1].Offset) } -func (s *jobChunkTest) TestJobChunkSizeForMultipleFileReads(t *testing.T) { +func (s *jobChunkTest) TestJobChunkSizeForMultipleFileReads() { var fileSize int64 = 16 * util.MiB var testFileNames [2]string var expectedOutcome [2]*Expected - testFileNames[0] = setupFileInTestDir(s.ctx, s.storageClient, testDirName, fileSize, t) - testFileNames[1] = setupFileInTestDir(s.ctx, s.storageClient, testDirName, fileSize, t) + testFileNames[0] = setupFileInTestDir(s.ctx, s.storageClient, fileSize, s.T()) + testFileNames[1] = setupFileInTestDir(s.ctx, s.storageClient, fileSize, s.T()) // Read 2 files in parallel. var wg sync.WaitGroup - for i := 0; i < 2; i++ { + for i := range 2 { wg.Add(1) i := i go func() { defer wg.Done() - expectedOutcome[i] = readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileNames[i], fileSize, false, t) + expectedOutcome[i] = readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileNames[i], fileSize, false, s.T()) }() } wg.Wait() // Parse the log file and validate cache hit or miss from the structured logs. - structuredJobLogs := read_logs.GetJobLogsSortedByTimestamp(setup.LogFile(), t) - require.Equal(t, 2, len(structuredJobLogs)) + structuredJobLogs := read_logs.GetJobLogsSortedByTimestamp(setup.LogFile(), s.T()) + require.Equal(s.T(), 2, len(structuredJobLogs)) // Goroutine execution order isn't guaranteed. // If the object name in expected outcome doesn't align with the logs, swap // the expected outcome objects and file names at positions 0 and 1. @@ -136,23 +127,23 @@ func (s *jobChunkTest) TestJobChunkSizeForMultipleFileReads(t *testing.T) { testFileNames[0], testFileNames[1] = testFileNames[1], testFileNames[0] } - for fileIndex := 0; fileIndex < 2; fileIndex++ { - assert.Equal(t, expectedOutcome[fileIndex].BucketName, structuredJobLogs[fileIndex].BucketName) - assert.Equal(t, expectedOutcome[fileIndex].ObjectName, structuredJobLogs[fileIndex].ObjectName) + for fileIndex := range 2 { + assert.Equal(s.T(), expectedOutcome[fileIndex].BucketName, structuredJobLogs[fileIndex].BucketName) + assert.Equal(s.T(), expectedOutcome[fileIndex].ObjectName, structuredJobLogs[fileIndex].ObjectName) // We need to check that downloadedOffset is always greater than the previous downloadedOffset // and is in multiples of chunkSize. entriesLen := len(structuredJobLogs[fileIndex].JobEntries) for entryIndex := 1; entryIndex < entriesLen; entryIndex++ { offsetDiff := structuredJobLogs[fileIndex].JobEntries[entryIndex].Offset - structuredJobLogs[fileIndex].JobEntries[entryIndex-1].Offset - assert.Greater(t, offsetDiff, int64(0)) + assert.Greater(s.T(), offsetDiff, int64(0)) // This is true for all entries except last one. // Will be true for last entry only if the fileSize is multiple of chunkSize. - assert.Equal(t, int64(0), offsetDiff%s.chunkSize) + assert.Equal(s.T(), int64(0), offsetDiff%s.chunkSize) } // Validate that last downloadedOffset is same as fileSize. - assert.Equal(t, fileSize, structuredJobLogs[fileIndex].JobEntries[entriesLen-1].Offset) + assert.Equal(s.T(), fileSize, structuredJobLogs[fileIndex].JobEntries[entriesLen-1].Offset) } } @@ -160,90 +151,31 @@ func (s *jobChunkTest) TestJobChunkSizeForMultipleFileReads(t *testing.T) { // Test Function (Runs once before all tests) //////////////////////////////////////////////////////////////////////// -func TestJobChunkTest(t *testing.T) { - ts := &jobChunkTest{ctx: context.Background()} - // Create storage client before running tests. - closeStorageClient := client.CreateStorageClientWithCancel(&ts.ctx, &ts.storageClient) - defer func() { - err := closeStorageClient() - if err != nil { - t.Errorf("closeStorageClient failed: %v", err) - } - }() - - // Run tests for mounted directory if the flag is set. - if setup.AreBothMountedDirectoryAndTestBucketFlagsSet() { - test_setup.RunTests(t, ts) +func (s *jobChunkTest) runTests(t *testing.T) { + t.Helper() + // Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, s) return } - var cacheSizeMB int64 = 48 + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, s.flags = range flagsSet { + log.Printf("Running tests with flags: %s", s.flags) + suite.Run(t, s) + } +} +func TestJobChunkTest(t *testing.T) { // Tests to validate chunk size when read cache parallel downloads are disabled. - var chunkSizeForReadCache int64 = 8 - ts.flags = []string{"--config-file=" + createConfigFile(&gcsfuseTestFlags{cacheSize: cacheSizeMB, cacheFileForRangeRead: true, fileName: configFileName, enableParallelDownloads: false, enableODirect: false, cacheDirPath: getDefaultCacheDirPathForTests(), clientProtocol: http1ClientProtocol})} - ts.chunkSize = chunkSizeForReadCache * util.MiB - log.Printf("Running tests with flags: %s", ts.flags) - test_setup.RunTests(t, ts) - - // Tests to validate chunk size when read cache parallel downloads are disabled with grpc client protocol. - ts.flags = []string{"--config-file=" + createConfigFile(&gcsfuseTestFlags{cacheSize: cacheSizeMB, cacheFileForRangeRead: true, fileName: configFileName, enableParallelDownloads: false, enableODirect: false, cacheDirPath: getDefaultCacheDirPathForTests(), clientProtocol: grpcClientProtocol})} - ts.chunkSize = chunkSizeForReadCache * util.MiB - log.Printf("Running tests with flags: %s", ts.flags) - test_setup.RunTests(t, ts) - - // Tests to validate chunk size when read cache parallel downloads are enabled - // with unlimited max parallel downloads. - ts.flags = []string{"--config-file=" + - createConfigFileForJobChunkTest(cacheSizeMB, false, "unlimitedMaxParallelDownloads", parallelDownloadsPerFile, maxParallelDownloads, downloadChunkSizeMB, http1ClientProtocol)} - ts.chunkSize = downloadChunkSizeMB * util.MiB - log.Printf("Running tests with flags: %s", ts.flags) - test_setup.RunTests(t, ts) - - // Tests to validate chunk size when read cache parallel downloads are enabled - // with unlimited max parallel downloads with grpc enabled. - ts.flags = []string{"--config-file=" + - createConfigFileForJobChunkTest(cacheSizeMB, false, "unlimitedMaxParallelDownloads", parallelDownloadsPerFile, maxParallelDownloads, downloadChunkSizeMB, grpcClientProtocol)} - ts.chunkSize = downloadChunkSizeMB * util.MiB - log.Printf("Running tests with flags: %s", ts.flags) - test_setup.RunTests(t, ts) - - // Tests to validate chunk size when read cache parallel downloads are enabled - // with go-routines not limited by max parallel downloads. - parallelDownloadsPerFile := 4 - maxParallelDownloads := 9 // maxParallelDownloads > parallelDownloadsPerFile * number of files being accessed concurrently. - downloadChunkSizeMB := 4 - ts.flags = []string{"--config-file=" + - createConfigFileForJobChunkTest(cacheSizeMB, false, "limitedMaxParallelDownloadsNotEffectingChunkSize", parallelDownloadsPerFile, maxParallelDownloads, downloadChunkSizeMB, http1ClientProtocol)} - ts.chunkSize = int64(downloadChunkSizeMB) * util.MiB - log.Printf("Running tests with flags: %s", ts.flags) - test_setup.RunTests(t, ts) - - // Tests to validate chunk size when read cache parallel downloads are enabled - // with go-routines not limited by max parallel downloads with grpc enabled. - ts.flags = []string{"--config-file=" + - createConfigFileForJobChunkTest(cacheSizeMB, false, "limitedMaxParallelDownloadsNotEffectingChunkSize", parallelDownloadsPerFile, maxParallelDownloads, downloadChunkSizeMB, grpcClientProtocol)} - ts.chunkSize = int64(downloadChunkSizeMB) * util.MiB - log.Printf("Running tests with flags: %s", ts.flags) - test_setup.RunTests(t, ts) - - // Tests to validate chunk size when read cache parallel downloads are enabled - // with go-routines limited by max parallel downloads. - parallelDownloadsPerFile = 4 - maxParallelDownloads = 2 - downloadChunkSizeMB = 4 - ts.flags = []string{"--config-file=" + - createConfigFileForJobChunkTest(cacheSizeMB, false, "limitedMaxParallelDownloadsEffectingChunkSize", parallelDownloadsPerFile, maxParallelDownloads, downloadChunkSizeMB, http1ClientProtocol)} - ts.chunkSize = int64(downloadChunkSizeMB) * util.MiB - log.Printf("Running tests with flags: %s", ts.flags) - test_setup.RunTests(t, ts) + ts := &jobChunkTest{ctx: context.Background(), storageClient: testEnv.storageClient, chunkSize: 8 * util.MiB, baseTestName: t.Name()} + ts.runTests(t) +} +func TestJobChunkTestWithParallelDownloads(t *testing.T) { // Tests to validate chunk size when read cache parallel downloads are enabled - // with go-routines limited by max parallel downloads with grpc enabled. - ts.flags = []string{"--config-file=" + - createConfigFileForJobChunkTest(cacheSizeMB, false, "limitedMaxParallelDownloadsEffectingChunkSize", parallelDownloadsPerFile, maxParallelDownloads, downloadChunkSizeMB, grpcClientProtocol)} - ts.chunkSize = int64(downloadChunkSizeMB) * util.MiB - log.Printf("Running tests with flags: %s", ts.flags) - test_setup.RunTests(t, ts) - + // The flag set combination is chosen in such a way that chunk size remains 4. + ts := &jobChunkTest{ctx: context.Background(), storageClient: testEnv.storageClient, chunkSize: 4 * util.MiB, baseTestName: t.Name()} + ts.runTests(t) } diff --git a/tools/integration_tests/read_cache/local_modification_test.go b/tools/integration_tests/read_cache/local_modification_test.go index a03e98b92f..685dd1dcc5 100644 --- a/tools/integration_tests/read_cache/local_modification_test.go +++ b/tools/integration_tests/read_cache/local_modification_test.go @@ -4,28 +4,30 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package read_cache import ( "context" "log" + "os" "path" "testing" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/log_parser/json_parser/read_logs" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/test_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" ) // ////////////////////////////////////////////////////////////////////// @@ -35,45 +37,76 @@ type localModificationTest struct { flags []string storageClient *storage.Client ctx context.Context + baseTestName string + suite.Suite +} + +func (s *localModificationTest) SetupSuite() { + setupLogFileAndCacheDir(s.baseTestName) + mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient) } -func (s *localModificationTest) Setup(t *testing.T) { - setupForMountedDirectoryTests() +func (s *localModificationTest) SetupTest() { + //Truncate log file created. + err := os.Truncate(testEnv.cfg.LogFile, 0) + require.NoError(s.T(), err) // Clean up the cache directory path as gcsfuse don't clean up on mounting. - operations.RemoveDir(cacheDirPath) - mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient, testDirName) + operations.RemoveDir(testEnv.cacheDirPath) + testEnv.testDirPath = client.SetupUniqueTestDirectory(s.ctx, s.storageClient, testDirPrefix) +} + +func (s *localModificationTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) } -func (s *localModificationTest) Teardown(t *testing.T) { - setup.UnmountGCSFuseAndDeleteLogFile(rootDir) +func (s *localModificationTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) } //////////////////////////////////////////////////////////////////////// // Test scenarios //////////////////////////////////////////////////////////////////////// -func (s *localModificationTest) TestReadAfterLocalGCSFuseWriteIsCacheMiss(t *testing.T) { - testFileName := testDirName + setup.GenerateRandomString(testFileNameSuffixLength) - operations.CreateFileOfSize(fileSize, path.Join(testDirPath, testFileName), t) +func (s *localModificationTest) TestReadAfterLocalGCSFuseWriteIsCacheMiss() { + testFileName := testDirPrefix + setup.GenerateRandomString(testFileNameSuffixLength) + operations.CreateFileOfSize(fileSize, path.Join(testEnv.testDirPath, testFileName), s.T()) // Read file 1st time. - expectedOutcome1 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, true, t) + expectedOutcome1 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, true, s.T()) // Append data in the same file to change object generation. smallContent, err := operations.GenerateRandomData(smallContentSize) if err != nil { - t.Errorf("TestReadAfterLocalGCSFuseWriteIsCacheMiss: could not generate randomm data: %v", err) + s.T().Errorf("TestReadAfterLocalGCSFuseWriteIsCacheMiss: could not generate randomm data: %v", err) } - err = operations.WriteFileInAppendMode(path.Join(testDirPath, testFileName), string(smallContent)) + err = operations.WriteFileInAppendMode(path.Join(testEnv.testDirPath, testFileName), string(smallContent)) if err != nil { - t.Errorf("Error in appending data in file: %v", err) + s.T().Errorf("Error in appending data in file: %v", err) } - // Read file 2nd time. - expectedOutcome2 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize+smallContentSize, true, t) + if !setup.IsZonalBucketRun() { + // Read file 2nd time. + expectedOutcome2 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize+smallContentSize, true, s.T()) - // Parse the log file and validate cache hit or miss from the structured logs. - structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t) - validate(expectedOutcome1, structuredReadLogs[0], true, false, chunksRead, t) - validate(expectedOutcome2, structuredReadLogs[1], true, false, chunksRead+1, t) + // Parse the log file and validate cache hit or miss from the structured logs. + structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + validate(expectedOutcome1, structuredReadLogs[0], true, false, chunksRead, s.T()) + validate(expectedOutcome2, structuredReadLogs[1], true, false, chunksRead+1, s.T()) + } else { + // Read file 2nd time. + expectedOutcome2 := readFileAndGetExpectedOutcome(testEnv.testDirPath, testFileName, true, zeroOffset, s.T()) + expectedPathOfCachedFile := getCachedFilePath(testFileName) + fileInfo, err := operations.StatFile(expectedPathOfCachedFile) + + // Validate cache size is within limit and cache file size is same as the original file size. + // This is because for unfinalized objects, we do not trigger a new download job due to appends, + // we simply fall back to another reader to serve newer reads (which are not cached). + validateCacheSizeWithinLimit(cacheCapacityInMB, s.T()) + assert.NoError(s.T(), err) + assert.Equal(s.T(), int64(fileSize), (*fileInfo).Size()) + + // Parse the log file and validate cache hit or miss from the structured logs. + structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + validate(expectedOutcome2, structuredReadLogs[1], true, true, chunksRead+1, s.T()) + } } //////////////////////////////////////////////////////////////////////// @@ -81,52 +114,18 @@ func (s *localModificationTest) TestReadAfterLocalGCSFuseWriteIsCacheMiss(t *tes //////////////////////////////////////////////////////////////////////// func TestLocalModificationTest(t *testing.T) { - ts := &localModificationTest{ctx: context.Background()} - // Create storage client before running tests. - closeStorageClient := client.CreateStorageClientWithCancel(&ts.ctx, &ts.storageClient) - defer func() { - err := closeStorageClient() - if err != nil { - t.Errorf("closeStorageClient failed: %v", err) - } - }() - - // Run tests for mounted directory if the flag is set. - if setup.AreBothMountedDirectoryAndTestBucketFlagsSet() { - test_setup.RunTests(t, ts) + ts := &localModificationTest{ctx: context.Background(), storageClient: testEnv.storageClient, baseTestName: t.Name()} + + // Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) return } - // Define flag set to run the tests. - flagsSet := []gcsfuseTestFlags{ - { - cliFlags: []string{"--implicit-dirs"}, - cacheSize: cacheCapacityInMB, - cacheFileForRangeRead: false, - fileName: configFileName, - enableParallelDownloads: false, - enableODirect: false, - cacheDirPath: getDefaultCacheDirPathForTests(), - }, - { - cliFlags: nil, - cacheSize: cacheCapacityInMB, - cacheFileForRangeRead: false, - fileName: configFileNameForParallelDownloadTests, - enableParallelDownloads: true, - enableODirect: false, - cacheDirPath: getDefaultCacheDirPathForTests(), - }, - } - flagsSet = appendClientProtocolConfigToFlagSet(flagsSet) - // Run tests. - for _, flags := range flagsSet { - configFilePath := createConfigFile(&flags) - ts.flags = []string{"--config-file=" + configFilePath} - if flags.cliFlags != nil { - ts.flags = append(ts.flags, flags.cliFlags...) - } + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { log.Printf("Running tests with flags: %s", ts.flags) - test_setup.RunTests(t, ts) + suite.Run(t, ts) } } diff --git a/tools/integration_tests/read_cache/range_read_test.go b/tools/integration_tests/read_cache/range_read_test.go index b9f01d1e5b..136ab46cd7 100644 --- a/tools/integration_tests/read_cache/range_read_test.go +++ b/tools/integration_tests/read_cache/range_read_test.go @@ -4,28 +4,30 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package read_cache import ( "context" + "fmt" "log" + "os" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/log_parser/json_parser/read_logs" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/test_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) //////////////////////////////////////////////////////////////////////// @@ -37,118 +39,115 @@ type rangeReadTest struct { storageClient *storage.Client ctx context.Context isParallelDownloadsEnabled bool + baseTestName string + suite.Suite +} + +func (s *rangeReadTest) SetupSuite() { + setupLogFileAndCacheDir(s.baseTestName) + mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient) } -func (s *rangeReadTest) Setup(t *testing.T) { - setupForMountedDirectoryTests() +func (s *rangeReadTest) SetupTest() { + //Truncate log file created. + err := os.Truncate(testEnv.cfg.LogFile, 0) + require.NoError(s.T(), err) // Clean up the cache directory path as gcsfuse don't clean up on mounting. - operations.RemoveDir(cacheDirPath) - mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient, testDirName) + operations.RemoveDir(testEnv.cacheDirPath) + testEnv.testDirPath = client.SetupUniqueTestDirectory(s.ctx, s.storageClient, testDirPrefix) } -func (s *rangeReadTest) Teardown(t *testing.T) { - setup.UnmountGCSFuseAndDeleteLogFile(rootDir) +func (s *rangeReadTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +func (s *rangeReadTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) } //////////////////////////////////////////////////////////////////////// // Test scenarios //////////////////////////////////////////////////////////////////////// -func (s *rangeReadTest) TestRangeReadsWithinReadChunkSize(t *testing.T) { +func (s *rangeReadTest) TestRangeReadsWithinReadChunkSize() { if s.isParallelDownloadsEnabled { // This test verifies that the reads are all cache hit within a downloaded chunk. // However, with parallel downloads, we cannot guarantee this behavior, so // we skip this test when parallel downloads are enabled. - t.SkipNow() + s.T().SkipNow() } - testFileName := setupFileInTestDir(s.ctx, s.storageClient, testDirName, largeFileSize, t) + testFileName := setupFileInTestDir(s.ctx, s.storageClient, largeFileSize, s.T()) - expectedOutcome1 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, zeroOffset, t) - expectedOutcome2 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, offsetForRangeReadWithin8MB, t) + expectedOutcome1 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, zeroOffset, s.T()) + expectedOutcome2 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, offsetForRangeReadWithin8MB, s.T()) - structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t) - validate(expectedOutcome1, structuredReadLogs[0], true, false, 1, t) - validate(expectedOutcome2, structuredReadLogs[1], false, true, 1, t) + structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + validate(expectedOutcome1, structuredReadLogs[0], true, false, 1, s.T()) + validate(expectedOutcome2, structuredReadLogs[1], false, true, 1, s.T()) } -func (s *rangeReadTest) TestRangeReadsBeyondReadChunkSizeWithFileCached(t *testing.T) { - testFileName := setupFileInTestDir(s.ctx, s.storageClient, testDirName, largeFileSize, t) - - expectedOutcome1 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, zeroOffset, t) - validateFileInCacheDirectory(testFileName, largeFileSize, ctx, s.storageClient, t) - expectedOutcome2 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, offset10MiB, t) - - structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t) - validate(expectedOutcome1, structuredReadLogs[0], true, false, 1, t) - validate(expectedOutcome2, structuredReadLogs[1], false, true, 1, t) - validateCacheSizeWithinLimit(largeFileCacheCapacity, t) +func (s *rangeReadTest) TestRangeReadsBeyondReadChunkSizeWithFileCached() { + testFileName := setupFileInTestDir(s.ctx, s.storageClient, largeFileSize, s.T()) + + // Read first chunk (0-128KB) to trigger the background file cache download job. + expectedOutcome1 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, zeroOffset, s.T()) + + // Wait until the background job downloads both the first 8MiB chunk and the second 8MiB-15MiB chunk. + // This ensures the read at 10MiB is always a cache hit, making the test deterministic. + s.T().Logf("Waiting for file cache Job with data reaching %d bytes", largeFileSize) + JobLog := operations.RetryUntil(s.ctx, s.T(), retryFrequency, retryDuration, func() ([]*read_logs.Job, error) { + logs := read_logs.GetJobLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + if len(logs) == 1 { + for _, entry := range logs[0].JobEntries { + if entry.Offset >= largeFileSize { + s.T().Logf("Found file cache Job with sufficient data (offset %d): %v", entry.Offset, logs[0]) + return logs, nil + } + } + } + return nil, fmt.Errorf("expected 1 Job with an entry >= %d bytes, found %d jobs", largeFileSize, len(logs)) + }) + require.Equal(s.T(), expectedOutcome1.ObjectName, JobLog[0].ObjectName) + + // Read the second chunk at offset 10MiB. This should be a cache hit since the background job + // was verified to have downloaded the second chunk (reaching up to 15MiB). + expectedOutcome2 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, offset10MiB, s.T()) + + // Validate results for both reads, verifying the second one is a cache hit. + structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + require.Len(s.T(), structuredReadLogs, 2, "Should have exactly 2 read records in logs") + validate(expectedOutcome1, structuredReadLogs[0], true, false, 1, s.T()) + validate(expectedOutcome2, structuredReadLogs[1], false, true, 1, s.T()) + + validateFileInCacheDirectory(testFileName, largeFileSize, testEnv.ctx, s.storageClient, s.T()) + validateCacheSizeWithinLimit(largeFileCacheCapacity, s.T()) } //////////////////////////////////////////////////////////////////////// // Test Function (Runs once before all tests) //////////////////////////////////////////////////////////////////////// -func TestRangeReadTest(t *testing.T) { - ts := &rangeReadTest{ctx: context.Background()} - // Create storage client before running tests. - closeStorageClient := client.CreateStorageClientWithCancel(&ts.ctx, &ts.storageClient) - defer func() { - err := closeStorageClient() - if err != nil { - t.Errorf("closeStorageClient failed: %v", err) - } - }() - - // Run tests for mounted directory if the flag is set. - if setup.AreBothMountedDirectoryAndTestBucketFlagsSet() { - test_setup.RunTests(t, ts) +func runTests(t *testing.T, ts *rangeReadTest) { + // Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) return } - // Run tests with parallel downloads disabled. - flagsSet := []gcsfuseTestFlags{ - { - cliFlags: []string{"--implicit-dirs"}, - cacheSize: largeFileCacheCapacity, - cacheFileForRangeRead: false, - fileName: configFileName + "1", - enableParallelDownloads: false, - enableODirect: false, - cacheDirPath: getDefaultCacheDirPathForTests(), - }, - } - flagsSet = appendClientProtocolConfigToFlagSet(flagsSet) - for _, flags := range flagsSet { - configFilePath := createConfigFile(&flags) - ts.flags = []string{"--config-file=" + configFilePath} - if flags.cliFlags != nil { - ts.flags = append(ts.flags, flags.cliFlags...) - } + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { log.Printf("Running tests with flags: %s", ts.flags) - test_setup.RunTests(t, ts) + suite.Run(t, ts) } +} - // Run tests with parallel downloads enabled. - flagsSet = []gcsfuseTestFlags{ - { - cliFlags: nil, - cacheSize: largeFileCacheCapacity, - cacheFileForRangeRead: true, - fileName: configFileName + "2", - enableParallelDownloads: false, - enableODirect: false, - cacheDirPath: getDefaultCacheDirPathForTests(), - }, - } - flagsSet = appendClientProtocolConfigToFlagSet(flagsSet) - for _, flags := range flagsSet { - configFilePath := createConfigFile(&flags) - ts.flags = []string{"--config-file=" + configFilePath} - if flags.cliFlags != nil { - ts.flags = append(ts.flags, flags.cliFlags...) - } - ts.isParallelDownloadsEnabled = true - log.Printf("Running tests with flags: %s", ts.flags) - test_setup.RunTests(t, ts) - } +func TestRangeReadTest(t *testing.T) { + ts := &rangeReadTest{ctx: context.Background(), storageClient: testEnv.storageClient, baseTestName: t.Name(), isParallelDownloadsEnabled: false} + runTests(t, ts) +} + +func TestRangeReadWithParallelDownloadsTest(t *testing.T) { + ts := &rangeReadTest{ctx: context.Background(), storageClient: testEnv.storageClient, baseTestName: t.Name(), isParallelDownloadsEnabled: true} + runTests(t, ts) } diff --git a/tools/integration_tests/read_cache/read_only_test.go b/tools/integration_tests/read_cache/read_only_test.go index f79458a5e5..5974090dac 100644 --- a/tools/integration_tests/read_cache/read_only_test.go +++ b/tools/integration_tests/read_cache/read_only_test.go @@ -4,27 +4,29 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package read_cache import ( "context" "log" + "os" + "path" "testing" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/log_parser/json_parser/read_logs" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/test_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" ) //////////////////////////////////////////////////////////////////////// @@ -35,25 +37,38 @@ type readOnlyTest struct { flags []string storageClient *storage.Client ctx context.Context + baseTestName string + suite.Suite +} + +func (s *readOnlyTest) SetupSuite() { + setupLogFileAndCacheDir(s.baseTestName) + mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient) } -func (s *readOnlyTest) Setup(t *testing.T) { - setupForMountedDirectoryTests() +func (s *readOnlyTest) SetupTest() { + //Truncate log file created. + err := os.Truncate(testEnv.cfg.LogFile, 0) + require.NoError(s.T(), err) // Clean up the cache directory path as gcsfuse don't clean up on mounting. - operations.RemoveDir(cacheDirPath) - mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient, testDirName) + operations.RemoveDir(testEnv.cacheDirPath) + testEnv.testDirPath = client.SetupUniqueTestDirectory(s.ctx, s.storageClient, testDirPrefix) } -func (s *readOnlyTest) Teardown(t *testing.T) { - setup.UnmountGCSFuseAndDeleteLogFile(rootDir) +func (s *readOnlyTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +func (s *readOnlyTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) } //////////////////////////////////////////////////////////////////////// // Helper functions //////////////////////////////////////////////////////////////////////// -func readMultipleFiles(numFiles int, ctx context.Context, storageClient *storage.Client, fileNames []string, fileSize int64, t *testing.T) (expectedOutcome []*Expected) { - for i := 0; i < numFiles; i++ { +func readMultipleFiles(numFiles int, ctx context.Context, storageClient *storage.Client, fileNames []string, t *testing.T) (expectedOutcome []*Expected) { + for i := range numFiles { expectedOutcome = append(expectedOutcome, readFileAndValidateCacheWithGCS(ctx, storageClient, fileNames[i], fileSize, true, t)) } return expectedOutcome @@ -71,74 +86,74 @@ func validateCacheOfMultipleObjectsUsingStructuredLogs(startIndex int, numFiles // Test scenarios //////////////////////////////////////////////////////////////////////// -func (s *readOnlyTest) TestSecondSequentialReadIsCacheHit(t *testing.T) { - testFileName := setupFileInTestDir(s.ctx, s.storageClient, testDirName, fileSize, t) +func (s *readOnlyTest) TestSecondSequentialReadIsCacheHit() { + testFileName := setupFileInTestDir(s.ctx, s.storageClient, fileSize, s.T()) // Read file 1st time. - expectedOutcome1 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, true, t) + expectedOutcome1 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, true, s.T()) // Read file 2nd time. - expectedOutcome2 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, true, t) + expectedOutcome2 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, true, s.T()) // Parse the log file and validate cache hit or miss from the structured logs. - structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t) - validate(expectedOutcome1, structuredReadLogs[0], true, false, chunksRead, t) - validate(expectedOutcome2, structuredReadLogs[1], true, true, chunksRead, t) + structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + validate(expectedOutcome1, structuredReadLogs[0], true, false, chunksRead, s.T()) + validate(expectedOutcome2, structuredReadLogs[1], true, true, chunksRead, s.T()) } -func (s *readOnlyTest) TestReadFileSequentiallyLargerThanCacheCapacity(t *testing.T) { +func (s *readOnlyTest) TestReadFileSequentiallyLargerThanCacheCapacity() { // Set up a file in test directory of size more than cache capacity. - client.SetupFileInTestDirectory(s.ctx, s.storageClient, testDirName, - largeFileName, largeFileSize, t) + fileName := setup.GenerateRandomString(7) + client.SetupFileInTestDirectory(s.ctx, s.storageClient, path.Base(testEnv.testDirPath), fileName, largeFileSize, s.T()) // Read file 1st time. - expectedOutcome1 := readFileAndValidateFileIsNotCached(s.ctx, s.storageClient, largeFileName, true, zeroOffset, t) + expectedOutcome1 := readFileAndValidateFileIsNotCached(s.ctx, s.storageClient, fileName, true, zeroOffset, s.T()) // Read file 2nd time. - expectedOutcome2 := readFileAndValidateFileIsNotCached(s.ctx, s.storageClient, largeFileName, true, zeroOffset, t) + expectedOutcome2 := readFileAndValidateFileIsNotCached(s.ctx, s.storageClient, fileName, true, zeroOffset, s.T()) // Parse the log file and validate cache hit or miss from the structured logs. - structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t) - validate(expectedOutcome1, structuredReadLogs[0], true, false, largeFileChunksRead, t) - validate(expectedOutcome2, structuredReadLogs[1], true, false, largeFileChunksRead, t) + structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + validate(expectedOutcome1, structuredReadLogs[0], true, false, largeFileChunksRead, s.T()) + validate(expectedOutcome2, structuredReadLogs[1], true, false, largeFileChunksRead, s.T()) } -func (s *readOnlyTest) TestReadFileRandomlyLargerThanCacheCapacity(t *testing.T) { +func (s *readOnlyTest) TestReadFileRandomlyLargerThanCacheCapacity() { // Set up a file in test directory of size more than cache capacity. - client.SetupFileInTestDirectory(s.ctx, s.storageClient, testDirName, - largeFileName, largeFileSize, t) + fileName := setup.GenerateRandomString(7) + client.SetupFileInTestDirectory(s.ctx, s.storageClient, path.Base(testEnv.testDirPath), fileName, largeFileSize, s.T()) // Do a random read on file. - expectedOutcome1 := readFileAndValidateFileIsNotCached(s.ctx, s.storageClient, largeFileName, false, randomReadOffset, t) + expectedOutcome1 := readFileAndValidateFileIsNotCached(s.ctx, s.storageClient, fileName, false, randomReadOffset, s.T()) // Read file sequentially again. - expectedOutcome2 := readFileAndValidateFileIsNotCached(s.ctx, s.storageClient, largeFileName, true, zeroOffset, t) + expectedOutcome2 := readFileAndValidateFileIsNotCached(s.ctx, s.storageClient, fileName, true, zeroOffset, s.T()) // Parse the log file and validate cache hit or miss from the structured logs. - structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t) - validate(expectedOutcome1, structuredReadLogs[0], false, false, 1, t) - validate(expectedOutcome2, structuredReadLogs[1], true, false, largeFileChunksRead, t) + structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + validate(expectedOutcome1, structuredReadLogs[0], false, false, 1, s.T()) + validate(expectedOutcome2, structuredReadLogs[1], true, false, largeFileChunksRead, s.T()) } -func (s *readOnlyTest) TestReadMultipleFilesMoreThanCacheLimit(t *testing.T) { - fileNames := client.CreateNFilesInDir(s.ctx, s.storageClient, NumberOfFilesMoreThanCacheLimit, testFileName, fileSize, testDirName, t) +func (s *readOnlyTest) TestReadMultipleFilesMoreThanCacheLimit() { + fileNames := client.CreateNFilesInDir(s.ctx, s.storageClient, NumberOfFilesMoreThanCacheLimit, testFileName, fileSize, path.Base(testEnv.testDirPath), s.T()) - expectedOutcome := readMultipleFiles(NumberOfFilesMoreThanCacheLimit, s.ctx, s.storageClient, fileNames, fileSize, t) - expectedOutcome = append(expectedOutcome, readMultipleFiles(NumberOfFilesMoreThanCacheLimit, s.ctx, s.storageClient, fileNames, fileSize, t)...) + expectedOutcome := readMultipleFiles(NumberOfFilesMoreThanCacheLimit, s.ctx, s.storageClient, fileNames, s.T()) + expectedOutcome = append(expectedOutcome, readMultipleFiles(NumberOfFilesMoreThanCacheLimit, s.ctx, s.storageClient, fileNames, s.T())...) // Parse the log file and validate cache hit or miss from the structured logs. - structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t) - validateCacheOfMultipleObjectsUsingStructuredLogs(0, NumberOfFilesMoreThanCacheLimit, expectedOutcome, structuredReadLogs, false, t) - validateCacheOfMultipleObjectsUsingStructuredLogs(NumberOfFilesMoreThanCacheLimit, NumberOfFilesMoreThanCacheLimit, expectedOutcome, structuredReadLogs, false, t) + structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + validateCacheOfMultipleObjectsUsingStructuredLogs(0, NumberOfFilesMoreThanCacheLimit, expectedOutcome, structuredReadLogs, false, s.T()) + validateCacheOfMultipleObjectsUsingStructuredLogs(NumberOfFilesMoreThanCacheLimit, NumberOfFilesMoreThanCacheLimit, expectedOutcome, structuredReadLogs, false, s.T()) } -func (s *readOnlyTest) TestReadMultipleFilesWithinCacheLimit(t *testing.T) { - fileNames := client.CreateNFilesInDir(s.ctx, s.storageClient, NumberOfFilesWithinCacheLimit, testFileName, fileSize, testDirName, t) +func (s *readOnlyTest) TestReadMultipleFilesWithinCacheLimit() { + fileNames := client.CreateNFilesInDir(s.ctx, s.storageClient, NumberOfFilesWithinCacheLimit, testFileName, fileSize, path.Base(testEnv.testDirPath), s.T()) - expectedOutcome := readMultipleFiles(NumberOfFilesWithinCacheLimit, s.ctx, s.storageClient, fileNames, fileSize, t) - expectedOutcome = append(expectedOutcome, readMultipleFiles(NumberOfFilesWithinCacheLimit, s.ctx, s.storageClient, fileNames, fileSize, t)...) + expectedOutcome := readMultipleFiles(NumberOfFilesWithinCacheLimit, s.ctx, s.storageClient, fileNames, s.T()) + expectedOutcome = append(expectedOutcome, readMultipleFiles(NumberOfFilesWithinCacheLimit, s.ctx, s.storageClient, fileNames, s.T())...) // Parse the log file and validate cache hit or miss from the structured logs. - structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t) - validateCacheOfMultipleObjectsUsingStructuredLogs(0, NumberOfFilesWithinCacheLimit, expectedOutcome, structuredReadLogs, false, t) - validateCacheOfMultipleObjectsUsingStructuredLogs(NumberOfFilesWithinCacheLimit, NumberOfFilesWithinCacheLimit, expectedOutcome, structuredReadLogs, true, t) + structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + validateCacheOfMultipleObjectsUsingStructuredLogs(0, NumberOfFilesWithinCacheLimit, expectedOutcome, structuredReadLogs, false, s.T()) + validateCacheOfMultipleObjectsUsingStructuredLogs(NumberOfFilesWithinCacheLimit, NumberOfFilesWithinCacheLimit, expectedOutcome, structuredReadLogs, true, s.T()) } //////////////////////////////////////////////////////////////////////// @@ -146,66 +161,18 @@ func (s *readOnlyTest) TestReadMultipleFilesWithinCacheLimit(t *testing.T) { //////////////////////////////////////////////////////////////////////// func TestReadOnlyTest(t *testing.T) { - ts := &readOnlyTest{ctx: context.Background()} - // Create storage client before running tests. - closeStorageClient := client.CreateStorageClientWithCancel(&ts.ctx, &ts.storageClient) - defer func() { - err := closeStorageClient() - if err != nil { - t.Errorf("closeStorageClient failed: %v", err) - } - }() - - // Run tests for mounted directory if the flag is set. - if setup.AreBothMountedDirectoryAndTestBucketFlagsSet() { - test_setup.RunTests(t, ts) + ts := &readOnlyTest{ctx: context.Background(), storageClient: testEnv.storageClient, baseTestName: t.Name()} + + // Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) return } - // Define flag set to run the tests. - flagsSet := []gcsfuseTestFlags{ - { - cliFlags: []string{"--implicit-dirs"}, - cacheSize: cacheCapacityInMB, - cacheFileForRangeRead: true, - fileName: configFileName, - enableParallelDownloads: false, - cacheDirPath: getDefaultCacheDirPathForTests(), - }, - { - cliFlags: nil, - cacheSize: cacheCapacityInMB, - cacheFileForRangeRead: false, - fileName: configFileNameForParallelDownloadTests, - enableParallelDownloads: true, - cacheDirPath: getDefaultCacheDirPathForTests(), - }, - { - cliFlags: []string{"--implicit-dirs", "--o=ro"}, - cacheSize: cacheCapacityInMB, - cacheFileForRangeRead: true, - fileName: configFileName, - enableParallelDownloads: false, - cacheDirPath: getDefaultCacheDirPathForTests(), - }, - { - cliFlags: []string{"--o=ro"}, - cacheSize: cacheCapacityInMB, - cacheFileForRangeRead: false, - fileName: configFileNameForParallelDownloadTests, - enableParallelDownloads: true, - cacheDirPath: getDefaultCacheDirPathForTests(), - }, - } - flagsSet = appendClientProtocolConfigToFlagSet(flagsSet) - // Run tests. - for _, flags := range flagsSet { - configFilePath := createConfigFile(&flags) - ts.flags = []string{"--config-file=" + configFilePath} - if flags.cliFlags != nil { - ts.flags = append(ts.flags, flags.cliFlags...) - } + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { log.Printf("Running tests with flags: %s", ts.flags) - test_setup.RunTests(t, ts) + suite.Run(t, ts) } } diff --git a/tools/integration_tests/read_cache/remount_test.go b/tools/integration_tests/read_cache/remount_test.go index f600eb281d..07c8d67640 100644 --- a/tools/integration_tests/read_cache/remount_test.go +++ b/tools/integration_tests/read_cache/remount_test.go @@ -4,31 +4,29 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package read_cache import ( "context" "log" + "os" "path" "testing" - "time" - - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/log_parser/json_parser/read_logs" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/dynamic_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/test_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" ) //////////////////////////////////////////////////////////////////////// @@ -39,15 +37,30 @@ type remountTest struct { flags []string storageClient *storage.Client ctx context.Context + baseTestName string + suite.Suite +} + +func (s *remountTest) SetupSuite() { + setupLogFileAndCacheDir(s.baseTestName) + mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient) } -func (s *remountTest) Setup(t *testing.T) { - operations.RemoveDir(cacheDirPath) - mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient, testDirName) +func (s *remountTest) SetupTest() { + //Truncate log file created. + err := os.Truncate(testEnv.cfg.LogFile, 0) + require.NoError(s.T(), err) + // Clean up the cache directory path as gcsfuse don't clean up on mounting. + operations.RemoveDir(testEnv.cacheDirPath) + testEnv.testDirPath = client.SetupUniqueTestDirectory(s.ctx, s.storageClient, testDirPrefix) } -func (s *remountTest) Teardown(t *testing.T) { - setup.UnmountGCSFuseAndDeleteLogFile(rootDir) +func (s *remountTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +func (s *remountTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) } //////////////////////////////////////////////////////////////////////// @@ -57,7 +70,7 @@ func (s *remountTest) Teardown(t *testing.T) { func readFileAndValidateCacheWithGCSForDynamicMount(bucketName string, ctx context.Context, storageClient *storage.Client, fileName string, checkCacheSize bool, t *testing.T) (expectedOutcome *Expected) { setup.SetDynamicBucketMounted(bucketName) defer setup.SetDynamicBucketMounted("") - testDirPath = path.Join(rootDir, bucketName, testDirName) + testEnv.testDirPath = path.Join(rootDir, bucketName, path.Base(testEnv.testDirPath)) expectedOutcome = readFileAndValidateCacheWithGCS(ctx, storageClient, fileName, fileSize, checkCacheSize, t) return expectedOutcome @@ -67,58 +80,48 @@ func readFileAndValidateCacheWithGCSForDynamicMount(bucketName string, ctx conte // Test scenarios //////////////////////////////////////////////////////////////////////// -func (s *remountTest) TestCacheIsNotReusedOnRemount(t *testing.T) { - testFileName := setupFileInTestDir(s.ctx, s.storageClient, testDirName, fileSize, t) +func (s *remountTest) TestCacheIsNotReusedOnRemount() { + testFileName := setupFileInTestDir(s.ctx, s.storageClient, fileSize, s.T()) // Run read operations on GCSFuse mount. - expectedOutcome1 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, true, t) - expectedOutcome2 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, true, t) - structuredReadLogsMount1 := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t) + expectedOutcome1 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, true, s.T()) + expectedOutcome2 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, true, s.T()) + structuredReadLogsMount1 := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) // Re-mount GCSFuse. - remountGCSFuse(s.flags, t) + remountGCSFuse(s.flags) // Run read operations again on GCSFuse mount. - expectedOutcome3 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, false, t) - expectedOutcome4 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, false, t) - structuredReadLogsMount2 := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t) - - validate(expectedOutcome1, structuredReadLogsMount1[0], true, false, chunksRead, t) - validate(expectedOutcome2, structuredReadLogsMount1[1], true, true, chunksRead, t) - validate(expectedOutcome3, structuredReadLogsMount2[0], true, false, chunksRead, t) - validate(expectedOutcome4, structuredReadLogsMount2[1], true, true, chunksRead, t) + expectedOutcome3 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, false, s.T()) + expectedOutcome4 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, false, s.T()) + structuredReadLogsMount2 := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + + validate(expectedOutcome1, structuredReadLogsMount1[0], true, false, chunksRead, s.T()) + validate(expectedOutcome2, structuredReadLogsMount1[1], true, true, chunksRead, s.T()) + validate(expectedOutcome3, structuredReadLogsMount2[0], true, false, chunksRead, s.T()) + validate(expectedOutcome4, structuredReadLogsMount2[1], true, true, chunksRead, s.T()) } -func (s *remountTest) TestCacheIsNotReusedOnDynamicRemount(t *testing.T) { - runTestsOnlyForDynamicMount(t) +func (s *remountTest) TestCacheIsNotReusedOnDynamicRemount() { + runTestsOnlyForDynamicMount(s.T()) testBucket1 := setup.TestBucket() - testFileName1 := setupFileInTestDir(s.ctx, s.storageClient, testDirName, fileSize, t) - testBucket2 := dynamic_mounting.CreateTestBucketForDynamicMounting(ctx, storageClient) - defer dynamic_mounting.DeleteTestBucketForDynamicMounting(ctx, storageClient, testBucket2) - setup.SetDynamicBucketMounted(testBucket2) - defer setup.SetDynamicBucketMounted("") - // Introducing a sleep of 10 seconds after bucket creation to address propagation delays. - time.Sleep(10 * time.Second) - client.SetupTestDirectory(s.ctx, s.storageClient, testDirName) - testFileName2 := setupFileInTestDir(s.ctx, s.storageClient, testDirName, fileSize, t) - - // Reading files in different buckets. - expectedOutcome1 := readFileAndValidateCacheWithGCSForDynamicMount(testBucket1, s.ctx, s.storageClient, testFileName1, true, t) - expectedOutcome2 := readFileAndValidateCacheWithGCSForDynamicMount(testBucket2, s.ctx, s.storageClient, testFileName2, true, t) - structuredReadLogs1 := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t) - remountGCSFuse(s.flags, t) - // Reading files in different buckets again. - expectedOutcome3 := readFileAndValidateCacheWithGCSForDynamicMount(testBucket1, s.ctx, s.storageClient, testFileName1, false, t) - expectedOutcome4 := readFileAndValidateCacheWithGCSForDynamicMount(testBucket2, s.ctx, s.storageClient, testFileName2, false, t) - // Reading same files in different buckets again without remount. - expectedOutcome5 := readFileAndValidateCacheWithGCSForDynamicMount(testBucket1, s.ctx, s.storageClient, testFileName1, false, t) - expectedOutcome6 := readFileAndValidateCacheWithGCSForDynamicMount(testBucket2, s.ctx, s.storageClient, testFileName2, false, t) - structuredReadLogs2 := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t) - - validate(expectedOutcome1, structuredReadLogs1[0], true, false, chunksRead, t) - validate(expectedOutcome2, structuredReadLogs1[1], true, false, chunksRead, t) - validate(expectedOutcome3, structuredReadLogs2[0], true, false, chunksRead, t) - validate(expectedOutcome4, structuredReadLogs2[1], true, false, chunksRead, t) - validate(expectedOutcome5, structuredReadLogs2[2], true, true, chunksRead, t) - validate(expectedOutcome6, structuredReadLogs2[3], true, true, chunksRead, t) + testFileName1 := setupFileInTestDir(s.ctx, s.storageClient, fileSize, s.T()) + + // 1. First read: This should result in a cache miss, and the file content will be cached. + expectedOutcome1 := readFileAndValidateCacheWithGCSForDynamicMount(testBucket1, s.ctx, s.storageClient, testFileName1, true, s.T()) + structuredReadLogs1 := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + // Remount GCSFuse. This should clear any in-memory cache. + remountGCSFuse(s.flags) + // 2. Second read (after remount): This should also result in a cache miss as the cache should be empty. + expectedOutcome2 := readFileAndValidateCacheWithGCSForDynamicMount(testBucket1, s.ctx, s.storageClient, testFileName1, false, s.T()) + // 3. Third read (without remount): This should result in a cache hit. + expectedOutcome3 := readFileAndValidateCacheWithGCSForDynamicMount(testBucket1, s.ctx, s.storageClient, testFileName1, false, s.T()) + structuredReadLogs2 := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + + // log1: First read -> cache miss + validate(expectedOutcome1, structuredReadLogs1[0], true, false, chunksRead, s.T()) + // log2: Second read (after remount) -> cache miss + validate(expectedOutcome2, structuredReadLogs2[0], true, false, chunksRead, s.T()) + // log3: Third read -> cache hit + validate(expectedOutcome3, structuredReadLogs2[1], true, true, chunksRead, s.T()) } //////////////////////////////////////////////////////////////////////// @@ -126,50 +129,21 @@ func (s *remountTest) TestCacheIsNotReusedOnDynamicRemount(t *testing.T) { //////////////////////////////////////////////////////////////////////// func TestRemountTest(t *testing.T) { - if setup.MountedDirectory() != "" { - t.Log("Not running remount tests for GKE environment...") - t.SkipNow() + ts := &remountTest{ + ctx: context.Background(), + storageClient: testEnv.storageClient, + baseTestName: t.Name(), } - // Define flag set to run the tests. - flagsSet := []gcsfuseTestFlags{ - { - cliFlags: []string{"--implicit-dirs"}, - cacheSize: cacheCapacityInMB, - cacheFileForRangeRead: false, - fileName: configFileName, - enableParallelDownloads: false, - enableODirect: false, - cacheDirPath: getDefaultCacheDirPathForTests(), - }, - { - cliFlags: nil, - cacheSize: cacheCapacityInMB, - cacheFileForRangeRead: false, - fileName: configFileNameForParallelDownloadTests, - enableParallelDownloads: true, - enableODirect: false, - cacheDirPath: getDefaultCacheDirPathForTests(), - }, + // Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return } - flagsSet = appendClientProtocolConfigToFlagSet(flagsSet) - // Create storage client before running tests. - ts := &remountTest{ctx: context.Background()} - closeStorageClient := client.CreateStorageClientWithCancel(&ts.ctx, &ts.storageClient) - defer func() { - err := closeStorageClient() - if err != nil { - t.Errorf("closeStorageClient failed: %v", err) - } - }() - - // Run tests. - for _, flags := range flagsSet { - configFilePath := createConfigFile(&flags) - ts.flags = []string{"--config-file=" + configFilePath} - if flags.cliFlags != nil { - ts.flags = append(ts.flags, flags.cliFlags...) - } + + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { log.Printf("Running tests with flags: %s", ts.flags) - test_setup.RunTests(t, ts) + suite.Run(t, ts) } } diff --git a/tools/integration_tests/read_cache/setup_test.go b/tools/integration_tests/read_cache/setup_test.go index 4d35d78fe9..875a073c25 100644 --- a/tools/integration_tests/read_cache/setup_test.go +++ b/tools/integration_tests/read_cache/setup_test.go @@ -19,138 +19,102 @@ import ( "log" "os" "path" - "strconv" + "strings" "testing" + "time" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/util" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/dynamic_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/only_dir_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/dynamic_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/only_dir_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" ) const ( - testDirName = "ReadCacheTest" - onlyDirMounted = "OnlyDirMountReadCache" - cacheSubDirectoryName = "gcsfuse-file-cache" - smallContentSize = 128 * util.KiB - chunkSizeToRead = 128 * util.KiB - fileSize = 3 * util.MiB - fileSizeSameAsCacheCapacity = cacheCapacityForRangeReadTestInMiB * util.MiB - fileSizeForRangeRead = 8 * util.MiB - chunksRead = fileSize / chunkSizeToRead - testFileName = "foo" - cacheCapacityInMB = 9 - NumberOfFilesWithinCacheLimit = (cacheCapacityInMB * util.MiB) / fileSize - NumberOfFilesMoreThanCacheLimit = (cacheCapacityInMB*util.MiB)/fileSize + 1 - largeFileSize = 15 * util.MiB - largeFileCacheCapacity = 15 - largeFileName = "15MBFile" - largeFileChunksRead = largeFileSize / chunkSizeToRead - chunksReadAfterUpdate = 1 - metadataCacheTTlInSec = 10 - testFileNameSuffixLength = 4 - zeroOffset = 0 - randomReadOffset = 9 * util.MiB - configFileName = "config" - configFileNameForParallelDownloadTests = "configForReadCacheWithParallelDownload" - offset5000 = 5000 - offset1000 = 1000 - offsetForRangeReadWithin8MB = 4 * util.MiB - offset10MiB = 10 * util.MiB - cacheCapacityForRangeReadTestInMiB = 50 - logFileNameForMountedDirectoryTests = "/tmp/gcsfuse_read_cache_test_logs/log.json" - parallelDownloadsPerFile = 4 - maxParallelDownloads = -1 - downloadChunkSizeMB = 4 - enableCrcCheck = true - http1ClientProtocol = "http1" - grpcClientProtocol = "grpc" + testDirPrefix = "ReadCacheTest" + onlyDirMounted = "OnlyDirMountReadCache" + cacheSubDirectoryName = "gcsfuse-file-cache" + smallContentSize = 128 * util.KiB + chunkSizeToRead = 128 * util.KiB + fileSize = 3 * util.MiB + fileSizeSameAsCacheCapacity = cacheCapacityForRangeReadTestInMiB * util.MiB + fileSizeForRangeRead = 8 * util.MiB + chunksRead = fileSize / chunkSizeToRead + testFileName = "foo" + cacheCapacityInMB = 9 + NumberOfFilesWithinCacheLimit = (cacheCapacityInMB * util.MiB) / fileSize + NumberOfFilesMoreThanCacheLimit = (cacheCapacityInMB*util.MiB)/fileSize + 1 + largeFileSize = 15 * util.MiB + largeFileCacheCapacity = 15 + largeFileChunksRead = largeFileSize / chunkSizeToRead + chunksReadAfterUpdate = 1 + metadataCacheTTlInSec = 10 + testFileNameSuffixLength = 4 + zeroOffset = 0 + randomReadOffset = 9 * util.MiB + offset5000 = 5000 + offset1000 = 1000 + offsetForRangeReadWithin8MB = 4 * util.MiB + offset10MiB = 10 * util.MiB + cacheCapacityForRangeReadTestInMiB = 50 + GKETempDir = "/gcsfuse-tmp" + retryFrequency = 5 * time.Second // Used in poll frequency for asynchronous test expecation. + retryDuration = 5 * time.Minute // Used for poll duration for asynchronous test expecation. ) var ( - testDirPath string cacheDirName string - cacheDirPath string - mountFunc func([]string) error + mountFunc func(*test_suite.TestConfig, []string) error // mount directory is where our tests run. mountDir string // root directory is the directory to be unmounted. - rootDir string - storageClient *storage.Client - ctx context.Context + rootDir string ) -type gcsfuseTestFlags struct { - cliFlags []string - cacheSize int64 - cacheFileForRangeRead bool - fileName string - enableParallelDownloads bool - enableODirect bool - cacheDirPath string - clientProtocol string +type env struct { + storageClient *storage.Client + ctx context.Context + testDirPath string + cfg *test_suite.TestConfig + bucketType string + cacheDirPath string } +var testEnv env + //////////////////////////////////////////////////////////////////////// // Helpers //////////////////////////////////////////////////////////////////////// -func setupForMountedDirectoryTests() { - if setup.MountedDirectory() != "" { - cacheDirPath = path.Join(os.TempDir(), cacheDirName) - mountDir = setup.MountedDirectory() - setup.SetLogFile(logFileNameForMountedDirectoryTests) +func setupLogFileAndCacheDir(testName string) { + var logFilePath string + testEnv.cacheDirPath = path.Join(setup.TestDir(), GKETempDir, testName) + logFilePath = path.Join(setup.TestDir(), GKETempDir, testName) + ".log" + if testEnv.cfg.GKEMountedDirectory != "" { + testEnv.cacheDirPath = path.Join(GKETempDir, testName) + mountDir = testEnv.cfg.GKEMountedDirectory + logFilePath = path.Join(GKETempDir, testName) + ".log" + if setup.ConfigFile() == "" { + // TODO: clean this up when GKE test migration completes. + logFilePath = "/tmp/gcsfuse_read_cache_test_logs/log.json" + if testEnv.bucketType == setup.FlatBucket { + testEnv.cacheDirPath = "/tmp/cache-dir-read-cache-hns-false" + } else { + testEnv.cacheDirPath = "/tmp/cache-dir-read-cache-hns-true" + } + } } + testEnv.cfg.LogFile = logFilePath + setup.SetLogFile(logFilePath) } -func mountGCSFuseAndSetupTestDir(flags []string, ctx context.Context, storageClient *storage.Client, testDirName string) { - setup.MountGCSFuseWithGivenMountFunc(flags, mountFunc) +func mountGCSFuseAndSetupTestDir(flags []string, ctx context.Context, storageClient *storage.Client) { + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, flags, mountFunc) setup.SetMntDir(mountDir) - testDirPath = client.SetupTestDirectory(ctx, storageClient, testDirName) -} - -func getDefaultCacheDirPathForTests() string { - return path.Join(setup.TestDir(), cacheDirName) -} - -func createConfigFile(flags *gcsfuseTestFlags) string { - cacheDirPath = flags.cacheDirPath - - // Set up config file for file cache. - mountConfig := map[string]interface{}{ - "file-cache": map[string]interface{}{ - "max-size-mb": flags.cacheSize, - "cache-file-for-range-read": flags.cacheFileForRangeRead, - "enable-parallel-downloads": flags.enableParallelDownloads, - "parallel-downloads-per-file": parallelDownloadsPerFile, - "max-parallel-downloads": maxParallelDownloads, - "download-chunk-size-mb": downloadChunkSizeMB, - "enable-crc": enableCrcCheck, - "enable-o-direct": flags.enableODirect, - }, - "cache-dir": cacheDirPath, - "gcs-connection": map[string]interface{}{ - "client-protocol": flags.clientProtocol, - }, - } - filePath := setup.YAMLConfigFile(mountConfig, flags.fileName) - return filePath -} - -func appendClientProtocolConfigToFlagSet(testFlagSet []gcsfuseTestFlags) (testFlagsWithHttpAndGrpc []gcsfuseTestFlags) { - for _, testFlags := range testFlagSet { - testFlagsWithHttp := testFlags - testFlagsWithHttp.clientProtocol = http1ClientProtocol - testFlagsWithHttpAndGrpc = append(testFlagsWithHttpAndGrpc, testFlagsWithHttp) - - testFlagsWithGrpc := testFlags - testFlagsWithGrpc.clientProtocol = grpcClientProtocol - testFlagsWithHttpAndGrpc = append(testFlagsWithHttpAndGrpc, testFlagsWithGrpc) - } - return + testEnv.testDirPath = client.SetupUniqueTestDirectory(ctx, storageClient, testDirPrefix) } //////////////////////////////////////////////////////////////////////// @@ -160,37 +124,246 @@ func appendClientProtocolConfigToFlagSet(testFlagSet []gcsfuseTestFlags) (testFl func TestMain(m *testing.M) { setup.ParseSetUpFlags() - ctx = context.Background() - closeStorageClient := client.CreateStorageClientWithCancel(&ctx, &storageClient) - defer func() { - err := closeStorageClient() - if err != nil { - log.Fatalf("closeStorageClient failed: %v", err) + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.ReadCache) == 0 { + log.Println("No configuration found for read_cache tests in config. Using flags instead.") + // Populate the config manually. + cfg.ReadCache = make([]test_suite.TestConfig, 1) + cfg.ReadCache[0].TestBucket = setup.TestBucket() + cfg.ReadCache[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.ReadCache[0].LogFile = setup.LogFile() + // Initialize the slice to hold 15 specific test configurations + cfg.ReadCache[0].Configs = make([]test_suite.ConfigItem, 20) + cfg.ReadCache[0].Configs[0].Flags = []string{ + "--metadata-cache-ttl-secs=10 --file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestSmallCacheTTLTest --log-file=/gcsfuse-tmp/TestSmallCacheTTLTest.log --log-severity=TRACE --implicit-dirs --enable-kernel-reader=false", + "--metadata-cache-ttl-secs=10 --file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=true -cache-dir=/gcsfuse-tmp/TestSmallCacheTTLTest --log-file=/gcsfuse-tmp/TestSmallCacheTTLTest.log --log-severity=TRACE --enable-kernel-reader=false", + "--metadata-cache-ttl-secs=10 --file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestSmallCacheTTLTest --log-file=/gcsfuse-tmp/TestSmallCacheTTLTest.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + "--metadata-cache-ttl-secs=10 --file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=true -cache-dir=/gcsfuse-tmp/TestSmallCacheTTLTest --log-file=/gcsfuse-tmp/TestSmallCacheTTLTest.log --log-severity=TRACE -client-protocol=grpc --implicit-dirs --enable-kernel-reader=false", } - }() + cfg.ReadCache[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ReadCache[0].Configs[0].Run = "TestSmallCacheTTLTest" - cacheDirName = "cache-dir-read-cache-hns-" + strconv.FormatBool(setup.IsHierarchicalBucket(ctx, storageClient)) + cfg.ReadCache[0].Configs[1].Flags = []string{ + "--file-cache-max-size-mb=9 --file-cache-cache-file-for-range-read=true --cache-dir=/gcsfuse-tmp/TestReadOnlyTest --log-file=/gcsfuse-tmp/TestReadOnlyTest.log --log-severity=TRACE --file-cache-enable-parallel-downloads=false -implicit-dirs --enable-kernel-reader=false", + "--file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestReadOnlyTest --log-file=/gcsfuse-tmp/TestReadOnlyTest.log --log-severity=TRACE --enable-kernel-reader=false", + "--file-cache-max-size-mb=9 --file-cache-cache-file-for-range-read=true --cache-dir=/gcsfuse-tmp/TestReadOnlyTest --log-file=/gcsfuse-tmp/TestReadOnlyTest.log --log-severity=TRACE --file-cache-enable-parallel-downloads=false --implicit-dirs --o=ro --enable-kernel-reader=false", + "--file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestReadOnlyTest --log-file=/gcsfuse-tmp/TestReadOnlyTest.log --log-severity=TRACE --o=ro --enable-kernel-reader=false", + "--file-cache-max-size-mb=9 --file-cache-cache-file-for-range-read=true --cache-dir=/gcsfuse-tmp/TestReadOnlyTest --log-file=/gcsfuse-tmp/TestReadOnlyTest.log --log-severity=TRACE --file-cache-enable-parallel-downloads=false -implicit-dirs --client-protocol=grpc --enable-kernel-reader=false", + "--file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestReadOnlyTest --log-file=/gcsfuse-tmp/TestReadOnlyTest.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + "--file-cache-max-size-mb=9 --file-cache-cache-file-for-range-read=true --cache-dir=/gcsfuse-tmp/TestReadOnlyTest --log-file=/gcsfuse-tmp/TestReadOnlyTest.log --log-severity=TRACE --file-cache-enable-parallel-downloads=false --implicit-dirs --o=ro --client-protocol=grpc --enable-kernel-reader=false", + "--file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestReadOnlyTest --log-file=/gcsfuse-tmp/TestReadOnlyTest.log --log-severity=TRACE --o=ro --client-protocol=grpc --enable-kernel-reader=false", + } + cfg.ReadCache[0].Configs[1].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ReadCache[0].Configs[1].Run = "TestReadOnlyTest" + + cfg.ReadCache[0].Configs[2].Flags = []string{ + "--implicit-dirs --file-cache-max-size-mb=15 --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestRangeReadTest --log-file=/gcsfuse-tmp/TestRangeReadTest.log --log-severity=TRACE --enable-kernel-reader=false", + "--implicit-dirs --file-cache-max-size-mb=15 --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestRangeReadTest --log-file=/gcsfuse-tmp/TestRangeReadTest.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + } + cfg.ReadCache[0].Configs[2].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ReadCache[0].Configs[2].Run = "TestRangeReadTest" + + cfg.ReadCache[0].Configs[3].Flags = []string{ + "--implicit-dirs --file-cache-max-size-mb=15 --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestRangeReadWithParallelDownloadsTest --log-file=/gcsfuse-tmp/TestRangeReadWithParallelDownloadsTest.log --log-severity=TRACE --enable-kernel-reader=false", + "--implicit-dirs --file-cache-max-size-mb=15 --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestRangeReadWithParallelDownloadsTest --log-file=/gcsfuse-tmp/TestRangeReadWithParallelDownloadsTest.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + } + cfg.ReadCache[0].Configs[3].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ReadCache[0].Configs[3].Run = "TestRangeReadWithParallelDownloadsTest" + + cfg.ReadCache[0].Configs[4].Flags = []string{ + "--file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestLocalModificationTest --log-file=/gcsfuse-tmp/TestLocalModificationTest.log --log-severity=TRACE --implicit-dirs --enable-kernel-reader=false", + "--file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestLocalModificationTest --log-file=/gcsfuse-tmp/TestLocalModificationTest.log --log-severity=TRACE --enable-kernel-reader=false", + "--file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestLocalModificationTest --log-file=/gcsfuse-tmp/TestLocalModificationTest.log --log-severity=TRACE --implicit-dirs --client-protocol=grpc --enable-kernel-reader=false", + "--file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestLocalModificationTest --log-file=/gcsfuse-tmp/TestLocalModificationTest.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + } + cfg.ReadCache[0].Configs[4].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ReadCache[0].Configs[4].Run = "TestLocalModificationTest" - setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() + cfg.ReadCache[0].Configs[5].Flags = []string{ + "--stat-cache-ttl=0s --file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestDisabledCacheTTLTest --log-file=/gcsfuse-tmp/TestDisabledCacheTTLTest.log --log-severity=TRACE --implicit-dirs --enable-kernel-reader=false", + "--stat-cache-ttl=0s --file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestDisabledCacheTTLTest --log-file=/gcsfuse-tmp/TestDisabledCacheTTLTest.log --log-severity=TRACE --enable-kernel-reader=false", + "--stat-cache-ttl=0s --file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestDisabledCacheTTLTest --log-file=/gcsfuse-tmp/TestDisabledCacheTTLTest.log --log-severity=TRACE --implicit-dirs --client-protocol=grpc --enable-kernel-reader=false", + "--stat-cache-ttl=0s --file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestDisabledCacheTTLTest --log-file=/gcsfuse-tmp/TestDisabledCacheTTLTest.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + } + cfg.ReadCache[0].Configs[5].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ReadCache[0].Configs[5].Run = "TestDisabledCacheTTLTest" - setup.RunTestsForMountedDirectoryFlag(m) + cfg.ReadCache[0].Configs[6].Flags = []string{ + "--file-cache-max-size-mb=50 --file-cache-cache-file-for-range-read=true --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest.log --log-severity=TRACE --implicit-dirs --enable-kernel-reader=false", + "--file-cache-max-size-mb=50 --file-cache-cache-file-for-range-read=true --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest.log --log-severity=TRACE --enable-kernel-reader=false", + "--file-cache-max-size-mb=50 --file-cache-cache-file-for-range-read=true --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest.log --log-severity=TRACE --file-cache-enable-o-direct=true --enable-kernel-reader=false", + "--file-cache-max-size-mb=50 --file-cache-cache-file-for-range-read=true --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest.log --log-severity=TRACE --implicit-dirs --client-protocol=grpc --enable-kernel-reader=false", + "--file-cache-max-size-mb=50 --file-cache-cache-file-for-range-read=true --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + "--file-cache-max-size-mb=50 --file-cache-cache-file-for-range-read=true --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest.log --log-severity=TRACE --file-cache-enable-o-direct=true --client-protocol=grpc --enable-kernel-reader=false", + } + cfg.ReadCache[0].Configs[6].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ReadCache[0].Configs[6].Run = "TestCacheFileForRangeReadTrueTest" + + //cfg.ReadCache[0].Configs[7].Flags = []string{ + // "--file-cache-max-size-mb=50 --file-cache-cache-file-for-range-read=true --file-cache-enable-parallel-downloads=true --cache-dir=/dev/shm/TestCacheFileForRangeReadTrueWithRamCache --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueWithRamCache.log --log-severity=TRACE", + // "--file-cache-max-size-mb=50 --file-cache-cache-file-for-range-read=true --file-cache-enable-parallel-downloads=true --cache-dir=/dev/shm/TestCacheFileForRangeReadTrueWithRamCache --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueWithRamCache.log --log-severity=TRACE --file-cache-enable-o-direct=true", + // "--file-cache-max-size-mb=50 --file-cache-cache-file-for-range-read=true --file-cache-enable-parallel-downloads=false --cache-dir=/dev/shm/TestCacheFileForRangeReadTrueWithRamCache --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueWithRamCache.log --log-severity=TRACE", + // "--file-cache-max-size-mb=50 --file-cache-cache-file-for-range-read=true --file-cache-enable-parallel-downloads=true --cache-dir=/dev/shm/TestCacheFileForRangeReadTrueWithRamCache --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueWithRamCache.log --log-severity=TRACE --client-protocol=grpc", + // "--file-cache-max-size-mb=50 --file-cache-cache-file-for-range-read=true --file-cache-enable-parallel-downloads=true --cache-dir=/dev/shm/TestCacheFileForRangeReadTrueWithRamCache --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueWithRamCache.log --log-severity=TRACE --file-cache-enable-o-direct=true --client-protocol=grpc", + // "--file-cache-max-size-mb=50 --file-cache-cache-file-for-range-read=true --file-cache-enable-parallel-downloads=false --cache-dir=/dev/shm/TestCacheFileForRangeReadTrueWithRamCache --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueWithRamCache.log --log-severity=TRACE --client-protocol=grpc", + //} + //cfg.ReadCache[0].Configs[7].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + //cfg.ReadCache[0].Configs[7].Run = "TestCacheFileForRangeReadTrueWithRamCache" + + cfg.ReadCache[0].Configs[8].Flags = []string{ + "--implicit-dirs --file-cache-max-size-mb=50 --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadFalseTest --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseTest.log --log-severity=TRACE --enable-kernel-reader=false", + "--implicit-dirs --file-cache-max-size-mb=50 --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadFalseTest --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseTest.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + } + cfg.ReadCache[0].Configs[8].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ReadCache[0].Configs[8].Run = "TestCacheFileForRangeReadFalseTest" + + //cfg.ReadCache[0].Configs[9].Flags = []string{ + // "--file-cache-max-size-mb=50 --file-cache-enable-parallel-downloads=false --cache-dir=/dev/shm/TestCacheFileForRangeReadFalseWithRamCache --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithRamCache.log --log-severity=TRACE", + // "--file-cache-max-size-mb=50 --file-cache-enable-parallel-downloads=false --cache-dir=/dev/shm/TestCacheFileForRangeReadFalseWithRamCache --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithRamCache.log --log-severity=TRACE --client-protocol=grpc", + //} + //cfg.ReadCache[0].Configs[9].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + //cfg.ReadCache[0].Configs[9].Run = "TestCacheFileForRangeReadFalseWithRamCache" + + cfg.ReadCache[0].Configs[10].Flags = []string{ + "--file-cache-max-size-mb=50 --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloads --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloads.log --log-severity=TRACE --enable-kernel-reader=false", + "--file-cache-max-size-mb=50 --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloads --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloads.log --log-severity=TRACE --file-cache-enable-o-direct=true --enable-kernel-reader=false", + "--file-cache-max-size-mb=50 --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloads --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloads.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + "--file-cache-max-size-mb=50 --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloads --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloads.log --log-severity=TRACE --file-cache-enable-o-direct=true --client-protocol=grpc --enable-kernel-reader=false", + } + cfg.ReadCache[0].Configs[10].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ReadCache[0].Configs[10].Run = "TestCacheFileForRangeReadFalseWithParallelDownloads" - // Else run tests for testBucket. + //cfg.ReadCache[0].Configs[11].Flags = []string{ + // "--file-cache-max-size-mb=50 --file-cache-enable-parallel-downloads=true --cache-dir=/dev/shm/TestCacheFileForRangeReadFalseWithParallelDownloadsAndRamCache --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloadsAndRamCache.log --log-severity=TRACE", + // "--file-cache-max-size-mb=50 --file-cache-enable-parallel-downloads=true --cache-dir=/dev/shm/TestCacheFileForRangeReadFalseWithParallelDownloadsAndRamCache --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloadsAndRamCache.log --log-severity=TRACE --file-cache-enable-o-direct=true", + // "--file-cache-max-size-mb=50 --file-cache-enable-parallel-downloads=true --cache-dir=/dev/shm/TestCacheFileForRangeReadFalseWithParallelDownloadsAndRamCache --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloadsAndRamCache.log --log-severity=TRACE --client-protocol=grpc", + // "--file-cache-max-size-mb=50 --file-cache-enable-parallel-downloads=true --cache-dir=/dev/shm/TestCacheFileForRangeReadFalseWithParallelDownloadsAndRamCache --log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloadsAndRamCache.log --log-severity=TRACE --file-cache-enable-o-direct=true --client-protocol=grpc", + //} + //cfg.ReadCache[0].Configs[11].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + //cfg.ReadCache[0].Configs[11].Run = "TestCacheFileForRangeReadFalseWithParallelDownloadsAndRamCache" + + cfg.ReadCache[0].Configs[12].Flags = []string{ + "--file-cache-max-size-mb=48 --file-cache-cache-file-for-range-read=true --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestJobChunkTest --log-file=/gcsfuse-tmp/TestJobChunkTest.log --log-severity=TRACE --enable-kernel-reader=false", + "--file-cache-max-size-mb=48 --file-cache-cache-file-for-range-read=true --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestJobChunkTest --log-file=/gcsfuse-tmp/TestJobChunkTest.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + } + cfg.ReadCache[0].Configs[12].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ReadCache[0].Configs[12].Run = "TestJobChunkTest" + + cfg.ReadCache[0].Configs[13].Flags = []string{ + //with unlimited max parallel downloads. + "--file-cache-max-size-mb=48 --file-cache-enable-parallel-downloads=true --file-cache-parallel-downloads-per-file=4 --file-cache-max-parallel-downloads=-1 --file-cache-download-chunk-size-mb=4 --file-cache-enable-crc=true --cache-dir=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads --log-file=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads.log --log-severity=TRACE --enable-kernel-reader=false", + "--file-cache-max-size-mb=48 --file-cache-enable-parallel-downloads=true --file-cache-parallel-downloads-per-file=4 --file-cache-max-parallel-downloads=-1 --file-cache-download-chunk-size-mb=4 --file-cache-enable-crc=true --cache-dir=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads --log-file=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + //with go-routines not limited by max parallel downloads. + //maxParallelDownloads > parallelDownloadsPerFile * number of files being accessed concurrently. + "--file-cache-max-size-mb=48 --file-cache-enable-parallel-downloads=true --file-cache-parallel-downloads-per-file=4 --file-cache-max-parallel-downloads=9 --file-cache-download-chunk-size-mb=4 --file-cache-enable-crc=true --cache-dir=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads --log-file=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads.log --log-severity=TRACE --enable-kernel-reader=false", + "--file-cache-max-size-mb=48 --file-cache-enable-parallel-downloads=true --file-cache-parallel-downloads-per-file=4 --file-cache-max-parallel-downloads=9 --file-cache-download-chunk-size-mb=4 --file-cache-enable-crc=true --cache-dir=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads --log-file=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + //with go-routines limited by max parallel downloads. + "--file-cache-max-size-mb=48 --file-cache-enable-parallel-downloads=true --file-cache-parallel-downloads-per-file=4 --file-cache-max-parallel-downloads=2 --file-cache-download-chunk-size-mb=4 --file-cache-enable-crc=true --cache-dir=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads --log-file=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads.log --log-severity=TRACE --enable-kernel-reader=false", + "--file-cache-max-size-mb=48 --file-cache-enable-parallel-downloads=true --file-cache-parallel-downloads-per-file=4 --file-cache-max-parallel-downloads=2 --file-cache-download-chunk-size-mb=4 --file-cache-enable-crc=true --cache-dir=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads --log-file=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + } + cfg.ReadCache[0].Configs[13].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ReadCache[0].Configs[13].Run = "TestJobChunkTestWithParallelDownloads" + + cfg.ReadCache[0].Configs[13].Flags = []string{ + //with unlimited max parallel downloads. + "--file-cache-max-size-mb=48 --file-cache-enable-parallel-downloads=true --file-cache-parallel-downloads-per-file=4 --file-cache-max-parallel-downloads=-1 --file-cache-download-chunk-size-mb=4 --file-cache-enable-crc=true --cache-dir=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads --log-file=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads.log --log-severity=TRACE --enable-kernel-reader=false", + "--file-cache-max-size-mb=48 --file-cache-enable-parallel-downloads=true --file-cache-parallel-downloads-per-file=4 --file-cache-max-parallel-downloads=-1 --file-cache-download-chunk-size-mb=4 --file-cache-enable-crc=true --cache-dir=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads --log-file=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + //with go-routines not limited by max parallel downloads. + //maxParallelDownloads > parallelDownloadsPerFile * number of files being accessed concurrently. + "--file-cache-max-size-mb=48 --file-cache-enable-parallel-downloads=true --file-cache-parallel-downloads-per-file=4 --file-cache-max-parallel-downloads=9 --file-cache-download-chunk-size-mb=4 --file-cache-enable-crc=true --cache-dir=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads --log-file=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads.log --log-severity=TRACE --enable-kernel-reader=false", + "--file-cache-max-size-mb=48 --file-cache-enable-parallel-downloads=true --file-cache-parallel-downloads-per-file=4 --file-cache-max-parallel-downloads=9 --file-cache-download-chunk-size-mb=4 --file-cache-enable-crc=true --cache-dir=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads --log-file=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + //with go-routines limited by max parallel downloads. + "--file-cache-max-size-mb=48 --file-cache-enable-parallel-downloads=true --file-cache-parallel-downloads-per-file=4 --file-cache-max-parallel-downloads=2 --file-cache-download-chunk-size-mb=4 --file-cache-enable-crc=true --cache-dir=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads --log-file=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads.log --log-severity=TRACE --enable-kernel-reader=false", + "--file-cache-max-size-mb=48 --file-cache-enable-parallel-downloads=true --file-cache-parallel-downloads-per-file=4 --file-cache-max-parallel-downloads=2 --file-cache-download-chunk-size-mb=4 --file-cache-enable-crc=true --cache-dir=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads --log-file=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + } + cfg.ReadCache[0].Configs[13].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ReadCache[0].Configs[13].Run = "TestJobChunkTestWithParallelDownloads" + + cfg.ReadCache[0].Configs[14].Flags = []string{ + "--file-cache-exclude-regex=. --file-cache-max-size-mb=50 --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest --log-file=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest.log --log-severity=TRACE --enable-kernel-reader=false", + "--file-cache-exclude-regex=. --file-cache-max-size-mb=50 --file-cache-cache-file-for-range-read=true --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest --log-file=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest.log --log-severity=TRACE --enable-kernel-reader=false", + "--file-cache-exclude-regex=^" + setup.TestBucket() + "/ --file-cache-max-size-mb=50 --file-cache-cache-file-for-range-read=true --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest --log-file=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest.log --log-severity=TRACE --enable-kernel-reader=false", + "--file-cache-exclude-regex=. --file-cache-max-size-mb=50 --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest --log-file=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + "--file-cache-exclude-regex=. --file-cache-max-size-mb=50 --file-cache-cache-file-for-range-read=true --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest --log-file=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + "--file-cache-exclude-regex=^" + setup.TestBucket() + "/ --file-cache-max-size-mb=50 --file-cache-cache-file-for-range-read=true --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest --log-file=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + } + cfg.ReadCache[0].Configs[14].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ReadCache[0].Configs[14].Run = "TestCacheFileForExcludeRegexTest" + + cfg.ReadCache[0].Configs[15].Flags = []string{ + "--implicit-dirs --file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestRemountTest --log-file=/gcsfuse-tmp/TestRemountTest.log --log-severity=TRACE --enable-kernel-reader=false", + "--implicit-dirs --file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestRemountTest --log-file=/gcsfuse-tmp/TestRemountTest.log --log-severity=TRACE --enable-kernel-reader=false", + "--implicit-dirs --file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestRemountTest --log-file=/gcsfuse-tmp/TestRemountTest.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + "--implicit-dirs --file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestRemountTest --log-file=/gcsfuse-tmp/TestRemountTest.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + } + cfg.ReadCache[0].Configs[15].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ReadCache[0].Configs[15].Run = "TestRemountTest" + + cfg.ReadCache[0].Configs[16].Flags = []string{ + "--file-cache-include-regex=^" + setup.TestBucket() + "/.*ReadCacheTest.*/foo* --file-cache-max-size-mb=9 --cache-dir=/gcsfuse-tmp/TestCacheFileForIncludeRegexTest --log-file=/gcsfuse-tmp/TestCacheFileForIncludeRegexTest.log --log-severity=TRACE --enable-kernel-reader=false", + "--file-cache-include-regex=^" + setup.TestBucket() + "/.*ReadCacheTest.*/foo* --file-cache-max-size-mb=9 --cache-dir=/gcsfuse-tmp/TestCacheFileForIncludeRegexTest --log-file=/gcsfuse-tmp/TestCacheFileForIncludeRegexTest.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + "--file-cache-include-regex=^" + setup.TestBucket() + "/.*ReadCacheTest.*/foo* --file-cache-exclude-regex=invalid --file-cache-max-size-mb=9 --cache-dir=/gcsfuse-tmp/TestCacheFileForIncludeRegexTest --log-file=/gcsfuse-tmp/TestCacheFileForIncludeRegexTest.log --log-severity=TRACE --enable-kernel-reader=false", + "--file-cache-include-regex=^" + setup.TestBucket() + "/.*ReadCacheTest.*/foo* --file-cache-exclude-regex=invalid --file-cache-max-size-mb=9 --cache-dir=/gcsfuse-tmp/TestCacheFileForIncludeRegexTest --log-file=/gcsfuse-tmp/TestCacheFileForIncludeRegexTest.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false", + } + cfg.ReadCache[0].Configs[16].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ReadCache[0].Configs[16].Run = "TestCacheFileForIncludeRegexTest" + + cfg.ReadCache[0].Configs[17].Flags = []string{ + "--file-cache-experimental-enable-chunk-cache=true --file-cache-download-chunk-size-mb=10 --file-cache-max-size-mb=-1 --cache-dir=/gcsfuse-tmp/TestChunkCacheTest --log-file=/gcsfuse-tmp/TestChunkCacheTest.log --log-severity=TRACE --enable-kernel-reader=false", + } + cfg.ReadCache[0].Configs[17].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ReadCache[0].Configs[17].Run = "TestChunkCacheTest" + + cfg.ReadCache[0].Configs[18].Flags = []string{ + "--file-cache-experimental-enable-chunk-cache=false --file-cache-max-size-mb=-1 --cache-dir=/gcsfuse-tmp/TestChunkCacheDisabledTest --log-file=/gcsfuse-tmp/TestChunkCacheDisabledTest.log --log-severity=TRACE --enable-kernel-reader=false", + } + cfg.ReadCache[0].Configs[18].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ReadCache[0].Configs[18].Run = "TestChunkCacheDisabledTest" + + cfg.ReadCache[0].Configs[19].Flags = []string{ + "--file-cache-experimental-enable-chunk-cache=true --file-cache-download-chunk-size-mb=10 --file-cache-max-size-mb=15 --cache-dir=/gcsfuse-tmp/TestChunkCacheEviction --log-file=/gcsfuse-tmp/TestChunkCacheEviction.log --log-severity=TRACE --enable-kernel-reader=false", + } + cfg.ReadCache[0].Configs[19].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ReadCache[0].Configs[19].Run = "TestChunkCacheEviction" + } + + testEnv.ctx = context.Background() + testEnv.bucketType = setup.TestEnvironment(testEnv.ctx, &cfg.ReadCache[0]) + testEnv.cfg = &cfg.ReadCache[0] + + // 2. Create storage client before running tests. + var err error + testEnv.storageClient, err = client.CreateStorageClient(testEnv.ctx) + if err != nil { + log.Printf("Error creating storage client: %v\n", err) + os.Exit(1) + } + defer testEnv.storageClient.Close() + + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + os.Exit(setup.RunTestsForMountedDirectory(testEnv.cfg.GKEMountedDirectory, m)) + } + + // Run tests for testBucket // Set up test directory. - setup.SetUpTestDirForTestBucketFlag() + setup.SetUpTestDirForTestBucket(testEnv.cfg) + // Override GKE specific paths with GCSFuse paths if running in GCE environment. + overrideFilePathsInFlagSet(testEnv.cfg, setup.TestDir()) // Save mount and root directory variables. mountDir, rootDir = setup.MntDir(), setup.MntDir() log.Println("Running static mounting tests...") - mountFunc = static_mounting.MountGcsfuseWithStaticMounting + mountFunc = static_mounting.MountGcsfuseWithStaticMountingWithConfigFile successCode := m.Run() if successCode == 0 { log.Println("Running dynamic mounting tests...") // Save mount directory variable to have path of bucket to run tests. mountDir = path.Join(setup.MntDir(), setup.TestBucket()) - mountFunc = dynamic_mounting.MountGcsfuseWithDynamicMounting + mountFunc = dynamic_mounting.MountGcsfuseWithDynamicMountingWithConfig successCode = m.Run() } @@ -198,12 +371,21 @@ func TestMain(m *testing.M) { log.Println("Running only dir mounting tests...") setup.SetOnlyDirMounted(onlyDirMounted + "/") mountDir = rootDir - mountFunc = only_dir_mounting.MountGcsfuseWithOnlyDir + mountFunc = only_dir_mounting.MountGcsfuseWithOnlyDirWithConfigFile successCode = m.Run() - setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(setup.TestBucket(), setup.OnlyDirMounted(), testDirName)) + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(setup.TestBucket(), setup.OnlyDirMounted(), testDirPrefix)) } // Clean up test directory created. - setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(setup.TestBucket(), testDirName)) + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(setup.TestBucket(), testDirPrefix)) os.Exit(successCode) } + +func overrideFilePathsInFlagSet(t *test_suite.TestConfig, GCSFuseTempDirPath string) { + for _, flags := range t.Configs { + for i := range flags.Flags { + // Iterate over the indices of the flags slice + flags.Flags[i] = strings.ReplaceAll(flags.Flags[i], "/gcsfuse-tmp", path.Join(GCSFuseTempDirPath, "gcsfuse-tmp")) + } + } +} diff --git a/tools/integration_tests/read_cache/small_cache_ttl_test.go b/tools/integration_tests/read_cache/small_cache_ttl_test.go index 8b9e008443..78eb63a865 100644 --- a/tools/integration_tests/read_cache/small_cache_ttl_test.go +++ b/tools/integration_tests/read_cache/small_cache_ttl_test.go @@ -18,16 +18,18 @@ import ( "context" "fmt" "log" + "os" "strings" "testing" "time" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/log_parser/json_parser/read_logs" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/test_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" ) //////////////////////////////////////////////////////////////////////// @@ -38,64 +40,112 @@ type smallCacheTTLTest struct { flags []string storageClient *storage.Client ctx context.Context + baseTestName string + suite.Suite } -func (s *smallCacheTTLTest) Setup(t *testing.T) { - setupForMountedDirectoryTests() +func (s *smallCacheTTLTest) SetupSuite() { + setupLogFileAndCacheDir(s.baseTestName) + mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient) +} + +func (s *smallCacheTTLTest) SetupTest() { + //Truncate log file created. + err := os.Truncate(testEnv.cfg.LogFile, 0) + require.NoError(s.T(), err) // Clean up the cache directory path as gcsfuse don't clean up on mounting. - operations.RemoveDir(cacheDirPath) - mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient, testDirName) + operations.RemoveDir(testEnv.cacheDirPath) + testEnv.testDirPath = client.SetupUniqueTestDirectory(s.ctx, s.storageClient, testDirPrefix) } -func (s *smallCacheTTLTest) Teardown(t *testing.T) { - setup.UnmountGCSFuseAndDeleteLogFile(rootDir) +func (s *smallCacheTTLTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +func (s *smallCacheTTLTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) } //////////////////////////////////////////////////////////////////////// // Test scenarios //////////////////////////////////////////////////////////////////////// -func (s *smallCacheTTLTest) TestReadAfterUpdateAndCacheExpiryIsCacheMiss(t *testing.T) { - testFileName := setupFileInTestDir(s.ctx, s.storageClient, testDirName, fileSize, t) +func (s *smallCacheTTLTest) TestReadAfterUpdateAndCacheExpiryIsCacheMiss() { + type initialReadState struct { + testFileName string + expectedOutcome1 *Expected + expectedOutcome2 *Expected + } + + s.T().Logf("Validating that stale data is served before cache expiry (%d seconds Metadata Cache TTL).", metadataCacheTTlInSec) + result := operations.RetryUntil(s.ctx, s.T(), retryFrequency, retryDuration, func() (initialReadState, error) { + // Truncate log file created. + err := os.Truncate(testEnv.cfg.LogFile, 0) + require.NoError(s.T(), err) + // Clean up the cache directory path as gcsfuse don't clean up on mounting. + operations.RemoveDir(testEnv.cacheDirPath) + testEnv.testDirPath = client.SetupUniqueTestDirectory(s.ctx, s.storageClient, testDirPrefix) - // Read file 1st time. - expectedOutcome1 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, true, t) - // Modify the file. - modifyFile(s.ctx, s.storageClient, testFileName, t) - // Read same file again immediately. - expectedOutcome2 := readFileAndGetExpectedOutcome(testDirPath, testFileName, true, zeroOffset, t) - validateFileSizeInCacheDirectory(testFileName, fileSize, t) + testFileName := setupFileInTestDir(s.ctx, s.storageClient, fileSize, s.T()) + + startTime := time.Now() + + // Read file 1st time. + expectedOutcome1 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, true, s.T()) + + // Modify the file. + modifyFile(s.ctx, s.storageClient, testFileName, s.T()) + + // Read same file again immediately. + expectedOutcome2 := readFileAndGetExpectedOutcome(testEnv.testDirPath, testFileName, true, zeroOffset, s.T()) + + if time.Since(startTime) >= metadataCacheTTlInSec*time.Second { + // Retry if the time taken is more (due to GCS Latency or high load) than the metadata cache TTL. + // In this scenario, the test would certainly fail as the stale data would not be served. + return initialReadState{}, fmt.Errorf("failed: because operation took %v seconds >= %d seconds (Metadata Cache TTL)", time.Since(startTime).Seconds(), metadataCacheTTlInSec) + } + s.T().Logf("Success: Operation took %v seconds < %d seconds (Metadata Cache TTL)", time.Since(startTime).Seconds(), metadataCacheTTlInSec) + return initialReadState{testFileName, expectedOutcome1, expectedOutcome2}, nil + }) + + testFileName := result.testFileName + expectedOutcome1 := result.expectedOutcome1 + expectedOutcome2 := result.expectedOutcome2 + + validateFileSizeInCacheDirectory(testFileName, fileSize, s.T()) // Validate that stale data is served from cache in this case. if strings.Compare(expectedOutcome1.content, expectedOutcome2.content) != 0 { - t.Errorf("content mismatch. Expected old data to be served again.") + s.T().Errorf("content mismatch. Expected old data to be served again.") } - // Wait for metadata cache expiry and read the file again. + s.T().Logf("Waiting for %d seconds for metadata cache to expire to get cache miss.", metadataCacheTTlInSec) time.Sleep(metadataCacheTTlInSec * time.Second) - expectedOutcome3 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, smallContentSize, true, t) + expectedOutcome3 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, smallContentSize, true, s.T()) // Parse the log file and validate cache hit or miss from the structured logs. - structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t) - validate(expectedOutcome1, structuredReadLogs[0], true, false, chunksRead, t) - validate(expectedOutcome2, structuredReadLogs[1], true, true, chunksRead, t) - validate(expectedOutcome3, structuredReadLogs[2], true, false, chunksReadAfterUpdate, t) + structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + require.Equal(s.T(), 3, len(structuredReadLogs)) + validate(expectedOutcome1, structuredReadLogs[0], true, false, chunksRead, s.T()) + validate(expectedOutcome2, structuredReadLogs[1], true, true, chunksRead, s.T()) + validate(expectedOutcome3, structuredReadLogs[2], true, false, chunksReadAfterUpdate, s.T()) } -func (s *smallCacheTTLTest) TestReadForLowMetaDataCacheTTLIsCacheHit(t *testing.T) { - testFileName := setupFileInTestDir(s.ctx, s.storageClient, testDirName, fileSize, t) +func (s *smallCacheTTLTest) TestReadForLowMetaDataCacheTTLIsCacheHit() { + testFileName := setupFileInTestDir(s.ctx, s.storageClient, fileSize, s.T()) // Read file 1st time. - expectedOutcome1 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, true, t) + expectedOutcome1 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, true, s.T()) // Wait for metadata cache expiry and read the file again. time.Sleep(metadataCacheTTlInSec * time.Second) - expectedOutcome2 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, true, t) + expectedOutcome2 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, true, s.T()) // Read same file again immediately. - expectedOutcome3 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, true, t) + expectedOutcome3 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, true, s.T()) // Parse the log file and validate cache hit or miss from the structured logs. - structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t) - validate(expectedOutcome1, structuredReadLogs[0], true, false, chunksRead, t) - validate(expectedOutcome2, structuredReadLogs[1], true, true, chunksRead, t) - validate(expectedOutcome3, structuredReadLogs[2], true, true, chunksRead, t) + structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(testEnv.cfg.LogFile, s.T()) + require.Equal(s.T(), 3, len(structuredReadLogs)) + validate(expectedOutcome1, structuredReadLogs[0], true, false, chunksRead, s.T()) + validate(expectedOutcome2, structuredReadLogs[1], true, true, chunksRead, s.T()) + validate(expectedOutcome3, structuredReadLogs[2], true, true, chunksRead, s.T()) } //////////////////////////////////////////////////////////////////////// @@ -103,53 +153,18 @@ func (s *smallCacheTTLTest) TestReadForLowMetaDataCacheTTLIsCacheHit(t *testing. //////////////////////////////////////////////////////////////////////// func TestSmallCacheTTLTest(t *testing.T) { - ts := &smallCacheTTLTest{ctx: context.Background()} - // Create storage client before running tests. - closeStorageClient := client.CreateStorageClientWithCancel(&ts.ctx, &ts.storageClient) - defer func() { - err := closeStorageClient() - if err != nil { - t.Errorf("closeStorageClient failed: %v", err) - } - }() + ts := &smallCacheTTLTest{ctx: context.Background(), storageClient: testEnv.storageClient, baseTestName: t.Name()} - // Run tests for mounted directory if the flag is set. - if setup.AreBothMountedDirectoryAndTestBucketFlagsSet() { - test_setup.RunTests(t, ts) + // Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) return } - // Define flag set to run the tests. - flagsSet := []gcsfuseTestFlags{ - { - cliFlags: []string{fmt.Sprintf("--stat-cache-ttl=%ds", metadataCacheTTlInSec), "--implicit-dirs"}, - cacheSize: cacheCapacityInMB, - cacheFileForRangeRead: false, - fileName: configFileName, - enableParallelDownloads: false, - enableODirect: false, - cacheDirPath: getDefaultCacheDirPathForTests(), - }, - { - cliFlags: []string{fmt.Sprintf("--stat-cache-ttl=%ds", metadataCacheTTlInSec)}, - cacheSize: cacheCapacityInMB, - cacheFileForRangeRead: false, - fileName: configFileNameForParallelDownloadTests, - enableParallelDownloads: true, - enableODirect: false, - cacheDirPath: getDefaultCacheDirPathForTests(), - }, - } - flagsSet = appendClientProtocolConfigToFlagSet(flagsSet) - - // Run tests. - for _, flags := range flagsSet { - configFilePath := createConfigFile(&flags) - ts.flags = []string{"--config-file=" + configFilePath} - if flags.cliFlags != nil { - ts.flags = append(ts.flags, flags.cliFlags...) - } + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { log.Printf("Running tests with flags: %s", ts.flags) - test_setup.RunTests(t, ts) + suite.Run(t, ts) } } diff --git a/tools/integration_tests/read_gcs_algo/concurrent_read_same_file_test.go b/tools/integration_tests/read_gcs_algo/concurrent_read_same_file_test.go new file mode 100644 index 0000000000..f5b6d5dc70 --- /dev/null +++ b/tools/integration_tests/read_gcs_algo/concurrent_read_same_file_test.go @@ -0,0 +1,79 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package read_gcs_algo + +import ( + "bytes" + "io" + "math/rand/v2" + "os" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "golang.org/x/sync/errgroup" +) + +func TestReadSameFileConcurrently(t *testing.T) { + fileSize := 30 * OneMB + filePathInLocalDisk, filePathInMntDir := operations.CreateFileAndCopyToMntDir(t, fileSize, DirForReadAlgoTests) + + var eG errgroup.Group + concurrentReaderCount := 3 + + // We will x numbers of concurrent threads trying to read from the same file. + for range concurrentReaderCount { + randomOffset := rand.Int64N(int64(fileSize)) + + eG.Go(func() error { + readAndCompare(t, filePathInMntDir, filePathInLocalDisk, randomOffset, 5*OneMB) + return nil + }) + } + + // Wait on threads to end. No error is returned by the read method. Hence, + // nothing handling it. + _ = eG.Wait() +} + +func readAndCompare(t *testing.T, filePathInMntDir string, filePathInLocalDisk string, offset int64, chunkSize int64) { + mountedFile, err := operations.OpenFileAsReadonly(filePathInMntDir) + if err != nil { + t.Fatalf("error in opening file from mounted directory :%d", err) + } + defer operations.CloseFileShouldNotThrowError(t, mountedFile) + + // Perform 5 reads on each file. + numberOfReads := 5 + for range numberOfReads { + mountContents := make([]byte, chunkSize) + // Reading chunk size randomly from the file. + _, err = mountedFile.ReadAt(mountContents, offset) + if err == io.EOF { + err = nil + } + if err != nil { + t.Fatalf("error in read file from mounted directory :%d", err) + } + + diskContents, err := operations.ReadChunkFromFile(filePathInLocalDisk, chunkSize, offset, os.O_RDONLY) + if err != nil { + t.Fatalf("error in read file from local directory :%d", err) + } + + if !bytes.Equal(mountContents, diskContents) { + t.Fatalf("data mismatch between mounted directory and local disk") + } + } +} diff --git a/tools/integration_tests/read_gcs_algo/read_gcs_algo_test.go b/tools/integration_tests/read_gcs_algo/read_gcs_algo_test.go new file mode 100644 index 0000000000..b27b49e550 --- /dev/null +++ b/tools/integration_tests/read_gcs_algo/read_gcs_algo_test.go @@ -0,0 +1,80 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package read_gcs_algo + +import ( + "context" + "log" + "os" + "testing" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" +) + +const OneMB = 1024 * 1024 +const DirForReadAlgoTests = "dirForReadAlgoTests" + +var ( + storageClient *storage.Client + ctx context.Context +) + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.ReadGCSAlgo) == 0 { + log.Println("No configuration found for list large dir tests in config. Using flags instead.") + // Populate the config manually. + cfg.ReadGCSAlgo = make([]test_suite.TestConfig, 1) + cfg.ReadGCSAlgo[0].TestBucket = setup.TestBucket() + cfg.ReadGCSAlgo[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.ReadGCSAlgo[0].Configs = make([]test_suite.ConfigItem, 2) + cfg.ReadGCSAlgo[0].Configs[0].Flags = []string{"--implicit-dirs=true"} + cfg.ReadGCSAlgo[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + } + + // 2. Create storage client before running tests. + ctx = context.Background() + bucketType := setup.TestEnvironment(ctx, &cfg.ReadGCSAlgo[0]) + closeStorageClient := client.CreateStorageClientWithCancel(&ctx, &storageClient) + defer func() { + err := closeStorageClient() + if err != nil { + log.Fatalf("closeStorageClient failed: %v", err) + } + }() + + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + // flags to be set, as ReadGCSAlgo tests validates content from the bucket. + if cfg.ReadGCSAlgo[0].GKEMountedDirectory != "" && cfg.ReadGCSAlgo[0].TestBucket != "" { + os.Exit(setup.RunTestsForMountedDirectory(cfg.ReadGCSAlgo[0].GKEMountedDirectory, m)) + } + + // Run tests for testBucket + // 4. Build the flag sets dynamically from the config. + flags := setup.BuildFlagSets(cfg.ReadGCSAlgo[0], bucketType, "") + + setup.SetUpTestDirForTestBucket(&cfg.ReadGCSAlgo[0]) + + successCode := static_mounting.RunTestsWithConfigFile(&cfg.ReadGCSAlgo[0], flags, m) + + os.Exit(successCode) +} diff --git a/tools/integration_tests/read_gcs_algo/seq_diff_block_size_test.go b/tools/integration_tests/read_gcs_algo/seq_diff_block_size_test.go new file mode 100644 index 0000000000..e65e4b85d5 --- /dev/null +++ b/tools/integration_tests/read_gcs_algo/seq_diff_block_size_test.go @@ -0,0 +1,61 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package read_gcs_algo + +import ( + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" +) + +type testCase struct { + name string // Name of the test case + offset int64 + chunkSize int64 +} + +func TestReadSequentialWithDifferentBlockSizes(t *testing.T) { + fileSize := 10 * OneMB + filePathInLocalDisk, filePathInMntDir := operations.CreateFileAndCopyToMntDir(t, fileSize, DirForReadAlgoTests) + + tests := []testCase{ + { + name: "0.5MB", // < 1MB + offset: 0, + chunkSize: OneMB / 2, + }, + { + name: "1MB", // Equal to kernel max buffer size i.e, 1MB + offset: 0, + chunkSize: OneMB, + }, + { + name: "1.5MB", // Not multiple of 1MB + offset: 0, + chunkSize: OneMB + (OneMB / 2), + }, + { + name: "5MB", // Multiple of 1MB + offset: 0, + chunkSize: 5 * OneMB, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + operations.ReadAndCompare(t, filePathInMntDir, filePathInLocalDisk, tc.offset, tc.chunkSize) + }) + } +} diff --git a/tools/integration_tests/read_gcs_algo/seq_to_ran_to_seq_read_test.go b/tools/integration_tests/read_gcs_algo/seq_to_ran_to_seq_read_test.go new file mode 100644 index 0000000000..5dcdedb599 --- /dev/null +++ b/tools/integration_tests/read_gcs_algo/seq_to_ran_to_seq_read_test.go @@ -0,0 +1,47 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package read_gcs_algo + +import ( + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" +) + +func TestSeqReadThenRandomThenSeqRead(t *testing.T) { + filePathInLocalDisk, filePathInMntDir := operations.CreateFileAndCopyToMntDir(t, 50*OneMB, DirForReadAlgoTests) + + // Current read algorithm: + // https://github.com/GoogleCloudPlatform/gcsfuse/blob/v2.5.1/internal/gcsx/random_reader.go#L275 + // First 2 reads are considered sequential. + offset := int64(40 * OneMB) + chunkSize := int64(OneMB) + operations.ReadAndCompare(t, filePathInMntDir, filePathInLocalDisk, offset, chunkSize) + offset = int64(35 * OneMB) + operations.ReadAndCompare(t, filePathInMntDir, filePathInLocalDisk, offset, chunkSize) + + // Perform a couple of random reads. + offset = int64(30 * OneMB) + operations.ReadAndCompare(t, filePathInMntDir, filePathInLocalDisk, offset, chunkSize) + offset = int64(25 * OneMB) + operations.ReadAndCompare(t, filePathInMntDir, filePathInLocalDisk, offset, chunkSize) + offset = int64(20 * OneMB) + operations.ReadAndCompare(t, filePathInMntDir, filePathInLocalDisk, offset, chunkSize) + + // Here we are reading a chunkSize of 40MB which gets converted to sequential because of our + // current read algorithm. + offset = int64(10 * OneMB) + operations.ReadAndCompare(t, filePathInMntDir, filePathInLocalDisk, offset, 40*OneMB) +} diff --git a/tools/integration_tests/read_large_files/concurrent_read_files_test.go b/tools/integration_tests/read_large_files/concurrent_read_files_test.go index 5643e47bbc..1e9f259257 100644 --- a/tools/integration_tests/read_large_files/concurrent_read_files_test.go +++ b/tools/integration_tests/read_large_files/concurrent_read_files_test.go @@ -15,14 +15,12 @@ package read_large_files import ( - "bytes" - "fmt" "os" "path" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" "golang.org/x/sync/errgroup" ) @@ -32,51 +30,49 @@ var FileThree = "fileThree" + setup.GenerateRandomString(5) + ".txt" const NumberOfFilesInLocalDiskForConcurrentRead = 3 -func readFile(fileInLocalDisk string, fileInMntDir string) error { - dataInMntDirFile, err := operations.ReadFile(fileInMntDir) - if err != nil { - return err - } - - dataInLocalDiskFile, err := operations.ReadFile(fileInLocalDisk) - if err != nil { - return err - } - - // Compare actual content and expect content. - if bytes.Equal(dataInLocalDiskFile, dataInMntDirFile) == false { - return fmt.Errorf("Reading incorrect file.") - } - - return nil -} - func TestReadFilesConcurrently(t *testing.T) { testDir := setup.SetupTestDirectory(DirForReadLargeFilesTests) - filesInLocalDisk := [NumberOfFilesInLocalDiskForConcurrentRead]string{FileOne, FileTwo, FileThree} - var filesPathInLocalDisk []string - var filesPathInMntDir []string + fileNames := [NumberOfFilesInLocalDiskForConcurrentRead]string{FileOne, FileTwo, FileThree} + localFilePaths := make([]string, NumberOfFilesInLocalDiskForConcurrentRead) + mntFilePaths := make([]string, NumberOfFilesInLocalDiskForConcurrentRead) + + var creationGroup errgroup.Group + for i := range NumberOfFilesInLocalDiskForConcurrentRead { + fileIndex := i + creationGroup.Go(func() error { + localFilePath := path.Join(os.Getenv("HOME"), fileNames[fileIndex]) + localFilePaths[fileIndex] = localFilePath - for i := 0; i < NumberOfFilesInLocalDiskForConcurrentRead; i++ { - fileInLocalDisk := path.Join(os.Getenv("HOME"), filesInLocalDisk[i]) - filesPathInLocalDisk = append(filesPathInLocalDisk, fileInLocalDisk) + mntFilePath := path.Join(testDir, fileNames[fileIndex]) + mntFilePaths[fileIndex] = mntFilePath - file := path.Join(testDir, filesInLocalDisk[i]) - filesPathInMntDir = append(filesPathInMntDir, file) + operations.CreateFileOnDiskAndCopyToMntDir(t, localFilePath, mntFilePath, FiveHundredMB) + return nil + }) + } + if err := creationGroup.Wait(); err != nil { + t.Fatalf("Error creating files: %v", err) + } - createFileOnDiskAndCopyToMntDir(fileInLocalDisk, file, FiveHundredMB, t) + // Register cleanup for local files. + for i := range NumberOfFilesInLocalDiskForConcurrentRead { + filePath := localFilePaths[i] + t.Cleanup(func() { + operations.RemoveFile(filePath) + }) } var eG errgroup.Group - for i := 0; i < NumberOfFilesInLocalDiskForConcurrentRead; i++ { + for i := range NumberOfFilesInLocalDiskForConcurrentRead { // Copy the current value of i into a local variable to avoid data races. fileIndex := i // Thread to read the current file. eG.Go(func() error { - return readFile(filesPathInLocalDisk[fileIndex], filesPathInMntDir[fileIndex]) + operations.ReadAndCompare(t, mntFilePaths[fileIndex], localFilePaths[fileIndex], 0, FiveHundredMB) + return nil }) } diff --git a/tools/integration_tests/read_large_files/random_read_large_file_test.go b/tools/integration_tests/read_large_files/random_read_large_file_test.go index 02140f483c..88f657f6b7 100644 --- a/tools/integration_tests/read_large_files/random_read_large_file_test.go +++ b/tools/integration_tests/read_large_files/random_read_large_file_test.go @@ -15,41 +15,20 @@ package read_large_files import ( - "bytes" "math/rand" - "os" - "path" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" ) func TestReadLargeFileRandomly(t *testing.T) { - testDir := setup.SetupTestDirectory(DirForReadLargeFilesTests) - fileInLocalDisk := path.Join(os.Getenv("HOME"), FiveHundredMBFile) - file := path.Join(testDir, FiveHundredMBFile) // Create and copy the local file in mountedDirectory. - createFileOnDiskAndCopyToMntDir(fileInLocalDisk, file, FiveHundredMB, t) + fileInLocalDisk, fileInMntDir := operations.CreateFileAndCopyToMntDir(t, FiveHundredMB, DirForReadLargeFilesTests) - for i := 0; i < NumberOfRandomReadCalls; i++ { + for range NumberOfRandomReadCalls { offset := rand.Int63n(MaxReadableByteFromFile - MinReadableByteFromFile) // Randomly read the data from file in mountedDirectory. - content, err := operations.ReadChunkFromFile(file, ChunkSize, offset, os.O_RDONLY) - if err != nil { - t.Errorf("Error in reading file: %v", err) - } - - // Read actual content from file located in local disk. - actualContent, err := operations.ReadChunkFromFile(fileInLocalDisk, ChunkSize, offset, os.O_RDONLY) - if err != nil { - t.Errorf("Error in reading file: %v", err) - } - - // Compare actual content and expect content. - if bytes.Equal(actualContent, content) == false { - t.Errorf("Error in reading file randomly.") - } + operations.ReadAndCompare(t, fileInMntDir, fileInLocalDisk, offset, RandomReadChunkSize) } // Removing file after testing. diff --git a/tools/integration_tests/read_large_files/read_large_files_test.go b/tools/integration_tests/read_large_files/read_large_files_test.go index cf82ed3ea5..81229e56b4 100644 --- a/tools/integration_tests/read_large_files/read_large_files_test.go +++ b/tools/integration_tests/read_large_files/read_large_files_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -19,101 +19,82 @@ import ( "context" "log" "os" - "path" - "strconv" "testing" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" ) -const OneMB = 1024 * 1024 -const FiveHundredMB = 500 * OneMB -const ChunkSize = 200 * OneMB +const FiveHundredMB = 500 * operations.MiB +const ChunkSize = 200 * operations.MiB +const RandomReadChunkSize = operations.MiB const NumberOfRandomReadCalls = 200 const MinReadableByteFromFile = 0 -const MaxReadableByteFromFile = 500 * OneMB +const MaxReadableByteFromFile = 500 * operations.MiB const DirForReadLargeFilesTests = "dirForReadLargeFilesTests" var ( storageClient *storage.Client ctx context.Context FiveHundredMBFile = "fiveHundredMBFile" + setup.GenerateRandomString(5) + ".txt" - cacheDir string ) -func createMountConfigsAndEquivalentFlags() (flags [][]string) { - cacheDirPath := path.Join(os.TempDir(), cacheDir) - - // Set up config file for file cache with cache-file-for-range-read: false - mountConfig1 := map[string]interface{}{ - "file-cache": map[string]interface{}{ - // Keeping the size as high because the operations are performed on large - // files. - "max-size-mb": 700, - "cache-file-for-range-read": true, - }, - "cache-dir": cacheDirPath, - } - filePath1 := setup.YAMLConfigFile(mountConfig1, "config1.yaml") - flags = append(flags, []string{"--implicit-dirs=true", "--config-file=" + filePath1}) - - // Set up config file for file cache with unlimited capacity - mountConfig2 := map[string]interface{}{ - "file-cache": map[string]interface{}{ - "max-size-mb": -1, - "cache-file-for-range-read": false, - }, - "cache-dir": cacheDirPath, - } - filePath2 := setup.YAMLConfigFile(mountConfig2, "config2.yaml") - flags = append(flags, []string{"--implicit-dirs=true", "--config-file=" + filePath2}) - - return flags -} - func TestMain(m *testing.M) { setup.ParseSetUpFlags() - var err error - ctx = context.Background() - storageClient, err = client.CreateStorageClient(ctx) - if err != nil { - log.Printf("Error creating storage client: %v\n", err) - os.Exit(1) + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.ReadLargeFiles) == 0 { + log.Println("No configuration found for read large files tests in config. Using flags instead.") + // Populate the config manually. + cfg.ReadLargeFiles = make([]test_suite.TestConfig, 1) + cfg.ReadLargeFiles[0].TestBucket = setup.TestBucket() + cfg.ReadLargeFiles[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.ReadLargeFiles[0].Configs = make([]test_suite.ConfigItem, 2) + cfg.ReadLargeFiles[0].Configs[0].Flags = []string{ + "--implicit-dirs", + "--implicit-dirs --client-protocol=grpc", + "--implicit-dirs=true --file-cache-max-size-mb=700 --file-cache-cache-file-for-range-read=true --cache-dir=/gcsfuse-tmp/read_large_files", + "--implicit-dirs=true --file-cache-max-size-mb=700 --file-cache-cache-file-for-range-read=true --client-protocol=grpc --cache-dir=/gcsfuse-tmp/read_large_files", + "--implicit-dirs=true --file-cache-max-size-mb=-1 --file-cache-cache-file-for-range-read=false --cache-dir=/gcsfuse-tmp/read_large_files", + "--implicit-dirs=true --file-cache-max-size-mb=-1 --file-cache-cache-file-for-range-read=false --client-protocol=grpc --cache-dir=/gcsfuse-tmp/read_large_files", + } + cfg.ReadLargeFiles[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ReadLargeFiles[0].Configs[1].Flags = []string{ + "--implicit-dirs --enable-kernel-reader=false", + } + cfg.ReadLargeFiles[0].Configs[1].Compatible = map[string]bool{"flat": false, "hns": false, "zonal": true} } - defer storageClient.Close() - cacheDir = "cache-dir-read-large-files-hns-" + strconv.FormatBool(setup.IsHierarchicalBucket(ctx, storageClient)) - flags := [][]string{{"--implicit-dirs"}} - mountConfigFlags := createMountConfigsAndEquivalentFlags() - flags = append(flags, mountConfigFlags...) - setup.AppendFlagsToAllFlagsInTheFlagsSet(&flags, "", "--client-protocol=grpc") - - setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() - - if setup.TestBucket() == "" && setup.MountedDirectory() != "" { - log.Print("Please pass the name of bucket mounted at mountedDirectory to --testBucket flag.") - os.Exit(1) + ctx = context.Background() + bucketType := setup.TestEnvironment(ctx, &cfg.ReadLargeFiles[0]) + + // 2. Create storage client before running tests. + closeStorageClient := client.CreateStorageClientWithCancel(&ctx, &storageClient) + defer func() { + err := closeStorageClient() + if err != nil { + log.Fatalf("closeStorageClient failed: %v", err) + } + }() + + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + // flags to be set, as ReadLargeFiles tests validates content from the bucket. + if cfg.ReadLargeFiles[0].GKEMountedDirectory != "" && cfg.ReadLargeFiles[0].TestBucket != "" { + os.Exit(setup.RunTestsForMountedDirectory(cfg.ReadLargeFiles[0].GKEMountedDirectory, m)) } - // Run tests for mountedDirectory only if --mountedDirectory flag is set. - setup.RunTestsForMountedDirectoryFlag(m) + setup.SetUpTestDirForTestBucket(&cfg.ReadLargeFiles[0]) + setup.OverrideFilePathsInFlagSet(&cfg.ReadLargeFiles[0], setup.TestDir()) - // Run tests for testBucket - setup.SetUpTestDirForTestBucketFlag() + // 4. Build the flag sets dynamically from the modified config. + flags := setup.BuildFlagSets(cfg.ReadLargeFiles[0], bucketType, "") - successCode := static_mounting.RunTests(flags, m) - os.Exit(successCode) -} + successCode := static_mounting.RunTestsWithConfigFile(&cfg.ReadLargeFiles[0], flags, m) -func createFileOnDiskAndCopyToMntDir(fileInLocalDisk string, fileInMntDir string, fileSize int, t *testing.T) { - setup.RunScriptForTestData("testdata/write_content_of_fix_size_in_file.sh", fileInLocalDisk, strconv.Itoa(fileSize)) - err := operations.CopyFile(fileInLocalDisk, fileInMntDir) - if err != nil { - t.Errorf("Error in copying file:%v", err) - } + os.Exit(successCode) } diff --git a/tools/integration_tests/read_large_files/seq_read_large_file_test.go b/tools/integration_tests/read_large_files/seq_read_large_file_test.go index 978296e179..8b78bb9eba 100644 --- a/tools/integration_tests/read_large_files/seq_read_large_file_test.go +++ b/tools/integration_tests/read_large_files/seq_read_large_file_test.go @@ -17,21 +17,21 @@ package read_large_files import ( "bytes" "os" - "path" + "syscall" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" ) func TestReadLargeFileSequentially(t *testing.T) { - testDir := setup.SetupTestDirectory(DirForReadLargeFilesTests) // Create file of 500 MB with random data in local disk and copy it in mntDir. - fileInLocalDisk := path.Join(os.Getenv("HOME"), FiveHundredMBFile) - file := path.Join(testDir, FiveHundredMBFile) - createFileOnDiskAndCopyToMntDir(fileInLocalDisk, file, FiveHundredMB, t) + fileInLocalDisk, fileInMntDir := operations.CreateFileAndCopyToMntDir(t, FiveHundredMB, DirForReadLargeFilesTests) // Sequentially read the data from file. + file, err := os.OpenFile(fileInMntDir, os.O_RDONLY|syscall.O_DIRECT, setup.FilePermission_0600) + require.NoError(t, err) content, err := operations.ReadFileSequentially(file, ChunkSize) if err != nil { t.Errorf("Error in reading file: %v", err) diff --git a/tools/integration_tests/read_large_files/testdata/write_content_of_fix_size_in_file.sh b/tools/integration_tests/read_large_files/testdata/write_content_of_fix_size_in_file.sh deleted file mode 100755 index bf4172b094..0000000000 --- a/tools/integration_tests/read_large_files/testdata/write_content_of_fix_size_in_file.sh +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FILE_PATH=$1 -FILE_SIZE=$2 - -# It will write filesize random data in a file. -head -c $FILE_SIZE </dev/urandom > $FILE_PATH diff --git a/tools/integration_tests/readdirplus/readdirplus_with_dentry_cache_test.go b/tools/integration_tests/readdirplus/readdirplus_with_dentry_cache_test.go new file mode 100644 index 0000000000..553c832a7a --- /dev/null +++ b/tools/integration_tests/readdirplus/readdirplus_with_dentry_cache_test.go @@ -0,0 +1,122 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package readdirplus + +import ( + "context" + "log" + "os" + "path" + "testing" + "time" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/jacobsa/fuse/fusetesting" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type ReaddirplusWithDentryCacheTest struct { + flags []string + storageClient *storage.Client + ctx context.Context + baseTestName string + suite.Suite +} + +func (s *ReaddirplusWithDentryCacheTest) SetupTest() { + testEnv.testDirPath = client.SetupTestDirectory(s.ctx, s.storageClient, path.Join(testDirName, s.T().Name())) +} + +func (s *ReaddirplusWithDentryCacheTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +func (s *ReaddirplusWithDentryCacheTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *ReaddirplusWithDentryCacheTest) SetupSuite() { + setup.SetUpLogFilePath(s.T().Name(), s.flags, GKETempDir, OldGKElogFilePath, testEnv.cfg) + mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient) +} + +func (s *ReaddirplusWithDentryCacheTest) TestReaddirplusWithDentryCache() { + // Create directory structure: + // testBucket/dirForReaddirplusTest/target_dir/ + // testBucket/dirForReaddirplusTest/target_dir/file + // testBucket/dirForReaddirplusTest/target_dir/emptySubDirectory/ + // testBucket/dirForReaddirplusTest/target_dir/subDirectory/ + // testBucket/dirForReaddirplusTest/target_dir/subDirectory/file1 + targetDir := path.Join(testEnv.testDirPath, targetDirName) + operations.CreateDirectory(targetDir, s.T()) + // Create a file in the target directory. + f1 := operations.CreateFile(path.Join(targetDir, "file"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f1) + // Create an empty subdirectory + operations.CreateDirectory(path.Join(targetDir, "emptySubDirectory"), s.T()) + // Create a subdirectory with file + operations.CreateDirectoryWithNFiles(1, path.Join(targetDir, "subDirectory"), "file", s.T()) + + // Call Readdirplus to list the directory. + startTime := time.Now() + entries, err := fusetesting.ReadDirPlusPicky(targetDir) + endTime := time.Now() + + require.NoError(s.T(), err, "ReadDirPlusPicky failed") + // Verify the entries. + expectedEntries := []struct { + name string + isDir bool + mode os.FileMode + }{ + {name: "emptySubDirectory", isDir: true, mode: os.ModeDir | 0755}, + {name: "file", isDir: false, mode: 0644}, + {name: "subDirectory", isDir: true, mode: os.ModeDir | 0755}, + } + assert.Equal(s.T(), len(expectedEntries), len(entries), "Number of entries mismatch") + for i, expected := range expectedEntries { + entry := entries[i] + assert.Equal(s.T(), expected.name, entry.Name(), "Name mismatch for entry %d", i) + assert.Equal(s.T(), expected.isDir, entry.IsDir(), "IsDir mismatch for entry %s", entry.Name()) + assert.Equal(s.T(), expected.mode, entry.Mode(), "Mode mismatch for entry %s", entry.Name()) + } + + // Dentry cache is enabled, so LookUpInode should also not be called. + // This applies even to the parent directory, as its inode is cached during + // the test setup phase when the directory structure is created. + validateLogsForReaddirplus(s.T(), testEnv.cfg.LogFile, true, startTime, endTime) +} + +func TestReaddirplusWithDentryCacheTest(t *testing.T) { + ts := &ReaddirplusWithDentryCacheTest{ctx: context.Background(), storageClient: testEnv.storageClient, baseTestName: t.Name()} + + // Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/readdirplus/readdirplus_without_dentry_cache_test.go b/tools/integration_tests/readdirplus/readdirplus_without_dentry_cache_test.go new file mode 100644 index 0000000000..442f6b9e3b --- /dev/null +++ b/tools/integration_tests/readdirplus/readdirplus_without_dentry_cache_test.go @@ -0,0 +1,121 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package readdirplus + +import ( + "context" + "log" + "os" + "path" + "testing" + "time" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/jacobsa/fuse/fusetesting" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type ReaddirplusWithoutDentryCacheTest struct { + flags []string + storageClient *storage.Client + ctx context.Context + baseTestName string + suite.Suite +} + +func (s *ReaddirplusWithoutDentryCacheTest) SetupTest() { + testEnv.testDirPath = client.SetupTestDirectory(s.ctx, s.storageClient, path.Join(testDirName, s.T().Name())) +} + +func (s *ReaddirplusWithoutDentryCacheTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +func (s *ReaddirplusWithoutDentryCacheTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *ReaddirplusWithoutDentryCacheTest) SetupSuite() { + setup.SetUpLogFilePath(s.T().Name(), s.flags, GKETempDir, OldGKElogFilePath, testEnv.cfg) + mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient) +} + +func (s *ReaddirplusWithoutDentryCacheTest) TestReaddirplusWithoutDentryCache() { + // Create directory structure: + // testBucket/dirForReaddirplusTest/target_dir/ + // testBucket/dirForReaddirplusTest/target_dir/file + // testBucket/dirForReaddirplusTest/target_dir/emptySubDirectory/ + // testBucket/dirForReaddirplusTest/target_dir/subDirectory/ + // testBucket/dirForReaddirplusTest/target_dir/subDirectory/file1 + targetDir := path.Join(testEnv.testDirPath, targetDirName) + operations.CreateDirectory(targetDir, s.T()) + // Create a file in the target directory. + f1 := operations.CreateFile(path.Join(targetDir, "file"), setup.FilePermission_0600, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), f1) + // Create an empty subdirectory + operations.CreateDirectory(path.Join(targetDir, "emptySubDirectory"), s.T()) + // Create a subdirectory with file + operations.CreateDirectoryWithNFiles(1, path.Join(targetDir, "subDirectory"), "file", s.T()) + + // Call Readdirplus to list the directory. + startTime := time.Now() + entries, err := fusetesting.ReadDirPlusPicky(targetDir) + endTime := time.Now() + + require.NoError(s.T(), err, "ReadDirPlusPicky failed") + // Verify the entries. + expectedEntries := []struct { + name string + isDir bool + mode os.FileMode + }{ + {name: "emptySubDirectory", isDir: true, mode: os.ModeDir | 0755}, + {name: "file", isDir: false, mode: 0644}, + {name: "subDirectory", isDir: true, mode: os.ModeDir | 0755}, + } + assert.Equal(s.T(), len(expectedEntries), len(entries), "Number of entries mismatch") + for i, expected := range expectedEntries { + entry := entries[i] + assert.Equal(s.T(), expected.name, entry.Name(), "Name mismatch for entry %d", i) + assert.Equal(s.T(), expected.isDir, entry.IsDir(), "IsDir mismatch for entry %s", entry.Name()) + assert.Equal(s.T(), expected.mode, entry.Mode(), "Mode mismatch for entry %s", entry.Name()) + } + + // Validate logs to check that ReadDirPlus was called and ReadDir was not. + // Dentry cache is not enabled, so LookUpInode should be called for + // parent directory as well as for all the entries. + validateLogsForReaddirplus(s.T(), testEnv.cfg.LogFile, false, startTime, endTime) +} + +func TestReaddirplusWithoutDentryCacheTest(t *testing.T) { + ts := &ReaddirplusWithoutDentryCacheTest{ctx: context.Background(), storageClient: testEnv.storageClient, baseTestName: t.Name()} + + // Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/readdirplus/setup_test.go b/tools/integration_tests/readdirplus/setup_test.go new file mode 100644 index 0000000000..1a0d9bac20 --- /dev/null +++ b/tools/integration_tests/readdirplus/setup_test.go @@ -0,0 +1,187 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Provides integration tests for Readdirplus +package readdirplus + +import ( + "context" + "io" + "log" + "os" + "path" + "strings" + "testing" + "time" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" + "github.com/stretchr/testify/require" +) + +const ( + testDirName = "dirForReaddirplusTest" + targetDirName = "target_dir" + GKETempDir = "/gcsfuse-tmp" + // // TODO: clean this up when GKE test migration completes. + OldGKElogFilePath = "/tmp/readdirplus_logs/log.json" +) + +var ( + testEnv env + mountFunc func(*test_suite.TestConfig, []string) error + // mount directory is where our tests run. + mountDir string + // root directory is the directory to be unmounted. + rootDir string +) + +type env struct { + storageClient *storage.Client + ctx context.Context + testDirPath string + cfg *test_suite.TestConfig + bucketType string +} + +func loadLogLines(reader io.Reader) ([]string, error) { + content, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + return strings.Split(string(content), "\n"), nil +} + +// validateLogsForReaddirplus checks that ReadDirPlus was called and ReadDir was not. +// It also checks that LookUpInode is not called when dentry cache is enabled. +func validateLogsForReaddirplus(t *testing.T, logFile string, dentryCacheEnabled bool, startTime, endTime time.Time) { + t.Helper() + + logForReadDirPlus := "ReadDirPlus (" + logForReadDir := "ReadDir (" + logForLookUpInode := "LookUpInode (" + + file, err := os.Open(logFile) + require.NoError(t, err, "Failed to open log file") + defer file.Close() + + logLines, err := loadLogLines(file) + require.NoError(t, err, "Failed to read log file") + + foundReadDirPlus := false + foundReadDir := false + foundLookUpInode := false + for _, line := range logLines { + logEntry, err := read_logs.ParseJsonLogLineIntoLogEntryStruct(line) // Assuming read_logs can parse general log lines too or a more generic parser is available. + // If parsing fails, it might be a non-JSON line or a different structured log. + // For this specific message, we expect it to be in the "Message" field of a structured log. + + if err == nil && logEntry != nil { + // Check if the log entry's timestamp is within the expected window. + if (logEntry.Timestamp.After(startTime) || logEntry.Timestamp.Equal(startTime)) && + (logEntry.Timestamp.Before(endTime) || logEntry.Timestamp.Equal(endTime)) { + if strings.Contains(logEntry.Message, logForReadDirPlus) { + foundReadDirPlus = true + } + if strings.Contains(logEntry.Message, logForReadDir) { + foundReadDir = true + } + if strings.Contains(logEntry.Message, logForLookUpInode) { + foundLookUpInode = true + } + } + } + } + + require.True(t, foundReadDirPlus, "ReadDirPlus not called") + require.False(t, foundReadDir, "ReadDir called unexpectedly") + if dentryCacheEnabled { + require.False(t, foundLookUpInode, "LookUpInode called unexpectedly") + } else { + require.True(t, foundLookUpInode, "LookUpInode not called") + } +} + +func mountGCSFuseAndSetupTestDir(flags []string, ctx context.Context, storageClient *storage.Client) { + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, flags, mountFunc) + setup.SetMntDir(mountDir) + testEnv.testDirPath = client.SetupTestDirectory(ctx, storageClient, testDirName) +} + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.ReadDirPlus) == 0 { + log.Println("No configuration found for readdirplus tests in config. Using flags instead.") + // Populate the config manually. + cfg.ReadDirPlus = make([]test_suite.TestConfig, 1) + cfg.ReadDirPlus[0].TestBucket = setup.TestBucket() + cfg.ReadDirPlus[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.ReadDirPlus[0].LogFile = setup.LogFile() + cfg.ReadDirPlus[0].Configs = make([]test_suite.ConfigItem, 2) + cfg.ReadDirPlus[0].Configs[0].Flags = []string{ + "--implicit-dirs --experimental-enable-readdirplus --experimental-enable-dentry-cache --log-file=/gcsfuse-tmp/TestReaddirplusWithDentryCacheTest.log --log-severity=TRACE", + } + cfg.ReadDirPlus[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ReadDirPlus[0].Configs[0].Run = "TestReaddirplusWithDentryCacheTest" + + cfg.ReadDirPlus[0].Configs[1].Flags = []string{ + "--implicit-dirs --experimental-enable-readdirplus --log-file=/gcsfuse-tmp/TestReaddirplusWithoutDentryCacheTest.log --log-severity=TRACE", + } + cfg.ReadDirPlus[0].Configs[1].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.ReadDirPlus[0].Configs[1].Run = "TestReaddirplusWithoutDentryCacheTest" + } + + testEnv.ctx = context.Background() + testEnv.cfg = &cfg.ReadDirPlus[0] + testEnv.bucketType = setup.TestEnvironment(testEnv.ctx, testEnv.cfg) + + // 2. Create storage client before running tests. + var err error + testEnv.storageClient, err = client.CreateStorageClient(testEnv.ctx) + if err != nil { + log.Printf("Error creating storage client: %v\n", err) + os.Exit(1) + } + defer testEnv.storageClient.Close() + + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + mountDir = testEnv.cfg.GKEMountedDirectory + os.Exit(setup.RunTestsForMountedDirectory(mountDir, m)) + } + + // Run tests for testBucket + // Set up test directory. + setup.SetUpTestDirForTestBucket(testEnv.cfg) + // Override GKE specific paths with GCSFuse paths if running in GCE environment. + setup.OverrideFilePathsInFlagSet(testEnv.cfg, setup.TestDir()) + + // Save mount and root directory variables. + mountDir, rootDir = setup.MntDir(), setup.MntDir() + + log.Println("Running static mounting tests...") + mountFunc = static_mounting.MountGcsfuseWithStaticMountingWithConfigFile + successCode := m.Run() + + // Clean up test directory created. + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(cfg.ReadDirPlus[0].TestBucket, testDirName)) + os.Exit(successCode) +} diff --git a/tools/integration_tests/readonly/copy_object_test.go b/tools/integration_tests/readonly/copy_object_test.go index 2a22bcad7c..578fe87799 100644 --- a/tools/integration_tests/readonly/copy_object_test.go +++ b/tools/integration_tests/readonly/copy_object_test.go @@ -16,26 +16,20 @@ package readonly_test import ( - "os" "path" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) // Copy srcFile in testBucket/testDirForReadOnlyTest/Test/b/b.txt destination. func checkIfFileCopyFailed(srcFilePath string, t *testing.T) { - // cp without destination file creates a destination file and create workflow is already covered separately. - copyFile := path.Join(setup.MntDir(), TestDirForReadOnlyTest, DirectoryNameInTestBucket, SubDirectoryNameInTestBucket, FileNameInSubDirectoryTestBucket) - // cp without destination file creates a destination file and create workflow is already covered separately. - // Checking if destination object exist. - if _, err := os.Stat(copyFile); err != nil { - t.Errorf("Copied file %s is not present", copyFile) - } + destFileName := FileNameInSubDirectoryTestBucket + setup.GenerateRandomString(5) + destFile := path.Join(setup.MntDir(), TestDirForReadOnlyTest, DirectoryNameInTestBucket, SubDirectoryNameInTestBucket, destFileName) - err := operations.CopyFile(srcFilePath, copyFile) + err := operations.CopyFile(srcFilePath, destFile) if err == nil { t.Errorf("File copied in read-only file system: %v", err) } diff --git a/tools/integration_tests/readonly/create_object_test.go b/tools/integration_tests/readonly/create_object_test.go index 9797e8c4fd..0ddc03b7d1 100644 --- a/tools/integration_tests/readonly/create_object_test.go +++ b/tools/integration_tests/readonly/create_object_test.go @@ -21,9 +21,9 @@ import ( "path" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) func checkIfFileCreationFailed(filePath string, t *testing.T) { @@ -33,7 +33,7 @@ func checkIfFileCreationFailed(filePath string, t *testing.T) { t.Errorf("File is created in read-only file system.") } - operations.CheckErrorForReadOnlyFileSystem(err, t) + operations.CheckErrorForReadOnlyFileSystem(t, err) defer file.Close() } @@ -57,7 +57,7 @@ func checkIfDirCreationFailed(dirPath string, t *testing.T) { t.Errorf("Directory is created in read-only file system.") } - operations.CheckErrorForReadOnlyFileSystem(err, t) + operations.CheckErrorForReadOnlyFileSystem(t, err) } func TestCreateDir(t *testing.T) { diff --git a/tools/integration_tests/readonly/delete_object_test.go b/tools/integration_tests/readonly/delete_object_test.go index b548bdb4eb..ac40ed1cba 100644 --- a/tools/integration_tests/readonly/delete_object_test.go +++ b/tools/integration_tests/readonly/delete_object_test.go @@ -20,9 +20,9 @@ import ( "path" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) func checkIfObjDeletionFailed(objPath string, t *testing.T) { @@ -32,7 +32,7 @@ func checkIfObjDeletionFailed(objPath string, t *testing.T) { t.Errorf("Objects are deleted in read-only file system.") } - operations.CheckErrorForReadOnlyFileSystem(err, t) + operations.CheckErrorForReadOnlyFileSystem(t, err) } func TestDeleteDir(t *testing.T) { diff --git a/tools/integration_tests/readonly/list_objects_test.go b/tools/integration_tests/readonly/list_objects_test.go index f0033f01f5..b090d8fd1b 100644 --- a/tools/integration_tests/readonly/list_objects_test.go +++ b/tools/integration_tests/readonly/list_objects_test.go @@ -21,7 +21,7 @@ import ( "path" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) func TestListObjectsInBucket(t *testing.T) { diff --git a/tools/integration_tests/readonly/read_test.go b/tools/integration_tests/readonly/read_test.go index 578b79977b..96421cef1b 100644 --- a/tools/integration_tests/readonly/read_test.go +++ b/tools/integration_tests/readonly/read_test.go @@ -21,8 +21,8 @@ import ( "syscall" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) func checkIfFileReadSucceeded(filePath string, expectedContent string, t *testing.T) { diff --git a/tools/integration_tests/readonly/readonly_test.go b/tools/integration_tests/readonly/readonly_test.go index 218c05761c..3ae418af00 100644 --- a/tools/integration_tests/readonly/readonly_test.go +++ b/tools/integration_tests/readonly/readonly_test.go @@ -17,19 +17,20 @@ package readonly_test import ( "context" + "fmt" "log" "os" "path" - "strconv" "strings" "testing" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/creds_tests" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/persistent_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/creds_tests" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/persistent_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" ) const TestDirForReadOnlyTest = "testDirForReadOnlyTest" @@ -52,10 +53,9 @@ const RenameDir = "rename" var ( storageClient *storage.Client ctx context.Context - cacheDir string ) -func createTestDataForReadOnlyTests(ctx context.Context, storageClient *storage.Client) { +func createTestDataForReadOnlyTests(ctx context.Context, storageClient *storage.Client) error { // Define the text to write and the files to create files := []struct { fileContent string @@ -74,17 +74,23 @@ func createTestDataForReadOnlyTests(ctx context.Context, storageClient *storage. filePath := path.Join(dirPath, file.filePath) // Create a storage writer for the destination object object := bucketHandle.Object(filePath) - writer := object.NewWriter(ctx) + writer, err := client.NewWriter(ctx, object, storageClient) + if err != nil { + return fmt.Errorf("Error opening writer for object %s: %w\n", file.filePath, err) + } // Write the text to the object - if _, err := writer.Write([]byte(file.fileContent + "\n")); err != nil { + if _, err = writer.Write([]byte(file.fileContent + "\n")); err != nil { log.Printf("Error writing to object %s: %v\n", file.filePath, err) } - err := writer.Close() + err = writer.Close() if err != nil { log.Printf("Error in closing writer: %v", err) + return err } } + + return nil } func checkErrorForObjectNotExist(err error, t *testing.T) { @@ -93,73 +99,64 @@ func checkErrorForObjectNotExist(err error, t *testing.T) { } } -func createMountConfigsAndEquivalentFlags() (flags [][]string) { - cacheDirPath := path.Join(os.TempDir(), cacheDir) - - // Set up config file for file cache. - mountConfig := map[string]interface{}{ - "file-cache": map[string]interface{}{ - // Keeping the size as small because the operations are performed on small - // files. - "max-size-mb": 3, - }, - "cache-dir": cacheDirPath, - } - filePath := setup.YAMLConfigFile(mountConfig, "config.yaml") - flags = append(flags, []string{"--o=ro", "--implicit-dirs=true", "--config-file=" + filePath}) - - return flags -} - func TestMain(m *testing.M) { setup.ParseSetUpFlags() - - var err error ctx = context.Background() - storageClient, err = client.CreateStorageClient(ctx) - if err != nil { - log.Printf("Error creating storage client: %v\n", err) - os.Exit(1) - } - defer storageClient.Close() - - cacheDir = "cache-dir-readonly-hns-" + strconv.FormatBool(setup.IsHierarchicalBucket(ctx, storageClient)) - flags := [][]string{{"--o=ro", "--implicit-dirs=true"}, {"--file-mode=544", "--dir-mode=544", "--implicit-dirs=true"}} - - if !testing.Short() { - flags = append(flags, []string{"--client-protocol=grpc", "--o=ro", "--implicit-dirs=true"}) + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.ReadOnly) == 0 { + log.Println("No configuration found for readonly tests in config. Using flags instead.") + // Populate the config manually. + cfg.ReadOnly = make([]test_suite.TestConfig, 1) + cfg.ReadOnly[0].TestBucket = setup.TestBucket() + cfg.ReadOnly[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.ReadOnly[0].Configs = make([]test_suite.ConfigItem, 1) + cfg.ReadOnly[0].Configs[0].Flags = []string{ + "--o=ro --implicit-dirs=true", + "--file-mode=544 --dir-mode=544 --implicit-dirs=true", + "--client-protocol=grpc --o=ro --implicit-dirs=true", + "--o=ro --implicit-dirs=true --cache-dir=/gcsfuse-tmp/readonly --file-cache-max-size-mb=3", + } + cfg.ReadOnly[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} } - setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() - - if setup.TestBucket() == "" && setup.MountedDirectory() != "" { - log.Print("Please pass the name of bucket mounted at mountedDirectory to --testBucket flag.") - os.Exit(1) - } + // 2. Create storage client before running tests. + bucketType := setup.TestEnvironment(ctx, &cfg.ReadOnly[0]) + closeStorageClient := client.CreateStorageClientWithCancel(&ctx, &storageClient) + defer func() { + err := closeStorageClient() + if err != nil { + log.Fatalf("closeStorageClient failed: %v", err) + } + }() // Create test data. - createTestDataForReadOnlyTests(ctx, storageClient) - - // Run tests for mountedDirectory only if --mountedDirectory flag is set. - setup.RunTestsForMountedDirectoryFlag(m) + if err := createTestDataForReadOnlyTests(ctx, storageClient); err != nil { + log.Fatalf("Failed creating test data for readonly tests: %v", err) + } - // Run tests for testBucket - setup.SetUpTestDirForTestBucketFlag() + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + // flags to be set. + if cfg.ReadOnly[0].GKEMountedDirectory != "" && cfg.ReadOnly[0].TestBucket != "" { + os.Exit(setup.RunTestsForMountedDirectory(cfg.ReadOnly[0].GKEMountedDirectory, m)) + } - // Setup config file for tests when --testbucket flag is enabled. - mountConfigFlags := createMountConfigsAndEquivalentFlags() - flags = append(flags, mountConfigFlags...) + setup.SetUpTestDirForTestBucket(&cfg.ReadOnly[0]) + setup.OverrideFilePathsInFlagSet(&cfg.ReadOnly[0], setup.TestDir()) - successCode := static_mounting.RunTests(flags, m) + // 4. Build the flag sets dynamically from the modified config. + flags := setup.BuildFlagSets(cfg.ReadOnly[0], bucketType, "") + // 5. Run tests. + successCode := static_mounting.RunTestsWithConfigFile(&cfg.ReadOnly[0], flags, m) if successCode == 0 { - successCode = persistent_mounting.RunTests(flags, m) + successCode = persistent_mounting.RunTestsWithConfigFile(&cfg.ReadOnly[0], flags, m) } - if successCode == 0 { - // Test for viewer permission on test bucket. - successCode = creds_tests.RunTestsForKeyFileAndGoogleApplicationCredentialsEnvVarSet(ctx, storageClient, flags, "objectViewer", m) + // These tests don't apply to GCSFuse sidecar. + // Validate that tests work with viewer permission on test bucket. + successCode = creds_tests.RunTestsForDifferentAuthMethods(ctx, &cfg.ReadOnly[0], storageClient, flags, "objectViewer", m) } os.Exit(successCode) diff --git a/tools/integration_tests/readonly/rename_object_test.go b/tools/integration_tests/readonly/rename_object_test.go index 759ac863eb..42f1cb74ad 100644 --- a/tools/integration_tests/readonly/rename_object_test.go +++ b/tools/integration_tests/readonly/rename_object_test.go @@ -20,8 +20,8 @@ import ( "path" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) // Rename oldFile to newFile @@ -31,7 +31,7 @@ func checkIfRenameFileFailed(oldFilePath string, newFilePath string, t *testing. t.Errorf("File renamed in read-only file system.") } - operations.CheckErrorForReadOnlyFileSystem(err, t) + operations.CheckErrorForReadOnlyFileSystem(t, err) if _, err := os.Stat(oldFilePath); err != nil { t.Errorf("Old file is deleted in read-only file system.") @@ -48,7 +48,7 @@ func checkIfRenameDirFailed(oldDirPath string, newDirPath string, t *testing.T) t.Errorf("Directory renamed in read-only file system.") } - operations.CheckErrorForReadOnlyFileSystem(err, t) + operations.CheckErrorForReadOnlyFileSystem(t, err) if _, err := os.Stat(oldDirPath); err != nil { t.Errorf("Old directory is deleted in read-only file system.") diff --git a/tools/integration_tests/readonly/stat_object_test.go b/tools/integration_tests/readonly/stat_object_test.go index ed06cd93c8..1a6fc81af8 100644 --- a/tools/integration_tests/readonly/stat_object_test.go +++ b/tools/integration_tests/readonly/stat_object_test.go @@ -20,7 +20,7 @@ import ( "path" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) func statObject(objPath string, t *testing.T) (file os.FileInfo) { diff --git a/tools/integration_tests/readonly/write_test.go b/tools/integration_tests/readonly/write_test.go index 9032957aa8..1003703eeb 100644 --- a/tools/integration_tests/readonly/write_test.go +++ b/tools/integration_tests/readonly/write_test.go @@ -19,8 +19,8 @@ import ( "path" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) const Content = "Testing" @@ -32,7 +32,7 @@ func checkIfFileFailedToOpenForWrite(filePath string, t *testing.T) { t.Errorf("File opened for writing in read-only mount.") } - operations.CheckErrorForReadOnlyFileSystem(err, t) + operations.CheckErrorForReadOnlyFileSystem(t, err) } // testBucket/testDirForReadOnlyTest/Test1.txt @@ -91,7 +91,7 @@ func checkIfFileFailedToOpenForAppend(filePath string, t *testing.T) { t.Errorf("File opened for appending content in read-only mount.") } - operations.CheckErrorForReadOnlyFileSystem(err, t) + operations.CheckErrorForReadOnlyFileSystem(t, err) } func TestOpenFileWithAppendAccess(t *testing.T) { diff --git a/tools/integration_tests/readonly_creds/failure_during_file_sync_test.go b/tools/integration_tests/readonly_creds/failure_during_file_sync_test.go index 8977a49d8a..a50d93acdd 100644 --- a/tools/integration_tests/readonly_creds/failure_during_file_sync_test.go +++ b/tools/integration_tests/readonly_creds/failure_during_file_sync_test.go @@ -20,9 +20,11 @@ import ( "strings" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/test_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" ) //////////////////////////////////////////////////////////////////////// @@ -31,13 +33,14 @@ import ( type readOnlyCredsTest struct { testDirPath string + suite.Suite } -func (r *readOnlyCredsTest) Setup(t *testing.T) { +func (r *readOnlyCredsTest) SetupTest() { r.testDirPath = path.Join(setup.MntDir(), testDirName) } -func (r *readOnlyCredsTest) Teardown(t *testing.T) { +func (r *readOnlyCredsTest) TearDownTest() { } //////////////////////////////////////////////////////////////////////// @@ -65,24 +68,34 @@ func (r *readOnlyCredsTest) assertFileSyncFailsWithPermissionError(fh *os.File, // Test scenarios //////////////////////////////////////////////////////////////////////// -func (r *readOnlyCredsTest) TestEmptyCreateFileFails_FailedFileNotInListing(t *testing.T) { +func (r *readOnlyCredsTest) TestEmptyCreateFileFails_FailedFileNotInListing() { filePath := path.Join(r.testDirPath, testFileName) - fh := operations.CreateFile(filePath, operations.FilePermission_0777, t) - r.assertFileSyncFailsWithPermissionError(fh, t) + fh, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, operations.FilePermission_0777) + if setup.IsZonalBucketRun() { + require.Error(r.T(), err) + assert.True(r.T(), strings.Contains(err.Error(), permissionDeniedError)) + } else { + r.assertFileSyncFailsWithPermissionError(fh, r.T()) + } - r.assertFailedFileNotInListing(t) + r.assertFailedFileNotInListing(r.T()) } -func (r *readOnlyCredsTest) TestNonEmptyCreateFileFails_FailedFileNotInListing(t *testing.T) { +func (r *readOnlyCredsTest) TestNonEmptyCreateFileFails_FailedFileNotInListing() { filePath := path.Join(r.testDirPath, testFileName) - fh := operations.CreateFile(filePath, operations.FilePermission_0777, t) - operations.WriteWithoutClose(fh, content, t) - operations.WriteWithoutClose(fh, content, t) - r.assertFileSyncFailsWithPermissionError(fh, t) + fh, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, operations.FilePermission_0777) + if setup.IsZonalBucketRun() { + require.Error(r.T(), err) + assert.True(r.T(), strings.Contains(err.Error(), permissionDeniedError)) + } else { + operations.WriteWithoutClose(fh, content, r.T()) + operations.WriteWithoutClose(fh, content, r.T()) + r.assertFileSyncFailsWithPermissionError(fh, r.T()) + } - r.assertFailedFileNotInListing(t) + r.assertFailedFileNotInListing(r.T()) } //////////////////////////////////////////////////////////////////////// @@ -93,5 +106,5 @@ func TestReadOnlyTest(t *testing.T) { ts := &readOnlyCredsTest{} // Run tests. - test_setup.RunTests(t, ts) + suite.Run(t, ts) } diff --git a/tools/integration_tests/readonly_creds/readonly_creds_test.go b/tools/integration_tests/readonly_creds/readonly_creds_test.go index 4c692fd851..3124ddba86 100644 --- a/tools/integration_tests/readonly_creds/readonly_creds_test.go +++ b/tools/integration_tests/readonly_creds/readonly_creds_test.go @@ -22,50 +22,89 @@ import ( "testing" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/creds_tests" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/creds_tests" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" ) const ( - testDirName = "ReadOnlyCredsTest" + testDirName = "ReadonlyCredsTest" testFileName = "fileName.txt" content = "write content." permissionDeniedError = "permission denied" ) +var ( + // mount directory is where our tests run. + mountDir string + // root directory is the directory to be unmounted. + rootDir string +) + +type env struct { + storageClient *storage.Client + ctx context.Context + cfg *test_suite.TestConfig + bucketType string +} + +var testEnv env + //////////////////////////////////////////////////////////////////////// // TestMain //////////////////////////////////////////////////////////////////////// func TestMain(m *testing.M) { setup.ParseSetUpFlags() - setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() - if setup.MountedDirectory() != "" { - log.Println("These tests will not run with mounted directory..") - return + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.ReadonlyCreds) == 0 { + log.Println("No configuration found for readonly_creds tests in config. Using flags instead.") + cfg.ReadonlyCreds = make([]test_suite.TestConfig, 1) + cfg.ReadonlyCreds[0].TestBucket = setup.TestBucket() + cfg.ReadonlyCreds[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.ReadonlyCreds[0].Configs = make([]test_suite.ConfigItem, 1) + cfg.ReadonlyCreds[0].Configs[0].Flags = []string{"--implicit-dirs=true", "--implicit-dirs=false"} + cfg.ReadonlyCreds[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} } - // Create test directory. - ctx := context.Background() - var storageClient *storage.Client - closeStorageClient := client.CreateStorageClientWithCancel(&ctx, &storageClient) + testEnv.ctx = context.Background() + testEnv.bucketType = setup.TestEnvironment(testEnv.ctx, &cfg.ReadonlyCreds[0]) + testEnv.cfg = &cfg.ReadonlyCreds[0] + + // 2. Create storage client before running tests. + closeStorageClient := client.CreateStorageClientWithCancel(&testEnv.ctx, &testEnv.storageClient) defer func() { err := closeStorageClient() if err != nil { - log.Printf("closeStorageClient failed: %v", err) + log.Printf("closeStorageClient failed: %v\n", err) } }() - client.SetupTestDirectory(ctx, storageClient, testDirName) + + // 3. Skip for GKE or mounted directory tests. + if cfg.ReadonlyCreds[0].GKEMountedDirectory != "" { + log.Print("These tests will not run for mountedDirectory flag.") + os.Exit(0) + } + + // Create test directory on GCS before dropping privileges. + client.SetupTestDirectory(testEnv.ctx, testEnv.storageClient, testDirName) // Run tests for testBucket - setup.SetUpTestDirForTestBucketFlag() + // Set up test directory. + setup.SetUpTestDirForTestBucket(testEnv.cfg) + // Override GKE specific paths with GCSFuse paths if running in GCE environment. + setup.OverrideFilePathsInFlagSet(testEnv.cfg, setup.TestDir()) + + // Save mount and root directory variables. + mountDir, rootDir = testEnv.cfg.GCSFuseMountedDirectory, testEnv.cfg.GCSFuseMountedDirectory + flags := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, "") // Test for viewer permission on test bucket. - flags := [][]string{{"--implicit-dirs=true"}, {"--implicit-dirs=false"}} - successCode := creds_tests.RunTestsForKeyFileAndGoogleApplicationCredentialsEnvVarSet(ctx, storageClient, flags, "objectViewer", m) + successCode := creds_tests.RunTestsForDifferentAuthMethods(testEnv.ctx, testEnv.cfg, testEnv.storageClient, flags, "objectViewer", m) - setup.CleanupDirectoryOnGCS(ctx, storageClient, path.Join(setup.TestBucket(), testDirName)) + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(testEnv.cfg.TestBucket, testDirName)) os.Exit(successCode) } diff --git a/tools/integration_tests/release_version/release_version_test.go b/tools/integration_tests/release_version/release_version_test.go new file mode 100644 index 0000000000..c20d1a0420 --- /dev/null +++ b/tools/integration_tests/release_version/release_version_test.go @@ -0,0 +1,50 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package release_version + +import ( + "os/exec" + "regexp" + "strings" + "testing" +) + +func TestReleaseVersion(t *testing.T) { + cmd := exec.Command("gcsfuse", "--version") + + outputBytes, err := cmd.CombinedOutput() + + if err != nil { + t.Fatalf("Failed to execute 'gcsfuse --version': %v\nOutput: %s", err, string(outputBytes)) + } + output := strings.TrimSpace(string(outputBytes)) + t.Logf("gcsfuse --version output:\n%s", output) // Log the output for debugging + expectedPattern := `^gcsfuse version (\d+\.\d+\.\d+) \(Go version (go.+)\)$` + r := regexp.MustCompile(expectedPattern) + // Match the output against the pattern + matches := r.FindStringSubmatch(output) + if len(matches) != 3 { // Expect 3 elements: full match, version, go version + t.Errorf("Output did not match expected pattern.\nExpected pattern: %q\nActual output: %q\nMatches: %v", expectedPattern, output, matches) + } else { + version := matches[1] + goVersion := matches[2] + if version == "" { + t.Errorf("Extracted gcsfuse version is empty") + } + if goVersion == "" { + t.Errorf("Extracted Go version is empty") + } + } +} diff --git a/tools/integration_tests/release_version/setup_test.go b/tools/integration_tests/release_version/setup_test.go new file mode 100644 index 0000000000..d0dd3969da --- /dev/null +++ b/tools/integration_tests/release_version/setup_test.go @@ -0,0 +1,50 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package release_version + +import ( + "log" + "os" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" +) + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.ReleaseVersion) == 0 { + log.Println("No configuration found for release_version tests in config. Using flags instead.") + cfg.ReleaseVersion = make([]test_suite.TestConfig, 1) + cfg.ReleaseVersion[0].TestBucket = setup.TestBucket() + cfg.ReleaseVersion[0].GKEMountedDirectory = setup.MountedDirectory() + } + + // 2. Not running mounted directory tests. + if cfg.ReleaseVersion[0].GKEMountedDirectory != "" { + log.Print("These tests will not run for mountedDirectory flag.") + os.Exit(0) + } + + // 3. The release_version test doesn't mount anything, but it needs the gcsfuse binary. + setup.SetUpTestDirForTestBucket(&cfg.ReleaseVersion[0]) + + // 4. Run tests. + successCode := m.Run() + os.Exit(successCode) +} diff --git a/tools/integration_tests/rename_dir_limit/move_dir_test.go b/tools/integration_tests/rename_dir_limit/move_dir_test.go index 19b4bb0389..756826cb20 100644 --- a/tools/integration_tests/rename_dir_limit/move_dir_test.go +++ b/tools/integration_tests/rename_dir_limit/move_dir_test.go @@ -23,8 +23,8 @@ import ( "path" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) const SrcMoveDirectory = "srcMoveDir" @@ -78,7 +78,7 @@ func createSrcDirectoryWithObjectsForMoveDirTest(dirPath string, t *testing.T) { } // Closing file at the end - defer operations.CloseFile(file) + defer operations.CloseFileShouldNotThrowError(t, file) err = operations.WriteFile(file.Name(), SrcMoveFileContent) if err != nil { diff --git a/tools/integration_tests/rename_dir_limit/rename_dir_limit_test.go b/tools/integration_tests/rename_dir_limit/rename_dir_limit_test.go index 3e53dcca49..a4794513b4 100644 --- a/tools/integration_tests/rename_dir_limit/rename_dir_limit_test.go +++ b/tools/integration_tests/rename_dir_limit/rename_dir_limit_test.go @@ -22,11 +22,12 @@ import ( "testing" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/only_dir_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/persistent_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/only_dir_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/persistent_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" ) const DirForRenameDirLimitTests = "dirForRenameDirLimitTests" @@ -51,9 +52,32 @@ var ( func TestMain(m *testing.M) { setup.ParseSetUpFlags() - var err error + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.RenameDirLimit) == 0 { + log.Println("No configuration found for rename dir limit tests in config. Using flags instead.") + // Populate the config manually. + cfg.RenameDirLimit = make([]test_suite.TestConfig, 1) + cfg.RenameDirLimit[0].TestBucket = setup.TestBucket() + cfg.RenameDirLimit[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.RenameDirLimit[0].Configs = make([]test_suite.ConfigItem, 2) + cfg.RenameDirLimit[0].Configs[0].Flags = []string{ + "--rename-dir-limit=3 --implicit-dirs --client-protocol=grpc", + "--rename-dir-limit=3", + "--rename-dir-limit=3 --client-protocol=grpc", + } + cfg.RenameDirLimit[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": false, "zonal": false} + cfg.RenameDirLimit[0].Configs[1].Flags = []string{ + "", + } + cfg.RenameDirLimit[0].Configs[1].Compatible = map[string]bool{"flat": false, "hns": true, "zonal": true} + } ctx = context.Background() + bucketType := setup.TestEnvironment(ctx, &cfg.RenameDirLimit[0]) + + // 2. Create storage client before running tests. + var err error storageClient, err = client.CreateStorageClient(ctx) if err != nil { log.Printf("Error creating storage client: %v\n", err) @@ -61,31 +85,24 @@ func TestMain(m *testing.M) { } defer storageClient.Close() - flags := [][]string{{"--rename-dir-limit=3", "--implicit-dirs"}, {"--rename-dir-limit=3"}} - if hnsFlagSet, err := setup.AddHNSFlagForHierarchicalBucket(ctx, storageClient); err == nil { - flags = [][]string{hnsFlagSet} + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + if cfg.RenameDirLimit[0].GKEMountedDirectory != "" && cfg.RenameDirLimit[0].TestBucket != "" { + os.Exit(setup.RunTestsForMountedDirectory(cfg.RenameDirLimit[0].GKEMountedDirectory, m)) } - setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() - - if setup.TestBucket() == "" && setup.MountedDirectory() != "" { - log.Print("Please pass the name of bucket mounted at mountedDirectory to --testBucket flag.") - os.Exit(1) - } - - // Run tests for mountedDirectory only if --mountedDirectory flag is set. - setup.RunTestsForMountedDirectoryFlag(m) // Run tests for testBucket - setup.SetUpTestDirForTestBucketFlag() + // 4. Build the flag sets dynamically from the config. + flags := setup.BuildFlagSets(cfg.RenameDirLimit[0], bucketType, "") + setup.SetUpTestDirForTestBucket(&cfg.RenameDirLimit[0]) - successCode := static_mounting.RunTests(flags, m) + successCode := static_mounting.RunTestsWithConfigFile(&cfg.RenameDirLimit[0], flags, m) if successCode == 0 { - successCode = only_dir_mounting.RunTests(flags, onlyDirMounted, m) + successCode = only_dir_mounting.RunTestsWithConfigFile(&cfg.RenameDirLimit[0], flags, onlyDirMounted, m) } if successCode == 0 { - successCode = persistent_mounting.RunTests(flags, m) + successCode = persistent_mounting.RunTestsWithConfigFile(&cfg.RenameDirLimit[0], flags, m) } os.Exit(successCode) diff --git a/tools/integration_tests/rename_dir_limit/rename_dir_test.go b/tools/integration_tests/rename_dir_limit/rename_dir_test.go index 1cc567b274..adf8c9d047 100644 --- a/tools/integration_tests/rename_dir_limit/rename_dir_test.go +++ b/tools/integration_tests/rename_dir_limit/rename_dir_test.go @@ -20,10 +20,11 @@ import ( "os" "os/exec" "path" + "syscall" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" "github.com/stretchr/testify/assert" ) @@ -80,7 +81,7 @@ func TestRenameDirectoryWithTwoFiles(t *testing.T) { // As --rename-directory-limit = 3, and the number of objects in the directory is two, // which is greater than the limit, the operation should get fail. func TestRenameDirectoryWithFourFiles(t *testing.T) { - if setup.IsHierarchicalBucket(ctx, storageClient) { + if setup.ResolveIsHierarchicalBucket(ctx, setup.TestBucket(), storageClient) { t.SkipNow() } testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests) @@ -134,7 +135,7 @@ func TestRenameDirectoryWithTwoFilesAndOneEmptyDirectory(t *testing.T) { // As --rename-directory-limit = 3, and the number of objects in the directory is Four, // which is greater than the limit, the operation should get fail. func TestRenameDirectoryWithTwoFilesAndOneNonEmptyDirectory(t *testing.T) { - if setup.IsHierarchicalBucket(ctx, storageClient) { + if setup.ResolveIsHierarchicalBucket(ctx, setup.TestBucket(), storageClient) { t.SkipNow() } testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests) @@ -197,3 +198,25 @@ func TestRenameDirectoryWithExistingEmptyDestDirectory(t *testing.T) { assert.Equal(t, "temp1", dirEntries[1].Name()) assert.False(t, dirEntries[1].IsDir()) } + +func TestRenameDirectoryWithExistingNonEmptyDestDirectory(t *testing.T) { + testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests) + // Creating directory structure + // testBucket/dirForRenameDirLimitTests/srcDirectory -- Dir + // testBucket/dirForRenameDirLimitTests/srcDirectory/temp1 -- File + // testBucket/dirForRenameDirLimitTests/destNonEmptyDirectory -- Dir + // testBucket/dirForRenameDirLimitTests/destNonEmptyDirectory/temp1 -- File + oldDirPath := path.Join(testDir, SrcDirectory) + operations.CreateDirectoryWithNFiles(1, oldDirPath, PrefixTempFile, t) + newDirPath := path.Join(testDir, "destNonEmptyDirectory") + operations.CreateDirectoryWithNFiles(1, newDirPath, PrefixTempFile, t) + + // Go's Rename function does not support renaming a directory into an existing directory. + // To achieve this, we call a Python rename function as a workaround. + // We catch OSError and exit with the errno so we can validate ENOTEMPTY. + cmd := exec.Command("python3", "-c", fmt.Sprintf("import os, sys; exec(\"try:\\n os.rename('%s', '%s')\\nexcept OSError as e:\\n sys.exit(e.errno)\")", oldDirPath, newDirPath)) + _, err := cmd.CombinedOutput() + + assert.Error(t, err) + assert.Equal(t, int(syscall.ENOTEMPTY), err.(*exec.ExitError).ExitCode()) +} diff --git a/tools/integration_tests/requester_pays_bucket/operations_test.go b/tools/integration_tests/requester_pays_bucket/operations_test.go new file mode 100644 index 0000000000..87ce00ae3e --- /dev/null +++ b/tools/integration_tests/requester_pays_bucket/operations_test.go @@ -0,0 +1,153 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package requester_pays_bucket + +import ( + "io/fs" + "os" + "path/filepath" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +type operationTests struct { + suite.Suite +} + +func (t *operationTests) SetupSuite() { + testEnv.testDirPath = filepath.Join(setup.MntDir(), testDirName) + err := os.MkdirAll(testEnv.testDirPath, 0755) + if err != nil { + t.T().Fatalf("Failed to create directory: %v", err) + } + +} + +func (t *operationTests) TearDownSuite() { + err := os.RemoveAll(testEnv.testDirPath) + assert.NoError(t.T(), err) +} + +//////////////////////////////////////////////////////////////////////// +// Test scenarios +//////////////////////////////////////////////////////////////////////// + +// Test all the folder operations in the mount. +func (t *operationTests) TestDirOperations() { + var fi fs.FileInfo + var err error + dirName := "dir" + setup.GenerateRandomString(5) + mountedDirPath := filepath.Join(testEnv.testDirPath, dirName) + + // Check that the directory does not exist initially. + _, err = os.Stat(mountedDirPath) + require.Error(t.T(), err) + require.True(t.T(), os.IsNotExist(err), "Directory should not exist before creation") + + // Create the directory. + err = os.Mkdir(mountedDirPath, setup.FilePermission_0600) + require.NoError(t.T(), err) + + // Check that the directory now exists. + fi, err = os.Stat(mountedDirPath) + require.NoError(t.T(), err) + require.True(t.T(), fi.IsDir(), "%q should be a directory", mountedDirPath) + + // Rename the directory. + renamedDirName := "dir" + setup.GenerateRandomString(5) + mountedRenamedDirPath := filepath.Join(testEnv.testDirPath, renamedDirName) + err = os.Rename(mountedDirPath, mountedRenamedDirPath) + require.NoError(t.T(), err) + + // Check that the old path no longer exists and the new one does. + _, err = os.Stat(mountedDirPath) + require.Error(t.T(), err) + require.True(t.T(), os.IsNotExist(err), "Old directory path should not exist after rename") + fi, err = os.Stat(mountedRenamedDirPath) + require.NoError(t.T(), err, "New directory path should exist after rename") + require.True(t.T(), fi.IsDir(), "%q should be a directory", mountedRenamedDirPath) + + // Remove the directory. + err = os.RemoveAll(mountedRenamedDirPath) + require.NoError(t.T(), err) + + // Check that the directory is removed. + _, err = os.Stat(mountedRenamedDirPath) + require.Error(t.T(), err) + require.True(t.T(), os.IsNotExist(err), "Directory should not exist after removal") +} + +// Test all the file operations in the mount. +func (t *operationTests) TestFileOperations() { + var fi fs.FileInfo + var err error + contentLength := 5 + content := setup.GenerateRandomString(contentLength) + fileName := "file" + setup.GenerateRandomString(5) + mountedFilePath := filepath.Join(testEnv.testDirPath, fileName) + + // Check that the file does not exist initially. + _, err = os.Stat(mountedFilePath) + require.Error(t.T(), err) + require.True(t.T(), os.IsNotExist(err), "File should not exist before creation") + + // Create the file. + operations.CreateFileWithContent(mountedFilePath, setup.FilePermission_0600, content, t.T()) + + // Check that the file now exists. + fi, err = os.Stat(mountedFilePath) + require.NoError(t.T(), err) + require.False(t.T(), fi.IsDir(), "%q should be a file", mountedFilePath) + + // Rename the file. + renamedFileName := "rename-output-file-" + setup.GenerateRandomString(5) + mountedRenamedFilePath := filepath.Join(testEnv.testDirPath, renamedFileName) + err = os.Rename(mountedFilePath, mountedRenamedFilePath) + require.NoError(t.T(), err) + + // Check that the old path no longer exists and the new one does. + _, err = os.Stat(mountedFilePath) + require.Error(t.T(), err) + require.True(t.T(), os.IsNotExist(err), "Old file path should not exist after rename") + fi, err = os.Stat(mountedRenamedFilePath) + require.NoError(t.T(), err, "New file path should exist after rename") + require.False(t.T(), fi.IsDir(), "%q should be a file", mountedRenamedFilePath) + + // Remove the renamed file. + err = os.RemoveAll(mountedRenamedFilePath) + require.NoError(t.T(), err) + + // Check that the renamed file is removed. + _, err = os.Stat(mountedRenamedFilePath) + require.Error(t.T(), err) + require.True(t.T(), os.IsNotExist(err), "file should not exist after removal") +} + +//////////////////////////////////////////////////////////////////////// +// Test Function (Runs once before all tests) +//////////////////////////////////////////////////////////////////////// + +func TestOperations(t *testing.T) { + suite.Run(t, new(operationTests)) +} diff --git a/tools/integration_tests/requester_pays_bucket/setup_test.go b/tools/integration_tests/requester_pays_bucket/setup_test.go new file mode 100644 index 0000000000..8fe4f3e907 --- /dev/null +++ b/tools/integration_tests/requester_pays_bucket/setup_test.go @@ -0,0 +1,186 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Provide tests for cases where bucket with requester-pays feature is +// mounted and used through gcsfuse. +package requester_pays_bucket + +import ( + "context" + "log" + "os" + "path" + "strings" + "testing" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/creds_tests" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/only_dir_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" +) + +const ( + testDirName = "RequesterPaysBucketTests" + onlyDirTestDirName = "OnlyDirRequesterPaysBucketTests" + + // requesterPaysServiceAccountName is the name of the service account used for requester-pays testing. + // This service account must exist in the active GCP project where tests are run + // (e.g., "gcs-fuse-test" or "gcs-fuse-test-ml"). + // The test expects a JSON key for this SA to be stored in Secret Manager + // within the same project, under the secret name specified by requesterPaysCredsSecretName. + // Note: The user or service account running the test package, as well as the + // requester-pays service account (requester-pays-tester), must be granted the + // Storage Admin and Service Usage Consumer roles on the project. + // For example, adding the Service Usage Consumer role to + // requester-pays-tester@gcs-fuse-test.iam.gserviceaccount.com in the gcs-fuse-test project. + requesterPaysServiceAccountName = "requester-pays-tester" + requesterPaysCredsSecretName = "requester-pays-tester" + targetBillingProject = "gcs-fuse-test" +) + +// To prevent global variable pollution, enhance code clarity, +// and avoid inadvertent errors. We strongly suggest that, all new package-level +// variables (which would otherwise be declared with `var` at the package root) should +// be added as fields to this 'env' struct instead. +type env struct { + testDirPath string + storageClient *storage.Client + ctx context.Context + bucketName string +} + +var ( + testEnv env +) + +//////////////////////////////////////////////////////////////////////// +// TestMain +//////////////////////////////////////////////////////////////////////// + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + if setup.IsZonalBucketRun() { + log.Fatal("Test not supported for zonal bucket as they don't support requester-pays feature") + } + + // Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.RequesterPaysBucket) == 0 { + log.Println("No configuration found for requester pays bucket tests in config. Using flags instead.") + // Populate the config manually. + cfg.RequesterPaysBucket = make([]test_suite.TestConfig, 1) + cfg.RequesterPaysBucket[0].TestBucket = setup.TestBucket() + cfg.RequesterPaysBucket[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.RequesterPaysBucket[0].Configs = make([]test_suite.ConfigItem, 1) + cfg.RequesterPaysBucket[0].Configs[0].Flags = []string{ + "--billing-project=${BILLING_PROJECT} --key-file=${KEY_FILE}", + "--billing-project=${BILLING_PROJECT} --client-protocol=grpc --key-file=${KEY_FILE}", + "--billing-project=${BILLING_PROJECT} --client-protocol=grpc --grpc-path-strategy=direct-path-only --key-file=${KEY_FILE}", + } + cfg.RequesterPaysBucket[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": false} + } + + testEnv.ctx = context.Background() + + // When not running in GKE environment. + if cfg.RequesterPaysBucket[0].GKEMountedDirectory == "" { + // Replace --billing-project= placeholder in flags with the default billing project. + for i := range cfg.RequesterPaysBucket[0].Configs { + for j, flag := range cfg.RequesterPaysBucket[0].Configs[i].Flags { + cfg.RequesterPaysBucket[0].Configs[i].Flags[j] = setup.ReplaceOrAppendFlag(flag, "${BILLING_PROJECT}", "--billing-project=", targetBillingProject) + } + } + // Setup service account credentials for requester-pays testing. + _, localKeyFilePath := creds_tests.CreateCredentialsForSA(testEnv.ctx, requesterPaysServiceAccountName, requesterPaysCredsSecretName) + defer func() { + if err := os.Remove(localKeyFilePath); err != nil { + log.Printf("Failed to delete temp credentials file %s: %v", localKeyFilePath, err) + } + }() + setup.SetKeyFile(localKeyFilePath) + for i := range cfg.RequesterPaysBucket[0].Configs { + for j, flag := range cfg.RequesterPaysBucket[0].Configs[i].Flags { + cfg.RequesterPaysBucket[0].Configs[i].Flags[j] = setup.ReplaceOrAppendFlag(flag, "${KEY_FILE}", "--key-file=", localKeyFilePath) + } + } + } + + // Extract billing project from flags. + var billingProject string + for _, flag := range cfg.RequesterPaysBucket[0].Configs[0].Flags { + if strings.Contains(flag, "--billing-project=") { + parts := strings.Split(flag, "--billing-project=") + if len(parts) > 1 { + // Split by comma first in case flags are comma-separated. + val := strings.Split(parts[1], ",")[0] + // Then take the first field in case they are space-separated. + fields := strings.Fields(val) + if len(fields) > 0 { + billingProject = fields[0] + } + break + } + } + } + if billingProject == "" { + log.Fatal("Billing project is not set. It must be set using environment variables in GKE envrionment and by replacing the '${BILLING_PROJECT}' string in non-GKE environements.") + } + setup.SetBillingProject(billingProject) + + // Create storage client before running tests. + closeStorageClient := client.CreateStorageClientWithCancel(&testEnv.ctx, &testEnv.storageClient) + defer func() { + err := closeStorageClient() + if err != nil { + log.Fatalf("closeStorageClient failed: %v", err) + } + }() + + testEnv.bucketName = strings.Split(cfg.RequesterPaysBucket[0].TestBucket, "/")[0] + wasEnabled := client.MustEnableRequesterPays(testEnv.storageClient, testEnv.ctx, testEnv.bucketName) + if wasEnabled { + defer client.MustDisableRequesterPays(testEnv.storageClient, testEnv.ctx, testEnv.bucketName) + } + + // To run mountedDirectory tests, we need both testBucket and mountedDirectory + // flags to be set, as RequesterPaysBucket tests validates content from the bucket. + if cfg.RequesterPaysBucket[0].GKEMountedDirectory != "" && cfg.RequesterPaysBucket[0].TestBucket != "" { + os.Exit(setup.RunTestsForMountedDirectory(cfg.RequesterPaysBucket[0].GKEMountedDirectory, m)) + } + + // Run tests for testBucket + // Build the flag sets dynamically from the config. + bucketType := setup.TestEnvironment(testEnv.ctx, &cfg.RequesterPaysBucket[0]) + flags := setup.BuildFlagSets(cfg.RequesterPaysBucket[0], bucketType, "") + setup.SetUpTestDirForTestBucket(&cfg.RequesterPaysBucket[0]) + + log.Println("Running static mounting tests...") + successCode := static_mounting.RunTestsWithConfigFile(&cfg.RequesterPaysBucket[0], flags, m) + + if successCode == 0 { + log.Printf("Running only-dir mounting tests ...") + successCode = only_dir_mounting.RunTestsWithConfigFile(&cfg.RequesterPaysBucket[0], flags, onlyDirTestDirName, m) + } + + // If failed, then save the gcsfuse log file(s). + setup.SaveLogFileInCaseOfFailure(successCode) + + // Clean up test directory created. + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(setup.TestBucket(), testDirName)) + os.Exit(successCode) +} diff --git a/tools/integration_tests/resource_usage.sh b/tools/integration_tests/resource_usage.sh new file mode 100755 index 0000000000..d826bf385e --- /dev/null +++ b/tools/integration_tests/resource_usage.sh @@ -0,0 +1,174 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Exit on error, treat unset variables as errors, and propagate pipeline errors. +set -euo pipefail + +# This script is used to collect the resource(mem,cpu,disk) usage of the machine in the file at given interval. +# This script can also be in use to print the resource usage visualisation in graph format. +usage() { + echo "A script to collect and visualize system resource usage." + echo "" + echo "========================================================================" + echo "" + echo "COMMAND: COLLECT" + echo "" + echo " Gathers resource usage (CPU, memory, disk) at predefined intervals." + echo "" + echo " USAGE:" + echo " $0 COLLECT <FILE_PATH>" + echo "" + echo " ARGUMENTS:" + echo "" + echo " <FILE_PATH> The path to the output file where resource data will be" + echo " saved. The script will create or overwrite this file." + echo "" + echo "========================================================================" + echo "" + echo "COMMAND: PRINT" + echo "" + echo " Visualizes resource data from a file in a graph format." + echo "" + echo " USAGE:" + echo " $0 PRINT <FILE_PATH>" + echo "" + echo " ARGUMENTS:" + echo " <FILE_PATH> The path to the input data file (generated by COLLECT)" + echo " that contains the statistics to be visualized." + echo "" + echo "========================================================================" + exit 1 +} + +readonly USAGE_COLLECTION_INTERVAL_SECONDS=10 # Collect usage every 10s. +readonly USAGE_COLLECTION_DURATION_SECONDS=$((3 * 60 * 60)) # Auto collect usage for upto 3 hrs only (VM timeout is 3 hrs in kokoro). +readonly COLLECT="COLLECT" +readonly PRINT="PRINT" + +# Validate number of arguments. +if [[ $# -ne 2 ]]; then + echo "Error: Invalid number of arguments." + usage +fi + +readonly COMMAND="$1" +readonly FILE_PATH="$2" + +# Calculates CPU usage as integer percentage using top command. +get_cpu_usage() { + # 'top -b -n 1 | awk '/^%Cpu/'' returns + # %Cpu(s): 1.1 us, 0.0 sy, 0.0 ni, 98.9 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st + # 8th entry is idle cpu % + top -b -n 1 | awk '/^%Cpu/ { usage = 100 - $8; print int(usage) }' || echo 0 +} + +# Calculates memory usage as integer percentage using free. +get_mem_usage() { + # 'free' returns + # total used free shared buff/cache available + # Mem: 371458720 6879376 285303364 9772 81770240 364579344 + # Swap: 0 0 0 + free | awk '/Mem:/ {p=($2-$7)*100/$2; print int(p) }' || echo 0 +} + +# Calculates disk usage as integer percentage using df. +get_disk_usage() { + # 'df -P /' returns + # Filesystem 1024-blocks Used Available Capacity Mounted on + # /dev/nvme0n1p1 515794480 82928236 411780644 17% / + df -P / | tail -n1 | awk '{sub(/%/, ""); printf "%.0f\n", $5}' || echo 0 +} + +# Helper function to collect the usage and write to file. +# Output: Appends new line to file with %cpu %mem %disk at given interval. +collect_all_metric() { + echo "CPU MEM DISK" >"$FILE_PATH" + while ((${SECONDS:-0} < USAGE_COLLECTION_DURATION_SECONDS)); do + { + cpu=$(get_cpu_usage) + mem=$(get_mem_usage) + disk=$(get_disk_usage) + echo "$cpu $mem $disk" + } >>"$FILE_PATH" + sleep "$USAGE_COLLECTION_INTERVAL_SECONDS" + done + echo "Exiting due to monitoring time limit." +} + +# Helper function to print usage of all metrics. +print_each_metric() { + # Read headers from file. + read -r -a headers <"$FILE_PATH" + declare -A data_columns + local line_num=0 + while read -r -a values; do + # Skip the header line we already read + ((line_num++ == 0)) && continue + + for i in "${!headers[@]}"; do + local header="${headers[$i]}" + local value="${values[$i]}" + # Append the value to the correct string in the associative array + data_columns["$header"]+="$value " + done + done <"$FILE_PATH" + + # Iterate through each resource and print its graph + for resource in "${headers[@]}"; do + local -a usage_values=(${data_columns[$resource]}) + local max_value=0 + for val in "${usage_values[@]}"; do + ((val > max_value)) && max_value=$val + done + + # Add some buffer to the graph ceiling, capping at 100 + local graph_ceiling=$((max_value + 5)) + ((graph_ceiling > 100)) && graph_ceiling=100 + + printf "\n%s usage in percentage every %s seconds:\n\n" "$resource" "$USAGE_COLLECTION_INTERVAL_SECONDS" + + # Print the graph from top to bottom + for ((y = graph_ceiling; y >= 0; y--)); do + printf "%s@ %3d%%|" "$resource" "$y" + for val in "${usage_values[@]}"; do + if ((val >= y)); then + printf "*" + else + printf " " + fi + done + printf "\n" + done + + # Print separator line + local total_width=$((${#usage_values[@]} + 15)) + for ((i = 0; i < total_width; i++)); do + printf "=" + done + printf "\n\n" + done +} + +# Exit the script if SIGTERM or SIGINT is received. +trap 'exit 0' SIGINT SIGTERM + +if [[ "$COMMAND" == "$COLLECT" ]]; then + collect_all_metric +elif [[ "$COMMAND" == "$PRINT" ]]; then + print_each_metric +else + echo "Error: Invalid command." + usage +fi diff --git a/tools/integration_tests/run_e2e_tests.sh b/tools/integration_tests/run_e2e_tests.sh index 48337c304c..33f6eb2390 100755 --- a/tools/integration_tests/run_e2e_tests.sh +++ b/tools/integration_tests/run_e2e_tests.sh @@ -39,6 +39,36 @@ if [ $# -ge 5 ] ; then RUN_TESTS_WITH_PRESUBMIT_FLAG=$5 fi +# 6th parameter is set to enable/disable run for zonal bucket(s). +# If it is set to true, then the run will be only on zonal bucket(s), +# otherwise the run will only on non-zonal bucket(s). +RUN_TESTS_WITH_ZONAL_BUCKET=false +if [[ $# -ge 6 ]] ; then + if [[ "$6" == "true" ]]; then + RUN_TESTS_WITH_ZONAL_BUCKET=true + elif [[ "$6" != "false" ]]; then + echo "Error: Invalid value for 6th argument: "$6" . Expected: true or false." + exit 1 + fi +fi + +# 7th parameter is to determine whether we want to disable build by the script +# and let every test package build its own GCSFuse binary. +BUILD_BINARY_IN_SCRIPT=true +if [[ $# -ge 7 ]] ; then + if [[ "$7" == "false" ]]; then + BUILD_BINARY_IN_SCRIPT=false + fi +fi + + +if ${RUN_TESTS_WITH_ZONAL_BUCKET}; then + if [ "${BUCKET_LOCATION}" != "us-west4" ] && [ "${BUCKET_LOCATION}" != "us-central1" ]; then + >&2 echo "For enabling zonal bucket run, BUCKET_LOCATION should be one of: us-west4, us-central1; passed: ${BUCKET_LOCATION}" + exit 1 + fi +fi + if [ "$#" -lt 3 ] then echo "Incorrect number of arguments passed, please refer to the script and pass the three arguments required..." @@ -64,10 +94,12 @@ echo "Setting the integration test timeout to: $INTEGRATION_TEST_TIMEOUT" readonly RANDOM_STRING_LENGTH=5 # Test directory arrays TEST_DIR_PARALLEL=( + "monitoring" "local_file" "log_rotation" "mounting" "read_cache" + # "grpc_validation" "gzip" "write_large_files" "list_large_dir" @@ -77,9 +109,22 @@ TEST_DIR_PARALLEL=( "implicit_dir" "interrupt" "operations" - "log_content" "kernel_list_cache" - "concurrent_operations" + "benchmarking" + "mount_timeout" + "stale_handle" + "negative_stat_cache" + "streaming_writes" + "inactive_stream_timeout" + "cloud_profiler" + "release_version" + "readdirplus" + "dentry_cache" + "buffered_read" + "requester_pays_bucket" + "flag_optimizations" + "unsupported_path" + "symlink_handling" ) # These tests never become parallel as it is changing bucket permissions. @@ -87,33 +132,138 @@ TEST_DIR_NON_PARALLEL=( "readonly" "managed_folders" "readonly_creds" + "concurrent_operations" +) + +# Subset of TEST_DIR_PARALLEL, +# but only those tests which currently +# pass for zonal buckets. +TEST_DIR_PARALLEL_FOR_ZB=( + "benchmarking" + "explicit_dir" + "gzip" + "implicit_dir" + "interrupt" + "kernel_list_cache" + "local_file" + "log_rotation" + "monitoring" + "mount_timeout" + "mounting" + "negative_stat_cache" + "operations" + "rapid_appends" + "read_cache" + "read_large_files" + "rename_dir_limit" + "stale_handle" + "streaming_writes" + "write_large_files" + "unfinalized_object" + "release_version" + "readdirplus" + "dentry_cache" + "flag_optimizations" + "unsupported_path" + "symlink_handling" +) + +# Subset of TEST_DIR_NON_PARALLEL, +# but only those tests which currently +# pass for zonal buckets. +TEST_DIR_NON_PARALLEL_FOR_ZB=( + "concurrent_operations" + "list_large_dir" + "managed_folders" + "readonly" + "readonly_creds" ) # Create a temporary file to store the log file name. TEST_LOGS_FILE=$(mktemp) +# This variable will store the path if the script builds GCSFuse binaries (gcsfuse, mount.gcsfuse) +BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR="" +# This variable will hold flag and its value to be passed to GCSFuse tests (--gcsfuse_prebuilt_dir=...) +USE_PREBUILT_GCSFUSE_BINARY="" + +build_gcsfuse_once() { + local build_output_dir # For the final gcsfuse binaries + build_output_dir=$(mktemp -d -t gcsfuse_e2e_run_build_XXXXXX) + echo "GCSFuse binaries will be built in ${build_output_dir}..." + + local gcsfuse_src_dir + # Determine GCSFuse source directory + # If this script is in tools/integration_tests, project root is ../../ + SCRIPT_DIR_REALPATH=$(realpath "$(dirname "${BASH_SOURCE[0]}")") + gcsfuse_src_dir=$(realpath "${SCRIPT_DIR_REALPATH}/../../") + + if [[ ! -f "${gcsfuse_src_dir}/go.mod" ]]; then + echo "Error: Could not reliably determine GCSFuse project root from ${SCRIPT_DIR_REALPATH}. Expected go.mod at ${gcsfuse_src_dir}" >&2 + rm -rf "${build_output_dir}" + exit 1 + fi + echo "Using GCSFuse source directory: ${gcsfuse_src_dir}" + + echo "Building GCSFuse using 'go run ./tools/build_gcsfuse/main.go'..." + (cd "${gcsfuse_src_dir}" && go run ./tools/build_gcsfuse/main.go . "${build_output_dir}" "e2e-$(date +%s)") + if [ $? -ne 0 ]; then + echo "Error building GCSFuse binaries using 'go run ./tools/build_gcsfuse/main.go'." + rm -rf "${build_output_dir}" # Clean up created temp dir + return 1 + fi + + # Set the directory path for use by the script (to form the go test flag) + BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR="${build_output_dir}" + echo "GCSFuse binaries built by script in: ${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}" + echo "GCSFuse executable: ${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}/bin/gcsfuse" + return 0 +} + + +cleanup_gcsfuse_once() { + if [ -n "${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}" ] && [ -d "${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}" ]; then + echo "Cleaning up GCSFuse build directory created by script: ${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}" + rm -rf "${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}" + fi +} + +# Delete contents of the buckets (and then the buckets themselves) whose names are in the passed file. +# Args: <bucket-names-file> +function delete_buckets_listed_in_file() { + local bucketNamesFile="${@}" + if test -f "${bucketNamesFile}"; then + cat "${bucketNamesFile}" | while read bucket; do + # Only if bucket-name is non-empty and contains + # something other than spaces. + if [ -n "${bucket}" ] && [ -n "${bucket// }" ]; then + # Delete the bucket and its contents. + if ! gcloud -q storage rm -r --verbosity=none gs://${bucket} ; then + >&2 echo "Failed to delete bucket ${bucket} !" + fi + fi + done + # At the end, delete the bucket-names file itself. + rm "${bucketNamesFile}" + else + echo "file ${bucketNamesFile} not found !" + fi +} + function upgrade_gcloud_version() { - sudo apt-get update - # Upgrade gcloud version. - # Kokoro machine's outdated gcloud version prevents the use of the "managed-folders" feature. - gcloud version - wget -O gcloud.tar.gz https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz -q - sudo tar xzf gcloud.tar.gz && sudo cp -r google-cloud-sdk /usr/local && sudo rm -r google-cloud-sdk - sudo /usr/local/google-cloud-sdk/install.sh - export PATH=/usr/local/google-cloud-sdk/bin:$PATH - echo 'export PATH=/usr/local/google-cloud-sdk/bin:$PATH' >> ~/.bashrc - gcloud version && rm gcloud.tar.gz - sudo /usr/local/google-cloud-sdk/bin/gcloud components update - sudo /usr/local/google-cloud-sdk/bin/gcloud components install alpha + # Install latest gcloud. + ./perfmetrics/scripts/install_latest_gcloud.sh + export PATH="/usr/local/google-cloud-sdk/bin:$PATH" + export CLOUDSDK_PYTHON="$HOME/.local/python-3.11.9/bin/python3.11" + export PATH="$HOME/.local/python-3.11.9/bin:$PATH"" } function install_packages() { - # e.g. architecture=arm64 or amd64 - architecture=$(dpkg --print-architecture) - echo "Installing go-lang 1.23.3..." - wget -O go_tar.tar.gz https://go.dev/dl/go1.23.3.linux-${architecture}.tar.gz -q - sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz && sudo mv go /usr/local - export PATH=$PATH:/usr/local/go/bin + # Install required go version. + ./perfmetrics/scripts/install_go.sh "$(cat .go-version)" + export PATH="/usr/local/go/bin:$PATH" + + sudo apt-get update sudo apt-get install -y python3 # install python3-setuptools tools. sudo apt-get install -y gcc python3-dev python3-setuptools @@ -126,7 +276,7 @@ function create_bucket() { bucket_prefix=$1 local -r project_id="gcs-fuse-test-ml" # Generate bucket name with random string - bucket_name=$bucket_prefix$(tr -dc 'a-z0-9' < /dev/urandom | head -c $RANDOM_STRING_LENGTH) + bucket_name=${bucket_prefix}$(date +%Y%m%d-%H%M%S)"-"$(tr -dc 'a-z0-9' < /dev/urandom | head -c $RANDOM_STRING_LENGTH) # We are using gcloud alpha because gcloud storage is giving issues running on Kokoro gcloud alpha storage buckets create gs://$bucket_name --project=$project_id --location=$BUCKET_LOCATION --uniform-bucket-level-access echo $bucket_name @@ -137,15 +287,31 @@ function create_hns_bucket() { # Generate bucket name with random string. # Adding prefix `golang-grpc-test` to white list the bucket for grpc # so that we can run grpc related e2e tests. - bucket_name="golang-grpc-test-gcsfuse-e2e-tests-hns-"$(tr -dc 'a-z0-9' < /dev/urandom | head -c $RANDOM_STRING_LENGTH) + bucket_name="golang-grpc-test-gcsfuse-e2e-tests-hns-"$(date +%Y%m%d-%H%M%S)"-"$(tr -dc 'a-z0-9' < /dev/urandom | head -c $RANDOM_STRING_LENGTH) gcloud alpha storage buckets create gs://$bucket_name --project=$hns_project_id --location=$BUCKET_LOCATION --uniform-bucket-level-access --enable-hierarchical-namespace echo "$bucket_name" } +function create_zonal_bucket() { + local -r project_id="gcs-fuse-test-ml" + local -r region=${BUCKET_LOCATION} + local -r zone=${region}"-a" + + local -r hns_project_id="gcs-fuse-test" + # Generate bucket name with random string. + bucket_name="gcsfuse-e2e-tests-zb-"$(date +%Y%m%d-%H%M%S)"-"$(tr -dc 'a-z0-9' < /dev/urandom | head -c $RANDOM_STRING_LENGTH) + gcloud alpha storage buckets create gs://$bucket_name --project=$project_id --location=$region --placement=${zone} --default-storage-class=RAPID --uniform-bucket-level-access --enable-hierarchical-namespace + echo "${bucket_name}" +} + function run_non_parallel_tests() { local exit_code=0 local -n test_array=$1 local bucket_name_non_parallel=$2 + local zonal=false + if [ $# -ge 3 ] && [ "$3" = "true" ] ; then + zonal=true + fi for test_dir_np in "${test_array[@]}" do @@ -156,11 +322,14 @@ function run_non_parallel_tests() { echo $log_file >> $TEST_LOGS_FILE # Executing integration tests - GODEBUG=asyncpreemptoff=1 go test $test_path_non_parallel -p 1 $GO_TEST_SHORT_FLAG $PRESUBMIT_RUN_FLAG --integrationTest -v --testbucket=$bucket_name_non_parallel --testInstalledPackage=$RUN_E2E_TESTS_ON_PACKAGE -timeout $INTEGRATION_TEST_TIMEOUT > "$log_file" 2>&1 + echo "Running test package in non-parallel (with zonal=${zonal}): ${test_dir_np} ..." + GODEBUG=asyncpreemptoff=1 go test $test_path_non_parallel -p 1 $GO_TEST_SHORT_FLAG $PRESUBMIT_RUN_FLAG --zonal=${zonal} --integrationTest -v --testbucket=$bucket_name_non_parallel --testInstalledPackage=$RUN_E2E_TESTS_ON_PACKAGE $USE_PREBUILT_GCSFUSE_BINARY -timeout $INTEGRATION_TEST_TIMEOUT > "$log_file" 2>&1 exit_code_non_parallel=$? if [ $exit_code_non_parallel != 0 ]; then exit_code=$exit_code_non_parallel - echo "test fail in non parallel on package: " $test_dir_np + echo "test fail in non parallel on package (with zonal=${zonal}): " $test_dir_np + else + echo "Passed test package in non-parallel (with zonal=${zonal}): " $test_dir_np fi done return $exit_code @@ -170,28 +339,49 @@ function run_parallel_tests() { local exit_code=0 local -n test_array=$1 local bucket_name_parallel=$2 - local pids=() + local zonal=false + if [ $# -ge 3 ] && [ "$3" = "true" ] ; then + zonal=true + fi + local benchmark_flags="" + declare -A pids for test_dir_p in "${test_array[@]}" do + # Unlike regular tests,benchmark tests are not executed by default when using go test . + # The -bench flag yells go test to run the benchmark tests and report their results by + # enabling the benchmarking framework. + # The -benchtime flag specifies exact number of iterations a benchmark should run , in this + # case, setting this to 100 to avoid flakiness. + if [ $test_dir_p == "benchmarking" ]; then + benchmark_flags="-bench=. -benchtime=100x" + fi test_path_parallel="./tools/integration_tests/$test_dir_p" # To make it clear whether tests are running on a flat or HNS bucket, We kept the log file naming # convention to include the bucket name as a suffix (e.g., package_name_bucket_name). local log_file="/tmp/${test_dir_p}_${bucket_name_parallel}.log" echo $log_file >> $TEST_LOGS_FILE # Executing integration tests - GODEBUG=asyncpreemptoff=1 go test $test_path_parallel $GO_TEST_SHORT_FLAG $PRESUBMIT_RUN_FLAG -p 1 --integrationTest -v --testbucket=$bucket_name_parallel --testInstalledPackage=$RUN_E2E_TESTS_ON_PACKAGE -timeout $INTEGRATION_TEST_TIMEOUT > "$log_file" 2>&1 & + echo "Queueing up test package in parallel (with zonal=${zonal}): ${test_dir_p} ..." + GODEBUG=asyncpreemptoff=1 go test $test_path_parallel $GO_TEST_SHORT_FLAG $PRESUBMIT_RUN_FLAG --zonal=${zonal} $benchmark_flags -p 1 --integrationTest -v --testbucket=$bucket_name_parallel --testInstalledPackage=$RUN_E2E_TESTS_ON_PACKAGE $USE_PREBUILT_GCSFUSE_BINARY -timeout $INTEGRATION_TEST_TIMEOUT > "$log_file" 2>&1 & pid=$! # Store the PID of the background process - pids+=("$pid") # Optionally add the PID to an array for later + echo "Queued up test package in parallel (with zonal=${zonal}): ${test_dir_p} with pid=${pid}" + pids[${test_dir_p}]=${pid} # Optionally add the PID to an array for later done # Wait for processes and collect exit codes - for pid in "${pids[@]}"; do + for package_name in "${!pids[@]}"; do + pid="${pids[${package_name}]}" + echo "Waiting on test package ${package_name} (with zonal=${zonal}) through pid=${pid} ..." + # What if the process for this test package completed long back and its PID got + # re-assigned to another process since then ? wait $pid exit_code_parallel=$? if [ $exit_code_parallel != 0 ]; then exit_code=$exit_code_parallel - echo "test fail in parallel on package: " $test_dir_p + echo "test fail in parallel on package (with zonal=${zonal}): " $package_name + else + echo "Passed test package in parallel (with zonal=${zonal}): " $package_name fi done return $exit_code @@ -214,44 +404,45 @@ function print_test_logs() { function run_e2e_tests_for_flat_bucket() { # Adding prefix `golang-grpc-test` to white list the bucket for grpc so that # we can run grpc related e2e tests. - bucketPrefix="golang-grpc-test-gcsfuse-non-parallel-e2e-tests-" + bucketPrefix="golang-grpc-test-gcsfuse-np-e2e-tests-" bucket_name_non_parallel=$(create_bucket $bucketPrefix) echo "Bucket name for non parallel tests: "$bucket_name_non_parallel + echo ${bucket_name_non_parallel}>>"${bucketNamesFile}" - bucketPrefix="golang-grpc-test-gcsfuse-parallel-e2e-tests-" + bucketPrefix="golang-grpc-test-gcsfuse-p-e2e-tests-" bucket_name_parallel=$(create_bucket $bucketPrefix) echo "Bucket name for parallel tests: "$bucket_name_parallel + echo ${bucket_name_parallel}>>"${bucketNamesFile}" echo "Running parallel tests..." run_parallel_tests TEST_DIR_PARALLEL $bucket_name_parallel & parallel_tests_pid=$! - echo "Running non parallel tests ..." - run_non_parallel_tests TEST_DIR_NON_PARALLEL $bucket_name_non_parallel & - non_parallel_tests_pid=$! - - # Wait for all tests to complete. - wait $parallel_tests_pid - parallel_tests_exit_code=$? - wait $non_parallel_tests_pid - non_parallel_tests_exit_code=$? + echo "Running non parallel tests ..." + run_non_parallel_tests TEST_DIR_NON_PARALLEL $bucket_name_non_parallel & + non_parallel_tests_pid=$! - flat_buckets=("$bucket_name_parallel" "$bucket_name_non_parallel") - clean_up flat_buckets + # Wait for all tests to complete. + wait $parallel_tests_pid + parallel_tests_exit_code=$? + wait $non_parallel_tests_pid + non_parallel_tests_exit_code=$? - if [ $non_parallel_tests_exit_code != 0 ] || [ $parallel_tests_exit_code != 0 ]; - then - return 1 - fi - return 0 + if [ $non_parallel_tests_exit_code != 0 ] || [ $parallel_tests_exit_code != 0 ]; + then + return 1 + fi + return 0 } function run_e2e_tests_for_hns_bucket(){ hns_bucket_name_parallel_group=$(create_hns_bucket) echo "Hns Bucket Created: "$hns_bucket_name_parallel_group + echo ${hns_bucket_name_parallel_group}>>"${bucketNamesFile}" hns_bucket_name_non_parallel_group=$(create_hns_bucket) echo "Hns Bucket Created: "$hns_bucket_name_non_parallel_group + echo ${hns_bucket_name_non_parallel_group}>>"${bucketNamesFile}" echo "Running tests for HNS bucket" run_parallel_tests TEST_DIR_PARALLEL "$hns_bucket_name_parallel_group" & @@ -265,9 +456,6 @@ function run_e2e_tests_for_hns_bucket(){ wait $non_parallel_tests_hns_group_pid non_parallel_tests_hns_group_exit_code=$? - hns_buckets=("$hns_bucket_name_parallel_group" "$hns_bucket_name_non_parallel_group") - clean_up hns_buckets - if [ $parallel_tests_hns_group_exit_code != 0 ] || [ $non_parallel_tests_hns_group_exit_code != 0 ]; then return 1 @@ -275,44 +463,73 @@ function run_e2e_tests_for_hns_bucket(){ return 0 } +function run_e2e_tests_for_zonal_bucket(){ + zonal_bucket_name_parallel_group=$(create_zonal_bucket) + echo "Zonal Bucket Created for parallel tests: "$zonal_bucket_name_parallel_group + echo ${zonal_bucket_name_parallel_group}>>"${bucketNamesFile}" + + zonal_bucket_name_non_parallel_group=$(create_zonal_bucket) + echo "Zonal Bucket Created for non-parallel tests: "$zonal_bucket_name_non_parallel_group + echo ${zonal_bucket_name_non_parallel_group}>>"${bucketNamesFile}" + + echo "Running tests for ZONAL bucket" + run_parallel_tests TEST_DIR_PARALLEL_FOR_ZB "$zonal_bucket_name_parallel_group" true & + parallel_tests_zonal_group_pid=$! + run_non_parallel_tests TEST_DIR_NON_PARALLEL_FOR_ZB "$zonal_bucket_name_non_parallel_group" true & + non_parallel_tests_zonal_group_pid=$! + + # Wait for all tests to complete. + wait $parallel_tests_zonal_group_pid + parallel_tests_zonal_group_exit_code=$? + wait $non_parallel_tests_zonal_group_pid + non_parallel_tests_zonal_group_exit_code=$? + + if [ $parallel_tests_zonal_group_exit_code != 0 ] || [ $non_parallel_tests_zonal_group_exit_code != 0 ]; + then + return 1 + fi + return 0 +} + function run_e2e_tests_for_tpc() { + local bucket=$1 + if [ "$bucket" == "" ]; + then + echo "Bucket name is required" + return 1 + fi + # Clean bucket before testing. - gcloud storage rm -r gs://gcsfuse-e2e-tests-tpc/** + gcloud --verbosity=error storage rm -r gs://"$bucket"/* # Run Operations e2e tests in TPC to validate all the functionality. - GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... --testOnTPCEndPoint=$RUN_TEST_ON_TPC_ENDPOINT $GO_TEST_SHORT_FLAG $PRESUBMIT_RUN_FLAG -p 1 --integrationTest -v --testbucket=gcsfuse-e2e-tests-tpc --testInstalledPackage=$RUN_E2E_TESTS_ON_PACKAGE -timeout $INTEGRATION_TEST_TIMEOUT + GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... --testOnTPCEndPoint=$RUN_TEST_ON_TPC_ENDPOINT $GO_TEST_SHORT_FLAG $PRESUBMIT_RUN_FLAG --zonal=false -p 1 --integrationTest -v --testbucket="$bucket" --testInstalledPackage=$RUN_E2E_TESTS_ON_PACKAGE $USE_PREBUILT_GCSFUSE_BINARY -timeout $INTEGRATION_TEST_TIMEOUT exit_code=$? set -e # Delete data after testing. - gcloud storage rm -r gs://gcsfuse-e2e-tests-tpc/** + gcloud --verbosity=error storage rm -r gs://"$bucket"/* if [ $exit_code != 0 ]; then - echo "The tests failed." + return 1 fi - exit $exit_code + return 0 } -#commenting it so cleanup and failure check happens for both -#set -e - -function clean_up() { - # Cleanup - # Delete bucket after testing. - local -n buckets=$1 - for bucket in "${buckets[@]}" - do - # Empty bucket name may cause deletions of all the buckets. - if [ "$bucket" != "" ]; - then - gcloud alpha storage rm --recursive gs://$bucket - fi - done +function run_e2e_tests_for_emulator() { + ./tools/integration_tests/emulator_tests/emulator_tests.sh $RUN_E2E_TESTS_ON_PACKAGE } function main(){ + # The name of a file containing the names of all the + # buckets to be cleaned-up while exiting this program. + bucketNamesFile=$(realpath ./bucketNames)"-"$(tr -dc 'a-z0-9' < /dev/urandom | head -c $RANDOM_STRING_LENGTH) + # Delete all these buckets when the program exits. + # Cleanup fuse build folder if created + trap "cleanup_gcsfuse_once; delete_buckets_listed_in_file ${bucketNamesFile}" EXIT + set -e upgrade_gcloud_version @@ -321,45 +538,105 @@ function main(){ set +e - # Run tpc test and exit in case RUN_TEST_ON_TPC_ENDPOINT is true. - if [ $RUN_TEST_ON_TPC_ENDPOINT == true ]; then - run_e2e_tests_for_tpc + # Decide whether to build GCSFuse based on RUN_E2E_TESTS_ON_PACKAGE + if [ "$RUN_E2E_TESTS_ON_PACKAGE" != "true" ] && [ "$BUILD_BINARY_IN_SCRIPT" == "true" ]; then + echo "RUN_E2E_TESTS_ON_PACKAGE is not 'true' (value: '${RUN_E2E_TESTS_ON_PACKAGE}') and BUILD_BINARY_IN_SCRIPT is 'true'. Building GCSFuse..." + build_gcsfuse_once + if [ $? -ne 0 ]; then + echo "build_gcsfuse_once failed. Exiting." + # The trap will handle cleanup + exit 1 + fi + + USE_PREBUILT_GCSFUSE_BINARY="--gcsfuse_prebuilt_dir=${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}" + echo "Script built GCSFuse at: ${BUILT_BY_SCRIPT_GCSFUSE_BUILD_DIR}" fi #run integration tests - run_e2e_tests_for_hns_bucket & - e2e_tests_hns_bucket_pid=$! + exit_code=0 - run_e2e_tests_for_flat_bucket & - e2e_tests_flat_bucket_pid=$! + if ${RUN_TESTS_WITH_ZONAL_BUCKET}; then + run_e2e_tests_for_zonal_bucket & + e2e_tests_zonal_bucket_pid=$! + wait $e2e_tests_zonal_bucket_pid + e2e_tests_zonal_bucket_status=$? - wait $e2e_tests_flat_bucket_pid - e2e_tests_flat_bucket_status=$? + if [ $e2e_tests_zonal_bucket_status != 0 ]; then + echo "The e2e tests for zonal bucket failed.." + exit_code=1 + fi + else + # Run tpc test and exit in case RUN_TEST_ON_TPC_ENDPOINT is true. + if [ "$RUN_TEST_ON_TPC_ENDPOINT" == true ]; then + # Run tests for flat bucket + run_e2e_tests_for_tpc gcsfuse-e2e-tests-tpc & + e2e_tests_tpc_flat_bucket_pid=$! + # Run tests for hns bucket + run_e2e_tests_for_tpc gcsfuse-e2e-tests-tpc-hns & + e2e_tests_tpc_hns_bucket_pid=$! + + wait $e2e_tests_tpc_flat_bucket_pid + e2e_tests_tpc_flat_bucket_status=$? + + wait $e2e_tests_tpc_hns_bucket_pid + e2e_tests_tpc_hns_bucket_status=$? + + if [ $e2e_tests_tpc_flat_bucket_status != 0 ]; + then + echo "The e2e tests for flat bucket failed.." + exit 1 + fi + if [ $e2e_tests_tpc_hns_bucket_status != 0 ]; + then + echo "The e2e tests for hns bucket failed.." + exit 1 + fi + # Exit to prevent the following code from executing for TPC. + exit 0 + fi - wait $e2e_tests_hns_bucket_pid - e2e_tests_hns_bucket_status=$? + run_e2e_tests_for_hns_bucket & + e2e_tests_hns_bucket_pid=$! - set -e + run_e2e_tests_for_flat_bucket & + e2e_tests_flat_bucket_pid=$! - print_test_logs + run_e2e_tests_for_emulator & + e2e_tests_emulator_pid=$! - if [ $e2e_tests_flat_bucket_status != 0 ] && [ $e2e_tests_hns_bucket_status != 0 ]; - then - echo "The e2e tests for both flat and hns bucket failed.." - exit 1 - fi + wait $e2e_tests_emulator_pid + e2e_tests_emulator_status=$? - if [ $e2e_tests_flat_bucket_status != 0 ]; - then - echo "The e2e tests for flat bucket failed.." - exit 1 - fi + wait $e2e_tests_flat_bucket_pid + e2e_tests_flat_bucket_status=$? - if [ $e2e_tests_hns_bucket_status != 0 ]; - then - echo "The e2e tests for hns bucket failed.." - exit 1 + wait $e2e_tests_hns_bucket_pid + e2e_tests_hns_bucket_status=$? + + if [ $e2e_tests_flat_bucket_status != 0 ]; + then + echo "The e2e tests for flat bucket failed.." + exit_code=1 + fi + + if [ $e2e_tests_hns_bucket_status != 0 ]; + then + echo "The e2e tests for hns bucket failed.." + exit_code=1 + fi + + if [ $e2e_tests_emulator_status != 0 ]; + then + echo "The e2e tests for emulator failed.." + exit_code=1 + fi fi + + set -e + + print_test_logs + + exit $exit_code } #Main method to run script diff --git a/tools/integration_tests/run_tests_mounted_directory.sh b/tools/integration_tests/run_tests_mounted_directory.sh index e7843299f5..a502cafcd2 100755 --- a/tools/integration_tests/run_tests_mounted_directory.sh +++ b/tools/integration_tests/run_tests_mounted_directory.sh @@ -23,75 +23,85 @@ TEST_BUCKET_NAME=$1 MOUNT_DIR=$2 export CGO_ENABLED=0 +ZONAL_BUCKET_ARG= +if [ $# -gt 2 ] ; then + if [ "$3" = "true" ]; then + ZONAL_BUCKET_ARG="--zonal=true" + elif [ "$3" != "false" ]; then + >&2 echo "Unexpected value of RUN_ZONAL_BUCKET: $3. Expected: true or false." + exit 1 + fi +fi + # package operations # Run test with static mounting. (flags: --implicit-dirs=true) gcsfuse --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with persistent mounting. (flags: --implicit-dirs=true) mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o implicit_dirs=true -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with static mounting. (flags: --implicit-dirs=false) gcsfuse --implicit-dirs=false $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with persistent mounting. (flags: --implicit-dirs=false) mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o implicit_dirs=false -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with static mounting. (flags: --experimental-enable-json-read) gcsfuse --experimental-enable-json-read $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with static mounting. (flags: --kernel-list-cache-ttl-secs=-1, --implicit-dirs=true) gcsfuse --kernel-list-cache-ttl-secs=-1 --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with persistent mounting. (flags: --experimental-enable-json-read, --implicit-dirs=true) mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o implicit_dirs=true,experimental_enable_json_read=true -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run tests with static mounting. (flags: --implicit-dirs=true, --only-dir testDir) gcsfuse --only-dir testDir --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run tests with persistent mounting. (flags: --implicit-dirs=true, --only-dir=testDir) mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o only_dir=testDir,implicit_dirs=true -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run tests with static mounting. (flags: --implicit-dirs=false, --only-dir testDir) gcsfuse --only-dir testDir --implicit-dirs=false $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run tests with persistent mounting. (flags: --implicit-dirs=false, --only-dir=testDir) mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o only_dir=testDir,implicit_dirs=false -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run tests with only-dir mounting. (flags: --experimental-enable-json-read, --only-dir testDir) gcsfuse --experimental-enable-json-read --only-dir testDir $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run tests with only-dir mounting. (flags: --kernel-list-cache-ttl-secs=-1, --implicit-dirs=true, --only-dir testDir) gcsfuse --kernel-list-cache-ttl-secs=-1 --implicit-dirs --only-dir testDir $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run tests with persistent mounting. (flags: --experimental-enable-json-read, --implicit-dirs=true, --only-dir=testDir) mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o only_dir=testDir,implicit_dirs=true,experimental_enable_json_read=true -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run tests with config "create-empty-file: true". @@ -99,7 +109,7 @@ echo "write: create-empty-file: true " > /tmp/gcsfuse_config.yaml gcsfuse --config-file=/tmp/gcsfuse_config.yaml $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run tests with config "file-cache: max-size-mb" static mounting. @@ -108,7 +118,7 @@ echo "file-cache: cache-dir: /tmp/cache-dir-operations-hns-false " > /tmp/gcsfuse_config.yaml gcsfuse --config-file=/tmp/gcsfuse_config.yaml $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run tests with config "metadata-cache: ttl-secs: 0" static mounting. @@ -116,48 +126,48 @@ echo "metadata-cache: ttl-secs: 0 " > /tmp/gcsfuse_config.yaml gcsfuse --config-file=/tmp/gcsfuse_config.yaml $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # package readonly # Run tests with static mounting. (flags: --implicit-dirs=true,--o=ro) gcsfuse --o=ro --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with persistent mounting. (flags: --implicit-dirs=true,--o=ro) mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o ro,implicit_dirs=true -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run tests with static mounting. (flags: --implicit-dirs=true, --file-mode=544, --dir-mode=544) gcsfuse --file-mode=544 --dir-mode=544 --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with persistent mounting. (flags: --implicit-dirs=true, --file-mode=544, --dir-mode=544) mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o file_mode=544,dir_mode=544,implicit_dirs=true -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run tests with static mounting. (flags: --implicit-dirs=true, --o=ro, --only-dir testDir) gcsfuse --only-dir testDir --o=ro --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with persistent mounting. (flags: --implicit-dirs=true,--o=ro,--only-dir=testDir) mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o ro,only_dir=testDir,implicit_dirs=true -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with static mounting. (flags: --implicit-dirs=true, --file-mode=544, --dir-mode=544, --only-dir testDir) gcsfuse --only-dir testDir --file-mode=544 --dir-mode=544 --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with persistent mounting. (flags: --implicit-dirs=true, --file-mode=544, --dir-mode=544, --only-dir=testDir) mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o only_dir=testDir,file_mode=544,dir_mode=544,implicit_dirs=true -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run tests with config "file-cache: max-size-mb" static mounting. @@ -166,112 +176,119 @@ echo "file-cache: cache-dir: /tmp/cache-dir-readonly-hns-false " > /tmp/gcsfuse_config.yaml gcsfuse --config-file /tmp/gcsfuse_config.yaml --only-dir testDir --file-mode=544 --dir-mode=544 --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # package rename_dir_limit # Run tests with static mounting. (flags: --rename-dir-limit=3, --implicit-dirs) gcsfuse --rename-dir-limit=3 --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with persistent mounting. (flags: --rename-dir-limit=3, --implicit-dirs) mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o rename_dir_limit=3,implicit_dirs -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run tests with static mounting. (flags: --rename-dir-limit=3) gcsfuse --rename-dir-limit=3 $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with persistent mounting. (flags: --rename-dir-limit=3) mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o rename_dir_limit=3 -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with static mounting. (flags: --rename-dir-limit=3, --implicit-dirs, --only-dir testDir) gcsfuse --only-dir testDir --rename-dir-limit=3 --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with persistent mounting . (flags: --rename-dir-limit=3, --implicit-dirs) mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o only_dir=testDir,rename_dir_limit=3,implicit_dirs -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with static mounting. (flags: --rename-dir-limit=3, --only-dir testDir) gcsfuse --only-dir testDir --rename-dir-limit=3 $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with persistent mounting . (flags: --rename-dir-limit=3, --implicit-dirs, --only-dir=testDir) mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o only_dir=testDir,rename_dir_limit=3,implicit_dirs -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # package implicit_dir # Run tests with static mounting. (flags: --implicit-dirs) gcsfuse --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/implicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/implicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with persistent mounting. (flags: --implicit-dirs) mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o implicit_dirs -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/implicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/implicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run tests with static mounting. (flags: --implicit-dirs, --only-dir testDir) gcsfuse --only-dir testDir --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/implicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/implicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with persistent mounting. (flags: --implicit-dirs,--only-dir=testDir) mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o only_dir=testDir,implicit_dirs -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/implicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/implicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # package explicit_dir # Run tests with static mounting. (flags: --implicit-dirs=false) gcsfuse --implicit-dirs=false $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/explicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/explicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with persistent mounting. (flags: --implicit-dirs=false) mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o implicit_dirs=false -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/explicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/explicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run tests with static mounting. (flags: --implicit-dirs=false, --only-dir testDir) gcsfuse --only-dir testDir --implicit-dirs=false $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/explicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/explicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with persistent mounting. (flags: --implicit-dirs=false, --only-dir=testDir) mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o only_dir=testDir,implicit_dirs=false -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/explicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/explicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # package list_large_dir # Run tests with static mounting. (flags: --implicit-dirs) gcsfuse --implicit-dirs --stat-cache-ttl=0 --kernel-list-cache-ttl-secs=-1 $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/list_large_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/list_large_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # package read_large_files # Run tests with static mounting. (flags: --implicit-dirs) gcsfuse --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/read_large_files/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/read_large_files/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR +if [ -n "${ZONAL_BUCKET_ARG}" ]; then + # Run tests with static mounting. (flags: --implicit-dirs, --enable-kernel-reader=false) + gcsfuse --implicit-dirs --enable-kernel-reader=false $TEST_BUCKET_NAME $MOUNT_DIR + GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/read_large_files/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} + sudo umount $MOUNT_DIR +fi + # Run tests with config "file-cache: max-size-mb, cache-file-for-range-read". echo "file-cache: max-size-mb: 700 cache-file-for-range-read: true cache-dir: /tmp/cache-dir-read-large-files-hns-false " > /tmp/gcsfuse_config.yaml -gcsfuse --config-file /tmp/gcsfuse_config.yaml --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/read_large_files/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +gcsfuse --config-file /tmp/gcsfuse_config.yaml --implicit-dirs=true --enable-kernel-reader=false $TEST_BUCKET_NAME $MOUNT_DIR +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/read_large_files/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run tests with config "file-cache: max-size-mb". @@ -280,47 +297,31 @@ echo "file-cache: cache-file-for-range-read: false cache-dir: /tmp/cache-dir-read-large-files-hns-false " > /tmp/gcsfuse_config.yaml -gcsfuse --config-file /tmp/gcsfuse_config.yaml --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/read_large_files/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +gcsfuse --config-file /tmp/gcsfuse_config.yaml --implicit-dirs=true --enable-kernel-reader=false $TEST_BUCKET_NAME $MOUNT_DIR +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/read_large_files/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # package write_large_files # Run tests with static mounting. (flags: --implicit-dirs) gcsfuse --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/write_large_files/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/write_large_files/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # package gzip # Run tests with static mounting. (flags: --implicit-dirs) gcsfuse --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/gzip/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/gzip/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # package local_file # Run test with static mounting. (flags: --implicit-dirs=true) -gcsfuse --implicit-dirs=true --rename-dir-limit=3 $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/local_file/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +gcsfuse --implicit-dirs=true --rename-dir-limit=3 --enable-streaming-writes=false $TEST_BUCKET_NAME $MOUNT_DIR +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/local_file/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with static mounting. (flags: --implicit-dirs=false) -gcsfuse --implicit-dirs=false --rename-dir-limit=3 $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/local_file/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME -sudo umount $MOUNT_DIR - -# Run tests with log rotation config. -rm -r /tmp/gcsfuse_integration_test_logs -mkdir /tmp/gcsfuse_integration_test_logs -echo "logging: - file-path: /tmp/gcsfuse_integration_test_logs/log.txt - format: text - severity: trace - log-rotate: - max-file-size-mb: 2 - backup-file-count: 3 - compress: true - " > /tmp/gcsfuse_config.yaml -gcsfuse --config-file=/tmp/gcsfuse_config.yaml $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/log_rotation/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR +gcsfuse --implicit-dirs=false --rename-dir-limit=3 --enable-streaming-writes=false $TEST_BUCKET_NAME $MOUNT_DIR +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/local_file/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run read cache functional tests. @@ -367,7 +368,6 @@ file-cache: metadata-cache: stat-cache-max-size-mb: 4 ttl-secs: $cache_ttl - type-cache-max-size-mb: 32 cache-dir: /tmp/cache-dir-read-cache-hns-false" > /tmp/gcsfuse_config.yaml } @@ -376,15 +376,28 @@ function run_read_cache_test() { local optional_flags=$2 if [ -n "$optional_flags" ]; then - gcsfuse "$optional_flags" --config-file=/tmp/gcsfuse_config.yaml "$TEST_BUCKET_NAME" "$MOUNT_DIR" > /dev/null + gcsfuse "$optional_flags" --enable-kernel-reader=false --config-file=/tmp/gcsfuse_config.yaml "$TEST_BUCKET_NAME" "$MOUNT_DIR" > /dev/null else - gcsfuse --config-file=/tmp/gcsfuse_config.yaml "$TEST_BUCKET_NAME" "$MOUNT_DIR" > /dev/null + gcsfuse --enable-kernel-reader=false --config-file=/tmp/gcsfuse_config.yaml "$TEST_BUCKET_NAME" "$MOUNT_DIR" > /dev/null fi GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/read_cache/... -p 1 --integrationTest -v --mountedDirectory="$MOUNT_DIR" --testbucket="$TEST_BUCKET_NAME" -run "$test_case" sudo umount "$MOUNT_DIR" cleanup_test_environment } +function run_chunk_cache_test() { + local test_case=$1 + local flags=$2 + + cleanup_test_environment + + gcsfuse $flags --log-file=/tmp/gcsfuse_read_cache_test_logs/log.json --log-format=json --log-severity=trace --cache-dir=/tmp/cache-dir-read-cache-hns-false "$TEST_BUCKET_NAME" "$MOUNT_DIR" > /dev/null + + GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/read_cache/... -p 1 --integrationTest -v --mountedDirectory="$MOUNT_DIR" --testbucket="$TEST_BUCKET_NAME" -run "$test_case" + sudo umount "$MOUNT_DIR" + cleanup_test_environment +} + # Read-cache test with cache-file-for-range-read:false. test_cases=( "TestCacheFileForRangeReadFalseTest/TestRangeReadsWithCacheMiss" @@ -486,18 +499,24 @@ for test_case in "${test_cases[@]}"; do run_read_cache_test "$test_case" done +# Chunk cache tests. +run_chunk_cache_test "TestChunkCacheTest" "--file-cache-experimental-enable-chunk-cache=true --file-cache-download-chunk-size-mb=10 --enable-kernel-reader=false" +run_chunk_cache_test "TestChunkCacheDisabledTest" "--file-cache-experimental-enable-chunk-cache=false --enable-kernel-reader=false" +run_chunk_cache_test "TestChunkCacheEviction" "--file-cache-experimental-enable-chunk-cache=true --file-cache-download-chunk-size-mb=10 --file-cache-max-size-mb=15 --enable-kernel-reader=false" + + # Package managed_folders echo "list: enable-empty-managed-folders: true" > /tmp/gcsfuse_config.yaml # Empty managed folders listing test. # Run test with static mounting (flags: --implicit-dirs) gcsfuse --implicit-dirs --config-file=/tmp/gcsfuse_config.yaml $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/managed_folders/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME -run TestEnableEmptyManagedFoldersTrue +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/managed_folders/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME -run TestEnableEmptyManagedFoldersTrue ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with persistent mounting. (flags: --implicit-dirs) mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o implicit_dirs -o config_file=/tmp/gcsfuse_config.yaml -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/managed_folders/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME -run TestEnableEmptyManagedFoldersTrue +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/managed_folders/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME -run TestEnableEmptyManagedFoldersTrue ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # For GRPC: running only core integration tests. @@ -505,34 +524,52 @@ sudo umount $MOUNT_DIR # Test packages: operations # Run test with static mounting. (flags: --client-protocol=grpc --implicit-dirs=true) gcsfuse --client-protocol=grpc --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with persistent mounting. (flags: --client-protocol=grpc --implicit-dirs=true) mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o implicit_dirs=true,client_protocol=grpc -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Test package: implicit_dir # Run tests with static mounting. (flags: --client-protocol=grpc --implicit-dirs=true) gcsfuse --implicit-dirs=true --client-protocol=grpc $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/implicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/implicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Run test with persistent mounting. (flags: --client-protocol=grpc --implicit-dirs=true) mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o implicit_dirs=true,client_protocol=grpc -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/implicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/implicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Test package: concurrent_operations +if [ -n "${ZONAL_BUCKET_ARG}" ]; then +# Run tests with static mounting. (flags: --kernel-list-cache-ttl-secs=-1 --implicit-dirs=true, --enable-kernel-reader=false) + gcsfuse --implicit-dirs=true --kernel-list-cache-ttl-secs=-1 --enable-kernel-reader=false $TEST_BUCKET_NAME $MOUNT_DIR + GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/concurrent_operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} + sudo umount $MOUNT_DIR +fi + # Run tests with static mounting. (flags: --kernel-list-cache-ttl-secs=-1 --implicit-dirs=true) gcsfuse --implicit-dirs=true --kernel-list-cache-ttl-secs=-1 $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/concurrent_operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/concurrent_operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR -# Run test with persistent mounting. (flags: --kernel-list-cache-ttl-secs=-1 --implicit-dirs=true) -mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o implicit_dirs=true,kernel_list_cache_ttl_secs=-1 -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/concurrent_operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +# Run test with persistent mounting. (flags: --kernel-list-cache-ttl-secs=-1 --implicit-dirs=true, --enable-kernel-reader=false) +mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o implicit_dirs=true,kernel_list_cache_ttl_secs=-1,enable-kernel-reader=false +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/concurrent_operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} +sudo umount $MOUNT_DIR + +# Test package: benchmarking +# Run tests with static mounting. (flags: --implicit-dirs=true) +gcsfuse --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/benchmarking/... --bench=. -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} +sudo umount $MOUNT_DIR + +# Run test with persistent mounting. (flags: --implicit-dirs=true) +mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o implicit_dirs=true +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/benchmarking/... --bench=. -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} sudo umount $MOUNT_DIR # Test package: kernel-list-cache @@ -547,11 +584,19 @@ test_cases=( "TestInfiniteKernelListCacheTest/TestKernelListCache_CacheMissOnAdditionOfDirectory" "TestInfiniteKernelListCacheTest/TestKernelListCache_CacheMissOnDeletionOfDirectory" "TestInfiniteKernelListCacheTest/TestKernelListCache_CacheMissOnDirectoryRename" - "TestInfiniteKernelListCacheTest/TestKernelListCache_ListAndDeleteDirectory" - "TestInfiniteKernelListCacheTest/TestKernelListCache_DeleteAndListDirectory" ) for test_case in "${test_cases[@]}"; do - gcsfuse --kernel-list-cache-ttl-secs=-1 "$TEST_BUCKET_NAME" "$MOUNT_DIR" + gcsfuse --kernel-list-cache-ttl-secs=-1 "$TEST_BUCKET_NAME" "$MOUNT_DIR" + GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/kernel_list_cache/... -p 1 --integrationTest -v --mountedDirectory="$MOUNT_DIR" --testbucket="$TEST_BUCKET_NAME" -run "$test_case" + sudo umount "$MOUNT_DIR" +done + +test_cases=( + "TestInfiniteKernelListCacheDeleteDirTest/TestKernelListCache_ListAndDeleteDirectory" + "TestInfiniteKernelListCacheDeleteDirTest/TestKernelListCache_DeleteAndListDirectory" +) +for test_case in "${test_cases[@]}"; do + gcsfuse --kernel-list-cache-ttl-secs=-1 --metadata-cache-ttl-secs=0 "$TEST_BUCKET_NAME" "$MOUNT_DIR" GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/kernel_list_cache/... -p 1 --integrationTest -v --mountedDirectory="$MOUNT_DIR" --testbucket="$TEST_BUCKET_NAME" -run "$test_case" sudo umount "$MOUNT_DIR" done @@ -575,3 +620,179 @@ for test_case in "${test_cases[@]}"; do GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/kernel_list_cache/... -p 1 --integrationTest -v --mountedDirectory="$MOUNT_DIR" --testbucket="$TEST_BUCKET_NAME" -run "$test_case" sudo umount "$MOUNT_DIR" done + +# Test package: stale_handle +# Run tests with static mounting. (flags: --metadata-cache-ttl-secs=0) +gcsfuse --metadata-cache-ttl-secs=0 $TEST_BUCKET_NAME $MOUNT_DIR +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/stale_handle/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} +sudo umount $MOUNT_DIR + +# Run test with persistent mounting. (flags: --metadata-cache-ttl-secs=0) +mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o metadata_cache_ttl_secs=0 +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/stale_handle/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} +sudo umount $MOUNT_DIR + +# Test package: streaming_writes +# Run streaming_writes tests. +gcsfuse --rename-dir-limit=3 --write-block-size-mb=1 --write-max-blocks-per-file=2 --write-global-max-blocks=-1 $TEST_BUCKET_NAME $MOUNT_DIR +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/streaming_writes/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} +sudo umount $MOUNT_DIR + +# Run write_large_files tests with streaming writes enabled. +gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/write_large_files/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} +sudo umount $MOUNT_DIR + +# Test package: inactive_stream_timeout +# Run tests when timeout is disabled. +log_dir="/tmp/inactive_stream_timeout_logs" +mkdir -p $log_dir +log_file="$log_dir/log.json" +gcsfuse --read-inactive-stream-timeout=0s --log-file $log_file --log-severity=trace --log-format=json $TEST_BUCKET_NAME $MOUNT_DIR +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/inactive_stream_timeout/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME -run "TestTimeoutDisabledSuite" +sudo umount $MOUNT_DIR +rm -rf $log_dir + +# Run tests when timeout is enabled. +test_cases=( + "TestTimeoutEnabledSuite/TestReaderCloses" + "TestTimeoutEnabledSuite/TestReaderStaysOpenWithinTimeout" +) +for test_case in "${test_cases[@]}"; do + log_dir="/tmp/inactive_stream_timeout_logs" + mkdir -p $log_dir + log_file="$log_dir/log.json" + gcsfuse --read-inactive-stream-timeout=1s --client-protocol grpc --log-file $log_file --log-severity=trace --log-format=json $TEST_BUCKET_NAME $MOUNT_DIR + GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/inactive_stream_timeout/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME -run $test_case + sudo umount $MOUNT_DIR + rm -rf $log_dir +done + +# Test package: cloud_profiler +# Run cloud_profiler tests. +random_profile_label="test" +gcsfuse --enable-cloud-profiler --cloud-profiler-goroutines --cloud-profiler-cpu --cloud-profiler-heap --cloud-profiler-allocated-heap --cloud-profiler-mutex --cloud-profiler-label $random_profile_label $TEST_BUCKET_NAME $MOUNT_DIR +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/cloud_profiler/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --profile_label=$random_profile_label +sudo umount $MOUNT_DIR + +# Test package: readdirplus +# Readdirplus test with dentry cache enabled (--experimental-enable-dentry-cache=true) +test_case="TestReaddirplusWithDentryCacheTest/TestReaddirplusWithDentryCache" +log_dir="/tmp/readdirplus_logs" +mkdir -p $log_dir +log_file="$log_dir/log.json" +gcsfuse --implicit-dirs --experimental-enable-readdirplus --experimental-enable-dentry-cache --log-file $log_file --log-severity=trace --log-format=json "$TEST_BUCKET_NAME" "$MOUNT_DIR" +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readdirplus/... -p 1 --integrationTest -v --mountedDirectory="$MOUNT_DIR" --testbucket="$TEST_BUCKET_NAME" -run $test_case +sudo umount "$MOUNT_DIR" +rm -rf $log_dir + +# Readdirplus test with dentry cache disabled (--experimental-enable-dentry-cache=false) +test_case="TestReaddirplusWithoutDentryCacheTest/TestReaddirplusWithoutDentryCache" +log_dir="/tmp/readdirplus_logs" +mkdir -p $log_dir +log_file="$log_dir/log.json" +gcsfuse --implicit-dirs --experimental-enable-readdirplus --log-file $log_file --log-severity=trace --log-format=json "$TEST_BUCKET_NAME" "$MOUNT_DIR" +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readdirplus/... -p 1 --integrationTest -v --mountedDirectory="$MOUNT_DIR" --testbucket="$TEST_BUCKET_NAME" -run $test_case +sudo umount "$MOUNT_DIR" +rm -rf $log_dir + +# Test package: dentry_cache +# Run stat with dentry cache enabled +test_cases=( +"TestStatWithDentryCacheEnabledTest/TestStatWithDentryCacheEnabled" +"TestStatWithDentryCacheEnabledTest/TestStatWhenFileIsDeletedDirectlyFromGCS" +) +for test_case in "${test_cases[@]}"; do + gcsfuse --implicit-dirs --experimental-enable-dentry-cache --metadata-cache-ttl-secs=1 "$TEST_BUCKET_NAME" "$MOUNT_DIR" + GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/dentry_cache/... -p 1 --integrationTest -v --mountedDirectory="$MOUNT_DIR" --testbucket="$TEST_BUCKET_NAME" -run $test_case + sudo umount "$MOUNT_DIR" +done + +# Run notifier tests +test_cases=( + "TestNotifierTest/TestReadFileWithDentryCacheEnabled" + "TestNotifierTest/TestWriteFileWithDentryCacheEnabled" + "TestNotifierTest/TestDeleteFileWithDentryCacheEnabled" +) +for test_case in "${test_cases[@]}"; do + gcsfuse --implicit-dirs --experimental-enable-dentry-cache --metadata-cache-ttl-secs=1000 "$TEST_BUCKET_NAME" "$MOUNT_DIR" + GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/dentry_cache/... -p 1 --integrationTest -v --mountedDirectory="$MOUNT_DIR" --testbucket="$TEST_BUCKET_NAME" -run $test_case + sudo umount "$MOUNT_DIR" +done + +# Run delete operation tests when dentry cache is enabled +test_case="TestDeleteOperationTest/TestDeleteFileWhenFileIsClobbered" +gcsfuse --implicit-dirs --experimental-enable-dentry-cache --metadata-cache-ttl-secs=1000 "$TEST_BUCKET_NAME" "$MOUNT_DIR" +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/dentry_cache/... -p 1 --integrationTest -v --mountedDirectory="$MOUNT_DIR" --testbucket="$TEST_BUCKET_NAME" -run $test_case +sudo umount "$MOUNT_DIR" + +# package buffered_read +log_dir="/tmp/gcsfuse_buffered_read_test_logs" +mkdir -p $log_dir +log_file="$log_dir/log.json" + +# Run TestSequentialReadSuite +sequential_read_test_case="TestSequentialReadSuite" +gcsfuse --log-severity=trace --enable-buffered-read=true --read-block-size-mb=8 --read-max-blocks-per-handle=20 --read-start-blocks-per-handle=1 --enable-kernel-reader=false \ +--read-min-blocks-per-handle=2 --log-file=$log_file $TEST_BUCKET_NAME $MOUNT_DIR +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/buffered_read/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR \ +--testbucket=$TEST_BUCKET_NAME -run ${sequential_read_test_case} ${ZONAL_BUCKET_ARG} +sudo umount $MOUNT_DIR + +# Run tests for fallback to another reader on random reads. +random_read_fallback_test_cases=( + "TestFallbackSuites/TestRandomRead_LargeFile_Fallback" + "TestFallbackSuites/TestRandomRead_SmallFile_NoFallback" +) +gcsfuse --log-severity=trace --enable-buffered-read=true --read-block-size-mb=8 --read-max-blocks-per-handle=20 --read-start-blocks-per-handle=2 --enable-kernel-reader=false \ +--read-min-blocks-per-handle=2 --log-file=$log_file $TEST_BUCKET_NAME $MOUNT_DIR +for test_case in "${random_read_fallback_test_cases[@]}"; do + GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/buffered_read/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR \ + --testbucket=$TEST_BUCKET_NAME -run ${test_case} ${ZONAL_BUCKET_ARG} +done +sudo umount $MOUNT_DIR + +# Run test for fallback when the global block pool is insufficient for buffered reader creation. +insufficient_pool_test_case="TestFallbackSuites/TestNewBufferedReader_InsufficientGlobalPool_NoReaderAdded" +gcsfuse --log-severity=trace --enable-buffered-read=true --read-block-size-mb=8 --read-max-blocks-per-handle=10 --read-start-blocks-per-handle=2 --enable-kernel-reader=false \ +--read-min-blocks-per-handle=2 --read-global-max-blocks=1 --log-file=$log_file $TEST_BUCKET_NAME $MOUNT_DIR +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/buffered_read/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR \ +--testbucket=$TEST_BUCKET_NAME -run ${insufficient_pool_test_case} ${ZONAL_BUCKET_ARG} +sudo umount $MOUNT_DIR + +rm -rf $log_dir + +# Package requester_pays_bucket +declare -A requester_pays_bucket_scenarios +requester_pays_bucket_scenarios["--billing-project=gcs-fuse-test-ml"]="" +for flags in "${!requester_pays_bucket_scenarios[@]}"; do + printf "\n=============================================================\n" + echo "Running requester_pays_bucket test with \"${flags}\" ... " + printf "\n=============================================================\n" + gcsfuse_mount_args=" --log-severity=trace ${flags} $TEST_BUCKET_NAME $MOUNT_DIR" + gcsfuse ${gcsfuse_mount_args} + testfilter="${requester_pays_bucket_scenarios[${flags}]}" + GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/requester_pays_bucket/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} -test.run ${testfilter} + sudo umount $MOUNT_DIR +done + +# Package flag_optimizations +declare -A flag_optimizations_scenarios +flag_optimizations_scenarios["--machine-type=low-end-machine"]="TestImplicitDirsNotEnabled/--machine-type=low-end-machine|TestRenameDirLimitNotSet/--machine-type=low-end-machine" +flag_optimizations_scenarios["--machine-type=a3-highgpu-8g"]="TestImplicitDirsEnabled/--machine-type=a3-highgpu-8g|TestRenameDirLimitSet/--machine-type=a3-highgpu-8g" +flag_optimizations_scenarios["--profile=aiml-training"]="TestImplicitDirsEnabled/--profile=aiml-training|TestRenameDirLimitNotSet/--profile=aiml-training" +flag_optimizations_scenarios["--profile=aiml-checkpointing"]="TestImplicitDirsEnabled/--profile=aiml-checkpointing|TestRenameDirLimitSet/--profile=aiml-checkpointing" +flag_optimizations_scenarios["--profile=aiml-serving"]="TestImplicitDirsEnabled/--profile=aiml-serving|TestRenameDirLimitNotSet/--profile=aiml-serving" +flag_optimizations_scenarios["--machine-type=low-end-machine --profile=aiml-training"]="TestImplicitDirsEnabled/--machine-type=low-end-machine_--profile=aiml-training" +flag_optimizations_scenarios["--machine-type=low-end-machine --profile=aiml-checkpointing"]="TestImplicitDirsEnabled/--machine-type=low-end-machine_--profile=aiml-checkpointing|TestRenameDirLimitSet/--machine-type=low-end-machine_--profile=aiml-checkpointing" +flag_optimizations_scenarios["--machine-type=low-end-machine --profile=aiml-serving"]="TestImplicitDirsEnabled/--machine-type=low-end-machine_--profile=aiml-serving" +for flags in "${!flag_optimizations_scenarios[@]}"; do + printf "\n=============================================================\n" + echo "Running flag_optimizations test with \"${flags}\" ... " + printf "\n=============================================================\n" + gcsfuse_mount_args=" --log-severity=trace ${flags} $TEST_BUCKET_NAME $MOUNT_DIR" + gcsfuse ${gcsfuse_mount_args} + testfilter="${flag_optimizations_scenarios[${flags}]}" + GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/flag_optimizations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME ${ZONAL_BUCKET_ARG} -test.run ${testfilter} + sudo umount $MOUNT_DIR +done diff --git a/tools/integration_tests/shared_chunk_cache/setup_test.go b/tools/integration_tests/shared_chunk_cache/setup_test.go new file mode 100644 index 0000000000..b1a4466192 --- /dev/null +++ b/tools/integration_tests/shared_chunk_cache/setup_test.go @@ -0,0 +1,113 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shared_chunk_cache + +import ( + "context" + "log" + "os" + "path" + "testing" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" +) + +const ( + testDirName = "SharedChunkCacheTest" + GKETempDir = "/gcsfuse-tmp" +) + +var ( + testEnv env +) + +type env struct { + storageClient *storage.Client + ctx context.Context + cfg *test_suite.TestConfig + bucketType string +} + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.SharedChunkCache) == 0 { + log.Println("No configuration found for shared_chunk_cache tests in config. Using flags instead.") + // Populate the config manually. + cfg.SharedChunkCache = make([]test_suite.TestConfig, 1) + cfg.SharedChunkCache[0].TestBucket = setup.TestBucket() + cfg.SharedChunkCache[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.SharedChunkCache[0].LogFile = setup.LogFile() + cfg.SharedChunkCache[0].Configs = make([]test_suite.ConfigItem, 1) + + // TestSharedChunkCacheTestSuite - dual mount with shared cache + cfg.SharedChunkCache[0].Configs[0].Flags = []string{ + "--enable-experimental-shared-chunk-cache --file-cache-max-size-mb=-1 --cache-dir=/gcsfuse-tmp/shared-cache", + } + cfg.SharedChunkCache[0].Configs[0].SecondaryFlags = []string{ + "--enable-experimental-shared-chunk-cache --file-cache-max-size-mb=-1 --cache-dir=/gcsfuse-tmp/shared-cache", + } + cfg.SharedChunkCache[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.SharedChunkCache[0].Configs[0].Run = "TestSharedChunkCacheTestSuite" + } + + testEnv.ctx = context.Background() + testEnv.cfg = &cfg.SharedChunkCache[0] + testEnv.bucketType = setup.TestEnvironment(testEnv.ctx, testEnv.cfg) + + // 2. Create storage client before running tests. + var err error + testEnv.storageClient, err = client.CreateStorageClient(testEnv.ctx) + if err != nil { + log.Printf("Error creating storage client: %v\n", err) + os.Exit(1) + } + defer func() { + if err := testEnv.storageClient.Close(); err != nil { + log.Printf("Error closing storage client: %v\n", err) + } + }() + + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + // For GKE, we expect both directories to be mounted if it's a dual mount test. + // If using config, GKEMountedDirectorySecondary should be set. + testEnv.cfg.GCSFuseMountedDirectory = testEnv.cfg.GKEMountedDirectory + testEnv.cfg.GCSFuseMountedDirectorySecondary = testEnv.cfg.GKEMountedDirectorySecondary + os.Exit(m.Run()) + } + + // For GCE environment + setup.SetUpTestDirForTestBucket(testEnv.cfg) + // Override GKE specific paths with GCSFuse paths if running in GCE environment. + setup.OverrideFilePathsInFlagSet(testEnv.cfg, setup.TestDir()) + + // For dual mount, we create another directory. + secondaryDir, err := os.MkdirTemp(setup.TestDir(), "gcsfuse-secondary-mount") + if err != nil { + log.Fatalf("Failed to create secondary mount directory: %v", err) + } + testEnv.cfg.GCSFuseMountedDirectorySecondary = secondaryDir + + successCode := m.Run() + + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(setup.TestBucket(), testDirName)) + os.Exit(successCode) +} diff --git a/tools/integration_tests/shared_chunk_cache/shared_chunk_cache_test.go b/tools/integration_tests/shared_chunk_cache/shared_chunk_cache_test.go new file mode 100644 index 0000000000..b6677b18a4 --- /dev/null +++ b/tools/integration_tests/shared_chunk_cache/shared_chunk_cache_test.go @@ -0,0 +1,371 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shared_chunk_cache + +import ( + "io/fs" + "log" + "os" + "path" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +//////////////////////////////////////////////////////////////////////// +// Suite Definitions +//////////////////////////////////////////////////////////////////////// + +// Struct to store the details of a mount point +type mountPoint struct { + rootDir string // Root directory of the test folder, which contains mnt and gcsfuse.log. + mntDir string // Directory where the GCS bucket is mounted. This is 'mnt' inside rootDir. + testDirPath string // Path to the 'SharedChunkCacheTest' directory inside mntDir. + logFilePath string // Path to the GCSFuse log file. This is gcsfuse.log inside rootDir. +} + +// BaseSuite provides the common structure and configuration-driven setup logic. +type BaseSuite struct { + suite.Suite + primaryFlags []string + secondaryFlags []string + primaryMount mountPoint + secondaryMount mountPoint + sharedCacheDir string +} + +// SharedChunkCacheTestSuite groups all shared chunk cache tests. +type SharedChunkCacheTestSuite struct{ BaseSuite } + +//////////////////////////////////////////////////////////////////////// +// Common Suite Logic +//////////////////////////////////////////////////////////////////////// + +func (t *BaseSuite) SetupTest() { + if testEnv.cfg.GKEMountedDirectory != "" { + t.sharedCacheDir = path.Join(GKETempDir, "shared-cache", "gcsfuse-shared-chunk-cache") + // Clean up cache directory before each test to ensure clean state + operations.RemoveDir(t.sharedCacheDir) + + // GKE Mode: Already mounted + t.primaryMount.mntDir = testEnv.cfg.GKEMountedDirectory + t.primaryMount.testDirPath = path.Join(t.primaryMount.mntDir, testDirName) + t.primaryMount.logFilePath = testEnv.cfg.LogFile // Might be empty, but that's fine for GKE + + if len(t.secondaryFlags) > 0 { + t.secondaryMount.mntDir = testEnv.cfg.GKEMountedDirectorySecondary + t.secondaryMount.testDirPath = path.Join(t.secondaryMount.mntDir, testDirName) + } + } else { + t.sharedCacheDir = path.Join(setup.TestDir(), GKETempDir, "shared-cache", "gcsfuse-shared-chunk-cache") + // Clean up cache directory before each test to ensure clean state + operations.RemoveDir(t.sharedCacheDir) + + // GCE Mode: Mount it + t.primaryMount.setupTestDir(testEnv.cfg.GCSFuseMountedDirectory, testEnv.cfg.LogFile) + t.mountGcsfuse(t.primaryMount, "primary", t.primaryFlags) + + if len(t.secondaryFlags) > 0 { + secondaryLog := path.Join(path.Dir(testEnv.cfg.LogFile), "gcsfuse_secondary.log") + t.secondaryMount.setupTestDir(testEnv.cfg.GCSFuseMountedDirectorySecondary, secondaryLog) + t.mountGcsfuse(t.secondaryMount, "secondary", t.secondaryFlags) + } + } +} + +func (t *BaseSuite) TearDownTest() { + if t.T().Failed() { + // Save logs for both mounts on failure to aid debugging. + testName := strings.ReplaceAll(t.T().Name(), "/", "_") + if t.primaryMount.logFilePath != "" { + setup.SaveLogFileAsArtifact(t.primaryMount.logFilePath, "gcsfuse-primary-log-"+testName) + } + if len(t.secondaryFlags) > 0 && t.secondaryMount.logFilePath != "" { + setup.SaveLogFileAsArtifact(t.secondaryMount.logFilePath, "gcsfuse-secondary-log-"+testName) + } + } + + if testEnv.cfg.GKEMountedDirectory != "" { + // GKE Mode: Just cleanup files + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(setup.TestBucket(), testDirName)) + } else { + // GCE Mode: Unmount and clean up + t.unmountAndCleanupMount(t.primaryMount, "primary") + if len(t.secondaryFlags) > 0 { + t.unmountAndCleanupMount(t.secondaryMount, "secondary") + } + } + + // Clean up shared cache directory after each test + operations.RemoveDir(t.sharedCacheDir) +} + +//////////////////////////////////////////////////////////////////////// +// Helpers +//////////////////////////////////////////////////////////////////////// + +func (mnt *mountPoint) setupTestDir(mountDir, logFile string) { + mnt.rootDir = setup.TestDir() + mnt.mntDir = mountDir + mnt.logFilePath = logFile + mnt.testDirPath = path.Join(mountDir, testDirName) +} + +func (t *BaseSuite) mountGcsfuse(mnt mountPoint, mountType string, flags []string) { + setup.SetMntDir(mnt.mntDir) + setup.SetLogFile(mnt.logFilePath) + log.Println("Running static mounting tests for shared chunk cache...") + err := static_mounting.MountGcsfuseWithStaticMounting(flags) + require.NoError(t.T(), err, "Unable to mount %s: %v", mountType, err) + mnt.testDirPath = setup.SetupTestDirectory(testDirName) + log.Printf("Running tests with %s mount flags %v", mountType, flags) +} + +func (t *BaseSuite) unmountAndCleanupMount(m mountPoint, name string) { + setup.UnmountGCSFuse(m.mntDir) + // Cleaning up the intermediate generated test files. + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(setup.TestBucket(), testDirName)) +} + +func (t *BaseSuite) createTestFile(fileName string, fileSize int) { + t.T().Helper() + testFilePath := path.Join(t.primaryMount.testDirPath, fileName) + operations.CreateFileOfSize(int64(fileSize), testFilePath, t.T()) +} + +func (t *BaseSuite) getCachedChunkCount() int { + t.T().Helper() + count := 0 + err := filepath.WalkDir(t.sharedCacheDir, func(filePath string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && strings.HasSuffix(filePath, ".bin") { + count++ + } + return nil + }) + if err != nil { + t.T().Logf("Error walking cache directory: %v", err) + } + return count +} + +func (t *BaseSuite) getCacheFileModTimes() map[string]os.FileInfo { + t.T().Helper() + modTimes := make(map[string]os.FileInfo) + err := filepath.WalkDir(t.sharedCacheDir, func(filePath string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && strings.HasSuffix(filePath, ".bin") { + info, err := os.Stat(filePath) + if err == nil { + modTimes[filePath] = info + } + } + return nil + }) + if err != nil { + t.T().Logf("Error walking cache directory: %v", err) + } + return modTimes +} + +//////////////////////////////////////////////////////////////////////// +// Test Cases +//////////////////////////////////////////////////////////////////////// + +// TestCacheMiss tests that when a small read triggers chunk caching from the first mount. +// Reading 2MB starting at 10MB offset will download and cache the entire 10MB chunk (2nd chunk). +func (t *SharedChunkCacheTestSuite) TestCacheMiss() { + const ( + testFileName = "test_cache_miss.txt" + fileSize = 30 * util.MiB + readSize = 2 * util.MiB // Read only 2MB + readOffset = 10 * util.MiB // Start at 10MB (in the 2nd chunk) + ) + + // Arrange: Set up test file and verify initial cache state + t.createTestFile(testFileName, fileSize) + initialCacheCount := t.getCachedChunkCount() + require.Equal(t.T(), 0, initialCacheCount, "Cache should be empty initially") + + // Act: Read 2MB from the 2nd chunk (triggers download and caching of entire 10MB chunk) + primaryFilePath := path.Join(t.primaryMount.testDirPath, testFileName) + startTime := time.Now() + chunk, err := operations.ReadChunkFromFile(primaryFilePath, readSize, readOffset, os.O_RDONLY) + cacheMissTime := time.Since(startTime) + + // Assert: Verify read succeeded and entire chunk was cached + require.NoError(t.T(), err, "Failed to read chunk from primary mount") + require.Equal(t.T(), int(readSize), len(chunk), "Read chunk size mismatch") + cachedChunkCount := t.getCachedChunkCount() + require.Equal(t.T(), 1, cachedChunkCount, "Cache should contain exactly 1 chunk after reading from 2nd chunk") + t.T().Logf("Cache miss test: Read %d bytes from offset %d in %v (cache miss), entire chunk cached (%d chunk)", + readSize, readOffset, cacheMissTime, cachedChunkCount) +} + +// TestCacheHit tests that when a small read is served from the shared chunk cache. +// Reading 2MB starting at 10MB offset should be served from the cached 10MB chunk (2nd chunk). +// This test verifies cache hits by checking that cache files are not modified. +func (t *SharedChunkCacheTestSuite) TestCacheHit() { + const ( + testFileName = "test_cache_hit.txt" + fileSize = 30 * util.MiB + readSize = 2 * util.MiB // Read only 2MB + readOffset = 10 * util.MiB // Start at 10MB (in the 2nd chunk) + ) + + // Arrange: Set up test file and populate cache via primary mount + t.createTestFile(testFileName, fileSize) + primaryFilePath := path.Join(t.primaryMount.testDirPath, testFileName) + primaryStartTime := time.Now() + primaryChunk, err := operations.ReadChunkFromFile(primaryFilePath, readSize, readOffset, os.O_RDONLY) + primaryCacheMissTime := time.Since(primaryStartTime) + require.NoError(t.T(), err, "Failed to read chunk from primary mount") + require.Equal(t.T(), int(readSize), len(primaryChunk), "Read chunk size mismatch on primary mount") + cachedChunkCount := t.getCachedChunkCount() + require.Equal(t.T(), 1, cachedChunkCount, "Cache should contain exactly 1 chunk after reading from 2nd chunk") + t.T().Logf("Primary read (cache miss): %d bytes from offset %d in %v, entire chunk cached (%d chunk)", + readSize, readOffset, primaryCacheMissTime, cachedChunkCount) + // Capture cache file modification times before secondary read + cacheFileTimes := t.getCacheFileModTimes() + require.NotEmpty(t.T(), cacheFileTimes, "Should have cache files to track") + t.T().Logf("Captured modification times for %d cache files", len(cacheFileTimes)) + + // Act: Read the same 2MB from the secondary mount (should be served from cache) + secondaryFilePath := path.Join(t.secondaryMount.testDirPath, testFileName) + // Warm up metadata cache on secondary mount by doing a stat first + // This ensures the timed read doesn't include initial metadata lookup overhead + _, err = os.Stat(secondaryFilePath) + require.NoError(t.T(), err, "Failed to stat file on secondary mount") + secondaryStartTime := time.Now() + secondaryChunk, err := operations.ReadChunkFromFile(secondaryFilePath, readSize, readOffset, os.O_RDONLY) + cacheHitTime := time.Since(secondaryStartTime) + + // Assert: Verify read succeeded from cache without re-downloading + require.NoError(t.T(), err, "Failed to read chunk from secondary mount") + require.Equal(t.T(), int(readSize), len(secondaryChunk), "Read chunk size mismatch on secondary mount") + require.Equal(t.T(), primaryChunk, secondaryChunk, "Chunk content from both mounts should match") + finalCacheCount := t.getCachedChunkCount() + require.Equal(t.T(), cachedChunkCount, finalCacheCount, + "Cache size should remain the same after reading from secondary mount (cache hit)") + // Verify cache files were NOT modified (proving they weren't re-downloaded) + newCacheFileTimes := t.getCacheFileModTimes() + for filePath, oldInfo := range cacheFileTimes { + newInfo, exists := newCacheFileTimes[filePath] + require.True(t.T(), exists, "Cache file should still exist: %s", filePath) + require.Equal(t.T(), oldInfo.ModTime(), newInfo.ModTime(), + "Cache file should not be modified (proving cache hit): %s", filePath) + } + speedup := float64(primaryCacheMissTime) / float64(cacheHitTime) + t.T().Logf("Secondary read (cache hit): %d bytes from offset %d in %v, cache files unchanged (%d chunk)", + readSize, readOffset, cacheHitTime, finalCacheCount) + t.T().Logf("Performance: Cache miss=%v, Cache hit=%v, Speedup=%.2fx", + primaryCacheMissTime, cacheHitTime, speedup) +} + +// TestCacheHitSingleMount tests cache behavior within a single mount. +// First read causes a cache miss (downloads and caches the chunk). +// Subsequent read of the same chunk should be a cache hit (served from cache). +func (t *SharedChunkCacheTestSuite) TestCacheHitSingleMount() { + const ( + testFileName = "test_cache_hit_single_mount.txt" + fileSize = 30 * util.MiB + readSize = 2 * util.MiB // Read only 2MB + readOffset = 10 * util.MiB // Start at 10MB (in the 2nd chunk) + ) + + // Arrange: Set up test file, verify empty cache, and perform first read to populate cache + t.createTestFile(testFileName, fileSize) + initialCacheCount := t.getCachedChunkCount() + require.Equal(t.T(), 0, initialCacheCount, "Cache should be empty initially") + primaryFilePath := path.Join(t.primaryMount.testDirPath, testFileName) + firstReadStart := time.Now() + firstChunk, err := operations.ReadChunkFromFile(primaryFilePath, readSize, readOffset, os.O_RDONLY) + cacheMissTime := time.Since(firstReadStart) + require.NoError(t.T(), err, "Failed to read chunk on first read (cache miss)") + require.Equal(t.T(), int(readSize), len(firstChunk), "Read chunk size mismatch on first read") + cachedChunkCount := t.getCachedChunkCount() + require.Equal(t.T(), 1, cachedChunkCount, "Cache should contain exactly 1 chunk after first read") + t.T().Logf("First read (cache miss): %d bytes from offset %d in %v, chunk cached (%d chunk)", + readSize, readOffset, cacheMissTime, cachedChunkCount) + // Capture cache file modification times before second read + cacheFileTimes := t.getCacheFileModTimes() + require.NotEmpty(t.T(), cacheFileTimes, "Should have cache files to track") + + // Act: Read the same chunk again from the same mount (should hit cache) + secondReadStart := time.Now() + secondChunk, err := operations.ReadChunkFromFile(primaryFilePath, readSize, readOffset, os.O_RDONLY) + cacheHitTime := time.Since(secondReadStart) + + // Assert: Verify second read succeeded from cache without re-downloading + require.NoError(t.T(), err, "Failed to read chunk on second read (cache hit)") + require.Equal(t.T(), int(readSize), len(secondChunk), "Read chunk size mismatch on second read") + require.Equal(t.T(), firstChunk, secondChunk, "Content should match between first and second read") + finalCacheCount := t.getCachedChunkCount() + require.Equal(t.T(), cachedChunkCount, finalCacheCount, + "Cache size should remain the same after second read (cache hit)") + // Verify cache files were NOT modified (proving cache hit) + newCacheFileTimes := t.getCacheFileModTimes() + for filePath, oldInfo := range cacheFileTimes { + newInfo, exists := newCacheFileTimes[filePath] + require.True(t.T(), exists, "Cache file should still exist: %s", filePath) + require.Equal(t.T(), oldInfo.ModTime(), newInfo.ModTime(), + "Cache file should not be modified (proving cache hit): %s", filePath) + } + speedup := float64(cacheMissTime) / float64(cacheHitTime) + t.T().Logf("Second read (cache hit): %d bytes from offset %d in %v, cache files unchanged (%d chunk)", + readSize, readOffset, cacheHitTime, finalCacheCount) + t.T().Logf("Single mount performance: Cache miss=%v, Cache hit=%v, Speedup=%.2fx", + cacheMissTime, cacheHitTime, speedup) +} + +//////////////////////////////////////////////////////////////////////// +// Test Suite Runner +//////////////////////////////////////////////////////////////////////// + +func RunTests(t *testing.T, runName string, factory func(primaryFlags, secondaryFlags []string) suite.TestingSuite) { + for _, cfg := range testEnv.cfg.Configs { + if cfg.Run == runName { + for i, flagStr := range cfg.Flags { + primaryFlags := strings.Fields(flagStr) + var secondaryFlags []string + if len(cfg.SecondaryFlags) > i { + secondaryFlags = strings.Fields(cfg.SecondaryFlags[i]) + } + suite.Run(t, factory(primaryFlags, secondaryFlags)) + } + } + } +} + +func TestSharedChunkCacheTestSuite(t *testing.T) { + RunTests(t, "TestSharedChunkCacheTestSuite", func(primaryFlags, secondaryFlags []string) suite.TestingSuite { + s := &SharedChunkCacheTestSuite{} + s.primaryFlags = primaryFlags + s.secondaryFlags = secondaryFlags + return s + }) +} diff --git a/tools/integration_tests/stale_handle/setup_test.go b/tools/integration_tests/stale_handle/setup_test.go new file mode 100644 index 0000000000..a1119961dd --- /dev/null +++ b/tools/integration_tests/stale_handle/setup_test.go @@ -0,0 +1,115 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package stale_handle + +import ( + "context" + "log" + "os" + "testing" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" +) + +const ( + testDirName = "StaleHandleTest" +) + +var ( + testEnv env + mountFunc func(*test_suite.TestConfig, []string) error + // mount directory is where our tests run. + mountDir string + rootDir string +) + +type env struct { + storageClient *storage.Client + ctx context.Context + testDirPath string + cfg *test_suite.TestConfig + bucketType string +} + +//////////////////////////////////////////////////////////////////////// +// TestMain +//////////////////////////////////////////////////////////////////////// + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.StaleHandle) == 0 { + log.Println("No configuration found for stale_handle tests in config. Using flags instead.") + if setup.MountedDirectory() != "" { + log.Println("Skip mounted directory tests if no config file has been passed.") + os.Exit(0) + } + // Populate the config manually. + cfg.StaleHandle = make([]test_suite.TestConfig, 1) + cfg.StaleHandle[0].TestBucket = setup.TestBucket() + cfg.StaleHandle[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.StaleHandle[0].LogFile = setup.LogFile() + cfg.StaleHandle[0].Configs = make([]test_suite.ConfigItem, 4) + cfg.StaleHandle[0].Configs[0].Flags = []string{ + "--metadata-cache-ttl-secs=0 --write-block-size-mb=1 --write-max-blocks-per-file=1", + "--metadata-cache-ttl-secs=0 --write-block-size-mb=1 --write-max-blocks-per-file=1 --client-protocol=grpc", + } + cfg.StaleHandle[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.StaleHandle[0].Configs[0].Run = "TestStaleHandleStreamingWritesEnabled" + + cfg.StaleHandle[0].Configs[1].Flags = []string{ + "--metadata-cache-ttl-secs=0 --enable-streaming-writes=false", + "--metadata-cache-ttl-secs=0 --enable-streaming-writes=false --client-protocol=grpc", + } + cfg.StaleHandle[0].Configs[1].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.StaleHandle[0].Configs[1].Run = "TestStaleHandleStreamingWritesDisabled" + } + + testEnv.ctx = context.Background() + testEnv.cfg = &cfg.StaleHandle[0] + testEnv.bucketType = setup.TestEnvironment(testEnv.ctx, testEnv.cfg) + + // 2. Create storage client before running tests. + var err error + testEnv.storageClient, err = client.CreateStorageClient(testEnv.ctx) + if err != nil { + log.Printf("Error creating storage client: %v\n", err) + os.Exit(1) + } + defer testEnv.storageClient.Close() + + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + // flags to be set, as stale handle tests validates content from the bucket. + // Note: These tests by default can only be run for non streaming mounts. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + mountDir, rootDir = testEnv.cfg.GKEMountedDirectory, testEnv.cfg.GKEMountedDirectory + os.Exit(setup.RunTestsForMountedDirectory(testEnv.cfg.GKEMountedDirectory, m)) + } + + // Run tests for testBucket + setup.SetUpTestDirForTestBucket(testEnv.cfg) + mountDir, rootDir = testEnv.cfg.GCSFuseMountedDirectory, testEnv.cfg.GCSFuseMountedDirectory + + log.Println("Running static mounting tests...") + mountFunc = static_mounting.MountGcsfuseWithStaticMountingWithConfigFile + successCode := m.Run() + + os.Exit(successCode) +} diff --git a/tools/integration_tests/stale_handle/stale_file_handle_common_test.go b/tools/integration_tests/stale_handle/stale_file_handle_common_test.go new file mode 100644 index 0000000000..43d98c8370 --- /dev/null +++ b/tools/integration_tests/stale_handle/stale_file_handle_common_test.go @@ -0,0 +1,110 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stale_handle + +import ( + "os" + "path" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +// ////////////////////////////////////////////////////////////////////// +// Boilerplate +// ////////////////////////////////////////////////////////////////////// + +type staleFileHandleCommon struct { + suite.Suite + flags []string + f1 *os.File + fileName string + data string + isStreamingWritesEnabled bool + isLocal bool +} + +func (s *staleFileHandleCommon) SetupSuite() { + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, s.flags, mountFunc) + setup.SetMntDir(mountDir) + testEnv.testDirPath = SetupTestDirectory(testEnv.ctx, testEnv.storageClient, testDirName) + s.data = setup.GenerateRandomString(5 * util.MiB) +} + +func (s *staleFileHandleCommon) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func (s *staleFileHandleCommon) TestClobberedFileSyncAndCloseThrowsStaleFileHandleError() { + // TODO(b/410698332): Remove skip condition once takeover support is available. + if s.isStreamingWritesEnabled && setup.IsZonalBucketRun() { + s.T().Skip("Skip test due to unable to overwrite the unfinalized zonal object.") + } + // Dirty the file by giving it some contents. + operations.WriteWithoutClose(s.f1, s.data, s.T()) + // Clobber file by replacing the underlying object with a new generation. + err := WriteToObject(testEnv.ctx, testEnv.storageClient, path.Join(testDirName, s.fileName), FileContents, storage.Conditions{}) + assert.NoError(s.T(), err) + + operations.ValidateSyncGivenThatFileIsClobbered(s.T(), s.f1, s.isStreamingWritesEnabled) + + err = s.f1.Close() + operations.ValidateESTALEError(s.T(), err) + ValidateObjectContentsFromGCS(testEnv.ctx, testEnv.storageClient, testDirName, s.fileName, FileContents, s.T()) +} + +func (s *staleFileHandleCommon) TestFileDeletedLocallySyncAndCloseDoNotThrowError() { + // Dirty the file by giving it some contents. + operations.WriteWithoutClose(s.f1, s.data, s.T()) + + // Delete the file. + operations.RemoveFile(s.f1.Name()) + + // Verify unlink operation succeeds. + operations.ValidateNoFileOrDirError(s.T(), s.f1.Name()) + // Attempt to write to file should not give any error. + operations.WriteWithoutClose(s.f1, s.data, s.T()) + operations.SyncFile(s.f1, s.T()) + operations.CloseFileShouldNotThrowError(s.T(), s.f1) + ValidateObjectNotFoundErrOnGCS(testEnv.ctx, testEnv.storageClient, testDirName, s.fileName, s.T()) +} + +func (s *staleFileHandleCommon) TestRenamedFileSyncAndCloseThrowsStaleFileHandleError() { + // Dirty the file by giving it some contents. + _, err := s.f1.WriteString(s.data) + assert.NoError(s.T(), err) + newFile := "new" + s.fileName + + err = operations.RenameFile(s.f1.Name(), path.Join(testEnv.testDirPath, newFile)) + + assert.NoError(s.T(), err) + _, err = s.f1.WriteString(s.data) + operations.ValidateESTALEError(s.T(), err) + // Sync/Flush call won't throw error as data couldn't be written after rename, so we don't have anything to upload. + err = s.f1.Sync() + require.NoError(s.T(), err) + err = s.f1.Close() + require.NoError(s.T(), err) +} diff --git a/tools/integration_tests/stale_handle/stale_file_handle_local_and_synced_file_test.go b/tools/integration_tests/stale_handle/stale_file_handle_local_and_synced_file_test.go new file mode 100644 index 0000000000..9fdd3da16c --- /dev/null +++ b/tools/integration_tests/stale_handle/stale_file_handle_local_and_synced_file_test.go @@ -0,0 +1,191 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stale_handle + +import ( + "log" + "path" + "testing" + + "cloud.google.com/go/storage" + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +// ////////////////////////////////////////////////////////////////////// +// Boilerplate +// ////////////////////////////////////////////////////////////////////// +type staleFileHandleLocalFile struct { + staleFileHandleCommon +} + +type staleFileHandleEmptyGcsFile struct { + staleFileHandleCommon +} + +// ////////////////////////////////////////////////////////////////////// +// Helpers +// ////////////////////////////////////////////////////////////////////// + +func (s *staleFileHandleLocalFile) SetupTest() { + // Create a local file. + s.fileName = path.Base(s.T().Name()) + setup.GenerateRandomString(5) + s.f1 = operations.OpenFileWithODirect(s.T(), path.Join(testEnv.testDirPath, s.fileName)) + s.isLocal = true +} +func (s *staleFileHandleLocalFile) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +func (s *staleFileHandleEmptyGcsFile) SetupTest() { + s.fileName = path.Base(s.T().Name()) + setup.GenerateRandomString(5) + err := CreateObjectOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(testDirName, s.fileName), "") + assert.NoError(s.T(), err) + s.f1 = operations.OpenFileWithODirect(s.T(), path.Join(testEnv.testDirPath, s.fileName)) +} +func (s *staleFileHandleEmptyGcsFile) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func (s *staleFileHandleEmptyGcsFile) TestClobberedFileReadThrowsStaleFileHandleError() { + // TODO(b/410698332): Remove skip condition once takeover support is available. + if s.isStreamingWritesEnabled && setup.IsZonalBucketRun() { + s.T().Skip("Skip test due to takeover support not available.") + } + // Dirty the file by giving it some contents. + _, err := s.f1.WriteAt([]byte(s.data), 0) + assert.NoError(s.T(), err) + operations.SyncFile(s.f1, s.T()) + + // Replace the underlying object with a new generation. + err = WriteToObject(testEnv.ctx, testEnv.storageClient, path.Join(testDirName, s.fileName), FileContents, storage.Conditions{}) + + assert.NoError(s.T(), err) + buffer := make([]byte, len(s.data)) + _, err = s.f1.Read(buffer) + operations.ValidateESTALEError(s.T(), err) +} + +func (s *staleFileHandleEmptyGcsFile) TestClobberedFileFirstWriteThrowsStaleFileHandleError() { + // TODO(b/410698332): Remove skip condition once takeover support is available. + if s.isStreamingWritesEnabled && setup.IsZonalBucketRun() { + s.T().Skip("Skip test due to takeover support not available.") + } + // Clobber file by replacing the underlying object with a new generation. + err := WriteToObject(testEnv.ctx, testEnv.storageClient, path.Join(testDirName, s.fileName), FileContents, storage.Conditions{}) + assert.NoError(s.T(), err) + + // Attempt first write to the file should give stale NFS file handle error. + _, err = s.f1.Write([]byte(s.data)) + + assert.NoError(s.T(), err) + operations.ValidateSyncGivenThatFileIsClobbered(s.T(), s.f1, s.isStreamingWritesEnabled) + err = s.f1.Close() + operations.ValidateESTALEError(s.T(), err) + ValidateObjectContentsFromGCS(testEnv.ctx, testEnv.storageClient, testDirName, s.fileName, FileContents, s.T()) +} + +func (s *staleFileHandleEmptyGcsFile) TestFileDeletedRemotelySyncAndCloseThrowsStaleFileHandleError() { + // TODO(mohitkyadav): Enable test once fix in b/415713332 is released + if s.isStreamingWritesEnabled && setup.IsZonalBucketRun() { + s.T().Skip("Skip test due to bug (b/415713332) in client.") + } + // Dirty the file by giving it some contents. + operations.WriteWithoutClose(s.f1, s.data, s.T()) + // Delete the file remotely. + err := DeleteObjectOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(testDirName, s.fileName)) + assert.NoError(s.T(), err) + // Verify unlink operation succeeds. + ValidateObjectNotFoundErrOnGCS(testEnv.ctx, testEnv.storageClient, testDirName, s.fileName, s.T()) + // Attempt to write to file should not give any error. + operations.WriteWithoutClose(s.f1, s.data, s.T()) + + operations.ValidateSyncGivenThatFileIsClobbered(s.T(), s.f1, s.isStreamingWritesEnabled) + + err = s.f1.Close() + operations.ValidateESTALEError(s.T(), err) + ValidateObjectNotFoundErrOnGCS(testEnv.ctx, testEnv.storageClient, testDirName, s.fileName, s.T()) +} + +//////////////////////////////////////////////////////////////////////// +// Test Function (Runs once before all tests) +//////////////////////////////////////////////////////////////////////// + +func TestStaleHandleStreamingWritesEnabled(t *testing.T) { + // Run tests for mounted directory if the flag is set and return. + if setup.AreBothMountedDirectoryAndTestBucketFlagsSet() { + // Run tests for local file. + suite.Run(t, &staleFileHandleLocalFile{staleFileHandleCommon{isStreamingWritesEnabled: true}}) + + // Run tests for empty gcs file. + suite.Run(t, &staleFileHandleEmptyGcsFile{staleFileHandleCommon{isStreamingWritesEnabled: true}}) + + return + } + + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, flags := range flagsSet { + // Run local file tests + sLocal := new(staleFileHandleLocalFile) + sLocal.flags = flags + log.Printf("Running local file tests with flags: %s", sLocal.flags) + sLocal.isStreamingWritesEnabled = true + suite.Run(t, sLocal) + + // Run empty GCS file tests + sEmptyGCS := new(staleFileHandleEmptyGcsFile) + sEmptyGCS.flags = flags + log.Printf("Running empty GCS file tests with flags: %s", sEmptyGCS.flags) + sEmptyGCS.isStreamingWritesEnabled = true + suite.Run(t, sEmptyGCS) + } +} + +func TestStaleHandleStreamingWritesDisabled(t *testing.T) { + // Run tests for mounted directory if the flag is set and return. + if setup.AreBothMountedDirectoryAndTestBucketFlagsSet() { + // Run tests for local file. + suite.Run(t, &staleFileHandleLocalFile{staleFileHandleCommon{isStreamingWritesEnabled: false}}) + + // Run tests for empty gcs file. + suite.Run(t, &staleFileHandleEmptyGcsFile{staleFileHandleCommon{isStreamingWritesEnabled: false}}) + + return + } + + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, flags := range flagsSet { + // Run local file tests + sLocal := new(staleFileHandleLocalFile) + sLocal.flags = flags + log.Printf("Running local file tests with flags: %s", sLocal.flags) + sLocal.isStreamingWritesEnabled = false + suite.Run(t, sLocal) + + // Run empty GCS file tests + sEmptyGCS := new(staleFileHandleEmptyGcsFile) + sEmptyGCS.flags = flags + log.Printf("Running empty GCS file tests with flags: %s", sEmptyGCS.flags) + sEmptyGCS.isStreamingWritesEnabled = false + suite.Run(t, sEmptyGCS) + } +} diff --git a/tools/integration_tests/streaming_writes/buffer_size_test.go b/tools/integration_tests/streaming_writes/buffer_size_test.go new file mode 100644 index 0000000000..38dcb90d37 --- /dev/null +++ b/tools/integration_tests/streaming_writes/buffer_size_test.go @@ -0,0 +1,99 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streaming_writes + +import ( + "os" + "path" + "testing" + + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWritesWithDifferentConfig(t *testing.T) { + // Do not run this test with mounted directory flag. + if testEnv.cfg.GKEMountedDirectory != "" { + t.SkipNow() + } + // Create a separate mountDir for these tests so it doesn't interfere with the other tests. + oldMntDir := testEnv.cfg.GCSFuseMountedDirectory + newMountDir := path.Join(setup.TestDir(), "mntTestWritesWithDifferentConfig") + err := os.MkdirAll(newMountDir, 0755) + assert.True(t, err == nil || os.IsExist(err)) + testEnv.cfg.GCSFuseMountedDirectory = newMountDir + defer func() { + testEnv.cfg.GCSFuseMountedDirectory = oldMntDir + }() + testCases := []struct { + name string + flags []string + fileSize int64 + }{ + { + name: "BlockSizeGreaterThanFileSize", + flags: []string{"--write-block-size-mb=5", "--write-max-blocks-per-file=2"}, + fileSize: 2 * 1024 * 1024, + }, + { + name: "BlockSizeLessThanFileSize", + flags: []string{"--write-block-size-mb=1", "--write-max-blocks-per-file=20"}, + fileSize: 5 * 1024 * 1024, + }, + { + // BlockSize*num_blocks < fileSize + name: "NumberOfBlocksLessThanFileSize", + flags: []string{"--write-block-size-mb=1", "--write-max-blocks-per-file=2"}, + fileSize: 10 * 1024 * 1024, + }, + { + name: "BlockSizeEqualToFileSize", + flags: []string{"--write-block-size-mb=5", "--write-max-blocks-per-file=2"}, + fileSize: 5 * 1024 * 1024, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := static_mounting.MountGcsfuseWithStaticMountingWithConfigFile(&testEnv.cfg, tc.flags) + require.NoError(t, err) + defer setup.SaveGCSFuseLogFileInCaseOfFailure(t) + defer setup.UnmountGCSFuseWithConfig(&testEnv.cfg) + testEnv.testDirPath = setup.SetupTestDirectory(testDirName) + // Create a local file. + fh := operations.CreateFile(path.Join(testEnv.testDirPath, FileName1), FilePerms, t) + testDirName := GetDirName(testEnv.testDirPath) + if setup.IsZonalBucketRun() { + ValidateObjectContentsFromGCS(testEnv.ctx, testEnv.storageClient, testDirName, FileName1, "", t) + } else { + ValidateObjectNotFoundErrOnGCS(testEnv.ctx, testEnv.storageClient, testDirName, FileName1, t) + } + data, err := operations.GenerateRandomData(tc.fileSize) + if err != nil { + t.Fatalf("Error in generating data: %v", err) + } + + // Write data to file. + operations.WriteAt(string(data[:]), 0, fh, t) + + // Close the file and validate that the file is created on GCS. + CloseFileAndValidateContentFromGCS(testEnv.ctx, testEnv.storageClient, fh, testDirName, FileName1, string(data[:]), t) + }) + } +} diff --git a/tools/integration_tests/streaming_writes/common_streaming_writes_suite_test.go b/tools/integration_tests/streaming_writes/common_streaming_writes_suite_test.go new file mode 100644 index 0000000000..f5a549222a --- /dev/null +++ b/tools/integration_tests/streaming_writes/common_streaming_writes_suite_test.go @@ -0,0 +1,51 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streaming_writes + +import ( + "os" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type StreamingWritesSuite struct { + f1 *os.File + fileName string + // filePath of the above file in the mounted directory. + filePath string + data string + test_suite.TestifySuite +} + +func (t *StreamingWritesSuite) SetupSuite() { + testEnv.testDirPath = setup.SetupTestDirectory(testDirName) + t.data = setup.GenerateRandomString(5 * util.MiB) +} + +func (t *StreamingWritesSuite) TearDownSuite() { + setup.SaveGCSFuseLogFileInCaseOfFailure(t.T()) +} + +func (t *StreamingWritesSuite) validateReadCall(fh *os.File, content string) { + readContent := make([]byte, len(content)) + n, err := fh.ReadAt(readContent, 0) + require.NoError(t.T(), err) + assert.Equal(t.T(), len(content), n) + assert.Equal(t.T(), content, string(readContent)) +} diff --git a/tools/integration_tests/streaming_writes/empty_gcs_file_test.go b/tools/integration_tests/streaming_writes/empty_gcs_file_test.go new file mode 100644 index 0000000000..f61a063526 --- /dev/null +++ b/tools/integration_tests/streaming_writes/empty_gcs_file_test.go @@ -0,0 +1,54 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streaming_writes + +import ( + "path" + "testing" + + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/suite" +) + +type streamingWritesEmptyGCSFileTestSuite struct { + StreamingWritesSuite + suite.Suite +} + +func (t *streamingWritesEmptyGCSFileTestSuite) SetupTest() { + t.createEmptyGCSFile() +} + +func (t *streamingWritesEmptyGCSFileTestSuite) SetupSubTest() { + t.createEmptyGCSFile() +} + +func (t *streamingWritesEmptyGCSFileTestSuite) createEmptyGCSFile() { + t.fileName = FileName1 + setup.GenerateRandomString(5) + // Create an empty file on GCS. + CreateObjectInGCSTestDir(testEnv.ctx, testEnv.storageClient, testDirName, t.fileName, "", t.T()) + ValidateObjectContentsFromGCS(testEnv.ctx, testEnv.storageClient, testDirName, t.fileName, "", t.T()) + t.filePath = path.Join(testEnv.testDirPath, t.fileName) + t.f1 = operations.OpenFileWithODirect(t.T(), t.filePath) +} + +// Executes all tests that run with single streamingWrites configuration for empty GCS Files. +func TestEmptyGCSFileTestSuiteTest(t *testing.T) { + s := new(streamingWritesEmptyGCSFileTestSuite) + s.StreamingWritesSuite.TestifySuite = &s.Suite + suite.Run(t, s) +} diff --git a/tools/integration_tests/streaming_writes/local_file_test.go b/tools/integration_tests/streaming_writes/local_file_test.go new file mode 100644 index 0000000000..cc85259693 --- /dev/null +++ b/tools/integration_tests/streaming_writes/local_file_test.go @@ -0,0 +1,52 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streaming_writes + +import ( + "path" + "testing" + + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/suite" +) + +type streamingWritesLocalFileTestSuite struct { + StreamingWritesSuite + suite.Suite +} + +func (t *streamingWritesLocalFileTestSuite) SetupTest() { + t.createLocalFile() +} + +func (t *streamingWritesLocalFileTestSuite) SetupSubTest() { + t.createLocalFile() +} + +func (t *streamingWritesLocalFileTestSuite) createLocalFile() { + t.fileName = FileName1 + setup.GenerateRandomString(5) + t.filePath = path.Join(testEnv.testDirPath, t.fileName) + // Create a local file with O_DIRECT. + t.f1 = operations.OpenFileWithODirect(t.T(), t.filePath) +} + +// Executes all tests that run with single streamingWrites configuration for localFiles. +func TestStreamingWritesLocalFileTestSuite(t *testing.T) { + s := new(streamingWritesLocalFileTestSuite) + s.StreamingWritesSuite.TestifySuite = &s.Suite + suite.Run(t, s) +} diff --git a/tools/integration_tests/streaming_writes/read_file_test.go b/tools/integration_tests/streaming_writes/read_file_test.go new file mode 100644 index 0000000000..f1b360e0f7 --- /dev/null +++ b/tools/integration_tests/streaming_writes/read_file_test.go @@ -0,0 +1,76 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streaming_writes + +import ( + "path" + + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func (t *StreamingWritesSuite) TestReadFileAfterSync() { + // Write some content to the file. + _, err := t.f1.WriteAt([]byte(t.data), 0) + assert.NoError(t.T(), err) + // Sync File to ensure buffers are flushed to GCS. + operations.SyncFile(t.f1, t.T()) + + t.validateReadCall(t.f1, t.data) + + // Close the file and validate that the file is created on GCS. + CloseFileAndValidateContentFromGCS(testEnv.ctx, testEnv.storageClient, t.f1, testDirName, t.fileName, t.data, t.T()) +} + +func (t *StreamingWritesSuite) TestReadBeforeFileIsFlushed() { + // Write data to file. + operations.WriteAt(t.data, 0, t.f1, t.T()) + + // Try to read the file. + t.validateReadCall(t.f1, t.data) + + // Validate if correct content is uploaded to GCS after read error. + CloseFileAndValidateContentFromGCS(testEnv.ctx, testEnv.storageClient, t.f1, testDirName, t.fileName, t.data, t.T()) +} + +func (t *StreamingWritesSuite) TestReadBeforeSyncThenWriteAgainAndRead() { + // Write data to file. + operations.WriteAt(t.data, 0, t.f1, t.T()) + + t.validateReadCall(t.f1, t.data) + + operations.WriteAt(t.data, int64(len(t.data)), t.f1, t.T()) + t.validateReadCall(t.f1, t.data+t.data) + // Validate if correct content is uploaded to GCS after read. + CloseFileAndValidateContentFromGCS(testEnv.ctx, testEnv.storageClient, t.f1, testDirName, t.fileName, t.data+t.data, t.T()) +} + +func (t *StreamingWritesSuite) TestReadAfterFlush() { + // Write data to file and flush. + operations.WriteAt(t.data, 0, t.f1, t.T()) + CloseFileAndValidateContentFromGCS(testEnv.ctx, testEnv.storageClient, t.f1, testDirName, t.fileName, t.data, t.T()) + + // Perform read and validate the contents. + var err error + t.f1, err = operations.OpenFileAsReadonly(path.Join(testEnv.testDirPath, t.fileName)) + require.NoError(t.T(), err) + buf := make([]byte, len(t.data)) + _, err = t.f1.Read(buf) + + require.NoError(t.T(), err) + require.Equal(t.T(), string(buf), t.data) +} diff --git a/tools/integration_tests/streaming_writes/rename_file_test.go b/tools/integration_tests/streaming_writes/rename_file_test.go new file mode 100644 index 0000000000..93e4edf446 --- /dev/null +++ b/tools/integration_tests/streaming_writes/rename_file_test.go @@ -0,0 +1,64 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streaming_writes + +import ( + "path" + + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/stretchr/testify/require" +) + +func (t *StreamingWritesSuite) TestRenameBeforeFileIsFlushed() { + operations.WriteWithoutClose(t.f1, t.data, t.T()) + operations.WriteWithoutClose(t.f1, t.data, t.T()) + operations.VerifyStatFile(t.filePath, int64(2*len(t.data)), FilePerms, t.T()) + err := t.f1.Sync() + require.NoError(t.T(), err) + + newFile := "new" + t.fileName + destDirPath := path.Join(testEnv.testDirPath, newFile) + err = operations.RenameFile(t.filePath, destDirPath) + + // Validate that move didn't throw any error. + require.NoError(t.T(), err) + // Verify the new object contents. + ValidateObjectContentsFromGCS(testEnv.ctx, testEnv.storageClient, testDirName, newFile, t.data+t.data, t.T()) + require.NoError(t.T(), t.f1.Close()) + // Check if old object is deleted. + ValidateObjectNotFoundErrOnGCS(testEnv.ctx, testEnv.storageClient, testDirName, t.fileName, t.T()) +} + +func (t *StreamingWritesSuite) TestSyncAfterRenameSucceeds() { + _, err := t.f1.WriteAt([]byte(t.data), 0) + require.NoError(t.T(), err) + operations.VerifyStatFile(t.filePath, int64(len(t.data)), FilePerms, t.T()) + err = t.f1.Sync() + require.NoError(t.T(), err) + newFile := "new" + t.fileName + err = operations.RenameFile(t.filePath, path.Join(testEnv.testDirPath, newFile)) + require.NoError(t.T(), err) + + err = t.f1.Sync() + + // Verify that sync succeeds after rename. + require.NoError(t.T(), err) + // Verify the new object contents. + ValidateObjectContentsFromGCS(testEnv.ctx, testEnv.storageClient, testDirName, newFile, string(t.data), t.T()) + require.NoError(t.T(), t.f1.Close()) + // Check if old object is deleted. + ValidateObjectNotFoundErrOnGCS(testEnv.ctx, testEnv.storageClient, testDirName, t.fileName, t.T()) +} diff --git a/tools/integration_tests/streaming_writes/setup_test.go b/tools/integration_tests/streaming_writes/setup_test.go new file mode 100644 index 0000000000..9f9e7b5c27 --- /dev/null +++ b/tools/integration_tests/streaming_writes/setup_test.go @@ -0,0 +1,89 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streaming_writes + +import ( + "context" + "log" + "os" + "testing" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" +) + +const ( + testDirName = "StreamingWritesTest" +) + +type env struct { + storageClient *storage.Client + ctx context.Context + testDirPath string + cfg test_suite.TestConfig +} + +var testEnv env + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.StreamingWrites) == 0 { + log.Println("No configuration found for streaming_writes tests in config. Using flags instead.") + // Populate the config manually. + cfg.StreamingWrites = make([]test_suite.TestConfig, 1) + cfg.StreamingWrites[0].TestBucket = setup.TestBucket() + cfg.StreamingWrites[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.StreamingWrites[0].LogFile = setup.LogFile() + cfg.StreamingWrites[0].Configs = make([]test_suite.ConfigItem, 1) + cfg.StreamingWrites[0].Configs[0].Flags = []string{ + "--rename-dir-limit=3 --write-block-size-mb=1 --write-max-blocks-per-file=2 --client-protocol=grpc --write-global-max-blocks=-1", + "--rename-dir-limit=3 --write-block-size-mb=1 --write-max-blocks-per-file=2 --write-global-max-blocks=-1", + } + cfg.StreamingWrites[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + } + + testEnv.ctx = context.Background() + bucketType := setup.TestEnvironment(testEnv.ctx, &cfg.StreamingWrites[0]) + testEnv.cfg = cfg.StreamingWrites[0] + + // 2. Create storage client before running tests. + var err error + testEnv.storageClient, err = client.CreateStorageClient(testEnv.ctx) + if err != nil { + log.Printf("Error creating storage client: %v\n", err) + os.Exit(1) + } + defer testEnv.storageClient.Close() + + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + os.Exit(setup.RunTestsForMountedDirectory(testEnv.cfg.GKEMountedDirectory, m)) + } + + // Run tests for testBucket + // 4. Build the flag sets dynamically from the config. + flagsSet := setup.BuildFlagSets(testEnv.cfg, bucketType, "") + setup.SetUpTestDirForTestBucket(&testEnv.cfg) + + successCode := static_mounting.RunTestsWithConfigFile(&testEnv.cfg, flagsSet, m) + setup.SaveLogFileInCaseOfFailure(successCode) + os.Exit(successCode) +} diff --git a/tools/integration_tests/streaming_writes/symlink_file_test.go b/tools/integration_tests/streaming_writes/symlink_file_test.go new file mode 100644 index 0000000000..bec93be904 --- /dev/null +++ b/tools/integration_tests/streaming_writes/symlink_file_test.go @@ -0,0 +1,66 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streaming_writes + +import ( + "os" + "path" + + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" +) + +func (t *StreamingWritesSuite) TestCreateSymlinkForLocalFileAndReadFromSymlink() { + // Create Symlink. + symlink := path.Join(testEnv.testDirPath, setup.GenerateRandomString(5)) + operations.CreateSymLink(t.filePath, symlink, t.T()) + _, err := t.f1.WriteAt([]byte(t.data), 0) + assert.NoError(t.T(), err) + // Verify read link. + operations.VerifyReadLink(t.filePath, symlink, t.T()) + + // Validate read file from symlink. + symlink_fh := operations.OpenFile(symlink, t.T()) + defer operations.CloseFileShouldNotThrowError(t.T(), symlink_fh) + t.validateReadCall(symlink_fh, t.data) + + // Close the file and validate that the file is created on GCS. + CloseFileAndValidateContentFromGCS(testEnv.ctx, testEnv.storageClient, t.f1, testDirName, t.fileName, t.data, t.T()) +} + +func (t *StreamingWritesSuite) TestReadingFromSymlinkForDeletedLocalFile() { + // Create Symlink. + symlink := path.Join(testEnv.testDirPath, setup.GenerateRandomString(5)) + operations.CreateSymLink(t.filePath, symlink, t.T()) + _, err := t.f1.WriteAt([]byte(t.data), 0) + assert.NoError(t.T(), err) + // Verify read link. + operations.VerifyReadLink(t.filePath, symlink, t.T()) + + // Validate read from symlink. + symlink_fh := operations.OpenFile(symlink, t.T()) + defer operations.CloseFileShouldNotThrowError(t.T(), symlink_fh) + t.validateReadCall(symlink_fh, t.data) + + // Remove filePath and then close the fileHandle to avoid syncing to GCS. + operations.RemoveFile(t.filePath) + operations.CloseFileShouldNotThrowError(t.T(), t.f1) + ValidateObjectNotFoundErrOnGCS(testEnv.ctx, testEnv.storageClient, testDirName, t.fileName, t.T()) + // Reading symlink should fail. + _, err = os.Stat(symlink) + assert.Error(t.T(), err) +} diff --git a/tools/integration_tests/streaming_writes/truncate_file_test.go b/tools/integration_tests/streaming_writes/truncate_file_test.go new file mode 100644 index 0000000000..774cd2a561 --- /dev/null +++ b/tools/integration_tests/streaming_writes/truncate_file_test.go @@ -0,0 +1,189 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streaming_writes + +import ( + "os" + + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func (t *StreamingWritesSuite) TestTruncate() { + truncateSize := 2 * 1024 * 1024 + + err := t.f1.Truncate(int64(truncateSize)) + + assert.NoError(t.T(), err) + data := make([]byte, truncateSize) + // Verify that GCSFuse is returning correct file size before the file is uploaded. + operations.VerifyStatFile(t.filePath, int64(truncateSize), FilePerms, t.T()) + // Close the file and validate that the file is created on GCS. + CloseFileAndValidateContentFromGCS(testEnv.ctx, testEnv.storageClient, t.f1, testDirName, t.fileName, string(data[:]), t.T()) +} + +func (t *StreamingWritesSuite) TestTruncateNegative() { + err := t.f1.Truncate(-1) + + assert.Error(t.T(), err) +} + +func (t *StreamingWritesSuite) TestWriteAfterTruncate() { + truncateSize := 10 + + testCases := []struct { + name string + offset int64 + fileSize int64 + }{ + { + name: "ZeroOffset", + offset: 0, + fileSize: 10, + }, + { + name: "RandomOffset", + offset: 5, + fileSize: 10, + }, + { + name: "Append", + offset: 10, + fileSize: 12, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func() { + data := make([]byte, tc.fileSize) + // Perform truncate. + err := t.f1.Truncate(int64(truncateSize)) + require.NoError(t.T(), err) + operations.VerifyStatFile(t.filePath, int64(truncateSize), FilePerms, t.T()) + + // Triggers writes after truncate. + newData := []byte("hi") + _, err = t.f1.WriteAt(newData, tc.offset) + + require.NoError(t.T(), err) + // Verify that GCSFuse is returning correct file size before the file is uploaded. + operations.VerifyStatFile(t.filePath, tc.fileSize, FilePerms, t.T()) + data[tc.offset] = newData[0] + data[tc.offset+1] = newData[1] + // Close the file and validate that the file is created on GCS. + CloseFileAndValidateContentFromGCS(testEnv.ctx, testEnv.storageClient, t.f1, testDirName, t.fileName, string(data[:]), t.T()) + }) + } + +} + +func (t *StreamingWritesSuite) TestWriteAndTruncate() { + testCases := []struct { + name string + intitialContent string + truncateSize int64 + finalContent string + }{ + { + name: "WriteTruncateToUpper", + intitialContent: "foobar", + truncateSize: 9, + finalContent: "foobar\x00\x00\x00", + }, + { + name: "WriteTruncateToLower", + intitialContent: "foobar", + truncateSize: 3, + finalContent: "foo", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func() { + // Write + operations.WriteWithoutClose(t.f1, tc.intitialContent, t.T()) + operations.VerifyStatFile(t.filePath, int64(len(tc.intitialContent)), FilePerms, t.T()) + + // Perform truncate + err := t.f1.Truncate(tc.truncateSize) + + require.NoError(t.T(), err) + operations.VerifyStatFile(t.filePath, tc.truncateSize, FilePerms, t.T()) + // Close the file and validate that the file is created on GCS. + CloseFileAndValidateContentFromGCS(testEnv.ctx, testEnv.storageClient, t.f1, testDirName, t.fileName, tc.finalContent, t.T()) + }) + } +} + +func (t *StreamingWritesSuite) TestWriteTruncateWrite() { + testCases := []struct { + name string + truncateSize int64 + initialContent string + writeContent string + finalContent string + }{ + { + name: "WriteTruncateToUpperWrite", + truncateSize: 12, + initialContent: "foobar", + writeContent: "foo", + finalContent: "foobarfoo\x00\x00\x00", + }, + { + name: "WriteTruncateToLowerWrite", + truncateSize: 3, + initialContent: "foobar", + writeContent: "foo", + finalContent: "foo\x00\x00\x00foo", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func() { + // Write + operations.WriteWithoutClose(t.f1, tc.initialContent, t.T()) + operations.VerifyStatFile(t.filePath, int64(len(tc.initialContent)), FilePerms, t.T()) + // Perform truncate + // Note: truncate operation does not change the position of the file pointer of file handle. + err := t.f1.Truncate(tc.truncateSize) + require.NoError(t.T(), err) + operations.VerifyStatFile(t.filePath, tc.truncateSize, FilePerms, t.T()) + + // Write again + operations.WriteWithoutClose(t.f1, tc.writeContent, t.T()) + + operations.VerifyStatFile(t.filePath, int64(len(tc.finalContent)), FilePerms, t.T()) + // Close the file and validate that the file is created on GCS. + CloseFileAndValidateContentFromGCS(testEnv.ctx, testEnv.storageClient, t.f1, testDirName, t.fileName, tc.finalContent, t.T()) + }) + } +} + +func (t *StreamingWritesSuite) TestTruncateDownAndDeleteFile() { + // Write + operations.WriteWithoutClose(t.f1, "foobar", t.T()) + operations.VerifyStatFile(t.filePath, int64(len("foobar")), FilePerms, t.T()) + // Perform truncate + err := t.f1.Truncate(3) + require.NoError(t.T(), err) + operations.VerifyStatFile(t.filePath, 3, FilePerms, t.T()) + ValidateObjectContentsFromGCS(testEnv.ctx, testEnv.storageClient, testDirName, t.fileName, "foobar", t.T()) + + err = os.Remove(t.filePath) + + require.NoError(t.T(), err) + ValidateObjectNotFoundErrOnGCS(testEnv.ctx, testEnv.storageClient, testDirName, t.fileName, t.T()) +} diff --git a/tools/integration_tests/streaming_writes/write_file_test.go b/tools/integration_tests/streaming_writes/write_file_test.go new file mode 100644 index 0000000000..34c3a01050 --- /dev/null +++ b/tools/integration_tests/streaming_writes/write_file_test.go @@ -0,0 +1,49 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streaming_writes + +import ( + "os" + + . "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/stretchr/testify/require" +) + +func (t *StreamingWritesSuite) TestOutOfOrderWriteSyncsFileToGcs() { + // Write + operations.WriteWithoutClose(t.f1, "foobar", t.T()) + operations.VerifyStatFile(t.filePath, int64(len("foobar")), FilePerms, t.T()) + + // Perform out of order write. + operations.WriteAt("foo", 3, t.f1, t.T()) + + ValidateObjectContentsFromGCS(testEnv.ctx, testEnv.storageClient, testDirName, t.fileName, "foobar", t.T()) + CloseFileAndValidateContentFromGCS(testEnv.ctx, testEnv.storageClient, t.f1, testDirName, t.fileName, "foofoo", t.T()) +} + +func (t *StreamingWritesSuite) TestOutOfOrderWriteSyncsFileToGcsAndDeletingFileDeletesFileFromGcs() { + // Write + operations.WriteWithoutClose(t.f1, "foobar", t.T()) + operations.VerifyStatFile(t.filePath, int64(len("foobar")), FilePerms, t.T()) + // Perform out of order write. + operations.WriteAt("foo", 3, t.f1, t.T()) + ValidateObjectContentsFromGCS(testEnv.ctx, testEnv.storageClient, testDirName, t.fileName, "foobar", t.T()) + + err := os.Remove(t.filePath) + + require.NoError(t.T(), err) + ValidateObjectNotFoundErrOnGCS(testEnv.ctx, testEnv.storageClient, testDirName, t.fileName, t.T()) +} diff --git a/tools/integration_tests/symlink_handling/symlink_handling_test.go b/tools/integration_tests/symlink_handling/symlink_handling_test.go new file mode 100644 index 0000000000..26887dc8a9 --- /dev/null +++ b/tools/integration_tests/symlink_handling/symlink_handling_test.go @@ -0,0 +1,104 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package symlink_handling + +import ( + "context" + "log" + "os" + "path" + "testing" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" +) + +const ( + TestDirName = "SymlinkHandlingTest" + SymlinkMetadataKey = "gcsfuse_symlink_target" + StandardSymlinkMetadataKey = "goog-reserved-file-is-symlink" +) + +var ( + testEnv env +) + +type env struct { + storageClient *storage.Client + ctx context.Context + cfg *test_suite.TestConfig +} + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.SymlinkHandling) == 0 { + log.Println("No configuration found for symlink handling tests in config. Using flags instead.") + // Populate the config manually. + cfg.SymlinkHandling = make([]test_suite.TestConfig, 1) + cfg.SymlinkHandling[0].TestBucket = setup.TestBucket() + cfg.SymlinkHandling[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.SymlinkHandling[0].LogFile = setup.LogFile() + cfg.SymlinkHandling[0].Configs = make([]test_suite.ConfigItem, 2) + + // 1. TestStandardSymlinksTestSuite + cfg.SymlinkHandling[0].Configs[0].Flags = []string{"--enable-standard-symlinks=true"} + cfg.SymlinkHandling[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.SymlinkHandling[0].Configs[0].Run = "TestStandardSymlinksTestSuite" + + // 2. TestLegacySymlinksTestSuite + cfg.SymlinkHandling[0].Configs[1].Flags = []string{"--enable-standard-symlinks=false"} + cfg.SymlinkHandling[0].Configs[1].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} + cfg.SymlinkHandling[0].Configs[1].Run = "TestLegacySymlinksTestSuite" + } + + testEnv.ctx = context.Background() + testEnv.cfg = &cfg.SymlinkHandling[0] + setup.TestEnvironment(testEnv.ctx, testEnv.cfg) + + // 2. Create storage client before running tests. + var err error + testEnv.storageClient, err = client.CreateStorageClient(testEnv.ctx) + if err != nil { + log.Printf("Error creating storage client: %v\n", err) + os.Exit(1) + } + defer func() { + if err := testEnv.storageClient.Close(); err != nil { + log.Printf("Error closing storage client: %v\n", err) + } + }() + + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + testEnv.cfg.GCSFuseMountedDirectory = testEnv.cfg.GKEMountedDirectory + os.Exit(m.Run()) + } + + // For GCE environment + setup.SetUpTestDirForTestBucket(testEnv.cfg) + // Override GKE specific paths with GCSFuse paths if running in GCE environment. + setup.OverrideFilePathsInFlagSet(testEnv.cfg, setup.TestDir()) + + log.Println("Running static mounting tests for symlink handling...") + successCode := m.Run() + + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(setup.TestBucket(), TestDirName)) + os.Exit(successCode) +} diff --git a/tools/integration_tests/symlink_handling/symlink_operations_test.go b/tools/integration_tests/symlink_handling/symlink_operations_test.go new file mode 100644 index 0000000000..57063085e0 --- /dev/null +++ b/tools/integration_tests/symlink_handling/symlink_operations_test.go @@ -0,0 +1,183 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package symlink_handling + +import ( + "os" + "os/exec" + "path" +) + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +// TestCreateSymlink tests the creation of symlinks. +func (s *BaseSymlinkSuite) TestCreateSymlink() { + // Create the symlink + _ = s.createSymlink(s.linkName, s.targetPath) + + // Validate the underlying GCS Object + s.validateBackingGCSObjectForSymlink(s.linkName, s.targetPath, s.isStandardSymlink) +} + +// TestReadSymlinkTest tests reading a symlink's target. +func (s *BaseSymlinkSuite) TestReadSymlinkTest() { + s.createGCSSymlinkObject(s.linkName, s.targetPath) + linkPath := path.Join(s.testDirPath, s.linkName) + + result, err := os.Readlink(linkPath) + + s.Require().NoError(err) + s.Assert().Equal(s.targetPath, result) +} + +// TestReadFileViaSymlink tests reading a file through a symlink. +func (s *BaseSymlinkSuite) TestReadFileViaSymlink() { + const content = "hello world" + // Create a target file with content. + err := os.WriteFile(s.targetPath, []byte(content), 0644) + s.Require().NoError(err) + // Create a symlink to the target file. + s.createGCSSymlinkObject(s.linkName, s.targetPath) + linkPath := path.Join(s.testDirPath, s.linkName) + + // Read file via symlink. + readContent, err := os.ReadFile(linkPath) + s.Require().NoError(err) + + // Verify content. + s.Assert().Equal(content, string(readContent)) +} + +// TestWriteFileViaSymlink tests writing to a file through a symlink. +func (s *BaseSymlinkSuite) TestWriteFileViaSymlink() { + const content = "new content" + // Create an empty target file. + f, err := os.Create(s.targetPath) + s.Require().NoError(err) + s.Require().NoError(f.Close()) + // Create a symlink to the target file. + s.createGCSSymlinkObject(s.linkName, s.targetPath) + linkPath := path.Join(s.testDirPath, s.linkName) + + // Write to file via symlink. + err = os.WriteFile(linkPath, []byte(content), 0644) + s.Require().NoError(err) + + // Verify content of the original file. + readContent, err := os.ReadFile(s.targetPath) + s.Require().NoError(err) + s.Assert().Equal(content, string(readContent)) +} + +// TestListDirViaSymlink tests listing a directory through a symlink. +func (s *BaseSymlinkSuite) TestListDirViaSymlink() { + fileName := "file_in_dir.txt" + // Create a target directory with a file. + err := os.Mkdir(s.targetPath, 0755) + s.Require().NoError(err) + filePath := path.Join(s.targetPath, fileName) + err = os.WriteFile(filePath, []byte("content"), 0644) + s.Require().NoError(err) + // Create a symlink to the target directory. + s.createGCSSymlinkObject(s.linkName, s.targetPath) + linkPath := path.Join(s.testDirPath, s.linkName) + + // List directory via symlink. + entries, err := os.ReadDir(linkPath) + + s.Require().NoError(err) + // Verify contents. + s.Assert().Len(entries, 1) + s.Assert().Equal(fileName, entries[0].Name()) +} + +// TestRenameSymlink tests renaming a symlink. +func (s *BaseSymlinkSuite) TestRenameSymlink() { + newLinkName := s.linkName + "_renamed" + // Create a target file. + err := os.WriteFile(s.targetPath, []byte("content"), 0644) + s.Require().NoError(err) + // Create a symlink to the target file. + s.createGCSSymlinkObject(s.linkName, s.targetPath) + linkPath := path.Join(s.testDirPath, s.linkName) + + newLinkPath := path.Join(s.testDirPath, newLinkName) + + // Rename the symlink. + err = os.Rename(linkPath, newLinkPath) + s.Require().NoError(err) + + // Verify old link is gone. + _, err = os.Lstat(linkPath) + s.Assert().True(os.IsNotExist(err)) + // Verify new link exists, is a symlink, and points to the correct target. + fi, err := os.Lstat(newLinkPath) + s.Require().NoError(err) + s.Assert().True(fi.Mode()&os.ModeSymlink != 0) + readTargetName, err := os.Readlink(newLinkPath) + s.Require().NoError(err) + s.Assert().Equal(s.targetPath, readTargetName) + // Verify target file is untouched. + _, err = os.Stat(s.targetPath) + s.Assert().NoError(err) +} + +// TestCopySymlink tests copying a symlink without dereferencing. +func (s *BaseSymlinkSuite) TestCopySymlink() { + newLinkName := s.linkName + "_copied" + // Create a target file. + err := os.WriteFile(s.targetPath, []byte("content"), 0644) + s.Require().NoError(err) + // Create a symlink to the target file. + s.createGCSSymlinkObject(s.linkName, s.targetPath) + linkPath := path.Join(s.testDirPath, s.linkName) + newLinkPath := path.Join(s.testDirPath, newLinkName) + + // Copy the symlink using cp -P to ensure no dereferencing. + cmd := exec.Command("cp", "-P", linkPath, newLinkPath) + err = cmd.Run() + s.Require().NoError(err) + + // Verify old link still exists. + _, err = os.Lstat(linkPath) + s.Assert().NoError(err) + // Verify new link exists, is a symlink, and points to the correct target. + fi, err := os.Lstat(newLinkPath) + s.Require().NoError(err) + s.Assert().True(fi.Mode()&os.ModeSymlink != 0) + readTargetName, err := os.Readlink(newLinkPath) + s.Require().NoError(err) + s.Assert().Equal(s.targetPath, readTargetName) + // Verify target file is untouched. + _, err = os.Stat(s.targetPath) + s.Assert().NoError(err) +} + +// TestReadStandardSymlinkInLegacyMode tests that a legacy mount can read a standard symlink. +func (s *LegacySymlinksTestSuite) TestReadStandardSymlinkInLegacyMode() { + // Temporarily enable standard symlink creation to create a standard symlink object. + s.isStandardSymlink = true + defer func() { s.isStandardSymlink = false }() + s.createGCSSymlinkObject(s.linkName, s.targetPath) + linkPath := path.Join(s.testDirPath, s.linkName) + + // Read the symlink via the legacy mount. + result, err := os.Readlink(linkPath) + + s.Require().NoError(err) + s.Assert().Equal(s.targetPath, result) +} diff --git a/tools/integration_tests/symlink_handling/symlink_suites_test.go b/tools/integration_tests/symlink_handling/symlink_suites_test.go new file mode 100644 index 0000000000..406c73b267 --- /dev/null +++ b/tools/integration_tests/symlink_handling/symlink_suites_test.go @@ -0,0 +1,185 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package symlink_handling + +import ( + "io" + "os" + "path" + "strings" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/suite" +) + +// BaseSymlinkSuite provides the common structure and configuration-driven setup logic. +type BaseSymlinkSuite struct { + suite.Suite + flags []string + mntDir string + testDirPath string + isStandardSymlink bool + // linkName is the name of the symlink (relative to testDirPath). + linkName string + // targetPath is the absolute path of the target file/dir . + targetPath string +} + +// StandardSymlinksTestSuite groups all test related to symlinks following standard representation. +type StandardSymlinksTestSuite struct{ BaseSymlinkSuite } + +// StandardSymlinksTestSuite groups all test related to symlinks following legacy representation. +type LegacySymlinksTestSuite struct{ BaseSymlinkSuite } + +//////////////////////////////////////////////////////////////////////// +// Common Suite Logic +//////////////////////////////////////////////////////////////////////// + +func (s *BaseSymlinkSuite) SetupTest() { + if testEnv.cfg.GKEMountedDirectory != "" { + s.mntDir = testEnv.cfg.GKEMountedDirectory + s.testDirPath = path.Join(s.mntDir, TestDirName) + } else { + s.mntDir = testEnv.cfg.GCSFuseMountedDirectory + setup.SetMntDir(s.mntDir) + err := static_mounting.MountGcsfuseWithStaticMountingWithConfigFile(testEnv.cfg, s.flags) + s.Require().NoError(err) + s.testDirPath = setup.SetupTestDirectory(TestDirName) + } + // Initialize common variables for symlink tests, ensuring they are unique for each test method. + s.linkName = setup.GenerateRandomString(5) + "_link" + s.targetPath = path.Join(s.testDirPath, setup.GenerateRandomString(5)) +} + +func (s *BaseSymlinkSuite) TearDownTest() { + if testEnv.cfg.GKEMountedDirectory == "" { + setup.UnmountGCSFuse(s.mntDir) + } + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) + setup.CleanupDirectoryOnGCS(testEnv.ctx, testEnv.storageClient, path.Join(setup.TestBucket(), TestDirName)) +} + +//////////////////////////////////////////////////////////////////////// +// Helpers +//////////////////////////////////////////////////////////////////////// + +func (s *BaseSymlinkSuite) createSymlink(linkName, target string) string { + linkPath := path.Join(s.testDirPath, linkName) + err := os.Symlink(target, linkPath) + s.Require().NoError(err) + return linkPath +} + +func (s *BaseSymlinkSuite) createTempFile() string { + targetFile, err := os.CreateTemp("", "symlink-target") + s.Require().NoError(err) + s.T().Cleanup(func() { + if err := os.Remove(targetFile.Name()); err != nil { + s.T().Logf("Error removing temporary file %s: %v", targetFile.Name(), err) + } + }) + s.Require().NoError(targetFile.Close()) + return targetFile.Name() +} + +// validateBackingGCSObjectForSymlink validates the GCS object created for a symlink. +func (s *BaseSymlinkSuite) validateBackingGCSObjectForSymlink(linkName, target string, isStandardSymlink bool) { + bucketName, objectName := setup.GetBucketAndObjectBasedOnTypeOfMount(path.Join(TestDirName, linkName)) + objHandle := testEnv.storageClient.Bucket(bucketName).Object(objectName) + attrs, err := objHandle.Attrs(testEnv.ctx) + s.Require().NoError(err) + + if isStandardSymlink { + // Validate the GCS Object content to be the symlink target + rc, err := objHandle.NewReader(testEnv.ctx) + s.Require().NoError(err) + defer func() { + s.Assert().NoError(rc.Close()) + }() + content, err := io.ReadAll(rc) + s.Require().NoError(err) + s.Assert().Equal(target, string(content), "Standard symlink content should match target") + s.Assert().Equal(int64(len(target)), attrs.Size, "Standard symlink size should match target length") + _, ok := attrs.Metadata[SymlinkMetadataKey] + s.Assert().True(ok) + val, ok := attrs.Metadata[StandardSymlinkMetadataKey] + s.Assert().True(ok) + s.Assert().Equal("true", val) + } else { + // Legacy symlink + // Validate the GCS Object content to be nil + s.Assert().Equal(int64(0), attrs.Size, "Legacy symlink size should be 0") + val, ok := attrs.Metadata[SymlinkMetadataKey] + s.Assert().True(ok, "Legacy symlink should have old metadata key (%s)", SymlinkMetadataKey) + s.Assert().Equal(target, val, "Legacy symlink metadata value should match target") + _, ok = attrs.Metadata[StandardSymlinkMetadataKey] + s.Assert().False(ok, "Legacy symlink should not have new metadata key (%s)", StandardSymlinkMetadataKey) + } +} + +// createGCSSymlinkObject creates a symlink object on GCS with appropriate metadata. +// The 'target' parameter is the symlink target path. +func (s *BaseSymlinkSuite) createGCSSymlinkObject(linkName, target string) { + fullLinkPath := path.Join(TestDirName, linkName) + bucketName, objectName := setup.GetBucketAndObjectBasedOnTypeOfMount(fullLinkPath) + objHandle := testEnv.storageClient.Bucket(bucketName).Object(objectName) + w, err := client.NewWriter(testEnv.ctx, objHandle, testEnv.storageClient) + s.Require().NoError(err) + + var content []byte + if s.isStandardSymlink { + w.Metadata = map[string]string{StandardSymlinkMetadataKey: "true", SymlinkMetadataKey: target} + content = []byte(target) // Standard symlinks store target in content + } else { + w.Metadata = map[string]string{SymlinkMetadataKey: target} + content = []byte("") // Legacy symlinks have empty content + } + + _, err = w.Write(content) + s.Require().NoError(err) + s.Require().NoError(w.Close()) + operations.WaitForSizeUpdate(setup.IsZonalBucketRun(), operations.WaitDurationAfterCloseZB) +} + +//////////////////////////////////////////////////////////////////////// +// Test Runner +//////////////////////////////////////////////////////////////////////// + +func TestStandardSymlinks(t *testing.T) { + RunTests(t, "TestStandardSymlinksTestSuite", func(flags []string) suite.TestingSuite { + return &StandardSymlinksTestSuite{BaseSymlinkSuite{flags: flags, isStandardSymlink: true}} + }) +} + +func TestLegacySymlinks(t *testing.T) { + RunTests(t, "TestLegacySymlinksTestSuite", func(flags []string) suite.TestingSuite { + return &LegacySymlinksTestSuite{BaseSymlinkSuite{flags: flags, isStandardSymlink: false}} + }) +} + +func RunTests(t *testing.T, runName string, factory func(flags []string) suite.TestingSuite) { + for _, cfg := range testEnv.cfg.Configs { + if cfg.Run == runName { + for _, flagStr := range cfg.Flags { + flags := strings.Fields(flagStr) + suite.Run(t, factory(flags)) + } + } + } +} diff --git a/tools/integration_tests/test_config.yaml b/tools/integration_tests/test_config.yaml new file mode 100644 index 0000000000..f02bd846a2 --- /dev/null +++ b/tools/integration_tests/test_config.yaml @@ -0,0 +1,1177 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +explicit_dir: + - mounted_directory: "${MOUNTED_DIR}" # To be passed by GKE after mounting + test_bucket: "${BUCKET_NAME}" # To be passed by both gcsfuse and gke tests + configs: + - flags: + - "--implicit-dirs=false" + - "--implicit-dirs=false,--client-protocol=grpc" + compatible: # Bucket type to run these tests with + flat: true + hns: false + zonal: false + run_on_gke: true + +implicit_dir: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + - "--implicit-dirs" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: true + - flags: + - "--implicit-dirs,--client-protocol=grpc" + compatible: + flat: true + hns: true + zonal: false + run_on_gke: true + +list_large_dir: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + - "--implicit-dirs,--stat-cache-ttl=0,--kernel-list-cache-ttl-secs=-1" + - "--client-protocol=grpc,--implicit-dirs,--stat-cache-ttl=0,--kernel-list-cache-ttl-secs=-1" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: true + run: TestListLargeDirWithKernelListCache + - flags: + - "--enable-metadata-prefetch,--implicit-dirs" + - "--client-protocol=grpc,--enable-metadata-prefetch,--implicit-dirs" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: true + run: TestListLargeDirWithoutKernelListCache +operations: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + only_dir: "${ONLY_DIR}" + configs: + - flags: + - "--metadata-cache-ttl-secs=0,--enable-streaming-writes=false" + - "--kernel-list-cache-ttl-secs=-1,--implicit-dirs,--enable-metadata-prefetch" + - "--experimental-enable-json-read,--enable-atomic-rename-object" + - "--client-protocol=grpc,--implicit-dirs,--enable-atomic-rename-object,--enable-metadata-prefetch" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: true + +write_large_files: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + - "--enable-streaming-writes=false" + # write-global-max-blocks=5 is for checking multiple file writes in parallel. + # concurrent_write_files_test.go- we are writing 3 files in parallel. + # with this config, we are giving 2 blocks to 2 files and 1 block to other file. + - "--write-max-blocks-per-file=2,--write-global-max-blocks=5" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: true + +gzip: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + - "--sequential-read-size-mb=1,--implicit-dirs" + - "--sequential-read-size-mb=1,--implicit-dirs,--client-protocol=grpc" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: true + +read_large_files: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + - "--implicit-dirs" + - "--implicit-dirs,--client-protocol=grpc" + - "--implicit-dirs,--file-cache-max-size-mb=700,--file-cache-cache-file-for-range-read,--cache-dir=/gcsfuse-tmp/read_large_files" + - "--implicit-dirs,--file-cache-max-size-mb=700,--file-cache-cache-file-for-range-read,--client-protocol=grpc,--cache-dir=/gcsfuse-tmp/read_large_files" + - "--implicit-dirs,--file-cache-max-size-mb=-1,--cache-dir=/gcsfuse-tmp/read_large_files" + - "--implicit-dirs,--file-cache-max-size-mb=-1,--client-protocol=grpc,--cache-dir=/gcsfuse-tmp/read_large_files" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: true + - flags: + - "--implicit-dirs --enable-kernel-reader=false" + compatible: + flat: false + hns: false + zonal: true + run_on_gke: true + +readonly: + - mounted_directory: "${MOUNTED_DIR}" # To be passed by GKE after mounting + test_bucket: "${BUCKET_NAME}" # To be passed by both gcsfuse and gke tests + configs: + - flags: + - "--o=ro,--implicit-dirs" + - "--file-mode=544,--dir-mode=544,--implicit-dirs" + - "--client-protocol=grpc,--o=ro,--implicit-dirs" + - "--o=ro,--implicit-dirs,--cache-dir=/gcsfuse-tmp/readonly,--file-cache-max-size-mb=3" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: true + +rename_dir_limit: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + only_dir: "${ONLY_DIR}" + configs: + - flags: + - "--rename-dir-limit=3,--implicit-dirs,--client-protocol=grpc" + - "--rename-dir-limit=3" + - "--rename-dir-limit=3,--client-protocol=grpc" + compatible: + flat: true + hns: false + zonal: false + run_on_gke: true + - flags: + - "" + compatible: + flat: false + hns: true + zonal: true + run_on_gke: true + +local_file: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + only_dir: "${ONLY_DIR}" + configs: + - flags: + - "--implicit-dirs,--rename-dir-limit=3,--enable-streaming-writes=false" + - "--implicit-dirs=false,--rename-dir-limit=3,--enable-streaming-writes=false,--client-protocol=grpc" + - "--rename-dir-limit=3,--write-block-size-mb=1,--write-max-blocks-per-file=2,--write-global-max-blocks=0" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: true + - flags: + - "--rename-dir-limit=3,--write-block-size-mb=1,--write-max-blocks-per-file=2,--write-global-max-blocks=-1" + - "--rename-dir-limit=3,--write-block-size-mb=1,--write-max-blocks-per-file=2,--write-global-max-blocks=-1,--client-protocol=grpc" + compatible: + flat: true + hns: true + zonal: false + run_on_gke: true + +streaming_writes: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + - "--rename-dir-limit=3,--write-block-size-mb=1,--write-max-blocks-per-file=2,--client-protocol=grpc,--write-global-max-blocks=-1" + - "--rename-dir-limit=3,--write-block-size-mb=1,--write-max-blocks-per-file=2,--write-global-max-blocks=-1" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: true + +cloud_profiler: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + # Set the 'PROFILE_LABEL' and 'PROFILE_SERVICE_NAME' environment variables for GKE + - "--enable-cloud-profiler,--cloud-profiler-cpu,--cloud-profiler-heap,--cloud-profiler-goroutines,--cloud-profiler-mutex,--cloud-profiler-allocated-heap,--cloud-profiler-label=${PROFILE_LABEL},--cloud-profiler-service-name=${PROFILE_SERVICE_NAME}" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: true + +requester_pays_bucket: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + only_dir: "${ONLY_DIR}" + configs: + - flags: + # Set the 'BILLING_PROJECT' environment variable for GKE + - "--billing-project=${BILLING_PROJECT},--key-file=${KEY_FILE}" + - "--billing-project=${BILLING_PROJECT},--client-protocol=grpc,--key-file=${KEY_FILE}" + - "--billing-project=${BILLING_PROJECT},--client-protocol=grpc,--grpc-path-strategy=direct-path-only,--key-file=${KEY_FILE}" + compatible: + flat: true + hns: true + zonal: false + run_on_gke: true + +read_cache: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + only_dir: "${ONLY_DIR}" + configs: + - flags: + - "--metadata-cache-ttl-secs=10,--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestSmallCacheTTLTest,--log-file=/gcsfuse-tmp/TestSmallCacheTTLTest.log,--log-severity=TRACE,--implicit-dirs,--enable-kernel-reader=false" + - "--metadata-cache-ttl-secs=10,--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestSmallCacheTTLTest,--log-file=/gcsfuse-tmp/TestSmallCacheTTLTest.log,--log-severity=TRACE,--enable-kernel-reader=false" + - "--metadata-cache-ttl-secs=10,--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestSmallCacheTTLTest,--log-file=/gcsfuse-tmp/TestSmallCacheTTLTest.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false" + - "--metadata-cache-ttl-secs=10,--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestSmallCacheTTLTest,--log-file=/gcsfuse-tmp/TestSmallCacheTTLTest.log,--log-severity=TRACE,--client-protocol=grpc,--implicit-dirs,--enable-kernel-reader=false" + compatible: + flat: true + hns: true + zonal: true + run: TestSmallCacheTTLTest + run_on_gke: true + - flags: + - "--file-cache-max-size-mb=9,--file-cache-cache-file-for-range-read,--cache-dir=/gcsfuse-tmp/TestReadOnlyTest,--log-file=/gcsfuse-tmp/TestReadOnlyTest.log,--log-severity=TRACE,--file-cache-enable-parallel-downloads=false,--implicit-dirs,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestReadOnlyTest,--log-file=/gcsfuse-tmp/TestReadOnlyTest.log,--log-severity=TRACE,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=9,--file-cache-cache-file-for-range-read,--cache-dir=/gcsfuse-tmp/TestReadOnlyTest,--log-file=/gcsfuse-tmp/TestReadOnlyTest.log,--log-severity=TRACE,--file-cache-enable-parallel-downloads=false,--implicit-dirs,--o=ro,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestReadOnlyTest,--log-file=/gcsfuse-tmp/TestReadOnlyTest.log,--log-severity=TRACE,--o=ro,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=9,--file-cache-cache-file-for-range-read,--cache-dir=/gcsfuse-tmp/TestReadOnlyTest,--log-file=/gcsfuse-tmp/TestReadOnlyTest.log,--log-severity=TRACE,--file-cache-enable-parallel-downloads=false,--implicit-dirs,--client-protocol=grpc,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestReadOnlyTest,--log-file=/gcsfuse-tmp/TestReadOnlyTest.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=9,--file-cache-cache-file-for-range-read,--cache-dir=/gcsfuse-tmp/TestReadOnlyTest,--log-file=/gcsfuse-tmp/TestReadOnlyTest.log,--log-severity=TRACE,--file-cache-enable-parallel-downloads=false,--implicit-dirs,--o=ro,--client-protocol=grpc,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestReadOnlyTest,--log-file=/gcsfuse-tmp/TestReadOnlyTest.log,--log-severity=TRACE,--o=ro,--client-protocol=grpc,--enable-kernel-reader=false" + compatible: + flat: true + hns: true + zonal: true + run: TestReadOnlyTest + run_on_gke: true + - flags: + - "--implicit-dirs,--file-cache-max-size-mb=15,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestRangeReadTest,--log-file=/gcsfuse-tmp/TestRangeReadTest.log,--log-severity=TRACE,--enable-kernel-reader=false" + - "--implicit-dirs,--file-cache-max-size-mb=15,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestRangeReadTest,--log-file=/gcsfuse-tmp/TestRangeReadTest.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false" + compatible: + flat: true + hns: true + zonal: true + run: TestRangeReadTest + run_on_gke: true + - flags: + - "--implicit-dirs,--file-cache-max-size-mb=15,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestRangeReadWithParallelDownloadsTest,--log-file=/gcsfuse-tmp/TestRangeReadWithParallelDownloadsTest.log,--log-severity=TRACE,--enable-kernel-reader=false" + - "--implicit-dirs,--file-cache-max-size-mb=15,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestRangeReadWithParallelDownloadsTest,--log-file=/gcsfuse-tmp/TestRangeReadWithParallelDownloadsTest.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false" + compatible: + flat: true + hns: true + zonal: true + run: TestRangeReadWithParallelDownloadsTest + run_on_gke: true + - flags: + - "--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestLocalModificationTest,--log-file=/gcsfuse-tmp/TestLocalModificationTest.log,--log-severity=TRACE,--implicit-dirs,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestLocalModificationTest,--log-file=/gcsfuse-tmp/TestLocalModificationTest.log,--log-severity=TRACE,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestLocalModificationTest,--log-file=/gcsfuse-tmp/TestLocalModificationTest.log,--log-severity=TRACE,--implicit-dirs,--client-protocol=grpc,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestLocalModificationTest,--log-file=/gcsfuse-tmp/TestLocalModificationTest.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false" + compatible: + flat: true + hns: true + zonal: true + run: TestLocalModificationTest + run_on_gke: true + - flags: + - "--stat-cache-ttl=0s,--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestDisabledCacheTTLTest,--log-file=/gcsfuse-tmp/TestDisabledCacheTTLTest.log,--log-severity=TRACE,--implicit-dirs,--enable-kernel-reader=false" + - "--stat-cache-ttl=0s,--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestDisabledCacheTTLTest,--log-file=/gcsfuse-tmp/TestDisabledCacheTTLTest.log,--log-severity=TRACE,--enable-kernel-reader=false" + - "--stat-cache-ttl=0s,--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestDisabledCacheTTLTest,--log-file=/gcsfuse-tmp/TestDisabledCacheTTLTest.log,--log-severity=TRACE,--implicit-dirs,--client-protocol=grpc,--enable-kernel-reader=false" + - "--stat-cache-ttl=0s,--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestDisabledCacheTTLTest,--log-file=/gcsfuse-tmp/TestDisabledCacheTTLTest.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false" + compatible: + flat: true + hns: true + zonal: true + run: TestDisabledCacheTTLTest + run_on_gke: true + - flags: + - "--file-cache-max-size-mb=50,--file-cache-cache-file-for-range-read,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest.log,--log-severity=TRACE,--implicit-dirs,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=50,--file-cache-cache-file-for-range-read,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest.log,--log-severity=TRACE,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=50,--file-cache-cache-file-for-range-read,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest.log,--log-severity=TRACE,--file-cache-enable-o-direct,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=50,--file-cache-cache-file-for-range-read,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest.log,--log-severity=TRACE,--implicit-dirs,--client-protocol=grpc,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=50,--file-cache-cache-file-for-range-read,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=50,--file-cache-cache-file-for-range-read,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueTest.log,--log-severity=TRACE,--file-cache-enable-o-direct,--client-protocol=grpc,--enable-kernel-reader=false" + compatible: + flat: true + hns: true + zonal: true + run: TestCacheFileForRangeReadTrueTest + run_on_gke: true + # TODO: Enable Ram cache tests after bug b/383682524 is fixed + # - flags: + # - "--file-cache-max-size-mb=50,--file-cache-cache-file-for-range-read,--file-cache-enable-parallel-downloads,--cache-dir=/dev/shm/TestCacheFileForRangeReadTrueWithRamCache,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueWithRamCache.log,--log-severity=TRACE" + # - "--file-cache-max-size-mb=50,--file-cache-cache-file-for-range-read,--file-cache-enable-parallel-downloads,--cache-dir=/dev/shm/TestCacheFileForRangeReadTrueWithRamCache,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueWithRamCache.log,--log-severity=TRACE,--file-cache-enable-o-direct" + # - "--file-cache-max-size-mb=50,--file-cache-cache-file-for-range-read,--file-cache-enable-parallel-downloads=false,--cache-dir=/dev/shm/TestCacheFileForRangeReadTrueWithRamCache,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueWithRamCache.log,--log-severity=TRACE" + # - "--file-cache-max-size-mb=50,--file-cache-cache-file-for-range-read,--file-cache-enable-parallel-downloads,--cache-dir=/dev/shm/TestCacheFileForRangeReadTrueWithRamCache,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueWithRamCache.log,--log-severity=TRACE,--client-protocol=grpc" + # - "--file-cache-max-size-mb=50,--file-cache-cache-file-for-range-read,--file-cache-enable-parallel-downloads,--cache-dir=/dev/shm/TestCacheFileForRangeReadTrueWithRamCache,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueWithRamCache.log,--log-severity=TRACE,--file-cache-enable-o-direct,--client-protocol=grpc" + # - "--file-cache-max-size-mb=50,--file-cache-cache-file-for-range-read,--file-cache-enable-parallel-downloads=false,--cache-dir=/dev/shm/TestCacheFileForRangeReadTrueWithRamCache,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadTrueWithRamCache.log,--log-severity=TRACE,--client-protocol=grpc" + # compatible: + # flat: true + # hns: true + # zonal: true + # run: TestCacheFileForRangeReadTrueWithRamCache + # run_on_gke: true + - flags: + - "--implicit-dirs,--file-cache-max-size-mb=50,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadFalseTest,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseTest.log,--log-severity=TRACE,--enable-kernel-reader=false" + - "--implicit-dirs,--file-cache-max-size-mb=50,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadFalseTest,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseTest.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false" + compatible: + flat: true + hns: true + zonal: true + run: TestCacheFileForRangeReadFalseTest + run_on_gke: true + # - flags: + # - "--file-cache-max-size-mb=50,--file-cache-enable-parallel-downloads=false,--cache-dir=/dev/shm/TestCacheFileForRangeReadFalseWithRamCache,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithRamCache.log,--log-severity=TRACE" + # - "--file-cache-max-size-mb=50,--file-cache-enable-parallel-downloads=false,--cache-dir=/dev/shm/TestCacheFileForRangeReadFalseWithRamCache,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithRamCache.log,--log-severity=TRACE,--client-protocol=grpc" + # compatible: + # flat: true + # hns: true + # zonal: true + # run: TestCacheFileForRangeReadFalseWithRamCache + # run_on_gke: true + - flags: + - "--file-cache-max-size-mb=50,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloads,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloads.log,--log-severity=TRACE,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=50,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloads,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloads.log,--log-severity=TRACE,--file-cache-enable-o-direct,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=50,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloads,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloads.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=50,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloads,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloads.log,--log-severity=TRACE,--file-cache-enable-o-direct,--client-protocol=grpc,--enable-kernel-reader=false" + compatible: + flat: true + hns: true + zonal: true + run: TestCacheFileForRangeReadFalseWithParallelDownloads + run_on_gke: true + # - flags: + # - "--file-cache-max-size-mb=50,--file-cache-enable-parallel-downloads,--cache-dir=/dev/shm/TestCacheFileForRangeReadFalseWithParallelDownloadsAndRamCache,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloadsAndRamCache.log,--log-severity=TRACE" + # - "--file-cache-max-size-mb=50,--file-cache-enable-parallel-downloads,--cache-dir=/dev/shm/TestCacheFileForRangeReadFalseWithParallelDownloadsAndRamCache,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloadsAndRamCache.log,--log-severity=TRACE,--file-cache-enable-o-direct" + # - "--file-cache-max-size-mb=50,--file-cache-enable-parallel-downloads,--cache-dir=/dev/shm/TestCacheFileForRangeReadFalseWithParallelDownloadsAndRamCache,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloadsAndRamCache.log,--log-severity=TRACE,--client-protocol=grpc" + # - "--file-cache-max-size-mb=50,--file-cache-enable-parallel-downloads,--cache-dir=/dev/shm/TestCacheFileForRangeReadFalseWithParallelDownloadsAndRamCache,--log-file=/gcsfuse-tmp/TestCacheFileForRangeReadFalseWithParallelDownloadsAndRamCache.log,--log-severity=TRACE,--file-cache-enable-o-direct,--client-protocol=grpc" + # compatible: + # flat: true + # hns: true + # zonal: true + # run: TestCacheFileForRangeReadFalseWithParallelDownloadsAndRamCache + # run_on_gke: true + - flags: + - "--file-cache-max-size-mb=48,--file-cache-cache-file-for-range-read,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestJobChunkTest,--log-file=/gcsfuse-tmp/TestJobChunkTest.log,--log-severity=TRACE,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=48,--file-cache-cache-file-for-range-read,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestJobChunkTest,--log-file=/gcsfuse-tmp/TestJobChunkTest.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false" + compatible: + flat: true + hns: true + zonal: true + run: TestJobChunkTest + run_on_gke: true + - flags: + # with unlimited max parallel downloads. + - "--file-cache-max-size-mb=48,--file-cache-enable-parallel-downloads,--file-cache-parallel-downloads-per-file=4,--file-cache-max-parallel-downloads=-1,--file-cache-download-chunk-size-mb=4,--file-cache-enable-crc,--cache-dir=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads,--log-file=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads.log,--log-severity=TRACE,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=48,--file-cache-enable-parallel-downloads,--file-cache-parallel-downloads-per-file=4,--file-cache-max-parallel-downloads=-1,--file-cache-download-chunk-size-mb=4,--file-cache-enable-crc,--cache-dir=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads,--log-file=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false" + # with go-routines not limited by max parallel downloads. + # maxParallelDownloads > parallelDownloadsPerFile * number of files being accessed concurrently. + - "--file-cache-max-size-mb=48,--file-cache-enable-parallel-downloads,--file-cache-parallel-downloads-per-file=4,--file-cache-max-parallel-downloads=9,--file-cache-download-chunk-size-mb=4,--file-cache-enable-crc,--cache-dir=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads,--log-file=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads.log,--log-severity=TRACE,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=48,--file-cache-enable-parallel-downloads,--file-cache-parallel-downloads-per-file=4,--file-cache-max-parallel-downloads=9,--file-cache-download-chunk-size-mb=4,--file-cache-enable-crc,--cache-dir=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads,--log-file=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false" + # with go-routines limited by max parallel downloads. + - "--file-cache-max-size-mb=48,--file-cache-enable-parallel-downloads,--file-cache-parallel-downloads-per-file=4,--file-cache-max-parallel-downloads=2,--file-cache-download-chunk-size-mb=4,--file-cache-enable-crc,--cache-dir=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads,--log-file=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads.log,--log-severity=TRACE,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=48,--file-cache-enable-parallel-downloads,--file-cache-parallel-downloads-per-file=4,--file-cache-max-parallel-downloads=2,--file-cache-download-chunk-size-mb=4,--file-cache-enable-crc,--cache-dir=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads,--log-file=/gcsfuse-tmp/TestJobChunkTestWithParallelDownloads.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false" + compatible: + flat: true + hns: true + zonal: true + run: TestJobChunkTestWithParallelDownloads + run_on_gke: true + - flags: + - "--file-cache-exclude-regex=.,--file-cache-max-size-mb=50,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest,--log-file=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest.log,--log-severity=TRACE,--enable-kernel-reader=false" + - "--file-cache-exclude-regex=.,--file-cache-max-size-mb=50,--file-cache-cache-file-for-range-read,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest,--log-file=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest.log,--log-severity=TRACE,--enable-kernel-reader=false" + - "--file-cache-exclude-regex=^${BUCKET_NAME}/,--file-cache-max-size-mb=50,--file-cache-cache-file-for-range-read,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest,--log-file=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest.log,--log-severity=TRACE,--enable-kernel-reader=false" + - "--file-cache-exclude-regex=.,--file-cache-max-size-mb=50,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest,--log-file=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false" + - "--file-cache-exclude-regex=.,--file-cache-max-size-mb=50,--file-cache-cache-file-for-range-read,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest,--log-file=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false" + - "--file-cache-exclude-regex=^${BUCKET_NAME}/,--file-cache-max-size-mb=50,--file-cache-cache-file-for-range-read,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest,--log-file=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false" + # Exclude regex flag takes precedence over include regex flag so files won't be cached. + - "--file-cache-include-regex=^${BUCKET_NAME}/,--file-cache-exclude-regex=^${BUCKET_NAME}/,--file-cache-max-size-mb=50,--file-cache-cache-file-for-range-read,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest,--log-file=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest.log,--log-severity=TRACE,--enable-kernel-reader=false" + - "--file-cache-include-regex=^${BUCKET_NAME}/,--file-cache-exclude-regex=^${BUCKET_NAME}/,--file-cache-max-size-mb=50,--file-cache-cache-file-for-range-read,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest,--log-file=/gcsfuse-tmp/TestCacheFileForExcludeRegexTest.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false" + compatible: + flat: true + hns: true + zonal: true + run: TestCacheFileForExcludeRegexTest + run_on_gke: true + - flags: + - "--implicit-dirs,--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestRemountTest,--log-file=/gcsfuse-tmp/TestRemountTest.log,--log-severity=TRACE,--enable-kernel-reader=false" + - "--implicit-dirs,--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestRemountTest,--log-file=/gcsfuse-tmp/TestRemountTest.log,--log-severity=TRACE,--enable-kernel-reader=false" + - "--implicit-dirs,--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestRemountTest,--log-file=/gcsfuse-tmp/TestRemountTest.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false" + - "--implicit-dirs,--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestRemountTest,--log-file=/gcsfuse-tmp/TestRemountTest.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false" + compatible: + flat: true + hns: true + zonal: true + run: TestRemountTest + run_on_gke: false + - flags: + - "--file-cache-include-regex=^${BUCKET_NAME}/.*ReadCacheTest.*/foo*,--file-cache-max-size-mb=9,--cache-dir=/gcsfuse-tmp/TestCacheFileForIncludeRegexTest,--log-file=/gcsfuse-tmp/TestCacheFileForIncludeRegexTest.log,--log-severity=TRACE,--enable-kernel-reader=false" + - "--file-cache-include-regex=^${BUCKET_NAME}/.*ReadCacheTest.*/foo*,--file-cache-max-size-mb=9,--cache-dir=/gcsfuse-tmp/TestCacheFileForIncludeRegexTest,--log-file=/gcsfuse-tmp/TestCacheFileForIncludeRegexTest.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false" + - "--file-cache-include-regex=^${BUCKET_NAME}/.*ReadCacheTest.*/foo*,--file-cache-exclude-regex=invalid,--file-cache-max-size-mb=9,--cache-dir=/gcsfuse-tmp/TestCacheFileForIncludeRegexTest,--log-file=/gcsfuse-tmp/TestCacheFileForIncludeRegexTest.log,--log-severity=TRACE,--enable-kernel-reader=false" + - "--file-cache-include-regex=^${BUCKET_NAME}/.*ReadCacheTest.*/foo*,--file-cache-exclude-regex=invalid,--file-cache-max-size-mb=9,--cache-dir=/gcsfuse-tmp/TestCacheFileForIncludeRegexTest,--log-file=/gcsfuse-tmp/TestCacheFileForIncludeRegexTest.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false" + compatible: + flat: true + hns: true + zonal: true + run: TestCacheFileForIncludeRegexTest + run_on_gke: true + - flags: + - "--file-cache-experimental-enable-chunk-cache=true,--file-cache-download-chunk-size-mb=10,--file-cache-max-size-mb=-1,--cache-dir=/gcsfuse-tmp/TestChunkCacheTest,--log-file=/gcsfuse-tmp/TestChunkCacheTest.log,--log-severity=TRACE,--enable-kernel-reader=false" + compatible: + flat: true + hns: true + zonal: true + run: TestChunkCacheTest + run_on_gke: true + - flags: + - "--file-cache-experimental-enable-chunk-cache=false,--file-cache-max-size-mb=-1,--cache-dir=/gcsfuse-tmp/TestChunkCacheDisabledTest,--log-file=/gcsfuse-tmp/TestChunkCacheDisabledTest.log,--log-severity=TRACE,--enable-kernel-reader=false" + compatible: + flat: true + hns: true + zonal: true + run: TestChunkCacheDisabledTest + run_on_gke: true + - flags: + - "--file-cache-experimental-enable-chunk-cache=true,--file-cache-download-chunk-size-mb=10,--file-cache-max-size-mb=15,--cache-dir=/gcsfuse-tmp/TestChunkCacheEviction,--log-file=/gcsfuse-tmp/TestChunkCacheEviction.log,--log-severity=TRACE,--enable-kernel-reader=false" + compatible: + flat: true + hns: true + zonal: true + run: TestChunkCacheEviction + run_on_gke: true + +stale_handle: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + - "--metadata-cache-ttl-secs=0,--write-block-size-mb=1,--write-max-blocks-per-file=1" + - "--metadata-cache-ttl-secs=0,--write-block-size-mb=1,--write-max-blocks-per-file=1,--client-protocol=grpc" + compatible: + flat: true + hns: true + zonal: true + run: "TestStaleHandleStreamingWritesEnabled" + run_on_gke: true + - flags: + - "--metadata-cache-ttl-secs=0,--enable-streaming-writes=false" + - "--metadata-cache-ttl-secs=0,--enable-streaming-writes=false,--client-protocol=grpc" + compatible: + flat: true + hns: true + zonal: true + run: "TestStaleHandleStreamingWritesDisabled" + run_on_gke: true + +readdirplus: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + - "--implicit-dirs,--experimental-enable-readdirplus,--experimental-enable-dentry-cache,--log-file=/gcsfuse-tmp/TestReaddirplusWithDentryCacheTest.log,--log-severity=TRACE" + compatible: + flat: true + hns: true + zonal: true + run: TestReaddirplusWithDentryCacheTest + run_on_gke: true + - flags: + - "--implicit-dirs,--experimental-enable-readdirplus,--log-file=/gcsfuse-tmp/TestReaddirplusWithoutDentryCacheTest.log,--log-severity=TRACE" + compatible: + flat: true + hns: true + zonal: true + run: TestReaddirplusWithoutDentryCacheTest + run_on_gke: true + +inactive_stream_timeout: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + - "--read-inactive-stream-timeout=1s,--client-protocol=http1,--log-format=json,--log-file=/gcsfuse-tmp/TestTimeoutEnabledSuite.log" + - "--read-inactive-stream-timeout=1s,--client-protocol=grpc,--log-format=json,--log-file=/gcsfuse-tmp/TestTimeoutEnabledSuite.log" + compatible: + flat: true + hns: true + zonal: true + run: TestTimeoutEnabledSuite + run_on_gke: true + - flags: + - "--read-inactive-stream-timeout=0s,--client-protocol=http1,--log-format=json,--log-file=/gcsfuse-tmp/TestTimeoutDisabledSuite.log" + compatible: + flat: true + hns: true + zonal: true + run: TestTimeoutDisabledSuite + run_on_gke: true + + +benchmarking: + - test_bucket: "${BUCKET_NAME}" + mounted_directory: "${MOUNTED_DIR}" + configs: + - flags: + - "--stat-cache-ttl=0" + - "--stat-cache-ttl=0 --client-protocol=grpc" + compatible: + flat: true + hns: true + zonal: true + run: "Benchmark_Stat" + run_on_gke: true + - flags: + - "--stat-cache-ttl=0,--enable-atomic-rename-object" + - "--stat-cache-ttl=0,--enable-atomic-rename-object,--client-protocol=grpc" + compatible: + flat: true + hns: true + zonal: true + run: "Benchmark_Rename" + run_on_gke: true + - flags: + - "--stat-cache-ttl=0" + - "--client-protocol=grpc,--stat-cache-ttl=0" + compatible: + flat: true + hns: true + zonal: true + run: "Benchmark_Delete" + run_on_gke: true + +dentry_cache: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + - "--implicit-dirs,--experimental-enable-dentry-cache,--metadata-cache-ttl-secs=2" + compatible: + flat: true + hns: true + zonal: true + run: TestStatWithDentryCacheEnabledTest + run_on_gke: true + - flags: + - "--implicit-dirs,--experimental-enable-dentry-cache,--metadata-cache-ttl-secs=1000" + compatible: + flat: true + hns: true + zonal: true + run: TestDeleteOperationTest + run_on_gke: true + - flags: + - "--implicit-dirs,--experimental-enable-dentry-cache,--metadata-cache-ttl-secs=1000" + compatible: + flat: true + hns: true + zonal: true + run: TestNotifierTest + run_on_gke: true + +read_gcs_algo: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + # Do not enable fileCache as we want to test gcs read flow. + - "--implicit-dirs" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: true + +unfinalized_object: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + - "--metadata-cache-ttl-secs=-1" + - "--metadata-cache-ttl-secs=-1,--enable-kernel-reader=false" + compatible: + flat: false + hns: false + zonal: true + run: TestUnfinalizedObjectReadTest + run_on_gke: true + - flags: + - "--metadata-cache-ttl-secs=0" + - "--metadata-cache-ttl-secs=0,--enable-kernel-reader=false" + compatible: + flat: false + hns: false + zonal: true + run: TestUnfinalizedObjectOperationTest + run_on_gke: true + - flags: + - "--metadata-cache-ttl-secs=2" + - "--metadata-cache-ttl-secs=2,--enable-kernel-reader=false" + compatible: + flat: false + hns: false + zonal: true + run: TestUnfinalizedObjectTailingReadTest + run_on_gke: true + +interrupt: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + #TODO(b/417136852): Enable this test for Zonal Bucket also once read start working. + - "--enable-streaming-writes" + compatible: + flat: true + hns: true + zonal: false + run_on_gke: true + - flags: + - "--implicit-dirs,--enable-streaming-writes=false" + - "--ignore-interrupts,--enable-streaming-writes=false" + - "--ignore-interrupts=false,--enable-streaming-writes=false" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: true + +log_rotation: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + - "--log-file=/gcsfuse-tmp/TestLogRotation.log,--log-rotate-max-file-size-mb=2,--log-rotate-backup-file-count=2,--log-rotate-compress=false,--log-severity=trace" + - "--log-file=/gcsfuse-tmp/TestLogRotation.log,--log-rotate-max-file-size-mb=2,--log-rotate-backup-file-count=2,--log-rotate-compress,--log-severity=trace" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: false + +readonly_creds: +- mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + - "--implicit-dirs=true" + - "--implicit-dirs=false" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: false + +mount_timeout: +- mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + - "" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: false + +release_version: +- mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + - "" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: false + +mounting: +- mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + - "" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: false + +flag_optimizations: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - run: TestMountFails + flags: + - "--profile=unknown-profile" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: false + - run: TestImplicitDirsNotEnabled + flags: + - "--machine-type=low-end-machine" + compatible: + flat: true + hns: false + zonal: false + run_on_gke: true + - run: TestRenameDirLimitNotSet + flags: + - "--machine-type=low-end-machine" + - "--profile=aiml-training" + - "--profile=aiml-serving" + compatible: + flat: true + hns: false + zonal: false + run_on_gke: true + - run: TestImplicitDirsEnabled + flags: + - "--machine-type=a3-highgpu-8g" + - "--profile=aiml-training" + - "--profile=aiml-serving" + - "--profile=aiml-checkpointing" + - "--machine-type=low-end-machine,--profile=aiml-training" + - "--machine-type=low-end-machine,--profile=aiml-serving" + - "--machine-type=low-end-machine,--profile=aiml-checkpointing" + compatible: + flat: true + hns: false + zonal: false + run_on_gke: true + - run: TestRenameDirLimitSet + flags: + - "--machine-type=a3-highgpu-8g" + - "--profile=aiml-checkpointing" + - "--machine-type=low-end-machine,--profile=aiml-checkpointing" + compatible: + flat: true + hns: false + zonal: false + run_on_gke: true + - run: TestZonalBucketOptimizations + flags: + - "--log-severity=trace" + compatible: + flat: false + hns: false + zonal: true + run_on_gke: false + - run: TestZonalBucketOptimizations_ExplicitOverrides + flags: + - "--implicit-dirs,--max-read-ahead-kb=2048,--max-background=50,--congestion-threshold=30,--log-severity=trace" + compatible: + flat: false + hns: false + zonal: true + run_on_gke: false + - run: TestZonalBucketOptimizations_Dynamic + flags: + - "--log-severity=trace" + compatible: + flat: false + hns: false + zonal: true + run_on_gke: false + - run: TestKernelReader_DefaultAndPrecedence + # Tests that kernel reader is used by default and takes precedence over buffered reader and file cache. + flags: + - "--implicit-dirs,--log-severity=trace" + - "--implicit-dirs,--log-severity=trace,--file-cache-max-size-mb=-1,--cache-dir=/gcsfuse-tmp/TestKernelReader_DefaultAndPrecedence_FileCache" + - "--implicit-dirs,--log-severity=trace,--enable-buffered-read=true" + - "--implicit-dirs,--log-severity=trace,--enable-buffered-read=true,--file-cache-max-size-mb=-1,--cache-dir=/gcsfuse-tmp/TestKernelReader_DefaultAndPrecedence_Both" + compatible: + flat: false + hns: false + zonal: true + run_on_gke: false + - run: TestFileCache_KernelReaderDisabled + # Tests that file cache is used when kernel reader is explicitly disabled. + flags: + - "--implicit-dirs,--log-severity=trace,--enable-kernel-reader=false,--file-cache-max-size-mb=-1,--cache-dir=/gcsfuse-tmp/TestFileCache_KernelReaderDisabled" + compatible: + flat: false + hns: false + zonal: true + run_on_gke: false + - run: TestBufferedReader_KernelReaderDisabled + # Tests that buffered reader is used when kernel reader is explicitly disabled. + flags: + - "--implicit-dirs,--log-severity=trace,--enable-kernel-reader=false,--enable-buffered-read" + compatible: + flat: false + hns: false + zonal: true + run_on_gke: false + - run: TestKernelReader_Dynamic + flags: + - "--implicit-dirs,--log-severity=trace" + compatible: + flat: false + hns: false + zonal: true + run_on_gke: false + +unsupported_path: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + - "--implicit-dirs,--client-protocol=grpc,--enable-unsupported-path-support,--rename-dir-limit=200,--metadata-cache-negative-ttl-secs=0" + compatible: + flat: true + hns: true + zonal: false + run_on_gke: true + - flags: + - "--implicit-dirs,--enable-unsupported-path-support,--rename-dir-limit=200,--metadata-cache-negative-ttl-secs=0" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: true + +kernel_list_cache: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + only_dir: "${ONLY_DIR}" + configs: + - flags: + - "--kernel-list-cache-ttl-secs=-1" + compatible: + flat: true + hns: true + zonal: true + run: "TestInfiniteKernelListCacheTest" + run_on_gke: true + - flags: + # Note: metadata cache is disabled to avoid cache consistency issue between gcsfuse cache and kernel cache. As + # gcsfuse cache might hold the entry which already became stale due to delete operation. + - "--kernel-list-cache-ttl-secs=-1,--metadata-cache-ttl-secs=0,--metadata-cache-negative-ttl-secs=0" + compatible: + flat: true + hns: true + zonal: true + run: "TestInfiniteKernelListCacheDeleteDirTest" + run_on_gke: true + - flags: + - "--kernel-list-cache-ttl-secs=5,--rename-dir-limit=10" + compatible: + flat: true + hns: true + zonal: true + run: "TestFiniteKernelListCacheTest" + run_on_gke: true + - flags: + - "--kernel-list-cache-ttl-secs=0,--stat-cache-ttl=0,--rename-dir-limit=10" + compatible: + flat: true + hns: true + zonal: true + run: "TestDisabledKernelListCacheTest" + run_on_gke: true + +rapid_appends: + - mounted_directory: "${MOUNTED_DIR}" + mounted_directory_secondary: "${MOUNTED_DIR_SECONDARY}" + test_bucket: "${BUCKET_NAME}" + configs: + - run: TestSingleMountAppendsTestSuite + flags: + - "--write-block-size-mb=1" + compatible: + flat: false + hns: false + zonal: true + run_on_gke: true + - run: TestDualMountAppendsTestSuite + flags: + - "--write-block-size-mb=1" + secondary_flags: + - "--write-block-size-mb=1" + compatible: + flat: false + hns: false + zonal: true + run_on_gke: true + - run: TestSingleMountReadsTestSuite + flags: + - "--metadata-cache-ttl-secs=0" # NoCache + - "--metadata-cache-ttl-secs=70" # MetadataCache + - "--file-cache-max-size-mb=-1,--cache-dir=/gcsfuse-tmp/cache,--metadata-cache-ttl-secs=0" # FileCache + - "--metadata-cache-ttl-secs=70,--file-cache-max-size-mb=-1,--cache-dir=/gcsfuse-tmp/cache" # MetadataAndFileCache + - "--metadata-cache-ttl-secs=0,--enable-kernel-reader=false" # NoCacheWithoutKernelReader + - "--metadata-cache-ttl-secs=70,--enable-kernel-reader=false" # MetadataCacheWithMRDWrapperWithoutKernelReader + - "--file-cache-max-size-mb=-1,--cache-dir=/gcsfuse-tmp/cache,--metadata-cache-ttl-secs=0,--enable-kernel-reader=false" # FileCacheWithoutKernelReader + - "--metadata-cache-ttl-secs=70,--file-cache-max-size-mb=-1,--cache-dir=/gcsfuse-tmp/cache,--enable-kernel-reader=false" # MetadataAndFileCacheWithoutKernelReader + compatible: + flat: false + hns: false + zonal: true + run_on_gke: true + - run: TestDualMountReadsTestSuiteWithMetadataCache + flags: + - "--metadata-cache-ttl-secs=70" + - "--metadata-cache-ttl-secs=70,--file-cache-max-size-mb=-1,--cache-dir=/gcsfuse-tmp/cache-primary" + - "--metadata-cache-ttl-secs=70,--enable-kernel-reader=false" + - "--metadata-cache-ttl-secs=70,--file-cache-max-size-mb=-1,--cache-dir=/gcsfuse-tmp/cache-primary,--enable-kernel-reader=false" + secondary_flags: + - "--write-block-size-mb=1" + - "--write-block-size-mb=1" + - "--write-block-size-mb=1" + - "--write-block-size-mb=1" + compatible: + flat: false + hns: false + zonal: true + run_on_gke: true + - run: TestDualMountReadsTestSuiteWithoutMetadataCache + flags: + - "--metadata-cache-ttl-secs=0" + - "--file-cache-max-size-mb=-1,--cache-dir=/gcsfuse-tmp/cache-primary,--metadata-cache-ttl-secs=0" + - "--metadata-cache-ttl-secs=0,--enable-kernel-reader=false" + - "--file-cache-max-size-mb=-1,--cache-dir=/gcsfuse-tmp/cache-primary,--metadata-cache-ttl-secs=0,--enable-kernel-reader=false" + secondary_flags: + - "--write-block-size-mb=1" + - "--write-block-size-mb=1" + - "--write-block-size-mb=1" + - "--write-block-size-mb=1" + compatible: + flat: false + hns: false + zonal: true + run_on_gke: true + +monitoring: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + - "--prometheus-port=9190,--cache-dir=/gcsfuse-tmp/PromOTELSuite,--file-cache-max-size-mb=-1,--log-file=/gcsfuse-tmp/TestPromOTELSuite.log,--enable-kernel-reader=false" + compatible: + flat: true + hns: false + zonal: false + run: "TestPromOTELSuite" + run_on_gke: false + - flags: + - "--prometheus-port=10190,--cache-dir=/gcsfuse-tmp/PromOTELSuite,--file-cache-max-size-mb=-1,--log-file=/gcsfuse-tmp/TestPromOTELSuite.log,--enable-kernel-reader=false" + compatible: + flat: false + hns: true + zonal: true + run: "TestPromOTELSuite" + run_on_gke: false + - flags: + - "--prometheus-port=9191,--enable-buffered-read,--read-block-size-mb=4,--read-random-seek-threshold=2,--read-global-max-blocks=5,--read-min-blocks-per-handle=2,--read-start-blocks-per-handle=2,--log-file=/gcsfuse-tmp/TestPromBufferedReadSuite.log,--enable-kernel-reader=false" + compatible: + flat: true + hns: false + zonal: false + run: "TestPromBufferedReadSuite" + run_on_gke: false + - flags: + - "--prometheus-port=10191,--enable-buffered-read,--read-block-size-mb=4,--read-random-seek-threshold=2,--read-global-max-blocks=5,--read-min-blocks-per-handle=2,--read-start-blocks-per-handle=2,--log-file=/gcsfuse-tmp/TestPromBufferedReadSuite.log,--enable-kernel-reader=false" + compatible: + flat: false + hns: true + zonal: true + run: "TestPromBufferedReadSuite" + run_on_gke: false + - flags: + - "--client-protocol=grpc,--experimental-enable-grpc-metrics,--prometheus-port=9192,--cache-dir=/gcsfuse-tmp/TestPromGrpcMetricsSuite,--file-cache-max-size-mb=-1,--log-file=/gcsfuse-tmp/TestPromGrpcMetricsSuite.log,--enable-kernel-reader=false" + compatible: + flat: true + hns: false + zonal: false + run: "TestPromGrpcMetricsSuite" + run_on_gke: false + - flags: + - "--client-protocol=grpc,--experimental-enable-grpc-metrics,--prometheus-port=10192,--cache-dir=/gcsfuse-tmp/TestPromGrpcMetricsSuite,--file-cache-max-size-mb=-1,--log-file=/gcsfuse-tmp/TestPromGrpcMetricsSuite.log,--enable-kernel-reader=false" + compatible: + flat: false + hns: true + zonal: true + run: "TestPromGrpcMetricsSuite" + run_on_gke: false + - flags: + - "--prometheus-port=9193 --log-file=/gcsfuse-tmp/TestPromKernelReaderSuite.log" + compatible: + flat: false + hns: false + zonal: true + run: "TestPromKernelReaderSuite" + run_on_gke: false + +symlink_handling: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - run: TestStandardSymlinksTestSuite + flags: + - "--enable-standard-symlinks=true" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: true + - run: TestLegacySymlinksTestSuite + flags: + - "--enable-standard-symlinks=false" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: true + +buffered_read: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + - "--enable-buffered-read,--read-block-size-mb=8,--read-max-blocks-per-handle=20,--read-start-blocks-per-handle=1,--read-min-blocks-per-handle=2,--enable-kernel-reader=false,--log-file=/gcsfuse-tmp/TestBufferedReadSuite.log,--log-severity=TRACE" + - "--client-protocol=grpc,--enable-buffered-read,--read-block-size-mb=8,--read-max-blocks-per-handle=20,--read-start-blocks-per-handle=1,--read-min-blocks-per-handle=2,--enable-kernel-reader=false,--log-file=/gcsfuse-tmp/TestBufferedReadSuite.log,--log-severity=TRACE" + compatible: + flat: true + hns: true + zonal: true + run: TestSequentialReadSuite + run_on_gke: true + - flags: + - "--enable-buffered-read,--read-block-size-mb=8,--read-min-blocks-per-handle=2,--read-global-max-blocks=1,--read-max-blocks-per-handle=10,--read-start-blocks-per-handle=2,--enable-kernel-reader=false,--log-file=/gcsfuse-tmp/TestInsufficientPoolCreationSuite.log,--log-severity=TRACE" + - "--client-protocol=grpc,--enable-buffered-read,--read-block-size-mb=8,--read-min-blocks-per-handle=2,--read-global-max-blocks=1,--read-max-blocks-per-handle=10,--read-start-blocks-per-handle=2,--enable-kernel-reader=false,--log-file=/gcsfuse-tmp/TestInsufficientPoolCreationSuite.log,--log-severity=TRACE" + compatible: + flat: true + hns: true + zonal: true + run: TestInsufficientPoolCreationSuite + run_on_gke: true + - flags: + - "--enable-buffered-read,--read-block-size-mb=8,--read-max-blocks-per-handle=20,--read-start-blocks-per-handle=2,--read-min-blocks-per-handle=2,--enable-kernel-reader=false,--log-file=/gcsfuse-tmp/TestRandomReadFallbackSuite.log,--log-severity=TRACE" + - "--client-protocol=grpc,--enable-buffered-read,--read-block-size-mb=8,--read-max-blocks-per-handle=20,--read-start-blocks-per-handle=2,--read-min-blocks-per-handle=2,--enable-kernel-reader=false,--log-file=/gcsfuse-tmp/TestRandomReadFallbackSuite.log,--log-severity=TRACE" + compatible: + flat: true + hns: true + zonal: true + run: TestRandomReadFallbackSuite + run_on_gke: true + +negative_stat_cache: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + only_dir: "${ONLY_DIR}" + configs: + - flags: + - "--metadata-cache-negative-ttl-secs=0" + compatible: + flat: true + hns: true + zonal: true + run: "TestDisabledNegativeStatCacheTest" + run_on_gke: true + - flags: + - "--metadata-cache-negative-ttl-secs=5" + compatible: + flat: true + hns: true + zonal: true + run: "TestFiniteNegativeStatCacheTest" + run_on_gke: true + - flags: + - "--metadata-cache-negative-ttl-secs=-1" + compatible: + flat: true + hns: true + zonal: true + run: "TestInfiniteNegativeStatCacheTest" + run_on_gke: true + +managed_folders: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + only_dir: "${ONLY_DIR}" + configs: + - run: TestManagedFolders_FolderViewPermission + flags: + - "--implicit-dirs,--key-file=${KEY_FILE},--rename-dir-limit=3" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: false + - run: TestEnableEmptyManagedFoldersTrue + flags: + - "--implicit-dirs,--enable-empty-managed-folders" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: false + - run: TestManagedFolders_FolderAdminPermission + flags: + - "--implicit-dirs,--key-file=${KEY_FILE},--rename-dir-limit=5,--stat-cache-ttl=0" + compatible: + flat: true + hns: true + zonal: true + run_on_gke: false + +concurrent_operations: + - mounted_directory: "${MOUNTED_DIR}" + test_bucket: "${BUCKET_NAME}" + configs: + - flags: + - "" + - "--file-cache-cache-file-for-range-read=true --file-cache-enable-parallel-downloads=true --enable-kernel-reader=false --cache-dir=/gcsfuse-tmp/read_large_files" + - "--enable-buffered-read --enable-kernel-reader=false --enable-metadata-prefetch" + compatible: + flat: true + hns: true + zonal: true + run: TestConcurrentRead + run_on_gke: true + - flags: + - "--enable-kernel-reader=false" + compatible: + flat: false + hns: false + zonal: true + run: TestConcurrentRead + run_on_gke: true + - flags: + - "--kernel-list-cache-ttl-secs=0 --enable-metadata-prefetch" + - "--kernel-list-cache-ttl-secs=0 --enable-metadata-prefetch --client-protocol=grpc" + - "--kernel-list-cache-ttl-secs=-1" + - "--kernel-list-cache-ttl-secs=-1 --client-protocol=grpc" + compatible: + flat: true + hns: true + zonal: true + run: TestConcurrentListing + run_on_gke: true diff --git a/tools/integration_tests/unfinalized_object/setup_test.go b/tools/integration_tests/unfinalized_object/setup_test.go new file mode 100644 index 0000000000..ed364d2042 --- /dev/null +++ b/tools/integration_tests/unfinalized_object/setup_test.go @@ -0,0 +1,119 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unfinalized_object + +import ( + "context" + "log" + "os" + "testing" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" +) + +const ( + testDirName = "UnfinalizedObjectTest" +) + +var ( + testEnv env + mountFunc func(*test_suite.TestConfig, []string) error + // mount directory is where our tests run. + mountDir string + rootDir string +) + +type env struct { + storageClient *storage.Client + ctx context.Context + testDirPath string + cfg *test_suite.TestConfig + bucketType string +} + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.UnfinalizedObject) == 0 { + log.Println("No configuration found for unfinalized_object tests in config. Using flags instead.") + // Populate the config manually. + cfg.UnfinalizedObject = make([]test_suite.TestConfig, 1) + cfg.UnfinalizedObject[0].TestBucket = setup.TestBucket() + cfg.UnfinalizedObject[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.UnfinalizedObject[0].LogFile = setup.LogFile() + cfg.UnfinalizedObject[0].Configs = make([]test_suite.ConfigItem, 3) + cfg.UnfinalizedObject[0].Configs[0].Flags = []string{ + "--metadata-cache-ttl-secs=-1", + "--metadata-cache-ttl-secs=-1 --enable-kernel-reader=false", + } + cfg.UnfinalizedObject[0].Configs[0].Compatible = map[string]bool{"flat": false, "hns": false, "zonal": true} + cfg.UnfinalizedObject[0].Configs[0].Run = "TestUnfinalizedObjectReadTest" + cfg.UnfinalizedObject[0].Configs[1].Flags = []string{ + "--metadata-cache-ttl-secs=0", + "--metadata-cache-ttl-secs=0 --enable-kernel-reader=false", + } + cfg.UnfinalizedObject[0].Configs[1].Compatible = map[string]bool{"flat": false, "hns": false, "zonal": true} + cfg.UnfinalizedObject[0].Configs[1].Run = "TestUnfinalizedObjectOperationTest" + cfg.UnfinalizedObject[0].Configs[2].Flags = []string{ + "--metadata-cache-ttl-secs=2", + "--metadata-cache-ttl-secs=2 --enable-kernel-reader=false", + } + cfg.UnfinalizedObject[0].Configs[2].Compatible = map[string]bool{"flat": false, "hns": false, "zonal": true} + cfg.UnfinalizedObject[0].Configs[2].Run = "TestUnfinalizedObjectTailingReadTest" + } + + testEnv.ctx = context.Background() + testEnv.cfg = &cfg.UnfinalizedObject[0] + testEnv.bucketType = setup.TestEnvironment(testEnv.ctx, testEnv.cfg) + + if !setup.IsZonalBucketRun() { + log.Printf("This test is only for Zonal buckets.") + os.Exit(0) + } + + // 2. Create storage client before running tests. + var err error + testEnv.storageClient, err = client.CreateStorageClient(testEnv.ctx) + if err != nil { + log.Printf("Error creating storage client: %v\n", err) + os.Exit(1) + } + defer testEnv.storageClient.Close() + + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + mountDir, rootDir = testEnv.cfg.GKEMountedDirectory, testEnv.cfg.GKEMountedDirectory + os.Exit(setup.RunTestsForMountedDirectory(testEnv.cfg.GKEMountedDirectory, m)) + } + + // Run tests for testBucket + // Set up test directory. + setup.SetUpTestDirForTestBucket(testEnv.cfg) + + // Save mount and root directory variables. + mountDir, rootDir = testEnv.cfg.GCSFuseMountedDirectory, testEnv.cfg.GCSFuseMountedDirectory + + log.Println("Running static mounting tests...") + mountFunc = static_mounting.MountGcsfuseWithStaticMountingWithConfigFile + successCode := m.Run() + + os.Exit(successCode) +} diff --git a/tools/integration_tests/unfinalized_object/tailing_reads_test.go b/tools/integration_tests/unfinalized_object/tailing_reads_test.go new file mode 100644 index 0000000000..adaf61cfcd --- /dev/null +++ b/tools/integration_tests/unfinalized_object/tailing_reads_test.go @@ -0,0 +1,127 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unfinalized_object + +import ( + "context" + "log" + "os" + "path" + "testing" + "time" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type unfinalizedObjectTailingReads struct { + flags []string + storageClient *storage.Client + ctx context.Context + testDirPath string + fileName string + suite.Suite +} + +func (t *unfinalizedObjectTailingReads) SetupTest() { + t.testDirPath = client.SetupTestDirectory(t.ctx, t.storageClient, testDirName) + t.fileName = path.Base(t.T().Name()) + setup.GenerateRandomString(5) +} + +func (s *unfinalizedObjectTailingReads) TearDownSuite() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *unfinalizedObjectTailingReads) SetupSuite() { + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, s.flags, mountFunc) + setup.SetMntDir(mountDir) + testEnv.testDirPath = client.SetupTestDirectory(s.ctx, s.storageClient, testDirName) +} + +func (t *unfinalizedObjectTailingReads) TestTailingRead() { + // 1. Create file + initialContent := setup.GenerateRandomString(initialSize) + _ = client.CreateUnfinalizedObject(t.ctx, t.T(), t.storageClient, path.Join(testDirName, t.fileName), initialContent) + + // 2. Open file for reading + readPath := path.Join(t.testDirPath, t.fileName) + readFile, err := os.OpenFile(readPath, os.O_RDONLY, setup.FilePermission_0600) + require.NoError(t.T(), err) + defer operations.CloseFileShouldNotThrowError(t.T(), readFile) + + // 3. Read initial content + buf := make([]byte, len(initialContent)) + n, err := readFile.Read(buf) + require.NoError(t.T(), err) + require.Equal(t.T(), len(initialContent), n) + require.Equal(t.T(), initialContent, string(buf)) + + // 4. Loop appends + numAppends := 2 + appendSize := 10 + for i := 0; i < numAppends; i++ { + // Open an appendable writer to the object at correct generation. + obj, err := t.storageClient.Bucket(setup.TestBucket()).Object(path.Join(testDirName, t.fileName)).Attrs(t.ctx) + require.NoError(t.T(), err) + writer, err := client.AppendableWriter(t.ctx, t.storageClient, path.Join(testDirName, t.fileName), obj.Generation) + require.NoError(t.T(), err) + + // Remotely append content to the object. + appendData := setup.GenerateRandomString(appendSize) + _, err = writer.Write([]byte(appendData)) + require.NoError(t.T(), err) + err = writer.Close() + require.NoError(t.T(), err) + + // Wait for metadata cache to expire if needed. + // Since we are running with --metadata-cache-ttl-secs=2, we should wait slightly more than that. + time.Sleep(3 * time.Second) + + // Check Stat (fstat on the read handle) + fi, err := readFile.Stat() + require.NoError(t.T(), err) + expectedSize := int64(len(initialContent) + (i+1)*appendSize) + require.Equal(t.T(), expectedSize, fi.Size(), "File size should update after append") + + // Read new data + newBuf := make([]byte, len(appendData)) + n, err = readFile.Read(newBuf) + require.NoError(t.T(), err) + require.Equal(t.T(), len(appendData), n) + require.Equal(t.T(), appendData, string(newBuf)) + } +} + +func TestUnfinalizedObjectTailingReadTest(t *testing.T) { + ts := &unfinalizedObjectTailingReads{ctx: context.Background(), storageClient: testEnv.storageClient} + + // Run tests for mounted directory if the flag is set. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/unfinalized_object/unfinalized_object_operations_test.go b/tools/integration_tests/unfinalized_object/unfinalized_object_operations_test.go new file mode 100644 index 0000000000..f6e0106421 --- /dev/null +++ b/tools/integration_tests/unfinalized_object/unfinalized_object_operations_test.go @@ -0,0 +1,219 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unfinalized_object + +import ( + "context" + "log" + "path" + "testing" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +type unfinalizedObjectOperations struct { + flags []string + storageClient *storage.Client + ctx context.Context + testDirPath string + fileName string + suite.Suite +} + +func (t *unfinalizedObjectOperations) SetupTest() { + t.testDirPath = client.SetupTestDirectory(t.ctx, t.storageClient, testDirName) + t.fileName = path.Base(t.T().Name()) + setup.GenerateRandomString(5) +} + +func (s *unfinalizedObjectOperations) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *unfinalizedObjectOperations) SetupSuite() { + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, s.flags, mountFunc) + setup.SetMntDir(mountDir) + testEnv.testDirPath = client.SetupTestDirectory(s.ctx, s.storageClient, testDirName) +} + +//////////////////////////////////////////////////////////////////////// +// Helper methods +//////////////////////////////////////////////////////////////////////// + +func (t *unfinalizedObjectOperations) setupUnfinalizedObjectAndGetInitialInode(initialContentSize int) (filePath string, initialInodeID uint64) { + filePath = path.Join(t.testDirPath, t.fileName) + initialContent := setup.GenerateRandomString(initialContentSize) + + // 1. Create an unfinalized object. + _ = client.CreateUnfinalizedObject(t.ctx, t.T(), t.storageClient, path.Join(testDirName, t.fileName), initialContent) + + // 2. Stat the file to get initial Inode ID. + initialStat := operations.StatFileOrFatal(filePath, t.T()) + initialInodeID = initialStat.Ino + return +} + +//////////////////////////////////////////////////////////////////////// +// Test scenarios +//////////////////////////////////////////////////////////////////////// + +func (t *unfinalizedObjectOperations) TestUnfinalizedObjectCreatedOutsideOfMountReportsNonZeroSize() { + size := operations.MiB + _ = client.CreateUnfinalizedObject(t.ctx, t.T(), t.storageClient, path.Join(testDirName, t.fileName), setup.GenerateRandomString(size)) + + statRes, err := operations.StatFile(path.Join(t.testDirPath, t.fileName)) + + require.NoError(t.T(), err) + assert.Equal(t.T(), t.fileName, (*statRes).Name()) + assert.EqualValues(t.T(), size, (*statRes).Size()) +} + +func (t *unfinalizedObjectOperations) TestUnfinalizedObjectCreatedFromSameMountReportsCorrectSize() { + size := operations.MiB + // Create un-finalized object via same mount. + fh := operations.CreateFile(path.Join(t.testDirPath, t.fileName), setup.FilePermission_0600, t.T()) + operations.WriteWithoutClose(fh, setup.GenerateRandomString(size), t.T()) + operations.SyncFile(fh, t.T()) + + statRes, err := operations.StatFile(path.Join(t.testDirPath, t.fileName)) + + require.NoError(t.T(), err) + assert.Equal(t.T(), t.fileName, (*statRes).Name()) + assert.EqualValues(t.T(), size, (*statRes).Size()) + // Write more data to the object and finalize. + operations.WriteWithoutClose(fh, setup.GenerateRandomString(size), t.T()) + err = fh.Close() + require.NoError(t.T(), err) + // After object is finalized, correct size should be reported. + statRes, err = operations.StatFile(path.Join(t.testDirPath, t.fileName)) + require.NoError(t.T(), err) + assert.EqualValues(t.T(), 2*size, (*statRes).Size()) +} + +func (t *unfinalizedObjectOperations) TestOverWritingUnfinalizedObjectsReturnsESTALE() { + // TODO(b/411333280): Enable the test once flush on unfinalized Object is fixed. + t.T().Skip("Skipping the test due to b/411333280") + size := operations.MiB + _ = client.CreateUnfinalizedObject(t.ctx, t.T(), t.storageClient, path.Join(testDirName, t.fileName), setup.GenerateRandomString(size)) + fh := operations.OpenFile(path.Join(t.testDirPath, t.fileName), t.T()) + + // Overwrite unfinalized object. + operations.WriteWithoutClose(fh, setup.GenerateRandomString(int(size)), t.T()) + err := fh.Close() + + operations.ValidateESTALEError(t.T(), err) +} + +func (t *unfinalizedObjectOperations) TestUnfinalizedObjectCanBeRenamedIfCreatedFromSameMount() { + size := operations.MiB + content := setup.GenerateRandomString(size) + newFileName := "new" + t.fileName + // Create un-finalized object via same mount. + fh := operations.CreateFile(path.Join(t.testDirPath, t.fileName), setup.FilePermission_0600, t.T()) + operations.WriteWithoutClose(fh, content, t.T()) + operations.SyncFile(fh, t.T()) + + err := operations.RenameFile(path.Join(t.testDirPath, t.fileName), path.Join(t.testDirPath, newFileName)) + + require.NoError(t.T(), err) + client.ValidateObjectNotFoundErrOnGCS(t.ctx, t.storageClient, testDirName, t.fileName, t.T()) + client.ValidateObjectContentsFromGCS(t.ctx, t.storageClient, testDirName, newFileName, content, t.T()) + // validate writing to the renamed file via stale file handle returns ESTALE error. + _, err = fh.Write([]byte(content)) + operations.ValidateESTALEError(t.T(), err) +} + +func (t *unfinalizedObjectOperations) TestUnfinalizedObjectCanBeRenamedIfCreatedFromDifferentMount() { + size := operations.MiB + _ = client.CreateUnfinalizedObject(t.ctx, t.T(), t.storageClient, path.Join(testDirName, t.fileName), setup.GenerateRandomString(size)) + + // Overwrite unfinalized object. + err := operations.RenameFile(path.Join(t.testDirPath, t.fileName), path.Join(t.testDirPath, "New"+t.fileName)) + + require.NoError(t.T(), err) +} + +func (t *unfinalizedObjectOperations) TestInodeIDPreservedOnRemoteAppend() { + // Setup and stat the file. + filePath, initialInodeID := t.setupUnfinalizedObjectAndGetInitialInode(initialSize) + appendContent := setup.GenerateRandomString(appendSize) + // Remotely append content to the object. This will increase the size of the unfinalized + // object without changing the generation. + obj, err := t.storageClient.Bucket(setup.TestBucket()).Object(path.Join(testDirName, t.fileName)).Attrs(t.ctx) + require.NoError(t.T(), err) + writer, err := client.AppendableWriter(t.ctx, t.storageClient, path.Join(testDirName, t.fileName), obj.Generation) + require.NoError(t.T(), err) + _, err = writer.Write([]byte(appendContent)) + require.NoError(t.T(), err) + err = writer.Close() + require.NoError(t.T(), err) + // Validate that the content was appended to the unfinalized object without changing the object generation. + finalObject, err := t.storageClient.Bucket(setup.TestBucket()).Object(path.Join(testDirName, t.fileName)).Attrs(t.ctx) + require.NoError(t.T(), err) + require.Equal(t.T(), obj.Generation, finalObject.Generation) + + // Stat the file again. + // Since we are using StatCacheTTL=0, this should trigger LookupInode. + newStat := operations.StatFileOrFatal(filePath, t.T()) + + // Assert Inode ID is preserved and Size is updated. + assert.Equal(t.T(), initialInodeID, newStat.Ino, "Inode ID should be preserved") + assert.Equal(t.T(), int64(initialSize+appendSize), newStat.Size, "Size should be updated") +} + +func (t *unfinalizedObjectOperations) TestInodeIDChangedOnRemoteOverwrite() { + // Setup and stat the file. + filePath, initialInodeID := t.setupUnfinalizedObjectAndGetInitialInode(initialSize) + newContent := setup.GenerateRandomString(initialSize) + // Remotely overwrite the object (this changes generation). + _ = client.CreateUnfinalizedObject(t.ctx, t.T(), t.storageClient, path.Join(testDirName, t.fileName), newContent) + + // Stat the file again. + newStat := operations.StatFileOrFatal(filePath, t.T()) + + // Assert Inode ID is DIFFERENT. + assert.NotEqual(t.T(), initialInodeID, newStat.Ino, "Inode ID should change when generation changes") + assert.Equal(t.T(), int64(initialSize), newStat.Size) +} + +//////////////////////////////////////////////////////////////////////// +// Test Function (Runs once before all tests) +//////////////////////////////////////////////////////////////////////// + +func TestUnfinalizedObjectOperationTest(t *testing.T) { + ts := &unfinalizedObjectOperations{ctx: context.Background(), storageClient: testEnv.storageClient} + + // Run tests for mounted directory if the flag is set. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/unfinalized_object/unfinalized_read_test.go b/tools/integration_tests/unfinalized_object/unfinalized_read_test.go new file mode 100644 index 0000000000..94839803dd --- /dev/null +++ b/tools/integration_tests/unfinalized_object/unfinalized_read_test.go @@ -0,0 +1,244 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unfinalized_object + +import ( + "context" + "io" + "log" + "os" + "path" + "syscall" + "testing" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/internal/util" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +const ( + initialSize = operations.MiB + appendSize = operations.MiB + readFlags = syscall.O_RDONLY + readFlagsWithODirect = syscall.O_RDONLY | syscall.O_DIRECT +) + +type unfinalizedObjectReads struct { + flags []string + storageClient *storage.Client + ctx context.Context + testDirPath string + fileName string + suite.Suite +} + +func (t *unfinalizedObjectReads) SetupTest() { + t.testDirPath = client.SetupTestDirectory(t.ctx, t.storageClient, testDirName) + t.fileName = path.Base(t.T().Name()) + setup.GenerateRandomString(5) +} + +func (s *unfinalizedObjectReads) TearDownSuite() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *unfinalizedObjectReads) SetupSuite() { + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, s.flags, mountFunc) + setup.SetMntDir(mountDir) + testEnv.testDirPath = client.SetupTestDirectory(s.ctx, s.storageClient, testDirName) +} + +//////////////////////////////////////////////////////////////////////// +// Helper Methods +//////////////////////////////////////////////////////////////////////// + +// setupAndAppend creates an unfinalized object of size `initialSize` and then attempts an +// initial reads via GCSFuse mount. Finally, it remotely appends to the unfinalized object +// by taking over at the same generation and returns the already open filehandle along with +// initialContent and appendContent to be further used by tests. +func (t *unfinalizedObjectReads) setupAndAppend(filePath string, initialSize, appendSize, openFlags int) (fh *os.File, initialContent, appendContent string) { + initialContent = setup.GenerateRandomString(initialSize) + appendContent = setup.GenerateRandomString(appendSize) + + // 1. Create an unfinalized object and open it. + _ = client.CreateUnfinalizedObject(t.ctx, t.T(), t.storageClient, path.Join(testDirName, t.fileName), initialContent) + fh = operations.OpenFileInMode(t.T(), filePath, openFlags) + + // 2. Read initial content to cache the state. + buffer := make([]byte, initialSize) + n, err := fh.Read(buffer) + require.NoError(t.T(), err) + require.Equal(t.T(), initialSize, n) + assert.Equal(t.T(), initialContent, string(buffer)) + + // 3. Remotely append content to the object. + obj, err := t.storageClient.Bucket(setup.TestBucket()).Object(path.Join(testDirName, t.fileName)).Attrs(t.ctx) + require.NoError(t.T(), err) + + writer, err := client.AppendableWriter(t.ctx, t.storageClient, path.Join(testDirName, t.fileName), obj.Generation) + require.NoError(t.T(), err) + n, err = writer.Write([]byte(appendContent)) + require.NoError(t.T(), err) + assert.Equal(t.T(), appendSize, n) + err = writer.Close() + require.NoError(t.T(), err) + + // Validate that the content was appended to the unfinalized object without changing the object generation. + finalObject, err := t.storageClient.Bucket(setup.TestBucket()).Object(path.Join(testDirName, t.fileName)).Attrs(t.ctx) + require.NoError(t.T(), err) + require.Equal(t.T(), obj.Generation, finalObject.Generation) + + return fh, initialContent, appendContent +} + +//////////////////////////////////////////////////////////////////////// +// Test scenarios +//////////////////////////////////////////////////////////////////////// + +func (t *unfinalizedObjectReads) TestUnfinalizedObjectsCanBeRead() { + var size int = operations.MiB + writtenContent := setup.GenerateRandomString(size) + // Create un-finalized object via same mount. + fh := operations.CreateFile(path.Join(t.testDirPath, t.fileName), setup.FilePermission_0600, t.T()) + operations.WriteWithoutClose(fh, writtenContent, t.T()) + defer operations.CloseFileShouldNotThrowError(t.T(), fh) + + // Read un-finalized object. + file, err := os.OpenFile(path.Join(t.testDirPath, t.fileName), os.O_RDONLY|syscall.O_DIRECT, setup.FilePermission_0600) + require.NoError(t.T(), err) + readContent, err := operations.ReadFileSequentially(file, util.MiB) + + require.NoError(t.T(), err) + assert.Equal(t.T(), writtenContent, string(readContent)) +} + +func (t *unfinalizedObjectReads) TestReadRemotelyModifiedUnfinalizedObject() { + testCases := []struct { + name string + openFlags int + readOffset int64 + bytesToRead int + expectedBytesRead int + expectedErr error + getExpectedContent func(initial, appended string) string + }{ + { + name: "WithODirect_Read_Beyond_Cached_Object_Size", + openFlags: readFlagsWithODirect, + readOffset: initialSize, + bytesToRead: appendSize, + expectedBytesRead: appendSize, + expectedErr: nil, + getExpectedContent: func(_, appended string) string { + return appended + }, + }, + { + name: "WithODirect_Read_Partially_Within_Cached_Object_Size", + openFlags: readFlagsWithODirect, + readOffset: initialSize / 2, + bytesToRead: appendSize, + expectedBytesRead: appendSize, + expectedErr: nil, + getExpectedContent: func(initial, appended string) string { + offset := initialSize / 2 + return initial[offset:] + appended[:offset] + }, + }, + { + name: "WithODirect_Requested_Data_Is_Partially_Present_On_GCS", + openFlags: readFlagsWithODirect, + readOffset: initialSize, + bytesToRead: appendSize * 2, + expectedBytesRead: appendSize, + expectedErr: io.EOF, + getExpectedContent: func(_, appended string) string { + return appended + }, + }, + { + name: "WithODirect_Read_Beyond_Actual_Object_Size", + openFlags: readFlagsWithODirect, + readOffset: initialSize + appendSize, + bytesToRead: appendSize, + expectedBytesRead: 0, + expectedErr: io.EOF, + }, + { + name: "WithoutODirect_Read_Beyond_Cached_Size", + openFlags: readFlags, + readOffset: initialSize, + bytesToRead: 1, + expectedBytesRead: 0, + expectedErr: io.EOF, + }, + } + + for _, tc := range testCases { + t.T().Run(tc.name, func(T *testing.T) { + // We need to use a new file for each subtest to ensure isolation. + t.fileName = path.Base(T.Name()) + setup.GenerateRandomString(5) + filePath := path.Join(t.testDirPath, t.fileName) + + fh, initialContent, appendContent := t.setupAndAppend(filePath, initialSize, appendSize, tc.openFlags) + defer operations.CloseFileShouldNotThrowError(T, fh) + + readBuffer := make([]byte, tc.bytesToRead) + bytesRead, err := fh.ReadAt(readBuffer, tc.readOffset) + + assert.Equal(T, tc.expectedBytesRead, bytesRead) + if tc.expectedErr != nil { + assert.Equal(T, tc.expectedErr, err) + } else { + require.NoError(T, err) + } + + if tc.getExpectedContent != nil { + expectedContent := tc.getExpectedContent(initialContent, appendContent) + assert.Equal(T, expectedContent, string(readBuffer[:bytesRead])) + } + }) + } +} + +//////////////////////////////////////////////////////////////////////// +// Test Function (Runs once before all tests) +//////////////////////////////////////////////////////////////////////// + +func TestUnfinalizedObjectReadTest(t *testing.T) { + ts := &unfinalizedObjectReads{ctx: context.Background(), storageClient: testEnv.storageClient} + + // Run tests for mounted directory if the flag is set. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + // Run tests for GCE environment otherwise. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + for _, ts.flags = range flagsSet { + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/unsupported_path/setup_test.go b/tools/integration_tests/unsupported_path/setup_test.go new file mode 100644 index 0000000000..96f56dadb0 --- /dev/null +++ b/tools/integration_tests/unsupported_path/setup_test.go @@ -0,0 +1,92 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package unsupported_path + +import ( + "context" + "log" + "os" + "testing" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" +) + +const DirForUnsupportedPathTests = "dirForUnsupportedPathTests" + +var ( + storageClient *storage.Client + ctx context.Context + bucketType string +) + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.UnsupportedPath) == 0 { + log.Println("No configuration found for unsupported path tests in config. Using flags instead.") + // Populate the config manually. + cfg.UnsupportedPath = []test_suite.TestConfig{ + { + TestBucket: setup.TestBucket(), + GKEMountedDirectory: setup.MountedDirectory(), + Configs: []test_suite.ConfigItem{ + { + Flags: []string{ + "--implicit-dirs --client-protocol=grpc --enable-unsupported-path-support=true --rename-dir-limit=200 --metadata-cache-negative-ttl-secs=0", + }, + Compatible: map[string]bool{"flat": true, "hns": true, "zonal": false}, + }, + { + Flags: []string{ + "--implicit-dirs --enable-unsupported-path-support=true --rename-dir-limit=200 --metadata-cache-negative-ttl-secs=0", + }, + Compatible: map[string]bool{"flat": true, "hns": true, "zonal": true}, + }, + }, + }, + } + } + + ctx = context.Background() + bucketType = setup.TestEnvironment(ctx, &cfg.UnsupportedPath[0]) + + // 2. Create storage client before running tests. + var err error + storageClient, err = client.CreateStorageClient(ctx) + if err != nil { + log.Printf("Error creating storage client: %v\n", err) + os.Exit(1) + } + defer storageClient.Close() + + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + if cfg.UnsupportedPath[0].GKEMountedDirectory != "" && cfg.UnsupportedPath[0].TestBucket != "" { + os.Exit(setup.RunTestsForMountedDirectory(cfg.UnsupportedPath[0].GKEMountedDirectory, m)) + } + + // Run tests for testBucket + // 4. Build the flag sets dynamically from the config. + flags := setup.BuildFlagSets(cfg.UnsupportedPath[0], bucketType, "") + setup.SetUpTestDirForTestBucket(&cfg.UnsupportedPath[0]) + + successCode := static_mounting.RunTestsWithConfigFile(&cfg.UnsupportedPath[0], flags, m) + + os.Exit(successCode) +} diff --git a/tools/integration_tests/unsupported_path/unsupported_path_test.go b/tools/integration_tests/unsupported_path/unsupported_path_test.go new file mode 100644 index 0000000000..ceb930cfca --- /dev/null +++ b/tools/integration_tests/unsupported_path/unsupported_path_test.go @@ -0,0 +1,201 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unsupported_path + +import ( + "os" + "path" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +const testDirName = "dirWithUnsupportedPaths" + +// UnsupportedPathSuite is a test suite for operations on directories containing paths that +// are unsupported by the GCSFuse file system (but can exist as objects in GCS). +type UnsupportedPathSuite struct { + suite.Suite + // The local test dir path for the test bucket. + testDir string + // The path in the GCS bucket for this test suite. + bucketPath string +} + +func TestUnsupportedPathSuite(t *testing.T) { + suite.Run(t, new(UnsupportedPathSuite)) +} + +func (s *UnsupportedPathSuite) SetupSuite() { + s.testDir = setup.SetupTestDirectory(DirForUnsupportedPathTests) + s.bucketPath = path.Join(DirForUnsupportedPathTests, testDirName) +} + +func (s *UnsupportedPathSuite) SetupTest() { + s.createTestObjects() +} + +func (s *UnsupportedPathSuite) TearDownTest() { + require.NoError(s.T(), os.RemoveAll(path.Join(s.testDir, testDirName))) +} + +func (s *UnsupportedPathSuite) TearDownSuite() { + require.NoError(s.T(), os.RemoveAll(s.testDir)) +} + +// createTestObjects populates the GCS bucket with objects having supported and unsupported names. +func (s *UnsupportedPathSuite) createTestObjects() { + s.T().Helper() + // Define objects with names that contain characters or sequences not supported by POSIX file systems. + unsupportedObjects := []string{ + s.bucketPath + "//unsupported_file1.txt", // Contains "//" (double slash) + s.bucketPath + "/./unsupported_file2.txt", // Contains "/./" (dot segment) + s.bucketPath + "/../unsupported_file3.txt", // Contains ".." (dot-dot segment) + s.bucketPath + "/.", // Is "." + s.bucketPath + "/..", // Is ".." + // Nested unsupported paths + s.bucketPath + "/unsupportedDir1//file2.txt", + s.bucketPath + "/unsupportedDir1//unsupportedDir2//file3.txt", + } + + for _, obj := range unsupportedObjects { + if bucketType == setup.ZonalBucket { + require.NoError(s.T(), client.CreateFinalizedObjectOnGCS(ctx, storageClient, obj, "unsupported content")) + } else { + require.NoError(s.T(), client.CreateObjectOnGCS(ctx, storageClient, obj, "unsupported content")) + } + } + + // Create objects with names that are supported and should be visible. + supportedFile := path.Join(s.bucketPath, "supported_file.txt") + supportedDir := path.Join(s.bucketPath, "supported_dir") + "/" + + if bucketType == setup.ZonalBucket { + require.NoError(s.T(), client.CreateFinalizedObjectOnGCS(ctx, storageClient, supportedFile, "content")) + require.NoError(s.T(), client.CreateFinalizedObjectOnGCS(ctx, storageClient, supportedDir, "")) + } else { + require.NoError(s.T(), client.CreateObjectOnGCS(ctx, storageClient, supportedFile, "content")) + require.NoError(s.T(), client.CreateObjectOnGCS(ctx, storageClient, supportedDir, "")) + } +} + +// --- Test Cases --- +// The core hypothesis is that GCSFuse will hide/ignore objects whose names +// contain unsupported path segments during file system operations. + +// TestListDirWithUnsupportedPaths verifies that os.ReadDir only returns supported objects. +func (s *UnsupportedPathSuite) TestListDirWithUnsupportedPaths() { + localPath := path.Join(s.testDir, testDirName) + + entries, err := os.ReadDir(localPath) + + require.NoError(s.T(), err, "os.ReadDir should succeed on a mounted directory.") + // Expect only the supported file and supported directory. + expectedEntriesCount := 3 + assert.Len(s.T(), entries, expectedEntriesCount, "The number of entries should only match supported objects.") + entryNames := make([]string, len(entries)) + for i, entry := range entries { + entryNames[i] = entry.Name() + } + expectedNames := []string{"supported_dir", "supported_file.txt", "unsupportedDir1"} + s.Assert().ElementsMatch(expectedNames, entryNames, "Only supported object names should be returned.") +} + +// TestCopyDirWithUnsupportedPaths verifies that operations.CopyDir only copies supported objects. +func (s *UnsupportedPathSuite) TestCopyDirWithUnsupportedPaths() { + destDirName := "copiedDir" + sourceLocalPath := path.Join(s.testDir, testDirName) + destLocalPath := path.Join(s.testDir, destDirName) + defer setup.CleanUpDir(destLocalPath) + destBucketPath := path.Join(DirForUnsupportedPathTests, destDirName) + expectedObjectNames := []string{ + path.Join(destBucketPath, "supported_file.txt"), + path.Join(destBucketPath, "supported_dir") + "/", + destBucketPath + "/unsupportedDir1/", + } + + err := operations.CopyDir(sourceLocalPath, destLocalPath) + + require.NoError(s.T(), err, "CopyDir operation should succeed.") + // List the contents of the destination directory in the GCS bucket (to check actual objects created). + entries, err := client.ListDirectory(ctx, storageClient, setup.TestBucket(), destBucketPath) + require.NoError(s.T(), err, "Listing the destination directory in GCS should succeed.") + // Verify that only supported objects were copied (3 objects). + assert.Len(s.T(), entries, 3, "The number of copied objects should only match supported objects.") + s.Assert().ElementsMatch(expectedObjectNames, entries, "The copied object names must match the expected supported names.") +} + +// TestRenameDirWithUnsupportedPaths verifies that operations.RenameDir successfully moves the directory +// and its contents, including the unsupported objects which exist in GCS. +func (s *UnsupportedPathSuite) TestRenameDirWithUnsupportedPaths() { + destDirName := "renamedDir" + sourceLocalPath := path.Join(s.testDir, testDirName) + destLocalPath := path.Join(s.testDir, destDirName) + destBucketPath := path.Join(DirForUnsupportedPathTests, destDirName) + defer setup.CleanUpDir(destLocalPath) + // In a rename operation, all GCS objects (supported and unsupported) are moved. + // The unsupported objects are expected to exist at the new location, though + // they will likely still be hidden from the fuse mount due to the unsupported + // path components. + expectedObjectNames := []string{ + // All objects are expected to be at the new destination path. + path.Join(destBucketPath, "supported_file.txt"), + path.Join(destBucketPath, "supported_dir") + "/", + destBucketPath + "//unsupported_file1.txt", + destBucketPath + "/./unsupported_file2.txt", + destBucketPath + "/../unsupported_file3.txt", + destBucketPath + "/../", + destBucketPath + "/./", + destBucketPath + "/.", + destBucketPath + "/..", + destBucketPath + "//", + destBucketPath + "/unsupportedDir1/", + destBucketPath + "/unsupportedDir1//file2.txt", + destBucketPath + "/unsupportedDir1//unsupportedDir2//file3.txt", + } + + err := operations.RenameDir(sourceLocalPath, destLocalPath) + + require.NoError(s.T(), err, "RenameDir operation should succeed.") + // List all objects under the destination prefix recursively to verify the move. + entries, err := client.ListDirectory(ctx, storageClient, setup.TestBucket(), destBucketPath) + require.NoError(s.T(), err, "Listing the destination directory in GCS should succeed.") + // Verify that ALL GCS objects (supported and unsupported) were moved. + assert.Len(s.T(), entries, 13, "The number of renamed objects should match all original GCS objects.") + s.Assert().ElementsMatch(expectedObjectNames, entries, "All GCS objects, including unsupported ones, should be moved.") +} + +// TestDeleteDirWithUnsupportedPaths verifies that os.RemoveAll successfully deletes the mounted directory +// and all corresponding GCS objects, including the unsupported ones. +func (s *UnsupportedPathSuite) TestDeleteDirWithUnsupportedPaths() { + localPath := path.Join(s.testDir, testDirName) + + err := os.RemoveAll(localPath) + + require.NoError(s.T(), err, "os.RemoveAll operation should succeed.") + // Verify the directory no longer exists in the mounted file system. + _, err = os.Stat(localPath) + require.Error(s.T(), err, "os.Stat on the removed directory should fail.") + assert.True(s.T(), os.IsNotExist(err), "The error should indicate 'no such file or directory'.") + // EXTRA: Verify all objects are deleted in GCS as well. + entries, err := client.ListDirectory(ctx, storageClient, setup.TestBucket(), s.bucketPath) + require.NoError(s.T(), err, "Listing the deleted directory prefix in GCS should succeed.") + assert.Empty(s.T(), entries, "The GCS directory prefix should be empty after RemoveAll.") +} diff --git a/tools/integration_tests/util/benchmark_setup/benchmark_setup_test.go b/tools/integration_tests/util/benchmark_setup/benchmark_setup_test.go index 0bdf7ef8fa..e15eacb0e0 100644 --- a/tools/integration_tests/util/benchmark_setup/benchmark_setup_test.go +++ b/tools/integration_tests/util/benchmark_setup/benchmark_setup_test.go @@ -18,7 +18,7 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/benchmark_setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/benchmark_setup" "github.com/stretchr/testify/assert" ) diff --git a/tools/integration_tests/util/client/control_client.go b/tools/integration_tests/util/client/control_client.go index fc7f6194e9..50199edf1f 100644 --- a/tools/integration_tests/util/client/control_client.go +++ b/tools/integration_tests/util/client/control_client.go @@ -22,12 +22,15 @@ import ( "context" "fmt" "log" + "path" "strings" "time" control "cloud.google.com/go/storage/control/apiv2" "cloud.google.com/go/storage/control/apiv2/controlpb" "github.com/googleapis/gax-go/v2" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" "google.golang.org/grpc/codes" ) @@ -102,3 +105,15 @@ func CreateManagedFoldersInBucket(ctx context.Context, client *control.StorageCo log.Fatalf("Error while creating managed folder: %v", err) } } + +func CreateFolderInBucket(ctx context.Context, client *control.StorageControlClient, folderPath string) (*controlpb.Folder, error) { + bucket, rootFolder := setup.GetBucketAndObjectBasedOnTypeOfMount("") + req := &controlpb.CreateFolderRequest{ + Parent: fmt.Sprintf(storage.FullBucketPathHNS, bucket), + FolderId: path.Join(rootFolder, folderPath), + } + + f, err := client.CreateFolder(ctx, req) + + return f, err +} diff --git a/tools/integration_tests/util/client/gcs_helper.go b/tools/integration_tests/util/client/gcs_helper.go index 55fd11206e..0601ccd0fb 100644 --- a/tools/integration_tests/util/client/gcs_helper.go +++ b/tools/integration_tests/util/client/gcs_helper.go @@ -22,28 +22,29 @@ import ( "path" "strings" "testing" + "time" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) const ( - FileName1 = "foo1" - FileName2 = "foo2" - FileName3 = "foo3" - ExplicitDirName = "explicit" - ExplicitFileName1 = "explicitFile1" - ImplicitDirName = "implicit" - ImplicitFileName1 = "implicitFile1" - FileContents = "testString" - SizeOfFileContents = 10 - GCSFileContent = "GCSteststring" - GCSFileSize = 13 - FilePerms = 0644 - SizeTruncate = 5 - NewFileName = "newName" - NewDirName = "newDirName" + FileName1 = "foo1" + FileName2 = "foo2" + ExplicitDirName = "explicit" + ExplicitFileName1 = "explicitFile1" + ImplicitDirName = "implicit" + ImplicitFileName1 = "implicitFile1" + FileContents = "testString" + SizeOfFileContents = 10 + GCSFileContent = "GCSteststring" + GCSFileSize = 13 + FilePerms = 0644 + SmallerSizeTruncate = 5 + NewDirName = "newDirName" ) func CreateImplicitDir(ctx context.Context, storageClient *storage.Client, @@ -58,16 +59,16 @@ func CreateImplicitDir(ctx context.Context, storageClient *storage.Client, } } -func ValidateObjectNotFoundErrOnGCS(ctx context.Context, storageClient *storage.Client, - testDirName string, fileName string, t *testing.T) { - _, err := ReadObjectFromGCS(ctx, storageClient, path.Join(testDirName, fileName)) +func ValidateObjectNotFoundErrOnGCS(ctx context.Context, storageClient *storage.Client, testDirName string, fileName string, t *testing.T) { + t.Helper() + _, err := StatObject(ctx, storageClient, path.Join(testDirName, fileName)) if err == nil || !strings.Contains(err.Error(), "storage: object doesn't exist") { t.Fatalf("Incorrect error returned from GCS for file %s: %v", fileName, err) } } -func ValidateObjectContentsFromGCS(ctx context.Context, storageClient *storage.Client, - testDirName string, fileName string, expectedContent string, t *testing.T) { +func ValidateObjectContentsFromGCS(ctx context.Context, storageClient *storage.Client, testDirName string, fileName string, expectedContent string, t *testing.T) { + t.Helper() gotContent, err := ReadObjectFromGCS(ctx, storageClient, path.Join(testDirName, fileName)) if err != nil { t.Fatalf("Error while reading file from GCS, Err: %v", err) @@ -95,12 +96,12 @@ func ValidateObjectChunkFromGCS(ctx context.Context, storageClient *storage.Clie func CloseFileAndValidateContentFromGCS(ctx context.Context, storageClient *storage.Client, fh *os.File, testDirName, fileName, content string, t *testing.T) { - operations.CloseFileShouldNotThrowError(fh, t) + operations.CloseFileShouldNotThrowError(t, fh) ValidateObjectContentsFromGCS(ctx, storageClient, testDirName, fileName, content, t) } -func CreateLocalFileInTestDir(ctx context.Context, storageClient *storage.Client, - testDirPath, fileName string, t *testing.T) (string, *os.File) { +func CreateLocalFileInTestDir(ctx context.Context, storageClient *storage.Client, testDirPath, fileName string, t *testing.T) (string, *os.File) { + t.Helper() filePath := path.Join(testDirPath, fileName) fh := operations.CreateFile(filePath, FilePerms, t) testDirName := GetDirName(testDirPath) @@ -122,6 +123,15 @@ func CreateObjectInGCSTestDir(ctx context.Context, storageClient *storage.Client } } +func CreateFinalizedObjectInGCSTestDir(ctx context.Context, storageClient *storage.Client, + testDirName, fileName, content string, t *testing.T) { + objectName := path.Join(testDirName, fileName) + err := CreateFinalizedObjectOnGCS(ctx, storageClient, objectName, content) + if err != nil { + t.Fatalf("Create Object %s on GCS: %v.", objectName, err) + } +} + func SetupFileInTestDirectory(ctx context.Context, storageClient *storage.Client, testDirName, testFileName string, size int64, t *testing.T) { randomData, err := operations.GenerateRandomData(size) @@ -146,8 +156,13 @@ func SetupTestDirectory(ctx context.Context, storageClient *storage.Client, test return testDirPath } +func SetupUniqueTestDirectory(ctx context.Context, storageClient *storage.Client, testDirPrefix string) string { + testDirName := testDirPrefix + "_" + setup.GenerateRandomString(5) + return SetupTestDirectory(ctx, storageClient, testDirName) +} + func CreateNFilesInDir(ctx context.Context, storageClient *storage.Client, numFiles int, fileName string, fileSize int64, dirName string, t *testing.T) (fileNames []string) { - for i := 0; i < numFiles; i++ { + for range numFiles { testFileName := fileName + setup.GenerateRandomString(4) fileNames = append(fileNames, testFileName) SetupFileInTestDirectory(ctx, storageClient, dirName, testFileName, fileSize, t) @@ -162,3 +177,61 @@ func GetCRCFromGCS(objectPath string, ctx context.Context, storageClient *storag } return attr.CRC32C, nil } + +// This method creates an Unfinalized Object with given content using appendable writer +// and performs a flush with Zonal Bucket Flush API for content to be available for read +// and returns the writer. +func CreateUnfinalizedObject(ctx context.Context, t *testing.T, client *storage.Client, object, content string) *storage.Writer { + writer, err := NewWriterWithPreconditionsSet(ctx, client, object, storage.Conditions{}) + require.NoError(t, err) + + bytesWritten, err := writer.Write([]byte(content)) + require.NoError(t, err) + assert.EqualValues(t, len(content), bytesWritten) + + err = writer.Close() + require.NoError(t, err) + // Sleep for a second after close to get correct size on stat. + time.Sleep(time.Second) + return writer +} + +// setRequesterPays sets requester-pays flag to given boolean for the given bucket. +func setRequesterPays(storageClient *storage.Client, ctx context.Context, bucketName string, enable bool) error { + bucket := getBucketHandle(storageClient, bucketName) + bucketAttrsToUpdate := storage.BucketAttrsToUpdate{ + RequesterPays: enable, + } + if _, err := bucket.Update(ctx, bucketAttrsToUpdate); err != nil { + return fmt.Errorf("failed to set requester-pays to %v for bucket %s: %w", enable, bucketName, err) + } + log.Printf("requester-pays set to %v for bucket %v\n", enable, bucketName) + return nil +} + +// MustEnableRequesterPays enables requester-pays for the given bucket if not already enabled. +// Returns true if it actually enabled it, false if it was already enabled. +func MustEnableRequesterPays(storageClient *storage.Client, ctx context.Context, bucketName string) bool { + bucket := getBucketHandle(storageClient, bucketName) + attrs, err := bucket.Attrs(ctx) + if err != nil { + panic(fmt.Sprintf("MustEnableRequesterPays: failed to get bucket attrs for %s: %v", bucketName, err)) + } + + if attrs.RequesterPays { + log.Printf("Requester pays is already enabled for bucket %s. Skipping enable.", bucketName) + return false + } + + if err := setRequesterPays(storageClient, ctx, bucketName, true); err != nil { + panic(fmt.Sprintf("MustEnableRequesterPays: failed to enable requester-pays for bucket %s: %v", bucketName, err)) + } + return true +} + +// MustDisableRequesterPays disables requester-pays for the given bucket and panics if it fails. +func MustDisableRequesterPays(storageClient *storage.Client, ctx context.Context, bucketName string) { + if err := setRequesterPays(storageClient, ctx, bucketName, false); err != nil { + panic(fmt.Sprintf("MustDisableRequesterPays: failed to disable requester-pays for bucket %s: %v", bucketName, err)) + } +} diff --git a/tools/integration_tests/util/client/storage_client.go b/tools/integration_tests/util/client/storage_client.go index 3b4ae9426e..1989496928 100644 --- a/tools/integration_tests/util/client/storage_client.go +++ b/tools/integration_tests/util/client/storage_client.go @@ -16,26 +16,94 @@ package client import ( "context" + "crypto/tls" + "errors" "fmt" "io" "log" + "net/http" "os" + "path" + "path/filepath" "reflect" + "runtime" + "strings" + "sync" "testing" "time" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "cloud.google.com/go/storage" + "cloud.google.com/go/storage/experimental" "github.com/googleapis/gax-go/v2" - "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" "golang.org/x/oauth2" "golang.org/x/oauth2/google" + "golang.org/x/sync/errgroup" "google.golang.org/api/iterator" "google.golang.org/api/option" storagev1 "google.golang.org/api/storage/v1" ) +const maxStorageClientRetryAttempts = 10 +const networkUnreachableError = "network is unreachable" + +func ShouldRetryForTest(err error) (b bool) { + b = storageutil.ShouldRetry(err) + if b { + log.Printf("Retrying for the error: %v", err) + return + } + + // Convert err.Error() to lowercase to make the check case-insensitive + if err != nil && strings.Contains(strings.ToLower(err.Error()), networkUnreachableError) { + b = true + log.Printf("Retrying for 'network is unreachable' error: %v", err) + return + } + return +} + +func CreateHttp1StorageClient(ctx context.Context) (*storage.Client, error) { + var tokenSrc oauth2.TokenSource + var err error + + if kf := setup.KeyFile(); kf != "" { + tokenSrc, err = getTokenSrc(kf) + } else { + tokenSrc, err = google.DefaultTokenSource(ctx, storagev1.DevstorageFullControlScope) + } + + if err != nil { + return nil, fmt.Errorf("unable to create token source: %w", err) + } + + httpClient := &http.Client{ + Transport: &oauth2.Transport{ + Base: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + MaxIdleConns: 0, // No connection limit. + MaxIdleConnsPerHost: 100, + TLSNextProto: make(map[string]func(authority string, c *tls.Conn) http.RoundTripper), // Disables HTTP/2 transport. + }, + Source: tokenSrc, + }, + Timeout: 0, // No timeout. + } + + return storage.NewClient(ctx, option.WithHTTPClient(httpClient)) +} + +func getBucketHandle(client *storage.Client, bucketName string) *storage.BucketHandle { + b := client.Bucket(bucketName) + if bp := setup.BillingProject(); bp != "" { + b = b.UserProject(bp) + } + return b +} + func CreateStorageClient(ctx context.Context) (client *storage.Client, err error) { // Create new storage client. if setup.TestOnTPCEndPoint() { @@ -47,7 +115,20 @@ func CreateStorageClient(ctx context.Context) (client *storage.Client, err error } client, err = storage.NewClient(ctx, option.WithEndpoint("storage.apis-tpczero.goog:443"), option.WithTokenSource(ts)) } else { - client, err = storage.NewClient(ctx) + if setup.IsZonalBucketRun() { + var opts []option.ClientOption + opts = append(opts, experimental.WithGRPCBidiReads()) + if kf := setup.KeyFile(); kf != "" { + ts, err := getTokenSrc(kf) + if err != nil { + return nil, err + } + opts = append(opts, option.WithTokenSource(ts)) + } + client, err = storage.NewGRPCClient(ctx, opts...) + } else { + client, err = CreateHttp1StorageClient(ctx) + } } if err != nil { return nil, fmt.Errorf("storage.NewClient: %w", err) @@ -62,8 +143,8 @@ func CreateStorageClient(ctx context.Context) (client *storage.Client, err error Multiplier: 2, }), storage.WithPolicy(storage.RetryAlways), - storage.WithErrorFunc(storageutil.ShouldRetry), - storage.WithMaxAttempts(5)) + storage.WithErrorFunc(ShouldRetryForTest), + storage.WithMaxAttempts(maxStorageClientRetryAttempts)) return client, nil } @@ -85,8 +166,11 @@ func getTokenSrc(path string) (tokenSrc oauth2.TokenSource, err error) { func ReadObjectFromGCS(ctx context.Context, client *storage.Client, object string) (string, error) { bucket, object := setup.GetBucketAndObjectBasedOnTypeOfMount(object) + if client == nil { + return "", fmt.Errorf("client is nil") + } // Create storage reader to read from GCS. - rc, err := client.Bucket(bucket).Object(object).NewReader(ctx) + rc, err := getBucketHandle(client, bucket).Object(object).NewReader(ctx) if err != nil { return "", fmt.Errorf("Object(%q).NewReader: %w", object, err) } @@ -106,7 +190,7 @@ func ReadChunkFromGCS(ctx context.Context, client *storage.Client, object string bucket, object := setup.GetBucketAndObjectBasedOnTypeOfMount(object) // Create storage reader to read from GCS. - rc, err := client.Bucket(bucket).Object(object).NewRangeReader(ctx, offset, size) + rc, err := getBucketHandle(client, bucket).Object(object).NewRangeReader(ctx, offset, size) if err != nil { return "", fmt.Errorf("Object(%q).NewReader: %w", object, err) } @@ -120,22 +204,52 @@ func ReadChunkFromGCS(ctx context.Context, client *storage.Client, object string return string(content), nil } +// NewWriter is a wrapper over storage.NewWriter which +// extends support to zonal buckets. +func NewWriter(ctx context.Context, o *storage.ObjectHandle, client *storage.Client) (wc *storage.Writer, err error) { + wc = o.NewWriter(ctx) + wc.FinalizeOnClose = true + + // Changes specific to zonal bucket + var attrs *storage.BucketAttrs + attrs, err = getBucketHandle(client, o.BucketName()).Attrs(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get attributes for bucket %q: %w", o.BucketName(), err) + } + if attrs.StorageClass == "RAPID" { + if setup.IsZonalBucketRun() { + // Zonal bucket writers require append-flag to be set. + wc.Append = true + // Zonal buckets with rapid appends should not finalize on close. + wc.FinalizeOnClose = false + } else { + return nil, fmt.Errorf("found zonal bucket %q in non-zonal e2e test run (--zonal=false)", o.BucketName()) + } + } + + return +} + func WriteToObject(ctx context.Context, client *storage.Client, object, content string, precondition storage.Conditions) error { bucket, object := setup.GetBucketAndObjectBasedOnTypeOfMount(object) - o := client.Bucket(bucket).Object(object) + o := getBucketHandle(client, bucket).Object(object) if !reflect.DeepEqual(precondition, storage.Conditions{}) { o = o.If(precondition) } // Upload an object with storage.Writer. - wc := o.NewWriter(ctx) + wc, err := NewWriter(ctx, o, client) + if err != nil { + return fmt.Errorf("Failed to open writer for object %q: %w", object, err) + } if _, err := io.WriteString(wc, content); err != nil { - return fmt.Errorf("io.WriteSTring: %w", err) + return fmt.Errorf("io.WriteString failed for object %q: %w", object, err) } if err := wc.Close(); err != nil { - return fmt.Errorf("Writer.Close: %w", err) + return fmt.Errorf("Writer.Close failed for object %q: %w", object, err) } + operations.WaitForSizeUpdate(setup.IsZonalBucketRun(), operations.WaitDurationAfterCloseZB) return nil } @@ -145,6 +259,24 @@ func CreateObjectOnGCS(ctx context.Context, client *storage.Client, object, cont return WriteToObject(ctx, client, object, content, storage.Conditions{DoesNotExist: true}) } +func CreateFinalizedObjectOnGCS(ctx context.Context, client *storage.Client, object, content string) error { + bucket, object := setup.GetBucketAndObjectBasedOnTypeOfMount(object) + o := getBucketHandle(client, bucket).Object(object) + + // Upload an object with storage.Writer with finalizeOnClose=true + wc := o.NewWriter(ctx) + wc.Append = true + wc.FinalizeOnClose = true + if _, err := io.WriteString(wc, content); err != nil { + return fmt.Errorf("io.WriteString failed for object %q: %w", object, err) + } + if err := wc.Close(); err != nil { + return fmt.Errorf("Writer.Close failed for object %q: %w", object, err) + } + operations.WaitForSizeUpdate(setup.IsZonalBucketRun(), operations.WaitDurationAfterCloseZB) + return nil +} + // CreateStorageClientWithCancel creates a new storage client with a cancelable context and returns a function that can be used to cancel the client's operations func CreateStorageClientWithCancel(ctx *context.Context, storageClient **storage.Client) func() error { var err error @@ -179,7 +311,7 @@ func DownloadObjectFromGCS(gcsFile string, destFileName string, t *testing.T) er } }() f := operations.CreateFile(destFileName, setup.FilePermission_0600, t) - defer operations.CloseFile(f) + defer operations.CloseFileShouldNotThrowError(t, f) rc, err := storageClient.Bucket(bucket).Object(gcsFile).NewReader(ctx) if err != nil { @@ -198,7 +330,7 @@ func DeleteObjectOnGCS(ctx context.Context, client *storage.Client, objectName s bucket, _ := setup.GetBucketAndObjectBasedOnTypeOfMount("") // Get handle to the object - object := client.Bucket(bucket).Object(objectName) + object := getBucketHandle(client, bucket).Object(objectName) // Delete the object err := object.Delete(ctx) @@ -208,46 +340,86 @@ func DeleteObjectOnGCS(ctx context.Context, client *storage.Client, objectName s return nil } +// DeleteAllObjectsWithPrefix deletes all objects with the specified prefix in a GCS bucket. +// It concurrently iterates through objects with the given prefix and deletes them using multiple goroutines, +// leveraging the number of CPU cores for optimal performance. func DeleteAllObjectsWithPrefix(ctx context.Context, client *storage.Client, prefix string) error { bucket, _ := setup.GetBucketAndObjectBasedOnTypeOfMount("") // Get an object iterator query := &storage.Query{Prefix: prefix} - objectItr := client.Bucket(bucket).Objects(ctx, query) + objectItr := getBucketHandle(client, bucket).Objects(ctx, query) + + // Create a buffered channel to receive errors from goroutines + errChan := make(chan error, 100) - // Iterate through objects with the specified prefix and delete them + // Determine the number of concurrent goroutines using CPU cores + numCores := max(16, runtime.NumCPU()) + sem := make(chan struct{}, numCores) // Semaphore to limit concurrency + + var wg sync.WaitGroup + + // Iterate through objects with the specified prefix for { attrs, err := objectItr.Next() if err == iterator.Done { break } - if err := DeleteObjectOnGCS(ctx, client, attrs.Name); err != nil { - return err + if err != nil { + return fmt.Errorf("error iterating through objects: %w", err) } + + wg.Add(1) + sem <- struct{}{} // Acquire a semaphore slot + go func(attrs *storage.ObjectAttrs) { + defer func() { + <-sem // Release the semaphore slot + wg.Done() + }() + if err := DeleteObjectOnGCS(ctx, client, attrs.Name); err != nil { + errChan <- fmt.Errorf("error deleting object %s: %w", attrs.Name, err) + } + }(attrs) } - return nil + + wg.Wait() + close(errChan) + + var errs []error + for err := range errChan { + errs = append(errs, err) + } + + return errors.Join(errs...) } func StatObject(ctx context.Context, client *storage.Client, object string) (*storage.ObjectAttrs, error) { bucket, object := setup.GetBucketAndObjectBasedOnTypeOfMount(object) - attrs, err := client.Bucket(bucket).Object(object).Attrs(ctx) + attrs, err := getBucketHandle(client, bucket).Object(object).Attrs(ctx) if err != nil { return nil, err } return attrs, nil } -// UploadGcsObject uploads a local file to a specified GCS bucket and object. +// UploadGcsObjectWithPreconditions uploads a local file to a specified GCS bucket and object with given preconditions. // Handles gzip compression if requested. -func UploadGcsObject(ctx context.Context, client *storage.Client, localPath, bucketName, objectName string, uploadGzipEncoded bool) error { +func UploadGcsObjectWithPreconditions(ctx context.Context, client *storage.Client, localPath, bucketName, objectName string, uploadGzipEncoded bool, preconditions *storage.Conditions) error { // Create a writer to upload the object. - obj := client.Bucket(bucketName).Object(objectName) - w := obj.NewWriter(ctx) + obj := getBucketHandle(client, bucketName).Object(objectName) + if preconditions != nil { + obj = obj.If(*preconditions) + } + w, err := NewWriter(ctx, obj, client) + if err != nil { + return fmt.Errorf("failed to open writer for GCS object gs://%s/%s: %w", bucketName, objectName, err) + } defer func() { if err := w.Close(); err != nil { log.Printf("Failed to close GCS object gs://%s/%s: %v", bucketName, objectName, err) } + operations.WaitForSizeUpdate(setup.IsZonalBucketRun(), operations.WaitDurationAfterCloseZB) }() filePathToUpload := localPath @@ -305,9 +477,271 @@ func ClearCacheControlOnGcsObject(ctx context.Context, client *storage.Client, o return nil } +// UploadGcsObject uploads a local file to a specified GCS bucket and object without any preconditions. +// Handles gzip compression if requested. +func UploadGcsObject(ctx context.Context, client *storage.Client, localPath, bucketName, objectName string, uploadGzipEncoded bool) error { + return UploadGcsObjectWithPreconditions(ctx, client, localPath, bucketName, objectName, uploadGzipEncoded, nil) +} + func CopyFileInBucket(ctx context.Context, storageClient *storage.Client, srcfilePath, destFilePath, bucket string) { err := UploadGcsObject(ctx, storageClient, srcfilePath, bucket, destFilePath, false) if err != nil { - log.Fatalf("Error while copying file in bucket: %v", err) + log.Fatalf("Error while copying file %q to GCS object \"gs://%s/%s\" : %v", srcfilePath, bucket, destFilePath, err) + } +} + +func CopyFileInBucketWithPreconditions(ctx context.Context, storageClient *storage.Client, srcfilePath, destFilePath, bucket string, preconditions *storage.Conditions) { + err := UploadGcsObjectWithPreconditions(ctx, storageClient, srcfilePath, bucket, destFilePath, false, preconditions) + if err != nil { + log.Fatalf("Error while copying file %q to GCS object \"gs://%s/%s\" : %v", srcfilePath, bucket, destFilePath, err) + } +} + +func DeleteBucket(ctx context.Context, client *storage.Client, bucketName string) error { + bucket := getBucketHandle(client, bucketName) + + // Iterate through objects and delete them + query := &storage.Query{} + it := bucket.Objects(ctx, query) + for { + objAttrs, err := it.Next() + if err == iterator.Done { + break // No more objects + } + if err != nil { + log.Fatalf("Error iterating through objects: %v", err) + } + + obj := bucket.Object(objAttrs.Name) + err = obj.Delete(ctx) + if err != nil { + log.Fatalf("Failed to delete object %s: %v", objAttrs.Name, err) + } + } + + if err := bucket.Delete(ctx); err != nil { + log.Printf("Bucket(%q).Delete: %v", bucketName, err) + return err + } + return nil +} + +func NewWriterWithPreconditionsSet(ctx context.Context, client *storage.Client, object string, precondition storage.Conditions) (*storage.Writer, error) { + bucket, object := setup.GetBucketAndObjectBasedOnTypeOfMount(object) + + o := getBucketHandle(client, bucket).Object(object) + if !reflect.DeepEqual(precondition, storage.Conditions{}) { + o = o.If(precondition) + } + + // Upload an object with storage.Writer. + wc, err := NewWriter(ctx, o, client) + if err != nil { + return nil, fmt.Errorf("failed to open writer for object %q: %w", o.ObjectName(), err) + } + return wc, nil +} + +func AppendableWriter(ctx context.Context, client *storage.Client, object string, gen int64) (*storage.Writer, error) { + bucket, object := setup.GetBucketAndObjectBasedOnTypeOfMount(object) + obj := getBucketHandle(client, bucket).Object(object) + + tw, _, err := obj.Generation(gen).NewWriterFromAppendableObject(ctx, &storage.AppendableWriterOpts{}) + return tw, err +} + +// CreateGcsDir creates a GCS object with trailing slash "/" to simulate a directory. +func CreateGcsDir(ctx context.Context, client *storage.Client, dirName, bucketName, objectName string) error { + // Combine objectName and dirName to form the full GCS object path + fullObjectPath := path.Join(objectName, dirName) + + // Ensure fullObjectPath ends with a "/" + if !strings.HasSuffix(fullObjectPath, "/") { + fullObjectPath += "/" + } + + // Create an empty object with the directory path + err := WriteToObject(ctx, client, fullObjectPath, "", storage.Conditions{}) + if err != nil { + return fmt.Errorf("failed to create GCS directory object %q in bucket %q: %w", fullObjectPath, bucketName, err) + } + + return nil +} + +func uploadGcsObjectWithPreconditionsWithoutIntermediateDelays(ctx context.Context, client *storage.Client, localPath, bucketName, objectName string, uploadGzipEncoded bool, preconditions *storage.Conditions) error { + // Create a writer to upload the object. + obj := getBucketHandle(client, bucketName).Object(objectName) + if preconditions != nil { + obj = obj.If(*preconditions) + } + w, err := NewWriter(ctx, obj, client) + if err != nil { + return fmt.Errorf("failed to open writer for GCS object gs://%s/%s: %w", bucketName, objectName, err) } + defer func() { + if err := w.Close(); err != nil { + log.Printf("Failed to close GCS object gs://%s/%s: %v", bucketName, objectName, err) + } + }() + + filePathToUpload := localPath + // Set content encoding if gzip compression is needed. + if uploadGzipEncoded { + data, err := os.ReadFile(localPath) + if err != nil { + return err + } + + content := string(data) + if filePathToUpload, err = operations.CreateLocalTempFile(content, true); err != nil { + return fmt.Errorf("failed to create local gzip file from %s for upload to bucket: %w", localPath, err) + } + defer func() { + if removeErr := os.Remove(filePathToUpload); removeErr != nil { + log.Printf("Error removing temporary gzip file %s: %v", filePathToUpload, removeErr) + } + }() + } + + // Open the local file for reading. + f, err := operations.OpenFileAsReadonly(filePathToUpload) + if err != nil { + return fmt.Errorf("failed to open local file %s: %w", filePathToUpload, err) + } + // Defer the Close() call immediately after the error check. + defer func() { + if closeErr := f.Close(); closeErr != nil { + log.Printf("Warning: error closing file %s: %v", filePathToUpload, closeErr) + } + }() + + // Copy the file contents to the object writer. + if _, err := io.Copy(w, f); err != nil { + return fmt.Errorf("failed to copy file %s to gs://%s/%s: %w", localPath, bucketName, objectName, err) + } + return nil +} + +func BatchUploadFilesWithoutIntermediateDelays(ctx context.Context, storageClient *storage.Client, bucketName, dirPathInBucket, srcDir, filesPrefix string) error { + srcFilesFullPathPrefix := filepath.Join(srcDir, filesPrefix) + matches, err := filepath.Glob(srcFilesFullPathPrefix + "*") + if err != nil { + return fmt.Errorf("failed to get files of pattern %s*: %w", srcFilesFullPathPrefix, err) + } + + if len(matches) == 0 { + return nil // No files to upload + } + + group, ctx := errgroup.WithContext(ctx) + + // Limit the number of concurrent uploads to avoid overwhelming resources. + maxConcurrentUploads := max(16, runtime.NumCPU()/2) + group.SetLimit(maxConcurrentUploads) + + for _, match := range matches { + srcLocalFilePath := match + + _, fileName := filepath.Split(match) + if len(fileName) == 0 { + continue + } + + dstGCSObjectPath := filepath.Join(dirPathInBucket, fileName) + + group.Go(func() error { + err := uploadGcsObjectWithPreconditionsWithoutIntermediateDelays(ctx, storageClient, srcLocalFilePath, bucketName, dstGCSObjectPath, false, &storage.Conditions{DoesNotExist: true}) + if err != nil { + return fmt.Errorf("failed to upload %s to gs://%s/%s: %w", srcLocalFilePath, bucketName, dstGCSObjectPath, err) + } + return nil + }) + } + + // Wait for all uploads to complete or for the first error. + if err := group.Wait(); err != nil { + return err + } + + // Wait for metadata updates after object creation. + time.Sleep(time.Second) + return nil +} + +// ListDirectory lists objects in the specified GCS bucket under the given prefix. +// It returns a slice of object names. +func ListDirectory(ctx context.Context, client *storage.Client, bucketName, prefix string) ([]string, error) { + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + bucket := getBucketHandle(client, bucketName) + + var entries []string + var mu sync.Mutex + g, ctx := errgroup.WithContext(ctx) + + // List objects recursively. + g.Go(func() error { + objQuery := &storage.Query{ + Prefix: prefix, + } + it := bucket.Objects(ctx, objQuery) + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return fmt.Errorf("error iterating GCS objects: %w", err) + } + if attrs.Name != prefix { + mu.Lock() + entries = append(entries, attrs.Name) + mu.Unlock() + } + } + return nil + }) + + g.Go(func() error { + folderQuery := &storage.Query{ + Prefix: prefix, + IncludeFoldersAsPrefixes: true, + Delimiter: "/", + } + it := bucket.Objects(ctx, folderQuery) + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return fmt.Errorf("error iterating GCS folders: %w", err) + } + if attrs.Prefix != "" && attrs.Prefix != prefix { + mu.Lock() + entries = append(entries, attrs.Prefix) + mu.Unlock() + } + } + return nil + }) + + if err := g.Wait(); err != nil { + return nil, err + } + + // Using a map to remove duplicates that might arise from listing objects and folders. + seen := make(map[string]struct{}) + var uniqueEntries []string + for _, entry := range entries { + if _, ok := seen[entry]; !ok { + seen[entry] = struct{}{} + uniqueEntries = append(uniqueEntries, entry) + } + } + + return uniqueEntries, nil } diff --git a/tools/integration_tests/util/creds_tests/creds.go b/tools/integration_tests/util/creds_tests/creds.go index cdac923280..50c7014b93 100644 --- a/tools/integration_tests/util/creds_tests/creds.go +++ b/tools/integration_tests/util/creds_tests/creds.go @@ -19,10 +19,8 @@ package creds_tests import ( "context" "fmt" - "io" "log" "os" - "path" "slices" "strings" "testing" @@ -33,9 +31,9 @@ import ( secretmanager "cloud.google.com/go/secretmanager/apiv1" "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" ) const NameOfServiceAccount = "creds-integration-tests" @@ -43,53 +41,73 @@ const CredentialsSecretName = "gcsfuse-integration-tests" var WhitelistedGcpProjects = []string{"gcs-fuse-test", "gcs-fuse-test-ml"} -func CreateCredentials(ctx context.Context) (serviceAccount, localKeyFilePath string) { - log.Println("Running credentials tests...") - +func projectID(ctx context.Context) string { // Fetching project-id to get service account id. - id, err := metadata.ProjectID() + id, err := metadata.ProjectIDWithContext(ctx) if err != nil { setup.LogAndExit(fmt.Sprintf("Error in fetching project id: %v", err)) } + if strings.Contains(id, "cloudtop") { + // In cloudtop environments, well known path is used for auth. So explicitly set the project as whitelisted. + id = WhitelistedGcpProjects[0] + } + // return if active GCP project is not in whitelisted gcp projects if !slices.Contains(WhitelistedGcpProjects, id) { log.Printf("The active GCP project is not one of: %s. So the credentials test will not run.", strings.Join(WhitelistedGcpProjects, ", ")) } + return id +} - // Service account id format is name@project-id.iam.gserviceaccount.com - serviceAccount = NameOfServiceAccount + "@" + id + ".iam.gserviceaccount.com" +func CreateCredentials(ctx context.Context) (serviceAccount, localKeyFilePath string) { + log.Println("Running credentials tests...") + return CreateCredentialsForSA(ctx, NameOfServiceAccount, CredentialsSecretName) +} + +func CreateCredentialsForSA(ctx context.Context, serviceAccountName, saCredentialsSecretName string) (serviceAccountEmail, localKeyFilePath string) { + log.Printf("Creating credentials for %s...", serviceAccountName) + + projID := projectID(ctx) - localKeyFilePath = path.Join(os.Getenv("HOME"), "creds.json") + // Service account id format is name@project-id.iam.gserviceaccount.com + serviceAccountEmail = serviceAccountName + "@" + projID + ".iam.gserviceaccount.com" - // Download credentials - client, err := secretmanager.NewClient(ctx) + // Create secretmanager client to download service account credential file. + smClient, err := secretmanager.NewClient(ctx) if err != nil { setup.LogAndExit(fmt.Sprintf("Failed to create secret manager client: %v", err)) } - defer client.Close() + defer func() { + if err := smClient.Close(); err != nil { + log.Printf("Failed to close secret manager client: %v", err) + } + }() req := &secretmanagerpb.AccessSecretVersionRequest{ - Name: fmt.Sprintf("projects/%s/secrets/%s/versions/latest", id, CredentialsSecretName), + Name: fmt.Sprintf("projects/%s/secrets/%s/versions/latest", projID, saCredentialsSecretName), } - creds, err := client.AccessSecretVersion(ctx, req) + secretVersion, err := smClient.AccessSecretVersion(ctx, req) if err != nil { setup.LogAndExit(fmt.Sprintf("Error while fetching key file %v", err)) } // Create and write creds to local file. - file, err := os.Create(localKeyFilePath) + keyFile, err := os.CreateTemp("", "creds-*.json") if err != nil { - setup.LogAndExit(fmt.Sprintf("Error while creating credentials file %v", err)) + setup.LogAndExit(fmt.Sprintf("Error while creating temp credentials file %v", err)) } - _, err = io.Writer.Write(file, creds.Payload.Data) + localKeyFilePath = keyFile.Name() + _, err = keyFile.Write(secretVersion.Payload.Data) if err != nil { setup.LogAndExit(fmt.Sprintf("Error while writing credentials to local file %v", err)) } - operations.CloseFile(file) + if err := keyFile.Close(); err != nil { + log.Printf("Failed to close key file: %v", err) + } return } -func ApplyPermissionToServiceAccount(ctx context.Context, storageClient *storage.Client, serviceAccount, permission, bucket string) { +func ApplyRoleToServiceAccountOnBucket(ctx context.Context, storageClient *storage.Client, serviceAccount, roleName, bucket string) { // Provide permission to service account for testing. bucketHandle := storageClient.Bucket(bucket) policy, err := bucketHandle.IAM().Policy(ctx) @@ -97,7 +115,7 @@ func ApplyPermissionToServiceAccount(ctx context.Context, storageClient *storage setup.LogAndExit(fmt.Sprintf("Error fetching: Bucket(%q).IAM().Policy: %v", bucket, err)) } identity := fmt.Sprintf("serviceAccount:%s", serviceAccount) - role := iam.RoleName(fmt.Sprintf("roles/storage.%s", permission)) + role := iam.RoleName(roleName) policy.Add(identity, role) if err := bucketHandle.IAM().SetPolicy(ctx, policy); err != nil { @@ -108,7 +126,16 @@ func ApplyPermissionToServiceAccount(ctx context.Context, storageClient *storage time.Sleep(120 * time.Second) } -func RevokePermission(ctx context.Context, storageClient *storage.Client, serviceAccount, permission, bucket string) { +func ApplyPermissionToServiceAccount(ctx context.Context, storageClient *storage.Client, serviceAccount, permission, bucket string) { + ApplyRoleToServiceAccountOnBucket(ctx, storageClient, serviceAccount, fmt.Sprintf("roles/storage.%s", permission), bucket) +} + +func ApplyCustomRoleToServiceAccountOnBucket(ctx context.Context, storageClient *storage.Client, serviceAccount, customRoleName, bucket string) { + projectID := projectID(ctx) + ApplyRoleToServiceAccountOnBucket(ctx, storageClient, serviceAccount, fmt.Sprintf("projects/%s/roles/%s", projectID, customRoleName), bucket) +} + +func RevokeRoleFromServiceAccountOnBucket(ctx context.Context, storageClient *storage.Client, serviceAccount, roleName, bucket string) { // Revoke the permission to service account after testing. bucketHandle := storageClient.Bucket(bucket) policy, err := bucketHandle.IAM().Policy(ctx) @@ -116,7 +143,7 @@ func RevokePermission(ctx context.Context, storageClient *storage.Client, servic setup.LogAndExit(fmt.Sprintf("Error fetching: Bucket(%q).IAM().Policy: %v", bucket, err)) } identity := fmt.Sprintf("serviceAccount:%s", serviceAccount) - role := iam.RoleName(fmt.Sprintf("roles/storage.%s", permission)) + role := iam.RoleName(roleName) policy.Remove(identity, role) if err := bucketHandle.IAM().SetPolicy(ctx, policy); err != nil { @@ -124,10 +151,24 @@ func RevokePermission(ctx context.Context, storageClient *storage.Client, servic } } -func RunTestsForKeyFileAndGoogleApplicationCredentialsEnvVarSet(ctx context.Context, storageClient *storage.Client, testFlagSet [][]string, permission string, m *testing.M) (successCode int) { +func RevokePermission(ctx context.Context, storageClient *storage.Client, serviceAccount, permission, bucket string) { + RevokeRoleFromServiceAccountOnBucket(ctx, storageClient, serviceAccount, fmt.Sprintf("roles/storage.%s", permission), bucket) +} + +func RevokeCustomRoleFromServiceAccountOnBucket(ctx context.Context, storageClient *storage.Client, serviceAccount, customRoleName, bucket string) { + projectID := projectID(ctx) + RevokeRoleFromServiceAccountOnBucket(ctx, storageClient, serviceAccount, fmt.Sprintf("projects/%s/roles/%s", projectID, customRoleName), bucket) +} + +func RunTestsForDifferentAuthMethods(ctx context.Context, cfg *test_suite.TestConfig, storageClient *storage.Client, testFlagSet [][]string, permission string, m *testing.M) (successCode int) { serviceAccount, localKeyFilePath := CreateCredentials(ctx) - ApplyPermissionToServiceAccount(ctx, storageClient, serviceAccount, permission, setup.TestBucket()) - defer RevokePermission(ctx, storageClient, serviceAccount, permission, setup.TestBucket()) + defer func() { + if err := os.Remove(localKeyFilePath); err != nil { + log.Printf("Failed to delete temp credentials file %s: %v", localKeyFilePath, err) + } + }() + ApplyPermissionToServiceAccount(ctx, storageClient, serviceAccount, permission, cfg.TestBucket) + defer RevokePermission(ctx, storageClient, serviceAccount, permission, cfg.TestBucket) // Without –key-file flag and GOOGLE_APPLICATION_CREDENTIALS // This case will not get covered as gcsfuse internally authenticates from a metadata server on GCE VM. @@ -139,7 +180,7 @@ func RunTestsForKeyFileAndGoogleApplicationCredentialsEnvVarSet(ctx context.Cont setup.LogAndExit(fmt.Sprintf("Error in setting environment variable: %v", err)) } - successCode = static_mounting.RunTests(testFlagSet, m) + successCode = static_mounting.RunTestsWithConfigFile(cfg, testFlagSet, m) if successCode != 0 { return @@ -148,11 +189,11 @@ func RunTestsForKeyFileAndGoogleApplicationCredentialsEnvVarSet(ctx context.Cont // Testing with --key-file and GOOGLE_APPLICATION_CREDENTIALS env variable set keyFileFlag := "--key-file=" + localKeyFilePath - for i := 0; i < len(testFlagSet); i++ { + for i := range testFlagSet { testFlagSet[i] = append(testFlagSet[i], keyFileFlag) } - successCode = static_mounting.RunTests(testFlagSet, m) + successCode = static_mounting.RunTestsWithConfigFile(cfg, testFlagSet, m) if successCode != 0 { return @@ -163,8 +204,8 @@ func RunTestsForKeyFileAndGoogleApplicationCredentialsEnvVarSet(ctx context.Cont setup.LogAndExit(fmt.Sprintf("Error in unsetting environment variable: %v", err)) } - // Testing with --key-file flag only - successCode = static_mounting.RunTests(testFlagSet, m) + // Testing with --key-file flag only. + successCode = static_mounting.RunTestsWithConfigFile(cfg, testFlagSet, m) if successCode != 0 { return @@ -172,3 +213,15 @@ func RunTestsForKeyFileAndGoogleApplicationCredentialsEnvVarSet(ctx context.Cont return successCode } + +// Deprecated: Use RunTestsForDifferentAuthMethods instead. +// TODO(b/438068132): cleanup deprecated methods after migration is complete. +func RunTestsForKeyFileAndGoogleApplicationCredentialsEnvVarSet(ctx context.Context, storageClient *storage.Client, testFlagSet [][]string, permission string, m *testing.M) (successCode int) { + config := &test_suite.TestConfig{ + TestBucket: setup.TestBucket(), + GKEMountedDirectory: setup.MountedDirectory(), + GCSFuseMountedDirectory: setup.MntDir(), + LogFile: setup.LogFile(), + } + return RunTestsForDifferentAuthMethods(ctx, config, storageClient, testFlagSet, permission, m) +} diff --git a/tools/integration_tests/util/log_parser/json_parser/read_logs/buffered_read_log_parser.go b/tools/integration_tests/util/log_parser/json_parser/read_logs/buffered_read_log_parser.go new file mode 100644 index 0000000000..628d56de1d --- /dev/null +++ b/tools/integration_tests/util/log_parser/json_parser/read_logs/buffered_read_log_parser.go @@ -0,0 +1,324 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package read_logs + +import ( + "encoding/json" + "fmt" + "io" + "regexp" + "strings" +) + +var readFileRegex = regexp.MustCompile(`fuse_debug: Op (0x[0-9a-fA-F]+)\s+connection\.go:\d+\] <- ReadFile \(inode (\d+), PID (\d+), handle (\d+), offset (\d+), (\d+) bytes\)`) +var readAtReqRegex = regexp.MustCompile(`([a-f0-9-]+) <- ReadAt\(([^:]+):/([^,]+), (\d+), (\d+), (\d+), (\d+)\)`) +var readAtSimpleRespRegex = regexp.MustCompile(`([a-f0-9-]+) -> ReadAt\(\): Ok\(([0-9.]+(?:s|ms|µs))\)`) +var fallbackFromHandleRegex = regexp.MustCompile(`Fallback to another reader for object "[^"]+", handle (\d+)\.(?: Random seek count (\d+) exceeded threshold \d+.*)?`) +var restartFromHandleRegex = regexp.MustCompile(`Restarting buffered reader.*handle (\d+)`) + +// ParseBufferedReadLogsFromLogReader parses buffered read logs from an io.Reader and +// returns a map of BufferedReadLogEntry keyed by file handle. +// BufferedReadLogEntry contains the common read log information and a slice of +// BufferedReadChunkData representing the chunk read from buffered reader. +// Example: +// +// { +// "25": { +// "CommonReadLog": { +// "Handle": 25, +// "StartTimeSeconds": 1704444226, +// "StartTimeNanos": 937309952, +// "ProcessID": 2270282, +// "InodeID": 2, +// "BucketName": "bucket_name", +// "ObjectName": "object/name" +// }, +// "Chunks": [ +// { +// "StartTimeSeconds": 1704444226, +// "StartTimeNanos": 937457664, +// "RequestID": "310f589d-20bf", +// "Offset": 0, +// "Size": 26214, +// "BlockIndex": 0, +// "ExecutionTime": "1.907320375s" +// }, +// ... +// ] +// }, +// ... +// } +func ParseBufferedReadLogsFromLogReader(reader io.Reader) (map[int64]*BufferedReadLogEntry, error) { + // file-handle to BufferedReadLogEntry map + bufferedReadLogsMap := make(map[int64]*BufferedReadLogEntry) + + // opReverseMap is used to map request ID to handle and chunk index. + opReverseMap := make(map[string]*handleAndChunkIndex) + + lines, err := loadLogLines(reader) + if err != nil { + return nil, fmt.Errorf("failed to load log lines: %v", err) + } + + for _, line := range lines { + if err := filterAndParseLogLineForBufferedRead(line, bufferedReadLogsMap, opReverseMap); err != nil { + return nil, fmt.Errorf("filterAndParseLogLineForBufferedRead failed for %s: %v", line, err) + } + } + + // Filter out entries that have no chunks, as they represent file handles + // that were opened but never read from using the buffered reader. + filteredLogsMap := make(map[int64]*BufferedReadLogEntry) + for handle, entry := range bufferedReadLogsMap { + if len(entry.Chunks) > 0 { + filteredLogsMap[handle] = entry + } + } + return filteredLogsMap, nil +} + +// filterAndParseLogLineForBufferedRead filters and parses a log line for buffered read logs. +func filterAndParseLogLineForBufferedRead( + logLine string, + bufferedReadLogsMap map[int64]*BufferedReadLogEntry, + opReverseMap map[string]*handleAndChunkIndex) error { + + jsonLog := make(map[string]any) + if err := json.Unmarshal([]byte(logLine), &jsonLog); err != nil { + return nil // Silently ignore the logs which are not in JSON format. + } + + if _, ok := jsonLog["timestamp"]; !ok { + return fmt.Errorf("filterAndParseLogLineForBufferedRead: log line does not contain timestamp: %s", logLine) + } + timestampSeconds := int64(jsonLog["timestamp"].(map[string]any)["seconds"].(float64)) + timestampNanos := int64(jsonLog["timestamp"].(map[string]any)["nanos"].(float64)) + + // Log message is expected to be in the "message" field. + if _, ok := jsonLog["message"]; !ok { + return fmt.Errorf("filterAndParseLogLineForBufferedRead: log line does not contain message: %s", logLine) + } + logMessage := jsonLog["message"].(string) + + // Parse the logs based on type. + switch { + case strings.Contains(logMessage, "<- ReadFile"): + if err := parseReadFileLogsUsingRegex(timestampSeconds, timestampNanos, logMessage, bufferedReadLogsMap); err != nil { + return fmt.Errorf("parseReadFileLog failed: %v", err) + } + case strings.Contains(logMessage, "<- ReadAt("): + if err := parseReadAtRequestLog(timestampSeconds, timestampNanos, logMessage, bufferedReadLogsMap, opReverseMap); err != nil { + return fmt.Errorf("parseReadAtLog failed: %v", err) + } + case strings.Contains(logMessage, "-> ReadAt("): + if err := parseReadAtResponseLog(logMessage, bufferedReadLogsMap, opReverseMap); err != nil { + return fmt.Errorf("parseReadAtResponseLog failed: %v", err) + } + case strings.Contains(logMessage, "Fallback to another reader for object"): + if err := parseFallbackLogFromHandle(logMessage, bufferedReadLogsMap); err != nil { + return fmt.Errorf("parseFallbackLogFromHandle failed: %v", err) + } + case strings.Contains(logMessage, "Restarting buffered reader"): + if err := parseRestartLogFromHandle(logMessage, bufferedReadLogsMap); err != nil { + return fmt.Errorf("parseRestartLogFromHandle failed: %v", err) + } + } + return nil +} + +func parseFallbackLogFromHandle( + logMessage string, + bufferedReadLogsMap map[int64]*BufferedReadLogEntry) error { + + matches := fallbackFromHandleRegex.FindStringSubmatch(logMessage) + if len(matches) < 2 { + // Not a fallback log we are interested in, might be from a different reader. + return nil + } + + handleID, err := parseToInt64(matches[1]) + if err != nil { + return fmt.Errorf("invalid handle ID in fallback log: %w", err) + } + + logEntry, ok := bufferedReadLogsMap[handleID] + if !ok { + return fmt.Errorf("log entry for handle %d not found for fallback log", handleID) + } + + logEntry.Fallback = true + if len(matches) > 2 && matches[2] != "" { + randomSeekCount, err := parseToInt64(matches[2]) + if err != nil { + return fmt.Errorf("invalid random seek count in fallback log: %v", err) + } + logEntry.RandomSeekCount = randomSeekCount + } + return nil +} + +func parseRestartLogFromHandle( + logMessage string, + bufferedReadLogsMap map[int64]*BufferedReadLogEntry) error { + + matches := restartFromHandleRegex.FindStringSubmatch(logMessage) + if len(matches) < 2 { + return nil + } + + handleID, err := parseToInt64(matches[1]) + if err != nil { + return fmt.Errorf("invalid handle ID in restart log: %w", err) + } + + if logEntry, ok := bufferedReadLogsMap[handleID]; ok { + logEntry.Restarted = true + } + return nil +} + +// parseReadFileLogsUsingRegex parses the ReadFile log using regex and updates the bufferedReadLogsMap map. +// It extracts the handle, PID, inode ID from the log message. +func parseReadFileLogsUsingRegex( + startTimeStampSec, startTimeStampNanos int64, + logMessage string, + bufferedReadLogsMap map[int64]*BufferedReadLogEntry) error { + + matches := readFileRegex.FindStringSubmatch(logMessage) + if len(matches) != 7 { + return fmt.Errorf("invalid ReadFile log format: %s", logMessage) + } + + handle, err := parseToInt64(matches[4]) + if err != nil { + return fmt.Errorf("invalid handle: %v", err) + } + pid, err := parseToInt64(matches[3]) + if err != nil { + return fmt.Errorf("invalid process ID: %v", err) + } + inodeID, err := parseToInt64(matches[2]) + if err != nil { + return fmt.Errorf("invalid inode ID: %v", err) + } + + // ReadFile log entries can come multiple times. + // Check if log entry exists in the map for file handle. + // If log entry doesn't exist, add it to the map. + _, ok := bufferedReadLogsMap[handle] + if !ok { + bufferedReadLogsMap[handle] = &BufferedReadLogEntry{ + CommonReadLog: CommonReadLog{ + Handle: handle, + StartTimeSeconds: startTimeStampSec, + StartTimeNanos: startTimeStampNanos, + ProcessID: pid, + InodeID: inodeID, + }, + Chunks: []BufferedReadChunkData{}, + } + } + return nil +} + +// parseReadAtRequestLog parses the ReadAt request log and updates the bufferedReadLogsMap map. +// It extracts the request ID, offset, size, and block index from the log message. +// It also populates the bucket and object name if they are not already set in the BufferedReadLogEntry. +func parseReadAtRequestLog( + startTimeStampSec, startTimeStampNanos int64, + logMessage string, + bufferedReadLogsMap map[int64]*BufferedReadLogEntry, + opReverseMap map[string]*handleAndChunkIndex) error { + + matches := readAtReqRegex.FindStringSubmatch(logMessage) + if len(matches) != 8 { + return fmt.Errorf("invalid ReadAt log format: %s", logMessage) + } + + handle, err := parseToInt64(matches[4]) // "1072693248" + if err != nil { + return fmt.Errorf("invalid handle: %v", err) + } + + logEntry, ok := bufferedReadLogsMap[handle] + if !ok || logEntry == nil { + return fmt.Errorf("BufferedReadLogEntry for handle %d not found", handle) + } + + if logEntry.BucketName == "" || logEntry.ObjectName == "" { + logEntry.BucketName = matches[2] // "bucket_name" + logEntry.ObjectName = matches[3] // "object/name" + } + + requestID := matches[1] // "37623d67-b6ee" + + offset, err := parseToInt64(matches[5]) // "0" + if err != nil { + return fmt.Errorf("invalid offset: %v", err) + } + size, err := parseToInt64(matches[6]) // "1048576" + if err != nil { + return fmt.Errorf("invalid size: %v", err) + } + blockIndex, err := parseToInt64(matches[7]) // "63" + if err != nil { + return fmt.Errorf("invalid block index: %v", err) + } + + chunkData := BufferedReadChunkData{ + StartTimeSeconds: startTimeStampSec, + StartTimeNanos: startTimeStampNanos, + RequestID: requestID, + Offset: offset, + Size: size, + BlockIndex: blockIndex, + ExecutionTime: "", // Execution time will be filled in the response log. + } + logEntry.Chunks = append(logEntry.Chunks, chunkData) + opReverseMap[requestID] = &handleAndChunkIndex{handle: handle, chunkIndex: len(logEntry.Chunks) - 1} + return nil +} + +// parseReadAtResponseLog parses the ReadAt response log and updates the bufferedReadLogsMap map. +// It extracts the request ID and execution time from the log message. +// It updates the corresponding chunk in the bufferedReadLogsMap map with the execution time. +// The request ID is looked up in the opReverseMap to find the corresponding handle and chunk index. +func parseReadAtResponseLog( + logMessage string, + bufferedReadLogsMap map[int64]*BufferedReadLogEntry, + opReverseMap map[string]*handleAndChunkIndex) error { + + matches := readAtSimpleRespRegex.FindStringSubmatch(logMessage) + if len(matches) != 3 { + return fmt.Errorf("invalid simple ReadAt response log format: %s", logMessage) + } + + requestID := matches[1] // "d88d347c-1b8c" + executionTime := matches[2] // "179.94µs" + + // Look up the request in the reverse map + handleAndChunk, exists := opReverseMap[requestID] + if !exists { + return fmt.Errorf("request ID %s not found in reverse map", requestID) + } + + // Update the execution time in the corresponding chunk + logEntry := bufferedReadLogsMap[handleAndChunk.handle] + if logEntry != nil && handleAndChunk.chunkIndex < len(logEntry.Chunks) { + logEntry.Chunks[handleAndChunk.chunkIndex].ExecutionTime = executionTime + } + + return nil +} diff --git a/tools/integration_tests/util/log_parser/json_parser/read_logs/buffered_read_log_parser_test.go b/tools/integration_tests/util/log_parser/json_parser/read_logs/buffered_read_log_parser_test.go new file mode 100644 index 0000000000..a70866d532 --- /dev/null +++ b/tools/integration_tests/util/log_parser/json_parser/read_logs/buffered_read_log_parser_test.go @@ -0,0 +1,348 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package read_logs_test + +import ( + "bytes" + "fmt" + "io" + "strings" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseBufferedReadLogsFromLogReaderSuccessful(t *testing.T) { + setup.IgnoreTestIfIntegrationTestFlagIsSet(t) + + tests := []struct { + name string // Name of the test case + reader io.Reader + expected map[int64]*read_logs.BufferedReadLogEntry + }{ + { + name: "Test buffered read logs with 1 chunk", + reader: bytes.NewReader([]byte(`{"timestamp":{"seconds":1754207548,"nanos":733110719},"severity":"TRACE","message":"fuse_debug: Op 0x0000004e connection.go:453] <- ReadFile (inode 2, PID 564246, handle 0, offset 34603008, 1048576 bytes)"} +{"timestamp":{"seconds":1754207548,"nanos":733199657},"severity":"TRACE","message":"2e4645d9-19a8 <- ReadAt(princer-working-dirs:/10G_file, 0, 34603008, 1048576, 2)"} +{"timestamp":{"seconds":1754207548,"nanos":733417812},"severity":"TRACE","message":"2e4645d9-19a8 -> ReadAt(): Ok(223.643µs)"} +{"timestamp":{"seconds":1754207548,"nanos":733444394},"severity":"TRACE","message":"fuse_debug: Op 0x0000004e connection.go:548] -> ReadFile ()"}`), + ), + expected: map[int64]*read_logs.BufferedReadLogEntry{ + 0: { + CommonReadLog: read_logs.CommonReadLog{ + Handle: 0, + StartTimeSeconds: 1754207548, + StartTimeNanos: 733110719, + ProcessID: 564246, + InodeID: 2, + BucketName: "princer-working-dirs", + ObjectName: "10G_file", + }, + Chunks: []read_logs.BufferedReadChunkData{ + { + StartTimeSeconds: 1754207548, + StartTimeNanos: 733199657, + RequestID: "2e4645d9-19a8", + Offset: 34603008, + Size: 1048576, + BlockIndex: 2, + ExecutionTime: "223.643µs", + }, + }, + }, + }, + }, + { + name: "Test buffered read logs with multiple chunks", + reader: bytes.NewReader([]byte(`{"timestamp":{"seconds":1754207548,"nanos":733110719},"severity":"TRACE","message":"fuse_debug: Op 0x0000004e connection.go:453] <- ReadFile (inode 2, PID 564246, handle 0, offset 34603008, 1048576 bytes)"} +{"timestamp":{"seconds":1754207548,"nanos":733199657},"severity":"TRACE","message":"2e4645d9-19a8 <- ReadAt(princer-working-dirs:/10G_file, 0, 34603008, 1048576, 2)"} +{"timestamp":{"seconds":1754207548,"nanos":733417812},"severity":"TRACE","message":"2e4645d9-19a8 -> ReadAt(): Ok(223.643µs)"} +{"timestamp":{"seconds":1754207548,"nanos":733444394},"severity":"TRACE","message":"fuse_debug: Op 0x0000004e connection.go:548] -> ReadFile ()"} +{"timestamp":{"seconds":1754207548,"nanos":733776221},"severity":"TRACE","message":"fuse_debug: Op 0x00000050 connection.go:453] <- ReadFile (inode 2, PID 564246, handle 0, offset 35651584, 1048576 bytes)"} +{"timestamp":{"seconds":1754207548,"nanos":733853084},"severity":"TRACE","message":"a8517095-54c0 <- ReadAt(princer-working-dirs:/10G_file, 0, 35651584, 1048576, 2)"} +{"timestamp":{"seconds":1754207548,"nanos":734027808},"severity":"TRACE","message":"a8517095-54c0 -> ReadAt(): Ok(173.476µs)"} +{"timestamp":{"seconds":1754207548,"nanos":734048914},"severity":"TRACE","message":"fuse_debug: Op 0x00000050 connection.go:548] -> ReadFile ()"} +{"timestamp":{"seconds":1754207548,"nanos":734299231},"severity":"TRACE","message":"fuse_debug: Op 0x00000052 connection.go:453] <- ReadFile (inode 2, PID 564246, handle 0, offset 36700160, 1048576 bytes)"} +{"timestamp":{"seconds":1754207548,"nanos":734358133},"severity":"TRACE","message":"4e8b1c9c-0012 <- ReadAt(princer-working-dirs:/10G_file, 0, 36700160, 1048576, 2)"} +{"timestamp":{"seconds":1754207548,"nanos":734532341},"severity":"TRACE","message":"4e8b1c9c-0012 -> ReadAt(): Ok(179.8µs)"}`), + ), + expected: map[int64]*read_logs.BufferedReadLogEntry{ + 0: { + CommonReadLog: read_logs.CommonReadLog{ + Handle: 0, + StartTimeSeconds: 1754207548, + StartTimeNanos: 733110719, + ProcessID: 564246, + InodeID: 2, + BucketName: "princer-working-dirs", + ObjectName: "10G_file", + }, + Chunks: []read_logs.BufferedReadChunkData{ + { + StartTimeSeconds: 1754207548, + StartTimeNanos: 733199657, + RequestID: "2e4645d9-19a8", + Offset: 34603008, + Size: 1048576, + BlockIndex: 2, + ExecutionTime: "223.643µs", + }, + { + StartTimeSeconds: 1754207548, + StartTimeNanos: 733853084, + RequestID: "a8517095-54c0", + Offset: 35651584, + Size: 1048576, + BlockIndex: 2, + ExecutionTime: "173.476µs", + }, + { + StartTimeSeconds: 1754207548, + StartTimeNanos: 734358133, + RequestID: "4e8b1c9c-0012", + Offset: 36700160, + Size: 1048576, + BlockIndex: 2, + ExecutionTime: "179.8µs", + }, + }, + }, + }, + }, + { + name: "Test buffered read logs with no fallback", + reader: bytes.NewReader([]byte(`{"timestamp":{"seconds":1754207548,"nanos":733110719},"severity":"TRACE","message":"fuse_debug: Op 0x0000004e connection.go:453] <- ReadFile (inode 2, PID 564246, handle 0, offset 34603008, 1048576 bytes)"} +{"timestamp":{"seconds":1754207548,"nanos":733199657},"severity":"TRACE","message":"2e4645d9-19a8 <- ReadAt(princer-working-dirs:/10G_file, 0, 34603008, 1048576, 2)"} +{"timestamp":{"seconds":1754207548,"nanos":733417812},"severity":"TRACE","message":"2e4645d9-19a8 -> ReadAt(): Ok(223.643µs)"} +{"timestamp":{"seconds":1754207548,"nanos":733444394},"severity":"TRACE","message":"fuse_debug: Op 0x0000004e connection.go:548] -> ReadFile ()"}`), + ), + expected: map[int64]*read_logs.BufferedReadLogEntry{ + 0: { + CommonReadLog: read_logs.CommonReadLog{ + Handle: 0, + StartTimeSeconds: 1754207548, + StartTimeNanos: 733110719, + ProcessID: 564246, + InodeID: 2, + BucketName: "princer-working-dirs", + ObjectName: "10G_file", + }, + Chunks: []read_logs.BufferedReadChunkData{ + { + StartTimeSeconds: 1754207548, + StartTimeNanos: 733199657, + RequestID: "2e4645d9-19a8", + Offset: 34603008, + Size: 1048576, + BlockIndex: 2, + ExecutionTime: "223.643µs", + }, + }, + Fallback: false, + RandomSeekCount: 0, + }, + }, + }, + { + name: "Test buffered read logs with no parsable logs", + reader: bytes.NewReader([]byte(`{"timestamp":{"seconds":1754207548,"nanos":742190759},"severity":"TRACE","message":"Scheduling block: (10G_file, 9, false)."} +{"timestamp":{"seconds":1754207548,"nanos":742296187},"severity":"TRACE","message":"Scheduling block: (10G_file, 10, false)."} +{"timestamp":{"seconds":1754207548,"nanos":742300356},"severity":"TRACE","message":"Download: <- block (10G_file, 9)."} +{"timestamp":{"seconds":1754207548,"nanos":742315339},"severity":"TRACE","message":"Scheduling block: (10G_file, 11, false)."} +{"timestamp":{"seconds":1754207548,"nanos":742323114},"severity":"TRACE","message":"Scheduling block: (10G_file, 12, false)."}`), + ), + expected: make(map[int64]*read_logs.BufferedReadLogEntry), + }, + { + name: "Test buffered read logs with fallback", + reader: bytes.NewReader([]byte(`{"timestamp":{"seconds":1754207548,"nanos":733110719},"severity":"TRACE","message":"fuse_debug: Op 0x0000004e connection.go:453] <- ReadFile (inode 2, PID 564246, handle 0, offset 34603008, 1048576 bytes)"} +{"timestamp":{"seconds":1754207548,"nanos":733199657},"severity":"TRACE","message":"2e4645d9-19a8 <- ReadAt(princer-working-dirs:/10G_file, 0, 34603008, 1048576, 2)"} +{"timestamp":{"seconds":1754207548,"nanos":733200000},"severity":"WARN","message":"Fallback to another reader for object \"10G_file\", handle 0. Random seek count 4 exceeded threshold 3."} +{"timestamp":{"seconds":1754207548,"nanos":733417812},"severity":"TRACE","message":"2e4645d9-19a8 -> ReadAt(): Ok(223.643µs)"} +{"timestamp":{"seconds":1754207548,"nanos":733444394},"severity":"TRACE","message":"fuse_debug: Op 0x0000004e connection.go:548] -> ReadFile ()"}`), + ), + expected: map[int64]*read_logs.BufferedReadLogEntry{ + 0: { + CommonReadLog: read_logs.CommonReadLog{ + Handle: 0, + StartTimeSeconds: 1754207548, + StartTimeNanos: 733110719, + ProcessID: 564246, + InodeID: 2, + BucketName: "princer-working-dirs", + ObjectName: "10G_file", + }, + Chunks: []read_logs.BufferedReadChunkData{ + { + StartTimeSeconds: 1754207548, + StartTimeNanos: 733199657, + RequestID: "2e4645d9-19a8", + Offset: 34603008, + Size: 1048576, + BlockIndex: 2, + ExecutionTime: "223.643µs", + }, + }, + Fallback: true, + RandomSeekCount: 4, + }, + }, + }, + { + name: "Test buffered read logs with generic fallback", + reader: bytes.NewReader([]byte(`{"timestamp":{"seconds":1754207548,"nanos":733110719},"severity":"TRACE","message":"fuse_debug: Op 0x0000004e connection.go:453] <- ReadFile (inode 2, PID 564246, handle 0, offset 34603008, 1048576 bytes)"} +{"timestamp":{"seconds":1754207548,"nanos":733199657},"severity":"TRACE","message":"2e4645d9-19a8 <- ReadAt(princer-working-dirs:/10G_file, 0, 34603008, 1048576, 2)"} +{"timestamp":{"seconds":1754207548,"nanos":733200000},"severity":"WARN","message":"Fallback to another reader for object \"10G_file\", handle 0. Due to freshStart failure: some error"} +{"timestamp":{"seconds":1754207548,"nanos":733417812},"severity":"TRACE","message":"2e4645d9-19a8 -> ReadAt(): Ok(223.643µs)"} +{"timestamp":{"seconds":1754207548,"nanos":733444394},"severity":"TRACE","message":"fuse_debug: Op 0x0000004e connection.go:548] -> ReadFile ()"}`), + ), + expected: map[int64]*read_logs.BufferedReadLogEntry{ + 0: { + CommonReadLog: read_logs.CommonReadLog{ + Handle: 0, + StartTimeSeconds: 1754207548, + StartTimeNanos: 733110719, + ProcessID: 564246, + InodeID: 2, + BucketName: "princer-working-dirs", + ObjectName: "10G_file", + }, + Chunks: []read_logs.BufferedReadChunkData{ + { + StartTimeSeconds: 1754207548, + StartTimeNanos: 733199657, + RequestID: "2e4645d9-19a8", + Offset: 34603008, + Size: 1048576, + BlockIndex: 2, + ExecutionTime: "223.643µs", + }, + }, + Fallback: true, + RandomSeekCount: 0, + }, + }, + }, + { + name: "Test buffered read logs with no JSON logs", + reader: bytes.NewReader([]byte(`hello 123`)), + expected: make(map[int64]*read_logs.BufferedReadLogEntry), + }, + { + name: "Test buffered read logs with restart", + reader: bytes.NewReader([]byte(`{"timestamp":{"seconds":1754207548,"nanos":733110719},"severity":"TRACE","message":"fuse_debug: Op 0x0000004e connection.go:453] <- ReadFile (inode 2, PID 564246, handle 0, offset 34603008, 1048576 bytes)"} +{"timestamp":{"seconds":1754207548,"nanos":733199657},"severity":"TRACE","message":"2e4645d9-19a8 <- ReadAt(princer-working-dirs:/10G_file, 0, 34603008, 1048576, 2)"} +{"timestamp":{"seconds":1754207548,"nanos":733200000},"severity":"INFO","message":"Restarting buffered reader due to sequential read pattern detected for object \"10G_file\", handle 0"} +{"timestamp":{"seconds":1754207548,"nanos":733417812},"severity":"TRACE","message":"2e4645d9-19a8 -> ReadAt(): Ok(223.643µs)"} +{"timestamp":{"seconds":1754207548,"nanos":733444394},"severity":"TRACE","message":"fuse_debug: Op 0x0000004e connection.go:548] -> ReadFile ()"}`), + ), + expected: map[int64]*read_logs.BufferedReadLogEntry{ + 0: { + CommonReadLog: read_logs.CommonReadLog{ + Handle: 0, + StartTimeSeconds: 1754207548, + StartTimeNanos: 733110719, + ProcessID: 564246, + InodeID: 2, + BucketName: "princer-working-dirs", + ObjectName: "10G_file", + }, + Chunks: []read_logs.BufferedReadChunkData{ + { + StartTimeSeconds: 1754207548, + StartTimeNanos: 733199657, + RequestID: "2e4645d9-19a8", + Offset: 34603008, + Size: 1048576, + BlockIndex: 2, + ExecutionTime: "223.643µs", + }, + }, + Fallback: false, + RandomSeekCount: 0, + Restarted: true, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + actual, err := read_logs.ParseBufferedReadLogsFromLogReader(tc.reader) + + require.NoError(t, err) + assert.Equal(t, tc.expected, actual, fmt.Sprintf("Expected: %v, Actual: %v", tc.expected, actual)) + }) + } +} + +func TestBufferedReadLogsFromLogReaderUnsuccessful(t *testing.T) { + setup.IgnoreTestIfIntegrationTestFlagIsSet(t) + + tests := []struct { + name string // Name of the test case + reader io.Reader + errorString string + }{ + { + name: "Test buffered read logs without Read File log", + reader: bytes.NewReader([]byte(`{"timestamp":{"seconds":1754207548,"nanos":733199657},"severity":"TRACE","message":"2e4645d9-19a8 <- ReadAt(princer-working-dirs:/10G_file, 0, 34603008, 1048576, 2)"} +{"timestamp":{"seconds":1754207548,"nanos":733417812},"severity":"TRACE","message":"2e4645d9-19a8 -> ReadAt(): Ok(223.643µs)"} +{"timestamp":{"seconds":1754207548,"nanos":733444394},"severity":"TRACE","message":"fuse_debug: Op 0x0000004e connection.go:548] -> ReadFile ()"}`), + ), + errorString: "BufferedReadLogEntry for handle 0 not found", + }, + { + name: "Test buffered read logs response log without request log", + reader: bytes.NewReader([]byte(`{"timestamp":{"seconds":1754207548,"nanos":733110719},"severity":"TRACE","message":"fuse_debug: Op 0x0000004e connection.go:453] <- ReadFile (inode 2, PID 564246, handle 0, offset 34603008, 1048576 bytes)"} +{"timestamp":{"seconds":1754207548,"nanos":733417812},"severity":"TRACE","message":"2e4645d9-19a8 -> ReadAt(): Ok(223.643µs)"} +{"timestamp":{"seconds":1754207548,"nanos":733444394},"severity":"TRACE","message":"fuse_debug: Op 0x0000004e connection.go:548] -> ReadFile ()"}`), + ), + errorString: "request ID 2e4645d9-19a8 not found in reverse map", + }, + { + name: "Test invalid read file log - invalid Inode ID", + reader: bytes.NewReader([]byte(`{"timestamp":{"seconds":1754207548,"nanos":733110719},"severity":"TRACE","message":"fuse_debug: Op 0x0000004e connection.go:453] <- ReadFile (inode abc, PID 564246, handle 0, offset 34603008, 1048576 bytes)"}`)), + errorString: "parseReadFileLog failed: invalid ReadFile log", + }, + { + name: "Test invalid read file log - invalid PID", + reader: bytes.NewReader([]byte(`{"timestamp": {"seconds": 1704458059, "nanos": 975956234}, "severity": "TRACE", "message": "fuse_debug: Op 0x00000182 connection.go:415] <- ReadFile (inode 6, PID abc, handle 29, offset 0, 4096 bytes)"}`)), + errorString: "parseReadFileLog failed: invalid ReadFile log format", + }, + { + name: "Test invalid read file log - invalid Handle", + reader: bytes.NewReader([]byte(`{"timestamp": {"seconds": 1704458059, "nanos": 975956234}, "severity": "TRACE", "message": "fuse_debug: Op 0x00000182 connection.go:415] <- ReadFile (inode 6, PID 2382526, handle abc, offset 0, 4096 bytes)"}`)), + errorString: "parseReadFileLog failed: invalid ReadFile log format", + }, + { + name: "Test fallback log for unknown handle", + reader: bytes.NewReader([]byte(`{"timestamp":{"seconds":1754207548,"nanos":733200000},"severity":"WARN","message":"Fallback to another reader for object \"10G_file\", handle 99. Random seek count 4 exceeded threshold 3."}`)), + errorString: "log entry for handle 99 not found for fallback log", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := read_logs.ParseBufferedReadLogsFromLogReader(tc.reader) + + require.Error(t, err) + assert.True(t, strings.Contains(err.Error(), tc.errorString), fmt.Sprintf("Unexpected error: %s", err)) + }) + } +} diff --git a/tools/integration_tests/util/log_parser/json_parser/read_logs/helpers.go b/tools/integration_tests/util/log_parser/json_parser/read_logs/helpers.go index bbb8b6aaed..ac40bcbc0c 100644 --- a/tools/integration_tests/util/log_parser/json_parser/read_logs/helpers.go +++ b/tools/integration_tests/util/log_parser/json_parser/read_logs/helpers.go @@ -64,12 +64,14 @@ func parseReadFileLog(startTimeStampSec, startTimeStampNanos int64, logs []strin _, ok := structuredLogs[handle] if !ok { structuredLogs[handle] = &StructuredReadLogEntry{ - Handle: handle, - StartTimeSeconds: startTimeStampSec, - StartTimeNanos: startTimeStampNanos, - ProcessID: pid, - InodeID: inodeID, - Chunks: []ReadChunkData{}, + CommonReadLog: CommonReadLog{ + Handle: handle, + StartTimeSeconds: startTimeStampSec, + StartTimeNanos: startTimeStampNanos, + ProcessID: pid, + InodeID: inodeID, + }, + Chunks: []ReadChunkData{}, } } return nil @@ -222,3 +224,58 @@ func parseJobFileLog(startTimeStampSec, startTimeStampNanos int64, logsMessage s } return nil } + +// parseChunkDownloadLog parses a chunk download log message and adds details +// to the structuredLogs map. +func parseChunkDownloadLog(startTimeStampSec, startTimeStampNanos int64, logsMessage string, structuredLogs map[string]*Job) error { + // Fetch bucket name, object name and offsets from the logs. + // Example: Job:0xc000aa65b0 (bucket:/obj) downloaded range [0, 10), added 10 bytes to sparse file + pattern := `Job:(\w+) \(([\w./_-]+):/([\w./_-]+)\) downloaded range \[(\d+), (\d+)\), added (\d+) bytes to sparse file` + re := regexp.MustCompile(pattern) + matches := re.FindStringSubmatch(logsMessage) + + var jobID, bucketName, objectName string + var startOffset, endOffset, bytesAdded int64 + var err error + + if len(matches) == 7 { + jobID = matches[1] + bucketName = matches[2] + objectName = matches[3] + startOffset, err = strconv.ParseInt(matches[4], 10, 64) + if err != nil { + return fmt.Errorf("error while parsing start offset: %v", err) + } + endOffset, err = strconv.ParseInt(matches[5], 10, 64) + if err != nil { + return fmt.Errorf("error while parsing end offset: %v", err) + } + bytesAdded, err = strconv.ParseInt(matches[6], 10, 64) + if err != nil { + return fmt.Errorf("error while parsing bytes added: %v", err) + } + } else { + return fmt.Errorf("string did not match the expected pattern for sparse download") + } + + entry := ChunkDownloadLogEntry{ + StartTimeSeconds: startTimeStampSec, + StartTimeNanos: startTimeStampNanos, + StartOffset: startOffset, + EndOffset: endOffset, + BytesAdded: bytesAdded, + } + + jobEntry, ok := structuredLogs[jobID] + if !ok { + structuredLogs[jobID] = &Job{ + JobID: jobID, + ObjectName: objectName, + BucketName: bucketName, + ChunkCacheDownloads: []ChunkDownloadLogEntry{entry}, + } + } else { + jobEntry.ChunkCacheDownloads = append(jobEntry.ChunkCacheDownloads, entry) + } + return nil +} diff --git a/tools/integration_tests/util/log_parser/json_parser/read_logs/json_job_log_parser.go b/tools/integration_tests/util/log_parser/json_parser/read_logs/json_job_log_parser.go index 4cf22d1479..db4895253e 100644 --- a/tools/integration_tests/util/log_parser/json_parser/read_logs/json_job_log_parser.go +++ b/tools/integration_tests/util/log_parser/json_parser/read_logs/json_job_log_parser.go @@ -23,7 +23,7 @@ import ( "strings" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) /* @@ -50,14 +50,14 @@ func parseJobLogsFromLogFile(reader io.Reader) (map[string]*Job, error) { func filterAndParseJobLogLine(logLine string, structuredLogs map[string]*Job) error { - jsonLog := make(map[string]interface{}) + jsonLog := make(map[string]any) if err := json.Unmarshal([]byte(logLine), &jsonLog); err != nil { return nil // Silently ignore the structuredLogs which are not in JSON format. } // Get timestamp from the jsonLog - timestampSeconds := int64(jsonLog["timestamp"].(map[string]interface{})["seconds"].(float64)) - timestampNanos := int64(jsonLog["timestamp"].(map[string]interface{})["nanos"].(float64)) + timestampSeconds := int64(jsonLog["timestamp"].(map[string]any)["seconds"].(float64)) + timestampNanos := int64(jsonLog["timestamp"].(map[string]any)["nanos"].(float64)) // Normalize whitespace in the log message. logMessage := strings.TrimSpace(regexp.MustCompile(`\s+`).ReplaceAllString(jsonLog["message"].(string), " ")) // Parse the logs based on type. @@ -66,6 +66,10 @@ func filterAndParseJobLogLine(logLine string, structuredLogs map[string]*Job) er if err := parseJobFileLog(timestampSeconds, timestampNanos, logMessage, structuredLogs); err != nil { return fmt.Errorf("parseJobFileLog failed: %v", err) } + case strings.Contains(logMessage, "downloaded range"): + if err := parseChunkDownloadLog(timestampSeconds, timestampNanos, logMessage, structuredLogs); err != nil { + return fmt.Errorf("parseChunkDownloadLog failed: %v", err) + } } return nil } diff --git a/tools/integration_tests/util/log_parser/json_parser/read_logs/json_job_log_parser_test.go b/tools/integration_tests/util/log_parser/json_parser/read_logs/json_job_log_parser_test.go index 9236376ec1..c37e1413ad 100644 --- a/tools/integration_tests/util/log_parser/json_parser/read_logs/json_job_log_parser_test.go +++ b/tools/integration_tests/util/log_parser/json_parser/read_logs/json_job_log_parser_test.go @@ -19,7 +19,7 @@ import ( "io" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -105,6 +105,26 @@ func TestParseJobLogsSuccessful(t *testing.T) { }, }, }, + { + name: "Test chunk download logs", + reader: bytes.NewReader([]byte(`{"timestamp":{"seconds":1721228431,"nanos":993427325},"severity":"TRACE","message":"Job:0xc000af6000 (bucket:/obj) downloaded range [0, 10), added 10 bytes to sparse file"}`)), + expected: map[string]*Job{ + "0xc000af6000": { + BucketName: "bucket", + ObjectName: "obj", + JobID: "0xc000af6000", + ChunkCacheDownloads: []ChunkDownloadLogEntry{ + { + StartTimeSeconds: 1721228431, + StartTimeNanos: 993427325, + StartOffset: 0, + EndOffset: 10, + BytesAdded: 10, + }, + }, + }, + }, + }, } for _, tc := range tests { diff --git a/tools/integration_tests/util/log_parser/json_parser/read_logs/json_read_log_parser.go b/tools/integration_tests/util/log_parser/json_parser/read_logs/json_read_log_parser.go index d0c852ddde..0dd46507b1 100644 --- a/tools/integration_tests/util/log_parser/json_parser/read_logs/json_read_log_parser.go +++ b/tools/integration_tests/util/log_parser/json_parser/read_logs/json_read_log_parser.go @@ -23,22 +23,23 @@ import ( "sort" "strings" "testing" + "time" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) func filterAndParseLogLine(logLine string, structuredLogs map[int64]*StructuredReadLogEntry, opReverseMap map[string]*handleAndChunkIndex) error { - jsonLog := make(map[string]interface{}) + jsonLog := make(map[string]any) if err := json.Unmarshal([]byte(logLine), &jsonLog); err != nil { return nil // Silently ignore the structuredLogs which are not in JSON format. } // Get timestamp from the jsonLog - timestampSeconds := int64(jsonLog["timestamp"].(map[string]interface{})["seconds"].(float64)) - timestampNanos := int64(jsonLog["timestamp"].(map[string]interface{})["nanos"].(float64)) + timestampSeconds := int64(jsonLog["timestamp"].(map[string]any)["seconds"].(float64)) + timestampNanos := int64(jsonLog["timestamp"].(map[string]any)["nanos"].(float64)) // Normalize whitespace in the log message. logMessage := strings.TrimSpace(regexp.MustCompile(`\s+`).ReplaceAllString(jsonLog["message"].(string), " ")) // Tokenize log message. @@ -47,7 +48,7 @@ func filterAndParseLogLine(logLine string, // Parse the logs based on type. switch { // TODO: change this to regex matching instead of strings.Contains. - case strings.Contains(logMessage, "ReadFile"): + case strings.Contains(logMessage, "<- ReadFile"): if err := parseReadFileLog(timestampSeconds, timestampNanos, tokenizedLogs, structuredLogs); err != nil { return fmt.Errorf("parseReadFileLog failed: %v", err) } @@ -121,7 +122,7 @@ func GetStructuredLogsSortedByTimestamp(logFilePath string, t *testing.T) []*Str // Open and parse log file. file, err := os.Open(logFilePath) if err != nil { - t.Errorf("Failed to open log file") + t.Errorf("Failed to open log file: %v", err) } logsMap, err := ParseReadLogsFromLogFile(file) if err != nil { @@ -149,3 +150,22 @@ func GetStructuredLogsSortedByTimestamp(logFilePath string, t *testing.T) []*Str return structuredReadLogs } + +func ParseJsonLogLineIntoLogEntryStruct(jsonLogEntry string) (*LogEntry, error) { + entry := &LogEntry{} + jsonLog := make(map[string]any) + err := json.Unmarshal([]byte(jsonLogEntry), &jsonLog) + if err != nil { + return entry, nil + } + + // Get timestamp from the jsonLog + timestampSeconds := int64(jsonLog["timestamp"].(map[string]any)["seconds"].(float64)) + timestampNanos := int64(jsonLog["timestamp"].(map[string]any)["nanos"].(float64)) + entry.Timestamp = time.Unix(timestampSeconds, timestampNanos) + + // Normalize whitespace in the log message. + entry.Message = strings.TrimSpace(regexp.MustCompile(`\s+`).ReplaceAllString(jsonLog["message"].(string), " ")) + + return entry, nil +} diff --git a/tools/integration_tests/util/log_parser/json_parser/read_logs/json_read_log_parser_test.go b/tools/integration_tests/util/log_parser/json_parser/read_logs/json_read_log_parser_test.go index d96f7b4a39..ade081d129 100644 --- a/tools/integration_tests/util/log_parser/json_parser/read_logs/json_read_log_parser_test.go +++ b/tools/integration_tests/util/log_parser/json_parser/read_logs/json_read_log_parser_test.go @@ -22,8 +22,8 @@ import ( "strings" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/log_parser/json_parser/read_logs" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -70,17 +70,20 @@ func TestParseLogFileSuccessful(t *testing.T) { reader: bytes.NewReader([]byte(`{"timestamp": {"seconds": 1704458059, "nanos": 975956234}, "severity": "TRACE", "message": "fuse_debug: Op 0x00000182 connection.go:415] <- ReadFile (inode 6, PID 2382526, handle 29, offset 0, 4096 bytes)"} {"timestamp": {"seconds": 1704458060, "nanos": 976093794}, "severity": "TRACE", "message": "f41c82a2-c891 <- FileCache(redacted:/smallfile.txt, offset: 0, size: 4096 handle: 29)"} {"timestamp": {"seconds": 1704458061, "nanos": 269924363}, "severity": "TRACE", "message": "Job:0xc000aa65b0 (redacted:/smallfile.txt) downloaded till 6 offset."} -{"timestamp": {"seconds": 1704458061, "nanos": 270075223}, "severity": "TRACE", "message": "f41c82a2-c891 -> OK (isSeq: true, hit: false) (293.935998ms)"}`), +{"timestamp": {"seconds": 1704458061, "nanos": 270075223}, "severity": "TRACE", "message": "f41c82a2-c891 -> OK (isSeq: true, hit: false) (293.935998ms)"} +{"timestamp": {"seconds": 1704458061, "nanos": 975956234}, "severity":"TRACE","message":"fuse_debug: Op 0x00000182 connection.go:497] -> ReadFile ()"}`), ), expected: map[int64]*read_logs.StructuredReadLogEntry{ handleId: { - Handle: handleId, - StartTimeSeconds: readTimestampSeconds, - StartTimeNanos: readTimestampNanos, - ProcessID: pid, - InodeID: inodeId, - BucketName: bucketName, - ObjectName: fileName, + CommonReadLog: read_logs.CommonReadLog{ + Handle: handleId, + StartTimeSeconds: readTimestampSeconds, + StartTimeNanos: readTimestampNanos, + ProcessID: pid, + InodeID: inodeId, + BucketName: bucketName, + ObjectName: fileName, + }, Chunks: []read_logs.ReadChunkData{ chunkData, }, @@ -104,13 +107,15 @@ func TestParseLogFileSuccessful(t *testing.T) { ), expected: map[int64]*read_logs.StructuredReadLogEntry{ 29: { - Handle: 29, - StartTimeSeconds: readTimestampSeconds, - StartTimeNanos: readTimestampNanos, - ProcessID: pid, - InodeID: inodeId, - BucketName: bucketName, - ObjectName: fileName, + CommonReadLog: read_logs.CommonReadLog{ + Handle: 29, + StartTimeSeconds: readTimestampSeconds, + StartTimeNanos: readTimestampNanos, + ProcessID: pid, + InodeID: inodeId, + BucketName: bucketName, + ObjectName: fileName, + }, Chunks: []read_logs.ReadChunkData{ chunkData, chunkData, chunkData, }, @@ -119,9 +124,9 @@ func TestParseLogFileSuccessful(t *testing.T) { }, { name: "Test file cache logs with no parsable logs", - reader: bytes.NewReader([]byte(`{"timestamp": {"seconds": 1704458059, "nanos": 975956234}, "severity":"TRACE","message":"fuse_debug: Op 0x00000182 connection.go:497] -> OK ()"} + reader: bytes.NewReader([]byte(`{"timestamp": {"seconds": 1704458059, "nanos": 975956234}, "severity":"TRACE","message":"fuse_debug: Op 0x00000182 connection.go:497] -> ReadFile ()"} {"timestamp": {"seconds": 1704458059, "nanos": 975956234}, "severity":"TRACE","message":"fuse_debug: Op 0x00000184 connection.go:415] <- FlushFile (inode 6, PID 2382526)"} -{"timestamp": {"seconds": 1704458059, "nanos": 975956234}, "severity":"TRACE","message":"fuse_debug: Op 0x00000184 connection.go:497] -> OK ()"}`), +{"timestamp": {"seconds": 1704458059, "nanos": 975956234}, "severity":"TRACE","message":"fuse_debug: Op 0x00000184 connection.go:497] -> FlushFile ()"}`), ), expected: make(map[int64]*read_logs.StructuredReadLogEntry), }, diff --git a/tools/integration_tests/util/log_parser/json_parser/read_logs/structure.go b/tools/integration_tests/util/log_parser/json_parser/read_logs/structure.go index 0d1e85d8b8..d0b2b29899 100644 --- a/tools/integration_tests/util/log_parser/json_parser/read_logs/structure.go +++ b/tools/integration_tests/util/log_parser/json_parser/read_logs/structure.go @@ -14,8 +14,11 @@ package read_logs -// StructuredReadLogEntry stores the structured format to be created from logs. -type StructuredReadLogEntry struct { +import ( + "time" +) + +type CommonReadLog struct { Handle int64 StartTimeSeconds int64 StartTimeNanos int64 @@ -23,9 +26,17 @@ type StructuredReadLogEntry struct { InodeID int64 BucketName string ObjectName string +} + +// StructuredReadLogEntry stores the structured format to be created from logs. +type StructuredReadLogEntry struct { + CommonReadLog + // It can be safely assumed that the Chunks will be sorted on timestamp as logs // are parsed in the order of timestamps. Chunks []ReadChunkData + // ChunkCacheReads contains logs related to chunk cache hit checks. + ChunkCacheReads []ChunkCacheReadLogEntry } // ReadChunkData stores the format of chunk to be stored StructuredReadLogEntry. @@ -45,6 +56,8 @@ type Job struct { BucketName string ObjectName string JobEntries []JobData + // ChunkCacheDownloads contains logs related to chunk downloads. + ChunkCacheDownloads []ChunkDownloadLogEntry } // JobData stores the job timestamp and offsets for a particular file. @@ -54,6 +67,24 @@ type JobData struct { Offset int64 } +// ChunkCacheReadLogEntry stores the details of a chunk cache hit check. +type ChunkCacheReadLogEntry struct { + StartTimeSeconds int64 + StartTimeNanos int64 + StartOffset int64 + EndOffset int64 + CacheHit bool +} + +// ChunkDownloadLogEntry stores the details of a chunk download. +type ChunkDownloadLogEntry struct { + StartTimeSeconds int64 + StartTimeNanos int64 + StartOffset int64 + EndOffset int64 + BytesAdded int64 +} + //////////////////////////////////////////////////////////////////////// // Helpers //////////////////////////////////////////////////////////////////////// @@ -64,3 +95,28 @@ type handleAndChunkIndex struct { handle int64 chunkIndex int } + +// LogEntry struct to match the JSON structure +type LogEntry struct { + Timestamp time.Time `json:"time"` + Message string `json:"message"` +} + +type BufferedReadLogEntry struct { + CommonReadLog + + Chunks []BufferedReadChunkData + Fallback bool + RandomSeekCount int64 + Restarted bool +} + +type BufferedReadChunkData struct { + StartTimeSeconds int64 + StartTimeNanos int64 + RequestID string + Offset int64 + Size int64 + BlockIndex int64 + ExecutionTime string +} diff --git a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go index 8093740130..9947a382cc 100644 --- a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go +++ b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go @@ -21,45 +21,45 @@ import ( "path" "testing" - "cloud.google.com/go/compute/metadata" "cloud.google.com/go/storage" - "google.golang.org/api/iterator" - - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" ) -// Adding prefix `golang-grpc-test` to white list the bucket for grpc so that -// we can run the grpc related e2e test. -const PrefixBucketForDynamicMountingTest = "golang-grpc-test-gcsfuse-dynamic-mounting-test-" - -var testBucketForDynamicMounting = PrefixBucketForDynamicMountingTest + setup.GenerateRandomString(5) +func MountGcsfuseWithDynamicMountingWithConfig(cfg *test_suite.TestConfig, flags []string) (err error) { + defaultArg := []string{"--log-severity=trace", + "--log-file=" + cfg.LogFile, + cfg.GCSFuseMountedDirectory} -func MountGcsfuseWithDynamicMounting(flags []string) (err error) { - defaultArg := []string{"--debug_gcs", - "--debug_fs", - "--debug_fuse", - "--log-file=" + setup.LogFile(), - setup.MntDir()} - - for i := 0; i < len(defaultArg); i++ { - flags = append(flags, defaultArg[i]) - } + flags = append(flags, defaultArg...) err = mounting.MountGcsfuse(setup.BinFile(), flags) return err } -func runTestsOnGivenMountedTestBucket(bucketName string, flags [][]string, rootMntDir string, m *testing.M) (successCode int) { - for i := 0; i < len(flags); i++ { - if err := MountGcsfuseWithDynamicMounting(flags[i]); err != nil { +// MountGcsfuseWithDynamicMounting is deprecated. Use MountGcsfuseWithDynamicMountingWithConfig instead. +func MountGcsfuseWithDynamicMounting(flags []string) (err error) { + cfg := &test_suite.TestConfig{ + GKEMountedDirectory: setup.MountedDirectory(), + GCSFuseMountedDirectory: setup.MntDir(), + TestBucket: setup.TestBucket(), + LogFile: setup.LogFile(), + } + return MountGcsfuseWithDynamicMountingWithConfig(cfg, flags) +} + +func runTestsOnGivenMountedTestBucket(cfg *test_suite.TestConfig, flags [][]string, rootMntDir string, m *testing.M) (successCode int) { + for i := range flags { + if err := MountGcsfuseWithDynamicMountingWithConfig(cfg, flags[i]); err != nil { setup.LogAndExit(fmt.Sprintf("mountGcsfuse: %v\n", err)) } // Changing mntDir to path of bucket mounted in mntDir for testing. - mntDirOfTestBucket := path.Join(setup.MntDir(), bucketName) - + mntDirOfTestBucket := path.Join(cfg.GCSFuseMountedDirectory, cfg.TestBucket) + cfg.GCSFuseMountedDirectory = mntDirOfTestBucket + // TODO: clean up MntDir. setup.SetMntDir(mntDirOfTestBucket) log.Printf("Running dynamic mounting tests with flags: %s", flags[i]) @@ -68,7 +68,9 @@ func runTestsOnGivenMountedTestBucket(bucketName string, flags [][]string, rootM // Currently mntDir is mntDir/bucketName. // Unmounting can happen on rootMntDir. Changing mntDir to rootMntDir for unmounting. + // TODO: clean up MntDir. setup.SetMntDir(rootMntDir) + cfg.GCSFuseMountedDirectory = rootMntDir setup.UnMountAndThrowErrorInFailure(flags[i], successCode) if successCode != 0 { return @@ -77,87 +79,36 @@ func runTestsOnGivenMountedTestBucket(bucketName string, flags [][]string, rootM return } -func executeTestsForDynamicMounting(flags [][]string, m *testing.M) (successCode int) { - rootMntDir := setup.MntDir() +func executeTestsForDynamicMounting(config *test_suite.TestConfig, flagsSet [][]string, m *testing.M) (successCode int) { + rootMntDir := config.GCSFuseMountedDirectory // In dynamic mounting all the buckets mounted in mntDir which user has permission. // mntDir - bucket1, bucket2, bucket3, ... - // We will test on passed testBucket and one created bucket. // SetDynamicBucketMounted to the passed test bucket. - setup.SetDynamicBucketMounted(setup.TestBucket()) - // Test on testBucket - successCode = runTestsOnGivenMountedTestBucket(setup.TestBucket(), flags, rootMntDir, m) - - // Test on created bucket. - // SetDynamicBucketMounted to the mounted bucket. - setup.SetDynamicBucketMounted(testBucketForDynamicMounting) - if successCode == 0 { - successCode = runTestsOnGivenMountedTestBucket(testBucketForDynamicMounting, flags, rootMntDir, m) - } + setup.SetDynamicBucketMounted(config.TestBucket) + successCode = runTestsOnGivenMountedTestBucket(config, flagsSet, rootMntDir, m) // Reset SetDynamicBucketMounted to empty after tests are done. setup.SetDynamicBucketMounted("") - // Setting back the original mntDir after testing. - setup.SetMntDir(rootMntDir) return } -func CreateTestBucketForDynamicMounting(ctx context.Context, client *storage.Client) (bucketName string) { - projectID, err := metadata.ProjectID() - if err != nil { - log.Printf("Error in fetching project id: %v", err) - } - - // Create bucket handle and attributes - storageClassAndLocation := &storage.BucketAttrs{ - Location: "us-west1", - } - - bucket := client.Bucket(testBucketForDynamicMounting) - if err := bucket.Create(ctx, projectID, storageClassAndLocation); err != nil { - log.Fatalf("DynamicBucket(%q).Create: %v", testBucketForDynamicMounting, err) - } - return testBucketForDynamicMounting -} - -func DeleteTestBucketForDynamicMounting(ctx context.Context, client *storage.Client, bucketName string) { - bucket := client.Bucket(bucketName) - - // Iterate through objects and delete them - query := &storage.Query{} - it := bucket.Objects(ctx, query) - for { - objAttrs, err := it.Next() - if err == iterator.Done { - break // No more objects - } - if err != nil { - log.Fatalf("Error iterating through objects: %v", err) - } - - obj := bucket.Object(objAttrs.Name) - err = obj.Delete(ctx) - if err != nil { - log.Fatalf("Failed to delete object %s: %v", objAttrs.Name, err) - } - } - - if err := bucket.Delete(ctx); err != nil { - log.Printf("Bucket(%q).Delete: %v", bucketName, err) +// Deprecated: Use RunTestsWithConfigFile instead. +// TODO(b/438068132): cleanup deprecated methods after migration is complete. +func RunTests(ctx context.Context, client *storage.Client, flags [][]string, m *testing.M) (successCode int) { + config := &test_suite.TestConfig{ + TestBucket: setup.TestBucket(), + GKEMountedDirectory: setup.MountedDirectory(), + GCSFuseMountedDirectory: setup.MntDir(), + LogFile: setup.LogFile(), } + return RunTestsWithConfigFile(config, flags, m) } -func RunTests(ctx context.Context, client *storage.Client, flags [][]string, m *testing.M) (successCode int) { +func RunTestsWithConfigFile(config *test_suite.TestConfig, flagsSet [][]string, m *testing.M) (successCode int) { log.Println("Running dynamic mounting tests...") - - CreateTestBucketForDynamicMounting(ctx, client) - - successCode = executeTestsForDynamicMounting(flags, m) - - log.Printf("Test log: %s\n", setup.LogFile()) - - DeleteTestBucketForDynamicMounting(ctx, client, testBucketForDynamicMounting) - + log.Printf("GCSFuse Log File for test: %s\n", config.LogFile) + successCode = executeTestsForDynamicMounting(config, flagsSet, m) return successCode } diff --git a/tools/integration_tests/util/mounting/mounting.go b/tools/integration_tests/util/mounting/mounting.go index b42690a489..f131922d5e 100644 --- a/tools/integration_tests/util/mounting/mounting.go +++ b/tools/integration_tests/util/mounting/mounting.go @@ -19,9 +19,10 @@ import ( "log" "os" "os/exec" + "path" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" ) func MountGcsfuse(binaryFile string, flags []string) error { @@ -31,16 +32,21 @@ func MountGcsfuse(binaryFile string, flags []string) error { ) // Adding mount command in LogFile + err := os.MkdirAll(path.Dir(setup.LogFile()), 0777) + if err != nil { + fmt.Println("error creating directory: ", err) + return err + } file, err := os.OpenFile(setup.LogFile(), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { - fmt.Println("Could not open logfile") + fmt.Println("Could not open logfile: ", err.Error()) } // Closing file at the end. defer operations.CloseFile(file) _, err = file.WriteString(mountCmd.String() + "\n") if err != nil { - fmt.Println("Could not write cmd to logFile") + fmt.Println("Could not write cmd to logFile: ", err.Error()) } output, err := mountCmd.CombinedOutput() diff --git a/tools/integration_tests/util/mounting/only_dir_mounting/only_dir_mounting.go b/tools/integration_tests/util/mounting/only_dir_mounting/only_dir_mounting.go index 2ac882a39d..c00e16e775 100644 --- a/tools/integration_tests/util/mounting/only_dir_mounting/only_dir_mounting.go +++ b/tools/integration_tests/util/mounting/only_dir_mounting/only_dir_mounting.go @@ -19,34 +19,43 @@ import ( "log" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" "golang.org/x/net/context" ) +// Deprecated: Use MountGcsfuseWithOnlyDirMountingWithConfigFile instead. +// TODO(b/438068132): cleanup deprecated methods after migration is complete. func MountGcsfuseWithOnlyDir(flags []string) (err error) { + config := &test_suite.TestConfig{ + TestBucket: setup.TestBucket(), + GKEMountedDirectory: setup.MountedDirectory(), + GCSFuseMountedDirectory: setup.MntDir(), + LogFile: setup.LogFile(), + } + return MountGcsfuseWithOnlyDirWithConfigFile(config, flags) +} + +func MountGcsfuseWithOnlyDirWithConfigFile(config *test_suite.TestConfig, flags []string) (err error) { defaultArg := []string{"--only-dir", setup.OnlyDirMounted(), - "--debug_gcs", - "--debug_fs", - "--debug_fuse", - "--log-file=" + setup.LogFile(), - setup.TestBucket(), - setup.MntDir()} - - for i := 0; i < len(defaultArg); i++ { - flags = append(flags, defaultArg[i]) - } + "--log-severity=trace", + "--log-file=" + config.LogFile, + config.TestBucket, + config.GCSFuseMountedDirectory} + + flags = append(flags, defaultArg...) err = mounting.MountGcsfuse(setup.BinFile(), flags) return err } -func mountGcsFuseForFlagsAndExecuteTests(flags [][]string, m *testing.M) (successCode int) { - for i := 0; i < len(flags); i++ { - if err := MountGcsfuseWithOnlyDir(flags[i]); err != nil { +func mountGcsFuseForFlagsAndExecuteTests(config *test_suite.TestConfig, flags [][]string, m *testing.M) (successCode int) { + for i := range flags { + if err := MountGcsfuseWithOnlyDirWithConfigFile(config, flags[i]); err != nil { setup.LogAndExit(fmt.Sprintf("mountGcsfuse: %v\n", err)) } log.Printf("Running only dir mounting tests with flags: %s", flags[i]) @@ -58,7 +67,7 @@ func mountGcsFuseForFlagsAndExecuteTests(flags [][]string, m *testing.M) (succes return } -func executeTestsForOnlyDirMounting(flags [][]string, dirName string, m *testing.M) (successCode int) { +func executeTestsForOnlyDirMounting(config *test_suite.TestConfig, flags [][]string, dirName string, m *testing.M) (successCode int) { ctx := context.Background() storageClient, err := client.CreateStorageClient(ctx) if err != nil { @@ -76,7 +85,7 @@ func executeTestsForOnlyDirMounting(flags [][]string, dirName string, m *testing if err != nil { log.Println("Error deleting object on GCS: %w", err) } - successCode = mountGcsFuseForFlagsAndExecuteTests(flags, m) + successCode = mountGcsFuseForFlagsAndExecuteTests(config, flags, m) if successCode != 0 { return } @@ -84,7 +93,7 @@ func executeTestsForOnlyDirMounting(flags [][]string, dirName string, m *testing // Test scenario when only-dir-mounted directory pre-exists in bucket. client.SetupTestDirectory(ctx, storageClient, dirName) - successCode = mountGcsFuseForFlagsAndExecuteTests(flags, m) + successCode = mountGcsFuseForFlagsAndExecuteTests(config, flags, m) err = client.DeleteAllObjectsWithPrefix(ctx, storageClient, dirName) if err != nil { log.Println("Error deleting object on GCS: %w", err) @@ -95,12 +104,21 @@ func executeTestsForOnlyDirMounting(flags [][]string, dirName string, m *testing return } +// Deprecated: Use RunTestsWithConfigFile instead. +// TODO(b/438068132): cleanup deprecated methods after migration is complete. func RunTests(flags [][]string, dirName string, m *testing.M) (successCode int) { - log.Println("Running only dir mounting tests...") - - successCode = executeTestsForOnlyDirMounting(flags, dirName, m) - - log.Printf("Test log: %s\n", setup.LogFile()) + config := &test_suite.TestConfig{ + TestBucket: setup.TestBucket(), + GKEMountedDirectory: setup.MountedDirectory(), + GCSFuseMountedDirectory: setup.MntDir(), + LogFile: setup.LogFile(), + } + return RunTestsWithConfigFile(config, flags, dirName, m) +} +func RunTestsWithConfigFile(config *test_suite.TestConfig, flagsSet [][]string, dirName string, m *testing.M) (successCode int) { + log.Println("Running only dir mounting tests...") + log.Printf("GCSFuse Log File for test: %s\n", config.LogFile) + successCode = executeTestsForOnlyDirMounting(config, flagsSet, dirName, m) return successCode } diff --git a/tools/integration_tests/util/mounting/persistent_mounting/perisistent_mounting.go b/tools/integration_tests/util/mounting/persistent_mounting/perisistent_mounting.go index e9678f2559..99544d68b3 100644 --- a/tools/integration_tests/util/mounting/persistent_mounting/perisistent_mounting.go +++ b/tools/integration_tests/util/mounting/persistent_mounting/perisistent_mounting.go @@ -20,44 +20,40 @@ import ( "strings" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" ) -// make e.g --debug_gcs in debug_gcs -func makePersistentMountingArgs(flags []string) (args []string, err error) { +// Change e.g --log_severity=trace to log_severity=trace +func makePersistentMountingArgs(flags []string) (args []string) { var s string for i := range flags { // We are already passing flags with -o flag. s = strings.Replace(flags[i], "--o=", "", -1) - // e.g. Convert --debug_gcs to __debug_gcs + // e.g. Convert --log-severity=trace to __log_severity=trace s = strings.Replace(s, "-", "_", -1) - // e.g. Convert __debug_gcs to debug_gcs + // e.g. Convert __log_severity=trace to log_severity=trace s = strings.Replace(s, "__", "", -1) + // e.g. Revert _1 to -1 + s = strings.Replace(s, "_1", "-1", -1) args = append(args, s) } return } -func mountGcsfuseWithPersistentMounting(flags []string) (err error) { - defaultArg := []string{setup.TestBucket(), - setup.MntDir(), +func mountGcsfuseWithPersistentMountingWithConfigFile(config *test_suite.TestConfig, flags []string) (err error) { + defaultArg := []string{config.TestBucket, + config.GCSFuseMountedDirectory, "-o", - "debug_gcs", + "log_severity=trace", "-o", - "debug_fs", - "-o", - "debug_fuse", - "-o", - "log_file=" + setup.LogFile(), + "log_file=" + config.LogFile, } - persistentMountingArgs, err := makePersistentMountingArgs(flags) - if err != nil { - setup.LogAndExit("Error in converting flags for persistent mounting.") - } + persistentMountingArgs := makePersistentMountingArgs(flags) - for i := 0; i < len(persistentMountingArgs); i++ { + for i := range persistentMountingArgs { // e.g. -o flag1, -o flag2, ... defaultArg = append(defaultArg, "-o", persistentMountingArgs[i]) } @@ -67,11 +63,11 @@ func mountGcsfuseWithPersistentMounting(flags []string) (err error) { return err } -func executeTestsForPersistentMounting(flagsSet [][]string, m *testing.M) (successCode int) { +func executeTestsForPersistentMountingWithConfigFile(config *test_suite.TestConfig, flagsSet [][]string, m *testing.M) (successCode int) { var err error - for i := 0; i < len(flagsSet); i++ { - if err = mountGcsfuseWithPersistentMounting(flagsSet[i]); err != nil { + for i := range flagsSet { + if err = mountGcsfuseWithPersistentMountingWithConfigFile(config, flagsSet[i]); err != nil { setup.LogAndExit(fmt.Sprintf("mountGcsfuse: %v\n", err)) } log.Printf("Running persistent mounting tests with flags: %s", flagsSet[i]) @@ -83,12 +79,21 @@ func executeTestsForPersistentMounting(flagsSet [][]string, m *testing.M) (succe return } +// Deprecated: Use RunTestsWithConfigFile instead. +// TODO(b/438068132): cleanup deprecated methods after migration is complete. func RunTests(flagsSet [][]string, m *testing.M) (successCode int) { - log.Println("Running persistent mounting tests...") - - successCode = executeTestsForPersistentMounting(flagsSet, m) - - log.Printf("Test log: %s\n", setup.LogFile()) + config := &test_suite.TestConfig{ + TestBucket: setup.TestBucket(), + GKEMountedDirectory: setup.MountedDirectory(), + GCSFuseMountedDirectory: setup.MntDir(), + LogFile: setup.LogFile(), + } + return RunTestsWithConfigFile(config, flagsSet, m) +} +func RunTestsWithConfigFile(config *test_suite.TestConfig, flagsSet [][]string, m *testing.M) (successCode int) { + log.Println("Running persistent mounting tests...") + log.Printf("GCSFuse Log File for test: %s\n", config.LogFile) + successCode = executeTestsForPersistentMountingWithConfigFile(config, flagsSet, m) return successCode } diff --git a/tools/integration_tests/util/mounting/static_mounting/static_mounting.go b/tools/integration_tests/util/mounting/static_mounting/static_mounting.go index 065551348f..60cc58bb9d 100644 --- a/tools/integration_tests/util/mounting/static_mounting/static_mounting.go +++ b/tools/integration_tests/util/mounting/static_mounting/static_mounting.go @@ -19,38 +19,47 @@ import ( "log" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" ) +// TODO(b/438068132): cleanup deprecated methods after migration is complete. func MountGcsfuseWithStaticMounting(flags []string) (err error) { + config := &test_suite.TestConfig{ + TestBucket: setup.TestBucket(), + GKEMountedDirectory: setup.MountedDirectory(), + GCSFuseMountedDirectory: setup.MntDir(), + LogFile: setup.LogFile(), + } + return MountGcsfuseWithStaticMountingWithConfigFile(config, flags) +} + +func MountGcsfuseWithStaticMountingWithConfigFile(config *test_suite.TestConfig, flags []string) (err error) { var defaultArg []string if setup.TestOnTPCEndPoint() { - defaultArg = append(defaultArg, "--custom-endpoint=storage.apis-tpczero.goog:443", + defaultArg = append(defaultArg, "--key-file=/tmp/sa.key.json") } - defaultArg = append(defaultArg, "--debug_gcs", - "--debug_fs", - "--debug_fuse", - "--log-file="+setup.LogFile(), - setup.TestBucket(), - setup.MntDir()) + defaultArg = append(defaultArg, "--log-severity=trace", + "--log-file="+config.LogFile, + config.TestBucket, + config.GCSFuseMountedDirectory) for i := 0; i < len(defaultArg); i++ { flags = append(flags, defaultArg[i]) } err = mounting.MountGcsfuse(setup.BinFile(), flags) - return err } -func executeTestsForStaticMounting(flagsSet [][]string, m *testing.M) (successCode int) { +func executeTestsForStaticMounting(config *test_suite.TestConfig, flagsSet [][]string, m *testing.M) (successCode int) { var err error - for i := 0; i < len(flagsSet); i++ { - if err = MountGcsfuseWithStaticMounting(flagsSet[i]); err != nil { + for i := range flagsSet { + if err = MountGcsfuseWithStaticMountingWithConfigFile(config, flagsSet[i]); err != nil { setup.LogAndExit(fmt.Sprintf("mountGcsfuse: %v\n", err)) } log.Printf("Running static mounting tests with flags: %s", flagsSet[i]) @@ -62,12 +71,21 @@ func executeTestsForStaticMounting(flagsSet [][]string, m *testing.M) (successCo return } +// Deprecated: Use RunTestsWithConfigFile instead. +// TODO(b/438068132): cleanup deprecated methods after migration is complete. func RunTests(flagsSet [][]string, m *testing.M) (successCode int) { - log.Println("Running static mounting tests...") - - successCode = executeTestsForStaticMounting(flagsSet, m) - - log.Printf("Test log: %s\n", setup.LogFile()) + config := &test_suite.TestConfig{ + TestBucket: setup.TestBucket(), + GKEMountedDirectory: setup.MountedDirectory(), + GCSFuseMountedDirectory: setup.MntDir(), + LogFile: setup.LogFile(), + } + return RunTestsWithConfigFile(config, flagsSet, m) +} +func RunTestsWithConfigFile(config *test_suite.TestConfig, flagsSet [][]string, m *testing.M) (successCode int) { + log.Println("Running static mounting tests...") + log.Printf("GCSFuse Log File for test: %s\n", config.LogFile) + successCode = executeTestsForStaticMounting(config, flagsSet, m) return successCode } diff --git a/tools/integration_tests/util/operations/dir_operations.go b/tools/integration_tests/util/operations/dir_operations.go index a2f73ab978..6077ede50c 100644 --- a/tools/integration_tests/util/operations/dir_operations.go +++ b/tools/integration_tests/util/operations/dir_operations.go @@ -17,7 +17,9 @@ package operations import ( "bytes" + "errors" "fmt" + "io/fs" "log" "os" "os/exec" @@ -25,6 +27,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "testing" ) @@ -91,22 +94,68 @@ func RenameDir(dirName string, newDirName string) (err error) { } func CreateDirectoryWithNFiles(numberOfFiles int, dirPath string, prefix string, t *testing.T) { + // 1. Create the directory. err := os.Mkdir(dirPath, FilePermission_0777) - if err != nil && !strings.Contains(err.Error(), "file exists") { - t.Errorf("Error in creating directory: %v", err) + if err != nil && !errors.Is(err, fs.ErrExist) { + t.Fatalf("Error in creating directory %q: %v", dirPath, err) } + // Limit the maximum number of I/O goroutines that can run simultaneously. + const maxConcurrency = 1024 + sem := make(chan struct{}, maxConcurrency) + + // 2. Setup a WaitGroup to manage concurrent Go routines + var wg sync.WaitGroup + + // 3. Setup a channel to collect and report any errors + // A buffered channel is used so a Go routine won't block if the main thread + // has already called t.Fatalf and stopped processing. + errCh := make(chan error, numberOfFiles) + + // 4. Loop to start concurrent file creation for i := 1; i <= numberOfFiles; i++ { - // Create file with name prefix + i - // e.g. If prefix = temp then temp1, temp2 - filePath := path.Join(dirPath, prefix+strconv.Itoa(i)) - file, err := os.Create(filePath) - if err != nil { - t.Errorf("Create file at %q: %v", dirPath, err) - } + // ACQUIRE TOKEN: This will block if 1024 goroutines are currently active + // to prevent thread limit exhaustion. + sem <- struct{}{} + + wg.Add(1) // Increment the counter for each Go routine started + + // Capture the loop variable locally to avoid race conditions + // where multiple Go routines might use the final value of i. + i := i + + go func() { + defer wg.Done() // Decrement the counter when the Go routine finishes + // RELEASE TOKEN: Execute this immediately before the goroutine exits + // to allow the next waiting goroutine to proceed. + defer func() { <-sem }() + + // Create file with name prefix + i (e.g., temp1, temp2) + filePath := path.Join(dirPath, prefix+strconv.Itoa(i)) + + file, err := os.Create(filePath) + if err != nil { + // Send the error to the channel instead of calling t.Fatalf directly + errCh <- err + return + } + + // Closing file at the end. + CloseFileShouldNotThrowError(t, file) + }() + } + + // 5. Wait for all Go routines to finish + wg.Wait() - // Closing file at the end. - CloseFile(file) + // 6. Check for errors + // We need to check if any errors were sent to the channel. + select { + case err := <-errCh: + // If an error is received, fail the test + t.Fatalf("Failed to create file during parallel execution: %v", err) + default: + // No errors were available, so proceed } } @@ -125,6 +174,7 @@ func ReadDirectory(dirPath string, t *testing.T) (entries []os.DirEntry) { } func VerifyDirectoryEntry(entry os.DirEntry, dirName string, t *testing.T) { + t.Helper() if !entry.IsDir() { t.Fatalf("Expected: directory entry, Got: file entry.") } @@ -140,6 +190,7 @@ func VerifyCountOfDirectoryEntries(expected, got int, t *testing.T) { } func CreateDirectory(dirPath string, t testing.TB) { + t.Helper() err := os.Mkdir(dirPath, DirPermission_0755) // Verify MkDir operation succeeds. @@ -164,26 +215,26 @@ func DirSizeMiB(dirPath string) (dirSizeMB int64, err error) { func DeleteManagedFoldersInBucket(managedFolderPath, bucket string) { gcloudDeleteManagedFolderCmd := fmt.Sprintf("alpha storage rm -r gs://%s/%s", bucket, managedFolderPath) - _, err := ExecuteGcloudCommandf(gcloudDeleteManagedFolderCmd) + _, err := ExecuteGcloudCommand(gcloudDeleteManagedFolderCmd) if err != nil && !strings.Contains(err.Error(), "The following URLs matched no objects or files") { - log.Fatalf(fmt.Sprintf("Error while deleting managed folder: %v", err)) + log.Fatalf("Error while deleting managed folder: %v", err) } } func CreateManagedFoldersInBucket(managedFolderPath, bucket string) { gcloudCreateManagedFolderCmd := fmt.Sprintf("alpha storage managed-folders create gs://%s/%s", bucket, managedFolderPath) - _, err := ExecuteGcloudCommandf(gcloudCreateManagedFolderCmd) + _, err := ExecuteGcloudCommand(gcloudCreateManagedFolderCmd) if err != nil && !strings.Contains(err.Error(), "The specified managed folder already exists") { - log.Fatalf(fmt.Sprintf("Error while creating managed folder: %v", err)) + log.Fatalf("Error while creating managed folder: %v", err) } } func CopyFileInBucket(srcfilePath, destFilePath, bucket string, t *testing.T) { gcloudCopyFileCmd := fmt.Sprintf("alpha storage cp %s gs://%s/%s/", srcfilePath, bucket, destFilePath) - _, err := ExecuteGcloudCommandf(gcloudCopyFileCmd) + _, err := ExecuteGcloudCommand(gcloudCopyFileCmd) if err != nil { - t.Fatalf(fmt.Sprintf("Error while copying file in bucket: %v", err)) + t.Fatalf("Error while copying file in bucket: %v", err) } } diff --git a/tools/integration_tests/util/operations/file_operations.go b/tools/integration_tests/util/operations/file_operations.go index 5ada65f988..3c9ad02c26 100644 --- a/tools/integration_tests/util/operations/file_operations.go +++ b/tools/integration_tests/util/operations/file_operations.go @@ -16,6 +16,7 @@ package operations import ( + "bufio" "bytes" "compress/gzip" "crypto/rand" @@ -31,6 +32,12 @@ import ( "syscall" "testing" "time" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/context" ) const ( @@ -45,7 +52,9 @@ const ( TimeSlop = 25 * time.Millisecond // TmpDirectory specifies the directory where temporary files will be created. // In this case, we are using the system's default temporary directory. - TmpDirectory = "/tmp" + TmpDirectory = "/tmp" + WaitDurationAfterFlushZB = time.Minute + WaitDurationAfterCloseZB = time.Second ) func copyFile(srcFileName, dstFileName string, allowOverwrite bool) (err error) { @@ -98,18 +107,9 @@ func CopyFileAllowOverwrite(srcFileName, newFileName string) (err error) { } func ReadFile(filePath string) (content []byte, err error) { - file, err := os.OpenFile(filePath, os.O_RDONLY|syscall.O_DIRECT, FilePermission_0600) - if err != nil { - err = fmt.Errorf("error in the opening the file %v", err) - return - } - - // Closing file at the end. - defer CloseFile(file) - - content, err = os.ReadFile(file.Name()) + content, err = os.ReadFile(filePath) if err != nil { - err = fmt.Errorf("ReadAll: %v", err) + err = fmt.Errorf("ReadFile: %v", err) return } return @@ -167,10 +167,20 @@ func WriteFile(fileName string, content string) (err error) { return } +func CloseFiles(t *testing.T, files []*os.File) { + t.Helper() + for _, file := range files { + err := file.Close() + assert.NoError(t, err) + } +} + +// Deprecated: please use CloseFileShouldNotThrowError instead. func CloseFile(file *os.File) { if err := file.Close(); err != nil { log.Fatalf("error in closing: %v", err) } + WaitForSizeUpdate(setup.IsZonalBucketRun(), WaitDurationAfterCloseZB) } func RemoveFile(filePath string) { @@ -180,16 +190,13 @@ func RemoveFile(filePath string) { } } -func ReadFileSequentially(filePath string, chunkSize int64) (content []byte, err error) { +func ReadFileSequentially(file *os.File, chunkSize int64) (content []byte, err error) { chunk := make([]byte, chunkSize) var offset int64 = 0 - file, err := os.OpenFile(filePath, os.O_RDONLY|syscall.O_DIRECT, FilePermission_0600) if err != nil { - log.Printf("Error in opening file: %v", err) + return nil, fmt.Errorf("error in opening file %q: %w", file.Name(), err) } - - // Closing the file at the end. defer CloseFile(file) for err != io.EOF { @@ -224,58 +231,49 @@ func ReadFileSequentially(filePath string, chunkSize int64) (content []byte, err return } -// Write data of chunkSize in file at given offset. -func WriteChunkOfRandomBytesToFile(file *os.File, chunkSize int, offset int64) error { +func WriteChunkOfRandomBytesToFiles(files []*os.File, chunkSize int, offset int64) error { // Generate random data of chunk size. - chunk := make([]byte, chunkSize) - _, err := rand.Read(chunk) + chunk, err := GenerateRandomData(int64(chunkSize)) if err != nil { - return fmt.Errorf("error while generating random string: %v", err) + return fmt.Errorf("error in generating random data: %v", err) } - // Write data in the file. - n, err := file.WriteAt(chunk, offset) - if err != nil { - return fmt.Errorf("error in writing randomly in file: %v", err) - } + for _, file := range files { + // Write data in the file. + n, err := file.WriteAt(chunk, offset) + if err != nil { + return fmt.Errorf("error in writing randomly in file: %s, %v", file.Name(), err) + } - if n != chunkSize { - return fmt.Errorf("incorrect number of bytes written in the file actual %d, expected %d", n, chunkSize) - } + if n != chunkSize { + return fmt.Errorf("incorrect number of bytes written in the file %s actual %d, expected %d", file.Name(), n, chunkSize) + } - err = file.Sync() - if err != nil { - return fmt.Errorf("error in syncing file: %v", err) + if !setup.IsZonalBucketRun() { + err = file.Sync() + if err != nil { + return fmt.Errorf("error in syncing file: %v", err) + } + WaitForSizeUpdate(setup.IsZonalBucketRun(), WaitDurationAfterFlushZB) + } } return nil } -func WriteFileSequentially(filePath string, fileSize int64, chunkSize int64) (err error) { - file, err := os.OpenFile(filePath, os.O_RDWR|syscall.O_DIRECT|os.O_CREATE, FilePermission_0600) - if err != nil { - log.Fatalf("Error in opening file: %v", err) - } - - // Closing file at the end. - defer CloseFile(file) +func WriteFilesSequentially(t *testing.T, filePaths []string, fileSize int64, chunkSize int64) { + t.Helper() + files := OpenFiles(t, filePaths) + defer CloseFiles(t, files) var offset int64 = 0 - for offset < fileSize { - // Get random chunkSize or remaining filesize data into chunk. - if (fileSize - offset) < chunkSize { - chunkSize = fileSize - offset - } - - err := WriteChunkOfRandomBytesToFile(file, int(chunkSize), offset) - if err != nil { - log.Fatalf("Error in writing chunk: %v", err) - } - + // Reduce chunk size to remaining file size in case chunk size is larger. + chunkSize = min(chunkSize, fileSize-offset) + err := WriteChunkOfRandomBytesToFiles(files, int(chunkSize), offset) + assert.NoError(t, err) offset = offset + chunkSize } - return } func ReadChunkFromFile(filePath string, chunkSize int64, offset int64, flag int) (chunk []byte, err error) { @@ -286,16 +284,14 @@ func ReadChunkFromFile(filePath string, chunkSize int64, offset int64, flag int) log.Printf("Error in opening file: %v", err) return } + defer CloseFile(file) - f, err := os.Stat(filePath) + f, err := file.Stat() if err != nil { log.Printf("Error in stating file: %v", err) return } - // Closing the file at the end. - defer CloseFile(file) - var numberOfBytes int // Reading chunk size randomly from the file. @@ -315,6 +311,29 @@ func ReadChunkFromFile(filePath string, chunkSize int64, offset int64, flag int) return } +func ReadFileBetweenOffset(t *testing.T, file *os.File, startOffset, endOffset, chunkSize int64) string { + t.Helper() + chunk := make([]byte, chunkSize) + var readData []byte + + for startOffset < endOffset { + readSize := min(chunkSize, endOffset-startOffset) + + n, err := file.ReadAt(chunk[:readSize], startOffset) + if err == io.EOF { + readData = append(readData, chunk[:n]...) + break + } else if err != nil { + t.Errorf("Failed to read file chunk at offset %d: %v", startOffset, err) + return "" + } + readData = append(readData, chunk[:n]...) + startOffset += int64(n) + } + + return string(readData) +} + // Returns the stats of a file. // Fails if the passed input is a directory. func StatFile(file string) (*fs.FileInfo, error) { @@ -337,6 +356,14 @@ func OpenFileAsReadonly(filepath string) (*os.File, error) { return f, nil } +// Open file in mode opens the given file in the flag mode provided. +func OpenFileInMode(t *testing.T, filepath string, flag int) *os.File { + t.Helper() + fh, err := os.OpenFile(filepath, flag, FilePermission_0600) + require.NoError(t, err) + return fh +} + func readBytesFromFile(f *os.File, numBytesToRead int, b []byte) error { numBytesRead, err := f.Read(b) if err != nil { @@ -464,14 +491,46 @@ func ClearCacheControlOnGcsObject(gcsObjPath string) error { } func CreateFile(filePath string, filePerms os.FileMode, t testing.TB) (f *os.File) { + t.Helper() // Creating a file shouldn't create file on GCS. - f, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, filePerms) + f, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC|syscall.O_DIRECT, filePerms) if err != nil { t.Fatalf("CreateFile(%s): %v", filePath, err) } return } +func OpenFiles(t *testing.T, filePaths []string) []*os.File { + t.Helper() + var files []*os.File + + // Open all files. + for _, filePath := range filePaths { + file, err := os.OpenFile(filePath, os.O_RDWR|syscall.O_DIRECT|os.O_CREATE, FilePermission_0600) + require.NoError(t, err) + files = append(files, file) + } + return files +} + +func OpenFile(filePath string, t *testing.T) (f *os.File) { + f, err := os.OpenFile(filePath, os.O_RDWR, FilePermission_0777) + if err != nil { + t.Fatalf("OpenFile(%s): %v", filePath, err) + } + return +} + +func OpenFileWithODirect(t *testing.T, filePath string) (f *os.File) { + t.Helper() + f, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC|syscall.O_DIRECT, FilePermission_0600) + if err != nil { + require.NoError(t, err) + } + return + +} + func CreateSymLink(filePath, symlink string, t *testing.T) { err := os.Symlink(filePath, symlink) @@ -555,9 +614,16 @@ func WriteAt(content string, offset int64, fh *os.File, t testing.TB) { } } -func CloseFileShouldNotThrowError(file *os.File, t *testing.T) { - if err := file.Close(); err != nil { - t.Fatalf("file.Close() for file %s: %v", file.Name(), err) +func CloseFileShouldNotThrowError(t testing.TB, file *os.File) { + err := file.Close() + assert.NoError(t, err) + WaitForSizeUpdate(setup.IsZonalBucketRun(), WaitDurationAfterCloseZB) +} + +func CloseFileShouldThrowError(t *testing.T, file *os.File) { + t.Helper() + if err := file.Close(); err == nil { + t.Fatalf("file.Close() for file %s should throw an error: %v", file.Name(), err) } } @@ -568,13 +634,27 @@ func SyncFile(fh *os.File, t *testing.T) { if err != nil { t.Fatalf("%s.Sync(): %v", fh.Name(), err) } + WaitForSizeUpdate(setup.IsZonalBucketRun(), WaitDurationAfterFlushZB) +} + +func SyncFiles(files []*os.File, t *testing.T) { + t.Helper() + for _, file := range files { + SyncFile(file, t) + } } -func CreateFileWithContent(filePath string, filePerms os.FileMode, - content string, t testing.TB) { +func SyncFileShouldThrowError(t *testing.T, file *os.File) { + t.Helper() + if err := file.Sync(); err == nil { + t.Fatalf("file.Close() for file %s should throw an error: %v", file.Name(), err) + } +} + +func CreateFileWithContent(filePath string, filePerms os.FileMode, content string, t testing.TB) { fh := CreateFile(filePath, filePerms, t) WriteAt(content, 0, fh, t) - CloseFile(fh) + CloseFileShouldNotThrowError(t, fh) } // CreateFileOfSize creates a file of given size with random data. @@ -683,3 +763,133 @@ func CreateLocalTempFile(content string, gzipCompress bool) (string, error) { return writeTextToFile(f, f.Name(), content, len(content)) } + +// ReadAndCompare reads content from the given file paths and compares them. +func ReadAndCompare(t *testing.T, filePathInMntDir string, filePathInLocalDisk string, offset int64, chunkSize int64) { + t.Helper() + mountContents, err := ReadChunkFromFile(filePathInMntDir, chunkSize, offset, os.O_RDONLY|syscall.O_DIRECT) + if err != nil { + t.Fatalf("error in read file from mounted directory :%d", err) + } + + diskContents, err := ReadChunkFromFile(filePathInLocalDisk, chunkSize, offset, os.O_RDONLY) + if err != nil { + t.Fatalf("error in read file from local directory :%d", err) + } + + if !bytes.Equal(mountContents, diskContents) { + t.Fatalf("data mismatch between mounted directory and local disk") + } +} + +func CreateLocalFile(ctx context.Context, t *testing.T, mntDir string, bucket gcs.Bucket, fileName string) (filePath string, f *os.File) { + t.Helper() + // Creating a file shouldn't create file on GCS. + filePath = path.Join(mntDir, fileName) + + f, err := os.Create(filePath) + + assert.Equal(t, nil, err) + ValidateObjectNotFoundErr(ctx, t, bucket, fileName) + return +} + +func CloseLocalFile(t *testing.T, f **os.File) error { + t.Helper() + err := (*f).Close() + *f = nil + return err +} + +func CheckLogFileForMessage(t *testing.T, expectedLog, logFile string) bool { + file, err := os.Open(logFile) + require.NoError(t, err, "Failed to open log file") + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + if strings.Contains(scanner.Text(), expectedLog) { + return true + } + } + return false +} + +// ValidateSyncGivenThatFileIsClobbered method validates sync operation on file which has already been clobbered. +// 1. With streaming writes sync operation only uploads pending buffers and it doesn't return any error. +// 2. Without streaming writes file is synced with GCS and returns ESTALE error. +func ValidateSyncGivenThatFileIsClobbered(t *testing.T, file *os.File, streamingWrites bool) { + t.Helper() + err := file.Sync() + if streamingWrites { + assert.NoError(t, err) + } else { + ValidateESTALEError(t, err) + } +} + +// CreateFileAndCopyToMntDir creates a file of given size. +// The same file will be copied to the mounted directory as well. +func CreateFileAndCopyToMntDir(t *testing.T, fileSize int, dirName string) (string, string) { + testDir := setup.SetupTestDirectory(dirName) + fileInLocalDisk := "test_file" + setup.GenerateRandomString(5) + ".txt" + filePathInLocalDisk := path.Join(os.TempDir(), fileInLocalDisk) + filePathInMntDir := path.Join(testDir, fileInLocalDisk) + CreateFileOnDiskAndCopyToMntDir(t, filePathInLocalDisk, filePathInMntDir, fileSize) + return filePathInLocalDisk, filePathInMntDir +} + +// CreateFileOnDiskAndCopyToMntDir creates a file of given size and copies to given path. +func CreateFileOnDiskAndCopyToMntDir(t *testing.T, filePathInLocalDisk string, filePathInMntDir string, fileSize int) { + // 1. Create the local file + localFile, err := os.Create(filePathInLocalDisk) + if err != nil { + t.Fatalf("Failed to create local file %q: %v", filePathInLocalDisk, err) + } + defer CloseFileShouldNotThrowError(t, localFile) + + buf := make([]byte, OneMiB) + + // 2. Write random data to the local file using the buffer + _, err = io.CopyBuffer(localFile, io.LimitReader(rand.Reader, int64(fileSize)), buf) + if err != nil { + t.Fatalf("Failed to write random data to local file: %v", err) + } + + // Ensure everything is flushed to disk + if err := localFile.Sync(); err != nil { + t.Fatalf("Failed to sync local file to disk: %v", err) + } + + // 3. Rewind the local file pointer to the beginning so we can copy it + if _, err := localFile.Seek(0, io.SeekStart); err != nil { + t.Fatalf("Failed to seek local file to start: %v", err) + } + + // 4. Create the destination file in the mount directory + mntFile, err := os.Create(filePathInMntDir) + if err != nil { + t.Fatalf("Failed to create destination file %q: %v", filePathInMntDir, err) + } + defer CloseFileShouldNotThrowError(t, mntFile) + + // 5. Copy the file over to the mnt dir using io.CopyBuffer with the same buffer + if _, err := io.CopyBuffer(mntFile, localFile, buf); err != nil { + t.Fatalf("Failed to copy file: %v", err) + } + + // Ensure the mount file is also fully flushed + if err := mntFile.Sync(); err != nil { + t.Fatalf("Failed to sync mnt file: %v", err) + } +} + +// StatFileOrFatal stats the given file path and returns the syscall.Stat_t struct. +// It fails the test if os.Stat or the type assertion fails. +func StatFileOrFatal(filePath string, t *testing.T) *syscall.Stat_t { + t.Helper() + fi, err := os.Stat(filePath) + require.NoError(t, err) + stat, ok := fi.Sys().(*syscall.Stat_t) + require.True(t, ok) + return stat +} diff --git a/tools/integration_tests/util/operations/operations.go b/tools/integration_tests/util/operations/operations.go index fdd4d71b62..4c02e78ef5 100644 --- a/tools/integration_tests/util/operations/operations.go +++ b/tools/integration_tests/util/operations/operations.go @@ -42,6 +42,14 @@ func executeToolCommandf(tool string, format string, args ...any) ([]byte, error return runCommand(cmd) } +// Executes any given tool (e.g. gsutil/gcloud). +func executeToolCommand(tool string, command string) ([]byte, error) { + cmdArgs := tool + " " + command + cmd := exec.Command("/bin/bash", "-c", cmdArgs) + + return runCommand(cmd) +} + // Executes any given tool (e.g. gsutil/gcloud) with given args in specified directory. func ExecuteToolCommandfInDirectory(dirPath, tool, format string, args ...any) ([]byte, error) { cmdArgs := tool + " " + fmt.Sprintf(format, args...) @@ -66,7 +74,20 @@ func runCommand(cmd *exec.Cmd) ([]byte, error) { return stdout.Bytes(), nil } -// Executes any given gcloud command with given args. +// ExecuteGcloudCommandf executes any given gcloud command with given args. func ExecuteGcloudCommandf(format string, args ...any) ([]byte, error) { return executeToolCommandf("gcloud", format, args...) } + +// ExecuteGcloudCommand executes any given gcloud command. +func ExecuteGcloudCommand(command string) ([]byte, error) { + return executeToolCommand("gcloud", command) +} + +// WaitForSizeUpdate waits for a specified time duration to ensure that stat() +// call returns correct size for unfinalized object. +func WaitForSizeUpdate(isZonal bool, duration time.Duration) { + if isZonal { + time.Sleep(duration) + } +} diff --git a/tools/integration_tests/util/operations/string_operations.go b/tools/integration_tests/util/operations/string_operations.go index 316e31ff1f..41c789bfbd 100644 --- a/tools/integration_tests/util/operations/string_operations.go +++ b/tools/integration_tests/util/operations/string_operations.go @@ -18,6 +18,8 @@ package operations import ( "strings" "testing" + + "github.com/google/uuid" ) func VerifyExpectedSubstrings(t *testing.T, input string, expectedSubstrings []string) { @@ -35,3 +37,21 @@ func VerifyUnexpectedSubstrings(t *testing.T, input string, unexpectedSubstrings } } } + +func GetRandomName(t *testing.T) string { + id, err := uuid.NewRandom() + if err != nil { + t.Errorf("Error while generating random string, err: %v", err) + } + return id.String() +} + +func SplitBucketNameAndDirPath(t *testing.T, bucketNameWithDirPath string) (bucketName, dirPathInBucket string) { + t.Helper() + + var found bool + if bucketName, dirPathInBucket, found = strings.Cut(bucketNameWithDirPath, "/"); !found { + t.Fatalf("Unexpected bucketNameWithDirPath: %q. Expected form: <bucket>/<object-name>", bucketNameWithDirPath) + } + return +} diff --git a/tools/integration_tests/util/operations/validation_helper.go b/tools/integration_tests/util/operations/validation_helper.go index 9fa2a7dce2..25f18fabca 100644 --- a/tools/integration_tests/util/operations/validation_helper.go +++ b/tools/integration_tests/util/operations/validation_helper.go @@ -15,12 +15,23 @@ package operations import ( + "context" + "errors" "os" "strings" + "syscall" "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/common" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func ValidateNoFileOrDirError(path string, t *testing.T) { +func ValidateNoFileOrDirError(t *testing.T, path string) { + t.Helper() _, err := os.Stat(path) if err == nil || !strings.Contains(err.Error(), "no such file or directory") { t.Fatalf("os.Stat(%s). Expected: %s, Got: %v", path, @@ -28,7 +39,30 @@ func ValidateNoFileOrDirError(path string, t *testing.T) { } } -func CheckErrorForReadOnlyFileSystem(err error, t *testing.T) { +func ValidateObjectNotFoundErr(ctx context.Context, t *testing.T, bucket gcs.Bucket, fileName string) { + t.Helper() + var notFoundErr *gcs.NotFoundError + _, err := storageutil.ReadObject(ctx, bucket, fileName) + + assert.Error(t, err) + assert.True(t, errors.As(err, ¬FoundErr)) +} + +func ValidateESTALEError(t *testing.T, err error) { + t.Helper() + + require.Error(t, err) + assert.Regexp(t, syscall.ESTALE.Error(), err.Error()) +} + +func ValidateEIOError(t *testing.T, err error) { + t.Helper() + + require.Error(t, err) + assert.Regexp(t, syscall.EIO.Error(), err.Error()) +} + +func CheckErrorForReadOnlyFileSystem(t *testing.T, err error) { if err == nil { t.Error("permission denied error expected but got nil error.") return @@ -38,3 +72,86 @@ func CheckErrorForReadOnlyFileSystem(err error, t *testing.T) { } t.Errorf("Incorrect error for readonly file system: %v", err.Error()) } + +func SkipKLCTestForUnsupportedKernelVersion(t *testing.T) { + t.Helper() + unsupported, err := common.IsKLCacheEvictionUnSupported() + assert.NoError(t, err) + if unsupported { + t.SkipNow() + } +} + +// RetryUntil executes the provided operation repeatedly until it succeeds +// (returns a nil error) or the retry deadline is reached. +// +// If the operation succeeds, the result (first return value) is returned. +// If the deadline is exceeded, the test fails via tb.Fatalf. +// +// Example: +// +// result := RetryUntil(ctx, t, 100*time.Millisecond, 5*time.Second, func() (int, error) { +// val, err := doSomething() +// if err != nil { +// return 0, err // retry +// } +// return val, nil // success +// }) +func RetryUntil[T any]( + ctx context.Context, + tb testing.TB, + retryFrequency time.Duration, + retryDeadline time.Duration, + operation func() (T, error), +) T { + tb.Helper() + + if retryFrequency <= 0 { + tb.Fatalf("retryFrequency must be greater than 0") + } + + ctx, cancel := context.WithTimeout(ctx, retryDeadline) + defer cancel() + + ticker := time.NewTicker(retryFrequency) + defer ticker.Stop() + + attempt := 1 + var finalResult T // Variable to hold the result once it succeeds + var lastErr error // Save last error for fatal log + + for { + result, err := operation() + if err == nil { + // It can be helpful to know if an operation was flaky + // but eventually succeeded during verbose test runs. + tb.Logf("Operation succeeded on attempt %d", attempt) + finalResult = result + break // Exit the loop on success + } + + lastErr = err // Save the error before select + + select { + case <-ctx.Done(): + // Log the total number of attempts in the fatal error. + // tb.Fatalf immediately terminates the test execution. + tb.Fatalf( + "Operation failed permanently after %d attempts (deadline: %v). Last error: %v, Context error: %v", + attempt, + retryDeadline, + lastErr, + ctx.Err(), + ) + + case <-ticker.C: + // Log the failure and intent to retry. + // Visible only on failure or with 'go test -v'. + tb.Logf("Attempt %d failed with error: %v. Retrying in %v...", attempt, lastErr, retryFrequency) + attempt++ + } + } + + // Single exit point for a successful execution + return finalResult +} diff --git a/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup/implicit_and_explicit_dir_setup.go b/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup/implicit_and_explicit_dir_setup.go index d20879c4da..2199d32ac2 100644 --- a/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup/implicit_and_explicit_dir_setup.go +++ b/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup/implicit_and_explicit_dir_setup.go @@ -15,15 +15,20 @@ package implicit_and_explicit_dir_setup import ( + "context" "log" "os" "path" + "strings" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/persistent_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/persistent_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" "github.com/stretchr/testify/require" ) @@ -42,24 +47,33 @@ const SecondFileInExplicitDirectory = "fileInExplicitDir2" const FileInImplicitDirectory = "fileInImplicitDir1" const FileInImplicitSubDirectory = "fileInImplicitDir2" -func RunTestsForImplicitDirAndExplicitDir(flags [][]string, m *testing.M) int { - setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() +func RunTestsForExplicitAndImplicitDir(config *test_suite.TestConfig, flags [][]string, m *testing.M) int { + if config == nil { + log.Println("config is nil") + return 1 + } - if setup.TestBucket() == "" && setup.MountedDirectory() != "" { - log.Print("Please pass the name of bucket mounted at mountedDirectory to --testBucket flag.") - os.Exit(1) + if len(flags) == 0 { + log.Println("flags empty: no tests to run") + return 0 } - // Run tests for mountedDirectory only if --mountedDirectory and --testbucket flag is set. - setup.RunTestsForMountedDirectoryFlag(m) + if config.GKEMountedDirectory != "" && config.TestBucket != "" { + successCode := setup.RunTestsForMountedDirectory(config.GKEMountedDirectory, m) + return successCode + } // Run tests for testBucket only if --testbucket flag is set. - setup.SetUpTestDirForTestBucketFlag() + if config.TestBucket == "" { + log.Print("pass test bucket to run the tests") + return 1 + } + setup.SetUpTestDirForTestBucket(config) - successCode := static_mounting.RunTests(flags, m) + successCode := static_mounting.RunTestsWithConfigFile(config, flags, m) if successCode == 0 { - successCode = persistent_mounting.RunTests(flags, m) + successCode = persistent_mounting.RunTestsWithConfigFile(config, flags, m) } return successCode } @@ -74,6 +88,40 @@ func RemoveAndCheckIfDirIsDeleted(dirPath string, dirName string, t *testing.T) } } +// testdataCreateObjects is equivalent of the script tools/integration_tests/util/setup/implicit_and_explicit_dir_setup/testdata/create_objects.sh . +// That script uses gcloud, but this function instead uses go client library. +func testdataCreateObjects(ctx context.Context, t *testing.T, storageClient *storage.Client, testDirWithoutBucketName string) { + t.Helper() + // Following is needed for error log, and because + // TestBucket can be of the form <bucket>/<onlydir> . + bucketName, _, _ := strings.Cut(setup.TestBucket(), "/") + + objectName := path.Join(testDirWithoutBucketName, ImplicitDirectory, FileInImplicitDirectory) + err := client.CreateObjectOnGCS(ctx, storageClient, objectName, "This is from directory fileInImplicitDir1 file implicitDirectory") + if err != nil { + t.Fatalf("Failed to create GCS object %q in bucket %q: %v", objectName, bucketName, err) + } + + objectName = path.Join(testDirWithoutBucketName, path.Join(ImplicitDirectory, ImplicitSubDirectory), FileInImplicitSubDirectory) + err = client.CreateObjectOnGCS(ctx, storageClient, objectName, "This is from directory implicitDirectory/implicitSubDirectory file fileInImplicitDir2") + if err != nil { + t.Fatalf("Failed to create GCS object %q in bucket %q: %v", objectName, bucketName, err) + } +} + +func CreateImplicitDirectoryStructureUsingStorageClient(ctx context.Context, t *testing.T, storageClient *storage.Client, testDir string) { + t.Helper() + + // Implicit Directory Structure + // testBucket/testDir/implicitDirectory -- Dir + // testBucket/testDir/implicitDirectory/fileInImplicitDir1 -- File + // testBucket/testDir/implicitDirectory/implicitSubDirectory -- Dir + // testBucket/testDir/implicitDirectory/implicitSubDirectory/fileInImplicitDir2 -- File + + // Create implicit directory in bucket for testing. + testdataCreateObjects(ctx, t, storageClient, testDir) +} + func CreateImplicitDirectoryStructure(testDir string) { // Implicit Directory Structure // testBucket/testDir/implicitDirectory -- Dir @@ -101,7 +149,7 @@ func CreateExplicitDirectoryStructure(testDir string, t *testing.T) { } // Closing file at the end. - defer operations.CloseFile(file) + defer operations.CloseFileShouldNotThrowError(t, file) } func CreateImplicitDirectoryInExplicitDirectoryStructure(testDir string, t *testing.T) { @@ -114,7 +162,28 @@ func CreateImplicitDirectoryInExplicitDirectoryStructure(testDir string, t *test // testBucket/testDir/explicitDirectory/implicitDirectory/implicitSubDirectory -- Dir // testBucket/testDir/explicitDirectory/implicitDirectory/implicitSubDirectory/fileInImplicitDir2 -- File + // CreateExplicitDirectoryStructure writes files using GCSFuse. CreateExplicitDirectoryStructure(testDir, t) + dirPathInBucket := path.Join(setup.TestBucket(), testDir, ExplicitDirectory) setup.RunScriptForTestData("../util/setup/implicit_and_explicit_dir_setup/testdata/create_objects.sh", dirPathInBucket) } + +func CreateImplicitDirectoryInExplicitDirectoryStructureUsingStorageClient(ctx context.Context, t *testing.T, storageClient *storage.Client, testDir string) { + t.Helper() + + // testBucket/testDir/explicitDirectory -- Dir + // testBucket/testDir/explictFile -- File + // testBucket/testDir/explicitDirectory/fileInExplicitDir1 -- File + // testBucket/testDir/explicitDirectory/fileInExplicitDir2 -- File + // testBucket/testDir/explicitDirectory/implicitDirectory -- Dir + // testBucket/testDir/explicitDirectory/implicitDirectory/fileInImplicitDir1 -- File + // testBucket/testDir/explicitDirectory/implicitDirectory/implicitSubDirectory -- Dir + // testBucket/testDir/explicitDirectory/implicitDirectory/implicitSubDirectory/fileInImplicitDir2 -- File + + // CreateExplicitDirectoryStructure writes files using GCSFuse. + CreateExplicitDirectoryStructure(testDir, t) + + dirPathInBucket := path.Join(testDir, ExplicitDirectory) + testdataCreateObjects(ctx, t, storageClient, dirPathInBucket) +} diff --git a/tools/integration_tests/util/setup/setup.go b/tools/integration_tests/util/setup/setup.go index eacad09a37..dac164b834 100644 --- a/tools/integration_tests/util/setup/setup.go +++ b/tools/integration_tests/util/setup/setup.go @@ -24,30 +24,43 @@ import ( "os/exec" "path" "path/filepath" + "regexp" "runtime/debug" "strings" "testing" "time" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/util" + "cloud.google.com/go/storage/experimental" + auth2 "github.com/googlecloudplatform/gcsfuse/v3/internal/auth" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" + "github.com/googlecloudplatform/gcsfuse/v3/tools/util" + "go.opentelemetry.io/contrib/detectors/gcp" + "go.opentelemetry.io/otel/sdk/resource" "google.golang.org/api/iterator" + "google.golang.org/api/option" ) var isPresubmitRun = flag.Bool("presubmit", false, "Boolean flag to indicate if test-run is a presubmit run.") +var isZonalBucketRun = flag.Bool("zonal", false, "Boolean flag to indicate if test-run should use a zonal bucket.") var testBucket = flag.String("testbucket", "", "The GCS bucket used for the test.") var mountedDirectory = flag.String("mountedDirectory", "", "The GCSFuse mounted directory used for the test.") var integrationTest = flag.Bool("integrationTest", false, "Run tests only when the flag value is true.") var testInstalledPackage = flag.Bool("testInstalledPackage", false, "[Optional] Run tests on the package pre-installed on the host machine. By default, integration tests build a new package to run the tests.") var testOnTPCEndPoint = flag.Bool("testOnTPCEndPoint", false, "Run tests on TPC endpoint only when the flag value is true.") -var seededRand *rand.Rand = rand.New(rand.NewSource(time.Now().UnixNano())) +var gcsfusePreBuiltDir = flag.String("gcsfuse_prebuilt_dir", "", "Path to the pre-built GCSFuse directory containing bin/gcsfuse and sbin/mount.gcsfuse.") +var configFile = flag.String("config-file", "", "Common GCSFuse config file to run tests with.") const ( - FilePermission_0600 = 0600 - DirPermission_0755 = 0755 - Charset = "abcdefghijklmnopqrstuvwxyz0123456789" - PathEnvVariable = "PATH" + FilePermission_0600 = 0600 + DirPermission_0755 = 0755 + Charset = "abcdefghijklmnopqrstuvwxyz0123456789" + PathEnvVariable = "PATH" + GCSFuseLogFilePrefix = "gcsfuse-failed-integration-test-logs-" + ProxyServerLogFilePrefix = "proxy-server-failed-integration-test-logs-" + zoneMatcherRegex = "^[a-z]+-[a-z0-9]+-[a-z]$" + regionMatcherRegex = "^[a-z]+-[a-z0-9]+$" + unsupportedCharactersInTestBucket = " " ) var ( @@ -58,8 +71,35 @@ var ( sbinFile string onlyDirMounted string dynamicBucketMounted string + billingProject string + keyFile string ) +func BillingProject() string { + return billingProject +} + +func SetBillingProject(bp string) { + billingProject = bp +} + +func KeyFile() string { + return keyFile +} + +func SetKeyFile(kf string) { + keyFile = kf +} + +// ReplaceOrAppendFlag replaces the placeholder in the flag if present, +// or appends the value to the flag prefix if the placeholder is not present. +func ReplaceOrAppendFlag(flag, placeholder, flagPrefix, value string) string { + if strings.Contains(flag, placeholder) { + return strings.ReplaceAll(flag, placeholder, value) + } + return strings.ReplaceAll(flag, flagPrefix, flagPrefix+value) +} + // Run the shell script to prepare the testData in the specified bucket. // First argument will be name of scipt script func RunScriptForTestData(args ...string) { @@ -75,6 +115,14 @@ func IsPresubmitRun() bool { return *isPresubmitRun } +func IsZonalBucketRun() bool { + return *isZonalBucketRun +} + +func SetIsZonalBucketRun(val bool) { + *isZonalBucketRun = val +} + func TestBucket() string { return *testBucket } @@ -150,17 +198,47 @@ func CompareFileContents(t *testing.T, fileName string, fileContent string) { } } -func SetUpTestDir() error { +// SetUpTestDir creates a test directory, builds GCSFuse into it and returns it's path. +// This function also creates the mountDir at path testDir/mnt. +func SetUpTestDir() (string, error) { var err error testDir, err = os.MkdirTemp("", "gcsfuse_readwrite_test_") if err != nil { - return fmt.Errorf("TempDir: %w\n", err) + return "", fmt.Errorf("TempDir: %w", err) } - if !TestInstalledPackage() { + // Order of priority to choose GCSFuse installation to run the tests + // 1. Installed package if explicitly asked to + // 2. Prebuilt GCSFuse dir if the said flag is passed + // 3. Build it yourself + if TestInstalledPackage() { + // when testInstalledPackage flag is set, gcsfuse is preinstalled on the + // machine. Hence, here we are overwriting binFile to gcsfuse. + log.Printf("Using GCSFuse installed on the target machine") + binFile = "gcsfuse" + sbinFile = "mount.gcsfuse" + } else if *gcsfusePreBuiltDir != "" { + prebuiltDir := *gcsfusePreBuiltDir + log.Printf("Using GCSFuse from pre-built directory specified by --gcsfuse_prebuilt_dir flag: %s", prebuiltDir) + binFile = filepath.Join(prebuiltDir, "bin/gcsfuse") + sbinFile = filepath.Join(prebuiltDir, "sbin/mount.gcsfuse") + + if _, statErr := os.Stat(binFile); statErr != nil { + return "", fmt.Errorf("gcsfuse binary from --gcsfuse_prebuilt_dir not found at %s: %w", binFile, statErr) + } + if _, statErr := os.Stat(sbinFile); statErr != nil { + return "", fmt.Errorf("mount helper from --gcsfuse_prebuilt_dir not found at %s: %w", sbinFile, statErr) + } + // Set PATH to include the bin directory of the pre-built gcsfuse + err = os.Setenv(PathEnvVariable, filepath.Dir(binFile)+string(filepath.ListSeparator)+os.Getenv(PathEnvVariable)) + if err != nil { + return "", fmt.Errorf("error setting PATH for --gcsfuse_prebuilt_dir: %v", err.Error()) + } + } else { + log.Printf("Building GCSFuse from source in the dir: %s ...", testDir) err = util.BuildGcsfuse(testDir) if err != nil { - return fmt.Errorf("BuildGcsfuse(%q): %w\n", TestDir(), err) + return "", fmt.Errorf("BuildGcsfuse(%q): %w", TestDir(), err) } binFile = path.Join(TestDir(), "bin/gcsfuse") sbinFile = path.Join(TestDir(), "sbin/mount.gcsfuse") @@ -170,30 +248,24 @@ func SetUpTestDir() error { // Setting PATH so that executable is found in test directory. err := os.Setenv(PathEnvVariable, path.Join(TestDir(), "bin")+string(filepath.ListSeparator)+os.Getenv(PathEnvVariable)) if err != nil { - log.Printf("Error in setting PATH environment variable: %v", err.Error()) + return "", fmt.Errorf("error in setting PATH environment variable: %v", err.Error()) } - } else { - // when testInstalledPackage flag is set, gcsfuse is preinstalled on the - // machine. Hence, here we are overwriting binFile to gcsfuse. - binFile = "gcsfuse" - sbinFile = "mount.gcsfuse" } - logFile = path.Join(TestDir(), "gcsfuse.log") - mntDir = path.Join(TestDir(), "mnt") + mntDir = path.Join(TestDir(), "mnt") err = os.Mkdir(mntDir, 0755) if err != nil { - return fmt.Errorf("Mkdir(%q): %v\n", MntDir(), err) + return "", fmt.Errorf("Mkdir(%q): %v", MntDir(), err) } - return nil + return TestDir(), nil } -func UnMount() error { +func UnMount(dir string) error { fusermount, err := exec.LookPath("fusermount") if err != nil { return fmt.Errorf("cannot find fusermount: %w", err) } - cmd := exec.Command(fusermount, "-uz", mntDir) + cmd := exec.Command(fusermount, "-uz", dir) if _, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("fusermount error: %w", err) } @@ -207,6 +279,7 @@ func ExecuteTest(m *testing.M) (successCode int) { } func GenerateRandomString(length int) string { + seededRand := rand.New(rand.NewSource(time.Now().UnixNano())) b := make([]byte, length) for i := range b { b[i] = Charset[seededRand.Intn(len(Charset))] @@ -215,7 +288,7 @@ func GenerateRandomString(length int) string { } func UnMountBucket() { - err := UnMount() + err := UnMount(mntDir) if err != nil { LogAndExit(fmt.Sprintf("Error in unmounting bucket: %v", err)) } @@ -223,15 +296,48 @@ func UnMountBucket() { func SaveLogFileInCaseOfFailure(successCode int) { if successCode != 0 { - // Logfile name will be gcsfuse-failed-integration-test-log-xxxxx - failedlogsFileName := "gcsfuse-failed-integration-test-logs-" + GenerateRandomString(5) - log.Printf("log file is available on kokoro artifacts with file name: %s", failedlogsFileName) - logFileInKokoroArtifact := path.Join(os.Getenv("KOKORO_ARTIFACTS_DIR"), failedlogsFileName) - err := operations.CopyFile(logFile, logFileInKokoroArtifact) - if err != nil { - log.Fatalf("Error in coping logfile in kokoro artifact: %v", err) - } + SaveLogFileAsArtifact(LogFile(), GCSFuseLogFilePrefix+GenerateRandomString(5)) + } +} + +// Saves logFile as given artifactName in KOKORO or +// TestDir based on where the test is ran. +func SaveLogFileAsArtifact(logFile, artifactName string) { + logDir := os.Getenv("KOKORO_ARTIFACTS_DIR") + if logDir == "" { + // Save log files in TestDir as this run is not on KOKORO. + logDir = TestDir() } + artifactPath := path.Join(logDir, artifactName) + logFileData, err := os.ReadFile(logFile) + if err != nil { + log.Fatalf("Error reading log file: %v", err) + } + err = os.WriteFile(artifactPath, logFileData, 0600) + if err != nil { + log.Fatalf("Error in writing log file to artifacts directory: %v", err) + } + log.Printf("Log file saved at %v", artifactPath) +} + +// In case of test failure saves GCSFuse log file to +// KOKORO artifacts directory if test ran on KOKORO +// or saves to TestDir if test ran on local. +func SaveGCSFuseLogFileInCaseOfFailure(tb testing.TB) { + if !tb.Failed() || MountedDirectory() != "" { + return + } + SaveLogFileAsArtifact(LogFile(), GCSFuseLogFilePrefix+strings.ReplaceAll(tb.Name(), "/", "_")+GenerateRandomString(5)) +} + +// In case of test failure saves ProxyServerLogFile to +// KOKORO artifacts directory if test ran on KOKORO +// or saves to TestDir if test ran on local. +func SaveProxyServerLogFileInCaseOfFailure(proxyServerLogFile string, tb testing.TB) { + if !tb.Failed() { + return + } + SaveLogFileAsArtifact(proxyServerLogFile, ProxyServerLogFilePrefix+strings.ReplaceAll(tb.Name(), "/", "_")+GenerateRandomString(5)) } func UnMountAndThrowErrorInFailure(flags []string, successCode int) { @@ -261,6 +367,18 @@ func ParseSetUpFlags() { } } +func ConfigFile() string { + if *configFile == "" { + return "" + } + + absPath, err := filepath.Abs(*configFile) + if err != nil { + log.Fatalf("error decoding config file path: %v", err) + } + return absPath +} + func IgnoreTestIfIntegrationTestFlagIsSet(t *testing.T) { flag.Parse() @@ -269,69 +387,87 @@ func IgnoreTestIfIntegrationTestFlagIsSet(t *testing.T) { } } -func ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() { - ParseSetUpFlags() +// IgnoreTestIfIntegrationTestFlagIsNotSet helps skip a test if --integrationTest flag is not set. +// If the test uses TestMain, then one usually calls os.Exit() to skip the test, +// but for non-TestMain tests, this helps skip integration tests if --integrationTest has not been passed. +func IgnoreTestIfIntegrationTestFlagIsNotSet(t *testing.T) { + flag.Parse() - if *testBucket == "" && *mountedDirectory == "" { - log.Print("--testbucket or --mountedDirectory must be specified") - os.Exit(1) + if !*integrationTest { + t.SkipNow() } } -func ExitWithFailureIfMountedDirectoryIsSetOrTestBucketIsNotSet() { - ParseSetUpFlags() +func IgnoreTestIfPresubmitFlagIsSet(b *testing.B) { + flag.Parse() - if *testBucket == "" { - log.Print("Please pass the name of bucket to be mounted to --testBucket flag. It is required for this test.") - os.Exit(1) + if *isPresubmitRun { + b.SkipNow() } +} - if *mountedDirectory != "" { - log.Print("Please do not pass the mountedDirectory at test runtime. It is not supported for this test.") +func ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() { + ParseSetUpFlags() + + if *testBucket == "" && *mountedDirectory == "" { + log.Print("--testbucket or --mountedDirectory must be specified") os.Exit(1) } } +// Deprecated: Use RunTestsForMountedDirectory instead. +// TODO(b/438068132): cleanup deprecated methods after migration is complete. func RunTestsForMountedDirectoryFlag(m *testing.M) { // Execute tests for the mounted directory. if *mountedDirectory != "" { - mntDir = *mountedDirectory - successCode := ExecuteTest(m) - os.Exit(successCode) + os.Exit(RunTestsForMountedDirectory(*mountedDirectory, m)) } } -func SetUpTestDirForTestBucketFlag() { - if TestBucket() == "" { - log.Fatal("Not running TestBucket tests as --testBucket flag is not set.") - } - if err := SetUpTestDir(); err != nil { - log.Printf("setUpTestDir: %v\n", err) - os.Exit(1) +// RunTestsForMountedDirectory executes tests for the mounted directory. +// User is expected to ensure that this function is called when mounted directory is set. +// Returns exit code. +func RunTestsForMountedDirectory(mountedDirectory string, m *testing.M) int { + // Execute tests for the mounted directory. + if mountedDirectory == "" { + log.Println("RunTestsForMountedDirectory failed: Mounted directory is not set.") + return 1 } + mntDir = mountedDirectory + return ExecuteTest(m) } -func SetUpLogDirForTestDirTests(logDirName string) (logDir string) { - logDir = path.Join(TestDir(), logDirName) - err := os.Mkdir(logDir, DirPermission_0755) - if err != nil { - log.Printf("os.Mkdir %s: %v\n", logDir, err) - os.Exit(1) +// SetUpTestDirForTestBucketFlag is Deprecated: Use SetUpTestDirForTestBucket instead. +// TODO(b/438068132): cleanup deprecated methods after migration is complete. +func SetUpTestDirForTestBucketFlag() { + cfg := &test_suite.TestConfig{ + GKEMountedDirectory: MountedDirectory(), + GCSFuseMountedDirectory: MntDir(), + TestBucket: TestBucket(), + LogFile: LogFile(), } - return + SetUpTestDirForTestBucket(cfg) } -func ValidateLogDirForMountedDirTests(logDirName string) (logDir string) { - if *mountedDirectory == "" { - return "" +// SetUpTestDirForTestBucket creates a test directory with GCSFuse binaries, mount point, log file, etc. +// Test config is passed by reference so it can set the LogFile, mountPath variables in the config +func SetUpTestDirForTestBucket(cfg *test_suite.TestConfig) { + if cfg.TestBucket == "" { + log.Fatal("Not running TestBucket tests as --testBucket flag is not set.") } - logDir = path.Join(os.TempDir(), logDirName) - _, err := os.Stat(logDir) + if strings.ContainsAny(cfg.TestBucket, unsupportedCharactersInTestBucket) { + log.Fatalf("Passed testBucket %q contains one or more of the following unsupported character(s): %q", cfg.TestBucket, unsupportedCharactersInTestBucket) + } + testDirPath, err := SetUpTestDir() if err != nil { - log.Printf("validateLogDirForMountedDirTests %s: %v\n", logDir, err) + log.Printf("setUpTestDir: %v\n", err) os.Exit(1) } - return + + cfg.GCSFuseMountedDirectory = path.Join(testDirPath, "mnt") + cfg.LogFile = path.Join(TestDir(), "gcsfuse.log") + // TODO: clean up this global variable up after migration is complete. + SetLogFile(cfg.LogFile) } func LogAndExit(s string) { @@ -355,11 +491,24 @@ func CleanUpDir(directoryPath string) { } } -// SetupTestDirectory creates a testDirectory in the mounted directory and cleans up -// any content present in it. +// SetupTestDirectory creates a test directory hierarchy in the mounted directory, +// cleaning up any content present. It takes a testDirName which can include +// slashes to create nested directories (e.g., "a/b/c"). func SetupTestDirectory(testDirName string) string { testDirPath := path.Join(MntDir(), testDirName) - err := os.Mkdir(testDirPath, DirPermission_0755) + err := os.MkdirAll(testDirPath, DirPermission_0755) + if err != nil && !strings.Contains(err.Error(), "file exists") { + log.Printf("Error while setting up directory %s for testing: %v", testDirPath, err) + } + CleanUpDir(testDirPath) + return testDirPath +} + +// SetupTestDirectoryRecursive recursively creates a testDirectory in the mounted directory and cleans up +// any content present in it. +func SetupTestDirectoryRecursive(testDirName string) string { + testDirPath := path.Join(MntDir(), testDirName) + err := os.MkdirAll(testDirPath, DirPermission_0755) if err != nil && !strings.Contains(err.Error(), "file exists") { log.Printf("Error while setting up directory %s for testing: %v", testDirPath, err) } @@ -369,8 +518,12 @@ func SetupTestDirectory(testDirName string) string { // CleanupDirectoryOnGCS cleans up the object/directory path passed in parameter. func CleanupDirectoryOnGCS(ctx context.Context, client *storage.Client, directoryPathOnGCS string) { - bucket, dirPath := GetBucketAndObjectBasedOnTypeOfMount(directoryPathOnGCS) + bucketAndDirPath := strings.Split(directoryPathOnGCS, "/") + bucket, dirPath := bucketAndDirPath[0], bucketAndDirPath[1] bucketHandle := client.Bucket(bucket) + if bp := BillingProject(); bp != "" { + bucketHandle = bucketHandle.UserProject(bp) + } it := bucketHandle.Objects(ctx, &storage.Query{Prefix: dirPath + "/"}) for { @@ -395,8 +548,14 @@ func AreBothMountedDirectoryAndTestBucketFlagsSet() bool { return false } +// Deprecated: use ResolveIsHierarchicalBucket instead. +// TODO(b/438068132): cleanup deprecated methods after migration is complete. func IsHierarchicalBucket(ctx context.Context, storageClient *storage.Client) bool { - attrs, err := storageClient.Bucket(TestBucket()).Attrs(ctx) + return ResolveIsHierarchicalBucket(ctx, TestBucket(), storageClient) +} + +func ResolveIsHierarchicalBucket(ctx context.Context, testBucket string, storageClient *storage.Client) bool { + attrs, err := storageClient.Bucket(testBucket).Attrs(ctx) if err != nil { return false } @@ -407,6 +566,101 @@ func IsHierarchicalBucket(ctx context.Context, storageClient *storage.Client) bo return false } +// TestEnvironment sets the global variables like test bucket, mount point, log file and isZonalBucket variable +// based on the bucket type. Also returns the bucket type. +func TestEnvironment(ctx context.Context, cfg *test_suite.TestConfig) string { + // TODO: clean up SetGlobalVars after migration completes. + SetGlobalVars(cfg) + bucketType, err := BucketType(ctx, cfg.TestBucket) + if err != nil { + log.Fatalf("BucketType failed: %v", err) + } + if bucketType == ZonalBucket { + SetIsZonalBucketRun(true) + } + + return bucketType +} + +const FlatBucket = "flat" +const HNSBucket = "hns" +const ZonalBucket = "zonal" + +func BucketType(ctx context.Context, testBucket string) (bucketType string, err error) { + // For only-dir mounts bucket name is passed as <test_bucket>/<only_dir> by GKE. + testBucket = strings.Split(testBucket, "/")[0] + ctx, cancel := context.WithCancel(ctx) + defer cancel() + var opts []option.ClientOption + opts = append(opts, experimental.WithGRPCBidiReads()) + if keyFile != "" { + cred, err := auth2.GetCredentials(keyFile) + if err != nil { + return "", fmt.Errorf("failed to get credentials: %w", err) + } + opts = append(opts, option.WithAuthCredentials(cred)) + } + storageClient, err := storage.NewGRPCClient(ctx, opts...) + if err != nil { + return "", fmt.Errorf("failed to create storage client: %w", err) + } + bucket := storageClient.Bucket(testBucket) + if billingProject != "" { + bucket = bucket.UserProject(billingProject) + } + attrs, err := bucket.Attrs(ctx) + if err != nil { + return "", fmt.Errorf("failed to get bucket attributes: %w", err) + } + if attrs.LocationType == "zone" { + return ZonalBucket, nil + } + if attrs.HierarchicalNamespace != nil && attrs.HierarchicalNamespace.Enabled { + return HNSBucket, nil + } + return FlatBucket, nil +} + +// BuildFlagSets dynamically builds a list of flag sets based on bucket compatibility. +// bucketType should be "flat", "hns", or "zonal". +// The run parameter filters flag sets based on the 'Run' field in the test +// configuration, which typically corresponds to a specific test name. If run is +// an empty string, all flag sets for the package are returned. +func BuildFlagSets(cfg test_suite.TestConfig, bucketType string, run string) [][]string { + // In case of mounted-directory, no need to + // parse flags. Just return a single + // set of empty flags to run only one test case + // for a single `go test` command. + if cfg.GKEMountedDirectory != "" { + return [][]string{{""}} + } + + var dynamicFlags [][]string + + // 1. Iterate through each defined test configuration (e.g., HTTP, gRPC). + for _, testCase := range cfg.Configs { + // 2. Check if the current test case is compatible with the bucket type. + // This is a safe and concise way to check the map. + isCompatible, ok := testCase.Compatible[bucketType] + if ok && isCompatible && (run == "" || run == testCase.Run) { + // 3. If compatible, process its flags and add them to the result. + for _, flagString := range testCase.Flags { + flagString = strings.ReplaceAll(flagString, ",", " ") + dynamicFlags = append(dynamicFlags, strings.Fields(flagString)) + } + } + } + return dynamicFlags +} + +func SetGlobalVars(cfg *test_suite.TestConfig) { + // TODO: clean global variables after test migration to config file completes. + testBucket = &cfg.TestBucket + logFile = cfg.LogFile + mntDir = cfg.GKEMountedDirectory + onlyDirMounted = cfg.OnlyDir +} + // Explicitly set the enable-hns config flag to true when running tests on the HNS bucket. func AddHNSFlagForHierarchicalBucket(ctx context.Context, storageClient *storage.Client) ([]string, error) { if !IsHierarchicalBucket(ctx, storageClient) { @@ -414,7 +668,7 @@ func AddHNSFlagForHierarchicalBucket(ctx context.Context, storageClient *storage } var flags []string - mountConfig4 := map[string]interface{}{ + mountConfig4 := map[string]any{ "enable-hns": true, } filePath4 := YAMLConfigFile(mountConfig4, "config_hns.yaml") @@ -458,6 +712,27 @@ func MountGCSFuseWithGivenMountFunc(flags []string, mountFunc func([]string) err } } +func MountGCSFuseWithGivenMountWithConfigFunc(config *test_suite.TestConfig, flags []string, mountFunc func(*test_suite.TestConfig, []string) error) { + if config.GKEMountedDirectory == "" { + // Mount GCSFuse only when tests are not running on mounted directory. + if err := mountFunc(config, flags); err != nil { + LogAndExit(fmt.Sprintf("Failed to mount GCSFuse: %v", err)) + } + } +} + +// MayMountGCSFuseWithGivenMountWithConfigFunc is similar to MountGCSFuseWithGivenMountWithConfigFunc, +// except that it returns error on failure, instead of panic'ing. +func MayMountGCSFuseWithGivenMountWithConfigFunc(config *test_suite.TestConfig, flags []string, mountFunc func(*test_suite.TestConfig, []string) error) error { + if config.GKEMountedDirectory == "" { + // Mount GCSFuse only when tests are not running on mounted directory. + if err := mountFunc(config, flags); err != nil { + return fmt.Errorf("Failed to mount GCSFuse: %w", err) + } + } + return nil +} + func UnmountGCSFuseAndDeleteLogFile(rootDir string) { UnmountGCSFuse(rootDir) // delete log file created @@ -473,7 +748,17 @@ func UnmountGCSFuse(rootDir string) { SetMntDir(rootDir) if *mountedDirectory == "" { // Unmount GCSFuse only when tests are not running on mounted directory. - err := UnMount() + err := UnMount(mntDir) + if err != nil { + LogAndExit(fmt.Sprintf("Error in unmounting bucket: %v", err)) + } + } +} + +func UnmountGCSFuseWithConfig(cfg *test_suite.TestConfig) { + if cfg.GKEMountedDirectory == "" { + // Unmount GCSFuse only when tests are not running on mounted directory. + err := UnMount(cfg.GCSFuseMountedDirectory) if err != nil { LogAndExit(fmt.Sprintf("Error in unmounting bucket: %v", err)) } @@ -505,3 +790,153 @@ func AppendFlagsToAllFlagsInTheFlagsSet(flagsSet *[][]string, newFlags ...string } *flagsSet = resultFlagsSet } + +func CreateProxyServerLogFile(t *testing.T) string { + proxyServerLogFile := path.Join(TestDir(), "proxy-server-log-"+GenerateRandomString(5)) + _, err := os.Create(proxyServerLogFile) + if err != nil { + t.Fatalf("Error in creating log file for proxy server: %v", err) + } + return proxyServerLogFile +} + +func AppendProxyEndpointToFlagSet(flagSet *[]string, port int) { + *flagSet = append(*flagSet, "--custom-endpoint="+fmt.Sprintf("http://localhost:%d/storage/v1/", port)) +} + +// GetGCEZone returns the GCE zone of the current machine from +// the GCP resource detector. +func GetGCEZone(ctx context.Context) (string, error) { + detectedAttrs, err := resource.New(ctx, resource.WithDetectors(gcp.NewDetector())) + if err != nil { + return "", fmt.Errorf("failed to fetch GCP resource detector: %w", err) + } + attrs := detectedAttrs.Set() + if zoneValue, exists := attrs.Value("cloud.availability_zone"); exists { + zone := zoneValue.AsString() + // Confirm that the zone string is in right format e.g. us-central1-a. + if match, err := regexp.MatchString(zoneMatcherRegex, zone); !match || err != nil { + return zone, fmt.Errorf("zone %q returned by GCP resource detector is not a valid zone-string: %w", zone, err) + } + return zone, nil + } + return "", fmt.Errorf("cloud.availability_zone not found in GCP resource detector") +} + +// GetGCERegion return the GCE region for a given GCE zone. +// E.g. from us-central1-a, it returns us-central1. +func GetGCERegion(gceZone string) (string, error) { + indexOfLastHyphen := strings.LastIndex(gceZone, "-") + if indexOfLastHyphen < 0 { + return "", fmt.Errorf("input gceZone %q is not proper. It is expected to be of the form <country>-<region>-<zone> e.g. us-central1-a.", gceZone) + } + region := gceZone[:indexOfLastHyphen] + + // Confirm that the region string is in right format e.g. us-central1. + if match, err := regexp.MatchString(regionMatcherRegex, region); !match || err != nil { + return region, fmt.Errorf("zone %q returned by GCE metadata server is not a valid zone-string: %w", region, err) + } + return region, nil +} + +// IsDynamicMount returns true if the mount is dynamic. +// In dynamic mounts, rootDir contains all buckets, and mountDir is the specific bucket directory. +func IsDynamicMount(mountDir, rootDir string) bool { + return mountDir != rootDir +} + +// ExtractServiceVersionFromFlags parses the cloud-profiler-label from a slice of flag strings. +func ExtractServiceVersionFromFlags(flags []string) string { + // Regex to find --cloud-profiler-label=some_value or --cloud-profiler-label some_value + re := regexp.MustCompile(`--cloud-profiler-label[=\s]([^\s]+)`) + for _, flagSet := range flags { + matches := re.FindStringSubmatch(flagSet) + // matches[0] is the full match, e.g., "--cloud-profiler-label=v1" + // matches[1] is the first capturing group, e.g., "v1" + if len(matches) > 1 { + return matches[1] + } + } + log.Fatal("Profile label should have been provided for mounted directory test.") + return "" +} + +// CloudProfilerServiceNameFromFlags parses the cloud-profiler-service-name from a slice of flag strings. +func CloudProfilerServiceNameFromFlags(flags []string) string { + // Regex to find --cloud-profiler-service-name=some_value or --cloud-profiler-service-name some_value + re := regexp.MustCompile(`--cloud-profiler-service-name[=\s]([^\s]+)`) + for _, flagSet := range flags { + matches := re.FindStringSubmatch(flagSet) + // matches[0] is the full match, e.g., "--cloud-profiler-service-name=v1" + // matches[1] is the first capturing group, e.g., "v1" + if len(matches) > 1 { + return matches[1] + } + } + // If not provided then return default value for profiler service name. + return "gcsfuse" +} + +func OverrideFilePathsInFlagSet(t *test_suite.TestConfig, GCSFuseTempDirPath string) { + for _, flags := range t.Configs { + for i := range flags.Flags { + // Iterate over the indices of the flags slice + flags.Flags[i] = strings.ReplaceAll(flags.Flags[i], "/gcsfuse-tmp", path.Join(GCSFuseTempDirPath, "gcsfuse-tmp")) + } + for i := range flags.SecondaryFlags { + // Iterate over the indices of the secondary flags slice + flags.SecondaryFlags[i] = strings.ReplaceAll(flags.SecondaryFlags[i], "/gcsfuse-tmp", path.Join(GCSFuseTempDirPath, "gcsfuse-tmp")) + } + } +} + +func ParseLogFileFromFlags(flags []string) string { + for _, flagStr := range flags { + flagStr = strings.ReplaceAll(flagStr, ",", " ") + parts := strings.Fields(flagStr) + for _, part := range parts { + if strings.HasPrefix(part, "--log-file=") { + // Get just the filename from the path + return path.Base(strings.TrimPrefix(part, "--log-file=")) + } + } + } + return "" +} + +func SetUpLogFilePath(testName string, flags []string, GKETempDir string, OldGKElogFilePath string, cfg *test_suite.TestConfig) { + var logFilePath string + parsedLogFileName := ParseLogFileFromFlags(flags) + + // Infer log filename directly from the parsed config block. + if parsedLogFileName == "" && cfg != nil { + currentTest := strings.Trim(testName, "^$") + currentTest = strings.Split(currentTest, "/")[0] + + if currentTest != "" { + for _, c := range cfg.Configs { + if c.Run == currentTest { + parsedLogFileName = ParseLogFileFromFlags(c.Flags) + break + } + } + } + } + + // Default logFile name. + if parsedLogFileName == "" { + parsedLogFileName = "gcsfuse-tmp.log" + } + + if cfg != nil && cfg.GKEMountedDirectory != "" { // GKE path + logFilePath = path.Join(GKETempDir, parsedLogFileName) + if ConfigFile() == "" { + // TODO: clean this up when GKE test migration completes. + logFilePath = OldGKElogFilePath + } + } else { + logFilePath = path.Join(TestDir(), GKETempDir, parsedLogFileName) + } + cfg.LogFile = logFilePath + SetLogFile(logFilePath) +} diff --git a/tools/integration_tests/util/setup/yaml-config.go b/tools/integration_tests/util/setup/yaml-config.go index 826d035fea..997777e4e2 100644 --- a/tools/integration_tests/util/setup/yaml-config.go +++ b/tools/integration_tests/util/setup/yaml-config.go @@ -22,7 +22,7 @@ import ( "gopkg.in/yaml.v3" ) -func YAMLConfigFile(yamlContent interface{}, fileName string) (filePath string) { +func YAMLConfigFile(yamlContent any, fileName string) (filePath string) { yamlData, err := yaml.Marshal(yamlContent) if err != nil { LogAndExit(fmt.Sprintf("Error while marshaling config file: %v", err)) diff --git a/tools/integration_tests/util/test_setup/test_setup.go b/tools/integration_tests/util/test_setup/test_setup.go deleted file mode 100644 index 075fc42354..0000000000 --- a/tools/integration_tests/util/test_setup/test_setup.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package test_setup implements Setup and Teardown methods to be used in tests. -package test_setup - -import ( - "reflect" - "strings" - "testing" -) - -// Testable defines Tester's methods for use in this package. -type Testable interface { - Setup(*testing.T) - Teardown(*testing.T) -} - -func getTestFunc(t *testing.T, xv reflect.Value, name string) func(*testing.T) { - if m := xv.MethodByName(name); m.IsValid() { - if f, ok := m.Interface().(func(*testing.T)); ok { - return f - } - // Method exists but has the wrong type signature. - t.Fatalf("test function %v has unexpected signature (%T)", name, m.Interface()) - } - return func(*testing.T) {} -} - -// RunTests runs all "Test*" functions that are member of x as subtests -// of the current test. Setup is run before the test function and Teardown is -// run after each test. -// x must extend Testable interface by implementing Setup and TearDown methods. -func RunTests(t *testing.T, x Testable) { - xt := reflect.TypeOf(x) - xv := reflect.ValueOf(x) - - for i := 0; i < xt.NumMethod(); i++ { - methodName := xt.Method(i).Name - if !strings.HasPrefix(methodName, "Test") { - continue - } - testFunc := getTestFunc(t, xv, methodName) - t.Run(methodName, func(t *testing.T) { - // Execute Teardown in t.Cleanup() to guarantee it is run even if test - // function or setup uses t.Fatal(). - t.Cleanup(func() { x.Teardown(t) }) - x.Setup(t) - testFunc(t) - }) - } -} diff --git a/tools/integration_tests/util/test_setup/test_setup_test.go b/tools/integration_tests/util/test_setup/test_setup_test.go deleted file mode 100644 index 0196ff61b7..0000000000 --- a/tools/integration_tests/util/test_setup/test_setup_test.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test_setup_test - -import ( - "testing" - - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/test_setup" - . "github.com/jacobsa/ogletest" -) - -type testStructure struct { - setupCtr, teardownCtr, test1, test2 int -} - -func (t *testStructure) Setup(*testing.T) { - t.setupCtr++ -} -func (t *testStructure) TestExample1(*testing.T) { - t.test1++ -} - -func (t *testStructure) TestExample2(*testing.T) { - t.test2++ -} - -func (t *testStructure) Teardown(*testing.T) { - t.teardownCtr++ -} - -func TestRunTests(t *testing.T) { - setup.IgnoreTestIfIntegrationTestFlagIsSet(t) - - testStruct := &testStructure{} - - test_setup.RunTests(t, testStruct) - - AssertEq(testStruct.setupCtr, 2) - AssertEq(testStruct.test1, 1) - AssertEq(testStruct.test2, 1) - AssertEq(testStruct.teardownCtr, 2) -} diff --git a/tools/integration_tests/util/test_suite/config.go b/tools/integration_tests/util/test_suite/config.go new file mode 100644 index 0000000000..f3cd26eedc --- /dev/null +++ b/tools/integration_tests/util/test_suite/config.go @@ -0,0 +1,109 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test_suite + +import ( + "log" + "os" + + "gopkg.in/yaml.v3" +) + +// BucketType represents the 'compatible' field. +type BucketType struct { + Flat bool `yaml:"flat"` + Hns bool `yaml:"hns"` + Zonal bool `yaml:"zonal"` +} + +// TestConfig represents the common configuration for test packages. +type TestConfig struct { + GKEMountedDirectory string `yaml:"mounted_directory"` + GKEMountedDirectorySecondary string `yaml:"mounted_directory_secondary"` + GCSFuseMountedDirectory string + GCSFuseMountedDirectorySecondary string + TestBucket string `yaml:"test_bucket"` + LogFile string + Configs []ConfigItem `yaml:"configs"` + OnlyDir string `yaml:"only_dir,omitempty"` +} + +// ConfigItem defines the variable parts of each test run. +type ConfigItem struct { + Flags []string `yaml:"flags"` + SecondaryFlags []string `yaml:"secondary_flags"` + Compatible map[string]bool `yaml:"compatible"` + Run string `yaml:"run,omitempty"` + RunOnGKE bool `yaml:"run_on_gke"` +} + +// Config holds all test configurations parsed from the YAML file. +type Config struct { + ImplicitDir []TestConfig `yaml:"implicit_dir"` + ExplicitDir []TestConfig `yaml:"explicit_dir"` + ListLargeDir []TestConfig `yaml:"list_large_dir"` + WriteLargeFiles []TestConfig `yaml:"write_large_files"` + Operations []TestConfig `yaml:"operations"` + ReadLargeFiles []TestConfig `yaml:"read_large_files"` + ReadOnly []TestConfig `yaml:"readonly"` + ReadCache []TestConfig `yaml:"read_cache"` + RenameDirLimit []TestConfig `yaml:"rename_dir_limit"` + Gzip []TestConfig `yaml:"gzip"` + LocalFile []TestConfig `yaml:"local_file"` + LogRotation []TestConfig `yaml:"log_rotation"` + ManagedFolders []TestConfig `yaml:"managed_folders"` + ConcurrentOperations []TestConfig `yaml:"concurrent_operations"` + Benchmarking []TestConfig `yaml:"benchmarking"` + BufferedRead []TestConfig `yaml:"buffered_read"` + StaleHandle []TestConfig `yaml:"stale_handle"` + StreamingWrites []TestConfig `yaml:"streaming_writes"` + InactiveStreamTimeout []TestConfig `yaml:"inactive_stream_timeout"` + CloudProfiler []TestConfig `yaml:"cloud_profiler"` + KernelListCache []TestConfig `yaml:"kernel_list_cache"` + ReadDirPlus []TestConfig `yaml:"readdirplus"` + DentryCache []TestConfig `yaml:"dentry_cache"` + RequesterPaysBucket []TestConfig `yaml:"requester_pays_bucket"` + ReadGCSAlgo []TestConfig `yaml:"read_gcs_algo"` + Interrupt []TestConfig `yaml:"interrupt"` + UnfinalizedObject []TestConfig `yaml:"unfinalized_object"` + RapidAppends []TestConfig `yaml:"rapid_appends"` + MountTimeout []TestConfig `yaml:"mount_timeout"` + Monitoring []TestConfig `yaml:"monitoring"` + FlagOptimizations []TestConfig `yaml:"flag_optimizations"` + UnsupportedPath []TestConfig `yaml:"unsupported_path"` + SymlinkHandling []TestConfig `yaml:"symlink_handling"` + SharedChunkCache []TestConfig `yaml:"shared_chunk_cache"` + ReadonlyCreds []TestConfig `yaml:"readonly_creds"` + ReleaseVersion []TestConfig `yaml:"release_version"` + Mounting []TestConfig `yaml:"mounting"` + NegativeStatCache []TestConfig `yaml:"negative_stat_cache"` +} + +// ReadConfigFile returns a Config struct from the YAML file. +func ReadConfigFile(configFilePath string) Config { + var cfg Config + if configFilePath != "" { + configData, err := os.ReadFile(configFilePath) + if err != nil { + log.Fatalf("could not read config file %q: %v", configFilePath, err) + } + expandedYaml := os.ExpandEnv(string(configData)) + if err := yaml.Unmarshal([]byte(expandedYaml), &cfg); err != nil { + log.Fatalf("Failed to parse config YAML: %v", err) + } + } + + return cfg +} diff --git a/tools/integration_tests/util/test_suite/testify_interface.go b/tools/integration_tests/util/test_suite/testify_interface.go new file mode 100644 index 0000000000..9320b11576 --- /dev/null +++ b/tools/integration_tests/util/test_suite/testify_interface.go @@ -0,0 +1,22 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test_suite + +import "github.com/stretchr/testify/suite" + +type TestifySuite interface { + suite.TestingSuite + Run(name string, subtest func()) bool +} diff --git a/tools/integration_tests/write_large_files/concurrent_write_files_test.go b/tools/integration_tests/write_large_files/concurrent_write_files_test.go index 29ecb8fa1d..7c076817fd 100644 --- a/tools/integration_tests/write_large_files/concurrent_write_files_test.go +++ b/tools/integration_tests/write_large_files/concurrent_write_files_test.go @@ -15,14 +15,13 @@ package write_large_files import ( - "fmt" - "os" "path" - "syscall" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ) @@ -30,60 +29,34 @@ const ( DirForConcurrentWrite = "dirForConcurrentWrite" ) -var FileOne = "fileOne" + setup.GenerateRandomString(5) + ".txt" -var FileTwo = "fileTwo" + setup.GenerateRandomString(5) + ".txt" -var FileThree = "fileThree" + setup.GenerateRandomString(5) + ".txt" - -func writeFile(fileName string, fileSize int64, t *testing.T) error { - filePath := path.Join(setup.MntDir(), DirForConcurrentWrite, fileName) - f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|syscall.O_DIRECT, WritePermission_0200) - if err != nil { - return fmt.Errorf("Open file for write at start: %v", err) - } - - // Closing file at the end. - defer operations.CloseFile(f) - - err = operations.WriteChunkOfRandomBytesToFile(f, int(fileSize), 0) - if err != nil { - return fmt.Errorf("Error: %v", err) - } - - filePathInGcsBucket := path.Join(DirForConcurrentWrite, fileName) - localFilePath := path.Join(TmpDir, fileName) - err = compareFileFromGCSBucketAndMntDir(filePathInGcsBucket, filePath, localFilePath, t) - if err != nil { - return fmt.Errorf("Error: %v", err) - } - - return nil -} - -func TestMultipleFilesAtSameTime(t *testing.T) { - concurrentWriteDir := path.Join(setup.MntDir(), DirForConcurrentWrite) - setup.SetupTestDirectory(DirForConcurrentWrite) - - // Clean up. - defer operations.RemoveDir(concurrentWriteDir) - +func TestWriteMultipleFilesConcurrently(t *testing.T) { + concurrentWriteDir := setup.SetupTestDirectory(DirForConcurrentWrite) + var FileOne = "fileOne" + setup.GenerateRandomString(5) + ".txt" + var FileTwo = "fileTwo" + setup.GenerateRandomString(5) + ".txt" + var FileThree = "fileThree" + setup.GenerateRandomString(5) + ".txt" files := []string{FileOne, FileTwo, FileThree} - var eG errgroup.Group - // Concurrently write three files. + // Concurrently write three different files. for i := range files { // Copy the current value of i into a local variable to avoid data races. fileIndex := i // Thread to write the current file. eG.Go(func() error { - return writeFile(files[fileIndex], FiveHundredMB, t) + mountedDirFilePath := path.Join(concurrentWriteDir, files[fileIndex]) + localFilePath := path.Join(TmpDir, setup.GenerateRandomString(5)) + t.Cleanup(func() { operations.RemoveFile(localFilePath) }) + + operations.WriteFilesSequentially(t, []string{localFilePath, mountedDirFilePath}, FiveHundredMB, ChunkSize) + + identical, err := operations.AreFilesIdentical(mountedDirFilePath, localFilePath) + require.NoError(t, err) + assert.True(t, identical) + return nil }) } // Wait on threads to end. - err := eG.Wait() - if err != nil { - t.Fatalf("Error: %v", err) - } + _ = eG.Wait() } diff --git a/tools/integration_tests/write_large_files/concurrent_write_to_same_file_test.go b/tools/integration_tests/write_large_files/concurrent_write_to_same_file_test.go new file mode 100644 index 0000000000..4e3551f277 --- /dev/null +++ b/tools/integration_tests/write_large_files/concurrent_write_to_same_file_test.go @@ -0,0 +1,75 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Provides integration tests for write large files sequentially and randomly. + +package write_large_files + +import ( + "path" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" +) + +func TestWriteToSameFileConcurrently(t *testing.T) { + // Setup Test directory and files to write to. + seqWriteDir := setup.SetupTestDirectory(DirForSeqWrite) + mountedDirFilePath := path.Join(seqWriteDir, setup.GenerateRandomString(5)) + localFilePath := path.Join(TmpDir, setup.GenerateRandomString(5)) + t.Cleanup(func() { operations.RemoveFile(localFilePath) }) + // We will have x numbers of concurrent writers trying to write to the same file. + // Every thread will start at offset = writer_index * (fileSize/thread_count). + var eG errgroup.Group + concurrentWriterCount := 5 + chunkSize := 50 * OneMiB / concurrentWriterCount + + for i := range concurrentWriterCount { + offset := i * chunkSize + eG.Go(func() error { + writeToFileSequentially(t, []string{localFilePath, mountedDirFilePath}, offset, offset+chunkSize) + return nil + }) + } + + // Wait on threads to end. + err := eG.Wait() + require.NoError(t, err) + identical, err := operations.AreFilesIdentical(mountedDirFilePath, localFilePath) + require.NoError(t, err) + assert.True(t, identical) +} + +func writeToFileSequentially(t *testing.T, filePaths []string, startOffset int, endOffset int) { + t.Helper() + filesToWrite := operations.OpenFiles(t, filePaths) + defer operations.CloseFiles(t, filesToWrite) + + var chunkSize = OneMiB + for startOffset < endOffset { + chunkSize = min(chunkSize, endOffset-startOffset) + + err := operations.WriteChunkOfRandomBytesToFiles(filesToWrite, chunkSize, int64(startOffset)) + require.NoError(t, err) + + startOffset = startOffset + chunkSize + } + if setup.IsZonalBucketRun() { + operations.SyncFiles(filesToWrite, t) + } +} diff --git a/tools/integration_tests/write_large_files/random_write_large_file_test.go b/tools/integration_tests/write_large_files/random_write_large_file_test.go index 267b0d4314..302cb62b77 100644 --- a/tools/integration_tests/write_large_files/random_write_large_file_test.go +++ b/tools/integration_tests/write_large_files/random_write_large_file_test.go @@ -16,57 +16,42 @@ package write_large_files import ( rand2 "math/rand" - "os" "path" - "syscall" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/util" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) const ( NumberOfRandomWriteCalls = 20 DirForRandomWrite = "dirForRandomWrite" - MaxWritableByteFromFile = 500 * OneMiB + MaxFileOffset = 500 * OneMiB ) -var FiveHundredMBFileForRandomWriteInLocalSystem string = "fiveHundredMBFileForRandomWriteInLocalSystem" + setup.GenerateRandomString(5) - func TestWriteLargeFileRandomly(t *testing.T) { - randomWriteDir := path.Join(setup.MntDir(), DirForRandomWrite) - setup.SetupTestDirectory(DirForRandomWrite) - filePath := path.Join(randomWriteDir, FiveHundredMBFile) - - // Clean up. - defer operations.RemoveDir(randomWriteDir) - - f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|syscall.O_DIRECT, WritePermission_0200) - if err != nil { - t.Fatalf("Open file for write at start: %v", err) + // Setup Test directory and files to write to. + randomWriteDir := setup.SetupTestDirectory(DirForRandomWrite) + mountedDirFilePath := path.Join(randomWriteDir, FiveHundredMBFile) + localFilePath := path.Join(TmpDir, setup.GenerateRandomString(5)) + t.Cleanup(func() { operations.RemoveFile(localFilePath) }) + // Open local file and mounted directory file for writing. + filesToWrite := operations.OpenFiles(t, []string{localFilePath, mountedDirFilePath}) + + for range NumberOfRandomWriteCalls { + offset := rand2.Int63n(MaxFileOffset - ChunkSize) + // Aligning to 4KiB page boundaries is required for O_DIRECT writes to local files. + offset = offset - offset%(4*util.KiB) + // Write chunk of random data to both local and mounted directory file. + err := operations.WriteChunkOfRandomBytesToFiles(filesToWrite, ChunkSize, offset) + require.NoError(t, err) } - defer operations.CloseFile(f) - - for i := 0; i < NumberOfRandomWriteCalls; i++ { - offset := rand2.Int63n(MaxWritableByteFromFile) - - // Generate chunk with random data. - chunkSize := ChunkSize - if offset+ChunkSize > MaxWritableByteFromFile { - chunkSize = int(MaxWritableByteFromFile - offset) - } - - err := operations.WriteChunkOfRandomBytesToFile(f, chunkSize, offset) - if err != nil { - t.Fatalf("Error: %v", err) - } - - filePathInGcsBucket := path.Join(DirForRandomWrite, FiveHundredMBFile) - localFilePath := path.Join(TmpDir, FiveHundredMBFileForRandomWriteInLocalSystem) - err = compareFileFromGCSBucketAndMntDir(filePathInGcsBucket, filePath, localFilePath, t) - if err != nil { - t.Fatalf("Error: %v", err) - } - } + operations.CloseFiles(t, filesToWrite) + identical, err := operations.AreFilesIdentical(mountedDirFilePath, localFilePath) + require.NoError(t, err) + assert.True(t, identical) } diff --git a/tools/integration_tests/write_large_files/seq_write_large_file_test.go b/tools/integration_tests/write_large_files/seq_write_large_file_test.go index 3e0c07efb1..35ee1a551b 100644 --- a/tools/integration_tests/write_large_files/seq_write_large_file_test.go +++ b/tools/integration_tests/write_large_files/seq_write_large_file_test.go @@ -18,8 +18,10 @@ import ( "path" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) const ( @@ -28,29 +30,18 @@ const ( DirForSeqWrite = "dirForSeqWrite" ) -var FiveHundredMBFile string = "fiveHundredMBFile" + setup.GenerateRandomString(5) + ".txt" -var FiveHundredMBFileForSeqWriteInLocalSystem string = "fiveHundredMBFileForSeqWriteInLocalSystem" + setup.GenerateRandomString(5) +var FiveHundredMBFile = "fiveHundredMBFile" + setup.GenerateRandomString(5) + ".txt" func TestWriteLargeFileSequentially(t *testing.T) { - seqWriteDir := path.Join(setup.MntDir(), DirForSeqWrite) - setup.SetupTestDirectory(DirForSeqWrite) - filePath := path.Join(seqWriteDir, FiveHundredMBFile) - - // Clean up. - defer operations.RemoveDir(seqWriteDir) - - // Sequentially write the data in file. - err := operations.WriteFileSequentially(filePath, FiveHundredMB, ChunkSize) - if err != nil { - t.Fatalf("Error in writing file: %v", err) - } - - // Download the file from a bucket in which we write the content and compare with - // the file content we wrote in mntDir. - filePathInGcsBucket := path.Join(DirForSeqWrite, FiveHundredMBFile) - localFilePath := path.Join(TmpDir, FiveHundredMBFileForSeqWriteInLocalSystem) - err = compareFileFromGCSBucketAndMntDir(filePathInGcsBucket, filePath, localFilePath, t) - if err != nil { - t.Fatalf("Error:%v", err) - } + seqWriteDir := setup.SetupTestDirectory(DirForSeqWrite) + mountedDirFilePath := path.Join(seqWriteDir, FiveHundredMBFile) + localFilePath := path.Join(TmpDir, setup.GenerateRandomString(5)) + t.Cleanup(func() { operations.RemoveFile(localFilePath) }) + + // Write sequentially to local and mounted directory file. + operations.WriteFilesSequentially(t, []string{localFilePath, mountedDirFilePath}, FiveHundredMB, ChunkSize) + + identical, err := operations.AreFilesIdentical(mountedDirFilePath, localFilePath) + require.NoError(t, err) + assert.True(t, identical) } diff --git a/tools/integration_tests/write_large_files/slow_file_write_test.go b/tools/integration_tests/write_large_files/slow_file_write_test.go new file mode 100644 index 0000000000..a8a34d4e85 --- /dev/null +++ b/tools/integration_tests/write_large_files/slow_file_write_test.go @@ -0,0 +1,50 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package write_large_files + +import ( + "path" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" +) + +const ( + DirForSlowWrite = "dirForSlowWrite" +) + +func TestSlowWriteToFile(t *testing.T) { + slowWriteDir := setup.SetupTestDirectory(DirForSlowWrite) + slowWriteFileName := "slowWriteFile" + setup.GenerateRandomString(5) + ".txt" + mountedDirFilePath := path.Join(slowWriteDir, slowWriteFileName) + fh := operations.OpenFileWithODirect(t, mountedDirFilePath) + defer operations.CloseFileShouldNotThrowError(t, fh) + data := setup.GenerateRandomString(33 * operations.MiB) + + // Write 2 blocks of 33MiB to file + for range 2 { + n, err := fh.Write([]byte(data)) + + assert.NoError(t, err) + assert.Equal(t, len(data), n) + operations.SyncFile(fh, t) + // Add a delay of 40 sec between each block to ensure 32 sec + // ChunkRetryDeadline is completely used while reading the second Chunk. + time.Sleep(40 * time.Second) + } +} diff --git a/tools/integration_tests/write_large_files/write_large_files_test.go b/tools/integration_tests/write_large_files/write_large_files_test.go index 84f749c4d4..b78220f714 100644 --- a/tools/integration_tests/write_large_files/write_large_files_test.go +++ b/tools/integration_tests/write_large_files/write_large_files_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -16,16 +16,16 @@ package write_large_files import ( - "fmt" + "context" "log" "os" "testing" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" - - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" - "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/test_suite" ) const ( @@ -34,41 +34,54 @@ const ( WritePermission_0200 = 0200 ) -func compareFileFromGCSBucketAndMntDir(gcsFile, mntDirFile, localFilePathToDownloadGcsFile string, t *testing.T) error { - err := client.DownloadObjectFromGCS(gcsFile, localFilePathToDownloadGcsFile, t) - if err != nil { - return fmt.Errorf("Error in downloading object: %v", err) - } - - // Remove file after testing. - defer operations.RemoveFile(localFilePathToDownloadGcsFile) - - identical, err := operations.AreFilesIdentical(mntDirFile, localFilePathToDownloadGcsFile) - if !identical { - return fmt.Errorf("Download of GCS object %s didn't match the Mounted local file (%s): %v", localFilePathToDownloadGcsFile, mntDirFile, err) - } - - return nil -} +var ( + storageClient *storage.Client + ctx context.Context +) func TestMain(m *testing.M) { setup.ParseSetUpFlags() - flags := [][]string{{"--implicit-dirs"}} - - setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() - - if setup.TestBucket() == "" && setup.MountedDirectory() != "" { - log.Print("Please pass the name of bucket mounted at mountedDirectory to --testBucket flag.") - os.Exit(1) + // 1. Load and parse the common configuration. + cfg := test_suite.ReadConfigFile(setup.ConfigFile()) + if len(cfg.WriteLargeFiles) == 0 { + log.Println("No configuration found for write large files tests in config. Using flags instead.") + // Populate the config manually. + cfg.WriteLargeFiles = make([]test_suite.TestConfig, 1) + cfg.WriteLargeFiles[0].TestBucket = setup.TestBucket() + cfg.WriteLargeFiles[0].GKEMountedDirectory = setup.MountedDirectory() + cfg.WriteLargeFiles[0].Configs = make([]test_suite.ConfigItem, 1) + cfg.WriteLargeFiles[0].Configs[0].Flags = []string{ + "--enable-streaming-writes=false", + "--write-max-blocks-per-file=2 --write-global-max-blocks=5", + } + cfg.WriteLargeFiles[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} } - // Run tests for mountedDirectory only if --mountedDirectory flag is set. - setup.RunTestsForMountedDirectoryFlag(m) + // 2. Create storage client before running tests. + ctx = context.Background() + bucketType := setup.TestEnvironment(ctx, &cfg.WriteLargeFiles[0]) + closeStorageClient := client.CreateStorageClientWithCancel(&ctx, &storageClient) + defer func() { + err := closeStorageClient() + if err != nil { + log.Fatalf("closeStorageClient failed: %v", err) + } + }() + + // 3. To run mountedDirectory tests, we need both testBucket and mountedDirectory + // flags to be set, as WriteLargeFiles tests validates content from the bucket. + if cfg.WriteLargeFiles[0].GKEMountedDirectory != "" && cfg.WriteLargeFiles[0].TestBucket != "" { + os.Exit(setup.RunTestsForMountedDirectory(cfg.WriteLargeFiles[0].GKEMountedDirectory, m)) + } // Run tests for testBucket - setup.SetUpTestDirForTestBucketFlag() + // 4. Build the flag sets dynamically from the config. + flags := setup.BuildFlagSets(cfg.WriteLargeFiles[0], bucketType, "") + + setup.SetUpTestDirForTestBucket(&cfg.WriteLargeFiles[0]) + + successCode := static_mounting.RunTestsWithConfigFile(&cfg.WriteLargeFiles[0], flags, m) - successCode := static_mounting.RunTests(flags, m) os.Exit(successCode) } diff --git a/tools/metrics-gen/main.go b/tools/metrics-gen/main.go new file mode 100644 index 0000000000..e20d4512b9 --- /dev/null +++ b/tools/metrics-gen/main.go @@ -0,0 +1,592 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "flag" + "fmt" + "log" + "os" + "sort" + "strconv" + "strings" + "text/template" // NOLINT + + "gopkg.in/yaml.v3" +) + +// Data structures to parse metrics.yaml +type Metric struct { + Name string `yaml:"metric-name"` + Description string `yaml:"description"` + Type string `yaml:"type"` + Unit string `yaml:"unit"` + Attributes []Attribute `yaml:"attributes"` + Boundaries []int64 `yaml:"boundaries"` +} + +type Attribute struct { + Name string `yaml:"attribute-name"` + Type string `yaml:"attribute-type"` + Values []string `yaml:"values"` +} + +// AttrValuePair is a helper struct for generating combinations. +type AttrValuePair struct { + Name string + Type string + Value string // "true"/"false" for bools +} + +// AttrCombination is a list of AttrValuePairs. +type AttrCombination []AttrValuePair + +// DistinctAttr represents a unique attribute with all its possible values. +type DistinctAttr struct { + TypeName string // e.g., ReadType + AttributeName string // e.g., read_type + Values []string // e.g., ["sequential", "random"] +} + +// Data structure to pass to the template. +type TemplateData struct { + Metrics []Metric + AttrCombinations map[string][]AttrCombination + DistinctAttrs []DistinctAttr +} + +// Helper functions for the template. +var funcMap = template.FuncMap{ + "toPascal": toPascal, + "toCamel": toCamel, + "getVarName": getVarName, + "getAtomicName": getAtomicName, + "getGoType": getGoType, + "getAttrConstName": getAttrConstName, + "getUnitMethod": getUnitMethod, + "joinInts": joinInts, + "isCounter": func(m Metric) bool { return m.Type == "int_counter" }, + "isUpDownCounter": func(m Metric) bool { return m.Type == "int_up_down_counter" }, + "isHistogram": func(m Metric) bool { return m.Type == "int_histogram" || m.Type == "time_histogram" }, + "isTimeHistogram": func(m Metric) bool { return m.Type == "time_histogram" }, + "buildSwitches": buildSwitches, + "getTestName": getTestName, + "getTestFuncArgs": getTestFuncArgs, + "getExpectedAttrs": getExpectedAttrs, + "getLatencyUnit": getLatencyUnit, + "getLatencyMethod": getLatencyMethod, + "getTestFuncArgsForHistogram": getTestFuncArgsForHistogram, +} + +func toPascal(s string) string { + s = strings.ReplaceAll(s, "::", "-") + s = strings.ReplaceAll(s, "/", "-") + s = strings.ReplaceAll(s, "_", "-") + parts := strings.Split(s, "-") + for i, p := range parts { + if len(p) > 0 { + parts[i] = strings.ToUpper(p[:1]) + p[1:] + } + } + return strings.Join(parts, "") +} + +func toCamel(s string) string { + pascal := toPascal(s) + if len(pascal) > 0 { + return strings.ToLower(pascal[:1]) + pascal[1:] + } + return "" +} + +func getVarName(metricName string, combo AttrCombination) string { + var parts []string + parts = append(parts, toCamel(metricName)) + for _, pair := range combo { + parts = append(parts, toPascal(pair.Name)) + parts = append(parts, toPascal(pair.Value)) + } + parts = append(parts, "AttrSet") + return strings.Join(parts, "") +} + +func getAtomicName(metricName string, combo AttrCombination) string { + var parts []string + parts = append(parts, toCamel(metricName)) + for _, pair := range combo { + parts = append(parts, toPascal(pair.Name)) + parts = append(parts, toPascal(pair.Value)) + } + parts = append(parts, "Atomic") + return strings.Join(parts, "") +} + +func getGoType(t string) string { + // If the type is not a primitive, it's a custom attribute type. + if t != "string" && t != "bool" { + return toPascal(t) + } + return t +} + +func getAttrConstName(typeName, valueName string) string { + return toPascal(typeName) + toPascal(valueName) + "Attr" +} + +func getUnitMethod(unit string) string { + switch unit { + case "us": + return ".Microseconds()" + case "ms": + return ".Milliseconds()" + case "s": + return ".Seconds()" + default: + // Assumes the value is already in the correct unit if not time-based. + return "" + } +} + +func joinInts(nums []int64) string { + var s []string + for _, n := range nums { + s = append(s, strconv.FormatInt(n, 10)) + } + return strings.Join(s, ", ") +} + +// getTestName generates a test name from an attribute combination. +func getTestName(combo AttrCombination) string { + if len(combo) == 0 { + return "no_attributes" + } + var parts = make([]string, 0, len(combo)*2) + for _, pair := range combo { + parts = append(parts, pair.Name) + parts = append(parts, pair.Value) + } + return strings.Join(parts, "_") +} + +// getTestFuncArgs generates arguments for the metric function call in tests. +func getTestFuncArgs(combo AttrCombination) string { + var parts []string + for _, pair := range combo { + if pair.Type != "bool" { + parts = append(parts, fmt.Sprintf(`"%s"`, pair.Value)) + } else { + parts = append(parts, pair.Value) + } + } + return strings.Join(parts, ", ") +} + +// getExpectedAttrs generates attribute set for test expectations. +func getExpectedAttrs(combo AttrCombination) string { + var parts []string + for _, pair := range combo { + if pair.Type != "bool" { + parts = append(parts, fmt.Sprintf(`attribute.String("%s", "%s")`, pair.Name, pair.Value)) + } else { // bool + parts = append(parts, fmt.Sprintf(`attribute.Bool("%s", %s)`, pair.Name, pair.Value)) + } + } + return strings.Join(parts, ", ") +} + +func getLatencyUnit(unit string) string { + switch unit { + case "us": + return "Microsecond" + case "ms": + return "Millisecond" + case "s": + return "Second" + default: + return "" + } +} + +func getLatencyMethod(unit string) string { + return toPascal(getLatencyUnit(unit)) + "s" +} + +func getTestFuncArgsForHistogram(prefix string, attrs []Attribute) string { + var parts []string + for _, attr := range attrs { + parts = append(parts, prefix+"."+toCamel(attr.Name)) + } + return strings.Join(parts, ", ") +} + +// generateCombinations creates all possible combinations of attribute values. +func generateCombinations(attributes []Attribute) []AttrCombination { + if len(attributes) == 0 { + return []AttrCombination{{}} + } + + firstAttr := attributes[0] + remainingAttrs := attributes[1:] + combsOfRest := generateCombinations(remainingAttrs) + + var firstAttrValues []AttrValuePair + if firstAttr.Type != "bool" { + for _, v := range firstAttr.Values { + // The type of the attribute is now the custom type, not "string". + firstAttrValues = append(firstAttrValues, AttrValuePair{Name: firstAttr.Name, Type: firstAttr.Type, Value: v}) + } + } else if firstAttr.Type == "bool" { + firstAttrValues = append(firstAttrValues, AttrValuePair{Name: firstAttr.Name, Type: "bool", Value: "true"}) + firstAttrValues = append(firstAttrValues, AttrValuePair{Name: firstAttr.Name, Type: "bool", Value: "false"}) + } + + var result []AttrCombination + for _, val := range firstAttrValues { + for _, comb := range combsOfRest { + newComb := append(AttrCombination{val}, comb...) + result = append(result, newComb) + } + } + return result +} + +func handleDefaultInSwitchCase(level int, attrName string, builder *strings.Builder) { + fmt.Fprintf(builder, "%sdefault:\n", strings.Repeat("\t", level+2)) + fmt.Fprintf(builder, "%supdateUnrecognizedAttribute(string(%s))\n", strings.Repeat("\t", level+3), toCamel(attrName)) + fmt.Fprintf(builder, "%sreturn\n", strings.Repeat("\t", level+3)) +} + +func validateMetric(m Metric) error { + if m.Name == "" { + return fmt.Errorf("metric-name is required") + } + if m.Description == "" { + return fmt.Errorf("description is required for metric %q", m.Name) + } + if m.Type != "int_counter" && m.Type != "int_histogram" && m.Type != "time_histogram" && m.Type != "int_up_down_counter" { + return fmt.Errorf("type for metric %q must be 'int_counter', 'int_histogram', 'time_histogram', or 'int_up_down_counter', got %q", m.Name, m.Type) + } + + if m.Type == "int_histogram" || m.Type == "time_histogram" { + if len(m.Boundaries) == 0 { + return fmt.Errorf("boundaries are required for histogram metric %q", m.Name) + } + } else { // int_counter + if len(m.Boundaries) > 0 { + return fmt.Errorf("boundaries should not be present for counter metric %q", m.Name) + } + } + + for _, a := range m.Attributes { + if a.Name == "" { + return fmt.Errorf("attribute-name is required for an attribute in metric %q", m.Name) + } + if a.Type != "string" && a.Type != "bool" { + return fmt.Errorf("attribute-type for attribute %q in metric %q must be 'string' or 'bool', got %q", a.Name, m.Name, a.Type) + } + + if a.Type == "string" { + if len(a.Values) == 0 { + return fmt.Errorf("values are required for string attribute %q in metric %q", a.Name, m.Name) + } + } + if a.Type == "bool" && len(a.Values) != 0 { + return fmt.Errorf("values should not be present for bool attribute %q in metric %q", a.Name, m.Name) + } + } + return nil +} + +// validateAttributeConstants checks if any two attributes would resolve to the +// same constant name. +func validateAttributeConstants(attrs []DistinctAttr) error { + constNames := make(map[string]string) + for _, attr := range attrs { + for _, val := range attr.Values { + constName := getAttrConstName(attr.TypeName, val) + if originalAttr, ok := constNames[constName]; ok { + return fmt.Errorf( + "constant name collision: attribute %q with value %q and attribute %q "+ + "both generate constant %q", + attr.AttributeName, val, originalAttr, constName) + } + constNames[constName] = attr.AttributeName + } + } + return nil +} + +func validateForDuplicates(metrics []Metric) error { + names := make(map[string]bool) + for _, m := range metrics { + if names[m.Name] { + return fmt.Errorf("duplicate metric-name: %q", m.Name) + } + names[m.Name] = true + } + return nil +} + +func validateSortOrder(metrics []Metric) error { + for i := 1; i < len(metrics); i++ { + if metrics[i-1].Name > metrics[i].Name { + return fmt.Errorf("metrics are not sorted by name. %q should come before %q", metrics[i].Name, metrics[i-1].Name) + } + } + return nil +} + +func validateAttributeSortOrder(metrics []Metric) error { + for _, m := range metrics { + for i := 1; i < len(m.Attributes); i++ { + if m.Attributes[i-1].Name > m.Attributes[i].Name { + return fmt.Errorf("attributes for metric %q are not sorted by name. %q should come before %q", m.Name, m.Attributes[i].Name, m.Attributes[i-1].Name) + } + } + } + return nil +} + +// validateMetrics checks for correctness of the metric definitions. +func validateMetrics(metrics []Metric) error { + if err := validateForDuplicates(metrics); err != nil { + return err + } + if err := validateSortOrder(metrics); err != nil { + return err + } + if err := validateAttributeSortOrder(metrics); err != nil { + return err + } + for _, m := range metrics { + if err := validateMetric(m); err != nil { + return err + } + } + return nil +} + +// buildSwitches generates the nested switch statement code for a metric method. +func buildSwitches(metric Metric) string { + var builder strings.Builder + var recorder func(level int, combo AttrCombination) + + recorder = func(level int, combo AttrCombination) { + if level == len(metric.Attributes) { + // Base case: record the metric + indent := strings.Repeat("\t", level+1) + if metric.Type == "int_counter" || metric.Type == "int_up_down_counter" { + atomicName := getAtomicName(metric.Name, combo) + fmt.Fprintf(&builder, "%so.%s.Add(inc)\n", indent, atomicName) + } else { // histogram + varName := getVarName(metric.Name, combo) + unitMethod := getUnitMethod(metric.Unit) + valVar := "latency" + if unitMethod == "" { + valVar = "value" + } + valueStr := fmt.Sprintf("%s%s", valVar, unitMethod) + if metric.Unit == "s" { + valueStr = fmt.Sprintf("int64(%s)", valueStr) + } + fmt.Fprintf(&builder, "%srecord = histogramRecord{ctx: ctx, instrument: o.%s, value: %s, attributes: %s}\n", indent, toCamel(metric.Name), valueStr, varName) + } + return + } + + attr := metric.Attributes[level] + indent := strings.Repeat("\t", level+1) + fmt.Fprintf(&builder, "%sswitch %s {\n", indent, toCamel(attr.Name)) + + var values []string + isBool := attr.Type == "bool" + if isBool { + values = []string{"true", "false"} + } else { + values = attr.Values + } + + for _, val := range values { + caseVal := val + if !isBool { + caseVal = getAttrConstName(attr.Type, val) + } + fmt.Fprintf(&builder, "%scase %s:\n", strings.Repeat("\t", level+2), caseVal) + currentCombo := append(combo, AttrValuePair{Name: attr.Name, Type: attr.Type, Value: val}) + recorder(level+1, currentCombo) + } + if !isBool { + handleDefaultInSwitchCase(level, attr.Name, &builder) + } + fmt.Fprintf(&builder, "%s}\n", indent) + } + + if len(metric.Attributes) == 0 { + switch metric.Type { + case "int_histogram", "time_histogram": + unitMethod := getUnitMethod(metric.Unit) + valVar := "latency" + if unitMethod == "" { + valVar = "value" + } + valueStr := fmt.Sprintf("%s%s", valVar, unitMethod) + if metric.Unit == "s" { + valueStr = fmt.Sprintf("int64(%s)", valueStr) + } + fmt.Fprintf(&builder, "\trecord := histogramRecord{ctx: ctx, instrument: o.%s, value: %s}\n", toCamel(metric.Name), valueStr) + case "int_counter", "int_up_down_counter": + atomicName := getAtomicName(metric.Name, AttrCombination{}) + fmt.Fprintf(&builder, "\to.%s.Add(inc)\n", atomicName) + } + } else { + if metric.Type == "int_histogram" || metric.Type == "time_histogram" { + builder.WriteString("\tvar record histogramRecord\n") + } + recorder(0, AttrCombination{}) + } + + return builder.String() +} + +// findDistinctAttributes finds all unique string attributes across all metrics +// and returns them as a slice of DistinctAttr, sorted by attribute name. +func findDistinctAttributes(metrics []Metric) []DistinctAttr { + distinctAttrsMap := make(map[string]map[string]bool) // map[attrName]map[value]bool + for _, m := range metrics { + for _, attr := range m.Attributes { + // We only generate constants for string attributes. + if attr.Type == "string" { + if _, ok := distinctAttrsMap[attr.Name]; !ok { + distinctAttrsMap[attr.Name] = make(map[string]bool) + } + for _, val := range attr.Values { + distinctAttrsMap[attr.Name][val] = true + } + } + } + } + var distinctAttrs []DistinctAttr + for attrName, valuesMap := range distinctAttrsMap { + var values []string + for val := range valuesMap { + values = append(values, val) + } + sort.Strings(values) + distinctAttrs = append(distinctAttrs, DistinctAttr{ + TypeName: toPascal(attrName), + AttributeName: attrName, + Values: values, + }) + } + return distinctAttrs +} + +func main() { + inputFile := flag.String("input", "metrics.yaml", "Input YAML file") + outputDir := flag.String("outDir", ".", "Output directory to dump artifacts.") + flag.Parse() + + yamlFile, err := os.ReadFile(*inputFile) + if err != nil { + log.Fatalf("error reading yaml file: %v", err) + } + + var metrics []Metric + err = yaml.Unmarshal(yamlFile, &metrics) + if err != nil { + log.Fatalf("error unmarshalling yaml: %v", err) + } + + // Validate metrics + if err := validateMetrics(metrics); err != nil { + log.Fatalf("invalid metrics.yaml: %v", err) + } + + // Sort attributes and their string values for deterministic output + for i := range metrics { + sort.Slice(metrics[i].Attributes, func(k, j int) bool { + return metrics[i].Attributes[k].Name < metrics[i].Attributes[j].Name + }) + for j := range metrics[i].Attributes { + if metrics[i].Attributes[j].Type == "string" { + sort.Strings(metrics[i].Attributes[j].Values) + } + } + } + + distinctAttrs := findDistinctAttributes(metrics) + // Sort for deterministic output. + sort.Slice(distinctAttrs, func(i, j int) bool { + return distinctAttrs[i].AttributeName < distinctAttrs[j].AttributeName + }) + + if err := validateAttributeConstants(distinctAttrs); err != nil { + log.Fatalf("error validating attribute constants: %v", err) + } + + // In the template data, update the attribute type to be the custom type name + // for the distinct attributes. This allows the templates to generate the + // correct type in function signatures. + for i, m := range metrics { + distinctAttrsMap := make(map[string]bool) + for _, da := range distinctAttrs { + distinctAttrsMap[da.AttributeName] = true + } + + for j, attr := range m.Attributes { + if distinctAttrsMap[attr.Name] { + metrics[i].Attributes[j].Type = attr.Name + } + } + } + + // Create the directory if it doesn't exist + if err := os.MkdirAll(*outputDir, 0755); err != nil { + log.Fatalf("error creating output directory: %v", err) + } + + attrCombinations := make(map[string][]AttrCombination) + for _, m := range metrics { + attrCombinations[m.Name] = generateCombinations(m.Attributes) + } + + data := TemplateData{ + Metrics: metrics, + AttrCombinations: attrCombinations, + DistinctAttrs: distinctAttrs, + } + createFile(&data, fmt.Sprintf("%s/metric_handle.go", *outputDir), "metric_handle.tpl") + createFile(&data, fmt.Sprintf("%s/noop_metrics.go", *outputDir), "noop_metrics.tpl") + createFile(&data, fmt.Sprintf("%s/otel_metrics.go", *outputDir), "otel_metrics.tpl") + createFile(&data, fmt.Sprintf("%s/otel_metrics_test.go", *outputDir), "otel_metrics_test.tpl") + +} + +func createFile(data *TemplateData, fName string, templateName string) { + tmpl, err := template.New(templateName).Funcs(funcMap).ParseFiles(templateName) + if err != nil { + log.Fatalf("error parsing template: %v", err) + } + var buf bytes.Buffer + err = tmpl.Execute(&buf, data) + if err != nil { + log.Fatalf("error executing template: %v", err) + } + + if err := os.WriteFile(fName, buf.Bytes(), 0644); err != nil { + log.Fatalf("error writing output file: %v", err) + } +} diff --git a/tools/metrics-gen/metric_handle.tpl b/tools/metrics-gen/metric_handle.tpl new file mode 100644 index 0000000000..595102f7cd --- /dev/null +++ b/tools/metrics-gen/metric_handle.tpl @@ -0,0 +1,52 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// **** DO NOT EDIT - FILE IS AUTO-GENERATED **** +package metrics + +import ( + "context" + "time" +) + +{{range .DistinctAttrs}} +{{- $attr := . -}} +// {{.TypeName}} is a custom type for the {{.AttributeName}} attribute. +type {{.TypeName}} string +const ( +{{- range .Values}} + {{getAttrConstName $attr.TypeName .}} {{$attr.TypeName}} = "{{.}}" +{{- end}} +) +{{end}} + +// MetricHandle provides an interface for recording metrics. +// The methods of this interface are auto-generated from metrics.yaml. +// Each method corresponds to a metric defined in metrics.yaml. +type MetricHandle interface { +{{- range .Metrics}} + // {{toPascal .Name}} - {{.Description}} + {{toPascal .Name}}( + {{- if or (isCounter .) (isUpDownCounter .) -}} + inc int64 + {{- else -}} + ctx context.Context, {{if isTimeHistogram .}}latency time.Duration{{else}}value int64{{end}} + {{- end }} + {{- if .Attributes}}, {{end}} + {{- range $i, $attr := .Attributes -}} + {{if $i}}, {{end}}{{toCamel $attr.Name}} {{getGoType $attr.Type}} + {{- end }}) + +{{end}} +} diff --git a/tools/metrics-gen/noop_metrics.tpl b/tools/metrics-gen/noop_metrics.tpl new file mode 100644 index 0000000000..3db0eb3828 --- /dev/null +++ b/tools/metrics-gen/noop_metrics.tpl @@ -0,0 +1,40 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// **** DO NOT EDIT - FILE IS AUTO-GENERATED **** +package metrics + +import ( + "context" + "time" +) + +type noopMetrics struct {} +{{- range .Metrics}} + func (*noopMetrics) {{toPascal .Name}}( + {{- if or (isCounter .) (isUpDownCounter .) -}} + inc int64 + {{- else -}} + ctx context.Context, {{if isTimeHistogram .}}latency time.Duration{{else}}value int64{{end}} + {{- end }} + {{- if .Attributes}}, {{end}} + {{- range $i, $attr := .Attributes -}} + {{if $i}}, {{end}}{{toCamel $attr.Name}} {{getGoType $attr.Type}} + {{- end }}){} +{{end}} + +func NewNoopMetrics() MetricHandle { + var n noopMetrics + return &n +} diff --git a/tools/metrics-gen/otel_metrics.tpl b/tools/metrics-gen/otel_metrics.tpl new file mode 100644 index 0000000000..df4761e08e --- /dev/null +++ b/tools/metrics-gen/otel_metrics.tpl @@ -0,0 +1,232 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// **** DO NOT EDIT - FILE IS AUTO-GENERATED **** + +package metrics + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +const logInterval = 5 * time.Minute + +var ( + unrecognizedAttr atomic.Value +{{- range $metric := .Metrics -}} +{{- if .Attributes}} +{{- range $combination := (index $.AttrCombinations $metric.Name)}} + {{getVarName $metric.Name $combination}} = metric.WithAttributeSet(attribute.NewSet( + {{- range $pair := $combination -}} + attribute.{{if eq $pair.Type "bool"}}Bool{{else}}String{{end}}("{{$pair.Name}}", {{if eq $pair.Type "bool"}}{{$pair.Value}}{{else}}"{{$pair.Value}}"{{end}}), + {{- end -}} + )) +{{- end -}} +{{- end -}} +{{- end -}} +) + +type histogramRecord struct { + ctx context.Context + instrument metric.Int64Histogram + value int64 + attributes metric.RecordOption +} + +type otelMetrics struct { + ch chan histogramRecord + wg *sync.WaitGroup + {{- range $metric := .Metrics}} + {{- if or (isCounter $metric) (isUpDownCounter $metric)}} + {{- range $combination := (index $.AttrCombinations $metric.Name)}} + {{getAtomicName $metric.Name $combination}} *atomic.Int64 + {{- end}} + {{- end}} + {{- end}} + {{- range $metric := .Metrics}} + {{- if isHistogram $metric}} + {{toCamel $metric.Name}} metric.Int64Histogram + {{- end}} + {{- end}} +} + +{{range .Metrics}} +func (o *otelMetrics) {{toPascal .Name}}( + {{- if or (isCounter .) (isUpDownCounter .)}} + inc int64 + {{- else }} + ctx context.Context, {{if isTimeHistogram .}}latency time.Duration{{else}}value int64{{end}} + {{- end }} + {{- if .Attributes}}, {{end}} + {{- range $i, $attr := .Attributes -}} + {{if $i}}, {{end}}{{toCamel $attr.Name}} {{getGoType $attr.Type}} + {{- end }}) { +{{- if or (isCounter .) (isUpDownCounter .)}} + {{- if isCounter . }} + if inc < 0 { + logger.Errorf("Counter metric {{.Name}} received a negative increment: %d", inc) + return + } + {{- end}} + {{buildSwitches .}} +{{- else }} + {{buildSwitches .}} + select { + case o.ch <- record: // Do nothing + default: // Unblock writes to channel if it's full. + } + {{- end -}} +} +{{end}} + +func NewOTelMetrics(ctx context.Context, workers int, bufferSize int) (*otelMetrics, error) { + ch := make(chan histogramRecord, bufferSize) + var wg sync.WaitGroup + startSampledLogging(ctx) + for range workers { + wg.Add(1) + go func() { + defer wg.Done() + for record := range ch { + if record.attributes != nil { + record.instrument.Record(record.ctx, record.value, record.attributes) + } else { + record.instrument.Record(record.ctx, record.value) + } + } + }() + } + meter := otel.Meter("gcsfuse") +{{- range $metric := .Metrics}} + {{- if or (isCounter $metric) (isUpDownCounter $metric) }} + var {{range $i, $combination := (index $.AttrCombinations $metric.Name)}}{{if $i}}, + {{end}}{{getAtomicName $metric.Name $combination}}{{end}} atomic.Int64 + {{- end}} + +{{end}} + +{{- range $i, $metric := .Metrics}} + {{- if or (isCounter $metric) (isUpDownCounter $metric)}} + {{- $instrumentCreationFunc := "" -}} + {{- $observationFunc := "" -}} + {{- if isCounter $metric -}} + {{- $instrumentCreationFunc = "meter.Int64ObservableCounter" -}} + {{- $observationFunc = "conditionallyObserve" -}} + {{- else -}} + {{- $instrumentCreationFunc = "meter.Int64ObservableUpDownCounter" -}} + {{- $observationFunc = "observeUpDownCounter" -}} + {{- end}} + + _, err{{$i}} := {{$instrumentCreationFunc}}("{{$metric.Name}}", + metric.WithDescription("{{.Description}}"), + metric.WithUnit("{{.Unit}}"), + metric.WithInt64Callback(func(_ context.Context, obsrv metric.Int64Observer) error { + {{- range $combination := (index $.AttrCombinations $metric.Name)}} + {{$observationFunc}}(obsrv, &{{getAtomicName $metric.Name $combination}}{{if $metric.Attributes}}, {{getVarName $metric.Name $combination}}{{end}}) + {{- end}} + return nil + })) + + {{- else}} + {{toCamel $metric.Name}}, err{{$i}} := meter.Int64Histogram("{{$metric.Name}}", + metric.WithDescription("{{.Description}}"), + metric.WithUnit("{{.Unit}}"), + {{- if .Boundaries}} + metric.WithExplicitBucketBoundaries({{joinInts .Boundaries}})) + {{- else}} + ) + {{- end}} + {{- end}} +{{end}} + + errs := []error{ + {{- range $i, $metric := .Metrics -}} + {{if $i}}, {{end}}err{{$i}} + {{- end -}} + } + if err := errors.Join(errs...); err != nil { + return nil, err + } + + return &otelMetrics{ + ch : ch, + wg: &wg, + {{- range $metric := .Metrics}} + {{- if or (isCounter $metric) (isUpDownCounter $metric)}} + {{- range $combination := (index $.AttrCombinations $metric.Name)}} + {{getAtomicName $metric.Name $combination}}: &{{getAtomicName $metric.Name $combination}}, + {{- end}} + {{- else}} + {{toCamel $metric.Name}}: {{toCamel $metric.Name}}, + {{- end}} + {{- end}} + }, nil +} + +func (o *otelMetrics) Close() { + close(o.ch) + o.wg.Wait() +} + +func conditionallyObserve(obsrv metric.Int64Observer, counter *atomic.Int64, obsrvOptions ...metric.ObserveOption) { + if val := counter.Load(); val > 0 { + obsrv.Observe(val, obsrvOptions...) + } +} + +func observeUpDownCounter(obsrv metric.Int64Observer, counter *atomic.Int64, obsrvOptions ...metric.ObserveOption) { + obsrv.Observe(counter.Load(), obsrvOptions...) +} + +func updateUnrecognizedAttribute(newValue string) { + unrecognizedAttr.CompareAndSwap("", newValue) +} + +// StartSampledLogging starts a goroutine that logs unrecognized attributes periodically. +func startSampledLogging(ctx context.Context) { + // Init the atomic.Value + unrecognizedAttr.Store("") + + go func() { + ticker := time.NewTicker(logInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + logUnrecognizedAttribute() + } + } + }() +} + +// logUnrecognizedAttribute retrieves and logs any unrecognized attributes. +func logUnrecognizedAttribute() { + // Atomically load and reset the attribute name, then generate a log + // if an unrecognized attribute was encountered. + if currentAttr := unrecognizedAttr.Swap("").(string); currentAttr != "" { + logger.Tracef("Attribute %s is not declared", currentAttr) + } +} diff --git a/tools/metrics-gen/otel_metrics_test.tpl b/tools/metrics-gen/otel_metrics_test.tpl new file mode 100644 index 0000000000..9721c4d1ac --- /dev/null +++ b/tools/metrics-gen/otel_metrics_test.tpl @@ -0,0 +1,331 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// **** DO NOT EDIT - FILE IS AUTO-GENERATED **** + +package metrics + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +// metricValueMap maps attribute sets to metric values. +type metricValueMap map[string]int64 + +// metricHistogramMap maps attribute sets to histogram data points. +type metricHistogramMap map[string]metricdata.HistogramDataPoint[int64] + +func waitForMetricsProcessing() { + time.Sleep(5 * time.Millisecond) +} + +func setupOTel(ctx context.Context, t *testing.T) (*otelMetrics, *metric.ManualReader) { + t.Helper() + reader := metric.NewManualReader() + provider := metric.NewMeterProvider(metric.WithReader(reader)) + otel.SetMeterProvider(provider) + + m, err := NewOTelMetrics(ctx, 10, 100) + require.NoError(t, err) + return m, reader +} + +// gatherHistogramMetrics collects all histogram metrics from the reader. +// It returns a map where the key is the metric name, and the value is another map. +// The inner map's key is a string representation of the attributes, +// and the value is the metricdata.HistogramDataPoint. +func gatherHistogramMetrics(ctx context.Context, t *testing.T, rd *metric.ManualReader) map[string]map[string]metricdata.HistogramDataPoint[int64] { + t.Helper() + var rm metricdata.ResourceMetrics + err := rd.Collect(ctx, &rm) + require.NoError(t, err) + + results := make(map[string]map[string]metricdata.HistogramDataPoint[int64]) + encoder := attribute.DefaultEncoder() // Using default encoder + + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + // We are interested in Histogram[int64]. + hist, ok := m.Data.(metricdata.Histogram[int64]) + if !ok { + continue + } + + metricMap := make(metricHistogramMap) + for _, dp := range hist.DataPoints { + if dp.Count == 0 { + continue + } + + metricMap[dp.Attributes.Encoded(encoder)] = dp + } + + if len(metricMap) > 0 { + results[m.Name] = metricMap + } + } + } + + return results +} + +// gatherNonZeroCounterMetrics collects all non-zero counter metrics from the reader. +// It returns a map where the key is the metric name, and the value is another map. +// The inner map's key is a string representation of the attributes, +// and the value is the metric's value. +func gatherNonZeroCounterMetrics(ctx context.Context, t *testing.T, rd *metric.ManualReader) map[string]map[string]int64 { + t.Helper() + var rm metricdata.ResourceMetrics + err := rd.Collect(ctx, &rm) + require.NoError(t, err) + + results := make(map[string]map[string]int64) + encoder := attribute.DefaultEncoder() + + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + // We are interested in Sum[int64] which corresponds to int_counter. + sum, ok := m.Data.(metricdata.Sum[int64]) + if !ok { + continue + } + + metricMap := make(metricValueMap) + for _, dp := range sum.DataPoints { + if dp.Value == 0 { + continue + } + + metricMap[dp.Attributes.Encoded(encoder)] = dp.Value + } + + if len(metricMap) > 0 { + results[m.Name] = metricMap + } + } + } + + return results +} + +{{range .Metrics}} +{{if or (isCounter .) (isUpDownCounter .)}} +func Test{{toPascal .Name}}(t *testing.T) { + {{- if .Attributes}} + tests := []struct { + name string + f func(m *otelMetrics) + expected map[attribute.Set]int64 + }{ + {{- $metric := . -}} + {{- range $combination := (index $.AttrCombinations $metric.Name)}} + { + name: "{{getTestName $combination}}", + f: func(m *otelMetrics) { + m.{{toPascal $metric.Name}}(5, {{getTestFuncArgs $combination}}) + }, + expected: map[attribute.Set]int64{ + attribute.NewSet({{getExpectedAttrs $combination}}): 5, + }, + }, + {{- end}} + {{- $combinations := (index $.AttrCombinations $metric.Name) -}} + {{- if and .Attributes (gt (len $combinations) 1) -}} + { + name: "multiple_attributes_summed", + f: func(m *otelMetrics) { + {{- $firstComb := (index $combinations 0) -}} + {{- $secondComb := (index $combinations 1) -}} + m.{{toPascal $metric.Name}}(5, {{getTestFuncArgs $firstComb}}) + m.{{toPascal $metric.Name}}(2, {{getTestFuncArgs $secondComb}}) + m.{{toPascal $metric.Name}}(3, {{getTestFuncArgs $firstComb}}) + }, + expected: map[attribute.Set]int64{ + {{- $firstComb := (index $combinations 0) -}} + {{- $secondComb := (index $combinations 1) -}} + attribute.NewSet({{getExpectedAttrs $firstComb}}): 8, + attribute.NewSet({{getExpectedAttrs $secondComb}}): 2, + }, + }, + {{- end}} + { + name: "negative_increment", + f: func(m *otelMetrics) { + {{- $firstComb := (index $combinations 0) -}} + m.{{toPascal $metric.Name}}(-5, {{getTestFuncArgs $firstComb}}) + m.{{toPascal $metric.Name}}(2, {{getTestFuncArgs $firstComb}}) + }, + expected: map[attribute.Set]int64{attribute.NewSet({{getExpectedAttrs (index $combinations 0)}}): {{if isUpDownCounter $metric}}-3{{else}}2{{end}}}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + + tc.f(m) + waitForMetricsProcessing() + + metrics := gatherNonZeroCounterMetrics(ctx, t, rd) + metric, ok := metrics["{{.Name}}"] + if len(tc.expected) == 0 { + assert.False(t, ok, "{{.Name}} metric should not be found") + return + } + require.True(t, ok, "{{.Name}} metric not found") + expectedMap := make(map[string]int64) + for k, v := range tc.expected { + expectedMap[k.Encoded(encoder)] = v + } + assert.Equal(t, expectedMap, metric) + }) + } + {{- else}} + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + + m.{{toPascal .Name}}(1024) + m.{{toPascal .Name}}(2048) + waitForMetricsProcessing() + + metrics := gatherNonZeroCounterMetrics(ctx, t, rd) + metric, ok := metrics["{{.Name}}"] + require.True(t, ok, "{{.Name}} metric not found") + s := attribute.NewSet() + assert.Equal(t, map[string]int64{s.Encoded(encoder): 3072}, metric, "Positive increments should be summed.") + + // Test negative increment + m.{{toPascal .Name}}(-100) + waitForMetricsProcessing() + + metrics = gatherNonZeroCounterMetrics(ctx, t, rd) + metric, ok = metrics["{{.Name}}"] + require.True(t, ok, "{{.Name}} metric not found after negative increment") + assert.Equal(t, map[string]int64{s.Encoded(encoder): {{if isUpDownCounter .}}2972{{else}}3072{{end}}}, metric, "Negative increment should {{if isUpDownCounter .}}change{{else}}not change{{end}} the metric value.") + {{- end}} +} +{{else if isHistogram .}} +func Test{{toPascal .Name}}(t *testing.T) { + {{- if .Attributes}} + tests := []struct { + name string + {{if isTimeHistogram .}}latencies []time.Duration{{else}}values []int64{{end}} + {{- range .Attributes}} + {{toCamel .Name}} {{getGoType .Type}} + {{- end}} + }{ + {{- $metric := . -}} + {{- range $combination := (index $.AttrCombinations $metric.Name)}} + { + name: "{{getTestName $combination}}", + {{if isTimeHistogram $metric -}} + latencies: []time.Duration{100 * time.{{getLatencyUnit $metric.Unit}}, 200 * time.{{getLatencyUnit $metric.Unit}}}, + {{- else -}} + values: []int64{100, 200}, + {{- end}} + {{- range $pair := $combination}} + {{toCamel $pair.Name}}: {{if eq $pair.Type "bool"}}{{$pair.Value}}{{else}}"{{$pair.Value}}"{{end}}, + {{- end}} + }, + {{- end}} + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + {{if isTimeHistogram . -}} + var totalLatency time.Duration + + for _, latency := range tc.latencies { + m.{{toPascal .Name}}(ctx, latency, {{getTestFuncArgsForHistogram "tc" .Attributes}}) + totalLatency += latency + } + {{- else -}} + var totalValue int64 + + for _, value := range tc.values { + m.{{toPascal .Name}}(ctx, value, {{getTestFuncArgsForHistogram "tc" .Attributes}}) + totalValue += value + } + {{- end}} + waitForMetricsProcessing() + + metrics := gatherHistogramMetrics(ctx, t, rd) + metric, ok := metrics["{{.Name}}"] + require.True(t, ok, "{{.Name}} metric not found") + + attrs := []attribute.KeyValue{ + {{- range .Attributes}} + attribute.{{if eq .Type "bool"}}Bool("{{.Name}}", tc.{{toCamel .Name}}){{else}}String("{{.Name}}", string(tc.{{toCamel .Name}})){{end}}, + {{- end}} + } + s := attribute.NewSet(attrs...) + expectedKey := s.Encoded(encoder) + dp, ok := metric[expectedKey] + require.True(t, ok, "DataPoint not found for key: %s", expectedKey) + assert.Equal(t, uint64(len(tc.{{if isTimeHistogram .}}latencies{{else}}values{{end}})), dp.Count) + assert.Equal(t, {{if isTimeHistogram .}}{{if eq .Unit "s"}}int64(totalLatency.{{getLatencyMethod .Unit}}()){{else}}totalLatency.{{getLatencyMethod .Unit}}(){{end}}{{else}}totalValue{{end}}, dp.Sum) + }) + } + {{- else}} + ctx := context.Background() + encoder := attribute.DefaultEncoder() + m, rd := setupOTel(ctx, t) + {{if isTimeHistogram . -}} + var totalLatency time.Duration + latencies := []time.Duration{100 * time.{{getLatencyUnit .Unit}}, 200 * time.{{getLatencyUnit .Unit}}} + + for _, latency := range latencies { + m.{{toPascal .Name}}(ctx, latency) + totalLatency += latency + } + {{- else -}} + var totalValue int64 + values := []int64{100, 200} + + for _, value := range values { + m.{{toPascal .Name}}(ctx, value) + totalValue += value + } + {{- end}} + waitForMetricsProcessing() + + metrics := gatherHistogramMetrics(ctx, t, rd) + metric, ok := metrics["{{.Name}}"] + require.True(t, ok, "{{.Name}} metric not found") + + s := attribute.NewSet() + expectedKey := s.Encoded(encoder) + dp, ok := metric[expectedKey] + require.True(t, ok, "DataPoint not found for key: %s", expectedKey) + assert.Equal(t, uint64(len({{if isTimeHistogram .}}latencies{{else}}values{{end}})), dp.Count) + assert.Equal(t, {{if isTimeHistogram .}}{{if eq .Unit "s"}}int64(totalLatency.{{getLatencyMethod .Unit}}()){{else}}totalLatency.{{getLatencyMethod .Unit}}(){{end}}{{else}}totalValue{{end}}, dp.Sum) + {{- end}} +} +{{end}} +{{end}} diff --git a/tools/mount_gcsfuse/main.go b/tools/mount_gcsfuse/main.go index 1bbfaadc24..e07ffec5f4 100644 --- a/tools/mount_gcsfuse/main.go +++ b/tools/mount_gcsfuse/main.go @@ -63,8 +63,8 @@ import ( "slices" "strings" - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/mount" + "github.com/googlecloudplatform/gcsfuse/v3/cfg" + "github.com/googlecloudplatform/gcsfuse/v3/internal/mount" "github.com/spf13/pflag" ) @@ -111,9 +111,10 @@ func makeGcsfuseArgs( return s == "o" }) nonBoolFlags = append(nonBoolFlags, cfg.ConfigFileFlagName) - // TODO: Clean this up after we gain enough confidence on CLI-Config Parity changes. - boolFlags = append(boolFlags, "disable-viper-config") - noopOptions := []string{"user", "nouser", "auto", "noauto", "_netdev", "no_netdev"} + // 'rw' mount option is implicitly passed as CLI parameter by /etc/fstab + // entries and overrides mount options in config file. Ignoring is safe, as + // 'rw' is default, so that config file mount options take effect. + noopOptions := []string{"rw", "user", "nouser", "auto", "noauto", "_netdev", "no_netdev"} // Deal with options. for name, value := range opts { @@ -267,6 +268,7 @@ func run(args []string) (err error) { // Run gcsfuse. cmd := exec.Command(gcsfusePath, gcsfuseArgs...) cmd.Env = append(cmd.Env, fmt.Sprintf("PATH=%s", path.Dir(fusermountPath))) + cmd.Env = append(cmd.Env, fmt.Sprintf("HOME=%s", os.Getenv("HOME"))) // Pass through the https_proxy/http_proxy environment variable, // in case the host requires a proxy server to reach the GCS endpoint. diff --git a/tools/mount_gcsfuse/main_test.go b/tools/mount_gcsfuse/main_test.go index e7ad274564..dfdb82103c 100644 --- a/tools/mount_gcsfuse/main_test.go +++ b/tools/mount_gcsfuse/main_test.go @@ -43,7 +43,7 @@ func TestMakeGcsfuseArgs(t *testing.T) { "ignore_interrupts": "", "anonymous_access": "false", "log_rotate_compress": "false", - "disable_viper_config": "true"}, + }, expectedFlags: []string{"--implicit-dirs=true", "--foreground=true", "--reuse-token-from-url=false", @@ -53,7 +53,7 @@ func TestMakeGcsfuseArgs(t *testing.T) { "--ignore-interrupts=true", "--anonymous-access=false", "--log-rotate-compress=false", - "--disable-viper-config=true"}, + }, }, { @@ -67,7 +67,7 @@ func TestMakeGcsfuseArgs(t *testing.T) { "ignore-interrupts": "", "anonymous-access": "false", "log_rotate-compress": "false", - "disable-viper-config": "false"}, + }, expectedFlags: []string{"--implicit-dirs=true", "--foreground=true", "--reuse-token-from-url=false", @@ -77,7 +77,7 @@ func TestMakeGcsfuseArgs(t *testing.T) { "--ignore-interrupts=true", "--anonymous-access=false", "--log-rotate-compress=false", - "--disable-viper-config=false"}, + }, }, { @@ -109,7 +109,7 @@ func TestMakeGcsfuseArgs(t *testing.T) { // Test ignored options { name: "TestMakeGcsfuseArgs with IgnoredOptions", - opts: map[string]string{"user": "nobody", "_netdev": ""}, + opts: map[string]string{"user": "nobody", "_netdev": "", "rw": ""}, expectedFlags: []string{}, }, diff --git a/tools/package_gcsfuse/build.go b/tools/package_gcsfuse/build.go index 6c624df384..7b080ddaca 100644 --- a/tools/package_gcsfuse/build.go +++ b/tools/package_gcsfuse/build.go @@ -28,8 +28,7 @@ import ( // root-relative file system structure we desire. func build( commit string, - version string, - osys string) (dir string, err error) { + version string) (dir string, err error) { log.Printf("Building version %s from %s.", version, commit) // Create a directory to become GOCACHE below. diff --git a/tools/package_gcsfuse/main.go b/tools/package_gcsfuse/main.go index d8c8104161..04d6e282ef 100644 --- a/tools/package_gcsfuse/main.go +++ b/tools/package_gcsfuse/main.go @@ -65,7 +65,7 @@ func run(args []string) (err error) { } // Assemble binaries, mount(8) helper scripts, etc. - buildDir, err := build(commit, version, osys) + buildDir, err := build(commit, version) if err != nil { err = fmt.Errorf("build: %w", err) return diff --git a/tools/package_gcsfuse/package.go b/tools/package_gcsfuse/package.go index 41e5810d9e..79af9119af 100644 --- a/tools/package_gcsfuse/package.go +++ b/tools/package_gcsfuse/package.go @@ -24,8 +24,6 @@ func packageFpm( packageType string, binDir string, version string, - osys string, - arch string, outputDir string) (err error) { // Call fpm. cmd := exec.Command( @@ -63,7 +61,7 @@ func packageDeb( outputDir string) (err error) { log.Println("Building a .deb package.") - err = packageFpm("deb", binDir, version, osys, arch, outputDir) + err = packageFpm("deb", binDir, version, outputDir) return } @@ -77,6 +75,6 @@ func packageRpm( outputDir string) (err error) { log.Println("Building a .rpm package.") - err = packageFpm("rpm", binDir, version, osys, arch, outputDir) + err = packageFpm("rpm", binDir, version, outputDir) return } diff --git a/tools/package_gcsfuse_docker/Dockerfile b/tools/package_gcsfuse_docker/Dockerfile index b338363201..8b51c1434c 100644 --- a/tools/package_gcsfuse_docker/Dockerfile +++ b/tools/package_gcsfuse_docker/Dockerfile @@ -12,26 +12,68 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Build an image with gcsfuse packages: -# > docker build . -t gcsfuse-release --build-arg GCSFUSE_VERSION=0.39.2 -# Copy the gcsfuse packages to the host: -# > docker run -it -v /tmp:/output gcsfuse-release cp -r /packages /output +# This Dockerfile builds gcsfuse DEB and RPM packages for different linux architectures. +# +# Build Arguments: +# GCSFUSE_VERSION - Version string for the package (required, e.g., "0.39.2") +# BRANCH_NAME - Git commit, branch or tag to build from (default: "v${GCSFUSE_VERSION}") +# ARCHITECTURE - Target architecture: "amd64" or "arm64" (default: "amd64") +# GCSFUSE_REPO - Repository URL (default: https://github.com/googlecloudplatform/gcsfuse/) +# +# Examples: +# +# Build from version tag (standard release): +# docker build . -t gcsfuse-release --build-arg GCSFUSE_VERSION=0.39.2 +# +# Build from specific branch/commit/tag (e.g., for testing unreleased changes): +# docker build . -t gcsfuse-release \ +# --build-arg GCSFUSE_VERSION=0.40.0 \ +# --build-arg BRANCH_NAME=<commit_hash_or_branch_name> +# +# Build for ARM64 architecture: +# docker build . -t gcsfuse-release \ +# --build-arg GCSFUSE_VERSION=0.39.2 \ +# --build-arg ARCHITECTURE=arm64 +# +# Extract packages from the container to host: +# docker run -it -v /tmp:/output gcsfuse-release cp -r /packages /output +# +# Output: +# - /packages/gcsfuse_{VERSION}_{ARCH}.deb +# - /packages/gcsfuse-{VERSION}-1.x86_64.rpm (or aarch64 for arm64) -FROM golang:1.23.3 as builder +FROM golang:1.24.5 as builder -RUN apt-get update -qq && apt-get install -y ruby ruby-dev rubygems build-essential rpm && gem install --no-document bundler +RUN apt-get update -qq && apt-get install -y ruby ruby-dev rubygems build-essential && gem install --no-document bundler -v "2.4.12" +RUN apt-get install -y rpm wget +ARG ARCHITECTURE="amd64" +RUN if [ "${ARCHITECTURE}" != "amd64" ] && [ "${ARCHITECTURE}" != "arm64" ]; then \ + echo "Architecture should be amd64 or arm64"; \ + exit 1; \ + fi + +ARG GCSFUSE_VERSION +ARG GCSFUSE_REPO="https://github.com/googlecloudplatform/gcsfuse/" +ARG BRANCH_NAME="v${GCSFUSE_VERSION}" +RUN git clone ${GCSFUSE_REPO} && \ + cd gcsfuse && \ + git checkout ${BRANCH_NAME} +RUN echo "Building from branch/commit/tag: ${BRANCH_NAME}" + +RUN rm -rf /usr/local/go && \ + GO_VERSION=$(cat gcsfuse/.go-version) && \ + wget -O go_tar.tar.gz https://go.dev/dl/go${GO_VERSION}.linux-${ARCHITECTURE}.tar.gz -q && \ + tar -C /usr/local -xzf go_tar.tar.gz && \ + rm go_tar.tar.gz +RUN PATH=$PATH:/usr/local/go/bin +RUN go version ENV CGO_ENABLED=0 ENV GOOS=linux ENV GO111MODULE=auto -ARG GCSFUSE_VERSION -ARG GCSFUSE_REPO="https://github.com/googlecloudplatform/gcsfuse/" -ARG BRANCH_NAME="v${GCSFUSE_VERSION}" -RUN git clone ${GCSFUSE_REPO} ARG GCSFUSE_PATH=${GOPATH}/gcsfuse WORKDIR ${GCSFUSE_PATH} -RUN git checkout ${BRANCH_NAME} ARG DEBEMAIL="gcs-fuse-maintainers@google.com" ARG DEBFULLNAME="GCSFuse Team" @@ -39,11 +81,6 @@ ARG DEBFULLNAME="GCSFuse Team" # Install fpm package using bundle RUN bundle install --gemfile=${GCSFUSE_PATH}/tools/gem_dependency/Gemfile -ARG ARCHITECTURE="amd64" -RUN if [ "${ARCHITECTURE}" != "amd64" ] && [ "${ARCHITECTURE}" != "arm64" ]; then \ - echo "Architecture should be amd64 or arm64"; \ - exit 1; \ -fi ARG GCSFUSE_BIN="/gcsfuse_${GCSFUSE_VERSION}_${ARCHITECTURE}" ARG GCSFUSE_DOC="${GCSFUSE_BIN}/usr/share/doc/gcsfuse" WORKDIR ${GCSFUSE_PATH}/tools/build_gcsfuse @@ -84,6 +121,7 @@ RUN fpm \ -C ${GCSFUSE_BIN} \ -v ${GCSFUSE_VERSION} \ -d fuse \ + --exclude DEBIAN \ --rpm-digest sha256 \ --license Apache-2.0 \ --vendor "" \ diff --git a/tools/prefetch_cache_gcsfuse/prefetch.go b/tools/prefetch_cache_gcsfuse/prefetch.go index 4631da0a95..cf0dba12ca 100644 --- a/tools/prefetch_cache_gcsfuse/prefetch.go +++ b/tools/prefetch_cache_gcsfuse/prefetch.go @@ -26,14 +26,14 @@ import ( "time" "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/v2/internal/contentcache" + "github.com/googlecloudplatform/gcsfuse/v3/internal/contentcache" "google.golang.org/api/iterator" ) const NUM_WORKERS = 10 func downloadFile(ctx context.Context, client *storage.Client, object *storage.ObjectAttrs, cacheDir string) (err error) { - log.Printf(fmt.Sprintf("downloading file %v from bucket %v into dir %v", object.Name, object.Bucket, cacheDir)) + log.Printf("downloading file %v from bucket %v into dir %v", object.Name, object.Bucket, cacheDir) // We may want a way to verify the files are fully downloaded // and either resuming the download or discarding and redownloading the file @@ -116,7 +116,7 @@ func prefetchCache(cacheDir, bucketName, prefix string) (err error) { }() // Consumers - for i := 0; i < NUM_WORKERS; i++ { + for i := range NUM_WORKERS { wg.Add(1) go func(i int) { defer wg.Done() diff --git a/tools/proxy_server/config.go b/tools/proxy_server/config.go new file mode 100644 index 0000000000..b807994500 --- /dev/null +++ b/tools/proxy_server/config.go @@ -0,0 +1,82 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "log" + + "github.com/spf13/viper" +) + +type RetryConfig struct { + // The name of the method to apply retries to (e.g., JsonCreate, JsonStat). + Method string `yaml:"method"` + // Retry instruction (e.g., return-503, stall-33s-after-20K). + RetryInstruction string `yaml:"retryInstruction"` + // Number of times to retry. + RetryCount int `yaml:"retryCount"` + // Number of starting retry attempts to skip. + SkipCount int `yaml:"skipCount"` +} + +type HeaderValidation struct { + // Header name to validate (e.g., "user-agent", "x-goog-api-client"). + HeaderName string `yaml:"headerName"` + // Expected value or pattern to find in the header. + ExpectedPattern string `yaml:"expectedPattern"` + // Whether to fail requests if header is missing or doesn't match. + FailOnMismatch bool `yaml:"failOnMismatch"` +} + +type Config struct { + // ProxyType specifies the protocol: "http" or "grpc" + ProxyType string `yaml:"proxyType"` + // TargetHost is the address of emulator server to which proxy server interacts. + TargetHost string `yaml:"targetHost"` + RetryConfig []RetryConfig `yaml:"retryConfig"` + HeaderValidation []HeaderValidation `yaml:"headerValidation"` +} + +func printConfig(config Config) { + log.Println("Proxy Type:", config.ProxyType) + log.Println("Target Host:", config.TargetHost) + for _, retry := range config.RetryConfig { + log.Println("Method:", retry.Method) + log.Println("Retry instructions:", retry.RetryInstruction) + log.Println("Retry Count:", retry.RetryCount) + log.Println("Skip Count:", retry.SkipCount) + } + for _, header := range config.HeaderValidation { + log.Println("Header Validation - Name:", header.HeaderName) + log.Println("Header Validation - Expected Pattern:", header.ExpectedPattern) + log.Println("Header Validation - Fail On Mismatch:", header.FailOnMismatch) + } +} + +func parseConfigFile(configPath string) (*Config, error) { + var config Config + + viper.SetConfigFile(configPath) + if err := viper.ReadInConfig(); err != nil { + return nil, fmt.Errorf("error reading config file, %s", err) + } + + if err := viper.Unmarshal(&config); err != nil { + return nil, fmt.Errorf("unable to decode into struct, %v", err) + } + + return &config, nil +} diff --git a/tools/proxy_server/config_test.go b/tools/proxy_server/config_test.go new file mode 100644 index 0000000000..9723e2a12c --- /dev/null +++ b/tools/proxy_server/config_test.go @@ -0,0 +1,102 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseConfigFile(t *testing.T) { + t.Run("ValidConfigFile", func(t *testing.T) { + // Create a temporary file with valid YAML content + validContent := ` +targetHost: "http://localhost:8080" +retryConfig: + - method: "JsonCreate" + retryInstruction: "return-503" + retryCount: 5 + skipCount: 1 + - method: "JsonStat" + retryInstruction: "stall-33s-after-20K" + retryCount: 3 + skipCount: 0 +` + tempFile, err := os.CreateTemp("", "valid-config-*.yaml") + assert.NoError(t, err) + defer os.Remove(tempFile.Name()) + + _, err = tempFile.Write([]byte(validContent)) + assert.NoError(t, err) + tempFile.Close() + + // Parse the file + config, err := parseConfigFile(tempFile.Name()) + assert.NoError(t, err) + + // Assertions + assert.Equal(t, "http://localhost:8080", config.TargetHost, "unexpected TargetHost value") + + assert.Len(t, config.RetryConfig, 2, "unexpected number of RetryConfig entries") + assert.Equal(t, "JsonCreate", config.RetryConfig[0].Method, "unexpected method in first RetryConfig entry") + assert.Equal(t, "return-503", config.RetryConfig[0].RetryInstruction, "unexpected retryInstruction in first RetryConfig entry") + assert.Equal(t, 5, config.RetryConfig[0].RetryCount, "unexpected retryCount in first RetryConfig entry") + assert.Equal(t, 1, config.RetryConfig[0].SkipCount, "unexpected skipCount in first RetryConfig entry") + + assert.Equal(t, "JsonStat", config.RetryConfig[1].Method, "unexpected method in second RetryConfig entry") + assert.Equal(t, "stall-33s-after-20K", config.RetryConfig[1].RetryInstruction, "unexpected retryInstruction in second RetryConfig entry") + assert.Equal(t, 3, config.RetryConfig[1].RetryCount, "unexpected retryCount in second RetryConfig entry") + assert.Equal(t, 0, config.RetryConfig[1].SkipCount, "unexpected skipCount in second RetryConfig entry") + }) + + t.Run("EmptyConfigFile", func(t *testing.T) { + // Create an empty temporary file + tempFile, err := os.CreateTemp("", "empty-config-*.yaml") + assert.NoError(t, err) + defer os.Remove(tempFile.Name()) + tempFile.Close() + + // Parse the file + config, err := parseConfigFile(tempFile.Name()) + + // Assertions + assert.NoError(t, err) + assert.Nil(t, config.RetryConfig, "config should be nil") + }) + + t.Run("InvalidConfigFile", func(t *testing.T) { + // Create a file with invalid content + invalidContent := ` +invalid_key: "invalid_value" +another_invalid_key: +` + tempFile, err := os.CreateTemp("", "invalid-config-*.yaml") + assert.NoError(t, err) + defer os.Remove(tempFile.Name()) + + _, err = tempFile.Write([]byte(invalidContent)) + assert.NoError(t, err) + tempFile.Close() + + // Parse the file + config, err := parseConfigFile(tempFile.Name()) + + // Assertions + assert.NoError(t, err) + assert.Nil(t, config.RetryConfig) + }) +} diff --git a/tools/proxy_server/emulator.go b/tools/proxy_server/emulator.go new file mode 100644 index 0000000000..85b1ef4f68 --- /dev/null +++ b/tools/proxy_server/emulator.go @@ -0,0 +1,105 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/url" +) + +type emulatorTest struct { + // host is the address of proxy server to which client interacts. + host *url.URL +} + +type RetryTestClient interface { + GetRetryID(instructions map[string][]string, transport string) (string, error) +} + +// GetRetryID creates a retry test resource in the emulator. +func (et *emulatorTest) GetRetryID(instructions map[string][]string, transport string) (string, error) { + c := http.DefaultClient + data := struct { + Instructions map[string][]string `json:"instructions"` + Transport string `json:"transport"` + }{ + Instructions: instructions, + Transport: transport, + } + + buf := new(bytes.Buffer) + if err := json.NewEncoder(buf).Encode(data); err != nil { + return "", fmt.Errorf("encoding request: %v\n", err) + } + + et.host.Path = "retry_test" + resp, err := c.Post(et.host.String(), "application/json", buf) + if err != nil || resp.StatusCode != 200 { + return "", fmt.Errorf("creating retry test: err: %v, resp: %+v\n", err, resp) + } + defer func() { + closeErr := resp.Body.Close() + if err == nil { + err = closeErr + } + }() + + testRes := struct { + TestID string `json:"id"` + }{} + if err := json.NewDecoder(resp.Body).Decode(&testRes); err != nil { + return "", fmt.Errorf("decoding test ID: %v\n", err) + } + + et.host.Path = "" + return testRes.TestID, nil +} + +// CreateRetryTest creates a retry test using the provided instructions. +// +// It takes emulator host (URL string) and a map of instructions. The instructions map +// specifies how the emulator should behave for specific requests. +// +// The keys of the `instructions` map are request paths, and the values are +// lists of strings representing actions to be taken for those requests. +// These actions can include things like returning specific error codes or +// simulating delays. +// +// Example `instructions` map: +// +// { +// "storage.objects.list": []string{"return-503"}, // Return a 503 error for this request +// "storage.objects.get": []string{"stall-100ms", "return-200"}, // Delay for 100ms then return success +// } +// +// The function returns a unique ID for the retry test, which can be used to +// identify and manage the test. It returns an error if there is a problem +// parsing the host URL or setting up the emulator test. +func CreateRetryTest(host string, instructions map[string][]string) (string, error) { + if len(instructions) == 0 { + return "", nil + } + + endpoint, err := url.Parse(host) + if err != nil { + return "", fmt.Errorf("Failed to parse host env: %v\n", err) + } + + et := &emulatorTest{host: endpoint} + return et.GetRetryID(instructions, "http") +} diff --git a/tools/proxy_server/emulator_test.go b/tools/proxy_server/emulator_test.go new file mode 100644 index 0000000000..7595b3cd5d --- /dev/null +++ b/tools/proxy_server/emulator_test.go @@ -0,0 +1,65 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestGetRetryID tests the GetRetryID function +func TestGetRetryID(t *testing.T) { + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/retry_test", r.URL.Path, "Unexpected URL path") + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(map[string]string{"id": "test-id-123"}) + assert.NoError(t, err) + })) + defer mockServer.Close() + hostURL, _ := url.Parse(mockServer.URL) + et := &emulatorTest{host: hostURL} + instructions := map[string][]string{"retry": {"retry-instruction"}} + + testID, err := et.GetRetryID(instructions, "http") + + assert.NoError(t, err) + assert.Equal(t, "test-id-123", testID, "Unexpected test ID returned") +} + +// TestCreateRetryTest tests the CreateRetryTest function +func TestCreateRetryTest(t *testing.T) { + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/retry_test", r.URL.Path, "Unexpected URL path") + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(map[string]string{"id": "test-id-123"}) + assert.NoError(t, err) + })) + defer mockServer.Close() + + instructions := map[string][]string{"retry": {"retry-instruction"}} + testID, err := CreateRetryTest(mockServer.URL, instructions) + assert.NoError(t, err) + assert.Equal(t, "test-id-123", testID, "Unexpected test ID returned") + + // Test with empty instructions + testID, err = CreateRetryTest(mockServer.URL, map[string][]string{}) + assert.NoError(t, err) + assert.Equal(t, "", testID, "Expected empty test ID for empty instructions") +} diff --git a/tools/proxy_server/grpc_proxy.go b/tools/proxy_server/grpc_proxy.go new file mode 100644 index 0000000000..be0998e1c7 --- /dev/null +++ b/tools/proxy_server/grpc_proxy.go @@ -0,0 +1,220 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "io" + "log" + "net" + "strings" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/encoding" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +// rawCodec is a Codec that passes through raw bytes without any encoding/decoding. +type rawCodec struct{} + +func (rawCodec) Marshal(v any) ([]byte, error) { + out, ok := v.(*[]byte) + if !ok { + return nil, fmt.Errorf("failed to marshal, message is %T, want *[]byte", v) + } + return *out, nil +} + +func (rawCodec) Unmarshal(data []byte, v any) error { + dst, ok := v.(*[]byte) + if !ok { + return fmt.Errorf("failed to unmarshal, message is %T, want *[]byte", v) + } + *dst = data + return nil +} + +func (rawCodec) Name() string { + return "raw-proxy-codec" +} + +func init() { + encoding.RegisterCodec(rawCodec{}) +} + +// validateGRPCMetadata validates gRPC metadata based on configured header validations +func validateGRPCMetadata(md metadata.MD, validations []HeaderValidation) error { + for _, validation := range validations { + headerValues := md.Get(validation.HeaderName) + + if len(headerValues) == 0 { + if validation.FailOnMismatch { + return status.Errorf(codes.InvalidArgument, "required metadata %s not found", validation.HeaderName) + } + log.Printf("Warning: metadata %s not found", validation.HeaderName) + continue + } + + // Check if any of the header values contain the expected pattern + found := false + for _, headerValue := range headerValues { + if validation.ExpectedPattern != "" && strings.Contains(headerValue, validation.ExpectedPattern) { + found = true + log.Printf("Metadata validation passed: %s = %s (contains '%s')", validation.HeaderName, headerValue, validation.ExpectedPattern) + break + } + } + + if validation.ExpectedPattern != "" && !found { + err := fmt.Errorf("metadata %s values %v do not contain expected pattern '%s'", + validation.HeaderName, headerValues, validation.ExpectedPattern) + if validation.FailOnMismatch { + return status.Errorf(codes.InvalidArgument, "%v", err) + } + log.Printf("Warning: %v", err) + } + } + return nil +} + +// startGRPCProxy creates a transparent gRPC proxy that validates metadata and forwards all requests to the target +func startGRPCProxy(listener net.Listener, targetHost string, validations []HeaderValidation) error { + // Create connection to target for forwarding with raw codec to avoid unmarshaling + targetConn, err := grpc.NewClient( + targetHost, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultCallOptions(grpc.ForceCodec(rawCodec{})), + ) + if err != nil { + return fmt.Errorf("failed to connect to target %s: %w", targetHost, err) + } + + // Create unknown service handler that validates metadata and forwards all calls + unknownHandler := func(srv any, stream grpc.ServerStream) error { + fullMethodName, ok := grpc.MethodFromServerStream(stream) + if !ok { + return status.Errorf(codes.Internal, "failed to get method name") + } + + // Log the gRPC call + log.Printf("=== Proxying Call: %s ===", fullMethodName) + + // Extract and validate metadata + md, ok := metadata.FromIncomingContext(stream.Context()) + if ok { + if err := validateGRPCMetadata(md, validations); err != nil { + log.Printf("Metadata validation failed: %v", err) + return err + } + } else if len(validations) > 0 { + log.Println("Warning: No metadata found in request") + } + + // Forward metadata to target + forwardCtx := metadata.NewOutgoingContext(stream.Context(), md) + + // Invoke the method on the target + clientStream, err := targetConn.NewStream(forwardCtx, &grpc.StreamDesc{ + StreamName: fullMethodName, + ServerStreams: true, + ClientStreams: true, + }, fullMethodName, grpc.ForceCodec(rawCodec{})) + if err != nil { + return status.Errorf(codes.Internal, "failed to create target stream: %v", err) + } + + // Proxy messages bidirectionally using raw bytes + // Create channels for error handling + clientToServer := make(chan error, 1) + serverToClient := make(chan error, 1) + + // Receive from client, send to target + go func() { + for { + req := new([]byte) + if err := stream.RecvMsg(req); err != nil { + if err == io.EOF { + err = clientStream.CloseSend() + clientToServer <- err + return + } + clientToServer <- fmt.Errorf("receiving from client: %w", err) + return + } + if err := clientStream.SendMsg(req); err != nil { + clientToServer <- fmt.Errorf("sending to server: %w", err) + return + } + } + }() + + // Receive from target, send to client + go func() { + for { + resp := new([]byte) + if err := clientStream.RecvMsg(resp); err != nil { + if err == io.EOF { + serverToClient <- nil + return + } + serverToClient <- fmt.Errorf("receiving from server: %w", err) + return + } + if err := stream.SendMsg(resp); err != nil { + serverToClient <- fmt.Errorf("sending to client: %w", err) + return + } + } + }() + + // Wait for BOTH directions to complete + // This is important for both unary and streaming RPCs + var err1, err2 error + for i := 0; i < 2; i++ { + select { + case err := <-clientToServer: + if err != nil { + log.Printf("Client to server error: %v", err) + err1 = err + } + case err := <-serverToClient: + if err != nil { + log.Printf("Server to client error: %v", err) + err2 = err + } + } + } + + // Return first error encountered, or nil if both succeeded + if err1 != nil { + return err1 + } + return err2 + } + + // Create gRPC server with unknown service handler and raw codec + opts := []grpc.ServerOption{ + grpc.UnknownServiceHandler(unknownHandler), + grpc.ForceServerCodec(rawCodec{}), + } + grpcServer := grpc.NewServer(opts...) + + log.Printf("gRPC proxy server ready to accept connections, forwarding to %s", targetHost) + + return grpcServer.Serve(listener) +} diff --git a/tools/proxy_server/main.go b/tools/proxy_server/main.go new file mode 100644 index 0000000000..a4095463f3 --- /dev/null +++ b/tools/proxy_server/main.go @@ -0,0 +1,293 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "log" + "net" + "net/http" + "net/url" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "time" +) + +const PortAndProxyProcessIdInfoLogFormat = "Listening Proxy Server On Port [%s] with Process ID [%d]" + +var ( + // Flag to accept config-file path. + fConfigPath = flag.String("config-path", "configs/config.yaml", "Path to the file") + // Flag to turn on/off debug logs. + fDebug = flag.Bool("debug", true, "Enable proxy server debug logs.") + // Log file to write proxy server logs. + fLogFilePath = flag.String("log-file", "", "Path to the log file") + // Initialized before the server gets started. + gConfig *Config + gOpManager *OperationManager + // Port number assigned to listener. + gPort string +) + +type ProxyHandler struct { + http.Handler +} + +// logRequestAndType is used for logging the request on proxy server. +// More fields can be added or removed as per requirement for debugging purpose. +func logRequestAndType(req *http.Request, r RequestType) { + // Print empty lines to separate each request in log. + log.Println("") + log.Println("") + log.Printf("RequestType: %s\n", r) + log.Printf("URL: %s\n", req.URL.String()) + log.Printf("Content-Length: %s\n", req.Header.Get("Content-Length")) + log.Printf("Content-Range: %s\n", req.Header.Get("Content-Range")) +} + +// AddRetryID creates mock error behavior on the target host for specific request types. +// It retrieves the corresponding operation from the operation manager based on the provided RequestTypeAndInstruction. +// If a matching operation is found, it creates a retry test with the target host and instruction, +// and attaches the generated test ID to the HTTP request header "x-retry-test-id". +// +// This function is used to simulate error scenarios for testing retry mechanisms. +func AddRetryID(req *http.Request, r RequestTypeAndInstruction) error { + plantOp := gOpManager.retrieveOperation(r.RequestType) + if *fDebug { + logRequestAndType(req, r.RequestType) + if plantOp != "" { + log.Println("Planting operation: ", plantOp) + } + } + + if plantOp != "" { + testID, err := CreateRetryTest(gConfig.TargetHost, map[string][]string{r.Instruction: {plantOp}}) + if err != nil { + return fmt.Errorf("CreateRetryTest: %v", err) + } + req.Header.Set("x-retry-test-id", testID) + } + return nil +} + +// ServeHTTP handles incoming HTTP requests. It acts as a proxy, forwarding requests +// to a target server specified in the configuration and then relaying the +// response back to the original client. +func (ph ProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + targetURL := fmt.Sprintf("%s%s", gConfig.TargetHost, r.RequestURI) + req, err := http.NewRequest(r.Method, targetURL, r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + for name, values := range r.Header { + for _, value := range values { + req.Header.Add(name, value) + } + } + // Determine the request type and instruction (e.g., read, write, metadata) based on the incoming request. + reqTypeAndInstruction := deduceRequestTypeAndInstruction(r) + + // Add a unique retry ID to the request headers, associating it with the + // deduced request type and instruction. This is used for adding custom failures on requests. + err = AddRetryID(req, reqTypeAndInstruction) + if err != nil { + log.Printf("AddRetryID: %v", err) + } + + // Send the request to the target server + client := &http.Client{} + start := time.Now() + resp, err := client.Do(req) + elapsed := time.Since(start) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + respURL, err := resp.Location() + // Change the response URL host to the proxy server host. + // This is necessary because, from the client's perspective, the proxy server is the endpoint. + // Therefore, the response must appear to originate from the proxy host. + if err == nil { + // Parse the original URL. + u, err := url.Parse(respURL.String()) + if err != nil { + log.Println("Error parsing URL:", err) + return + } + + u.Host = "localhost:" + gPort + resp.Header.Set("Location", u.String()) + } + + defer resp.Body.Close() + + // Copy headers from the target server's response + for name, values := range resp.Header { + for _, value := range values { + w.Header().Add(name, value) + } + } + + // Copy the response body + w.WriteHeader(resp.StatusCode) + _, err = io.Copy(w, resp.Body) + if err != nil { + log.Printf("Error in coping response body: %v", err) + } + if *fDebug { + log.Printf("Respnse Status: %d\n", resp.StatusCode) + log.Printf("Elapsed Time: %.3fs\n", elapsed.Seconds()) + } +} + +// ProxyServer represents a simple proxy server over GCS storage based API endpoint. +type ProxyServer struct { + server *http.Server + shutdown chan os.Signal +} + +// NewProxyServer creates a new ProxyServer instance +func NewProxyServer() *ProxyServer { + return &ProxyServer{ + shutdown: make(chan os.Signal, 1), + } +} + +// Start starts the proxy server. +func (ps *ProxyServer) Start() { + // Create a listener on random available port. + listener, err := net.Listen("tcp", ":0") + if err != nil { + log.Fatalf("Error on listening: %v", err) + } + gPort = strconv.Itoa(listener.Addr().(*net.TCPAddr).Port) + // Log port number and proxy process Id for the proxy server. + log.Printf(PortAndProxyProcessIdInfoLogFormat, gPort, os.Getpid()) + ps.server = &http.Server{ + Addr: ":" + gPort, + Handler: ProxyHandler{}, + } + + // Start the server in a new goroutine + go func() { + if err := ps.server.Serve(listener); err != nil && err != http.ErrServerClosed { + log.Fatalf("Server error: %v", err) + } + }() + + // Handle graceful shutdown + signal.Notify(ps.shutdown, syscall.SIGINT, syscall.SIGTERM) + // Blocks until one of the Signal is recieved. + <-ps.shutdown + log.Println("Shutting down proxy server...") + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := ps.server.Shutdown(ctx); err != nil { + log.Fatalf("Proxy server forced to shutdown: %v", err) + } else { + log.Println("Proxy server exiting") + } +} + +func main() { + // Parse the command-line flags + flag.Parse() + + var err error + gConfig, err = parseConfigFile(*fConfigPath) + if err != nil { + log.Printf("Parsing error: %v\n", err) + os.Exit(1) + } + + if *fLogFilePath == "" { + log.Println("No log file path for proxy server provided.") + os.Exit(1) + } + logFile, err := os.OpenFile(*fLogFilePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + log.Printf("Error opening log file: %v\n", err) + os.Exit(1) + } + defer logFile.Close() + log.SetOutput(logFile) + + if *fDebug { + printConfig(*gConfig) + } + + gOpManager = NewOperationManager(*gConfig) + + // Determine proxy type from config (default to http if not specified) + switch strings.ToLower(gConfig.ProxyType) { + case "grpc": + log.Println("Starting gRPC proxy server...") + NewGRPCProxyServer().Start() + default: + log.Println("Starting HTTP proxy server...") + NewProxyServer().Start() + } +} + +// GRPCProxyServer represents a gRPC proxy server +type GRPCProxyServer struct { + shutdown chan os.Signal +} + +// NewGRPCProxyServer creates a new GRPCProxyServer instance +func NewGRPCProxyServer() *GRPCProxyServer { + return &GRPCProxyServer{ + shutdown: make(chan os.Signal, 1), + } +} + +// Start starts the gRPC proxy server +func (gs *GRPCProxyServer) Start() { + // Create a listener on random available port + listener, err := net.Listen("tcp", ":0") + if err != nil { + log.Fatalf("Error on listening: %v", err) + } + gPort = strconv.Itoa(listener.Addr().(*net.TCPAddr).Port) + log.Printf(PortAndProxyProcessIdInfoLogFormat, gPort, os.Getpid()) + + // Start gRPC server in goroutine + go func() { + if err := startGRPCProxy(listener, gConfig.TargetHost, gConfig.HeaderValidation); err != nil && !errors.Is(err, net.ErrClosed) { + log.Fatalf("gRPC server error: %v", err) + } + }() + + // Handle graceful shutdown + signal.Notify(gs.shutdown, syscall.SIGINT, syscall.SIGTERM) + <-gs.shutdown + log.Println("Shutting down gRPC proxy server...") + if err = listener.Close(); err != nil { + log.Printf("Error closing listener: %v", err) + } else { + log.Println("gRPC proxy server exited") + } +} diff --git a/tools/proxy_server/main_test.go b/tools/proxy_server/main_test.go new file mode 100644 index 0000000000..4006cd1014 --- /dev/null +++ b/tools/proxy_server/main_test.go @@ -0,0 +1,51 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestAddRetryID tests the AddRetryID function +func TestAddRetryID(t *testing.T) { + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(map[string]string{"id": "test-id-123"}) + assert.NoError(t, err) + })) + defer mockServer.Close() + + gConfig = &Config{TargetHost: mockServer.URL} + gOpManager = &OperationManager{ + retryConfigs: map[RequestType][]RetryConfig{ + "TestType": {{Method: "TestType", RetryInstruction: "retry-instruction", RetryCount: 1, SkipCount: 0}}, + }, + } + req, _ := http.NewRequest("GET", "http://example.com", nil) + r := RequestTypeAndInstruction{ + RequestType: "TestType", + Instruction: "retry", + } + + err := AddRetryID(req, r) + + assert.NoError(t, err) + assert.Equal(t, "test-id-123", req.Header.Get("x-retry-test-id"), "Unexpected x-retry-test-id header value") +} diff --git a/tools/proxy_server/operation_manager.go b/tools/proxy_server/operation_manager.go new file mode 100644 index 0000000000..d002f15b36 --- /dev/null +++ b/tools/proxy_server/operation_manager.go @@ -0,0 +1,80 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "sync" +) + +type OperationManager struct { + retryConfigs map[RequestType][]RetryConfig + mu sync.Mutex +} + +func NewOperationManager(config Config) *OperationManager { + rc := make(map[RequestType][]RetryConfig) + om := &OperationManager{ + retryConfigs: rc, + } + for _, retryConfig := range config.RetryConfig { + om.addRetryConfig(retryConfig) + } + + if *fDebug { + println(om) + } + + return om +} + +// Empty string represent there is no plantation required. +func (om *OperationManager) retrieveOperation(requestType RequestType) string { + om.mu.Lock() + defer om.mu.Unlock() + + configs, ok := om.retryConfigs[requestType] + if !ok { + return "" + } + + for len(configs) > 0 { + cc := &configs[0] + if cc.SkipCount > 0 { + cc.SkipCount-- + return "" + } else if cc.RetryCount > 0 { + cc.RetryCount-- + return cc.RetryInstruction + } else { + configs = configs[1:] + om.retryConfigs[requestType] = configs + } + } + return "" +} + +func (om *OperationManager) addRetryConfig(rc RetryConfig) { + rt := RequestType(rc.Method) + if *fDebug { + println(rt) + } + if om.retryConfigs[rt] != nil { + // Key exists, append the new retryConfig to the existing list + om.retryConfigs[rt] = append(om.retryConfigs[rt], rc) + } else { + // Key doesn't exist, getRetryID a new list with the retryConfig + om.retryConfigs[rt] = []RetryConfig{rc} + } +} diff --git a/tools/proxy_server/operation_manager_test.go b/tools/proxy_server/operation_manager_test.go new file mode 100644 index 0000000000..45663c0eb3 --- /dev/null +++ b/tools/proxy_server/operation_manager_test.go @@ -0,0 +1,155 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewOperationManager(t *testing.T) { + config := Config{ + RetryConfig: []RetryConfig{ + {Method: "JsonCreate", RetryInstruction: "return-503", RetryCount: 2, SkipCount: 1}, + {Method: "JsonStat", RetryInstruction: "stall-33s-after-20K", RetryCount: 3, SkipCount: 0}, + }, + } + + om := NewOperationManager(config) + + // Assert that retryConfigs are initialized correctly + assert.Len(t, om.retryConfigs, 2) + assert.Len(t, om.retryConfigs["JsonCreate"], 1) + assert.Len(t, om.retryConfigs["JsonStat"], 1) + + assert.Equal(t, "return-503", om.retryConfigs["JsonCreate"][0].RetryInstruction) + assert.Equal(t, "stall-33s-after-20K", om.retryConfigs["JsonStat"][0].RetryInstruction) +} + +func TestRetrieveOperation(t *testing.T) { + t.Run("One config test", func(t *testing.T) { + config := Config{ + RetryConfig: []RetryConfig{ + {Method: "JsonCreate", RetryInstruction: "return-503", RetryCount: 2, SkipCount: 1}, + }, + } + om := NewOperationManager(config) + + // First call: Skip count is decremented, so no retry instruction should be returned + result := om.retrieveOperation("JsonCreate") + assert.Equal(t, "", result, "Expected empty result due to SkipCount") + + // Second call: Retry instruction should be returned + result = om.retrieveOperation("JsonCreate") + assert.Equal(t, "return-503", result, "Expected 'return-503' as RetryInstruction") + + // Third call: Retry instruction should be returned again + result = om.retrieveOperation("JsonCreate") + assert.Equal(t, "return-503", result, "Expected 'return-503' as RetryInstruction") + + // Fourth call: Retry count is exhausted, so no retry instruction should be returned + result = om.retrieveOperation("JsonCreate") + assert.Equal(t, "", result, "Expected empty result as RetryCount is exhausted") + }) + + t.Run("Multiple config tests with same request types", func(t *testing.T) { + // Initialize OperationManager with two retry configs + config := Config{ + RetryConfig: []RetryConfig{ + {Method: "RequestTypeA", RetryInstruction: "retry-503", RetryCount: 2, SkipCount: 1}, + {Method: "RequestTypeA", RetryInstruction: "retry-202", RetryCount: 1, SkipCount: 0}, + }, + } + om := NewOperationManager(config) + + // Test for RequestTypeA + // First call: SkipCount is decremented, so no retry instruction should be returned + result := om.retrieveOperation("RequestTypeA") + assert.Equal(t, "", result, "Expected no result due to SkipCount") + + // Second call: First retry instruction should be returned + result = om.retrieveOperation("RequestTypeA") + assert.Equal(t, "retry-503", result, "Expected 'retry-503' as RetryInstruction") + + // Third call: Second retry instruction should be returned + result = om.retrieveOperation("RequestTypeA") + assert.Equal(t, "retry-503", result, "Expected 'retry-503' as RetryInstruction") + + // Fourth call: Move to the second config for RequestTypeA + result = om.retrieveOperation("RequestTypeA") + assert.Equal(t, "retry-202", result, "Expected 'retry-202' as RetryInstruction") + + // Fifth call: All retry instructions exhausted, so no result + result = om.retrieveOperation("RequestTypeA") + assert.Equal(t, "", result, "Expected no result as all retries are exhausted") + }) + + t.Run("Multiple config tests with different request types", func(t *testing.T) { + // Initialize OperationManager with two retry configs + config := Config{ + RetryConfig: []RetryConfig{ + {Method: "RequestTypeA", RetryInstruction: "retry-503", RetryCount: 2, SkipCount: 1}, + {Method: "RequestTypeB", RetryInstruction: "retry-202", RetryCount: 1, SkipCount: 0}, + }, + } + om := NewOperationManager(config) + + // Test for RequestTypeA + // First call: SkipCount is decremented, so no retry instruction should be returned + result := om.retrieveOperation("RequestTypeA") + assert.Equal(t, "", result, "Expected no result due to SkipCount") + + // Second call: First retry instruction should be returned + result = om.retrieveOperation("RequestTypeA") + assert.Equal(t, "retry-503", result, "Expected 'retry-503' as RetryInstruction") + + // Third call: Second retry instruction should be returned + result = om.retrieveOperation("RequestTypeA") + assert.Equal(t, "retry-503", result, "Expected 'retry-503' as RetryInstruction") + + // Forth call: All retry instructions for A exhausted, so no result + result = om.retrieveOperation("RequestTypeA") + assert.Equal(t, "", result, "Expected no result as all retries are exhausted") + + // Test for RequestTypeB + // Fifth call: Move to the config for RequestTypeB + result = om.retrieveOperation("RequestTypeB") + assert.Equal(t, "retry-202", result, "Expected 'retry-202' as RetryInstruction") + }) +} + +func TestAddRetryConfig(t *testing.T) { + om := &OperationManager{ + retryConfigs: make(map[RequestType][]RetryConfig), + } + + retryConfig := RetryConfig{Method: "JsonUpdate", RetryInstruction: "retry-202", RetryCount: 1, SkipCount: 0} + om.addRetryConfig(retryConfig) + + // Assert the retryConfig is added to the map + assert.Len(t, om.retryConfigs, 1) + assert.Len(t, om.retryConfigs["JsonUpdate"], 1) + assert.Equal(t, "retry-202", om.retryConfigs["JsonUpdate"][0].RetryInstruction) + + // Add another retryConfig for the same method + retryConfig2 := RetryConfig{Method: "JsonUpdate", RetryInstruction: "retry-503", RetryCount: 2, SkipCount: 1} + om.addRetryConfig(retryConfig2) + + // Assert both retryConfigs are stored under the same key + assert.Len(t, om.retryConfigs["JsonUpdate"], 2) + assert.Equal(t, "retry-202", om.retryConfigs["JsonUpdate"][0].RetryInstruction) + assert.Equal(t, "retry-503", om.retryConfigs["JsonUpdate"][1].RetryInstruction) +} diff --git a/tools/proxy_server/request_mapper.go b/tools/proxy_server/request_mapper.go new file mode 100644 index 0000000000..3d9a9ef47f --- /dev/null +++ b/tools/proxy_server/request_mapper.go @@ -0,0 +1,80 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "net/http" + "strings" +) + +type RequestType string + +const ( + XmlRead RequestType = "XmlRead" + JsonStat RequestType = "JsonStat" + JsonDelete RequestType = "JsonDelete" + JsonUpdate RequestType = "JsonUpdate" + JsonCreate RequestType = "JsonCreate" + JsonCopy RequestType = "JsonCopy" + JsonList RequestType = "JsonList" + JsonCompose RequestType = "JsonCompose" + Unknown RequestType = "Unknown" +) + +type RequestTypeAndInstruction struct { + RequestType RequestType + Instruction string +} + +// deduceRequestTypeAndInstruction determines the type of request and its corresponding instruction +func deduceRequestTypeAndInstruction(r *http.Request) RequestTypeAndInstruction { + path := r.URL.Path + method := r.Method + + if isJsonAPI(path) { + switch method { + // TODO: Implement logic to differentiate JSON read requests from general HTTP GET requests for the purpose of testing JSON read functionality. + case http.MethodGet: + // Check if path ends with `/o` (indicates listing objects) + if strings.HasSuffix(path, "/o") { + return RequestTypeAndInstruction{JsonList, "storage.objects.list"} + } + // Check if path has `/o/<object-name>` (indicates stat operation) + if strings.Contains(path, "/o/") { + return RequestTypeAndInstruction{JsonStat, "storage.objects.get"} + } + case http.MethodPost: + return RequestTypeAndInstruction{JsonCreate, "storage.objects.insert"} + case http.MethodDelete: + return RequestTypeAndInstruction{JsonDelete, "storage.objects.delete"} + case http.MethodPut: + return RequestTypeAndInstruction{JsonUpdate, "storage.objects.update"} + default: + return RequestTypeAndInstruction{Unknown, ""} + } + } + switch method { + case http.MethodGet: + return RequestTypeAndInstruction{XmlRead, "storage.objects.get"} + default: + return RequestTypeAndInstruction{Unknown, ""} + } +} + +// isJsonAPI checks if the request is targeting the JSON API +func isJsonAPI(path string) bool { + // The JSON API path includes "/storage/v1", while the XML API path does not. + return strings.Contains(path, "/storage/v1") +} diff --git a/tools/proxy_server/request_mappper_test.go b/tools/proxy_server/request_mappper_test.go new file mode 100644 index 0000000000..5b856e68f2 --- /dev/null +++ b/tools/proxy_server/request_mappper_test.go @@ -0,0 +1,59 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "net/http" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDeduceRequestTypeAndInstruction(t *testing.T) { + tests := []struct { + name string + method string + path string + expectedReq RequestType + expectedIns string + }{ + // JSON API Tests + {"JsonStat GET", http.MethodGet, "/storage/v1/bucket/o/object", JsonStat, "storage.objects.get"}, + {"JsonList GET", http.MethodGet, "/storage/v1/bucket/o", JsonList, "storage.objects.list"}, + {"JsonCreate POST", http.MethodPost, "/storage/v1/bucket/o", JsonCreate, "storage.objects.insert"}, + {"JsonDelete DELETE", http.MethodDelete, "/storage/v1/bucket/o", JsonDelete, "storage.objects.delete"}, + {"JsonUpdate PUT", http.MethodPut, "/storage/v1/bucket/o", JsonUpdate, "storage.objects.update"}, + {"JsonUnknown PATCH", http.MethodPatch, "/storage/v1/bucket/o", Unknown, ""}, + + // XML API Tests + {"XmlRead GET", http.MethodGet, "/bucket/object", XmlRead, "storage.objects.get"}, + {"XmlUnknown POST", http.MethodPost, "/bucket/object", Unknown, ""}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req := &http.Request{ + Method: test.method, + URL: &url.URL{Path: test.path}, + } + + result := deduceRequestTypeAndInstruction(req) + + assert.Equal(t, test.expectedReq, result.RequestType) + assert.Equal(t, test.expectedIns, result.Instruction) + }) + } +} diff --git a/tools/util/build_gcsfuse.go b/tools/util/build_gcsfuse.go index 0ba19bb3f4..bf02cfda3c 100644 --- a/tools/util/build_gcsfuse.go +++ b/tools/util/build_gcsfuse.go @@ -53,7 +53,7 @@ func BuildGcsfuse(dstDir string) (err error) { { var pkg *build.Package pkg, err = build.Import( - "github.com/googlecloudplatform/gcsfuse/v2", + "github.com/googlecloudplatform/gcsfuse/v3", "", build.FindOnly) @@ -73,7 +73,7 @@ func BuildGcsfuse(dstDir string) (err error) { toolPath, srcDir, dstDir, - "fake_version", + "0.0.0", ) var output []byte @@ -94,7 +94,7 @@ func buildBuildGcsfuse(dst string) (err error) { { var pkg *build.Package pkg, err = build.Import( - "github.com/googlecloudplatform/gcsfuse/v2/tools/build_gcsfuse", + "github.com/googlecloudplatform/gcsfuse/v3/tools/build_gcsfuse", "", build.FindOnly) diff --git a/tracing/benchmark_test.go b/tracing/benchmark_test.go new file mode 100644 index 0000000000..38d253a2b1 --- /dev/null +++ b/tracing/benchmark_test.go @@ -0,0 +1,141 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tracing + +import ( + "context" + "fmt" + "testing" +) + +func BenchmarkTrace(b *testing.B) { + traceHandlers := []struct { + prefix string + traceHandle TraceHandle + }{ + { + prefix: "Otel", + traceHandle: NewOTELTracer(), + }, + { + prefix: "Noop", + traceHandle: NewNoopTracer(), + }, + } + for _, tc := range traceHandlers { + th := tc.traceHandle + prefix := tc.prefix + + b.Run(fmt.Sprintf("BenchmarkStartSpan_%s", prefix), func(b *testing.B) { + ctx := context.Background() + + for b.Loop() { + _, span := th.StartSpan(ctx, "TestSpanName") + th.EndSpan(span) + } + }) + + b.Run(fmt.Sprintf("BenchmarkStartServerSpan_%s", prefix), func(b *testing.B) { + ctx := context.Background() + + for b.Loop() { + _, span := th.StartServerSpan(ctx, "TestSpanName") + th.EndSpan(span) + } + }) + + b.Run(fmt.Sprintf("BenchmarkRecordError_%s", prefix), func(b *testing.B) { + ctx := context.Background() + err := fmt.Errorf("TestError") + _, span := th.StartSpan(ctx, "TestSpanName") + + for b.Loop() { + th.RecordError(span, err) + } + + th.EndSpan(span) + }) + + b.Run(fmt.Sprintf("BenchmarkTraceUploadWithErrorNoBytes_%s", prefix), func(b *testing.B) { + ctx := context.Background() + err := fmt.Errorf("TestError") + bytes := int64(0) + + for b.Loop() { + _, finishSpan := th.TraceUpload(ctx, "TestSpanName", "A/B/C/test_file.text", &bytes, &err) + finishSpan() + } + }) + + b.Run(fmt.Sprintf("BenchmarkTraceUploadWithErrorWithBytes_%s", prefix), func(b *testing.B) { + ctx := context.Background() + err := fmt.Errorf("TestError") + bytes := int64(33554432) + + for b.Loop() { + _, finishSpan := th.TraceUpload(ctx, "TestSpanName", "A/B/C/test_file.text", &bytes, &err) + finishSpan() + } + }) + + b.Run(fmt.Sprintf("BenchmarkTraceUploadWithoutErrorNoBytes_%s", prefix), func(b *testing.B) { + ctx := context.Background() + bytes := int64(0) + + for b.Loop() { + _, finishSpan := th.TraceUpload(ctx, "TestSpanName", "A/B/C/test_file.text", &bytes, nil) + finishSpan() + } + }) + + b.Run(fmt.Sprintf("BenchmarkTraceUploadWithoutErrorWithBytes_%s", prefix), func(b *testing.B) { + ctx := context.Background() + bytes := int64(33554432) + + for b.Loop() { + _, finishSpan := th.TraceUpload(ctx, "TestSpanName", "A/B/C/test_file.text", &bytes, nil) + finishSpan() + } + }) + + b.Run(fmt.Sprintf("BenchmarkSetCacheReadAttributes_%s", prefix), func(b *testing.B) { + ctx := context.Background() + + _, span := th.StartSpan(ctx, "TestSpanName") + for b.Loop() { + th.SetCacheReadAttributes(span, true, 100) + } + th.EndSpan(span) + }) + + b.Run(fmt.Sprintf("BenchmarkSetUploadAttributes_%s", prefix), func(b *testing.B) { + ctx := context.Background() + + _, span := th.StartSpan(ctx, "TestSpanName") + for b.Loop() { + th.SetUploadAttributes(span, 100, "A/B/C/test_file.text") + } + th.EndSpan(span) + }) + + b.Run(fmt.Sprintf("BenchmarkPropagateTraceContext_%s", prefix), func(b *testing.B) { + ctx := context.Background() + + for b.Loop() { + _ = th.PropagateTraceContext(ctx, ctx) + } + }) + } +} diff --git a/tracing/noop_tracer.go b/tracing/noop_tracer.go new file mode 100644 index 0000000000..4119945f4c --- /dev/null +++ b/tracing/noop_tracer.go @@ -0,0 +1,57 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tracing + +import ( + "context" + + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" +) + +var ( + emptyFinisher = func() {} +) + +type noopTracer struct{} + +func (*noopTracer) StartSpan(ctx context.Context, traceName string) (context.Context, trace.Span) { + return ctx, noop.Span{} +} + +func (*noopTracer) StartServerSpan(ctx context.Context, traceName string) (context.Context, trace.Span) { + return ctx, noop.Span{} +} + +func (*noopTracer) EndSpan(span trace.Span) {} + +func (*noopTracer) RecordError(span trace.Span, err error) {} + +func (o *noopTracer) SetCacheReadAttributes(span trace.Span, isCacheHit bool, bytesRead int) {} + +func (o *noopTracer) SetUploadAttributes(span trace.Span, bytesUploaded int64, objectName string) {} + +func (*noopTracer) TraceUpload(ctx context.Context, name string, objName string, bytes *int64, err *error) (context.Context, func()) { + return ctx, emptyFinisher +} + +// Return the new context as it is as this is a no-op implementation +func (*noopTracer) PropagateTraceContext(newCtx context.Context, _ context.Context) context.Context { + return newCtx +} + +func NewNoopTracer() TraceHandle { + return new(noopTracer) +} diff --git a/tracing/otel_tracer.go b/tracing/otel_tracer.go new file mode 100644 index 0000000000..90b538e5c7 --- /dev/null +++ b/tracing/otel_tracer.go @@ -0,0 +1,110 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tracing + +import ( + "context" + "sync" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" +) + +type otelTracer struct { + tracer trace.Tracer + slicePool *sync.Pool +} + +var ( + bytesReadKey = attribute.Key(BYTES_READ) + bytesUploadedKey = attribute.Key(BYTES_UPLOADED) + objectNameKey = attribute.Key(OBJECT_NAME) + cacheHit = attribute.Bool(IS_CACHE_HIT, true) + cacheMiss = attribute.Bool(IS_CACHE_HIT, false) +) + +func (o *otelTracer) StartSpan(ctx context.Context, traceName string) (context.Context, trace.Span) { + return o.tracer.Start(ctx, traceName) +} + +func (o *otelTracer) StartServerSpan(ctx context.Context, traceName string) (context.Context, trace.Span) { + return o.tracer.Start(ctx, traceName, trace.WithSpanKind(trace.SpanKindServer)) +} + +func (o *otelTracer) EndSpan(span trace.Span) { + span.End() +} + +func (o *otelTracer) RecordError(span trace.Span, err error) { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) +} + +func (o *otelTracer) SetCacheReadAttributes(span trace.Span, isCacheHit bool, bytesRead int) { + attrSetPtr := o.slicePool.Get().(*[]attribute.KeyValue) + attrSet := *attrSetPtr + defer o.slicePool.Put(attrSetPtr) + attrSet[0] = bytesReadKey.Int(bytesRead) + if isCacheHit { + attrSet[1] = cacheHit + } else { + attrSet[1] = cacheMiss + } + span.SetAttributes(attrSet...) +} + +func (o *otelTracer) SetUploadAttributes(span trace.Span, bytesUploaded int64, objectName string) { + attrSetPtr := o.slicePool.Get().(*[]attribute.KeyValue) + attrSet := *attrSetPtr + defer o.slicePool.Put(attrSetPtr) + attrSet[0] = bytesUploadedKey.Int64(bytesUploaded) + attrSet[1] = objectNameKey.String(objectName) + span.SetAttributes(attrSet...) +} + +func (o *otelTracer) TraceUpload(ctx context.Context, name string, objName string, bytes *int64, err *error) (context.Context, func()) { + ctx, span := o.StartSpan(ctx, name) + + return ctx, func() { + if bytes != nil { + o.SetUploadAttributes(span, *bytes, objName) + } + if err != nil && *err != nil { + o.RecordError(span, *err) + } + o.EndSpan(span) + } +} + +func (o *otelTracer) PropagateTraceContext(newCtx context.Context, oldCtx context.Context) context.Context { + span := trace.SpanFromContext(oldCtx) + return trace.ContextWithSpan(newCtx, span) +} + +func NewOTELTracer() TraceHandle { + slicePool := sync.Pool{ + New: func() any { + s := make([]attribute.KeyValue, 2) + return &s + }, + } + + return &otelTracer{ + tracer: otel.Tracer(name), + slicePool: &slicePool, + } +} diff --git a/tracing/otel_tracer_test.go b/tracing/otel_tracer_test.go new file mode 100644 index 0000000000..0e1abad1ee --- /dev/null +++ b/tracing/otel_tracer_test.go @@ -0,0 +1,142 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tracing + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel/trace" +) + +func TestOtelTracer_StartEndSpan(t *testing.T) { + recorder := tracetest.NewSpanRecorder() + provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + otel.SetTracerProvider(provider) + tracer := NewOTELTracer() + spanName := "test-span" + + ctx, span := tracer.StartSpan(context.Background(), spanName) + tracer.EndSpan(span) + + assert.NotNil(t, ctx) + assert.NotNil(t, span) + spans := recorder.Ended() + assert.Len(t, spans, 1) + assert.Equal(t, spanName, spans[0].Name()) +} + +func TestOtelTracer_StartServerSpan(t *testing.T) { + recorder := tracetest.NewSpanRecorder() + provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + otel.SetTracerProvider(provider) + tracer := NewOTELTracer() + spanName := "test-server-span" + + ctx, span := tracer.StartServerSpan(context.Background(), spanName) + tracer.EndSpan(span) + + assert.NotNil(t, ctx) + assert.NotNil(t, span) + spans := recorder.Ended() + assert.Len(t, spans, 1) + assert.Equal(t, spanName, spans[0].Name()) + assert.Equal(t, trace.SpanKindServer, spans[0].SpanKind()) +} + +func TestOtelTracer_RecordError(t *testing.T) { + recorder := tracetest.NewSpanRecorder() + provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + otel.SetTracerProvider(provider) + tracer := NewOTELTracer() + spanName := "test-error-span" + errMsg := "test-error" + err := errors.New(errMsg) + + _, span := tracer.StartSpan(context.Background(), spanName) + tracer.RecordError(span, err) + tracer.EndSpan(span) + + spans := recorder.Ended() + assert.Len(t, spans, 1) + assert.Len(t, spans[0].Events(), 1) + assert.Equal(t, "exception", spans[0].Events()[0].Name) + assert.Contains(t, spans[0].Events()[0].Attributes, attribute.String("exception.message", errMsg)) + assert.Equal(t, codes.Error, spans[0].Status().Code) + assert.Equal(t, errMsg, spans[0].Status().Description) +} + +func TestOtelTracer_SetCacheReadAttributes(t *testing.T) { + recorder := tracetest.NewSpanRecorder() + provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + otel.SetTracerProvider(provider) + tracer := NewOTELTracer() + spanName := "test-cache-read-span" + bytesRead := 123 + + testCases := []struct { + name string + cacheHit bool + }{ + { + name: "cache_hit", + cacheHit: true, + }, + { + name: "cache_miss", + cacheHit: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + recorder.Reset() + + _, span := tracer.StartSpan(context.Background(), spanName) + tracer.SetCacheReadAttributes(span, tc.cacheHit, bytesRead) + tracer.EndSpan(span) + + spans := recorder.Ended() + assert.Len(t, spans, 1) + assert.Len(t, spans[0].Attributes(), 2) + assert.Contains(t, spans[0].Attributes(), attribute.Int(BYTES_READ, bytesRead)) + assert.Contains(t, spans[0].Attributes(), attribute.Bool(IS_CACHE_HIT, tc.cacheHit)) + }) + } +} + +func TestOtelTracer_PropagateTraceContext(t *testing.T) { + recorder := tracetest.NewSpanRecorder() + provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + otel.SetTracerProvider(provider) + tracer := NewOTELTracer() + spanName := "test-propagation-span" + oldCtx, oldSpan := tracer.StartSpan(context.Background(), spanName) + + newCtx := tracer.PropagateTraceContext(context.Background(), oldCtx) + tracer.EndSpan(oldSpan) + + newSpan := trace.SpanFromContext(newCtx) + assert.Equal(t, oldSpan, newSpan) + spans := recorder.Ended() + assert.Len(t, spans, 1) +} diff --git a/tracing/span_attributes.go b/tracing/span_attributes.go new file mode 100644 index 0000000000..380c5c5ffd --- /dev/null +++ b/tracing/span_attributes.go @@ -0,0 +1,22 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tracing + +const ( + IS_CACHE_HIT = "cache.hit" // Indicates if the response was served from cache or not. + BYTES_READ = "read.size" // Indicates the number of bytes read from the given reader + BYTES_UPLOADED = "write.chunk.size" // Indicates the number of bytes uploaded + OBJECT_NAME = "write.object_name" // Indicates the object name uploaded +) diff --git a/tracing/span_names.go b/tracing/span_names.go new file mode 100644 index 0000000000..af6a3de69c --- /dev/null +++ b/tracing/span_names.go @@ -0,0 +1,136 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tracing contains the constants and utilities for OpenTelemetry +// instrumentation within gcsfuse. +package tracing + +// Span name constants for GCSFuse operations. +// These constants define the canonical names for spans created during FUSE +// operations. Using these constants ensures consistency and better readability and a single source of truth where all the span names are listed. + +const ( + // --- Cache and Prefetching Operations --- + + // FileCacheRead tracks read operations specifically from the local file cache. + FileCacheRead = "file.cache.read" + // FileCacheWrite tracks write or population operations into the local file cache. + FileCacheWrite = "file.cache.write" + // ReadPrefetchBlockPoolGen monitors the generation/lifecycle of the prefetch block pool. + ReadPrefetchBlockPoolGen = "prefetch.block_pool_gen.read" + // DownloadPrefetchBlock triggers a network request to pre-fill a specific data block. + DownloadPrefetchBlock = "prefetch.block.download" + // WaitForPrefetchBlock measures the time a process waits for a prefetch operation to complete. + WaitForPrefetchBlock = "prefetch.block.wait" + // ReadFromPrefetchBlock executes a read directly from a successfully prefetched data block. + ReadFromPrefetchBlock = "prefetch.block.read" + // ScheduleBlockForDownload adds a specific file block to the asynchronous download queue. + ScheduleBlockForDownload = "download.block.schedule" + + // --- Metadata and Inode Operations --- + + // StatFS retrieves aggregate filesystem statistics (e.g., total capacity, free space). + StatFS = "fs.stat_fs" + // LookUpInode resolves a specific filename within a directory to its unique inode. + LookUpInode = "fs.inode.lookup" + // GetInodeAttributes retrieves metadata such as size, mode, and timestamps for an inode. + GetInodeAttributes = "fs.inode.get_attributes" + // SetInodeAttributes updates metadata (e.g., permissions or ownership) for an inode. + SetInodeAttributes = "fs.inode.set_attributes" + // ForgetInode informs the kernel that it no longer needs to track a specific inode. + ForgetInode = "fs.inode.forget" + // BatchForget allows the kernel to release multiple inode references in a single call. + BatchForget = "fs.batch_forget" + + // --- Directory Lifecycle --- + + // MkDir creates a new directory entry. + MkDir = "fs.dir.mk" + // RmDir removes an existing, empty directory. + RmDir = "fs.dir.rm" + // OpenDir opens a directory for content enumeration. + OpenDir = "fs.dir.open" + // ReadDir reads entries from an open directory handle. + ReadDir = "fs.dir.read" + // ReadDirPlus reads directory entries along with their associated metadata/attributes. + ReadDirPlus = "fs.dir.read_plus" + // ReleaseDirHandle closes the handle for a directory and releases associated resources. + ReleaseDirHandle = "fs.dir.release_handle" + + // --- File Lifecycle and I/O --- + + // CreateFile creates and opens a new file within the filesystem. + CreateFile = "fs.file.create" + // OpenFile opens an existing file for reading or writing. + OpenFile = "fs.file.open" + // ReadFile executes a standard read operation from a file handle. + ReadFile = "fs.file.read" + // WriteFile executes a standard write operation to a file handle. + WriteFile = "fs.file.write" + // SyncFile flushes buffered data for a specific file to stable storage. + SyncFile = "fs.file.sync" + // FlushFile is called on every close of a file descriptor to flush changes. + FlushFile = "fs.file.flush" + // ReleaseFileHandle closes the file handle and releases system resources. + ReleaseFileHandle = "fs.file.release_handle" + + // --- Links and Extended Attributes --- + + // CreateLink creates a hard link to an existing inode. + CreateLink = "fs.link.create" + // CreateSymlink creates a symbolic (soft) link. + CreateSymlink = "fs.symlink.create" + // ReadSymlink reads the target path stored within a symbolic link. + ReadSymlink = "fs.symlink.read" + // Rename changes the name or location of a file or directory. + Rename = "fs.rename" + // Unlink removes a name from the filesystem; if it was the last link, the file is deleted. + Unlink = "fs.unlink" + // GetXattr retrieves the value of an extended attribute. + GetXattr = "fs.xattr.get" + // SetXattr sets or updates an extended attribute. + SetXattr = "fs.xattr.set" + // ListXattr lists the names of extended attributes assigned to a file. + ListXattr = "fs.xattr.list" + // RemoveXattr deletes an extended attribute from a file. + RemoveXattr = "fs.xattr.remove" + + // --- Advanced Filesystem Operations --- + + // MkNode creates a filesystem node (file, device special file, or named pipe). + MkNode = "fs.mknode" + // Fallocate ensures that disk space is pre-allocated for a file. + Fallocate = "fs.fallocate" + // SyncFS flushes all buffered data for the entire filesystem to disk. + SyncFS = "fs.sync_fs" + + // --- Write Flow traces --- + + // WriteFileStaged traces a write operation using the legacy staged writes. + WriteFileStaged = "write.staged" + // SyncFileStaged traces the synchronization of the staged temp file to GCS. + SyncFileStaged = "write.staged.sync" + // WriteFileStreaming traces a write operation using the buffered writes handler. + WriteFileStreaming = "write.streaming" + // SyncFileStreaming traces the synchronization of buffered writes to GCS. + SyncFileStreaming = "write.streaming.sync" + // StreamingUploadBlock traces the upload of a single streaming block to GCS. + StreamingUploadBlock = "write.streaming.upload.block" + // StreamingUploadFinalize traces the finalization of the streaming upload. + StreamingUploadFinalize = "write.streaming.upload.finalize" + // StreamingUploadFlush traces the flushing of pending streaming writes. + StreamingUploadFlush = "write.streaming.upload.flush" + // Tracks the complete go routine that trigger the async upload of the write blocks received + StreamingUploader = "write.streaming.uploader" +) diff --git a/tracing/trace_handle.go b/tracing/trace_handle.go new file mode 100644 index 0000000000..a47114ccde --- /dev/null +++ b/tracing/trace_handle.go @@ -0,0 +1,54 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tracing + +import ( + "context" + + "go.opentelemetry.io/otel/trace" +) + +const name = "cloud.google.com/gcsfuse" + +// TraceHandle provides an interface for recording traces, trace links and everything related to tracing. This allows easier switching between various trace-implementations, especially with a custom no-op tracer. +type TraceHandle interface { + // Start a span with a given name & context + StartSpan(ctx context.Context, traceName string) (context.Context, trace.Span) + + // Start a span of span kind server given name & context + StartServerSpan(ctx context.Context, traceName string) (context.Context, trace.Span) + + // End a span + EndSpan(span trace.Span) + + // Record an error on the span for export in case of failure + RecordError(span trace.Span, err error) + + // A handle interface method to set attributes for file cache read + // attribute creation and generic interface using variadic operator is a costly affair both from memory allocation and CPU time perspectives - (3.883 ns/op 0 B/op 0 allocs/op) vs (90.21 ns/op 128 B/op 1 allocs/op) + // This method is specifically created so that the caller doesn't have to create the attributes themselves. + // Instead the implementation of the TraceHandle that's chosen decides whether to create the attributes. + // This allows skipping the attribute creation entirely in case of noop tracer which is selected when tracing is disabled. + SetCacheReadAttributes(span trace.Span, isCacheHit bool, bytesRead int) + + // A handle interface method to set attributes for upload + SetUploadAttributes(span trace.Span, bytesUploaded int64, objectName string) + + // TraceUpload starts a span and returns a finisher function that can set upload attributes, record error and end span + TraceUpload(ctx context.Context, name string, objName string, bytes *int64, err *error) (context.Context, func()) + + // A handle interface method to retain relevant span data in new context from the older context + PropagateTraceContext(newCtx context.Context, oldCtx context.Context) context.Context +}