diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index d40d5c6..aba1385 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -1,31 +1,31 @@ { "plugins": [ { - "version": "0.2.4", + "version": "0.3.0", "name": "elastic-elasticsearch", "source": "./plugins/elasticsearch", "description": "Elasticsearch skills — ES|QL queries, data ingestion, security (authn, authz, audit), and troubleshooting" }, { - "version": "0.2.4", + "version": "0.3.0", "name": "elastic-kibana", "source": "./plugins/kibana", "description": "Kibana skills — dashboards, alerting rules, connectors, Vega visualizations, Agent Builder, streams, and audit logging" }, { - "version": "0.2.4", + "version": "0.3.0", "name": "elastic-observability", "source": "./plugins/observability", "description": "Elastic Observability skills — OpenTelemetry instrumentation and migration (.NET, Java, Python), LLM observability, log search, SLOs, and service health" }, { - "version": "0.2.4", + "version": "0.3.0", "name": "elastic-security", "source": "./plugins/security", "description": "Elastic Security skills — alert triage, case management, detection rule management, and sample data generation" }, { - "version": "0.2.4", + "version": "0.3.0", "name": "elastic-cloud", "source": "./plugins/cloud", "description": "Elastic Cloud skills — project setup, access management, network security, and Serverless project lifecycle" diff --git a/.github/.release-manifest.json b/.github/.release-manifest.json index 26b257d..2e73de4 100644 --- a/.github/.release-manifest.json +++ b/.github/.release-manifest.json @@ -1,7 +1,7 @@ { - "version": "0.2.4", + "version": "0.3.0", "source_repo": "elastic/agent-skills", - "sandbox_sha": "b1798448ff8f0fd7ea371606a0e0b95d33a1d70a", + "sandbox_sha": "092aaa6a6c68a4f0553145c90c86ea157721ae2e", "skills": { "skills/cloud/access-management": { "name": "cloud-access-management", @@ -37,7 +37,7 @@ }, "skills/elasticsearch/elasticsearch-esql": { "name": "elasticsearch-esql", - "version": "0.1.1" + "version": "0.3.0" }, "skills/elasticsearch/elasticsearch-file-ingest": { "name": "elasticsearch-file-ingest", @@ -59,6 +59,10 @@ "name": "kibana-alerting-rules", "version": "0.1.0" }, + "skills/kibana/kibana-anomaly-detection": { + "name": "kibana-anomaly-detection", + "version": "0.2.0" + }, "skills/kibana/kibana-audit": { "name": "kibana-audit", "version": "0.1.0" @@ -69,7 +73,7 @@ }, "skills/kibana/kibana-dashboards": { "name": "kibana-dashboards", - "version": "0.1.1" + "version": "0.1.2" }, "skills/kibana/kibana-vega": { "name": "kibana-vega", @@ -103,6 +107,10 @@ "name": "observability-edot-python-migrate", "version": "0.1.0" }, + "skills/observability/k8s-investigation": { + "name": "observability-k8s-investigation", + "version": "0.2.0" + }, "skills/observability/llm-obs": { "name": "observability-llm-obs", "version": "0.1.0" diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index 06a9bc0..d78d94b 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -11,31 +11,31 @@ "name": "elastic-elasticsearch", "source": "./plugins/elasticsearch", "description": "Elasticsearch skills — ES|QL queries, data ingestion, security (authn, authz, audit), and troubleshooting", - "version": "0.2.4" + "version": "0.3.0" }, { "name": "elastic-kibana", "source": "./plugins/kibana", "description": "Kibana skills — dashboards, alerting rules, connectors, Vega visualizations, Agent Builder, streams, and audit logging", - "version": "0.2.4" + "version": "0.3.0" }, { "name": "elastic-observability", "source": "./plugins/observability", "description": "Elastic Observability skills — OpenTelemetry instrumentation and migration (.NET, Java, Python), LLM observability, log search, SLOs, and service health", - "version": "0.2.4" + "version": "0.3.0" }, { "name": "elastic-security", "source": "./plugins/security", "description": "Elastic Security skills — alert triage, case management, detection rule management, and sample data generation", - "version": "0.2.4" + "version": "0.3.0" }, { "name": "elastic-cloud", "source": "./plugins/cloud", "description": "Elastic Cloud skills — project setup, access management, network security, and Serverless project lifecycle", - "version": "0.2.4" + "version": "0.3.0" } ] } diff --git a/CHANGELOG.md b/CHANGELOG.md index 05f1872..4f4ea2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## v0.3.0 + +### New Skills + +- `skills/kibana/kibana-anomaly-detection` (v0.2.0) +- `skills/observability/k8s-investigation` (v0.2.0) + +### Updated Skills + +- `skills/elasticsearch/elasticsearch-esql` (v0.1.1 → v0.3.0) +- `skills/elasticsearch/elasticsearch-onboarding` (v0.1.0) +- `skills/kibana/kibana-dashboards` (v0.1.1 → v0.1.2) + +### Generated Artifacts + +- Regenerated README skill table + ## v0.2.4 ### Updated Skills diff --git a/README.md b/README.md index 5197a1c..0bd3b94 100644 --- a/README.md +++ b/README.md @@ -59,30 +59,31 @@ Skills in this repository focus on: | [elasticsearch-audit](skills/elasticsearch/elasticsearch-audit/SKILL.md) | Enable, configure, and query Elasticsearch security audit logs. Use when the task involves audit logging setup, event filtering, or investigating security incidents like failed logins. | 0.1.0 | elastic | | [elasticsearch-authn](skills/elasticsearch/elasticsearch-authn/SKILL.md) | Authenticate to Elasticsearch using native, file-based, LDAP/AD, SAML, OIDC, Kerberos, JWT, or certificate realms. Use when connecting with credentials, choosing a realm, or managing API keys. Assumes the target realms are already configured. | 0.1.0 | elastic | | [elasticsearch-authz](skills/elasticsearch/elasticsearch-authz/SKILL.md) | Manage Elasticsearch RBAC: native users, roles, role mappings, document- and field-level security. Use when creating users or roles, assigning privileges, or mapping external realms like LDAP/SAML. | 0.1.1 | elastic | -| [elasticsearch-esql](skills/elasticsearch/elasticsearch-esql/SKILL.md) | Execute ES\|QL (Elasticsearch Query Language) queries, use when the user wants to query Elasticsearch data, analyze logs, aggregate metrics, explore data, or create charts and dashboards from ES\|QL results. | 0.1.1 | elastic | +| [elasticsearch-esql](skills/elasticsearch/elasticsearch-esql/SKILL.md) | Execute ES\|QL (Elasticsearch Query Language) queries, use when the user wants to query Elasticsearch data, analyze logs, aggregate metrics, explore data, or create charts and dashboards from ES\|QL results. | 0.3.0 | elastic | | [elasticsearch-file-ingest](skills/elasticsearch/elasticsearch-file-ingest/SKILL.md) | Ingest and transform data files (CSV/JSON/Parquet/Arrow IPC) into Elasticsearch with stream processing and custom transforms. Use when loading files or batch importing data — not for reindexing, general ingest pipeline design, or bulk API patterns. | 0.2.0 | elastic | -| [elasticsearch-onboarding](skills/elasticsearch/elasticsearch-onboarding/SKILL.md) | Help developers new to Elasticsearch get from zero to a working search experience. Guide them through understanding their intent, mapping their data, and building a search experience with best practices baked in. Use this when developers are new to Elasticsearch and need help getting started with their search use case. | 0.1.0 | elastic | +| [elasticsearch-onboarding](skills/elasticsearch/elasticsearch-onboarding/SKILL.md) | Help developers new to Elasticsearch get from zero to a working search experience. Guide them through understanding their intent, mapping their data, and building a search experience with best practices baked in. Use this when the user shows intent to build search-related functionality, asks about Elasticsearch-related concepts for their use case, or expresses the need for help getting started with Elasticsearch. | 0.1.0 | elastic | | [elasticsearch-security-troubleshooting](skills/elasticsearch/elasticsearch-security-troubleshooting/SKILL.md) | Diagnose and resolve Elasticsearch security errors: 401/403 failures, TLS problems, expired API keys, role mapping mismatches, and Kibana login issues. Use when the user reports a security error. | 0.1.0 | elastic |
-Kibana (7) +Kibana (8) | Skill | Description | Version | Author | | ----- | ----------- | ------- | ------ | | [kibana-agent-builder](skills/kibana/agent-builder/SKILL.md) | Create and manage Agent Builder agents and custom tools in Kibana. Use when asked to create, update, delete, test, or inspect agents or tools in Agent Builder. | 0.2.0 | elastic | | [kibana-alerting-rules](skills/kibana/kibana-alerting-rules/SKILL.md) | Create and manage Kibana alerting rules via REST API or Terraform. Use when creating, updating, or managing rule lifecycle (enable, disable, mute, snooze) or rules-as-code workflows. | 0.1.0 | elastic | +| [kibana-anomaly-detection](skills/kibana/kibana-anomaly-detection/SKILL.md) | Elastic ML anomaly detection skill — investigation/RCA, score explanation, job operations (create, datafeed, start/stop, results), and troubleshooting (missing docs, memory limits, datafeed health, lifecycle). Operates against Kibana Agent Builder MCP tools (`ad_*`) on `.ml-anomalies-*`, `.ml-config`, `.ml-notifications-*`, `.ml-annotations-*`. Use when answering "what broke?"/"which entity?"/RCA, "why is score high/low?"/renormalization, "datafeed stopped"/"memory limit", or any request to set up or configure an ML anomaly detection job. | 0.2.0 | elastic | | [kibana-audit](skills/kibana/kibana-audit/SKILL.md) | Enable and configure Kibana audit logging for saved object access, logins, and space operations. Use when setting up Kibana audit, filtering events, or correlating Kibana and ES audit logs. | 0.1.0 | elastic | | [kibana-connectors](skills/kibana/kibana-connectors/SKILL.md) | Create and manage Kibana connectors for Slack, PagerDuty, Jira, webhooks, and more via REST API or Terraform. Use when configuring third-party integrations or managing connectors as code. | 0.1.1 | elastic | -| [kibana-dashboards](skills/kibana/kibana-dashboards/SKILL.md) | Create and manage Kibana Dashboards and visualizations. Use when you need to define dashboards and visualizations declaratively, version control them, or automate their deployment. | 0.1.1 | elastic | +| [kibana-dashboards](skills/kibana/kibana-dashboards/SKILL.md) | Create and manage Kibana Dashboards and visualizations. Use when you need to define dashboards and visualizations declaratively, version control them, or automate their deployment. | 0.1.2 | elastic | | [kibana-vega](skills/kibana/kibana-vega/SKILL.md) | Create Vega and Vega-Lite visualizations with ES\|QL data sources in Kibana. Use when building custom charts, dashboards, or programmatic panel layouts beyond standard Lens charts. | 0.1.0 | elastic | | [kibana-streams](skills/kibana/streams/SKILL.md) | List, inspect, enable, disable, and resync Kibana Streams via the REST API. Use when the user needs stream details, ingest/query settings, queries, significant events, or attachments. | 0.1.0 | elastic |
-Observability (10) +Observability (11) | Skill | Description | Version | Author | | ----- | ----------- | ------- | ------ | @@ -92,6 +93,7 @@ Skills in this repository focus on: | [observability-edot-java-migrate](skills/observability/edot-java-migrate/SKILL.md) | Migrate a Java application from the classic Elastic APM Java agent to the EDOT Java agent. Use when switching from elastic-apm-agent.jar to elastic-otel-javaagent.jar. | 0.1.1 | elastic | | [observability-edot-python-instrument](skills/observability/edot-python-instrument/SKILL.md) | Instrument a Python application with the Elastic Distribution of OpenTelemetry (EDOT) Python agent for automatic tracing, metrics, and logs. Use when adding observability to a Python service that has no existing APM agent. | 0.1.0 | elastic | | [observability-edot-python-migrate](skills/observability/edot-python-migrate/SKILL.md) | Migrate a Python application from the classic Elastic APM Python agent to the EDOT Python agent. Use when switching from elastic-apm to elastic-opentelemetry. | 0.1.0 | elastic | +| [observability-k8s-investigation](skills/observability/k8s-investigation/SKILL.md) | Investigate Kubernetes workload, node, and control-plane issues using OTel telemetry (EDOT). Use when diagnosing pod failures (CrashLoopBackOff, OOMKilled, Error), node pressure, resource exhaustion, image pull failures, admission rejections, autoscaling anomalies, or correlating K8s state with application signals. OTel ingest path only — the legacy ECS Kubernetes integration shape is out of scope. | 0.2.0 | elastic | | [observability-llm-obs](skills/observability/llm-obs/SKILL.md) | Monitor LLMs and agentic apps: performance, token/cost, response quality, and workflow orchestration. Use when the user asks about LLM monitoring, GenAI observability, or AI cost/quality. | 0.1.0 | elastic | | [observability-logs-search](skills/observability/logs-search/SKILL.md) | Search and filter Observability logs using ES\|QL. Use when investigating log spikes, errors, or anomalies; getting volume and trends; or drilling into services or containers during incidents. | 0.2.0 | elastic | | [observability-manage-slos](skills/observability/manage-slos/SKILL.md) | Create and manage SLOs in Elastic Observability using the Kibana API. Use when defining SLIs, setting error budgets, or managing SLO lifecycle. | 0.2.0 | elastic | diff --git a/package.json b/package.json index a80c778..9296b3c 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "version": "0.2.4", + "version": "0.3.0", "name": "elastic-agent-skills", "description": "Official Elastic agent skills for Elasticsearch, Kibana, Observability, Security, and Cloud", "license": "Apache-2.0", diff --git a/plugins/cloud/.claude-plugin/plugin.json b/plugins/cloud/.claude-plugin/plugin.json index 94d0a40..0d7cb10 100644 --- a/plugins/cloud/.claude-plugin/plugin.json +++ b/plugins/cloud/.claude-plugin/plugin.json @@ -1,5 +1,5 @@ { - "version": "0.2.4", + "version": "0.3.0", "name": "elastic-cloud", "description": "Elastic Cloud skills - project setup, access management, network security, and Serverless project lifecycle", "author": { diff --git a/plugins/cloud/plugin.json b/plugins/cloud/plugin.json index 4fd4d78..4fe79ba 100644 --- a/plugins/cloud/plugin.json +++ b/plugins/cloud/plugin.json @@ -1,7 +1,7 @@ { "name": "elastic-cloud", "description": "Elastic Cloud skills - project setup, access management, network security, and Serverless project lifecycle", - "version": "0.2.4", + "version": "0.3.0", "author": { "name": "Elastic", "url": "https://www.elastic.co" @@ -9,10 +9,5 @@ "repository": "https://github.com/elastic/agent-skills", "homepage": "https://github.com/elastic/agent-skills/tree/main/plugins/cloud", "license": "Apache-2.0", - "keywords": [ - "cloud", - "serverless", - "elastic-cloud", - "elastic" - ] + "keywords": ["cloud", "serverless", "elastic-cloud", "elastic"] } diff --git a/plugins/elasticsearch/.claude-plugin/plugin.json b/plugins/elasticsearch/.claude-plugin/plugin.json index 57313fa..9d65e4c 100644 --- a/plugins/elasticsearch/.claude-plugin/plugin.json +++ b/plugins/elasticsearch/.claude-plugin/plugin.json @@ -1,5 +1,5 @@ { - "version": "0.2.4", + "version": "0.3.0", "name": "elastic-elasticsearch", "description": "Elasticsearch skills - ES|QL queries, data ingestion, security (authn, authz, audit), and troubleshooting", "author": { diff --git a/plugins/elasticsearch/plugin.json b/plugins/elasticsearch/plugin.json index 09fb57b..51af6bd 100644 --- a/plugins/elasticsearch/plugin.json +++ b/plugins/elasticsearch/plugin.json @@ -1,7 +1,7 @@ { "name": "elastic-elasticsearch", "description": "Elasticsearch skills for GitHub Copilot — translate natural language to ES|QL, ingest data, manage security (authn, authz, audit), and troubleshoot clusters.", - "version": "0.2.4", + "version": "0.3.0", "author": { "name": "Elastic", "url": "https://www.elastic.co" diff --git a/plugins/elasticsearch/skills/elasticsearch-esql/SKILL.md b/plugins/elasticsearch/skills/elasticsearch-esql/SKILL.md index 175d901..4d03d81 100644 --- a/plugins/elasticsearch/skills/elasticsearch-esql/SKILL.md +++ b/plugins/elasticsearch/skills/elasticsearch-esql/SKILL.md @@ -6,7 +6,7 @@ description: > charts and dashboards from ES|QL results. metadata: author: elastic - version: 0.1.1 + version: 0.3.0 --- # Elasticsearch ES|QL @@ -120,11 +120,24 @@ node scripts/esql.js test curl -s "$ELASTICSEARCH_URL//_settings/index.mode" -H "Authorization: ApiKey $ELASTICSEARCH_API_KEY" ``` + For TSDS indices on 9.4+, prefer the in-language discovery commands `METRICS_INFO` and `TS_INFO` (both GA) over + inspecting mappings — they enumerate the metric catalogue and the dimension labels of each time series directly. Both + must follow `TS` and must precede `STATS`/`SORT`/`LIMIT`. See + [Time Series Queries](references/time-series-queries.md#metric-and-time-series-discovery). + + ```bash + node scripts/esql.js raw "TS metrics-tsds | METRICS_INFO | SORT metric_name" --tsv + node scripts/esql.js raw "TS metrics-tsds | TS_INFO | KEEP metric_name, dimensions | SORT metric_name" --tsv + ``` + 3. **Choose the right ES|QL feature for the task**: Before writing queries, match the user's intent to the most appropriate ES|QL feature. Prefer a single advanced query over multiple basic ones. - "find patterns," "categorize," "group similar messages" → `CATEGORIZE(field)` - "spike," "dip," "anomaly," "when did X change" → `CHANGE_POINT value ON key` - "trend over time," "time series" → `STATS ... BY BUCKET(@timestamp, interval)` or `TS` for TSDB + - "PromQL", "Prometheus query/dashboard/alert", `sum by (instance) (...)`, label matchers like `{cluster="prod"}` → + `PROMQL` source command (9.4+ preview); see [PROMQL Command](references/promql-command.md). Prefer `TS` for native + ES|QL phrasing. - "search," "find documents matching" → `MATCH` (default), `QSTR` (advanced boolean), `KQL` (Kibana migration). For content/document relevance search, follow the [ES|QL Search Strategy](references/esql-search-strategy.md) - "count," "average," "breakdown" → `STATS` with aggregation functions @@ -134,6 +147,8 @@ node scripts/esql.js test CIDR_MATCH), common templates, and ambiguity handling - [Time Series Queries](references/time-series-queries.md) - **read before any TS query**: inner/outer aggregation model, TBUCKET syntax, RATE constraints + - [PROMQL Command](references/promql-command.md) — **read before any PROMQL query**: options, output schema, + limitations, and `PROMQL` vs `TS` decision matrix (9.4+ preview) - [ES|QL Complete Reference](references/esql-reference.md) - full syntax for all commands and functions - [ES|QL Search Strategy](references/esql-search-strategy.md) — for content/document relevance search (retrieve → fuse → rerank) @@ -281,6 +296,25 @@ TS metrics-tsds | SORT bucket ``` +**Time series with PromQL syntax (9.4+ preview):** Use the `PROMQL` source command when the user explicitly asks for +PromQL, references Prometheus syntax (`sum by (instance) (...)`, label matchers like `{cluster="prod"}`), or is +migrating a Prometheus dashboard or alert. The `PROMQL` command accepts standard PromQL with optional `index`, `step`, +`buckets`, `start`, `end`, and `scrape_interval` options, and produces a table that the rest of the ES|QL pipeline can +process. Range selectors are optional — when omitted, the window is `max(step, scrape_interval)`. Otherwise prefer `TS` +(GA in 9.4). `PROMQL` does **not** support group modifiers, set operators (`or`/`and`/`unless`), or functions like +`histogram_quantile`, `predict_linear`, and `label_join` — fall back to `TS` for those. See +[PROMQL Command](references/promql-command.md) for the full reference. + +```esql +// Adaptive Kibana query — date picker drives time range and step +PROMQL index=metrics-* sum by (instance) (rate(http_requests_total)) + +// Named result, post-processed with ES|QL +PROMQL index=k8s step=1h bytes=(max by (cluster) (network.bytes_in)) +| STATS max_bytes = MAX(bytes) BY cluster +| SORT cluster +``` + **Data enrichment with LOOKUP JOIN:** The basic `ON` clause matches fields by name in both indices (`LOOKUP JOIN idx ON field_name`). When the join key has a different name in the source, use `RENAME` first to align names. 9.2+ tech preview also supports expression predicates (`ON expr == expr`); see @@ -351,6 +385,7 @@ For complete ES|QL syntax including all commands, functions, and operators, read - [Query Patterns](references/query-patterns.md) - Natural language to ES|QL translation - [Generation Tips](references/generation-tips.md) - Best practices for query generation - [Time Series Queries](references/time-series-queries.md) - TS command, time series aggregation functions, TBUCKET +- [PROMQL Command](references/promql-command.md) - PromQL source command for TSDS indices (9.4+ preview) - [DSL to ES|QL Migration](references/dsl-to-esql-migration.md) - Convert Query DSL to ES|QL - [Environment Setup](references/environment-setup.md) - Connection configuration options diff --git a/plugins/elasticsearch/skills/elasticsearch-esql/references/dsl-to-esql-migration.md b/plugins/elasticsearch/skills/elasticsearch-esql/references/dsl-to-esql-migration.md index e9ae400..e138a91 100644 --- a/plugins/elasticsearch/skills/elasticsearch-esql/references/dsl-to-esql-migration.md +++ b/plugins/elasticsearch/skills/elasticsearch-esql/references/dsl-to-esql-migration.md @@ -958,28 +958,29 @@ FROM sales Features not available in ES|QL as of version 9.3: -| Feature | Query DSL | ES\|QL | -| ---------------------------- | --------- | ----------------------------------- | -| Highlighting | ✅ | ❌ | -| Nested queries | ✅ | ❌ | -| Parent-child queries | ✅ | ❌ | -| Scroll/pagination beyond 10k | ✅ | ❌ | -| Percolate queries | ✅ | ❌ | -| Complex boosting | ✅ | Limited | -| Geo distance sorting | ✅ | ❌ | -| Runtime fields | ✅ | Use EVAL | -| Suggest API | ✅ | ❌ | -| Collapse (field collapsing) | ✅ | ❌ | -| Inner hits | ✅ | ❌ | -| Timezone in date functions | ✅ | ❌ (UTC only) | -| JOIN (non-lookup) | N/A | ❌ (only LEFT JOIN on lookup index) | +| Feature | Query DSL | ES\|QL | +| ---------------------------- | --------- | ----------------------------------------- | +| Highlighting | ✅ | ❌ | +| Nested queries | ✅ | ❌ | +| Parent-child queries | ✅ | ❌ | +| Scroll/pagination beyond 10k | ✅ | ❌ | +| Percolate queries | ✅ | ❌ | +| Complex boosting | ✅ | Limited | +| Geo distance sorting | ✅ | ❌ | +| Runtime fields | ✅ | Use EVAL | +| Suggest API | ✅ | ❌ | +| Collapse (field collapsing) | ✅ | ❌ | +| Inner hits | ✅ | ❌ | +| Timezone support | ✅ | ✅ `SET time_zone` (Serverless GA) | +| JOIN (non-lookup) | N/A | ❌ (only LEFT JOIN on lookup index) | +| Subqueries / UNION ALL | N/A | ✅ `FROM` subqueries (Serverless preview) | ### Unsupported Field Types in ES|QL - `nested` - `binary` - `completion` -- `flattened` +- `flattened` (use `METADATA _source` + `JSON_EXTRACT` to access sub-keys) - Range types (`date_range`, `integer_range`, etc.) - `rank_feature`, `rank_features` - `search_as_you_type` diff --git a/plugins/elasticsearch/skills/elasticsearch-esql/references/esql-reference.md b/plugins/elasticsearch/skills/elasticsearch-esql/references/esql-reference.md index 662323a..3597fac 100644 --- a/plugins/elasticsearch/skills/elasticsearch-esql/references/esql-reference.md +++ b/plugins/elasticsearch/skills/elasticsearch-esql/references/esql-reference.md @@ -52,23 +52,26 @@ Query directives modify the behavior of an ES|QL query. They appear before the s ### SET (9.3+, tech preview) -Controls query-level settings. +Controls query-level settings. Every `SET` directive must end with a semicolon before the source command. **Syntax:** ```esql -SET setting = value; [SET settingN = valueN;] +SET setting = "value"; [SET setting = "value";] source-command | processing-commands ``` **`unmapped_fields`** (9.3+ preview) -- controls how unmapped fields are treated: -- `FAIL` (default) -- the query fails if it references unmapped fields -- `NULLIFY` -- treats unmapped fields as null values +- `"default"` / `"fail"` -- the query fails if it references unmapped fields +- `"nullify"` -- treats unmapped fields as null values +- `"load"` -- loads unmapped fields dynamically. **Limitation:** `"load"` is incompatible with subqueries and views. Use + `"nullify"` when composing subqueries or querying views. **`time_zone`** (Serverless GA; self-managed planned) -- sets the default timezone for the query, overriding UTC -default. +default. Accepts any IANA timezone string or UTC offset. Applies to all date/time operations: `DATE_TRUNC`, +`DATE_FORMAT`, `NOW()`, etc. **Examples:** @@ -79,6 +82,12 @@ FROM employees | SORT emp_no | LIMIT 1 +SET time_zone = "America/Los_Angeles"; +FROM error_triage +| EVAL hour = DATE_TRUNC(1 hour, @timestamp) +| STATS errors = COUNT(*) BY hour, service +| SORT hour DESC + SET time_zone = "+05:00"; TS k8s | WHERE @timestamp == "2024-05-10T00:04:49.000Z" @@ -86,7 +95,11 @@ TS k8s ``` > **When to use:** `unmapped_fields` is useful when querying across multiple indices where some indices may not have all -> fields mapped. `time_zone` shifts date functions and display to a non-UTC zone. +> fields mapped. `time_zone` shifts date functions and display to a non-UTC zone. There is no per-function timezone +> argument — `DATE_TRUNC(1 hour, @timestamp, "America/Los_Angeles")` does **not** work. +> +> **Restriction:** `SET` directives cannot be used inside view definitions. The caller must apply `SET` when querying +> the view. --- @@ -123,6 +136,33 @@ FROM FROM cluster_one:logs-*, cluster_two:logs-* ``` +**Subqueries (Serverless tech preview):** `FROM` supports parenthesized subqueries with UNION ALL semantics. Each branch +is a complete ES|QL pipeline. Columns present in one branch but not another are filled with `null`. + +```esql +// Combine logs from different indices with independent pipelines +FROM + (FROM web_logs + | WHERE status_code >= 500 + | KEEP @timestamp, message, service.name), + (FROM app_logs + | WHERE level == "error" + | KEEP @timestamp, message, service.name) +| STATS errors = COUNT(*) BY service.name + +// Mix bare index patterns and subqueries +FROM raw_index, (FROM other_index | WHERE active == true | KEEP id, name) +``` + +**Subquery constraints:** + +- Non-correlated only — branches cannot reference columns from the outer query +- Columns with the same name must have compatible types across branches +- `FORK` cannot be used inside or after subqueries +- `SET unmapped_fields="load"` is incompatible with subqueries + +**Subqueries vs FORK:** Different data sources → subqueries. Same data, different analyses → FORK. + **Note:** Without explicit `LIMIT`, queries default to 1000 rows (or whatever the cluster setting esql.query.result_truncation_default_size is set to). @@ -147,7 +187,7 @@ ROW greeting = "hello", pi = 3.14159 ### TS Retrieves data from time series data streams (TSDS). Similar to `FROM` but enables time series aggregation functions in -`STATS` and targets only time series indices. Available since 9.2. +`STATS` and targets only time series indices. **Preview from 9.2 to 9.3, GA since 9.4**; GA on Elastic Cloud Serverless. **Syntax:** @@ -161,6 +201,11 @@ TS index_pattern [METADATA fields] - Time series functions are evaluated per time series first, then aggregated by group using an outer function - If no inner time series function is specified, `LAST_OVER_TIME()` is assumed implicitly - Cannot be combined with `FORK` before `STATS` is applied +- When the query has no `STATS`, `TS` returns rows sorted by `@timestamp` descending by default +- When the first `STATS` after `TS` uses a **bare** time series function (not wrapped in an outer aggregation like + `AVG()` / `SUM()`), results are implicitly grouped by every dimension and include a `_timeseries` JSON column. Use + `BY WITHOUT(dim, ...)` (GA in 9.4) to narrow this grouping. Bare dimension columns in `BY` are rejected; only grouping + functions (`TBUCKET`, `WITHOUT`) are allowed alongside a bare time series function. **Examples:** @@ -177,6 +222,11 @@ TS metrics // Average of per-time-series averages (explicit inner function) TS metrics | STATS AVG(AVG_OVER_TIME(memory_usage)) + +// Bare time series function — group by every dimension except `pod` (9.4+ GA) +TS k8s +| STATS total_cost = SUM(network.cost) BY WITHOUT(pod) +| SORT total_cost ``` **Best practices:** @@ -185,6 +235,69 @@ TS metrics - Use `TS` instead of `FROM` for aggregations on time series data - Avoid aggregating metrics with different dimensional cardinalities in the same query +### PROMQL + +Queries time series data streams (TSDS) using **Prometheus Query Language (PromQL)** instead of ES|QL syntax. Like `TS`, +it produces a table that the rest of the ES|QL pipeline can process. Available since **9.4 (preview)** and on Elastic +Cloud Serverless. See [promql-command.md](promql-command.md) for the full reference. + +**Syntax:** + +```esql +PROMQL [