diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 29ef74e..163a323 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -13,9 +13,9 @@ "name": "chicago-data-portal", "source": "./skills/chicago-data-portal", "description": "Query Chicago's open data using Socrata/SODA API", - "version": "1.1.0", + "version": "1.2.0", "license": "MIT", - "keywords": ["chicago", "open-data", "socrata", "soda"] + "keywords": ["chicago", "open-data", "socrata", "soda", "crime", "311", "permits"] }, { "name": "cook-county-data-portal", diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c612d3..256ff92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,47 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- **chicago-data-portal** skill (v1.2.0): Major refactor for better usability and progressive disclosure + - **SKILL.md**: Streamlined from ~200 lines to ~110 lines focused on quick-start workflow + - New quick reference table for common endpoints + - Simplified auth check section + - Essential SoQL examples inline, detailed reference moved to `references/` + - Cleaner output template + - **references/popular-datasets.md**: Expanded with decision tree lookup table ("If user asks about X, use Y dataset") + - Added more datasets: Employee Salaries, Divvy Stations, Lobbyist Data + - Richer column documentation and common values + - Better categorization (Public Safety, City Services, Business & Permits, Transportation, Government) + - **references/soql-quick-ref.md**: Enhanced with more practical examples + - Added common query patterns section (pagination, time series, aggregations) + - Better date function examples with `date_trunc_ym` for grouping + - Added `contains()` text function + - **references/geospatial.md**: New reference file for location-based queries + - Coordinate ordering warning (lat/lon vs lon/lat gotcha) + - Common Chicago landmark coordinates + - Examples combining spatial + other filters + - **examples/typescript-query.ts**: New TypeScript/JavaScript example + - Async/await with fetch API + - Typed interfaces for query params and metadata + - AsyncGenerator for pagination + - **examples/python-query.py**: Expanded with more practical examples + - Added `get_all_results()` helper for pagination + - Time series example (monthly counts) + - Food inspections analysis example + - **examples/curl-examples.sh**: More organized with clear sections + - Added filtering, aggregation, and geospatial sections + - Dataset-specific examples for permits, licenses, crashes + ### Added +- **Test Infrastructure**: Integration tests for skill API examples + - `tests/` directory with pytest-based test suite + - `test_chicago_data_portal.py`: 17 tests covering basic queries, metadata, filtering, aggregation, geospatial, pagination, and error handling + - Tests marked with `@pytest.mark.live` for easy skip when offline + - Shared fixtures in `conftest.py` for app tokens across skills + - Documentation in `tests/README.md` + - **Plugin Marketplace Support**: Repository can now be added as a Claude Code plugin marketplace - `.claude-plugin/marketplace.json` manifest with all plugins listed - Individual `plugin.json` files for each skill diff --git a/skills/chicago-data-portal/.claude-plugin/plugin.json b/skills/chicago-data-portal/.claude-plugin/plugin.json index 2b465eb..8a88952 100644 --- a/skills/chicago-data-portal/.claude-plugin/plugin.json +++ b/skills/chicago-data-portal/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "chicago-data-portal", - "version": "1.1.0", + "version": "1.2.0", "description": "Query Chicago's open data using Socrata/SODA API", "license": "MIT", "skills": "./" diff --git a/skills/chicago-data-portal/README.md b/skills/chicago-data-portal/README.md index 369ae2c..a46a1ef 100644 --- a/skills/chicago-data-portal/README.md +++ b/skills/chicago-data-portal/README.md @@ -1,6 +1,6 @@ # Chicago Data Portal -Query and download datasets from the City of Chicago Data Portal using the Socrata Open Data API (SODA) and SoQL. +Query and download datasets from the City of Chicago Open Data Portal using the Socrata SODA API and SoQL. ## Triggers @@ -12,7 +12,7 @@ This skill activates when you ask Claude to: - "write a SODA query for Chicago" - "search data.cityofchicago.org" -Or when you mention Chicago city data (311 requests, permits, licenses, inspections, crimes, etc.). +Or when you mention Chicago city data (311 requests, permits, licenses, inspections, crimes, traffic crashes, etc.). ## Installation @@ -22,7 +22,12 @@ Copy this skill folder to your project's `.claude/plugins/` directory: cp -r skills/chicago-data-portal /path/to/your/project/.claude/plugins/ ``` -Or clone the entire repo and reference it in your Claude Code settings for global access. +Or install from the plugin marketplace: + +``` +/plugin marketplace add MisterClean/claude-plugins +/plugin install chicago-data-portal@misterclean-plugins +``` ## Usage @@ -34,20 +39,46 @@ Once installed, just ask Claude naturally: > "Get 311 service requests for potholes by ward" -Claude will automatically use this skill to discover datasets, build SoQL queries, and retrieve the data. +> "Show me food inspections that failed near Wrigley Field" + +Claude will automatically discover datasets, build SoQL queries, and retrieve data. ## Contents | File | Description | |------|-------------| | [SKILL.md](./SKILL.md) | Core instructions and workflow | -| [references/popular-datasets.md](./references/popular-datasets.md) | Commonly requested datasets with IDs | -| [references/soql-quick-ref.md](./references/soql-quick-ref.md) | SoQL syntax reference | -| [examples/curl-examples.sh](./examples/curl-examples.sh) | Sample curl commands | -| [examples/python-query.py](./examples/python-query.py) | Python query example | +| [references/popular-datasets.md](./references/popular-datasets.md) | Dataset lookup table with IDs and columns | +| [references/soql-quick-ref.md](./references/soql-quick-ref.md) | Complete SoQL syntax reference | +| [references/geospatial.md](./references/geospatial.md) | Location queries and coordinate systems | +| [examples/curl-examples.sh](./examples/curl-examples.sh) | Ready-to-run curl commands | +| [examples/python-query.py](./examples/python-query.py) | Python with requests + pandas | +| [examples/typescript-query.ts](./examples/typescript-query.ts) | TypeScript/JavaScript fetch examples | + +## Popular Datasets + +| Dataset | ID | Use Case | +|---------|-----|----------| +| Crimes | `ijzp-q8t2` | Crime analysis, safety research | +| 311 Service Requests | `v6vf-nfxy` | City services, complaints | +| Building Permits | `ydr8-5enu` | Construction, development | +| Food Inspections | `4ijn-s7e5` | Restaurant health scores | +| Traffic Crashes | `85ca-t3if` | Accident analysis | +| Business Licenses | `r5kz-chrr` | Business activity | + +See [references/popular-datasets.md](./references/popular-datasets.md) for the full list. + +## Authentication + +Queries work without authentication but are rate-limited. For production use, get a free app token: + +1. Sign up at https://data.cityofchicago.org/signup +2. Go to Developer Settings and create an app token +3. Add to your `.env` file: `CHICAGO_DATA_PORTAL_TOKEN=your_token_here` ## Resources - [Chicago Data Portal](https://data.cityofchicago.org) - [SODA API Documentation](https://dev.socrata.com/) +- [SoQL Reference](https://dev.socrata.com/docs/queries/) - [Main Repository](../../README.md) diff --git a/skills/chicago-data-portal/SKILL.md b/skills/chicago-data-portal/SKILL.md index fb99a60..af4cb59 100644 --- a/skills/chicago-data-portal/SKILL.md +++ b/skills/chicago-data-portal/SKILL.md @@ -1,209 +1,113 @@ --- name: chicago-data-portal description: This skill should be used when the user asks to "query Chicago data", "find Chicago datasets", "get Chicago crime data", "download Chicago permits", "write a SODA query for Chicago", "search data.cityofchicago.org", or mentions Chicago city data (311, permits, licenses, inspections, crimes, etc.). -version: 1.1.0 +version: 1.2.0 --- # Chicago Data Portal Skill -Query and download datasets from the City of Chicago Data Portal using the Socrata Open Data API (SODA) and SoQL. +Query Chicago's open data at `data.cityofchicago.org` using the Socrata SODA API. -## Prerequisites +## Quick Reference -Before querying, check if the user has an app token: +| Action | Endpoint | +|--------|----------| +| Query data | `GET https://data.cityofchicago.org/resource/{ID}.json?$where=...` | +| Get schema | `GET https://data.cityofchicago.org/api/views/{ID}` | +| Search catalog | `GET https://api.us.socrata.com/api/catalog/v1?domains=data.cityofchicago.org&q=...` | +| Download CSV | `GET https://data.cityofchicago.org/api/views/{ID}/rows.csv?accessType=DOWNLOAD` | -1. Look for `CHICAGO_DATA_PORTAL_TOKEN` in the user's `.env` file -2. If found, use it in requests via header: `X-App-Token: ` -3. If not found, instruct the user to: - - Sign up at https://data.cityofchicago.org/signup - - Create an app token in Developer Settings - - Add to `.env`: `CHICAGO_DATA_PORTAL_TOKEN=your_token_here` +## Auth Check -Queries work without a token but are rate-limited. - -## Quick Start - -The Chicago Data Portal is at `data.cityofchicago.org`. Each dataset has a unique 4x4 ID (e.g., `ijzp-q8t2` for crimes). Use the catalog API to discover datasets, then query via SODA. +Look for `CHICAGO_DATA_PORTAL_TOKEN` in `.env`. If found, add header `X-App-Token: {token}`. Queries work without tokens but are rate-limited. Get one free at https://data.cityofchicago.org/signup. ## Workflow -### Step 1: Clarify the Data Need +### 1. Find the Dataset -Ask the user: -- **Topic**: What data? (crimes, permits, 311 requests, businesses, etc.) -- **Geography**: Citywide, ward, community area, or specific location/radius? -- **Time window**: Date range or "most recent"? -- **Output**: JSON (code) or CSV (Excel)? -- **Granularity**: Raw rows or aggregated counts? +Check `references/popular-datasets.md` first - it has the most common ones. Otherwise search: -### Step 2: Find the Dataset - -**Option A - Catalog Search API:** -``` -GET https://api.us.socrata.com/api/catalog/v1?domains=data.cityofchicago.org&q= +```bash +curl "https://api.us.socrata.com/api/catalog/v1?domains=data.cityofchicago.org&q=YOUR_KEYWORDS" ``` -**Option B - Portal UI:** -Browse https://data.cityofchicago.org and use the search bar. - -Deliverable: Dataset name, 4x4 ID, and API endpoint. +Extract the 4x4 ID (e.g., `ijzp-q8t2`) from results. -See `references/popular-datasets.md` for commonly requested datasets. +### 2. Get Column Names -### Step 3: Get Dataset Metadata +**Always fetch metadata before building queries:** -Fetch schema and column info: -``` -GET https://data.cityofchicago.org/api/views/<4x4-ID> +```bash +curl "https://data.cityofchicago.org/api/views/{ID}" | jq '.columns[] | {fieldName, dataTypeName}' ``` -Key fields in response: -- `columns[].fieldName` - exact column names for queries -- `columns[].dataTypeName` - data type (text, number, calendar_date, location, etc.) -- `columns[].description` - what the column means -- `rowsUpdatedAt` - last data update timestamp +### 3. Build Query -Always verify column names from metadata before building queries. - -### Step 4: Build the Query - -**Legacy GET (simple, recommended for most cases):** ``` -https://data.cityofchicago.org/resource/<4x4-ID>.json?$where=&$limit=1000 +https://data.cityofchicago.org/resource/{ID}.json?$select=col1,col2&$where=...&$limit=1000 ``` -**SODA3 POST (complex queries):** -```bash -curl -X POST \ - -H "X-App-Token: YOUR_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{"query": "SELECT * WHERE date > '\''2024-01-01'\''", "page": {"pageNumber": 1, "pageSize": 1000}}' \ - https://data.cityofchicago.org/api/v3/views/<4x4-ID>/query.json -``` +**Core parameters:** `$select`, `$where`, `$group`, `$having`, `$order`, `$limit`, `$offset` -### Step 5: Handle Pagination +See `references/soql-quick-ref.md` for syntax details. -Default limit is 1000 rows. For larger extracts: -``` -$limit=1000&$offset=0 # Page 1 -$limit=1000&$offset=1000 # Page 2 -``` +### 4. Handle Large Results -Always include `$order` for stable paging: -``` -$order=date DESC&$limit=1000&$offset=0 -``` +Default max is 1000 rows. For pagination, always include `$order` for stable results: -For full dataset export, use CSV: ``` -https://data.cityofchicago.org/api/views/<4x4-ID>/rows.csv?accessType=DOWNLOAD +$order=date DESC&$limit=1000&$offset=0 # Page 1 +$order=date DESC&$limit=1000&$offset=1000 # Page 2 ``` -## SoQL Essentials - -### Query Parameters -| Param | Purpose | Example | -|-------|---------|---------| -| `$select` | Columns to return | `$select=date,primary_type,ward` | -| `$where` | Filter rows | `$where=year=2024` | -| `$group` | Aggregate | `$group=primary_type` | -| `$having` | Filter aggregates | `$having=count(*)>100` | -| `$order` | Sort results | `$order=date DESC` | -| `$limit` | Max rows | `$limit=500` | -| `$offset` | Skip rows | `$offset=1000` | - -### Syntax Rules -- Backticks around column names: `` `column_name` `` -- Single quotes for strings: `'value'` -- Dates as ISO strings: `'2024-01-01T00:00:00'` - -### Common Filters +## Essential SoQL + ```sql --- Date range +-- Date filter (ISO format) $where=date >= '2024-01-01' AND date < '2025-01-01' --- Text matching (case-insensitive) +-- Text (case-insensitive) $where=upper(primary_type) = 'THEFT' --- Null handling -$where=ward IS NOT NULL - -- Multiple values $where=primary_type IN ('THEFT', 'BATTERY', 'ASSAULT') -``` - -### Aggregations -```sql -$select=primary_type, count(*) as total -$group=primary_type -$order=total DESC -``` - -See `references/soql-quick-ref.md` for full function reference. - -## Geospatial Queries -If the dataset has a location field (Point type): +-- Aggregation +$select=primary_type, count(*) as total&$group=primary_type&$order=total DESC -```sql --- Within radius (meters) +-- Geospatial (within 1km of a point) $where=within_circle(location, 41.8781, -87.6298, 1000) - --- Within bounding box -$where=within_box(location, 42.0, -87.9, 41.6, -87.5) - --- Within polygon -$where=within_polygon(location, 'MULTIPOLYGON(((-87.6 41.8, -87.5 41.8, -87.5 41.9, -87.6 41.9, -87.6 41.8)))') ``` -## App Tokens - -Unauthenticated requests are rate-limited. Register for a free app token: +## Output Template -1. Create account at https://data.cityofchicago.org -2. Go to Developer Settings -3. Create New App Token -4. Use via header: `X-App-Token: YOUR_TOKEN` +Always provide: -## Output Format - -Provide the user with: -1. **Dataset**: Name + 4x4 ID + portal link -2. **Columns used**: Exact field names -3. **Query**: Formatted SoQL -4. **How to run**: curl command or full URL -5. **Assumptions**: Time zone, update frequency, any caveats - -### Example Response Format ``` -Dataset: Crimes - 2001 to Present (ijzp-q8t2) -https://data.cityofchicago.org/d/ijzp-q8t2 +**Dataset:** {Name} ({ID}) +{portal_link} -Query: -SELECT date, primary_type, description, ward, latitude, longitude -WHERE date >= '2024-01-01' AND primary_type = 'THEFT' -ORDER BY date DESC -LIMIT 100 +**Query:** +{formatted_soql} -Run it: -curl "https://data.cityofchicago.org/resource/ijzp-q8t2.json?\$select=date,primary_type,description,ward,latitude,longitude&\$where=date%20%3E=%20%272024-01-01%27%20AND%20primary_type%20=%20%27THEFT%27&\$order=date%20DESC&\$limit=100" +**Run it:** +curl "{full_url}" -Note: Data updates daily. Dates are in Chicago local time (America/Chicago). +**Notes:** {update frequency, timezone (America/Chicago), any caveats} ``` ## Troubleshooting -| Issue | Fix | +| Error | Fix | |-------|-----| -| 404 / "unknown column" | Wrong dataset ID or field name. Check metadata endpoint. | -| Empty results | Filters too strict, wrong date format, or nulls. | -| 429 throttled | Add X-App-Token header. | -| Slow query | Select fewer columns, add filters, reduce limit. | -| Encoding errors | URL-encode special chars: space=%20, >=%3E, '=%27 | - -## Additional Resources - -- **`references/popular-datasets.md`** - Common Chicago datasets with IDs -- **`references/soql-quick-ref.md`** - All SoQL functions -- **`examples/python-query.py`** - Python code snippet -- **`examples/curl-examples.sh`** - curl command templates +| 404 / unknown column | Wrong ID or column name. Check `/api/views/{ID}` | +| Empty results | Filters too strict, wrong date format, or all nulls | +| 429 throttled | Add `X-App-Token` header | +| Encoding errors | URL-encode: space=%20, >=%3E, '=%27 | + +## References + +- `references/popular-datasets.md` - Dataset IDs organized by category +- `references/soql-quick-ref.md` - Full SoQL syntax +- `references/geospatial.md` - Location queries and coordinate systems +- `examples/` - Working code in Python, JavaScript, and curl diff --git a/skills/chicago-data-portal/examples/curl-examples.sh b/skills/chicago-data-portal/examples/curl-examples.sh index be735c6..586ea2a 100644 --- a/skills/chicago-data-portal/examples/curl-examples.sh +++ b/skills/chicago-data-portal/examples/curl-examples.sh @@ -2,53 +2,123 @@ # Chicago Data Portal curl examples # Replace YOUR_TOKEN with your app token (optional but recommended) -# --- DISCOVERY --- +# ============================================================ +# DISCOVERY & METADATA +# ============================================================ -# Search catalog for datasets +# Search catalog for datasets by keyword curl "https://api.us.socrata.com/api/catalog/v1?domains=data.cityofchicago.org&q=building%20permits" # Get dataset metadata (columns, types, descriptions) curl "https://data.cityofchicago.org/api/views/ijzp-q8t2" | jq '.columns[] | {fieldName, dataTypeName}' -# --- BASIC QUERIES (Legacy GET) --- +# Get just column names +curl "https://data.cityofchicago.org/api/views/ijzp-q8t2" | jq '.columns[].fieldName' -# Recent crimes (JSON) +# ============================================================ +# BASIC QUERIES +# ============================================================ + +# Recent crimes (JSON, 10 most recent) curl "https://data.cityofchicago.org/resource/ijzp-q8t2.json?\$limit=10&\$order=date%20DESC" -# Recent crimes (CSV) +# Same query but CSV format curl "https://data.cityofchicago.org/resource/ijzp-q8t2.csv?\$limit=10&\$order=date%20DESC" -# Filter by date and type -curl "https://data.cityofchicago.org/resource/ijzp-q8t2.json?\$where=date%20%3E=%20%272024-01-01%27%20AND%20primary_type%20=%20%27THEFT%27&\$limit=100" - -# With app token +# With app token (higher rate limits) curl -H "X-App-Token: YOUR_TOKEN" \ "https://data.cityofchicago.org/resource/ijzp-q8t2.json?\$limit=100" -# --- AGGREGATION --- +# ============================================================ +# FILTERING +# ============================================================ + +# Thefts in 2024 +curl "https://data.cityofchicago.org/resource/ijzp-q8t2.json?\$where=date%20%3E=%20%272024-01-01%27%20AND%20primary_type%20=%20%27THEFT%27&\$limit=100" + +# Multiple crime types +curl "https://data.cityofchicago.org/resource/ijzp-q8t2.json?\$where=primary_type%20IN%20(%27THEFT%27,%27BATTERY%27,%27ASSAULT%27)&\$limit=100" + +# Specific ward +curl "https://data.cityofchicago.org/resource/ijzp-q8t2.json?\$where=ward%20=%20%2742%27&\$limit=100" + +# Date range +curl "https://data.cityofchicago.org/resource/ijzp-q8t2.json?\$where=date%20%3E=%20%272024-01-01%27%20AND%20date%20%3C%20%272024-02-01%27&\$limit=1000" + +# ============================================================ +# AGGREGATION +# ============================================================ # Count crimes by type curl "https://data.cityofchicago.org/resource/ijzp-q8t2.json?\$select=primary_type,count(*)%20as%20total&\$group=primary_type&\$order=total%20DESC&\$limit=10" -# --- GEOSPATIAL --- +# Count by ward +curl "https://data.cityofchicago.org/resource/ijzp-q8t2.json?\$select=ward,count(*)%20as%20total&\$where=date%20%3E=%20%272024-01-01%27&\$group=ward&\$order=total%20DESC" + +# Count by month +curl "https://data.cityofchicago.org/resource/ijzp-q8t2.json?\$select=date_trunc_ym(date)%20as%20month,count(*)%20as%20total&\$where=date%20%3E=%20%272024-01-01%27&\$group=date_trunc_ym(date)&\$order=month" + +# 311 requests by type (top 20) +curl "https://data.cityofchicago.org/resource/v6vf-nfxy.json?\$select=sr_type,count(*)%20as%20total&\$where=created_date%20%3E=%20%272024-01-01%27&\$group=sr_type&\$order=total%20DESC&\$limit=20" + +# ============================================================ +# GEOSPATIAL +# ============================================================ # Crimes within 1km of downtown (41.8781, -87.6298) -curl "https://data.cityofchicago.org/resource/ijzp-q8t2.json?\$where=within_circle(location,41.8781,-87.6298,1000)&\$limit=50" +curl "https://data.cityofchicago.org/resource/ijzp-q8t2.json?\$where=within_circle(location,41.8781,-87.6298,1000)&\$limit=100" + +# Crimes within 500m of Willis Tower +curl "https://data.cityofchicago.org/resource/ijzp-q8t2.json?\$where=within_circle(location,41.8789,-87.6359,500)&\$order=date%20DESC&\$limit=50" + +# Food inspections near Wrigley Field that failed +curl "https://data.cityofchicago.org/resource/4ijn-s7e5.json?\$where=within_circle(location,41.9484,-87.6553,500)%20AND%20results=%27Fail%27&\$limit=50" -# --- PAGINATION --- +# Bounding box (The Loop area) +curl "https://data.cityofchicago.org/resource/ijzp-q8t2.json?\$where=within_box(location,41.887,-87.6425,41.875,-87.619)&\$limit=100" -# Page 1 +# ============================================================ +# PAGINATION +# ============================================================ + +# Page 1 (first 1000) curl "https://data.cityofchicago.org/resource/ijzp-q8t2.json?\$order=date%20DESC&\$limit=1000&\$offset=0" -# Page 2 +# Page 2 (next 1000) curl "https://data.cityofchicago.org/resource/ijzp-q8t2.json?\$order=date%20DESC&\$limit=1000&\$offset=1000" -# --- FULL EXPORT (large datasets) --- +# Page 3 +curl "https://data.cityofchicago.org/resource/ijzp-q8t2.json?\$order=date%20DESC&\$limit=1000&\$offset=2000" + +# ============================================================ +# FULL EXPORTS (large datasets) +# ============================================================ -# Download entire dataset as CSV +# Download entire dataset as CSV (WARNING: can be very large!) curl -o crimes.csv "https://data.cityofchicago.org/api/views/ijzp-q8t2/rows.csv?accessType=DOWNLOAD" -# --- SODA3 POST (complex queries) --- +# Download with filters (more practical) +curl -o thefts_2024.csv "https://data.cityofchicago.org/resource/ijzp-q8t2.csv?\$where=date%20%3E=%20%272024-01-01%27%20AND%20primary_type=%27THEFT%27&\$limit=50000" + +# ============================================================ +# SPECIFIC DATASET EXAMPLES +# ============================================================ + +# Building permits in last 30 days +curl "https://data.cityofchicago.org/resource/ydr8-5enu.json?\$where=issue_date%20%3E=%20%272024-12-01%27&\$order=issue_date%20DESC&\$limit=100" + +# Business licenses issued this year +curl "https://data.cityofchicago.org/resource/r5kz-chrr.json?\$where=date_issued%20%3E=%20%272024-01-01%27&\$order=date_issued%20DESC&\$limit=100" + +# Towed vehicles (rolling 90-day window) +curl "https://data.cityofchicago.org/resource/ygr5-vcbg.json?\$order=tow_date%20DESC&\$limit=50" + +# Traffic crashes with injuries +curl "https://data.cityofchicago.org/resource/85ca-t3if.json?\$where=injuries_total%20%3E%200&\$order=crash_date%20DESC&\$limit=100" + +# ============================================================ +# ADVANCED: SODA3 POST (for complex queries) +# ============================================================ curl -X POST \ -H "X-App-Token: YOUR_TOKEN" \ diff --git a/skills/chicago-data-portal/examples/python-query.py b/skills/chicago-data-portal/examples/python-query.py index 402da17..5398732 100644 --- a/skills/chicago-data-portal/examples/python-query.py +++ b/skills/chicago-data-portal/examples/python-query.py @@ -1,58 +1,205 @@ """Chicago Data Portal query examples using requests + pandas.""" -import requests +import os +from typing import Any, Iterator + import pandas as pd +import requests -# Optional: Set your app token for higher rate limits -APP_TOKEN = None # or "your-token-here" +# Optional: Load from .env or environment +APP_TOKEN = os.getenv("CHICAGO_DATA_PORTAL_TOKEN") HEADERS = {"X-App-Token": APP_TOKEN} if APP_TOKEN else {} BASE_URL = "https://data.cityofchicago.org/resource" +METADATA_URL = "https://data.cityofchicago.org/api/views" def query_dataset(dataset_id: str, params: dict) -> pd.DataFrame: """Query a Chicago dataset and return as DataFrame.""" url = f"{BASE_URL}/{dataset_id}.json" - resp = requests.get(url, params=params, headers=HEADERS) + resp = requests.get(url, params=params, headers=HEADERS, timeout=30) resp.raise_for_status() return pd.DataFrame(resp.json()) def get_metadata(dataset_id: str) -> dict: """Fetch dataset metadata including columns.""" - url = f"https://data.cityofchicago.org/api/views/{dataset_id}" - resp = requests.get(url, headers=HEADERS) + url = f"{METADATA_URL}/{dataset_id}" + resp = requests.get(url, headers=HEADERS, timeout=30) resp.raise_for_status() return resp.json() -# Example 1: Recent crimes in a ward -crimes = query_dataset("ijzp-q8t2", { - "$select": "date, primary_type, description, block", - "$where": "date >= '2024-01-01' AND ward = '42'", - "$order": "date DESC", - "$limit": 100 -}) - -# Example 2: 311 requests by type (aggregated) -requests_by_type = query_dataset("v6vf-nfxy", { - "$select": "sr_type, count(*) as total", - "$where": "created_date >= '2024-01-01'", - "$group": "sr_type", - "$order": "total DESC", - "$limit": 20 -}) - -# Example 3: Paginate through large results -def get_all_pages(dataset_id: str, base_params: dict, page_size: int = 1000): - """Fetch all rows with pagination.""" - all_data = [] +def paginate_dataset( + dataset_id: str, params: dict, page_size: int = 1000 +) -> Iterator[pd.DataFrame]: + """Generator that yields pages of results.""" offset = 0 while True: - params = {**base_params, "$limit": page_size, "$offset": offset} - df = query_dataset(dataset_id, params) + page_params = {**params, "$limit": page_size, "$offset": offset} + df = query_dataset(dataset_id, page_params) if df.empty: break - all_data.append(df) + yield df offset += page_size - return pd.concat(all_data, ignore_index=True) if all_data else pd.DataFrame() + if len(df) < page_size: + break + + +def get_all_results(dataset_id: str, params: dict, page_size: int = 1000) -> pd.DataFrame: + """Fetch all pages and concatenate into single DataFrame.""" + all_dfs = list(paginate_dataset(dataset_id, params, page_size)) + return pd.concat(all_dfs, ignore_index=True) if all_dfs else pd.DataFrame() + + +# ============================================================ +# Example 1: Recent crimes in a ward +# ============================================================ +def example_recent_crimes(): + """Get recent crimes in a specific ward.""" + crimes = query_dataset( + "ijzp-q8t2", + { + "$select": "date, primary_type, description, block, latitude, longitude", + "$where": "date >= '2024-01-01' AND ward = '42'", + "$order": "date DESC", + "$limit": 100, + }, + ) + print(f"Found {len(crimes)} crimes") + print(crimes.head()) + return crimes + + +# ============================================================ +# Example 2: 311 requests by type (aggregated) +# ============================================================ +def example_311_by_type(): + """Get 311 service requests aggregated by type.""" + requests_by_type = query_dataset( + "v6vf-nfxy", + { + "$select": "sr_type, count(*) as total", + "$where": "created_date >= '2024-01-01'", + "$group": "sr_type", + "$order": "total DESC", + "$limit": 20, + }, + ) + print("Top 311 request types:") + print(requests_by_type) + return requests_by_type + + +# ============================================================ +# Example 3: Geospatial - crimes near a location +# ============================================================ +def example_crimes_near_location(lat: float, lon: float, radius_m: int): + """Get crimes within a radius of a point.""" + crimes = query_dataset( + "ijzp-q8t2", + { + "$where": f"within_circle(location, {lat}, {lon}, {radius_m})", + "$order": "date DESC", + "$limit": 100, + }, + ) + print(f"Found {len(crimes)} crimes within {radius_m}m of ({lat}, {lon})") + return crimes + + +# ============================================================ +# Example 4: Time series - monthly crime counts +# ============================================================ +def example_monthly_crime_counts(): + """Get crime counts by month.""" + monthly = query_dataset( + "ijzp-q8t2", + { + "$select": "date_trunc_ym(date) as month, count(*) as total", + "$where": "date >= '2024-01-01'", + "$group": "date_trunc_ym(date)", + "$order": "month", + }, + ) + print("Monthly crime counts:") + print(monthly) + return monthly + + +# ============================================================ +# Example 5: Inspect dataset columns before querying +# ============================================================ +def example_inspect_dataset(dataset_id: str): + """Print column information for a dataset.""" + meta = get_metadata(dataset_id) + print(f"Dataset: {meta['name']}") + print(f"Description: {meta.get('description', 'N/A')}") + print("\nColumns:") + for col in meta["columns"]: + desc = col.get("description", "No description") + print(f" {col['fieldName']} ({col['dataTypeName']}): {desc[:60]}...") + + +# ============================================================ +# Example 6: Paginate through large results +# ============================================================ +def example_paginate_permits(): + """Get all building permits for 2024 using pagination.""" + all_permits = get_all_results( + "ydr8-5enu", + { + "$where": "issue_date >= '2024-01-01'", + "$order": "issue_date DESC", + }, + page_size=2000, + ) + print(f"Total permits: {len(all_permits)}") + return all_permits + + +# ============================================================ +# Example 7: Food inspections analysis +# ============================================================ +def example_food_inspections(): + """Analyze food inspection results.""" + inspections = query_dataset( + "4ijn-s7e5", + { + "$select": "results, count(*) as total", + "$where": "inspection_date >= '2024-01-01'", + "$group": "results", + "$order": "total DESC", + }, + ) + print("Food inspection results:") + print(inspections) + return inspections + + +if __name__ == "__main__": + # Run examples + print("=" * 60) + print("Example 1: Recent crimes") + print("=" * 60) + example_recent_crimes() + + print("\n" + "=" * 60) + print("Example 2: 311 by type") + print("=" * 60) + example_311_by_type() + + print("\n" + "=" * 60) + print("Example 3: Crimes near Willis Tower") + print("=" * 60) + example_crimes_near_location(41.8789, -87.6359, 500) + + print("\n" + "=" * 60) + print("Example 4: Monthly crime counts") + print("=" * 60) + example_monthly_crime_counts() + + print("\n" + "=" * 60) + print("Example 5: Inspect crimes dataset") + print("=" * 60) + example_inspect_dataset("ijzp-q8t2") diff --git a/skills/chicago-data-portal/examples/typescript-query.ts b/skills/chicago-data-portal/examples/typescript-query.ts new file mode 100644 index 0000000..e8788d2 --- /dev/null +++ b/skills/chicago-data-portal/examples/typescript-query.ts @@ -0,0 +1,195 @@ +/** + * Chicago Data Portal query examples using TypeScript/JavaScript. + * Works in Node.js, Deno, Bun, or browsers (with CORS). + */ + +const BASE_URL = "https://data.cityofchicago.org/resource"; +const METADATA_URL = "https://data.cityofchicago.org/api/views"; + +// Optional: Set via environment variable for higher rate limits +const APP_TOKEN = process.env.CHICAGO_DATA_PORTAL_TOKEN; + +interface QueryParams { + $select?: string; + $where?: string; + $group?: string; + $having?: string; + $order?: string; + $limit?: number; + $offset?: number; +} + +interface ColumnMetadata { + fieldName: string; + dataTypeName: string; + description: string; +} + +/** + * Query a Chicago dataset and return JSON results. + */ +async function queryDataset>( + datasetId: string, + params: QueryParams +): Promise { + const url = new URL(`${BASE_URL}/${datasetId}.json`); + + for (const [key, value] of Object.entries(params)) { + if (value !== undefined) { + url.searchParams.set(key, String(value)); + } + } + + const headers: HeadersInit = { + Accept: "application/json", + }; + if (APP_TOKEN) { + headers["X-App-Token"] = APP_TOKEN; + } + + const response = await fetch(url.toString(), { headers }); + + if (!response.ok) { + throw new Error(`API error: ${response.status} ${response.statusText}`); + } + + return response.json(); +} + +/** + * Get dataset metadata including column definitions. + */ +async function getMetadata(datasetId: string): Promise<{ + name: string; + description: string; + rowsUpdatedAt: number; + columns: ColumnMetadata[]; +}> { + const response = await fetch(`${METADATA_URL}/${datasetId}`); + + if (!response.ok) { + throw new Error(`Metadata error: ${response.status}`); + } + + return response.json(); +} + +/** + * Paginate through large result sets. + */ +async function* paginateDataset>( + datasetId: string, + params: QueryParams, + pageSize = 1000 +): AsyncGenerator { + let offset = 0; + + while (true) { + const results = await queryDataset(datasetId, { + ...params, + $limit: pageSize, + $offset: offset, + }); + + if (results.length === 0) break; + + yield results; + offset += pageSize; + + if (results.length < pageSize) break; + } +} + +// ============================================================ +// Example Usage +// ============================================================ + +// Example 1: Recent crimes in a specific ward +async function getRecentCrimes() { + const crimes = await queryDataset("ijzp-q8t2", { + $select: "date, primary_type, description, block, latitude, longitude", + $where: "date >= '2024-01-01' AND ward = '42'", + $order: "date DESC", + $limit: 100, + }); + + console.log(`Found ${crimes.length} crimes`); + return crimes; +} + +// Example 2: 311 requests aggregated by type +async function get311ByType() { + const results = await queryDataset("v6vf-nfxy", { + $select: "sr_type, count(*) as total", + $where: "created_date >= '2024-01-01'", + $group: "sr_type", + $order: "total DESC", + $limit: 20, + }); + + return results; +} + +// Example 3: Geospatial - crimes near a location +async function getCrimesNearLocation(lat: number, lon: number, radiusMeters: number) { + const crimes = await queryDataset("ijzp-q8t2", { + $where: `within_circle(location, ${lat}, ${lon}, ${radiusMeters})`, + $order: "date DESC", + $limit: 100, + }); + + return crimes; +} + +// Example 4: Get all pages of a large result set +async function getAllBuildingPermits2024() { + const allPermits: Record[] = []; + + for await (const page of paginateDataset("ydr8-5enu", { + $where: "issue_date >= '2024-01-01'", + $order: "issue_date DESC", + })) { + allPermits.push(...page); + console.log(`Fetched ${allPermits.length} permits so far...`); + } + + return allPermits; +} + +// Example 5: Check column names before querying +async function inspectDataset(datasetId: string) { + const meta = await getMetadata(datasetId); + + console.log(`Dataset: ${meta.name}`); + console.log(`Description: ${meta.description}`); + console.log(`Last updated: ${new Date(meta.rowsUpdatedAt * 1000).toISOString()}`); + console.log("\nColumns:"); + + for (const col of meta.columns) { + console.log(` ${col.fieldName} (${col.dataTypeName}): ${col.description || "No description"}`); + } +} + +// Run examples +async function main() { + try { + // Inspect the crimes dataset + await inspectDataset("ijzp-q8t2"); + + // Get recent crimes + const crimes = await getRecentCrimes(); + console.log("\nRecent crimes sample:", crimes.slice(0, 3)); + + // Get 311 by type + const requests = await get311ByType(); + console.log("\n311 requests by type:", requests.slice(0, 5)); + + // Geospatial query - crimes near Willis Tower + const nearbycrimes = await getCrimesNearLocation(41.8789, -87.6359, 500); + console.log(`\nCrimes within 500m of Willis Tower: ${nearbycrimes.length}`); + } catch (error) { + console.error("Error:", error); + } +} + +main(); diff --git a/skills/chicago-data-portal/references/geospatial.md b/skills/chicago-data-portal/references/geospatial.md new file mode 100644 index 0000000..01ac00f --- /dev/null +++ b/skills/chicago-data-portal/references/geospatial.md @@ -0,0 +1,136 @@ +# Geospatial Queries + +Chicago datasets with location data support powerful spatial queries. Most datasets have a `location` column (Point type) with lat/lon coordinates. + +## Coordinate Ordering Warning + +**Different functions use different coordinate orders:** + +| Function | Order | Example | +|----------|-------|---------| +| `within_circle` | lat, lon | `within_circle(location, 41.8781, -87.6298, 1000)` | +| `within_box` | north_lat, west_lon, south_lat, east_lon | `within_box(location, 42.0, -87.9, 41.6, -87.5)` | +| WKT (POLYGON, POINT) | lon, lat | `'POINT(-87.6298 41.8781)'` | +| GeoJSON | lon, lat | `[-87.6298, 41.8781]` | + +**Remember:** SoQL functions use `lat, lon` but WKT/GeoJSON use `lon, lat`. + +## Functions + +### within_circle + +Find records within a radius (in meters) of a point: + +```sql +$where=within_circle(location, {lat}, {lon}, {radius_meters}) +``` + +**Examples:** +```sql +-- Crimes within 500m of Willis Tower (41.8789, -87.6359) +$where=within_circle(location, 41.8789, -87.6359, 500) + +-- 311 requests within 1km of Wrigley Field (41.9484, -87.6553) +$where=within_circle(location, 41.9484, -87.6553, 1000) +``` + +### within_box + +Find records within a bounding box: + +```sql +$where=within_box(location, {north_lat}, {west_lon}, {south_lat}, {east_lon}) +``` + +**Example:** +```sql +-- The Loop (roughly) +$where=within_box(location, 41.8870, -87.6425, 41.8750, -87.6190) +``` + +### within_polygon + +Find records within an arbitrary polygon using WKT (Well-Known Text): + +```sql +$where=within_polygon(location, 'MULTIPOLYGON((({lon1} {lat1}, {lon2} {lat2}, ...)))') +``` + +**Note:** Coordinates in WKT are `lon lat` (longitude first), opposite of the other functions. + +**Example:** +```sql +-- Triangle in downtown +$where=within_polygon(location, 'MULTIPOLYGON(((-87.63 41.88, -87.62 41.88, -87.625 41.875, -87.63 41.88)))') +``` + +### distance_in_meters + +Calculate distance from a point (useful for sorting by proximity): + +```sql +$select=*, distance_in_meters(location, 'POINT({lon} {lat})') as distance +$order=distance ASC +``` + +**Example:** +```sql +-- Nearest food inspections to an address +$select=dba_name, address, results, distance_in_meters(location, 'POINT(-87.6298 41.8781)') as distance +&$order=distance ASC +&$limit=20 +``` + +### intersects + +Check if geometries overlap (for datasets with polygon geometry): + +```sql +$where=intersects(the_geom, 'POLYGON(({lon1} {lat1}, ...))') +``` + +## Common Chicago Locations + +| Location | Latitude | Longitude | +|----------|----------|-----------| +| City Hall | 41.8838 | -87.6319 | +| Willis Tower | 41.8789 | -87.6359 | +| Wrigley Field | 41.9484 | -87.6553 | +| Soldier Field | 41.8623 | -87.6167 | +| O'Hare Airport | 41.9742 | -87.9073 | +| Midway Airport | 41.7868 | -87.7522 | +| Navy Pier | 41.8917 | -87.6086 | +| Millennium Park | 41.8826 | -87.6226 | +| United Center | 41.8807 | -87.6742 | +| University of Chicago | 41.7886 | -87.5987 | + +## Combining Spatial with Other Filters + +```sql +-- Thefts in 2024 within 1km of downtown +$where=within_circle(location, 41.8781, -87.6298, 1000) + AND date >= '2024-01-01' + AND primary_type = 'THEFT' +$limit=500 +``` + +## Checking for Location Data + +Not all records have coordinates. Filter out nulls: + +```sql +$where=location IS NOT NULL AND within_circle(location, 41.8781, -87.6298, 1000) +``` + +Or check for null lat/lon: + +```sql +$where=latitude IS NOT NULL AND longitude IS NOT NULL +``` + +## Full URL Example + +```bash +# Food inspections that failed within 500m of a location +curl "https://data.cityofchicago.org/resource/4ijn-s7e5.json?\$where=within_circle(location,41.8838,-87.6319,500)%20AND%20results=%27Fail%27&\$limit=50" +``` diff --git a/skills/chicago-data-portal/references/popular-datasets.md b/skills/chicago-data-portal/references/popular-datasets.md index ba0221d..23c86a1 100644 --- a/skills/chicago-data-portal/references/popular-datasets.md +++ b/skills/chicago-data-portal/references/popular-datasets.md @@ -1,47 +1,127 @@ # Popular Chicago Datasets -Quick reference for commonly requested datasets. Always verify columns via metadata endpoint before querying. +Quick reference for commonly requested datasets. **Always verify column names via metadata endpoint before querying.** + +## Dataset Lookup by Topic + +| If user asks about... | Use this dataset | +|-----------------------|------------------| +| Crime, shootings, theft | Crimes - 2001 to Present (`ijzp-q8t2`) | +| Car accidents, traffic crashes | Traffic Crashes (`85ca-t3if`) | +| 311, service requests, complaints | 311 Service Requests (`v6vf-nfxy`) | +| Towed cars, impound | Towed Vehicles (`ygr5-vcbg`) | +| Construction, building permits | Building Permits (`ydr8-5enu`) | +| Business licenses, new businesses | Business Licenses (`r5kz-chrr`) | +| Restaurant inspections, health | Food Inspections (`4ijn-s7e5`) | +| Bike sharing, Divvy | Divvy Trips (`fg6s-gzvg`) | +| Potholes, street repairs | 311 Service Requests (filter: `sr_type LIKE '%pothole%'`) | +| Graffiti removal | 311 Service Requests (filter: `sr_type LIKE '%graffiti%'`) | +| Employee salaries | Employee Salaries (`xzkq-xp2w`) | +| Lobbyist activity | Lobbyist Data (`tq3e-t5yq`) | + +--- ## Public Safety -| Dataset | ID | Key Columns | Update | -|---------|-----|-------------|--------| -| [Crimes - 2001 to Present](https://data.cityofchicago.org/d/ijzp-q8t2) | `ijzp-q8t2` | date, primary_type, description, ward, community_area, latitude, longitude | Daily | -| [Traffic Crashes - Crashes](https://data.cityofchicago.org/d/85ca-t3if) | `85ca-t3if` | crash_date, crash_type, injuries_total, latitude, longitude | Daily | +### Crimes - 2001 to Present +- **ID:** `ijzp-q8t2` +- **Link:** https://data.cityofchicago.org/d/ijzp-q8t2 +- **Key columns:** `date`, `primary_type`, `description`, `arrest`, `domestic`, `beat`, `district`, `ward`, `community_area`, `latitude`, `longitude`, `location` +- **Update:** Daily (excludes most recent 7 days) +- **Notes:** Privacy-protected - addresses shown at block level only. Types include: THEFT, BATTERY, CRIMINAL DAMAGE, ASSAULT, BURGLARY, NARCOTICS, MOTOR VEHICLE THEFT, ROBBERY, etc. + +### Traffic Crashes - Crashes +- **ID:** `85ca-t3if` +- **Link:** https://data.cityofchicago.org/d/85ca-t3if +- **Key columns:** `crash_date`, `crash_type`, `posted_speed_limit`, `traffic_control_device`, `weather_condition`, `lighting_condition`, `first_crash_type`, `injuries_total`, `injuries_fatal`, `latitude`, `longitude` +- **Update:** Daily +- **Related:** Vehicles (`68nd-jvt3`), People (`u6pd-qa9d`) + +--- ## City Services -| Dataset | ID | Key Columns | Update | -|---------|-----|-------------|--------| -| [311 Service Requests](https://data.cityofchicago.org/d/v6vf-nfxy) | `v6vf-nfxy` | sr_number, sr_type, created_date, status, ward, latitude, longitude | Daily | -| [Towed Vehicles](https://data.cityofchicago.org/d/ygr5-vcbg) | `ygr5-vcbg` | tow_date, make, color, plate, towed_to_address | Daily (90-day window) | +### 311 Service Requests +- **ID:** `v6vf-nfxy` +- **Link:** https://data.cityofchicago.org/d/v6vf-nfxy +- **Key columns:** `sr_number`, `sr_type`, `sr_short_code`, `created_date`, `closed_date`, `status`, `ward`, `community_area`, `latitude`, `longitude` +- **Update:** Daily +- **Common sr_types:** "Pothole in Street", "Graffiti Removal Request", "Street Light Out", "Tree Trim Request", "Abandoned Vehicle", "Rodent Baiting/Rat Complaint" + +### Towed Vehicles +- **ID:** `ygr5-vcbg` +- **Link:** https://data.cityofchicago.org/d/ygr5-vcbg +- **Key columns:** `tow_date`, `make`, `style`, `model`, `color`, `plate`, `state`, `towed_to_address`, `inventory_number` +- **Update:** Daily +- **Notes:** Rolling 90-day window only. No location of tow origin. + +--- ## Business & Permits -| Dataset | ID | Key Columns | Update | -|---------|-----|-------------|--------| -| [Building Permits](https://data.cityofchicago.org/d/ydr8-5enu) | `ydr8-5enu` | issue_date, permit_type, work_description, total_fee, latitude, longitude | Daily | -| [Business Licenses](https://data.cityofchicago.org/d/r5kz-chrr) | `r5kz-chrr` | license_id, doing_business_as_name, license_description, date_issued, ward | Daily | -| [Food Inspections](https://data.cityofchicago.org/d/4ijn-s7e5) | `4ijn-s7e5` | inspection_date, dba_name, facility_type, results, violations, latitude, longitude | Daily | +### Building Permits +- **ID:** `ydr8-5enu` +- **Link:** https://data.cityofchicago.org/d/ydr8-5enu +- **Key columns:** `id`, `permit_`, `permit_type`, `issue_date`, `work_description`, `total_fee`, `street_number`, `street_direction`, `street_name`, `suffix`, `latitude`, `longitude` +- **Update:** Daily +- **permit_type values:** "PERMIT - NEW CONSTRUCTION", "PERMIT - RENOVATION/ALTERATION", "PERMIT - ELECTRIC WIRING", etc. + +### Business Licenses +- **ID:** `r5kz-chrr` +- **Link:** https://data.cityofchicago.org/d/r5kz-chrr +- **Key columns:** `license_id`, `account_number`, `legal_name`, `doing_business_as_name`, `license_description`, `business_activity`, `license_status`, `date_issued`, `expiration_date`, `ward`, `latitude`, `longitude` +- **Update:** Daily +- **Notes:** One row per license; businesses may have multiple licenses. + +### Food Inspections +- **ID:** `4ijn-s7e5` +- **Link:** https://data.cityofchicago.org/d/4ijn-s7e5 +- **Key columns:** `inspection_id`, `dba_name`, `aka_name`, `license_`, `facility_type`, `risk`, `inspection_date`, `inspection_type`, `results`, `violations`, `latitude`, `longitude` +- **Update:** Daily +- **results values:** "Pass", "Fail", "Pass w/ Conditions", "Out of Business", "No Entry" +- **Notes:** `violations` is a pipe-delimited text field with violation codes and comments. + +--- ## Transportation -| Dataset | ID | Key Columns | Update | -|---------|-----|-------------|--------| -| [Divvy Trips](https://data.cityofchicago.org/d/fg6s-gzvg) | `fg6s-gzvg` | start_time, end_time, from_station_name, to_station_name, tripduration | Monthly | +### Divvy Trips +- **ID:** `fg6s-gzvg` +- **Link:** https://data.cityofchicago.org/d/fg6s-gzvg +- **Key columns:** `trip_id`, `start_time`, `end_time`, `bikeid`, `tripduration`, `from_station_id`, `from_station_name`, `to_station_id`, `to_station_name`, `usertype` +- **Update:** Monthly +- **Notes:** Excludes trips under 60 seconds and staff service trips. + +### Divvy Bicycle Stations +- **ID:** `bbyy-e7gq` +- **Link:** https://data.cityofchicago.org/d/bbyy-e7gq +- **Key columns:** `id`, `station_name`, `total_docks`, `docks_in_service`, `status`, `latitude`, `longitude` +- **Update:** Near real-time + +--- + +## Government & Finance -## Quick Metadata Check +### Employee Salaries +- **ID:** `xzkq-xp2w` +- **Link:** https://data.cityofchicago.org/d/xzkq-xp2w +- **Key columns:** `name`, `job_titles`, `department`, `full_or_part_time`, `salary_or_hourly`, `annual_salary`, `hourly_rate` +- **Update:** Annually + +--- + +## Metadata Quick Check Get full schema for any dataset: ```bash -curl "https://data.cityofchicago.org/api/views/" | jq '.columns[] | {fieldName, dataTypeName, description}' +curl "https://data.cityofchicago.org/api/views/{ID}" | jq '.columns[] | {fieldName, dataTypeName, description}' ``` -## Finding More Datasets +## Find More Datasets -Search the catalog: -``` -https://api.us.socrata.com/api/catalog/v1?domains=data.cityofchicago.org&q=YOUR_KEYWORDS +Search the catalog API: +```bash +curl "https://api.us.socrata.com/api/catalog/v1?domains=data.cityofchicago.org&q=YOUR_KEYWORDS" ``` Or browse: https://data.cityofchicago.org diff --git a/skills/chicago-data-portal/references/soql-quick-ref.md b/skills/chicago-data-portal/references/soql-quick-ref.md index 5221a25..7d26e7d 100644 --- a/skills/chicago-data-portal/references/soql-quick-ref.md +++ b/skills/chicago-data-portal/references/soql-quick-ref.md @@ -1,93 +1,233 @@ # SoQL Quick Reference +Complete syntax reference for Socrata Query Language (SoQL). + +## Query Parameters + +| Param | Purpose | Example | +|-------|---------|---------| +| `$select` | Columns/expressions to return | `$select=date, primary_type, count(*) as total` | +| `$where` | Filter rows | `$where=year = 2024 AND ward = 42` | +| `$group` | Group for aggregation | `$group=primary_type` | +| `$having` | Filter aggregated results | `$having=count(*) > 100` | +| `$order` | Sort results | `$order=date DESC` | +| `$limit` | Max rows (default 1000, max 50000) | `$limit=5000` | +| `$offset` | Skip rows for pagination | `$offset=1000` | + +--- + +## Comparison Operators + +| Operator | Example | +|----------|---------| +| `=` | `ward = 42` | +| `!=` or `<>` | `status != 'CLOSED'` | +| `<`, `>`, `<=`, `>=` | `total_fee > 1000` | +| `IS NULL` | `latitude IS NULL` | +| `IS NOT NULL` | `latitude IS NOT NULL` | +| `IN (...)` | `primary_type IN ('THEFT', 'BATTERY')` | +| `NOT IN (...)` | `ward NOT IN (1, 2, 3)` | +| `BETWEEN` | `fee BETWEEN 100 AND 500` | + +## Logical Operators + +```sql +$where=year = 2024 AND ward = 42 +$where=primary_type = 'THEFT' OR primary_type = 'BATTERY' +$where=NOT (status = 'CLOSED') +$where=(ward = 1 OR ward = 2) AND year = 2024 +``` + +--- + ## Aggregate Functions + | Function | Example | |----------|---------| | `count(*)` | `$select=count(*) as total` | -| `sum(column)` | `$select=sum(amount) as total_amount` | +| `count(column)` | `$select=count(arrest) as arrest_count` | +| `sum(column)` | `$select=sum(total_fee) as total_fees` | | `avg(column)` | `$select=avg(fee) as avg_fee` | | `min(column)` | `$select=min(date) as earliest` | | `max(column)` | `$select=max(date) as latest` | | `stddev_pop(column)` | Population standard deviation | | `stddev_samp(column)` | Sample standard deviation | +**Aggregation example:** +``` +$select=primary_type, count(*) as total, avg(community_area) as avg_area +&$group=primary_type +&$having=count(*) > 1000 +&$order=total DESC +``` + +--- + ## Date Functions -| Function | Example | Returns | + +### Extract Parts + +| Function | Returns | Example | |----------|---------|---------| -| `date_extract_y(col)` | `date_extract_y(date)` | Year (2024) | -| `date_extract_m(col)` | `date_extract_m(date)` | Month (1-12) | -| `date_extract_d(col)` | `date_extract_d(date)` | Day (1-31) | -| `date_extract_dow(col)` | `date_extract_dow(date)` | Day of week (0=Sun) | -| `date_extract_hh(col)` | `date_extract_hh(date)` | Hour (0-23) | -| `date_trunc_y(col)` | `date_trunc_y(date)` | Truncate to year | -| `date_trunc_ym(col)` | `date_trunc_ym(date)` | Truncate to month | -| `date_trunc_ymd(col)` | `date_trunc_ymd(date)` | Truncate to day | - -**Date comparison:** +| `date_extract_y(col)` | Year (2024) | `date_extract_y(date) = 2024` | +| `date_extract_m(col)` | Month (1-12) | `date_extract_m(date) = 6` | +| `date_extract_d(col)` | Day (1-31) | `date_extract_d(date) = 15` | +| `date_extract_dow(col)` | Day of week (0=Sun, 6=Sat) | `date_extract_dow(date) = 0` | +| `date_extract_hh(col)` | Hour (0-23) | `date_extract_hh(date) = 14` | + +### Truncate for Grouping + +| Function | Truncates to | +|----------|--------------| +| `date_trunc_y(col)` | Year start (2024-01-01) | +| `date_trunc_ym(col)` | Month start (2024-06-01) | +| `date_trunc_ymd(col)` | Day start (2024-06-15) | + +**Group by month example:** +``` +$select=date_trunc_ym(date) as month, count(*) as total +&$group=date_trunc_ym(date) +&$order=month DESC +``` + +### Date Comparison + +Use ISO 8601 format with single quotes: + ```sql -$where=date >= '2024-01-01T00:00:00' AND date < '2025-01-01T00:00:00' +-- Exact date +$where=date = '2024-06-15' + +-- Date range +$where=date >= '2024-01-01' AND date < '2025-01-01' + +-- With time +$where=date >= '2024-01-01T00:00:00' AND date < '2024-01-02T00:00:00' + +-- Relative (this year) +$where=date_extract_y(date) = 2024 ``` +--- + ## Text Functions + | Function | Example | |----------|---------| | `upper(col)` | `upper(primary_type) = 'THEFT'` | | `lower(col)` | `lower(status) = 'open'` | -| `starts_with(col, str)` | `starts_with(address, '123')` | -| `like` | `address like '%MICHIGAN%'` | -| `not like` | `address not like '%APT%'` | +| `starts_with(col, str)` | `starts_with(address, '123 N')` | +| `contains(col, str)` | `contains(description, 'VEHICLE')` | +| `like` | `address LIKE '%MICHIGAN%'` | +| `not like` | `address NOT LIKE '%APT%'` | + +**LIKE wildcards:** +- `%` = any characters +- `_` = single character + +```sql +-- Addresses on Michigan Ave +$where=address LIKE '%MICHIGAN%' + +-- Starts with specific block +$where=address LIKE '100 N%' +``` + +--- ## Geospatial Functions + +See `references/geospatial.md` for detailed examples. + | Function | Syntax | |----------|--------| | `within_circle` | `within_circle(location, lat, lon, radius_meters)` | | `within_box` | `within_box(location, north_lat, west_lon, south_lat, east_lon)` | | `within_polygon` | `within_polygon(location, 'MULTIPOLYGON(...)')` | -| `distance_in_meters` | `distance_in_meters(location, 'POINT(-87.6 41.8)')` | +| `distance_in_meters` | `distance_in_meters(location, 'POINT(lon lat)')` | | `intersects` | `intersects(geometry, 'POLYGON(...)')` | -**Radius example (1km from downtown):** -```sql -$where=within_circle(location, 41.8781, -87.6298, 1000) -``` +--- -## Comparison Operators -| Operator | Example | -|----------|---------| -| `=` | `ward = 42` | -| `!=` or `<>` | `status != 'CLOSED'` | -| `<`, `>`, `<=`, `>=` | `total_fee > 1000` | -| `IS NULL` | `latitude IS NULL` | -| `IS NOT NULL` | `latitude IS NOT NULL` | -| `IN (...)` | `primary_type IN ('THEFT', 'BATTERY')` | -| `NOT IN (...)` | `ward NOT IN (1, 2, 3)` | -| `BETWEEN` | `fee BETWEEN 100 AND 500` | +## System Fields -## Logical Operators -```sql -$where=year = 2024 AND ward = 42 -$where=primary_type = 'THEFT' OR primary_type = 'BATTERY' -$where=NOT (status = 'CLOSED') -``` +Every record has hidden system fields: -## System Fields | Field | Description | |-------|-------------| -| `:id` | Internal record ID | -| `:created_at` | Row creation timestamp | -| `:updated_at` | Row last modified timestamp | +| `:id` | Internal row ID | +| `:created_at` | When row was created | +| `:updated_at` | When row was last modified | -**Include system fields:** -``` +```sql $select=:id, :updated_at, * +$where=:updated_at > '2024-01-01' +$order=:updated_at DESC ``` +--- + ## URL Encoding + +When building URLs, encode special characters: + | Character | Encoded | |-----------|---------| -| space | `%20` | +| space | `%20` or `+` | | `=` | `%3D` | | `>` | `%3E` | | `<` | `%3C` | | `'` | `%27` | | `,` | `%2C` | +| `(` | `%28` | +| `)` | `%29` | + +**Example:** +``` +# Human readable +$where=date >= '2024-01-01' AND primary_type = 'THEFT' + +# URL encoded +$where=date%20%3E=%20%272024-01-01%27%20AND%20primary_type%20=%20%27THEFT%27 +``` + +--- + +## Common Query Patterns + +### Recent records by date +``` +$order=date DESC&$limit=100 +``` + +### Count by category +``` +$select=primary_type, count(*) as total +&$group=primary_type +&$order=total DESC +``` + +### Count by time period +``` +$select=date_trunc_ym(date) as month, count(*) as total +&$where=date >= '2024-01-01' +&$group=date_trunc_ym(date) +&$order=month +``` + +### Filter + aggregate +``` +$select=ward, count(*) as total +&$where=primary_type = 'THEFT' AND date >= '2024-01-01' +&$group=ward +&$order=total DESC +``` + +### Pagination +``` +# Page 1 +$order=date DESC&$limit=1000&$offset=0 + +# Page 2 +$order=date DESC&$limit=1000&$offset=1000 +``` diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..55c6137 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,63 @@ +# Skill Tests + +Integration tests that verify the API examples in skills work against live endpoints. + +## Setup + +```bash +pip install -r tests/requirements.txt +``` + +## Running Tests + +```bash +# Run all tests +pytest tests/ -v + +# Run only Chicago data portal tests +pytest tests/test_chicago_data_portal.py -v + +# Skip live API tests (useful for CI without network) +pytest tests/ -m "not live" + +# Run with app token for higher rate limits +export CHICAGO_DATA_PORTAL_TOKEN="your_token" +pytest tests/test_chicago_data_portal.py -v +``` + +## Test Structure + +Each skill has a corresponding test file: + +| Skill | Test File | +|-------|-----------| +| chicago-data-portal | `test_chicago_data_portal.py` | +| cook-county-data-portal | `test_cook_county_data_portal.py` (planned) | +| us-census-data | `test_us_census_data.py` (planned) | + +## What's Tested + +Tests verify that the query patterns documented in skill examples actually work: + +- **Basic queries**: JSON/CSV responses, column existence +- **Metadata**: Dataset schema retrieval, column info +- **Filtering**: Date ranges, category filters, IN clauses +- **Aggregation**: GROUP BY, count, date_trunc functions +- **Geospatial**: within_circle, within_box queries +- **Pagination**: Offset-based paging, no overlap +- **Error handling**: Invalid datasets, bad column names + +## Markers + +- `@pytest.mark.live` - Tests that hit live APIs (may be slow, need network) +- `@pytest.mark.slow` - Long-running tests + +## Environment Variables + +| Variable | Purpose | +|----------|---------| +| `CHICAGO_DATA_PORTAL_TOKEN` | App token for higher rate limits | +| `COOK_COUNTY_DATA_PORTAL_TOKEN` | Cook County app token | +| `CENSUS_API_KEY` | US Census API key | + +Tests work without tokens but may hit rate limits on repeated runs. diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..43ba68a --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,31 @@ +"""Shared pytest fixtures and configuration for skill tests.""" + +import os + +import pytest + + +def pytest_configure(config): + """Register custom markers.""" + config.addinivalue_line("markers", "live: marks tests as requiring live API access") + config.addinivalue_line("markers", "slow: marks tests as slow-running") + + +@pytest.fixture +def chicago_app_token(): + """Get Chicago Data Portal app token from environment if available.""" + return os.getenv("CHICAGO_DATA_PORTAL_TOKEN") + + +@pytest.fixture +def cook_county_app_token(): + """Get Cook County Data Portal app token from environment if available.""" + return os.getenv("COOK_COUNTY_DATA_PORTAL_TOKEN") or os.getenv( + "CHICAGO_DATA_PORTAL_TOKEN" + ) + + +@pytest.fixture +def census_api_key(): + """Get Census API key from environment if available.""" + return os.getenv("CENSUS_API_KEY") diff --git a/tests/pytest.ini b/tests/pytest.ini new file mode 100644 index 0000000..6eb49b3 --- /dev/null +++ b/tests/pytest.ini @@ -0,0 +1,11 @@ +[pytest] +markers = + live: marks tests as requiring live API access (may be slow, requires network) + slow: marks tests as slow-running + +# Default: run all tests +# To skip live tests: pytest -m "not live" +# To run only live tests: pytest -m "live" + +# Show extra test summary for failures +addopts = --tb=short diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..235c9ce --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,7 @@ +# Test dependencies for claude-plugins skill tests + +pytest>=7.0.0 +requests>=2.28.0 + +# Optional but recommended +pytest-timeout>=2.0.0 # Prevent hung tests diff --git a/tests/test_chicago_data_portal.py b/tests/test_chicago_data_portal.py new file mode 100644 index 0000000..3cabb20 --- /dev/null +++ b/tests/test_chicago_data_portal.py @@ -0,0 +1,419 @@ +""" +Integration tests for Chicago Data Portal skill examples. + +These tests verify that the SODA API endpoints and query patterns +documented in the skill actually work against the live API. + +Run with: pytest tests/test_chicago_data_portal.py -v +Skip live tests: pytest tests/test_chicago_data_portal.py -v -m "not live" +""" + +import pytest +import requests + +# Base URLs +BASE_URL = "https://data.cityofchicago.org/resource" +METADATA_URL = "https://data.cityofchicago.org/api/views" +CATALOG_URL = "https://api.us.socrata.com/api/catalog/v1" + +# Dataset IDs +CRIMES_ID = "ijzp-q8t2" +SERVICE_REQUESTS_ID = "v6vf-nfxy" +BUILDING_PERMITS_ID = "ydr8-5enu" +FOOD_INSPECTIONS_ID = "4ijn-s7e5" + +# Test timeout +TIMEOUT = 30 + + +def get_headers(app_token: str | None) -> dict: + """Build request headers with optional app token.""" + if app_token: + return {"X-App-Token": app_token} + return {} + + +# ============================================================ +# Basic Query Tests +# ============================================================ + + +@pytest.mark.live +def test_basic_query_json(chicago_app_token): + """Test basic JSON query returns data.""" + url = f"{BASE_URL}/{CRIMES_ID}.json" + params = {"$limit": 5, "$order": "date DESC"} + + resp = requests.get( + url, params=params, headers=get_headers(chicago_app_token), timeout=TIMEOUT + ) + + assert resp.status_code == 200 + data = resp.json() + assert isinstance(data, list) + assert len(data) <= 5 + if data: + # Verify expected columns exist + assert "date" in data[0] + assert "primary_type" in data[0] + + +@pytest.mark.live +def test_basic_query_csv(chicago_app_token): + """Test basic CSV query returns data.""" + url = f"{BASE_URL}/{CRIMES_ID}.csv" + params = {"$limit": 5} + + resp = requests.get( + url, params=params, headers=get_headers(chicago_app_token), timeout=TIMEOUT + ) + + assert resp.status_code == 200 + assert "text/csv" in resp.headers.get("Content-Type", "") + # CSV should have header row + data rows + lines = resp.text.strip().split("\n") + assert len(lines) >= 1 # At least header + + +# ============================================================ +# Metadata Tests +# ============================================================ + + +@pytest.mark.live +def test_get_dataset_metadata(chicago_app_token): + """Test fetching dataset metadata including columns.""" + url = f"{METADATA_URL}/{CRIMES_ID}" + + resp = requests.get(url, headers=get_headers(chicago_app_token), timeout=TIMEOUT) + + assert resp.status_code == 200 + meta = resp.json() + + # Verify structure + assert "name" in meta + assert "columns" in meta + assert isinstance(meta["columns"], list) + + # Verify we can extract column info + column_names = [col["fieldName"] for col in meta["columns"]] + assert "date" in column_names + assert "primary_type" in column_names + + +@pytest.mark.live +def test_catalog_search(chicago_app_token): + """Test searching the catalog API.""" + params = {"domains": "data.cityofchicago.org", "q": "crimes"} + + resp = requests.get( + CATALOG_URL, params=params, headers=get_headers(chicago_app_token), timeout=TIMEOUT + ) + + assert resp.status_code == 200 + data = resp.json() + + # Should return results + assert "results" in data + assert len(data["results"]) > 0 + + +# ============================================================ +# Filter Tests +# ============================================================ + + +@pytest.mark.live +def test_where_filter_date(chicago_app_token): + """Test date filtering with $where clause.""" + url = f"{BASE_URL}/{CRIMES_ID}.json" + params = { + "$where": "date >= '2024-01-01'", + "$limit": 5, + "$order": "date DESC", + } + + resp = requests.get( + url, params=params, headers=get_headers(chicago_app_token), timeout=TIMEOUT + ) + + assert resp.status_code == 200 + data = resp.json() + assert isinstance(data, list) + # All dates should be >= 2024-01-01 + for row in data: + assert row["date"] >= "2024-01-01" + + +@pytest.mark.live +def test_where_filter_type(chicago_app_token): + """Test filtering by category.""" + url = f"{BASE_URL}/{CRIMES_ID}.json" + params = { + "$where": "primary_type = 'THEFT'", + "$limit": 10, + } + + resp = requests.get( + url, params=params, headers=get_headers(chicago_app_token), timeout=TIMEOUT + ) + + assert resp.status_code == 200 + data = resp.json() + for row in data: + assert row["primary_type"] == "THEFT" + + +@pytest.mark.live +def test_where_filter_in_clause(chicago_app_token): + """Test IN clause for multiple values.""" + url = f"{BASE_URL}/{CRIMES_ID}.json" + params = { + "$where": "primary_type IN ('THEFT', 'BATTERY', 'ASSAULT')", + "$limit": 20, + } + + resp = requests.get( + url, params=params, headers=get_headers(chicago_app_token), timeout=TIMEOUT + ) + + assert resp.status_code == 200 + data = resp.json() + for row in data: + assert row["primary_type"] in ("THEFT", "BATTERY", "ASSAULT") + + +# ============================================================ +# Aggregation Tests +# ============================================================ + + +@pytest.mark.live +def test_aggregation_count_by_type(chicago_app_token): + """Test GROUP BY aggregation.""" + url = f"{BASE_URL}/{CRIMES_ID}.json" + params = { + "$select": "primary_type, count(*) as total", + "$group": "primary_type", + "$order": "total DESC", + "$limit": 10, + } + + resp = requests.get( + url, params=params, headers=get_headers(chicago_app_token), timeout=TIMEOUT + ) + + assert resp.status_code == 200 + data = resp.json() + assert len(data) > 0 + + # Each row should have aggregated fields + for row in data: + assert "primary_type" in row + assert "total" in row + assert int(row["total"]) > 0 + + +@pytest.mark.live +def test_aggregation_date_trunc(chicago_app_token): + """Test date_trunc_ym for monthly aggregation.""" + url = f"{BASE_URL}/{CRIMES_ID}.json" + params = { + "$select": "date_trunc_ym(date) as month, count(*) as total", + "$where": "date >= '2024-01-01'", + "$group": "date_trunc_ym(date)", + "$order": "month", + "$limit": 12, + } + + resp = requests.get( + url, params=params, headers=get_headers(chicago_app_token), timeout=TIMEOUT + ) + + assert resp.status_code == 200 + data = resp.json() + assert len(data) > 0 + + for row in data: + assert "month" in row + assert "total" in row + + +# ============================================================ +# Geospatial Tests +# ============================================================ + + +@pytest.mark.live +def test_geospatial_within_circle(chicago_app_token): + """Test within_circle geospatial query (downtown Chicago).""" + url = f"{BASE_URL}/{CRIMES_ID}.json" + # Downtown Chicago: 41.8781, -87.6298, 500m radius + params = { + "$where": "within_circle(location, 41.8781, -87.6298, 500)", + "$limit": 10, + } + + resp = requests.get( + url, params=params, headers=get_headers(chicago_app_token), timeout=TIMEOUT + ) + + assert resp.status_code == 200 + data = resp.json() + # Should find some crimes in downtown (high-traffic area) + assert isinstance(data, list) + + +@pytest.mark.live +def test_geospatial_within_box(chicago_app_token): + """Test within_box geospatial query (The Loop).""" + url = f"{BASE_URL}/{CRIMES_ID}.json" + # The Loop bounding box + params = { + "$where": "within_box(location, 41.887, -87.6425, 41.875, -87.619)", + "$limit": 10, + } + + resp = requests.get( + url, params=params, headers=get_headers(chicago_app_token), timeout=TIMEOUT + ) + + assert resp.status_code == 200 + data = resp.json() + assert isinstance(data, list) + + +# ============================================================ +# Pagination Tests +# ============================================================ + + +@pytest.mark.live +def test_pagination_offset(chicago_app_token): + """Test pagination with limit and offset.""" + url = f"{BASE_URL}/{CRIMES_ID}.json" + + # Get page 1 + params_p1 = {"$order": "date DESC", "$limit": 5, "$offset": 0} + resp1 = requests.get( + url, params=params_p1, headers=get_headers(chicago_app_token), timeout=TIMEOUT + ) + + # Get page 2 + params_p2 = {"$order": "date DESC", "$limit": 5, "$offset": 5} + resp2 = requests.get( + url, params=params_p2, headers=get_headers(chicago_app_token), timeout=TIMEOUT + ) + + assert resp1.status_code == 200 + assert resp2.status_code == 200 + + data1 = resp1.json() + data2 = resp2.json() + + # Pages should have different data (no overlap) + if data1 and data2: + ids1 = {row.get("id") for row in data1} + ids2 = {row.get("id") for row in data2} + # Should have no overlap + assert ids1.isdisjoint(ids2) + + +# ============================================================ +# Multi-Dataset Tests +# ============================================================ + + +@pytest.mark.live +def test_311_service_requests(chicago_app_token): + """Test 311 service requests dataset.""" + url = f"{BASE_URL}/{SERVICE_REQUESTS_ID}.json" + params = { + "$select": "sr_type, count(*) as total", + "$where": "created_date >= '2024-01-01'", + "$group": "sr_type", + "$order": "total DESC", + "$limit": 5, + } + + resp = requests.get( + url, params=params, headers=get_headers(chicago_app_token), timeout=TIMEOUT + ) + + assert resp.status_code == 200 + data = resp.json() + assert len(data) > 0 + assert "sr_type" in data[0] + + +@pytest.mark.live +def test_building_permits(chicago_app_token): + """Test building permits dataset.""" + url = f"{BASE_URL}/{BUILDING_PERMITS_ID}.json" + params = { + "$order": "issue_date DESC", + "$limit": 5, + } + + resp = requests.get( + url, params=params, headers=get_headers(chicago_app_token), timeout=TIMEOUT + ) + + assert resp.status_code == 200 + data = resp.json() + assert isinstance(data, list) + if data: + assert "permit_type" in data[0] or "permit_" in data[0] + + +@pytest.mark.live +def test_food_inspections(chicago_app_token): + """Test food inspections dataset.""" + url = f"{BASE_URL}/{FOOD_INSPECTIONS_ID}.json" + params = { + "$select": "results, count(*) as total", + "$where": "inspection_date >= '2024-01-01'", + "$group": "results", + "$order": "total DESC", + } + + resp = requests.get( + url, params=params, headers=get_headers(chicago_app_token), timeout=TIMEOUT + ) + + assert resp.status_code == 200 + data = resp.json() + assert len(data) > 0 + + # Should have common results + result_types = {row["results"] for row in data} + assert "Pass" in result_types or "Fail" in result_types + + +# ============================================================ +# Error Handling Tests +# ============================================================ + + +@pytest.mark.live +def test_invalid_dataset_returns_404(chicago_app_token): + """Test that invalid dataset ID returns 404.""" + url = f"{BASE_URL}/invalid-dataset-id.json" + + resp = requests.get(url, headers=get_headers(chicago_app_token), timeout=TIMEOUT) + + assert resp.status_code == 404 + + +@pytest.mark.live +def test_invalid_column_returns_error(chicago_app_token): + """Test that invalid column name returns error.""" + url = f"{BASE_URL}/{CRIMES_ID}.json" + params = {"$where": "nonexistent_column = 'value'"} + + resp = requests.get( + url, params=params, headers=get_headers(chicago_app_token), timeout=TIMEOUT + ) + + # Should return 400 Bad Request for invalid column + assert resp.status_code in (400, 404)