Skip to content

fix: polymarket deprecated endpoint update #233

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
nikbpetrov wants to merge 2 commits into
base: main
Choose a base branch
from
+318 −86
Open

fix: polymarket deprecated endpoint update #233

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4e7ebaf
fix: polymarket deprecated endpoint update
nikbpetrov Jul 10, 2026
df31aef
fix: exclude fetched polymarket Qs that cannot be used before deadline
nikbpetrov Jul 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/orchestration/func_polymarket_fetch/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ requests
certifi
backoff
numpy
scipy
1 change: 1 addition & 0 deletions src/orchestration/func_polymarket_update/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ requests
certifi
backoff
numpy
scipy
202 changes: 146 additions & 56 deletions src/sources/polymarket.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import logging
import os
import time
from datetime import datetime, timedelta
from datetime import date, datetime, timedelta
from typing import Any, ClassVar

import backoff
Expand All @@ -19,15 +19,16 @@

from _fb_types import UpdateResult
from _schemas import PolymarketFetchFrame, QuestionFrame, ResolutionFrame
from helpers import constants, data_utils, dates
from helpers import constants, data_utils, dates, question_curation

from ._market import MarketSource

logger = logging.getLogger(__name__)

_GAMMA_API_URL = "https://gamma-api.polymarket.com/markets"
_GAMMA_API_URL = "https://gamma-api.polymarket.com/markets/keyset"
_CLOB_API_URL = "https://clob.polymarket.com/prices-history"
_MIN_MARKET_LIQUIDITY = 25000
_REQUEST_TIMEOUT_SECONDS = 30

# Set CHECK_AND_FIX_RESOLVED_DATA=1 to re-fetch resolved questions whose resolution files are
# missing or have non-contiguous dates. This needs every resolved file downloaded, so it's costly
Expand Down Expand Up @@ -225,69 +226,104 @@ def update(
# Private: API calls
# ------------------------------------------------------------------

@backoff.on_exception(
backoff.expo,
requests.exceptions.RequestException,
max_time=20,
on_backoff=data_utils.print_error_info_handler,
)
def _fetch_active_markets_from_api(self) -> list[dict]:
Comment thread
nikbpetrov marked this conversation as resolved.
"""Fetch active binary markets from the Gamma API with price history attached.

Paginates through all active, non-archived, non-closed markets ordered by liquidity,
keeps binary markets with sufficient liquidity that aren't catch-all ("other") markets,
and attaches each qualifying market's price history.
Paginates through active, non-archived, non-closed markets above the liquidity floor,
keeps the binary ones that aren't catch-all ("other") markets,
and attaches each qualifying market's price history. A transient request failure retries
the whole paginated fetch via backoff rather than returning a truncated result.
Comment on lines +240 to +241

@coderabbitai coderabbitai Bot Jul 10, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🗄️ Data Integrity & Integration | 🟠 Major | ⚡ Quick win

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
set -euo pipefail

printf '\n== outline ==\n'
ast-grep outline src/sources/polymarket.py --view expanded || true

printf '\n== search symbols ==\n'
rg -n "_fetch_price_history|backoff|test_skips_when_price_history_none|RequestException|price history" src tests || true

printf '\n== line ranges ==\n'
nl -ba src/sources/polymarket.py | sed -n '210,330p'

Repository: forecastingresearch/forecastbench

Length of output: 11274

🏁 Script executed:

#!/bin/bash
set -euo pipefail

printf '\n== polymarket slices ==\n'
sed -n '229,310p' src/sources/polymarket.py
printf '\n== fetch_price_history slice ==\n'
sed -n '348,398p' src/sources/polymarket.py
printf '\n== relevant tests slice ==\n'
sed -n '430,530p' src/tests/test_polymarket.py

Repository: forecastingresearch/forecastbench

Length of output: 10022

🏁 Script executed:

#!/bin/bash
set -euo pipefail

sed -n '130,175p' src/sources/polymarket.py
sed -n '175,235p' src/sources/polymarket.py

Repository: forecastingresearch/forecastbench

Length of output: 4660

Do not swallow price-history failures

_fetch_price_history catches RequestException and returns None, so its backoff wrapper never retries. In _fetch_active_markets_from_api, that None just drops the market, which leaves the fetched set incomplete instead of retrying the failing request. Re-raise a retryable exception here, or handle None as a hard failure, and update test_skips_when_price_history_none accordingly.

🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/sources/polymarket.py` around lines 240 - 241, Ensure
_fetch_price_history does not swallow RequestException: re-raise a retryable
exception so its backoff wrapper retries, or make _fetch_active_markets_from_api
treat a None result as a hard failure instead of silently dropping the market.
Update test_skips_when_price_history_none to assert the revised failure and
retry behavior.

"""
all_markets: list[dict] = []
offset = 0
limit = 500 # max page size: 500
n_markets_fetched = 0
after_cursor: str | None = None
seen_ids: set[str] = set()

# Markets that resolve within the freeze window can never appear on a question set (it's
# published FREEZE_WINDOW_IN_DAYS before forecasts are due), so drop them at fetch time.
min_resolution_date = dates.get_date_today() + timedelta(
days=question_curation.FREEZE_WINDOW_IN_DAYS
)

# https://docs.polymarket.com/api-reference/markets/list-markets-keyset-pagination
# /markets/keyset uses opaque cursor pagination
# Note that docs for `order` param are wrong; they specify:
# "Comma-separated list of JSON field names to order by, e.g. volume_num,liquidity_num"
# but those snake_case fields are ignored; camelCase is correct
# Also, liquidity field == liquidityNum (former is forced to a number)
params: dict[str, Any] = {
"limit": limit,
"limit": 100, # limit as per API docs
"archived": False,
"active": True,
"closed": False,
"order": "liquidity",
"order": "liquidityNum",
"ascending": False,
Comment thread
nikbpetrov marked this conversation as resolved.
Comment on lines +265 to 266

@coderabbitai coderabbitai Bot Jul 10, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🗄️ Data Integrity & Integration | 🟠 Major | ⚡ Quick win

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
set -euo pipefail

# Inspect the target file around the cited lines and relevant helpers.
python3 - <<'PY'
from pathlib import Path
path = Path("src/sources/polymarket.py")
lines = path.read_text().splitlines()
for start, end in [(1, 360)]:
    for i in range(start-1, min(end, len(lines))):
        if 220 <= i+1 <= 340:
            print(f"{i+1:4d}: {lines[i]}")
PY

Repository: forecastingresearch/forecastbench

Length of output: 6231

🏁 Script executed:

#!/bin/bash
set -euo pipefail

# Map nearby symbols to focus on pagination and retry behavior.
ast-grep outline src/sources/polymarket.py --view expanded

Repository: forecastingresearch/forecastbench

Length of output: 1368

🏁 Script executed:

#!/bin/bash
set -euo pipefail

# Inspect price-history handling and surrounding retry logic.
python3 - <<'PY'
from pathlib import Path
path = Path("src/sources/polymarket.py")
lines = path.read_text().splitlines()
for start, end in [(350, 390)]:
    for i in range(start-1, min(end, len(lines))):
        print(f"{i+1:4d}: {lines[i]}")
PY

Repository: forecastingresearch/forecastbench

Length of output: 1947

🏁 Script executed:

#!/bin/bash
set -euo pipefail

# Continue to the exception handling and any caller behavior.
python3 - <<'PY'
from pathlib import Path
path = Path("src/sources/polymarket.py")
lines = path.read_text().splitlines()
for start, end in [(389, 405)]:
    for i in range(start-1, min(end, len(lines))):
        print(f"{i+1:4d}: {lines[i]}")
PY

Repository: forecastingresearch/forecastbench

Length of output: 1035

🏁 Script executed:

#!/bin/bash
set -euo pipefail

# Inspect the rest of the price-history exception path.
python3 - <<'PY'
from pathlib import Path
path = Path("src/sources/polymarket.py")
lines = path.read_text().splitlines()
for start, end in [(389, 405)]:
    for i in range(start-1, min(end, len(lines))):
        print(f"{i+1:4d}: {lines[i]}")
PY

Repository: forecastingresearch/forecastbench

Length of output: 1035

🏁 Script executed:

#!/bin/bash
set -euo pipefail

# Inspect the remainder of the price-history exception path to see how failures are surfaced.
python3 - <<'PY'
from pathlib import Path
path = Path("src/sources/polymarket.py")
lines = path.read_text().splitlines()
for start, end in [(389, 410)]:
    for i in range(start-1, min(end, len(lines))):
        print(f"{i+1:4d}: {lines[i]}")
PY

Repository: forecastingresearch/forecastbench

Length of output: 1257

🌐 Web query:

Polymarket Gamma API markets keyset pagination order after_cursor liquidityNum docs

💡 Result:

The Polymarket Gamma API utilizes keyset (cursor-based) pagination for efficient data retrieval, explicitly rejecting offset-based pagination [1][2][3]. Keyset Pagination Mechanics: - Use the after_cursor query parameter to request subsequent pages [1][3]. - The API response contains a next_cursor field; pass this value into the after_cursor parameter of the next request [1][3]. - Note: Offset-based pagination is not supported and will return an HTTP 422 error [1][3]. - Technical reports as of April 2026 have noted potential issues where the keyset endpoint may fail to advance past the first page when the cursor parameter is provided [4]. Developers are encouraged to use the provided TypeScript or Python SDKs, which abstract these pagination patterns [5][6]. LiquidityNum: - liquidityNum represents a market's liquidity expressed as a numerical value [7][8][9]. - It is available as a filterable field in various endpoints (e.g., liquidity_num_min, liquidity_num_max) to refine market search results [10][9]. General API Tips: - The Gamma API base URL is https://gamma-api.polymarket.com [3]. - SDKs for TypeScript and Python are available to simplify handling of pagination, typing, and request flows [5][6].

Citations:

Avoid paginating on mutable liquidity. A market can move across the cursor between requests and be skipped entirely; the conditionId dedupe only removes repeats, not misses. Also, _fetch_price_history catches RequestException and returns None, so the backoff wrapper never retries and transient CLOB failures silently drop markets from the output.

"liquidity_num_min": _MIN_MARKET_LIQUIDITY,
}

while True:
params["offset"] = offset
try:
logger.info(f"Fetching markets with offset {offset}.")
response = requests.get(_GAMMA_API_URL, params=params)
response.raise_for_status()
markets = response.json()
if not markets:
logger.info(
f"Fetched total of {n_markets_fetched} markets, "
f"{len(all_markets)} satisfy criteria."
)
break

n_markets_fetched += len(markets)
for market in markets:
binary_market = self._is_market_binary(market)
# Avoids questions like the following, which don't make sense without the other
# questions in the event:
# * Will any other Republican Politician win the popular vote in the 2024
# Presidential Election?
catch_all_market = "other" in market["slug"] # no need to test "another" also
liquid_market = (
"liquidityNum" in market.keys()
and market["liquidityNum"] > _MIN_MARKET_LIQUIDITY
)
if binary_market and liquid_market and not catch_all_market:
price_history = self._fetch_price_history(self._get_yes_token(market))
if price_history is not None:
logger.info(
"Binary question satisfying criteria: "
f"https://polymarket.com/market/{market['slug']}"
)
market["price_history"] = price_history
all_markets.append(market)

except requests.exceptions.RequestException as e:
logger.error(f"Error fetching markets: {e}")
if after_cursor:
params["after_cursor"] = after_cursor
logger.info(f"Fetching markets (cursor={after_cursor}).")
response = requests.get(_GAMMA_API_URL, params=params, timeout=_REQUEST_TIMEOUT_SECONDS)
response.raise_for_status()
payload = response.json()
markets = payload.get("markets", [])

n_markets_fetched += len(markets)
for market in markets:
# Keyset orders by liquidity, which changes as trades land, so a market
# can shift across the cursor and recur on a later page; dedupe by id.
condition_id = market["conditionId"]
if condition_id in seen_ids:
continue
seen_ids.add(condition_id)
binary_market = self._is_market_binary(market)
# Avoids questions like the following, which don't make sense without the other
# questions in the event:
# * Will any other Republican Politician win the popular vote in the 2024
# Presidential Election?
catch_all_market = "other" in market["slug"] # no need to test "another" also
liquid_market = (
"liquidityNum" in market.keys()
and market["liquidityNum"] > _MIN_MARKET_LIQUIDITY
)
resolves_too_soon = self._resolves_before_forecast_window(
market, min_resolution_date
)
if (
binary_market
and liquid_market
and not catch_all_market
and not resolves_too_soon
):
price_history = self._fetch_price_history(self._get_yes_token(market))
if price_history is not None:
logger.info(
"Binary question satisfying criteria: "
f"https://polymarket.com/market/{market['slug']}"
)
market["price_history"] = price_history
all_markets.append(market)

next_cursor = payload.get("next_cursor")

if not next_cursor:
logger.info(
f"Fetched total of {n_markets_fetched} markets, "
f"{len(all_markets)} satisfy criteria."
)
break

time.sleep(1)
offset += limit
after_cursor = next_cursor
# cap at ~20 req/s, under the /markets limit of 300 req/10s (30 req/s)
# https://docs.polymarket.com/api-reference/rate-limits
time.sleep(0.05)

return all_markets

Expand All @@ -313,9 +349,11 @@ def _get_market(self, condition_id: str) -> dict:
{"condition_ids": condition_id, "closed": False},
{"condition_ids": condition_id, "closed": True},
]:
response = requests.get(_GAMMA_API_URL, params=params_market)
response = requests.get(
_GAMMA_API_URL, params=params_market, timeout=_REQUEST_TIMEOUT_SECONDS
)
response.raise_for_status()
markets = response.json()
markets = response.json().get("markets", [])
if len(markets) == 1:
return markets[0]
logger.error(f"Problem getting market for condition id {condition_id}.")
Expand Down Expand Up @@ -346,7 +384,12 @@ def _fetch_price_history(self, market_id: str) -> list[dict] | None:
}

try:
response = requests.get(_CLOB_API_URL, params=params, verify=certifi.where())
response = requests.get(
_CLOB_API_URL,
params=params,
verify=certifi.where(),
timeout=_REQUEST_TIMEOUT_SECONDS,
)
if not response.ok:
logger.error(
f"Request to endpoint failed for {_CLOB_API_URL}: "
Expand Down Expand Up @@ -381,6 +424,54 @@ def _get_yes_token(market: dict) -> str:
yes_token_index = PolymarketSource._get_yes_index(market)
return json.loads(market["clobTokenIds"])[yes_token_index]

@staticmethod
def _get_market_end_date_str(market: dict) -> str | None:
"""Return the market's raw Zulu close-date string, or ``None`` if unavailable.

Prefers the market's own ``endDate`` and falls back to its first event's ``endDate``. The
Gamma API omits the market-level ``endDate`` on some markets (e.g. season/futures markets
grouped only under an event), hence the event fallback.

Args:
market (dict): Raw Gamma API market.
"""
end_date = market.get("endDate")
if not end_date:
events = market.get("events") or []
end_date = events[0].get("endDate") if events else None
return end_date or None

@staticmethod
def _get_market_close_date(market: dict) -> date | None:
"""Return the market's scheduled close date, or ``None`` if it can't be determined.

Args:
market (dict): Raw Gamma API market.
"""
end_date = PolymarketSource._get_market_end_date_str(market)
if end_date is None:
return None
close_datetime_str = dates.convert_zulu_to_iso(end_date)
return datetime.fromisoformat(close_datetime_str).replace(tzinfo=None).date()

@staticmethod
def _resolves_before_forecast_window(market: dict, min_resolution_date: date) -> bool:
"""Return True if the market resolves too soon to appear on a question set.

A question set is published ``FREEZE_WINDOW_IN_DAYS`` before forecasts are due, so any
market closing on or before ``min_resolution_date`` (``today + FREEZE_WINDOW_IN_DAYS``) can
never be forecast and is dropped at fetch time. Markets with no discoverable close date are
kept here and left for ``_transform_question`` to drop.

Args:
market (dict): Raw Gamma API market.
min_resolution_date (date): Earliest close date a market may have and still be usable.
"""
close_date = PolymarketSource._get_market_close_date(market)
if close_date is None:
return False
return close_date <= min_resolution_date

# ------------------------------------------------------------------
# Private: price history helpers
# ------------------------------------------------------------------
Expand Down Expand Up @@ -456,9 +547,8 @@ def _transform_question(
current_prob = price_history[-1]["p"] if len(price_history) > 1 else np.nan
resolved_datetime = resolved_datetime_str = "N/A"

try:
end_date = market["endDate"] if "endDate" in market else market["events"][0]["endDate"]
except KeyError:
end_date = PolymarketSource._get_market_end_date_str(market)
if end_date is None:
# endDate unexpectedly missing from:
# https://polymarket.com/event/will-trump-meet-with-khamenei-before-august
return None
Expand Down
Loading
Loading