diff --git a/src/orchestration/func_polymarket_fetch/requirements.txt b/src/orchestration/func_polymarket_fetch/requirements.txt index ecae5538..05fb7afc 100644 --- a/src/orchestration/func_polymarket_fetch/requirements.txt +++ b/src/orchestration/func_polymarket_fetch/requirements.txt @@ -5,3 +5,4 @@ requests certifi backoff numpy +scipy \ No newline at end of file diff --git a/src/orchestration/func_polymarket_update/requirements.txt b/src/orchestration/func_polymarket_update/requirements.txt index ecae5538..576fab5e 100644 --- a/src/orchestration/func_polymarket_update/requirements.txt +++ b/src/orchestration/func_polymarket_update/requirements.txt @@ -5,3 +5,4 @@ requests certifi backoff numpy +scipy diff --git a/src/sources/polymarket.py b/src/sources/polymarket.py index 6cd0ce11..835c9c6a 100644 --- a/src/sources/polymarket.py +++ b/src/sources/polymarket.py @@ -6,7 +6,7 @@ import logging import os import time -from datetime import datetime, timedelta +from datetime import date, datetime, timedelta from typing import Any, ClassVar import backoff @@ -19,15 +19,16 @@ from _fb_types import UpdateResult from _schemas import PolymarketFetchFrame, QuestionFrame, ResolutionFrame -from helpers import constants, data_utils, dates +from helpers import constants, data_utils, dates, question_curation from ._market import MarketSource logger = logging.getLogger(__name__) -_GAMMA_API_URL = "https://gamma-api.polymarket.com/markets" +_GAMMA_API_URL = "https://gamma-api.polymarket.com/markets/keyset" _CLOB_API_URL = "https://clob.polymarket.com/prices-history" _MIN_MARKET_LIQUIDITY = 25000 +_REQUEST_TIMEOUT_SECONDS = 30 # Set CHECK_AND_FIX_RESOLVED_DATA=1 to re-fetch resolved questions whose resolution files are # missing or have non-contiguous dates. This needs every resolved file downloaded, so it's costly @@ -225,69 +226,104 @@ def update( # Private: API calls # ------------------------------------------------------------------ + @backoff.on_exception( + backoff.expo, + requests.exceptions.RequestException, + max_time=20, + on_backoff=data_utils.print_error_info_handler, + ) def _fetch_active_markets_from_api(self) -> list[dict]: """Fetch active binary markets from the Gamma API with price history attached. - Paginates through all active, non-archived, non-closed markets ordered by liquidity, - keeps binary markets with sufficient liquidity that aren't catch-all ("other") markets, - and attaches each qualifying market's price history. + Paginates through active, non-archived, non-closed markets above the liquidity floor, + keeps the binary ones that aren't catch-all ("other") markets, + and attaches each qualifying market's price history. A transient request failure retries + the whole paginated fetch via backoff rather than returning a truncated result. """ all_markets: list[dict] = [] - offset = 0 - limit = 500 # max page size: 500 n_markets_fetched = 0 + after_cursor: str | None = None + seen_ids: set[str] = set() + + # Markets that resolve within the freeze window can never appear on a question set (it's + # published FREEZE_WINDOW_IN_DAYS before forecasts are due), so drop them at fetch time. + min_resolution_date = dates.get_date_today() + timedelta( + days=question_curation.FREEZE_WINDOW_IN_DAYS + ) + # https://docs.polymarket.com/api-reference/markets/list-markets-keyset-pagination + # /markets/keyset uses opaque cursor pagination + # Note that docs for `order` param are wrong; they specify: + # "Comma-separated list of JSON field names to order by, e.g. volume_num,liquidity_num" + # but those snake_case fields are ignored; camelCase is correct + # Also, liquidity field == liquidityNum (former is forced to a number) params: dict[str, Any] = { - "limit": limit, + "limit": 100, # limit as per API docs "archived": False, "active": True, "closed": False, - "order": "liquidity", + "order": "liquidityNum", "ascending": False, + "liquidity_num_min": _MIN_MARKET_LIQUIDITY, } while True: - params["offset"] = offset - try: - logger.info(f"Fetching markets with offset {offset}.") - response = requests.get(_GAMMA_API_URL, params=params) - response.raise_for_status() - markets = response.json() - if not markets: - logger.info( - f"Fetched total of {n_markets_fetched} markets, " - f"{len(all_markets)} satisfy criteria." - ) - break - - n_markets_fetched += len(markets) - for market in markets: - binary_market = self._is_market_binary(market) - # Avoids questions like the following, which don't make sense without the other - # questions in the event: - # * Will any other Republican Politician win the popular vote in the 2024 - # Presidential Election? - catch_all_market = "other" in market["slug"] # no need to test "another" also - liquid_market = ( - "liquidityNum" in market.keys() - and market["liquidityNum"] > _MIN_MARKET_LIQUIDITY - ) - if binary_market and liquid_market and not catch_all_market: - price_history = self._fetch_price_history(self._get_yes_token(market)) - if price_history is not None: - logger.info( - "Binary question satisfying criteria: " - f"https://polymarket.com/market/{market['slug']}" - ) - market["price_history"] = price_history - all_markets.append(market) - - except requests.exceptions.RequestException as e: - logger.error(f"Error fetching markets: {e}") + if after_cursor: + params["after_cursor"] = after_cursor + logger.info(f"Fetching markets (cursor={after_cursor}).") + response = requests.get(_GAMMA_API_URL, params=params, timeout=_REQUEST_TIMEOUT_SECONDS) + response.raise_for_status() + payload = response.json() + markets = payload.get("markets", []) + + n_markets_fetched += len(markets) + for market in markets: + # Keyset orders by liquidity, which changes as trades land, so a market + # can shift across the cursor and recur on a later page; dedupe by id. + condition_id = market["conditionId"] + if condition_id in seen_ids: + continue + seen_ids.add(condition_id) + binary_market = self._is_market_binary(market) + # Avoids questions like the following, which don't make sense without the other + # questions in the event: + # * Will any other Republican Politician win the popular vote in the 2024 + # Presidential Election? + catch_all_market = "other" in market["slug"] # no need to test "another" also + liquid_market = ( + "liquidityNum" in market.keys() + and market["liquidityNum"] > _MIN_MARKET_LIQUIDITY + ) + resolves_too_soon = self._resolves_before_forecast_window( + market, min_resolution_date + ) + if ( + binary_market + and liquid_market + and not catch_all_market + and not resolves_too_soon + ): + price_history = self._fetch_price_history(self._get_yes_token(market)) + if price_history is not None: + logger.info( + "Binary question satisfying criteria: " + f"https://polymarket.com/market/{market['slug']}" + ) + market["price_history"] = price_history + all_markets.append(market) + + next_cursor = payload.get("next_cursor") + + if not next_cursor: + logger.info( + f"Fetched total of {n_markets_fetched} markets, " + f"{len(all_markets)} satisfy criteria." + ) break - - time.sleep(1) - offset += limit + after_cursor = next_cursor + # cap at ~20 req/s, under the /markets limit of 300 req/10s (30 req/s) + # https://docs.polymarket.com/api-reference/rate-limits + time.sleep(0.05) return all_markets @@ -313,9 +349,11 @@ def _get_market(self, condition_id: str) -> dict: {"condition_ids": condition_id, "closed": False}, {"condition_ids": condition_id, "closed": True}, ]: - response = requests.get(_GAMMA_API_URL, params=params_market) + response = requests.get( + _GAMMA_API_URL, params=params_market, timeout=_REQUEST_TIMEOUT_SECONDS + ) response.raise_for_status() - markets = response.json() + markets = response.json().get("markets", []) if len(markets) == 1: return markets[0] logger.error(f"Problem getting market for condition id {condition_id}.") @@ -346,7 +384,12 @@ def _fetch_price_history(self, market_id: str) -> list[dict] | None: } try: - response = requests.get(_CLOB_API_URL, params=params, verify=certifi.where()) + response = requests.get( + _CLOB_API_URL, + params=params, + verify=certifi.where(), + timeout=_REQUEST_TIMEOUT_SECONDS, + ) if not response.ok: logger.error( f"Request to endpoint failed for {_CLOB_API_URL}: " @@ -381,6 +424,54 @@ def _get_yes_token(market: dict) -> str: yes_token_index = PolymarketSource._get_yes_index(market) return json.loads(market["clobTokenIds"])[yes_token_index] + @staticmethod + def _get_market_end_date_str(market: dict) -> str | None: + """Return the market's raw Zulu close-date string, or ``None`` if unavailable. + + Prefers the market's own ``endDate`` and falls back to its first event's ``endDate``. The + Gamma API omits the market-level ``endDate`` on some markets (e.g. season/futures markets + grouped only under an event), hence the event fallback. + + Args: + market (dict): Raw Gamma API market. + """ + end_date = market.get("endDate") + if not end_date: + events = market.get("events") or [] + end_date = events[0].get("endDate") if events else None + return end_date or None + + @staticmethod + def _get_market_close_date(market: dict) -> date | None: + """Return the market's scheduled close date, or ``None`` if it can't be determined. + + Args: + market (dict): Raw Gamma API market. + """ + end_date = PolymarketSource._get_market_end_date_str(market) + if end_date is None: + return None + close_datetime_str = dates.convert_zulu_to_iso(end_date) + return datetime.fromisoformat(close_datetime_str).replace(tzinfo=None).date() + + @staticmethod + def _resolves_before_forecast_window(market: dict, min_resolution_date: date) -> bool: + """Return True if the market resolves too soon to appear on a question set. + + A question set is published ``FREEZE_WINDOW_IN_DAYS`` before forecasts are due, so any + market closing on or before ``min_resolution_date`` (``today + FREEZE_WINDOW_IN_DAYS``) can + never be forecast and is dropped at fetch time. Markets with no discoverable close date are + kept here and left for ``_transform_question`` to drop. + + Args: + market (dict): Raw Gamma API market. + min_resolution_date (date): Earliest close date a market may have and still be usable. + """ + close_date = PolymarketSource._get_market_close_date(market) + if close_date is None: + return False + return close_date <= min_resolution_date + # ------------------------------------------------------------------ # Private: price history helpers # ------------------------------------------------------------------ @@ -456,9 +547,8 @@ def _transform_question( current_prob = price_history[-1]["p"] if len(price_history) > 1 else np.nan resolved_datetime = resolved_datetime_str = "N/A" - try: - end_date = market["endDate"] if "endDate" in market else market["events"][0]["endDate"] - except KeyError: + end_date = PolymarketSource._get_market_end_date_str(market) + if end_date is None: # endDate unexpectedly missing from: # https://polymarket.com/event/will-trump-meet-with-khamenei-before-august return None diff --git a/src/tests/test_polymarket.py b/src/tests/test_polymarket.py index 337995ac..d49b3a9d 100644 --- a/src/tests/test_polymarket.py +++ b/src/tests/test_polymarket.py @@ -1,14 +1,16 @@ """Tests for PolymarketSource fetch/update logic.""" import os -from datetime import datetime +from datetime import date, datetime, timedelta from unittest.mock import Mock, patch import pytest import requests from _schemas import PolymarketFetchFrame, QuestionFrame, ResolutionFrame +from helpers import question_curation from sources.polymarket import ( + _MIN_MARKET_LIQUIDITY, ConditionIdMarketNotFoundError, FailedConditionIdsError, PolymarketSource, @@ -283,6 +285,19 @@ def test_missing_end_date_returns_none(self): result = PolymarketSource._transform_question(market, self.FETCH_DT, set()) assert result is None + def test_falls_back_to_event_end_date_when_market_end_date_missing(self): + """With no market-level endDate, the event's endDate drives the close datetime.""" + market = make_polymarket_api_market( + events=[{"endDate": "2026-06-15T00:00:00Z"}], + price_history=make_polymarket_price_history([(1736380800, 0.5)]), + ) + del market["endDate"] + + result = PolymarketSource._transform_question(market, self.FETCH_DT, set()) + + assert result is not None + assert result["market_info_close_datetime"].startswith("2026-06-15") + def test_single_price_history_entry(self): """Single price history entry: probability and freeze value are N/A.""" market = make_polymarket_api_market( @@ -327,10 +342,30 @@ def test_invalid_market_skips_resolution_branch(self): class TestFetchActiveMarketsFromApi: """Tests for PolymarketSource._fetch_active_markets_from_api.""" - def _mock_response(self, data): + _FROZEN_TODAY = date(2026, 1, 15) + + @pytest.fixture(autouse=True) + def _freeze_today_for_window(self, freeze_today): + """Freeze 'today' so the resolution-window filter is deterministic across wall-clock time. + + The default market fixture closes 2026-06-01, comfortably after the frozen window cutoff, so + the existing (non-date) tests are unaffected. + """ + freeze_today(self._FROZEN_TODAY) + + def _end_date(self, days_from_cutoff: int) -> str: + """Return an endDate string offset by ``days_from_cutoff`` days from the window cutoff. + + The cutoff is ``today + FREEZE_WINDOW_IN_DAYS``; deriving from the constant keeps these + tests correct if the freeze window changes. + """ + cutoff = self._FROZEN_TODAY + timedelta(days=question_curation.FREEZE_WINDOW_IN_DAYS) + return (cutoff + timedelta(days=days_from_cutoff)).strftime("%Y-%m-%dT00:00:00Z") + + def _mock_response(self, data, next_cursor=None): resp = Mock() resp.ok = True - resp.json.return_value = data + resp.json.return_value = {"markets": data, "next_cursor": next_cursor} resp.raise_for_status = Mock() return resp @@ -340,10 +375,7 @@ def _mock_response(self, data): def test_basic_returns_qualifying(self, mock_get, mock_price, mock_sleep, polymarket_source): """Binary, liquid, non-catch-all markets are returned.""" market = make_polymarket_api_market() - mock_get.side_effect = [ - self._mock_response([market]), - self._mock_response([]), # End pagination - ] + mock_get.return_value = self._mock_response([market]) mock_price.return_value = [{"t": 1736380800, "p": 0.5}] result = polymarket_source._fetch_active_markets_from_api() @@ -358,10 +390,7 @@ def test_basic_returns_qualifying(self, mock_get, mock_price, mock_sleep, polyma def test_filters_non_binary(self, mock_get, mock_price, mock_sleep, polymarket_source): """Non-binary markets are excluded.""" market = make_polymarket_api_market(outcomes='["Over", "Under"]') - mock_get.side_effect = [ - self._mock_response([market]), - self._mock_response([]), - ] + mock_get.return_value = self._mock_response([market]) result = polymarket_source._fetch_active_markets_from_api() @@ -374,10 +403,7 @@ def test_filters_non_binary(self, mock_get, mock_price, mock_sleep, polymarket_s def test_filters_low_liquidity(self, mock_get, mock_price, mock_sleep, polymarket_source): """Markets with liquidityNum below the threshold are excluded.""" market = make_polymarket_api_market(liquidityNum=10000) - mock_get.side_effect = [ - self._mock_response([market]), - self._mock_response([]), - ] + mock_get.return_value = self._mock_response([market]) result = polymarket_source._fetch_active_markets_from_api() @@ -389,10 +415,7 @@ def test_filters_low_liquidity(self, mock_get, mock_price, mock_sleep, polymarke def test_filters_catch_all(self, mock_get, mock_price, mock_sleep, polymarket_source): """Markets with 'other' in the slug are excluded.""" market = make_polymarket_api_market(slug="who-will-win-other-candidates") - mock_get.side_effect = [ - self._mock_response([market]), - self._mock_response([]), - ] + mock_get.return_value = self._mock_response([market]) result = polymarket_source._fetch_active_markets_from_api() @@ -406,34 +429,76 @@ def test_pagination(self, mock_get, mock_price, mock_sleep, polymarket_source): m1 = make_polymarket_api_market(conditionId="0x001") m2 = make_polymarket_api_market(conditionId="0x002") mock_get.side_effect = [ - self._mock_response([m1]), + self._mock_response([m1], next_cursor="cursor1"), self._mock_response([m2]), - self._mock_response([]), ] mock_price.return_value = [{"t": 1736380800, "p": 0.5}] result = polymarket_source._fetch_active_markets_from_api() assert len(result) == 2 + # The 2nd request must forward the cursor from the 1st response; offset is never sent. + assert mock_get.call_count == 2 + second_params = mock_get.call_args_list[1].kwargs["params"] + assert second_params["after_cursor"] == "cursor1" + assert "offset" not in second_params + # Liquidity floor is pushed server-side (client-side check stays the authoritative cutoff). + assert second_params["liquidity_num_min"] == _MIN_MARKET_LIQUIDITY @patch("sources.polymarket.time.sleep") @patch.object(PolymarketSource, "_fetch_price_history") @patch("sources.polymarket.requests.get") - def test_skips_when_price_history_none( + def test_dedupes_market_recurring_across_pages( self, mock_get, mock_price, mock_sleep, polymarket_source ): - """Markets where _fetch_price_history returns None are excluded.""" - market = make_polymarket_api_market() + """A market re-served on a later page (keyset orders by mutable liquidity) is kept once.""" + market = make_polymarket_api_market(conditionId="0xdupe") mock_get.side_effect = [ + self._mock_response([market], next_cursor="cursor1"), self._mock_response([market]), - self._mock_response([]), ] + mock_price.return_value = [{"t": 1736380800, "p": 0.5}] + + result = polymarket_source._fetch_active_markets_from_api() + + assert len(result) == 1 + assert result[0]["conditionId"] == "0xdupe" + + @patch("sources.polymarket.time.sleep") + @patch.object(PolymarketSource, "_fetch_price_history") + @patch("sources.polymarket.requests.get") + def test_skips_when_price_history_none( + self, mock_get, mock_price, mock_sleep, polymarket_source + ): + """Markets where _fetch_price_history returns None are excluded.""" + market = make_polymarket_api_market() + mock_get.return_value = self._mock_response([market]) mock_price.return_value = None result = polymarket_source._fetch_active_markets_from_api() assert len(result) == 0 + @patch("sources.polymarket.time.sleep") + @patch.object(PolymarketSource, "_fetch_price_history") + @patch("sources.polymarket.requests.get") + def test_transient_error_retried(self, mock_get, mock_price, mock_sleep, polymarket_source): + """A transient HTTP error retries the whole fetch via backoff instead of truncating.""" + market = make_polymarket_api_market() + err_resp = Mock() + err_resp.raise_for_status.side_effect = requests.exceptions.HTTPError("500") + mock_get.side_effect = [ + err_resp, # first attempt: transient 500 + self._mock_response([market]), # retry: success + ] + mock_price.return_value = [{"t": 1736380800, "p": 0.5}] + + result = polymarket_source._fetch_active_markets_from_api() + + assert len(result) == 1 + assert result[0]["conditionId"] == "0xabc123" + assert mock_get.call_count == 2 + @patch("sources.polymarket.time.sleep") @patch.object(PolymarketSource, "_fetch_price_history") @patch("sources.polymarket.requests.get") @@ -443,14 +508,89 @@ def test_filters_missing_liquidity_key( """Markets without a liquidityNum key are excluded.""" market = make_polymarket_api_market() del market["liquidityNum"] - mock_get.side_effect = [ - self._mock_response([market]), - self._mock_response([]), - ] + mock_get.return_value = self._mock_response([market]) + + result = polymarket_source._fetch_active_markets_from_api() + + assert len(result) == 0 + + @patch("sources.polymarket.time.sleep") + @patch.object(PolymarketSource, "_fetch_price_history") + @patch("sources.polymarket.requests.get") + def test_filters_market_resolving_within_freeze_window( + self, mock_get, mock_price, mock_sleep, polymarket_source + ): + """Markets closing within the freeze window are excluded (they can't be forecast).""" + market = make_polymarket_api_market(endDate=self._end_date(-1)) + mock_get.return_value = self._mock_response([market]) result = polymarket_source._fetch_active_markets_from_api() assert len(result) == 0 + # The expensive price-history call is skipped for markets that resolve too soon. + mock_price.assert_not_called() + + @patch("sources.polymarket.time.sleep") + @patch.object(PolymarketSource, "_fetch_price_history") + @patch("sources.polymarket.requests.get") + def test_filters_market_resolving_on_freeze_window_boundary( + self, mock_get, mock_price, mock_sleep, polymarket_source + ): + """A market closing exactly on the cutoff date is excluded (boundary is inclusive).""" + market = make_polymarket_api_market(endDate=self._end_date(0)) + mock_get.return_value = self._mock_response([market]) + + result = polymarket_source._fetch_active_markets_from_api() + + assert len(result) == 0 + + @patch("sources.polymarket.time.sleep") + @patch.object(PolymarketSource, "_fetch_price_history") + @patch("sources.polymarket.requests.get") + def test_keeps_market_resolving_after_freeze_window( + self, mock_get, mock_price, mock_sleep, polymarket_source + ): + """Markets closing after the window cutoff are kept.""" + market = make_polymarket_api_market(endDate=self._end_date(1)) + mock_get.return_value = self._mock_response([market]) + mock_price.return_value = [{"t": 1736380800, "p": 0.5}] + + result = polymarket_source._fetch_active_markets_from_api() + + assert len(result) == 1 + + @patch("sources.polymarket.time.sleep") + @patch.object(PolymarketSource, "_fetch_price_history") + @patch("sources.polymarket.requests.get") + def test_window_filter_falls_back_to_event_end_date( + self, mock_get, mock_price, mock_sleep, polymarket_source + ): + """With no top-level endDate, the event's endDate drives the window filter.""" + market = make_polymarket_api_market(events=[{"endDate": self._end_date(-1)}]) + del market["endDate"] + mock_get.return_value = self._mock_response([market]) + + result = polymarket_source._fetch_active_markets_from_api() + + assert len(result) == 0 + mock_price.assert_not_called() + + @patch("sources.polymarket.time.sleep") + @patch.object(PolymarketSource, "_fetch_price_history") + @patch("sources.polymarket.requests.get") + def test_keeps_market_when_close_date_undeterminable( + self, mock_get, mock_price, mock_sleep, polymarket_source + ): + """A market with an undeterminable close date is kept here (left for _transform_question).""" + market = make_polymarket_api_market() + del market["endDate"] + del market["events"] + mock_get.return_value = self._mock_response([market]) + mock_price.return_value = [{"t": 1736380800, "p": 0.5}] + + result = polymarket_source._fetch_active_markets_from_api() + + assert len(result) == 1 # --------------------------------------------------------------------------- @@ -511,7 +651,7 @@ class TestGetMarket: def _mock_response(self, data): resp = Mock() resp.ok = True - resp.json.return_value = data + resp.json.return_value = {"markets": data} resp.raise_for_status = Mock() return resp