From f2568c0bcce80ff1d069494761738d5b72de3821 Mon Sep 17 00:00:00 2001 From: Bruno Leonardo Michels Date: Sat, 18 Jul 2026 23:34:58 -0300 Subject: [PATCH 1/5] feat: add TripAdvisor username scan module --- user_scanner/user_scan/other/tripadvisor.py | 26 +++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 user_scanner/user_scan/other/tripadvisor.py diff --git a/user_scanner/user_scan/other/tripadvisor.py b/user_scanner/user_scan/other/tripadvisor.py new file mode 100644 index 00000000..e4b80550 --- /dev/null +++ b/user_scanner/user_scan/other/tripadvisor.py @@ -0,0 +1,26 @@ +from user_scanner.core.orchestrator import generic_validate, Result + + +def validate_tripadvisor(user): + url = f"https://www.tripadvisor.com/Profile/{user}" + show_url = url + + headers = { + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0" + } + + def process(response): + if response.status_code == 200: + return Result.taken() + # Handles are case-canonicalized: a non-canonical casing 301-redirects + # to the real profile, so a redirect back to /Profile/ still means the + # account exists. Only a genuinely missing handle returns 404. + if response.status_code in (301, 302): + location = response.headers.get("location", "") + if "/profile/" in location.lower(): + return Result.taken() + if response.status_code == 404: + return Result.available() + return Result.error(f"Unexpected status: {response.status_code}") + + return generic_validate(url, process, headers=headers, show_url=show_url) From cc1b31b7bcfb90235dd61404b508ceb7cb219728 Mon Sep 17 00:00:00 2001 From: Bruno Leonardo Michels Date: Sun, 19 Jul 2026 09:15:10 -0300 Subject: [PATCH 2/5] feat: extract profile metadata in TripAdvisor username scan Capture the member name, about text and avatar from OpenGraph tags via Result.taken(extra=...) when a profile is found. --- user_scanner/user_scan/other/tripadvisor.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/user_scanner/user_scan/other/tripadvisor.py b/user_scanner/user_scan/other/tripadvisor.py index e4b80550..359c1333 100644 --- a/user_scanner/user_scan/other/tripadvisor.py +++ b/user_scanner/user_scan/other/tripadvisor.py @@ -1,3 +1,4 @@ +import re from user_scanner.core.orchestrator import generic_validate, Result @@ -11,7 +12,7 @@ def validate_tripadvisor(user): def process(response): if response.status_code == 200: - return Result.taken() + return Result.taken(extra=_extract_profile(response.text)) # Handles are case-canonicalized: a non-canonical casing 301-redirects # to the real profile, so a redirect back to /Profile/ still means the # account exists. Only a genuinely missing handle returns 404. @@ -24,3 +25,21 @@ def process(response): return Result.error(f"Unexpected status: {response.status_code}") return generic_validate(url, process, headers=headers, show_url=show_url) + + +def _extract_profile(html: str) -> dict: + extra = {} + + name = re.search(r']+property="og:title"[^>]+content="([^"]+)"', html, re.IGNORECASE) + if name: + extra["name"] = re.sub(r"\s*[-|]\s*Tripadvisor.*$", "", name.group(1), flags=re.IGNORECASE).strip() + + about = re.search(r']+property="og:description"[^>]+content="([^"]+)"', html, re.IGNORECASE) + if about: + extra["about"] = about.group(1).strip() + + avatar = re.search(r']+property="og:image"[^>]+content="([^"]+)"', html, re.IGNORECASE) + if avatar: + extra["avatar"] = avatar.group(1).strip() + + return extra From b23add52df2af1141ef955d6a5be72a19e8458a2 Mon Sep 17 00:00:00 2001 From: Bruno Leonardo Michels Date: Wed, 22 Jul 2026 08:59:17 -0300 Subject: [PATCH 3/5] feat: bypass DataDome and extract profile metadata in TripAdvisor scan TripAdvisor sits behind DataDome, which fingerprints the TLS handshake and returns 403 to Python's default HTTP stack regardless of headers. Route the module through a warmed, browser-impersonating curl_cffi session: a one-time homepage warm-up obtains the clearance cookie, and Chrome TLS impersonation clears the fingerprint check. Add reusable impersonate_validate/impersonate_request core helpers so other bot-walled modules can reuse the same warmed session, and fetch profile metadata (name, bio, hometown, joined, website, avatar) from TripAdvisor's persisted GraphQL query. Parsing is null-safe and fully guarded so metadata enrichment never turns a found account into an error. --- pyproject.toml | 3 +- requirements.txt | 1 + tests/test_impersonate.py | 89 +++++++++++++++++++ user_scanner/core/impersonate.py | 79 +++++++++++++++++ user_scanner/user_scan/other/tripadvisor.py | 95 ++++++++++++++++----- 5 files changed, 245 insertions(+), 22 deletions(-) create mode 100644 tests/test_impersonate.py create mode 100644 user_scanner/core/impersonate.py diff --git a/pyproject.toml b/pyproject.toml index b068e79d..e369baa4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,8 @@ authors = [ dependencies = [ "httpx[http2]>=0.27,<0.29", "socksio>=1.0,<2", - "colorama>=0.4,<1" + "colorama>=0.4,<1", + "curl_cffi>=0.7,<1" ] requires-python = ">=3.10" diff --git a/requirements.txt b/requirements.txt index 1277d247..037f37db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ httpx[http2]>=0.27,<0.29 socksio>=1.0,<2 colorama>=0.4,<1 +curl_cffi>=0.7,<1 diff --git a/tests/test_impersonate.py b/tests/test_impersonate.py new file mode 100644 index 00000000..66caa32b --- /dev/null +++ b/tests/test_impersonate.py @@ -0,0 +1,89 @@ +from types import SimpleNamespace + +from user_scanner.core import impersonate +from user_scanner.core.result import Result, Status + + +class FakeSession: + instances = [] + + def __init__(self, impersonate=None, proxies=None): + self.impersonate = impersonate + self.proxies = proxies + self.calls = [] + FakeSession.instances.append(self) + + def get(self, url, **kwargs): + return self.request("GET", url, **kwargs) + + def request(self, method, url, **kwargs): + self.calls.append((url, kwargs)) + return SimpleNamespace(status_code=200, headers={}, text="ok") + + +def _reset(monkeypatch): + impersonate._sessions.clear() + impersonate._warmed.clear() + FakeSession.instances.clear() + monkeypatch.setattr(impersonate.cffi, "Session", FakeSession) + monkeypatch.setattr(impersonate, "get_proxy", lambda: None) + + +def _process(response): + return Result.taken() if response.status_code == 200 else Result.error("bad") + + +def test_warmup_runs_once_and_session_is_reused(monkeypatch): + _reset(monkeypatch) + + r1 = impersonate.impersonate_validate( + "https://site/a", _process, warmup_url="https://site/", show_url="SHOWN" + ) + r2 = impersonate.impersonate_validate( + "https://site/b", _process, warmup_url="https://site/" + ) + + assert r1.status == Status.TAKEN + assert r1.url == "SHOWN" + assert r2.status == Status.TAKEN + + assert len(FakeSession.instances) == 1 + session = FakeSession.instances[0] + urls = [call[0] for call in session.calls] + assert urls == ["https://site/", "https://site/a", "https://site/b"] + + profile_kwargs = session.calls[1][1] + assert profile_kwargs.get("allow_redirects") is False + assert profile_kwargs.get("timeout") == impersonate.DEFAULT_TIMEOUT + + +def test_request_reuses_warm_session(monkeypatch): + _reset(monkeypatch) + + impersonate.impersonate_validate( + "https://site/a", _process, warmup_url="https://site/" + ) + resp = impersonate.impersonate_request( + "https://site/graphql", method="POST", warmup_url="https://site/", json=[{"q": 1}] + ) + + assert resp.status_code == 200 + assert len(FakeSession.instances) == 1 + session = FakeSession.instances[0] + urls = [call[0] for call in session.calls] + # warm-up happens once, then the GET and the follow-up POST reuse it + assert urls == ["https://site/", "https://site/a", "https://site/graphql"] + + +def test_exception_becomes_error_result(monkeypatch): + _reset(monkeypatch) + + def boom(response): + raise RuntimeError("kaboom") + + res = impersonate.impersonate_validate( + "https://site/a", boom, warmup_url="https://site/", show_url="SHOWN" + ) + + assert res.status == Status.ERROR + assert res.url == "SHOWN" diff --git a/user_scanner/core/impersonate.py b/user_scanner/core/impersonate.py new file mode 100644 index 00000000..f04ff116 --- /dev/null +++ b/user_scanner/core/impersonate.py @@ -0,0 +1,79 @@ +import threading +from typing import Callable, Optional + +from curl_cffi import requests as cffi + +from user_scanner.core.helpers import get_proxy +from user_scanner.core.result import Result + +DEFAULT_IMPERSONATE = "chrome" +DEFAULT_TIMEOUT = 15.0 + +_sessions: dict[tuple, cffi.Session] = {} +_warmed: set[tuple] = set() +_lock = threading.Lock() + + +def impersonate_validate( + url: str, + func: Callable[[cffi.Response], Result], + warmup_url: Optional[str] = None, + impersonate: str = DEFAULT_IMPERSONATE, + show_url: Optional[str] = None, + **kwargs, +) -> Result: + """Like ``generic_validate`` but routes the request through a browser- + impersonating curl_cffi session, so it clears TLS-fingerprint bot walls + (e.g. DataDome) that reject Python's default TLS stack regardless of headers. + + ``warmup_url`` is fetched once per session to obtain the clearance cookie the + protected endpoint requires; a blocked warm-up still sets that cookie. + """ + display_url = show_url or url + try: + response = impersonate_request( + url, warmup_url=warmup_url, impersonate=impersonate, **kwargs + ) + return func(response).update(url=display_url) + except Exception as e: + return Result.error(e, url=display_url) + + +def impersonate_request( + url: str, + method: str = "GET", + warmup_url: Optional[str] = None, + impersonate: str = DEFAULT_IMPERSONATE, + **kwargs, +) -> cffi.Response: + """Issue a single request through a warmed, cookie-persistent browser- + impersonating session and return the raw response. Reuses the same cached + session as ``impersonate_validate`` for a given (impersonate, proxy), so a + follow-up call (e.g. a profile API request) inherits the clearance cookie. + """ + session = _get_warm_session(impersonate, get_proxy(), warmup_url) + kwargs.setdefault("timeout", DEFAULT_TIMEOUT) + kwargs.setdefault("allow_redirects", False) + return session.request(method, url, **kwargs) + + +def _get_warm_session( + impersonate: str, proxy: Optional[str], warmup_url: Optional[str] +) -> cffi.Session: + key = (impersonate, proxy) + with _lock: + session = _sessions.get(key) + if session is None: + session = cffi.Session( + impersonate=impersonate, + proxies={"http": proxy, "https": proxy} if proxy else None, + ) + _sessions[key] = session + + if warmup_url and key not in _warmed: + # A blocked (403) warm-up still returns normally and sets the cookie; + # only a network error leaves the session unwarmed for a later retry. + session.get(warmup_url, timeout=DEFAULT_TIMEOUT) + _warmed.add(key) + + return session diff --git a/user_scanner/user_scan/other/tripadvisor.py b/user_scanner/user_scan/other/tripadvisor.py index 359c1333..3bdcffb3 100644 --- a/user_scanner/user_scan/other/tripadvisor.py +++ b/user_scanner/user_scan/other/tripadvisor.py @@ -1,45 +1,98 @@ +import base64 import re -from user_scanner.core.orchestrator import generic_validate, Result + +from user_scanner.core.impersonate import impersonate_request, impersonate_validate +from user_scanner.core.result import Result + +WARMUP_URL = "https://www.tripadvisor.com/" +GRAPHQL_URL = "https://www.tripadvisor.com/data/graphql/ids" + +# TripAdvisor renders the profile client-side, so name/hometown/bio/etc. are not +# in the page HTML — they come from this persisted GraphQL query. The id is +# registered server-side by TripAdvisor and can change when they redeploy; if it +# does, metadata degrades to empty while found/available detection keeps working. +PROFILE_QUERY_ID = "b7f6eb32ff629f60" def validate_tripadvisor(user): url = f"https://www.tripadvisor.com/Profile/{user}" - show_url = url - - headers = { - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0" - } def process(response): if response.status_code == 200: - return Result.taken(extra=_extract_profile(response.text)) + return Result.taken(extra=_fetch_profile(user)) # Handles are case-canonicalized: a non-canonical casing 301-redirects # to the real profile, so a redirect back to /Profile/ still means the # account exists. Only a genuinely missing handle returns 404. if response.status_code in (301, 302): location = response.headers.get("location", "") if "/profile/" in location.lower(): - return Result.taken() + return Result.taken(extra=_fetch_profile(user)) if response.status_code == 404: return Result.available() return Result.error(f"Unexpected status: {response.status_code}") - return generic_validate(url, process, headers=headers, show_url=show_url) + return impersonate_validate(url, process, warmup_url=WARMUP_URL, show_url=url) -def _extract_profile(html: str) -> dict: - extra = {} +def _fetch_profile(user: str) -> dict: + payload = [ + { + "variables": {"username": user}, + "extensions": {"preRegisteredQueryId": PROFILE_QUERY_ID}, + } + ] + # Metadata is best-effort: a parsing failure must never turn a found + # account into an error, so the whole enrichment is guarded. + try: + response = impersonate_request( + GRAPHQL_URL, method="POST", warmup_url=WARMUP_URL, json=payload + ) + profiles = response.json()[0]["data"]["memberProfiles"] + profile = profiles[0] if profiles else None + return _parse_profile(profile) if profile else {} + except Exception: + return {} + + +def _parse_profile(profile: dict) -> dict: + sizes = (profile.get("avatar") or {}).get("photoSizes") or [] + avatar = max(sizes, key=lambda p: p.get("width") or 0).get("url") if sizes else None + + return { + "name": profile.get("displayName"), + "user_id": profile.get("userId"), + "bio": profile.get("bio"), + "joined": profile.get("created"), + "hometown": _extract_hometown(profile), + "website": _decode_website(profile.get("website")), + "verified": profile.get("isVerified"), + "avatar": avatar, + } - name = re.search(r']+property="og:title"[^>]+content="([^"]+)"', html, re.IGNORECASE) - if name: - extra["name"] = re.sub(r"\s*[-|]\s*Tripadvisor.*$", "", name.group(1), flags=re.IGNORECASE).strip() - about = re.search(r']+property="og:description"[^>]+content="([^"]+)"', html, re.IGNORECASE) - if about: - extra["about"] = about.group(1).strip() +def _extract_hometown(profile: dict) -> str | None: + # Nested fields can be present-but-null, so coalesce with `or {}` at each + # hop rather than relying on .get() defaults (which a null value bypasses). + hometown = profile.get("hometown") or {} + location = hometown.get("location") or {} + long_name = (location.get("additionalNames") or {}).get("long") + if long_name: + return long_name + # No resolved location: fall back to the free-text hometown, unless it is + # just the numeric locationId echoed back as a string. + fallback = hometown.get("fallbackString") + if fallback and not fallback.isdigit(): + return fallback + return None - avatar = re.search(r']+property="og:image"[^>]+content="([^"]+)"', html, re.IGNORECASE) - if avatar: - extra["avatar"] = avatar.group(1).strip() - return extra +def _decode_website(encoded: str | None) -> str | None: + if not encoded: + return None + try: + decoded = base64.b64decode(encoded).decode("utf-8", "replace") + except Exception: + return None + # TripAdvisor wraps the real URL as __; drop both wrappers. + inner = decoded.split("_", 1)[-1].rsplit("_", 1)[0].strip() + return inner if inner.startswith(("http://", "https://")) else None From e7418067fc02c46539b5234d13228bc1263e652e Mon Sep 17 00:00:00 2001 From: Bruno Leonardo Michels Date: Wed, 22 Jul 2026 09:22:27 -0300 Subject: [PATCH 4/5] fix: satisfy ruff and mypy for the curl_cffi module Drop the leftover `re` import in tripadvisor.py (the website decoder no longer uses a regex). curl_cffi is fully typed but was not installed in the lint job. Install it there alongside httpx so mypy checks against its real types, and fix the two genuine mismatches instead of masking them: narrow the request method to Literal["GET", "POST"] and waive only the browser-name Literal on the impersonate argument. --- .github/workflows/lint-pr.yml | 2 +- .github/workflows/lint-push.yml | 2 +- user_scanner/core/impersonate.py | 8 +++++--- user_scanner/user_scan/other/tripadvisor.py | 1 - 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index c6a1ec2e..b05674f3 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -25,7 +25,7 @@ jobs: - name: Install linters run: | python -m pip install --upgrade pip - pip install httpx colorama types-colorama + pip install httpx curl_cffi colorama types-colorama pip install ruff mypy - name: Run ruff diff --git a/.github/workflows/lint-push.yml b/.github/workflows/lint-push.yml index ca709b10..574798a0 100644 --- a/.github/workflows/lint-push.yml +++ b/.github/workflows/lint-push.yml @@ -26,7 +26,7 @@ jobs: - name: Install linters run: | python -m pip install --upgrade pip - pip install httpx colorama types-colorama + pip install httpx curl_cffi colorama types-colorama pip install ruff mypy - name: Run ruff diff --git a/user_scanner/core/impersonate.py b/user_scanner/core/impersonate.py index f04ff116..17ba5a9e 100644 --- a/user_scanner/core/impersonate.py +++ b/user_scanner/core/impersonate.py @@ -1,5 +1,5 @@ import threading -from typing import Callable, Optional +from typing import Callable, Literal, Optional from curl_cffi import requests as cffi @@ -41,7 +41,7 @@ def impersonate_validate( def impersonate_request( url: str, - method: str = "GET", + method: Literal["GET", "POST"] = "GET", warmup_url: Optional[str] = None, impersonate: str = DEFAULT_IMPERSONATE, **kwargs, @@ -65,7 +65,9 @@ def _get_warm_session( session = _sessions.get(key) if session is None: session = cffi.Session( - impersonate=impersonate, + # curl_cffi types this as a browser-name Literal; the value is a + # validated runtime string, so widen it here only. + impersonate=impersonate, # type: ignore[arg-type] proxies={"http": proxy, "https": proxy} if proxy else None, ) _sessions[key] = session diff --git a/user_scanner/user_scan/other/tripadvisor.py b/user_scanner/user_scan/other/tripadvisor.py index 3bdcffb3..482c5825 100644 --- a/user_scanner/user_scan/other/tripadvisor.py +++ b/user_scanner/user_scan/other/tripadvisor.py @@ -1,5 +1,4 @@ import base64 -import re from user_scanner.core.impersonate import impersonate_request, impersonate_validate from user_scanner.core.result import Result From 405b398333f700f58cb0787ec75f5c65df7c20c3 Mon Sep 17 00:00:00 2001 From: Bruno Leonardo Michels Date: Wed, 22 Jul 2026 14:10:36 -0300 Subject: [PATCH 5/5] fix: honor the global -t timeout flag in the impersonate helper impersonate_request() and the session warm-up used a hardcoded DEFAULT_TIMEOUT, so they ignored the CLI -t flag. Route both through a _timeout() helper that reads get_global_timeout() and falls back to DEFAULT_TIMEOUT, matching make_request() / generic_validate(). --- user_scanner/core/impersonate.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/user_scanner/core/impersonate.py b/user_scanner/core/impersonate.py index 17ba5a9e..d60ca9f8 100644 --- a/user_scanner/core/impersonate.py +++ b/user_scanner/core/impersonate.py @@ -3,7 +3,7 @@ from curl_cffi import requests as cffi -from user_scanner.core.helpers import get_proxy +from user_scanner.core.helpers import get_global_timeout, get_proxy from user_scanner.core.result import Result DEFAULT_IMPERSONATE = "chrome" @@ -52,7 +52,7 @@ def impersonate_request( follow-up call (e.g. a profile API request) inherits the clearance cookie. """ session = _get_warm_session(impersonate, get_proxy(), warmup_url) - kwargs.setdefault("timeout", DEFAULT_TIMEOUT) + kwargs.setdefault("timeout", _timeout()) kwargs.setdefault("allow_redirects", False) return session.request(method, url, **kwargs) @@ -75,7 +75,14 @@ def _get_warm_session( if warmup_url and key not in _warmed: # A blocked (403) warm-up still returns normally and sets the cookie; # only a network error leaves the session unwarmed for a later retry. - session.get(warmup_url, timeout=DEFAULT_TIMEOUT) + session.get(warmup_url, timeout=_timeout()) _warmed.add(key) return session + + +def _timeout() -> float: + # Honour the CLI -t flag (get_global_timeout), matching make_request(); + # fall back to DEFAULT_TIMEOUT when the user has not set one. + global_timeout = get_global_timeout() + return global_timeout if global_timeout is not None else DEFAULT_TIMEOUT