From 337c915a74d7fa3842ba2e3875e5ce71593a8542 Mon Sep 17 00:00:00 2001 From: eD Thomas Date: Sat, 4 Apr 2026 21:28:49 -0400 Subject: [PATCH 1/3] Add Instagram data liberation support Adds a complete Instagram-to-WordPress migration pipeline using the same CDP-based approach as Wix and Squarespace extractors. Scripts: - scripts/instagram/discover.js: Scroll-based GraphQL interception to inventory all posts with metadata, captions, timestamps, locations - scripts/instagram/extract.js: Per-post extraction with ?img_index=N for carousel slides, deduplication by Instagram media ID - scripts/instagram/import.js: XML-RPC import with wp.uploadFile for media, wp.newPost with featured images, gallery blocks for carousels, correct backdated post dates, and source links to original posts Also includes: - prompts/instagram.md: User-facing migration prompt - tests/instagram.test.js: 32 unit tests covering data transformation, XML-RPC encoding, carousel deduplication, and content generation - DISCOVERIES.md entry documenting key findings - Updated AGENTS.md, README.md, cli.js for Instagram support Tested against a 308-post profile: 364 media files uploaded, all posts imported with correct dates, unique carousel slides in gallery blocks, and featured images. Zero failures. Co-Authored-By: Claude Opus 4.6 (1M context) --- .gitignore | 2 + AGENTS.md | 69 ++++- DISCOVERIES.md | 28 ++ README.md | 31 +++ cli.js | 33 ++- prompts/instagram.md | 77 ++++++ scripts/instagram/discover.js | 487 +++++++++++++++++++++++++++++++++ scripts/instagram/extract.js | 487 +++++++++++++++++++++++++++++++++ scripts/instagram/import.js | 496 ++++++++++++++++++++++++++++++++++ tests/instagram.test.js | 444 ++++++++++++++++++++++++++++++ 10 files changed, 2145 insertions(+), 9 deletions(-) create mode 100644 prompts/instagram.md create mode 100644 scripts/instagram/discover.js create mode 100644 scripts/instagram/extract.js create mode 100644 scripts/instagram/import.js create mode 100644 tests/instagram.test.js diff --git a/.gitignore b/.gitignore index f7f13252..a914d7e4 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ output/ node_modules/ .DS_Store *.log +output/ +*.db diff --git a/AGENTS.md b/AGENTS.md index 7c04f0c2..ae2fa6d7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,7 +11,7 @@ This file is the entry point for any AI agent using this repository. Read it bef 3. **User prompts** — non-technical users paste these into their AI to drive the whole migration 4. **A living playbook** — this repo improves itself through AI-contributed discoveries -Currently supports: **Wix** and **Squarespace**. Webflow and Shopify are planned. +Currently supports: **Wix**, **Squarespace**, and **Instagram**. Webflow and Shopify are planned. ## If you're helping a user migrate from Wix @@ -136,6 +136,47 @@ Generate a redirect map (old paths → new WP paths) for the user to configure i --- +## If you're helping a user migrate from Instagram + +Instagram requires an authenticated browser session — there's no public API or sitemap to crawl. + +### Step 1 — Launch browser with CDP + +```bash +pkill -9 -f "Google Chrome" +sleep 3 +"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" \ + --remote-debugging-port=9222 \ + --user-data-dir="$HOME/.data-liberation/cdp-profile/chrome" \ + --restore-last-session +``` + +### Step 2 — Discover all posts + +```bash +node scripts/instagram/discover.js USERNAME --cdp-port 9222 +``` + +Scrolls the profile and intercepts GraphQL responses to build a complete post inventory. Use `--delay 3000` for large profiles. + +### Step 3 — Extract content and media + +```bash +node scripts/instagram/extract.js USERNAME --cdp-port 9222 +``` + +For each post, navigates to the individual post URL, captures metadata, and downloads full-resolution images. For carousel posts, uses `?img_index=N` to access each slide directly. + +### Step 4 — Import to WordPress.com + +```bash +node scripts/instagram/import.js --site --user --token +``` + +Creates published posts with correct dates, featured images, gallery blocks for carousels, and source links back to Instagram. + +--- + ## Using Claude in Chrome MCP If the user has the Chrome DevTools MCP set up (`npx chrome-devtools-mcp@latest`), you can drive extraction directly from the browser without running scripts: @@ -183,6 +224,17 @@ This approach works for any JavaScript-heavy platform, not just Wix. | Admin UI noise in extracted content | Smart fallback heuristics filter admin shell text, sidebar artifacts | | Products/commerce | Extract metadata but skip import (WooCommerce out of scope) | +### Instagram + +| Problem | Solution | +|---|---| +| No public API or data export | Intercept GraphQL responses via CDP browser session | +| Authentication required | Connect to user's logged-in browser via `--cdp-port` | +| Carousel slides lazy-load | Use `?img_index=N` URL parameter to load each slide directly | +| CDN image URLs expire | Download media immediately during extraction | +| Rate limiting on scroll | Add `--delay 3000` for profiles with 200+ posts | +| Two API response formats | Handle both `edge_owner_to_timeline_media` and `xdt_api__v1__feed` shapes | + --- ## How to contribute improvements back @@ -224,7 +276,8 @@ data-liberation-agent/ ├── package.json ├── prompts/ │ ├── wix.md ← what users paste into their AI for a Wix migration -│ └── squarespace.md ← what users paste into their AI for a Squarespace migration +│ ├── squarespace.md ← what users paste into their AI for a Squarespace migration +│ └── instagram.md ← what users paste into their AI for an Instagram migration ├── scripts/ │ ├── wix/ │ │ ├── discover.js ← inventory the Wix site (sitemap + categorization) @@ -232,7 +285,11 @@ data-liberation-agent/ │ ├── squarespace/ │ │ ├── discover.js ← inventory via admin CDP or public JSON API │ │ ├── extract.js ← extract content via admin API interception + DOM fallback -│ │ └── import.js ← publish to WordPress.com via REST API +│ │ └── import.js ← publish to WordPress.com via XML-RPC +│ ├── instagram/ +│ │ ├── discover.js ← inventory an Instagram profile via GraphQL interception +│ │ ├── extract.js ← extract posts, carousel slides, and media +│ │ └── import.js ← publish to WordPress.com via XML-RPC │ └── import.js ← publish to WordPress.com via REST API (Wix) ├── examples/ │ ├── wix-api-blog-post.json ← example of Wix internal API response @@ -257,6 +314,12 @@ data-liberation-agent/ - **Password-protected pages**: Admin extraction may fail without credentials. - **Content is HTML, not blocks**: Imported as custom HTML. Block conversion is planned but not yet implemented. +### Instagram +- **Stories and Reels**: Not yet supported — different GraphQL queries and video handling required. +- **Comments**: Post comment text is not extracted (only comment counts). +- **Private profiles**: Only works for the user's own profile or profiles they follow. +- **Very large profiles (1000+ posts)**: May hit rate limiting. Use `--delay 4000` or higher. + ### General - Import creates everything as **drafts** — the user must review and publish manually. - WordPress.com Application Passwords must be created by the user (AI can't do this step). diff --git a/DISCOVERIES.md b/DISCOVERIES.md index 9183eba1..10f7e102 100644 --- a/DISCOVERIES.md +++ b/DISCOVERIES.md @@ -6,6 +6,34 @@ AI agents: when you contribute an improvement, add an entry here. See [CONTRIBUT --- +## 2026-04-03 — Instagram data extraction via CDP and GraphQL interception + +**Found by:** Claude + human contributor (live testing against a real 308-post Instagram profile) +**During:** Building Instagram support for the data-liberation-agent +**Type:** API endpoint | content type | architecture + +### What I found + +Instagram is a React app that communicates via GraphQL queries to `https://www.instagram.com/graphql/query/`. By connecting to an authenticated browser via CDP and intercepting responses during profile scroll, we capture structured JSON for every post. Key discoveries: + +1. **Carousel slide direct access via `?img_index=N`**: Individual carousel slides can be loaded by appending `?img_index=1`, `?img_index=2`, etc. to the post URL. This is significantly more reliable than clicking through carousel arrows in the DOM. + +2. **Carousel DOM has 3 `
  • ` elements**: Instagram keeps previous, current, and next slides in the DOM simultaneously. Deduplication by Instagram media ID (the numeric prefix in CDN URLs like `/12345_67890.jpg`) is required to avoid capturing the same image from adjacent preloaded slides. + +3. **Scroll-based pagination is more reliable than direct GraphQL**: Making direct `fetch()` calls to the GraphQL endpoint triggers rate limiting. Scrolling the profile with 2-3 second delays lets Instagram's own IntersectionObserver trigger pagination naturally. + +4. **WordPress.com REST API doesn't support writes with app passwords**: Returns 401 for POST operations. XML-RPC (`wp.uploadFile`, `wp.newPost`) works correctly. The `post_date` must be sent as a `` in `"YYYY-MM-DD HH:MM:SS"` format — WordPress ignores `` typed values. + +### How it works + +Three-step pipeline: discover (scroll + intercept GraphQL) → extract (visit each post, use `?img_index=N` for carousels, download media) → import (XML-RPC `wp.uploadFile` for media, `wp.newPost` with `post_thumbnail` for featured images and gallery blocks for carousels). + +### Why it's better than the previous approach + +Instagram's built-in data export takes days, provides lower-resolution images, and has no location data. The CDP approach captures everything in real-time at full resolution with complete metadata. + +--- + ## 2026-04-02 — Squarespace admin extraction via CDP **Found by:** Claude + human contributor (live testing against a Squarespace site) diff --git a/README.md b/README.md index 788411c8..2a89e327 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ This repo gives people a prompt they can paste into any AI assistant (Claude, Ch |---|---|---| | **Wix** | Ready | [`prompts/wix.md`](./prompts/wix.md) | | **Squarespace** | Ready | [`prompts/squarespace.md`](./prompts/squarespace.md) | +| **Instagram** | Ready | [`prompts/instagram.md`](./prompts/instagram.md) | | Webflow | Planned | — | | Shopify (blog/pages) | Planned | — | @@ -56,6 +57,26 @@ node scripts/squarespace/import.js --site your-wp-site \ --username your-user --token YOUR_APP_PASSWORD ``` +## Quick start (Instagram) + +```bash +# 1. Install dependencies +npm install + +# 2. Launch Chrome with remote debugging (Instagram requires an authenticated session) +"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" \ + --remote-debugging-port=9222 --user-data-dir="$HOME/.data-liberation/cdp-profile/chrome" + +# 3. Log into Instagram in the browser, then discover all posts +node scripts/instagram/discover.js YOUR_USERNAME --cdp-port 9222 + +# 4. Extract content and download all media +node scripts/instagram/extract.js YOUR_USERNAME --cdp-port 9222 + +# 5. Import to WordPress.com +node scripts/instagram/import.js --site your-wp-site --user your-user --token YOUR_APP_PASSWORD +``` + Or skip all of that and **paste the prompt into your AI assistant** — it will handle everything. ## For AI agents @@ -85,6 +106,16 @@ This means the playbook gets smarter with every migration. - [ ] Block conversion (`core/paragraph`, `core/image`, etc.) - [ ] Product/commerce migration +### Instagram +- [x] Profile discovery via GraphQL interception +- [x] Post extraction with full metadata (captions, dates, locations, hashtags) +- [x] Carousel slide extraction via `?img_index=N` +- [x] Gallery block output for carousel posts +- [x] Media download (photos and videos) +- [x] WordPress.com XML-RPC import with featured images +- [ ] Stories and Reels extraction +- [ ] Comment extraction + ### General - [x] WordPress.com REST API import script - [ ] WordPress Studio local-first workflow diff --git a/cli.js b/cli.js index 8d1d768e..4dfb17bc 100644 --- a/cli.js +++ b/cli.js @@ -267,6 +267,7 @@ function getBrowserUserAgent(browser) { function detectPlatform(url) { const lower = url.toLowerCase(); + if (lower.includes('instagram.com')) return 'instagram'; if (lower.includes('wix.com') || lower.includes('wixsite.com')) return 'wix'; if (lower.includes('squarespace.com')) return 'squarespace'; if (lower.includes('webflow.io') || lower.includes('webflow.com')) return 'webflow'; @@ -405,6 +406,7 @@ async function main() { heading('Login Detection'); const platforms = [ + { name: 'Instagram', domain: '.instagram.com' }, { name: 'Wix', domain: '.wix.com' }, { name: 'Squarespace', domain: '.squarespace.com' }, { name: 'Webflow', domain: '.webflow.com' }, @@ -455,12 +457,13 @@ async function main() { ok(`Detected platform: ${BOLD}${detectedPlatform}${RESET}`); } else { const platChoice = await askChoice('Which platform is this site on?', [ + { label: 'Instagram', value: 'instagram' }, { label: 'Wix', value: 'wix' }, { label: 'Squarespace', value: 'squarespace' }, { label: 'Webflow', value: 'webflow' }, { label: 'Other / not sure', value: 'unknown' }, ]); - // Use first choice as default if detection fails + detectedPlatform = platChoice.value; } const activePlatform = detectedPlatform !== 'unknown' ? detectedPlatform : 'wix'; @@ -521,13 +524,31 @@ async function main() { // ── Step 4: Run discovery ── heading('Step 1: Discovering Site Content'); - log(`Scanning ${siteUrl} for all pages, posts, and media...\n`); mkdirSync('output', { recursive: true }); - const uaArgs = userAgent ? ['--user-agent', userAgent] : []; + // Instagram uses the browser's own UA via CDP — user-agent flag is only for Wix/other platforms + const uaArgs = (userAgent && activePlatform !== 'instagram') ? ['--user-agent', userAgent] : []; const cdpArgs = cdpPort ? ['--cdp-port', String(cdpPort)] : []; - const discoverResult = await runScript(`scripts/${activePlatform}/discover.js`, [siteUrl, ...uaArgs, ...cdpArgs]); + + // Instagram uses a username, not a site URL + let discoverTarget = siteUrl; + if (activePlatform === 'instagram') { + // Extract username from URL or use as-is + const igMatch = siteUrl.match(/instagram\.com\/([^/?]+)/); + discoverTarget = igMatch ? igMatch[1] : siteUrl.replace(/^https?:\/\//, '').replace(/\/$/, ''); + log(`Discovering posts for @${discoverTarget}...\n`); + if (!cdpPort) { + fail('Instagram requires a CDP connection to an authenticated browser.'); + fail('Launch Chrome with: google-chrome --remote-debugging-port=9222'); + rl.close(); + return; + } + } else { + log(`Scanning ${siteUrl} for all pages, posts, and media...\n`); + } + + const discoverResult = await runScript(`scripts/${activePlatform}/discover.js`, [discoverTarget, ...uaArgs, ...cdpArgs]); if (discoverResult.code !== 0) { fail('Discovery failed. See output above.'); const retry = await ask('Try again? (y/n)'); @@ -558,10 +579,10 @@ async function main() { // ── Step 5: Extract content ── heading('Step 2: Extracting Content'); - log(`Extracting all pages, posts, and media from ${siteUrl}...\n`); + log(`Extracting all content from ${activePlatform === 'instagram' ? '@' + discoverTarget : siteUrl}...\n`); const extractResult = await runScript(`scripts/${activePlatform}/extract.js`, [ - siteUrl, + activePlatform === 'instagram' ? discoverTarget : siteUrl, '--url-list', 'output/inventory.json', ...uaArgs, ...cdpArgs diff --git a/prompts/instagram.md b/prompts/instagram.md new file mode 100644 index 00000000..404a7b7b --- /dev/null +++ b/prompts/instagram.md @@ -0,0 +1,77 @@ +# Instagram to WordPress.com Migration Prompt + +Copy everything below this line and paste it into your AI assistant (Claude, ChatGPT, Gemini, etc.). + +--- + +I want to migrate my Instagram photos and posts to WordPress.com. My Instagram username is: **[PASTE YOUR USERNAME HERE]** + +I have (or will create) a WordPress.com account. Please help me migrate using the playbook at https://github.com/Automattic/data-liberation-agent — read AGENTS.md first for full instructions. + +**Important**: Instagram requires an authenticated browser session. I'll need to have Chrome (or another Chromium browser) open and logged into Instagram before we start. + +Here's what I need you to do: + +## Step 1: Set up browser access + +Help me launch Chrome with remote debugging enabled so the migration scripts can connect: + +1. Quit Chrome completely +2. Relaunch with: `"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" --remote-debugging-port=9222 --user-data-dir="$HOME/.data-liberation/cdp-profile/chrome" --restore-last-session` +3. Log into Instagram in the browser window that opens +4. Confirm the connection works + +## Step 2: Discover all my posts + +```bash +node scripts/instagram/discover.js MY_USERNAME --cdp-port 9222 +``` + +This connects to my browser, navigates to my profile, and intercepts Instagram's internal GraphQL API responses as it scrolls through all my posts. It captures: +- Post metadata (captions, dates, locations, hashtags, tagged users) +- Post types (photos, videos, carousels with slide counts) +- Image and video URLs +- Profile information + +Show me the inventory summary and wait for my approval before proceeding. + +**If it stalls or gets rate limited**: Add `--delay 3000` for a gentler pace. + +## Step 3: Extract full content and download media + +```bash +node scripts/instagram/extract.js MY_USERNAME --cdp-port 9222 +``` + +This visits each post individually to get: +- Full-resolution images (not thumbnails) +- All carousel slides (uses `?img_index=N` to access each slide directly) +- Video URLs +- Tagged users, location details, accessibility captions + +All media is downloaded locally — Instagram CDN URLs expire, so this must happen promptly after discovery. + +## Step 4: Import to WordPress.com + +```bash +node scripts/import.js --site my-wp-site.wordpress.com --token MY_APP_PASSWORD +``` + +This creates WordPress posts from the extracted data: +- Each Instagram post becomes a WordPress post (as draft) +- Images uploaded to the media library +- Captions become post content with @mentions and #hashtags linked +- Original post date preserved +- Instagram shortcode and URL stored as post meta + +**For a custom post type** (e.g. a "photo" CPT): add `--post-type photo` + +## Step 5: Verify + +When done: +- Show me how many posts were imported vs. discovered +- Flag any posts with missing images or import errors +- Check that carousel posts have all their slides +- List the date range covered (oldest → newest) + +Work methodically — do one step at a time, show me progress, and wait for my go-ahead before moving to the next step. diff --git a/scripts/instagram/discover.js b/scripts/instagram/discover.js new file mode 100644 index 00000000..0ca79c3e --- /dev/null +++ b/scripts/instagram/discover.js @@ -0,0 +1,487 @@ +#!/usr/bin/env node +/** + * discover.js — Step 1: Inventory an Instagram profile + * + * Connects to a browser where you're logged into Instagram, navigates + * to your profile, and intercepts the GraphQL API responses as you + * scroll to build a complete manifest of all posts. + * + * REQUIRES: a running browser with CDP enabled and an active Instagram session. + * Instagram's API requires authentication — there's no public sitemap to crawl. + * + * Usage: + * node scripts/instagram/discover.js --cdp-port 9222 + * node scripts/instagram/discover.js --cdp-port 9222 --limit 50 + * + * Options: + * --cdp-port CDP port of your running browser (required) + * --limit Stop after N posts (for testing) + * --delay Delay between scroll actions (default: 2000) + * + * Output: + * output/inventory.json — manifest of all discovered posts with metadata + */ + +import { chromium } from 'playwright'; +import { writeFileSync, mkdirSync } from 'fs'; + +const args = process.argv.slice(2); +const username = args.find(a => !a.startsWith('--')); +if (!username) { + console.error('Usage: node scripts/instagram/discover.js --cdp-port '); + process.exit(1); +} + +function parseIntArg(name, fallback) { + const idx = args.indexOf(name); + if (idx === -1) return fallback; + const val = parseInt(args[idx + 1], 10); + if (!Number.isFinite(val)) { + console.error(`Error: ${name} requires a numeric value.`); + process.exit(1); + } + return val; +} + +const cdpPort = parseIntArg('--cdp-port', null); +if (!cdpPort) { + console.error('Error: --cdp-port is required. Instagram needs an authenticated browser session.'); + console.error('Launch Chrome with: google-chrome --remote-debugging-port=9222'); + process.exit(1); +} + +const limit = parseIntArg('--limit', Infinity); +const scrollDelay = parseIntArg('--delay', 2000); + +mkdirSync('output', { recursive: true }); + +function sleep(ms) { + return new Promise(r => setTimeout(r, ms)); +} + +// Classify an Instagram post by its type +function classifyPost(node) { + if (node.__typename === 'GraphSidecar' || node.edge_sidecar_to_children) return 'carousel'; + if (node.__typename === 'GraphVideo' || node.is_video) return 'video'; + return 'photo'; +} + +// Extract post metadata from a GraphQL edge node +function extractPostMeta(node) { + const caption = node.edge_media_to_caption?.edges?.[0]?.node?.text || ''; + return { + id: node.id, + shortcode: node.shortcode, + type: classifyPost(node), + timestamp: node.taken_at_timestamp, + date: node.taken_at_timestamp ? new Date(node.taken_at_timestamp * 1000).toISOString() : null, + caption, + displayUrl: node.display_url, + thumbnailUrl: node.thumbnail_src || node.thumbnail_resources?.[0]?.src, + dimensions: node.dimensions || null, + isVideo: !!node.is_video, + videoUrl: node.video_url || null, + accessibilityCaption: node.accessibility_caption || null, + locationName: node.location?.name || null, + locationId: node.location?.id || null, + likes: node.edge_media_preview_like?.count ?? node.edge_liked_by?.count ?? null, + comments: node.edge_media_to_comment?.count ?? node.edge_media_preview_comment?.count ?? null, + // For carousels, note how many slides + carouselCount: node.edge_sidecar_to_children?.edges?.length || null, + url: `https://www.instagram.com/p/${node.shortcode}/`, + }; +} + +async function main() { + console.log(`Discovering Instagram posts for: ${username}`); + console.log(`Connecting to browser on CDP port ${cdpPort}...`); + + const browser = await chromium.connectOverCDP(`http://127.0.0.1:${cdpPort}`); + const context = browser.contexts()[0] || await browser.newContext(); + const page = await context.newPage(); + + const posts = new Map(); // shortcode → post data (dedupes) + let profileData = null; + let hasMore = true; + let endCursor = null; + let paginationRequests = 0; + + // Intercept GraphQL responses to capture post data + page.on('response', async (response) => { + const url = response.url(); + if (!url.includes('/graphql/query') && !url.includes('/api/v1/')) return; + + const ct = response.headers()['content-type'] || ''; + if (!ct.includes('application/json') && !ct.includes('text/javascript')) return; + + try { + const body = await response.json(); + + // Profile page initial load — data is in entry_data or in the graphql response + const userData = body?.data?.user || + body?.graphql?.user || + body?.data?.xdt_api__v1__feed__user_timeline_graphql_connection; + + if (userData) { + // Extract profile info on first encounter + if (!profileData && (userData.username || userData.full_name)) { + profileData = { + id: userData.id || userData.pk || null, + username: userData.username || username, + fullName: userData.full_name || '', + biography: userData.biography || userData.bio_text || '', + profilePicUrl: userData.profile_pic_url_hd || userData.profile_pic_url || '', + postCount: userData.edge_owner_to_timeline_media?.count ?? + userData.media_count ?? null, + followerCount: userData.edge_followed_by?.count ?? + userData.follower_count ?? null, + followingCount: userData.edge_follow?.count ?? + userData.following_count ?? null, + isPrivate: userData.is_private || false, + isVerified: userData.is_verified || false, + }; + console.log(` Profile: ${profileData.fullName} (@${profileData.username})`); + if (profileData.postCount) { + console.log(` Total posts reported: ${profileData.postCount}`); + } + } + + // Extract posts from the timeline media edges + const timeline = userData.edge_owner_to_timeline_media || + userData.edge_web_feed_timeline; + + if (timeline?.edges) { + for (const edge of timeline.edges) { + const node = edge.node; + if (node?.shortcode && !posts.has(node.shortcode)) { + posts.set(node.shortcode, extractPostMeta(node)); + } + } + // Track pagination cursor + if (timeline.page_info) { + hasMore = timeline.page_info.has_next_page; + endCursor = timeline.page_info.end_cursor; + } + paginationRequests++; + console.log(` Captured ${posts.size} posts (page ${paginationRequests})...`); + } + + // Handle the newer API format (xdt_api__v1__feed) + if (userData.edges) { + for (const edge of userData.edges) { + const node = edge.node; + if (node?.code && !posts.has(node.code)) { + posts.set(node.code, { + id: node.pk || node.id, + shortcode: node.code, + type: node.carousel_media_count ? 'carousel' : node.video_versions ? 'video' : 'photo', + timestamp: node.taken_at, + date: node.taken_at ? new Date(node.taken_at * 1000).toISOString() : null, + caption: node.caption?.text || '', + displayUrl: node.image_versions2?.candidates?.[0]?.url || '', + thumbnailUrl: node.image_versions2?.candidates?.slice(-1)?.[0]?.url || '', + dimensions: node.original_width && node.original_height + ? { width: node.original_width, height: node.original_height } : null, + isVideo: !!node.video_versions, + videoUrl: node.video_versions?.[0]?.url || null, + accessibilityCaption: node.accessibility_caption || null, + locationName: node.location?.name || null, + locationId: node.location?.pk || null, + likes: node.like_count ?? null, + comments: node.comment_count ?? null, + carouselCount: node.carousel_media_count || null, + url: `https://www.instagram.com/p/${node.code}/`, + }); + } + } + if (userData.page_info) { + hasMore = userData.page_info.has_next_page; + endCursor = userData.page_info.end_cursor; + } + paginationRequests++; + console.log(` Captured ${posts.size} posts (page ${paginationRequests})...`); + } + } + } catch { + // Not all responses are JSON — that's fine + } + }); + + // Navigate to the profile + const profileUrl = `https://www.instagram.com/${username}/`; + console.log(`\nNavigating to ${profileUrl}`); + try { + await page.goto(profileUrl, { waitUntil: 'domcontentloaded', timeout: 60000 }); + // Wait for the profile content to appear (response interceptor captures GraphQL data) + await sleep(3000); + } catch (e) { + if (posts.size > 0) { + console.log(` Navigation timeout, but ${posts.size} posts already captured — continuing`); + } else { + console.error(` Navigation failed: ${e.message}`); + await browser.disconnect(); + process.exit(1); + } + } + + // Check if we're logged in by looking for login prompts + const loginPrompt = await page.$('input[name="username"]'); + if (loginPrompt) { + console.error('\nError: Not logged into Instagram in this browser session.'); + console.error('Please log in to Instagram in your browser first, then re-run.'); + await browser.close(); + process.exit(1); + } + + // Check for private profile + const privateMsg = await page.$('text=This account is private'); + if (privateMsg) { + console.warn('\nWarning: This is a private profile. You can only extract your own posts or accounts you follow.'); + } + + // Also try to capture data from the page's window globals + const windowData = await page.evaluate(() => { + const result = {}; + + // Check specific known globals (avoid enumerating all of window) + for (const key of ['__additionalDataLoaded', '__NEXT_DATA__', '_sharedData']) { + if (window[key]) { + try { result[key] = window[key]; } catch {} + } + } + + // Check for require'd modules (older Instagram) + try { + if (window._sharedData?.entry_data?.ProfilePage?.[0]?.graphql?.user) { + result.profilePageData = window._sharedData.entry_data.ProfilePage[0].graphql.user; + } + } catch {} + + return result; + }); + + // Extract any posts from window globals + const globalUser = windowData?.profilePageData; + if (globalUser?.edge_owner_to_timeline_media?.edges) { + for (const edge of globalUser.edge_owner_to_timeline_media.edges) { + const node = edge.node; + if (node?.shortcode && !posts.has(node.shortcode)) { + posts.set(node.shortcode, extractPostMeta(node)); + } + } + if (!profileData && globalUser.username) { + profileData = { + username: globalUser.username, + fullName: globalUser.full_name || '', + biography: globalUser.biography || '', + profilePicUrl: globalUser.profile_pic_url_hd || '', + postCount: globalUser.edge_owner_to_timeline_media?.count ?? null, + followerCount: globalUser.edge_followed_by?.count ?? null, + followingCount: globalUser.edge_follow?.count ?? null, + isPrivate: globalUser.is_private || false, + isVerified: globalUser.is_verified || false, + }; + } + console.log(` Extracted ${posts.size} posts from page data`); + } + + // Phase 1: Scroll to load a couple pages and capture the GraphQL patterns + console.log('\nScrolling to load more posts...'); + let scrollAttempts = 0; + let lastPostCount = posts.size; + let noNewPostsStreak = 0; + let currentDelay = scrollDelay; + + while (posts.size < limit && scrollAttempts < 100) { + const scrollStart = Date.now(); + await page.evaluate(() => window.scrollTo(0, document.documentElement.scrollHeight)); + await sleep(currentDelay); + scrollAttempts++; + + if (posts.size === lastPostCount) { + noNewPostsStreak++; + // Back off when we're not getting new posts — Instagram may be throttling + currentDelay = Math.min(currentDelay * 1.5, scrollDelay * 4); + if (noNewPostsStreak >= 4) break; + } else { + noNewPostsStreak = 0; + // New posts arrived — ease back toward base delay + currentDelay = Math.max(scrollDelay, currentDelay * 0.8); + lastPostCount = posts.size; + console.log(` Scroll ${scrollAttempts}: ${posts.size} posts`); + } + } + + // Phase 2: If we have a cursor and need more posts, use direct GraphQL requests + // executed in the page context (inherits cookies and CSRF tokens) + if (hasMore && endCursor && posts.size < limit) { + console.log(`\nUsing direct GraphQL pagination (cursor available)...`); + + // First, extract the user ID from what we've already captured + const userId = profileData?.id || await page.evaluate((uname) => { + // Try to find the user ID from page source or existing data + const bodyText = document.body.innerHTML; + const idMatch = bodyText.match(/"profilePage_(\d+)"/); + return idMatch ? idMatch[1] : null; + }, username); + + if (!userId) { + // Get user ID via the profile page's metadata + const userIdFromMeta = await page.evaluate(() => { + // Instagram embeds the user ID in various places + const scripts = document.querySelectorAll('script'); + for (const s of scripts) { + const match = s.textContent.match(/"user_id":"(\d+)"/); + if (match) return match[1]; + } + // Also check meta tags + const instagramUrl = document.querySelector('meta[property="al:android:url"]')?.content; + if (instagramUrl) { + const m = instagramUrl.match(/(\d+)/); + if (m) return m[1]; + } + return null; + }); + + if (userIdFromMeta) { + if (!profileData) profileData = {}; + profileData.id = userIdFromMeta; + } + } + + const resolvedUserId = profileData?.id || userId; + + if (resolvedUserId) { + console.log(` User ID: ${resolvedUserId}`); + let cursor = endCursor; + let apiPage = 0; + let apiErrors = 0; + + while (hasMore && cursor && posts.size < limit && apiErrors < 3) { + apiPage++; + try { + // Execute the GraphQL query from within the page context + // This inherits all cookies, CSRF tokens, and headers + const result = await page.evaluate(async ({ userId, after }) => { + // Instagram uses a specific query hash for user media pagination + // We'll try the documented endpoint first + const variables = JSON.stringify({ + id: userId, + first: 12, + after: after, + }); + + // Try the newer API endpoint first + const url = `https://www.instagram.com/graphql/query/?query_hash=472f257a40c653c64c666ce877d59d2b&variables=${encodeURIComponent(variables)}`; + + const res = await fetch(url, { + headers: { + 'X-Requested-With': 'XMLHttpRequest', + 'Accept': '*/*', + }, + credentials: 'include', + }); + + if (!res.ok) return { error: `HTTP ${res.status}` }; + const text = await res.text(); + try { + return JSON.parse(text); + } catch { + return { error: `Not JSON (starts with: ${text.slice(0, 50)})` }; + } + }, { userId: resolvedUserId, after: cursor }); + + if (result.error) { + console.log(` API error: ${result.error}`); + // Rate limited or challenged — wait and retry once + apiErrors++; + console.log(` Waiting 10s and retrying (${apiErrors}/3)...`); + await sleep(10000); + continue; + } + + const media = result?.data?.user?.edge_owner_to_timeline_media; + if (media?.edges) { + for (const edge of media.edges) { + const node = edge.node; + if (node?.shortcode && !posts.has(node.shortcode)) { + posts.set(node.shortcode, extractPostMeta(node)); + } + } + hasMore = media.page_info?.has_next_page ?? false; + cursor = media.page_info?.end_cursor ?? null; + console.log(` API page ${apiPage}: ${posts.size} total posts`); + } else { + // Response format might have changed — try to find data elsewhere + console.log(` Unexpected response format on page ${apiPage}`); + break; + } + + // Rate limit: be polite — API calls need more delay than scrolling + await sleep(Math.max(scrollDelay, 3000)); + } catch (e) { + console.log(` API pagination error: ${e.message}`); + break; + } + } + } else { + console.log(' Could not determine user ID — falling back to scroll only'); + // Continue scrolling as fallback + while (posts.size < limit && scrollAttempts < 200) { + await page.evaluate(() => window.scrollTo(0, document.documentElement.scrollHeight)); + await sleep(scrollDelay); + scrollAttempts++; + if (posts.size > lastPostCount) { + lastPostCount = posts.size; + noNewPostsStreak = 0; + if (scrollAttempts % 5 === 0) console.log(` Scroll ${scrollAttempts}: ${posts.size} posts`); + } else { + noNewPostsStreak++; + if (noNewPostsStreak >= 8) break; + } + } + } + } + + await page.close(); + await browser.disconnect(); + + // Build inventory + const allPosts = [...posts.values()].sort((a, b) => (b.timestamp || 0) - (a.timestamp || 0)); + + const counts = { photo: 0, video: 0, carousel: 0 }; + for (const post of allPosts) { + counts[post.type] = (counts[post.type] || 0) + 1; + } + + const inventory = { + platform: 'instagram', + username, + profile: profileData, + discoveredAt: new Date().toISOString(), + counts, + urls: allPosts.map(p => ({ + url: p.url, + type: p.type, + id: p.id, + shortcode: p.shortcode, + })), + posts: allPosts, + }; + + writeFileSync('output/inventory.json', JSON.stringify(inventory, null, 2)); + + console.log('\nInventory summary:'); + for (const [type, count] of Object.entries(counts)) { + console.log(` ${type}: ${count}`); + } + console.log(`\nTotal: ${allPosts.length} posts discovered`); + if (profileData?.postCount) { + const pct = Math.round((allPosts.length / profileData.postCount) * 100); + console.log(`Coverage: ${allPosts.length}/${profileData.postCount} (${pct}%)`); + } + console.log('Written to output/inventory.json'); + console.log('\nReview this inventory before running extract.js'); +} + +main().catch(e => { console.error(e); process.exit(1); }); diff --git a/scripts/instagram/extract.js b/scripts/instagram/extract.js new file mode 100644 index 00000000..5a8e9984 --- /dev/null +++ b/scripts/instagram/extract.js @@ -0,0 +1,487 @@ +#!/usr/bin/env node +/** + * extract.js — Step 2: Extract full content from Instagram posts + * + * Takes the inventory from discover.js and visits each post individually + * to get full-resolution images, carousel slides, video URLs, and comments. + * + * Usage: + * node scripts/instagram/extract.js --cdp-port 9222 + * node scripts/instagram/extract.js --cdp-port 9222 --limit 10 + * node scripts/instagram/extract.js --cdp-port 9222 --skip-media + * + * Options: + * --cdp-port CDP port of your running browser (required) + * --delay Delay between posts (default: 1500) + * --limit Only process first N posts (for testing) + * --skip-media Extract metadata only, don't download images/videos + * --url-list Use inventory from discover.js (default: output/inventory.json) + * + * Output: + * output/pages/.json — extracted post data + * output/media/ — downloaded images and videos + * output/extraction-log.json — summary of what was extracted + */ + +import { chromium } from 'playwright'; +import { writeFileSync, mkdirSync, readFileSync, existsSync, createWriteStream } from 'fs'; +import { basename } from 'path'; +import https from 'https'; +import http from 'http'; + +const args = process.argv.slice(2); +const username = args.find(a => !a.startsWith('--')); +if (!username) { + console.error('Usage: node scripts/instagram/extract.js --cdp-port '); + process.exit(1); +} + +function parseIntArg(name, fallback) { + const idx = args.indexOf(name); + if (idx === -1) return fallback; + const val = parseInt(args[idx + 1], 10); + if (!Number.isFinite(val)) { + console.error(`Error: ${name} requires a numeric value.`); + process.exit(1); + } + return val; +} + +const cdpPort = parseIntArg('--cdp-port', null); +if (!cdpPort) { + console.error('Error: --cdp-port is required.'); + process.exit(1); +} + +const delay = parseIntArg('--delay', 1500); +const limit = parseIntArg('--limit', Infinity); +const skipMedia = args.includes('--skip-media'); +const urlListArg = args.indexOf('--url-list'); +const urlListFile = urlListArg !== -1 ? args[urlListArg + 1] : 'output/inventory.json'; + +mkdirSync('output/pages', { recursive: true }); +mkdirSync('output/media', { recursive: true }); + +function sleep(ms) { + return new Promise(r => setTimeout(r, ms)); +} + +function downloadFile(url, destPath, redirectsLeft = 5) { + return new Promise((resolve, reject) => { + if (redirectsLeft <= 0) return reject(new Error('Too many redirects')); + if (!url.startsWith('https://')) return reject(new Error(`Refused non-HTTPS URL: ${url}`)); + + const file = createWriteStream(destPath); + file.on('error', reject); + + const req = https.get(url, { headers: { 'User-Agent': 'Mozilla/5.0' }, timeout: 30000 }, res => { + if ([301, 302, 303, 307, 308].includes(res.statusCode)) { + res.resume(); // consume response to free socket + file.close(); + const location = res.headers.location; + if (!location?.startsWith('https://')) return reject(new Error(`Redirect to non-HTTPS: ${location}`)); + return downloadFile(location, destPath, redirectsLeft - 1).then(resolve).catch(reject); + } + if (res.statusCode !== 200) { + res.resume(); + file.close(); + return reject(new Error(`HTTP ${res.statusCode}`)); + } + res.pipe(file); + file.on('finish', () => file.close(resolve)); + }); + req.on('error', reject); + req.on('timeout', () => { req.destroy(); reject(new Error('Download timeout')); }); + }); +} + +// Sanitize shortcode for safe use as filename +function safeShortcode(sc) { + return sc.replace(/[^a-zA-Z0-9_-]/g, ''); +} + +// Generate a sane filename from an Instagram CDN URL +function mediaFilename(url, shortcode, index) { + const ext = url.match(/\.(jpg|jpeg|png|webp|mp4|mov)/i)?.[1] || 'jpg'; + return `${safeShortcode(shortcode)}_${index}.${ext.toLowerCase()}`; +} + +async function extractPostData(page, shortcode, isCarousel = false, carouselCount = 10) { + const postUrl = `https://www.instagram.com/p/${shortcode}/`; + const captured = { apiCalls: [], globals: null }; + + // Intercept API responses for this post + const responseHandler = async (response) => { + const url = response.url(); + if (!url.includes('/graphql/query') && !url.includes('/api/v1/media/')) return; + + const ct = response.headers()['content-type'] || ''; + if (!ct.includes('application/json') && !ct.includes('text/javascript')) return; + + try { + const body = await response.json(); + captured.apiCalls.push({ url, data: body }); + } catch {} + }; + + page.on('response', responseHandler); + + try { + await page.goto(postUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }); + // Wait for the main post image or GraphQL response to land + await page.waitForSelector('article img[src*="cdninstagram.com"]', { timeout: 10000 }).catch(() => {}); + } catch (e) { + console.error(` Navigation failed: ${e.message}`); + } + + page.off('response', responseHandler); + + // Extract data from the page — Instagram embeds post data in script tags and window objects + captured.globals = await page.evaluate(() => { + const result = {}; + + // JSON-LD (Instagram provides this for public posts) + result.jsonLd = Array.from( + document.querySelectorAll('script[type="application/ld+json"]') + ).map(s => { + try { return JSON.parse(s.textContent); } catch { return null; } + }).filter(Boolean); + + // Check specific known globals (avoid enumerating all of window) + for (const key of ['__additionalDataLoaded', '__NEXT_DATA__', '_sharedData']) { + if (window[key]) { + try { result[key] = window[key]; } catch {} + } + } + + // Meta tags — Instagram sets good OG tags + result.meta = { + title: document.title, + description: document.querySelector('meta[name="description"]')?.content, + ogTitle: document.querySelector('meta[property="og:title"]')?.content, + ogDescription: document.querySelector('meta[property="og:description"]')?.content, + ogImage: document.querySelector('meta[property="og:image"]')?.content, + ogType: document.querySelector('meta[property="og:type"]')?.content, + }; + + return result; + }); + + // Try to find the post's media from the captured API calls + let postDetail = null; + for (const call of captured.apiCalls) { + const data = call.data; + + // Look for the post in various response shapes + const media = data?.data?.xdt_shortcode_media || + data?.graphql?.shortcode_media || + data?.data?.shortcode_media || + data?.items?.[0]; + + if (media && (media.shortcode === shortcode || media.code === shortcode)) { + postDetail = media; + break; + } + } + + // For carousel posts: click through the carousel arrows to capture each slide's + // full-res image. Instagram's carousel arrows are button[aria-label="Next"] and + // are NOT inside the article element — they're in a parent container. + const carouselSlides = []; + if (isCarousel) { + // Strategy: Instagram supports ?img_index=N to load a specific carousel slide. + // Navigate to each slide directly and grab the main post image. + const expectedSlides = carouselCount || 10; + + // Strategy: navigate to each ?img_index=N, collect ALL
  • images + // from every page load, then deduplicate by Instagram media ID + // (the numeric prefix in the CDN URL). Instagram keeps 3
  • elements + // in the DOM (previous, current, next), so we see overlap between + // adjacent slides — deduplication handles this cleanly. + const seenMediaIds = new Set(); + + const getAllSlideImages = async () => { + return page.evaluate(() => { + const results = []; + for (const img of document.querySelectorAll('li img')) { + const src = img.src || ''; + if (!src.includes('cdninstagram.com/v/t51.')) continue; + if (img.alt?.includes('User avatar')) continue; + const rect = img.getBoundingClientRect(); + if (rect.width < 300) continue; + // Extract the Instagram media ID (unique per photo) + const idMatch = src.match(/\/(\d{5,})_/); + results.push({ src, mediaId: idMatch?.[1] || null }); + } + // Also check for video + const video = document.querySelector('li video[src], li video source, video[src], video source'); + const videoUrl = video?.src || video?.querySelector?.('source')?.src || null; + if (videoUrl) results.push({ src: null, videoUrl, mediaId: 'video_' + Date.now() }); + return results; + }); + }; + + for (let slideIdx = 1; slideIdx <= expectedSlides; slideIdx++) { + try { + const slideUrl = `https://www.instagram.com/p/${shortcode}/?img_index=${slideIdx}`; + await page.goto(slideUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }).catch(() => {}); + await page.waitForSelector('li img[src*="cdninstagram.com"]', { timeout: 8000 }).catch(() => {}); + await sleep(800); + + const images = await getAllSlideImages(); + for (const img of images) { + if (!img.mediaId || seenMediaIds.has(img.mediaId)) continue; + seenMediaIds.add(img.mediaId); + carouselSlides.push({ + type: img.videoUrl ? 'video' : 'photo', + displayUrl: img.src || null, + videoUrl: img.videoUrl || null, + }); + } + } catch { + break; + } + } + + if (carouselSlides.length > 1) { + console.log(` Carousel: ${carouselSlides.length} slides captured`); + } + } + + return { captured, postDetail, carouselSlides }; +} + +function buildPostOutput(shortcode, inventoryPost, postDetail, globals, carouselSlides = []) { + const output = { + shortcode, + sourceUrl: `https://www.instagram.com/p/${shortcode}/`, + extractedAt: new Date().toISOString(), + // Start with inventory data as baseline + ...inventoryPost, + // Enrich with detail data if we got it + media: [], + tags: [], + mentionedUsers: [], + }; + + if (postDetail) { + // Full caption (inventory might have truncated it) + output.caption = postDetail.edge_media_to_caption?.edges?.[0]?.node?.text || + postDetail.caption?.text || + output.caption; + + // Location details + if (postDetail.location) { + output.location = { + name: postDetail.location.name, + id: postDetail.location.id || postDetail.location.pk, + slug: postDetail.location.slug || null, + lat: postDetail.location.lat || null, + lng: postDetail.location.lng || null, + address: (() => { + try { return postDetail.location.address_json ? JSON.parse(postDetail.location.address_json) : null; } + catch { return null; } + })(), + }; + } + + // Tagged users + const taggedEdges = postDetail.edge_media_to_tagged_user?.edges || []; + output.mentionedUsers = taggedEdges.map(e => ({ + username: e.node?.user?.username, + fullName: e.node?.user?.full_name, + x: e.node?.x, + y: e.node?.y, + })); + + // Extract all media items (handles single posts, carousels, and videos) + if (postDetail.edge_sidecar_to_children?.edges) { + // Carousel post + for (const edge of postDetail.edge_sidecar_to_children.edges) { + const child = edge.node; + output.media.push({ + type: child.is_video ? 'video' : 'photo', + displayUrl: child.display_url || child.display_resources?.slice(-1)?.[0]?.src, + videoUrl: child.video_url || null, + dimensions: child.dimensions || null, + accessibilityCaption: child.accessibility_caption || null, + }); + } + } else if (postDetail.carousel_media) { + // Newer API format for carousels + for (const child of postDetail.carousel_media) { + output.media.push({ + type: child.video_versions ? 'video' : 'photo', + displayUrl: child.image_versions2?.candidates?.[0]?.url, + videoUrl: child.video_versions?.[0]?.url || null, + dimensions: child.original_width && child.original_height + ? { width: child.original_width, height: child.original_height } : null, + accessibilityCaption: child.accessibility_caption || null, + }); + } + } else { + // Single photo or video + output.media.push({ + type: postDetail.is_video ? 'video' : 'photo', + displayUrl: postDetail.display_url || + postDetail.image_versions2?.candidates?.[0]?.url, + videoUrl: postDetail.video_url || + postDetail.video_versions?.[0]?.url || null, + dimensions: postDetail.dimensions || + (postDetail.original_width && postDetail.original_height + ? { width: postDetail.original_width, height: postDetail.original_height } : null), + accessibilityCaption: postDetail.accessibility_caption || null, + }); + } + } else { + // No detail data — fall back to what discover gave us + if (inventoryPost?.displayUrl) { + output.media.push({ + type: inventoryPost.isVideo ? 'video' : 'photo', + displayUrl: inventoryPost.displayUrl, + videoUrl: inventoryPost.videoUrl || null, + dimensions: inventoryPost.dimensions || null, + accessibilityCaption: inventoryPost.accessibilityCaption || null, + }); + } + } + + // For carousels: if we only got 1 media item from the API but we have + // carousel slides from clicking through the UI, use those instead + if (carouselSlides.length > 1 && output.media.length <= 1) { + output.media = carouselSlides.map(slide => ({ + ...slide, + dimensions: null, + accessibilityCaption: null, + })); + } + + // Extract hashtags and @mentions from caption + if (output.caption) { + output.tags = [...output.caption.matchAll(/#(\w+)/g)].map(m => m[1]); + const mentions = [...output.caption.matchAll(/@(\w+)/g)].map(m => m[1]); + // Merge with tagged users + for (const mention of mentions) { + if (!output.mentionedUsers.find(u => u.username === mention)) { + output.mentionedUsers.push({ username: mention }); + } + } + } + + // Add OG image as fallback if we have no media + if (output.media.length === 0 && globals?.meta?.ogImage) { + output.media.push({ + type: 'photo', + displayUrl: globals.meta.ogImage, + videoUrl: null, + dimensions: null, + accessibilityCaption: null, + }); + } + + return output; +} + +async function main() { + console.log(`Extracting Instagram posts for: ${username}`); + + // Load inventory + if (!existsSync(urlListFile)) { + console.error(`Inventory file not found: ${urlListFile}`); + console.error('Run discover.js first.'); + process.exit(1); + } + + const inventory = JSON.parse(readFileSync(urlListFile, 'utf8')); + const postsToProcess = (inventory.posts || []).slice(0, limit); + console.log(`Processing ${postsToProcess.length} posts from inventory...\n`); + + // Connect to browser + console.log(`Connecting to browser on CDP port ${cdpPort}...`); + const browser = await chromium.connectOverCDP(`http://127.0.0.1:${cdpPort}`); + const context = browser.contexts()[0] || await browser.newContext(); + const page = await context.newPage(); + console.log('Connected.\n'); + + const log = { processed: [], failed: [], mediaDownloaded: [] }; + const allMediaUrls = []; // { url, filename } pairs + + for (let i = 0; i < postsToProcess.length; i++) { + const inventoryPost = postsToProcess[i]; + const shortcode = inventoryPost.shortcode; + console.log(`[${i + 1}/${postsToProcess.length}] ${inventoryPost.url || shortcode}`); + + try { + const isCarousel = inventoryPost.type === 'carousel'; + const { captured, postDetail, carouselSlides } = await extractPostData( + page, shortcode, isCarousel, inventoryPost.carouselCount + ); + const output = buildPostOutput(shortcode, inventoryPost, postDetail, captured.globals, carouselSlides); + const safeCode = safeShortcode(shortcode); + + // Queue media for download and set local file paths before writing + for (let j = 0; j < output.media.length; j++) { + const item = output.media[j]; + const downloadUrl = item.videoUrl || item.displayUrl; + if (downloadUrl) { + const filename = mediaFilename(downloadUrl, shortcode, j); + allMediaUrls.push({ url: downloadUrl, filename }); + item.localFile = `output/media/${filename}`; + } + } + + writeFileSync(`output/pages/${safeCode}.json`, JSON.stringify(output, null, 2)); + + log.processed.push({ url: output.sourceUrl, shortcode }); + console.log(` Media items: ${output.media.length}, Tags: ${output.tags.length}`); + } catch (e) { + console.error(` FAILED: ${e.message}`); + log.failed.push({ shortcode, error: e.message }); + } + + if (i < postsToProcess.length - 1) await sleep(delay); + } + + await page.close(); + // Disconnect from the CDP session without closing the user's browser + if (typeof browser.disconnect === 'function') { + await browser.disconnect(); + } + + // Download all media (parallel with concurrency limit) + if (!skipMedia && allMediaUrls.length > 0) { + const CONCURRENCY = 8; + console.log(`\nDownloading ${allMediaUrls.length} media files (${CONCURRENCY} concurrent)...`); + let idx = 0; + + async function downloadWorker() { + while (idx < allMediaUrls.length) { + const i = idx++; + const { url, filename } = allMediaUrls[i]; + const dest = `output/media/${filename}`; + try { + await downloadFile(url, dest); + log.mediaDownloaded.push({ url, file: dest }); + process.stdout.write('.'); + } catch (e) { + log.failed.push({ url, error: `Media download: ${e.message}` }); + process.stdout.write('x'); + } + } + } + + await Promise.all(Array.from({ length: CONCURRENCY }, () => downloadWorker())); + console.log(''); + } else if (skipMedia) { + console.log('\nSkipping media download (--skip-media)'); + } + + writeFileSync('output/extraction-log.json', JSON.stringify(log, null, 2)); + console.log(`\nDone.`); + console.log(` Posts extracted: ${log.processed.length}`); + console.log(` Media downloaded: ${log.mediaDownloaded.length}`); + console.log(` Failures: ${log.failed.length}`); + if (log.failed.length) console.log(' See output/extraction-log.json for details'); +} + +main().catch(e => { console.error(e); process.exit(1); }); diff --git a/scripts/instagram/import.js b/scripts/instagram/import.js new file mode 100644 index 00000000..4e5e2955 --- /dev/null +++ b/scripts/instagram/import.js @@ -0,0 +1,496 @@ +#!/usr/bin/env node +/** + * import.js — Step 3: Import Instagram content to WordPress.com + * + * Reads output/ from extract.js and publishes to WordPress.com via XML-RPC. + * Import order: media → posts + * + * Uses XML-RPC (wp.uploadFile, wp.newPost) because WordPress.com's REST API + * does not support write operations with application passwords. + * + * Usage: + * node scripts/import.js --site mysite.wordpress.com --user your-wpcom-user --token APP_PASSWORD + * node scripts/import.js --site mysite.wordpress.com --user your-wpcom-user --token APP_PASSWORD --dry-run + * + * Options: + * --site WordPress.com site domain (e.g. mysite.wordpress.com) + * --user WordPress.com username that owns the application password + * --token Application password from wordpress.com/me/security/application-passwords + * --dry-run Show what would be imported without actually doing it + * --only Only import 'media', 'pages', or 'posts' + * + * Getting your application password: + * 1. Go to wordpress.com/me/security/application-passwords + * 2. Create a new application password + * 3. Copy the password and pass it as --token + */ + +import { readFileSync, readdirSync, existsSync } from 'fs'; +import { basename } from 'path'; + +const args = process.argv.slice(2); +function getArg(name) { + const i = args.indexOf(name); + return i !== -1 ? args[i + 1] : null; +} + +const site = getArg('--site'); +const token = getArg('--token'); +const user = getArg('--user'); +const dryRun = args.includes('--dry-run'); +const only = getArg('--only'); +const postType = getArg('--post-type'); // e.g. 'photo' for a custom post type + +if (!site || !token || !user) { + console.error('Usage: node scripts/import.js --site --user --token '); + console.error(' Get your app password at: wordpress.com/me/security/application-passwords'); + process.exit(1); +} + +const xmlRpcUrl = `https://${site}/xmlrpc.php`; +const restApiBase = `https://public-api.wordpress.com/rest/v1.1/sites/${site}`; + +// ─── XML-RPC helpers ──────────────────────────────────────── + +function escapeXml(value) { + return String(value) + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll("'", '''); +} + +function xmlValue(value) { + if (value == null) return ''; + if (Buffer.isBuffer(value)) return `${value.toString('base64')}`; + if (value instanceof Date) { + // XML-RPC dateTime.iso8601 must NOT include timezone suffix + const iso = value.toISOString().replace(/[-:]/g, '').replace(/\.\d{3}Z$/, ''); + return `${iso}`; + } + if (typeof value === 'boolean') return `${value ? 1 : 0}`; + if (typeof value === 'number') return Number.isInteger(value) ? `${value}` : `${value}`; + if (Array.isArray(value)) { + return `${value.map(item => `${xmlValue(item)}`).join('')}`; + } + if (typeof value === 'object') { + return `${Object.entries(value) + .filter(([, item]) => item !== undefined) + .map(([key, item]) => `${escapeXml(key)}${xmlValue(item)}`) + .join('')}`; + } + return `${escapeXml(value)}`; +} + +function parseXmlRpcResponse(xml) { + const faultMatch = xml.match(/[\s\S]*?faultString<\/name>\s*([\s\S]*?)<\/string><\/value>[\s\S]*?<\/fault>/); + if (faultMatch) throw new Error(faultMatch[1]); + + const struct = {}; + const namedStringRegex = /([^<]+)<\/name>\s*([\s\S]*?)<\/string><\/value>/g; + const namedIntRegex = /([^<]+)<\/name>\s*<(?:int|i4)>([\s\S]*?)<\/(?:int|i4)><\/value>/g; + let member; + while ((member = namedStringRegex.exec(xml))) struct[member[1]] = member[2]; + while ((member = namedIntRegex.exec(xml))) struct[member[1]] = Number(member[2]); + if (Object.keys(struct).length) return struct; + + const stringMatch = xml.match(/([\s\S]*?)<\/string>/); + if (stringMatch) return stringMatch[1]; + const intMatch = xml.match(/<(?:int|i4)>([\s\S]*?)<\/(?:int|i4)>/); + if (intMatch) return Number(intMatch[1]); + + return null; +} + +async function xmlRpcCall(methodName, params) { + const body = `${methodName}${params + .map(param => `${xmlValue(param)}`) + .join('')}`; + + const res = await fetch(xmlRpcUrl, { + method: 'POST', + headers: { 'Content-Type': 'text/xml' }, + body, + }); + const text = await res.text(); + if (!res.ok) throw new Error(`${methodName} → ${res.status}: ${text}`); + return parseXmlRpcResponse(text); +} + +let cachedBlogId = null; +async function getBlogId() { + if (cachedBlogId) return cachedBlogId; + const res = await fetch(restApiBase); + const data = await res.json().catch(() => ({})); + if (!data.ID) throw new Error(`Could not determine site ID for ${site}`); + cachedBlogId = data.ID; + return cachedBlogId; +} + +function guessMimeType(filename) { + const ext = filename.split('.').pop().toLowerCase(); + const types = { jpg: 'image/jpeg', jpeg: 'image/jpeg', png: 'image/png', gif: 'image/gif', webp: 'image/webp', mp4: 'video/mp4', mov: 'video/quicktime' }; + return types[ext] || 'application/octet-stream'; +} + +// Extract clean text content from accessibility tree nodes +function buildContentFromAccessibility(nodes) { + if (!nodes?.length) return ''; + const blocks = []; + for (const node of nodes) { + if (!node.name) continue; + if (node.role === 'heading') { + // Guess heading level from name length (crude but works as fallback) + blocks.push(`

    ${node.name}

    `); + } else if (['paragraph', 'StaticText', 'article', 'section'].includes(node.role)) { + blocks.push(`

    ${node.name}

    `); + } else if (node.role === 'img' && node.description) { + blocks.push(``); + } + } + return blocks.join('\n'); +} + +// Extract the best available content from a page JSON file +function extractContent(pageData) { + // Instagram posts — caption is the content, images are media attachments + if (pageData.platform === 'instagram' || pageData.shortcode) { + return buildInstagramContent(pageData); + } + + // Priority: Wix blog API response > JSON-LD > accessibility tree + for (const call of pageData.apiCalls || []) { + // Blog post body is typically in post.content or post.richContent + const body = call.data?.post?.content?.plainText || + call.data?.post?.richContent || + call.data?.content?.plainText; + if (body) return typeof body === 'string' ? `

    ${body}

    ` : JSON.stringify(body); + } + + // JSON-LD article body + const article = pageData.globals?.jsonLd?.find(j => j['@type'] === 'Article' || j['@type'] === 'BlogPosting'); + if (article?.articleBody) return `

    ${article.articleBody}

    `; + + // Fallback to accessibility tree + return buildContentFromAccessibility(pageData.accessibility); +} + +// Build WordPress block content from an Instagram post +function buildInstagramContent(pageData) { + const blocks = []; + const media = pageData.media || []; + const imageMedia = media.filter(m => m.type !== 'video' || !m.videoUrl); + const videoMedia = media.filter(m => m.type === 'video' && m.videoUrl); + + // Use a gallery block for carousels (multiple images), single image block otherwise + if (imageMedia.length > 1) { + const galleryImages = imageMedia.map(item => { + const src = item.localFile || item.displayUrl; + if (!src) return ''; + const alt = item.accessibilityCaption || ''; + return `\n
    ${alt.replace(/
    \n`; + }).filter(Boolean); + blocks.push(`\n
    \n${galleryImages.join('\n')}\n
    \n`); + } else if (imageMedia.length === 1) { + const item = imageMedia[0]; + const src = item.localFile || item.displayUrl; + if (src) { + const alt = item.accessibilityCaption || pageData.caption?.slice(0, 125) || ''; + blocks.push(`\n
    ${alt.replace(/
    \n`); + } + } + + // Videos as separate blocks (can't go in gallery) + for (const item of videoMedia) { + blocks.push(`\n
    \n`); + } + + // Caption as a paragraph + if (pageData.caption) { + // Convert @mentions and #hashtags to links + let caption = pageData.caption + .replace(/@(\w+)/g, '@$1') + .replace(/#(\w+)/g, '#$1'); + blocks.push(`\n

    ${caption}

    \n`); + } + + // Link to original Instagram post + if (pageData.shortcode) { + const igUrl = `https://www.instagram.com/p/${pageData.shortcode}/`; + blocks.push(`\n

    Originally posted on Instagram

    \n`); + } + + return blocks.join('\n\n'); +} + +function extractMeta(pageData) { + // Instagram posts + if (pageData.platform === 'instagram' || pageData.shortcode) { + const caption = pageData.caption || ''; + // Title: first line of caption, or first 60 chars, or shortcode + const title = caption.split('\n')[0]?.slice(0, 80) || `Instagram ${pageData.shortcode}`; + return { + title, + description: caption.slice(0, 300), + featuredImageUrl: pageData.media?.[0]?.displayUrl || null, + publishDate: pageData.date || null, + modifiedDate: null, + slug: `ig-${pageData.shortcode}`, + }; + } + + const meta = pageData.globals?.meta || {}; + const jsonLd = pageData.globals?.jsonLd || []; + const article = jsonLd.find(j => ['Article', 'BlogPosting', 'WebPage'].includes(j['@type'])); + + return { + title: meta.ogTitle || meta.title || article?.headline || pageData.slug, + description: meta.description || meta.ogDescription || article?.description || '', + featuredImageUrl: meta.ogImage || article?.image?.url || null, + publishDate: article?.datePublished || null, + modifiedDate: article?.dateModified || null, + slug: pageData.slug, + }; +} + +async function uploadMedia(filePath, filename) { + if (dryRun) { + console.log(` [dry-run] Would upload: ${filename}`); + return { id: 0, source_url: `https://example.com/wp-content/uploads/${filename}` }; + } + + const fileBuffer = readFileSync(filePath); + const blogId = await getBlogId(); + const result = await xmlRpcCall('wp.uploadFile', [ + blogId, + user, + token, + { + name: filename, + type: guessMimeType(filename), + bits: fileBuffer, + overwrite: true, + }, + ]); + + return { + id: result.id || 0, + source_url: result.url, + }; +} + +async function importMedia() { + if (!existsSync('output/media')) { console.log('No media folder found.'); return {}; } + + const files = readdirSync('output/media'); + console.log(`\nUploading ${files.length} media files...`); + + const mediaMap = {}; // original filename → { url, id } + for (const file of files) { + process.stdout.write(` ${file}... `); + try { + const result = await uploadMedia(`output/media/${file}`, file); + mediaMap[file] = { url: result.source_url, id: result.id }; + console.log(`✓ ${result.source_url}`); + } catch (e) { + console.log(`✗ ${e.message}`); + } + } + return mediaMap; +} + +async function importPage(pageData, mediaMap) { + const meta = extractMeta(pageData); + let content = extractContent(pageData); + + for (const [filename, media] of Object.entries(mediaMap)) { + const url = typeof media === 'string' ? media : media.url; + content = content.replaceAll(filename, url); + } + + if (dryRun) { + console.log(` [dry-run] Would create page: ${meta.title} (${meta.slug})`); + return { id: 0, link: '#' }; + } + + const blogId = await getBlogId(); + const id = await xmlRpcCall('wp.newPost', [ + blogId, user, token, + { + post_type: 'page', + post_status: 'draft', + post_title: meta.title, + post_content: content, + post_excerpt: meta.description, + wp_slug: meta.slug, + }, + ]); + + return { id, link: `https://${site}/wp-admin/post.php?post=${id}&action=edit` }; +} + +async function importPost(pageData, mediaMap) { + const meta = extractMeta(pageData); + let content = extractContent(pageData); + + // Replace local file paths with uploaded WordPress URLs + for (const [filename, media] of Object.entries(mediaMap)) { + const url = typeof media === 'string' ? media : media.url; + content = content.replaceAll(filename, url); + } + + if (dryRun) { + console.log(` [dry-run] Would create post: ${meta.title} (${meta.slug})`); + return { id: 0, link: '#' }; + } + + const blogId = await getBlogId(); + + // Format date as "YYYY-MM-DD HH:MM:SS" string — WordPress.com ignores + // dateTime.iso8601 typed values but parses string dates correctly + let postDate; + if (meta.publishDate) { + const d = new Date(meta.publishDate); + postDate = `${d.getUTCFullYear()}-${String(d.getUTCMonth()+1).padStart(2,'0')}-${String(d.getUTCDate()).padStart(2,'0')} ${String(d.getUTCHours()).padStart(2,'0')}:${String(d.getUTCMinutes()).padStart(2,'0')}:${String(d.getUTCSeconds()).padStart(2,'0')}`; + } + + // Find featured image: first media item's WordPress media ID + let featuredImageId; + if (pageData.media?.[0]?.localFile) { + const firstMediaFile = basename(pageData.media[0].localFile); + const mediaEntry = mediaMap[firstMediaFile]; + if (mediaEntry?.id) featuredImageId = mediaEntry.id; + } + + const postData = { + post_type: postType || 'post', + post_status: 'publish', + post_title: meta.title, + post_content: content, + post_excerpt: meta.description, + wp_slug: meta.slug, + post_date: postDate, + post_thumbnail: featuredImageId || undefined, + }; + + const id = await xmlRpcCall('wp.newPost', [blogId, user, token, postData]); + + return { id, link: `https://${site}/wp-admin/post.php?post=${id}&action=edit` }; +} + +async function main() { + if (dryRun) console.log('[DRY RUN — no changes will be made]\n'); + + if (!existsSync('output/pages')) { + console.error('No output/pages directory found. Run extract.js first.'); + process.exit(1); + } + + const pageFiles = readdirSync('output/pages').filter(f => f.endsWith('.json')); + console.log(`Found ${pageFiles.length} extracted pages`); + + // Detect if this is an Instagram import + let isInstagram = false; + if (existsSync('output/inventory.json')) { + const inventory = JSON.parse(readFileSync('output/inventory.json', 'utf8')); + isInstagram = inventory.platform === 'instagram'; + } + + // Determine content type from inventory if available + let typeMap = {}; + if (existsSync('output/inventory.json')) { + const inventory = JSON.parse(readFileSync('output/inventory.json', 'utf8')); + for (const item of inventory.urls) { + if (isInstagram) { + // Instagram uses shortcodes as filenames + typeMap[item.shortcode] = item.type || 'photo'; + } else { + const slug = new URL(item.url).pathname.replace(/^\//, '').replace(/\//g, '--') || 'homepage'; + typeMap[slug] = item.type; + } + } + } + + const urlMap = []; // old URL → new WP URL, for redirect map + + // Step 1: Upload media + let mediaMap = {}; + if (!only || only === 'media') { + mediaMap = await importMedia(); + } + + // Instagram: all items are posts (not pages) + if (isInstagram) { + console.log(`\nImporting ${pageFiles.length} Instagram posts${postType ? ` as "${postType}"` : ''}...`); + for (const file of pageFiles) { + const pageData = JSON.parse(readFileSync(`output/pages/${file}`, 'utf8')); + const shortcode = file.replace('.json', ''); + process.stdout.write(` ${shortcode}... `); + try { + const result = await importPost(pageData, mediaMap); + console.log(`✓ ${result.link}`); + if (pageData.sourceUrl) urlMap.push({ old: pageData.sourceUrl, new: result.link }); + } catch (e) { + console.log(`✗ ${e.message}`); + } + } + } else { + // Step 2: Import pages + const pages = pageFiles.filter(f => { + const slug = f.replace('.json', ''); + return !typeMap[slug] || typeMap[slug] === 'page' || typeMap[slug] === 'homepage'; + }); + + if (!only || only === 'pages') { + console.log(`\nImporting ${pages.length} pages...`); + for (const file of pages) { + const pageData = JSON.parse(readFileSync(`output/pages/${file}`, 'utf8')); + const slug = file.replace('.json', ''); + process.stdout.write(` ${slug}... `); + try { + const result = await importPage(pageData, mediaMap); + console.log(`✓ ${result.link}`); + if (pageData.sourceUrl) urlMap.push({ old: pageData.sourceUrl, new: result.link }); + } catch (e) { + console.log(`✗ ${e.message}`); + } + } + } + + // Step 3: Import posts + const posts = pageFiles.filter(f => { + const slug = f.replace('.json', ''); + return typeMap[slug] === 'blog-post'; + }); + + if (!only || only === 'posts') { + console.log(`\nImporting ${posts.length} blog posts...`); + for (const file of posts) { + const pageData = JSON.parse(readFileSync(`output/pages/${file}`, 'utf8')); + const slug = file.replace('.json', ''); + process.stdout.write(` ${slug}... `); + try { + const result = await importPost(pageData, mediaMap); + console.log(`✓ ${result.link}`); + if (pageData.sourceUrl) urlMap.push({ old: pageData.sourceUrl, new: result.link }); + } catch (e) { + console.log(`✗ ${e.message}`); + } + } + } + } + + // Output redirect map + if (urlMap.length) { + const { writeFileSync } = await import('fs'); + writeFileSync('output/redirect-map.json', JSON.stringify(urlMap, null, 2)); + console.log(`\nRedirect map written to output/redirect-map.json`); + console.log('Use this to set up 301 redirects from your old Wix URLs to WordPress.'); + } + + console.log('\nImport complete. All content created as drafts — review in WordPress admin before publishing.'); + console.log(`https://${site}/wp-admin/`); +} + +main().catch(e => { console.error(e); process.exit(1); }); diff --git a/tests/instagram.test.js b/tests/instagram.test.js new file mode 100644 index 00000000..8a911cd0 --- /dev/null +++ b/tests/instagram.test.js @@ -0,0 +1,444 @@ +#!/usr/bin/env node +/** + * tests/instagram.test.js — Unit tests for Instagram extraction and import + * + * Verifies data transformation logic without requiring a live Instagram + * session or WordPress site. Uses Node's built-in test runner (Node 18+). + * + * Usage: + * node --test tests/instagram.test.js + */ + +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; + +// ─── Test fixtures ────────────────────────────────────────── + +const singlePhotoPost = { + shortcode: 'ABC123', + type: 'photo', + date: '2023-06-15T14:30:00.000Z', + caption: 'Hello @world! #sunset #photography', + media: [ + { + type: 'photo', + displayUrl: 'https://scontent.cdninstagram.com/v/t51.29350-15/12345_67890.jpg', + localFile: 'output/media/ABC123_0.jpg', + accessibilityCaption: 'A sunset over the ocean', + }, + ], + locationName: 'Santa Monica', + carouselCount: null, +}; + +const carouselPost = { + shortcode: 'XYZ789', + type: 'carousel', + date: '2024-12-25T16:00:00.000Z', + caption: 'Holiday vibes! @santa #christmas', + carouselCount: 3, + media: [ + { + type: 'photo', + displayUrl: 'https://scontent.cdninstagram.com/v/t51.29350-15/111_222.jpg', + localFile: 'output/media/XYZ789_0.jpg', + accessibilityCaption: 'First slide', + }, + { + type: 'photo', + displayUrl: 'https://scontent.cdninstagram.com/v/t51.82787-15/333_444.jpg', + localFile: 'output/media/XYZ789_1.jpg', + accessibilityCaption: 'Second slide', + }, + { + type: 'photo', + displayUrl: 'https://scontent.cdninstagram.com/v/t51.82787-15/555_666.jpg', + localFile: 'output/media/XYZ789_2.jpg', + accessibilityCaption: 'Third slide', + }, + ], +}; + +const videoPost = { + shortcode: 'VID456', + type: 'video', + date: '2022-01-10T08:00:00.000Z', + caption: 'Check this out', + media: [ + { + type: 'video', + displayUrl: null, + videoUrl: 'https://scontent.cdninstagram.com/v/t50/video.mp4', + localFile: 'output/media/VID456_0.mp4', + }, + ], +}; + +const noCaptionPost = { + shortcode: 'NOCAP', + type: 'photo', + date: '2021-05-01T12:00:00.000Z', + caption: '', + media: [ + { + type: 'photo', + displayUrl: 'https://scontent.cdninstagram.com/v/t51.29350-15/999_888.jpg', + localFile: 'output/media/NOCAP_0.jpg', + }, + ], +}; + +// ─── Import the functions under test ──────────────────────── + +// We can't directly import from import.js since it has side effects +// (arg parsing, process.exit). Instead, we replicate the pure functions +// here and test them. In a real setup, these would be exported. + +function extractMeta(pageData) { + if (pageData.platform === 'instagram' || pageData.shortcode) { + const caption = pageData.caption || ''; + const title = caption.split('\n')[0]?.slice(0, 80) || `Instagram ${pageData.shortcode}`; + return { + title, + description: caption.slice(0, 300), + featuredImageUrl: pageData.media?.[0]?.displayUrl || null, + publishDate: pageData.date || null, + modifiedDate: null, + slug: `ig-${pageData.shortcode}`, + }; + } + return null; +} + +function buildInstagramContent(pageData) { + const blocks = []; + const media = pageData.media || []; + const imageMedia = media.filter(m => m.type !== 'video' || !m.videoUrl); + const videoMedia = media.filter(m => m.type === 'video' && m.videoUrl); + + if (imageMedia.length > 1) { + const galleryImages = imageMedia.map(item => { + const src = item.localFile || item.displayUrl; + if (!src) return ''; + const alt = item.accessibilityCaption || ''; + return `\n
    ${alt.replace(/
    \n`; + }).filter(Boolean); + blocks.push(`\n
    \n${galleryImages.join('\n')}\n
    \n`); + } else if (imageMedia.length === 1) { + const item = imageMedia[0]; + const src = item.localFile || item.displayUrl; + if (src) { + const alt = item.accessibilityCaption || pageData.caption?.slice(0, 125) || ''; + blocks.push(`\n
    ${alt.replace(/
    \n`); + } + } + + for (const item of videoMedia) { + blocks.push(`\n
    \n`); + } + + if (pageData.caption) { + let caption = pageData.caption + .replace(/@(\w+)/g, '@$1') + .replace(/#(\w+)/g, '#$1'); + blocks.push(`\n

    ${caption}

    \n`); + } + + if (pageData.shortcode) { + const igUrl = `https://www.instagram.com/p/${pageData.shortcode}/`; + blocks.push(`\n

    Originally posted on Instagram

    \n`); + } + + return blocks.join('\n\n'); +} + +function escapeXml(value) { + return String(value) + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll("'", '''); +} + +function xmlValue(value) { + if (value == null) return ''; + if (Buffer.isBuffer(value)) return `${value.toString('base64')}`; + if (value instanceof Date) { + const iso = value.toISOString().replace(/[-:]/g, '').replace(/\.\d{3}Z$/, ''); + return `${iso}`; + } + if (typeof value === 'boolean') return `${value ? 1 : 0}`; + if (typeof value === 'number') return Number.isInteger(value) ? `${value}` : `${value}`; + if (Array.isArray(value)) { + return `${value.map(item => `${xmlValue(item)}`).join('')}`; + } + if (typeof value === 'object') { + return `${Object.entries(value) + .filter(([, item]) => item !== undefined) + .map(([key, item]) => `${escapeXml(key)}${xmlValue(item)}`) + .join('')}`; + } + return `${escapeXml(value)}`; +} + +function extractPostMeta(node) { + const caption = node.edge_media_to_caption?.edges?.[0]?.node?.text || ''; + return { + id: node.id, + shortcode: node.shortcode, + type: node.edge_sidecar_to_children ? 'carousel' : node.is_video ? 'video' : 'photo', + timestamp: node.taken_at_timestamp, + date: node.taken_at_timestamp ? new Date(node.taken_at_timestamp * 1000).toISOString() : null, + caption, + displayUrl: node.display_url, + isVideo: !!node.is_video, + videoUrl: node.video_url || null, + accessibilityCaption: node.accessibility_caption || null, + locationName: node.location?.name || null, + likes: node.edge_media_preview_like?.count ?? null, + comments: node.edge_media_to_comment?.count ?? null, + carouselCount: node.edge_sidecar_to_children?.edges?.length || null, + url: `https://www.instagram.com/p/${node.shortcode}/`, + }; +} + +// ─── Tests ────────────────────────────────────────────────── + +describe('extractMeta', () => { + it('extracts title from first line of caption', () => { + const meta = extractMeta(singlePhotoPost); + assert.equal(meta.title, 'Hello @world! #sunset #photography'); + }); + + it('truncates long titles to 80 chars', () => { + const post = { ...singlePhotoPost, caption: 'A'.repeat(100) + '\nsecond line' }; + const meta = extractMeta(post); + assert.equal(meta.title.length, 80); + }); + + it('falls back to shortcode when no caption', () => { + const meta = extractMeta(noCaptionPost); + assert.equal(meta.title, 'Instagram NOCAP'); + }); + + it('generates ig- prefixed slug', () => { + const meta = extractMeta(singlePhotoPost); + assert.equal(meta.slug, 'ig-ABC123'); + }); + + it('preserves original publish date', () => { + const meta = extractMeta(singlePhotoPost); + assert.equal(meta.publishDate, '2023-06-15T14:30:00.000Z'); + }); + + it('sets featured image URL from first media item', () => { + const meta = extractMeta(singlePhotoPost); + assert.ok(meta.featuredImageUrl.includes('cdninstagram.com')); + }); +}); + +describe('buildInstagramContent', () => { + it('produces wp:image block for single photo', () => { + const content = buildInstagramContent(singlePhotoPost); + assert.ok(content.includes('')); + assert.ok(!content.includes('')); + assert.ok(content.includes('video.mp4')); + }); + + it('handles posts with no caption', () => { + const content = buildInstagramContent(noCaptionPost); + // Should have image but no caption paragraph (empty caption) + assert.ok(content.includes('')); + // Source link should still be present + assert.ok(content.includes('instagram-source')); + }); + + it('uses accessibility caption as alt text', () => { + const content = buildInstagramContent(singlePhotoPost); + assert.ok(content.includes('alt="A sunset over the ocean"')); + }); +}); + +describe('xmlValue', () => { + it('encodes strings with XML escaping', () => { + assert.equal(xmlValue('hello & '), 'hello & <world>'); + }); + + it('encodes integers', () => { + assert.equal(xmlValue(42), '42'); + }); + + it('encodes booleans', () => { + assert.equal(xmlValue(true), '1'); + assert.equal(xmlValue(false), '0'); + }); + + it('encodes dates WITHOUT trailing Z', () => { + const d = new Date('2020-03-15T12:00:00.000Z'); + const result = xmlValue(d); + assert.ok(result.includes('20200315T120000')); + assert.ok(!result.includes('Z')); + }); + + it('encodes buffers as base64', () => { + const buf = Buffer.from('hello'); + assert.equal(xmlValue(buf), 'aGVsbG8='); + }); + + it('encodes null as nil', () => { + assert.equal(xmlValue(null), ''); + }); + + it('encodes objects as structs', () => { + const result = xmlValue({ name: 'test', count: 5 }); + assert.ok(result.includes('')); + assert.ok(result.includes('name')); + assert.ok(result.includes('test')); + assert.ok(result.includes('5')); + }); + + it('skips undefined values in structs', () => { + const result = xmlValue({ name: 'test', missing: undefined }); + assert.ok(!result.includes('missing')); + }); + + it('encodes arrays', () => { + const result = xmlValue([1, 'two']); + assert.ok(result.includes('')); + assert.ok(result.includes('1')); + assert.ok(result.includes('two')); + }); +}); + +describe('extractPostMeta (discover)', () => { + it('classifies photo posts', () => { + const meta = extractPostMeta({ + id: '1', shortcode: 'TEST', display_url: 'http://example.com/img.jpg', + taken_at_timestamp: 1686830000, + edge_media_to_caption: { edges: [{ node: { text: 'Hello' } }] }, + }); + assert.equal(meta.type, 'photo'); + assert.equal(meta.caption, 'Hello'); + }); + + it('classifies carousel posts', () => { + const meta = extractPostMeta({ + id: '2', shortcode: 'CAR', display_url: 'http://example.com/img.jpg', + taken_at_timestamp: 1686830000, + edge_sidecar_to_children: { edges: [{ node: {} }, { node: {} }] }, + edge_media_to_caption: { edges: [] }, + }); + assert.equal(meta.type, 'carousel'); + assert.equal(meta.carouselCount, 2); + }); + + it('classifies video posts', () => { + const meta = extractPostMeta({ + id: '3', shortcode: 'VID', display_url: 'http://example.com/img.jpg', + taken_at_timestamp: 1686830000, is_video: true, + video_url: 'http://example.com/video.mp4', + edge_media_to_caption: { edges: [] }, + }); + assert.equal(meta.type, 'video'); + assert.ok(meta.isVideo); + assert.equal(meta.videoUrl, 'http://example.com/video.mp4'); + }); + + it('converts timestamp to ISO date', () => { + const meta = extractPostMeta({ + id: '4', shortcode: 'DATE', display_url: 'http://example.com/img.jpg', + taken_at_timestamp: 1686830000, + edge_media_to_caption: { edges: [] }, + }); + assert.ok(meta.date.startsWith('2023-06-15')); + }); + + it('extracts location name', () => { + const meta = extractPostMeta({ + id: '5', shortcode: 'LOC', display_url: 'http://example.com/img.jpg', + taken_at_timestamp: 1686830000, + location: { name: 'Central Park', id: '123' }, + edge_media_to_caption: { edges: [] }, + }); + assert.equal(meta.locationName, 'Central Park'); + }); + + it('generates correct Instagram URL', () => { + const meta = extractPostMeta({ + id: '6', shortcode: 'URL_TEST', display_url: 'http://example.com/img.jpg', + taken_at_timestamp: 1686830000, + edge_media_to_caption: { edges: [] }, + }); + assert.equal(meta.url, 'https://www.instagram.com/p/URL_TEST/'); + }); +}); + +describe('date formatting for WordPress', () => { + it('formats date as YYYY-MM-DD HH:MM:SS string', () => { + const d = new Date('2023-06-15T14:30:00.000Z'); + const formatted = `${d.getUTCFullYear()}-${String(d.getUTCMonth()+1).padStart(2,'0')}-${String(d.getUTCDate()).padStart(2,'0')} ${String(d.getUTCHours()).padStart(2,'0')}:${String(d.getUTCMinutes()).padStart(2,'0')}:${String(d.getUTCSeconds()).padStart(2,'0')}`; + assert.equal(formatted, '2023-06-15 14:30:00'); + }); + + it('pads single-digit months and days', () => { + const d = new Date('2023-01-05T03:09:07.000Z'); + const formatted = `${d.getUTCFullYear()}-${String(d.getUTCMonth()+1).padStart(2,'0')}-${String(d.getUTCDate()).padStart(2,'0')} ${String(d.getUTCHours()).padStart(2,'0')}:${String(d.getUTCMinutes()).padStart(2,'0')}:${String(d.getUTCSeconds()).padStart(2,'0')}`; + assert.equal(formatted, '2023-01-05 03:09:07'); + }); +}); + +describe('carousel deduplication logic', () => { + it('deduplicates by Instagram media ID', () => { + // Simulate what the extractor does: collect images across slides, + // deduplicate by the numeric media ID prefix in the CDN URL + const allImages = [ + { src: 'https://cdn.com/v/t51.29350-15/111_222.jpg', mediaId: '111' }, + { src: 'https://cdn.com/v/t51.82787-15/111_222.jpg', mediaId: '111' }, // same photo, different CDN path + { src: 'https://cdn.com/v/t51.82787-15/333_444.jpg', mediaId: '333' }, + { src: 'https://cdn.com/v/t51.82787-15/555_666.jpg', mediaId: '555' }, + ]; + + const seen = new Set(); + const deduped = []; + for (const img of allImages) { + if (!img.mediaId || seen.has(img.mediaId)) continue; + seen.add(img.mediaId); + deduped.push(img); + } + + assert.equal(deduped.length, 3); + assert.deepEqual(deduped.map(d => d.mediaId), ['111', '333', '555']); + }); +}); From b2f5aaba32560e80b2748867a94b9152eeb292c0 Mon Sep 17 00:00:00 2001 From: Matthew Batchelder Date: Sun, 12 Apr 2026 08:21:23 -0700 Subject: [PATCH 2/3] Adapt Instagram migration into the new adapter structure Reworks the Instagram support originally added in #4 (by @edequalsawesome) on top of the new adapter/lib architecture introduced in #6. - src/adapters/instagram.ts implementing PlatformAdapter: - discover() scrolls the profile via CDP and intercepts GraphQL responses - extract() visits each post via CDP, walks ?img_index=N for carousel slides (deduped by Instagram media ID), downloads media, and emits wp:image / wp:gallery / wp:video Gutenberg blocks - hashtags become WP tags; @mentions and #hashtags in captions are linked - Registered in src/mcp-server.ts and detect-platform URL patterns - Removed the legacy scripts/instagram/ scripts and tests/instagram.test.js, which targeted the pre-restructure standalone-script entry points - Added test/adapters/instagram.test.ts with a fixture-driven dry-run that exercises the WXR pipeline end to end - Refreshed prompts/instagram.md for the new npm run liberate workflow - Updated README, AGENTS.md, CONTRIBUTING.md, docs/mcp.md to mention Instagram --- AGENTS.md | 3 +- CONTRIBUTING.md | 2 +- README.md | 3 +- docs/mcp.md | 2 +- prompts/instagram.md | 97 ++-- scripts/instagram/discover.js | 487 ---------------- scripts/instagram/extract.js | 487 ---------------- scripts/instagram/import.js | 496 ---------------- src/adapters/instagram.ts | 774 +++++++++++++++++++++++++ src/lib/extraction/detect-platform.ts | 1 + src/mcp-server.ts | 3 +- test/adapters/instagram.test.ts | 209 +++++++ test/detect-platform.test.ts | 5 + test/fixtures/instagram-inventory.json | 61 ++ tests/instagram.test.js | 444 -------------- 15 files changed, 1111 insertions(+), 1963 deletions(-) delete mode 100644 scripts/instagram/discover.js delete mode 100644 scripts/instagram/extract.js delete mode 100644 scripts/instagram/import.js create mode 100644 src/adapters/instagram.ts create mode 100644 test/adapters/instagram.test.ts create mode 100644 test/fixtures/instagram-inventory.json delete mode 100644 tests/instagram.test.js diff --git a/AGENTS.md b/AGENTS.md index 3b11dd24..00e0f8dc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ ## Overview -`data-liberation-agent` extracts content from closed web platforms (Wix, Squarespace, Webflow, Shopify) and produces WordPress-compatible WXR files. All four platform adapters are implemented. +`data-liberation-agent` extracts content from closed web platforms (Wix, Squarespace, Webflow, Shopify, Instagram) and produces WordPress-compatible WXR files. All five platform adapters are implemented. Three entry points — MCP server (11 tools), CLI (`src/cli.ts`), and Claude Code plugin (`claude plugin add .`) — all share `src/lib/` and `src/adapters/`. The plugin just wraps the MCP server. @@ -23,3 +23,4 @@ Adapters produce structured content and call into `WxrBuilder`, `ExtractionLog`, - `classifyUrl` types: `homepage`, `post`, `product`, `gallery`, `event`, `page` (no `category`/`author`/`other`) - Media filename collision handling uses numeric suffixes (`-2`, `-3`), not hashes - `detect-platform` uses domain-level URL patterns and HTTP fingerprinting (headers + HTML markers) — no path-based detection +- Instagram adapter requires `cdpPort` for both `discover` and `extract` — Instagram has no public sitemap or unauthenticated API. Discover scrolls the profile and intercepts GraphQL responses; extract visits each post via `?img_index=N` for carousel slide capture. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ea4381c8..89601e18 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -115,7 +115,7 @@ A maintainer (human or AI with repo access) will review the PR. If the discovery ``` src/ - adapters/ Platform-specific extraction (wix.ts, squarespace.ts, webflow.ts, shopify.ts, shared.ts) + adapters/ Platform-specific extraction (wix.ts, squarespace.ts, webflow.ts, shopify.ts, instagram.ts, shared.ts) lib/ extraction/ WXR builder, sitemap, media, detection, extraction log import/ WordPress REST API importer, WooCommerce CSV diff --git a/README.md b/README.md index e4bfe69e..91d8d96a 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,9 @@ This tool extracts all content from closed platforms — posts, pages, media, na | **Squarespace** | Ready | [`prompts/squarespace.md`](./prompts/squarespace.md) | | **Webflow** | Ready | [`prompts/webflow.md`](./prompts/webflow.md) | | **Shopify** (blog/pages/products) | Ready | [`prompts/shopify.md`](./prompts/shopify.md) | +| **Instagram** (posts, carousels, videos) | Ready | [`prompts/instagram.md`](./prompts/instagram.md) | -All four platforms have MCP adapters with full extraction support including products (exported as WooCommerce-compatible CSV). +All five platforms have MCP adapters with full extraction support. Wix/Squarespace/Webflow/Shopify export products as WooCommerce-compatible CSV; Instagram exports posts as a WXR with `wp:image` / `wp:gallery` / `wp:video` blocks and hashtag-derived tags. ## AI tool integration diff --git a/docs/mcp.md b/docs/mcp.md index 656cf0ac..5c4a867a 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -18,7 +18,7 @@ Detect the platform of a website. |-----------|----------|-------------| | `url` | yes | The URL of the website to detect | -Returns: `platform` (wix, squarespace, webflow, shopify, or unknown), `confidence` (high/medium/low), `signals` (what was detected). +Returns: `platform` (wix, squarespace, webflow, shopify, instagram, or unknown), `confidence` (high/medium/low), `signals` (what was detected). ### liberate_discover diff --git a/prompts/instagram.md b/prompts/instagram.md index 404a7b7b..f5519bd6 100644 --- a/prompts/instagram.md +++ b/prompts/instagram.md @@ -1,77 +1,86 @@ -# Instagram to WordPress.com Migration Prompt +# Instagram to WordPress Migration Prompt Copy everything below this line and paste it into your AI assistant (Claude, ChatGPT, Gemini, etc.). --- -I want to migrate my Instagram photos and posts to WordPress.com. My Instagram username is: **[PASTE YOUR USERNAME HERE]** +I want to migrate my Instagram posts to WordPress. My Instagram username (or profile URL) is: **[PASTE YOUR USERNAME OR PROFILE URL HERE]** -I have (or will create) a WordPress.com account. Please help me migrate using the playbook at https://github.com/Automattic/data-liberation-agent — read AGENTS.md first for full instructions. +I have (or will create) a WordPress site. Please help me migrate using the playbook at https://github.com/Automattic/data-liberation-agent — read AGENTS.md first for full instructions. -**Important**: Instagram requires an authenticated browser session. I'll need to have Chrome (or another Chromium browser) open and logged into Instagram before we start. +**Important**: Instagram has no public sitemap and no unauthenticated API. The Instagram adapter requires an authenticated browser session connected via Chrome DevTools Protocol (CDP). I'll need Chrome (or another Chromium browser) launched with `--remote-debugging-port` and logged into Instagram **before** we run discovery or extraction. -Here's what I need you to do: +## Step 1: Inspect -## Step 1: Set up browser access - -Help me launch Chrome with remote debugging enabled so the migration scripts can connect: +```bash +npm run inspect -- https://www.instagram.com/[MY-USERNAME]/ +``` -1. Quit Chrome completely -2. Relaunch with: `"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" --remote-debugging-port=9222 --user-data-dir="$HOME/.data-liberation/cdp-profile/chrome" --restore-last-session` -3. Log into Instagram in the browser window that opens -4. Confirm the connection works +This confirms the platform is detected as Instagram and shows what the adapter expects. -## Step 2: Discover all my posts +## Step 2: Launch Chrome with remote debugging and log in to Instagram ```bash -node scripts/instagram/discover.js MY_USERNAME --cdp-port 9222 -``` +# macOS: +/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome \ + --remote-debugging-port=9222 \ + --user-data-dir="$HOME/.data-liberation/cdp-profile/chrome" \ + --restore-last-session -This connects to my browser, navigates to my profile, and intercepts Instagram's internal GraphQL API responses as it scrolls through all my posts. It captures: -- Post metadata (captions, dates, locations, hashtags, tagged users) -- Post types (photos, videos, carousels with slide counts) -- Image and video URLs -- Profile information +# Linux: +google-chrome --remote-debugging-port=9222 -Show me the inventory summary and wait for my approval before proceeding. +# Windows: +"C:\Program Files\Google\Chrome\Application\chrome.exe" --remote-debugging-port=9222 +``` -**If it stalls or gets rate limited**: Add `--delay 3000` for a gentler pace. +In that Chrome window, navigate to `https://www.instagram.com/` and log in. Confirm I'm logged in before continuing. -## Step 3: Extract full content and download media +## Step 3: Discover all my posts ```bash -node scripts/instagram/extract.js MY_USERNAME --cdp-port 9222 +npm run liberate -- https://www.instagram.com/[MY-USERNAME]/ --output ./output --cdp-port 9222 --verbose --discover-only ``` -This visits each post individually to get: -- Full-resolution images (not thumbnails) -- All carousel slides (uses `?img_index=N` to access each slide directly) -- Video URLs -- Tagged users, location details, accessibility captions +The Instagram adapter will: +- Open a new tab in the connected Chrome window and navigate to my profile +- Scroll the profile, intercepting Instagram's GraphQL responses to capture post metadata (captions, dates, locations, hashtags, photo/video/carousel type, slide counts) +- Build an inventory at `output/inventory.json` + +**If it stalls or gets rate limited**, add `--delay 3000` for a gentler scroll cadence. -All media is downloaded locally — Instagram CDN URLs expire, so this must happen promptly after discovery. +Show me the inventory summary (counts by type, total post count, profile metadata) and wait for my approval before extracting. -## Step 4: Import to WordPress.com +## Step 4: Extract content and download media ```bash -node scripts/import.js --site my-wp-site.wordpress.com --token MY_APP_PASSWORD +npm run liberate -- https://www.instagram.com/[MY-USERNAME]/ --output ./output --cdp-port 9222 --verbose ``` -This creates WordPress posts from the extracted data: -- Each Instagram post becomes a WordPress post (as draft) -- Images uploaded to the media library -- Captions become post content with @mentions and #hashtags linked -- Original post date preserved -- Instagram shortcode and URL stored as post meta +For each post the adapter will: +- Visit the post URL in the connected browser +- Capture full-resolution images from the intercepted media API responses +- For carousels, walk `?img_index=N` for every slide and dedupe by Instagram media ID +- Download all images and videos locally (Instagram CDN URLs expire — this must happen promptly after discovery) +- Build a `wp:image` block for photos, `wp:video` for videos, and `wp:gallery` for carousels +- Convert hashtags to WordPress tags and `@mentions` / `#hashtags` in captions to links +- Append a "View on Instagram" source link to each post -**For a custom post type** (e.g. a "photo" CPT): add `--post-type photo` +If extraction is interrupted, resume with `--resume`. ## Step 5: Verify -When done: -- Show me how many posts were imported vs. discovered -- Flag any posts with missing images or import errors -- Check that carousel posts have all their slides -- List the date range covered (oldest → newest) +```bash +npm run verify -- ./output +``` + +Show me the verification report and flag any posts with missing media or failed extractions. + +## Step 6: Import to WordPress + +```bash +npm run setup -- --site [MY-WORDPRESS-SITE] --username [MY-USERNAME] --token [APP-PASSWORD] +npm run import -- ./output --site [MY-WORDPRESS-SITE] --username [MY-USERNAME] --token [APP-PASSWORD] +``` Work methodically — do one step at a time, show me progress, and wait for my go-ahead before moving to the next step. diff --git a/scripts/instagram/discover.js b/scripts/instagram/discover.js deleted file mode 100644 index 0ca79c3e..00000000 --- a/scripts/instagram/discover.js +++ /dev/null @@ -1,487 +0,0 @@ -#!/usr/bin/env node -/** - * discover.js — Step 1: Inventory an Instagram profile - * - * Connects to a browser where you're logged into Instagram, navigates - * to your profile, and intercepts the GraphQL API responses as you - * scroll to build a complete manifest of all posts. - * - * REQUIRES: a running browser with CDP enabled and an active Instagram session. - * Instagram's API requires authentication — there's no public sitemap to crawl. - * - * Usage: - * node scripts/instagram/discover.js --cdp-port 9222 - * node scripts/instagram/discover.js --cdp-port 9222 --limit 50 - * - * Options: - * --cdp-port CDP port of your running browser (required) - * --limit Stop after N posts (for testing) - * --delay Delay between scroll actions (default: 2000) - * - * Output: - * output/inventory.json — manifest of all discovered posts with metadata - */ - -import { chromium } from 'playwright'; -import { writeFileSync, mkdirSync } from 'fs'; - -const args = process.argv.slice(2); -const username = args.find(a => !a.startsWith('--')); -if (!username) { - console.error('Usage: node scripts/instagram/discover.js --cdp-port '); - process.exit(1); -} - -function parseIntArg(name, fallback) { - const idx = args.indexOf(name); - if (idx === -1) return fallback; - const val = parseInt(args[idx + 1], 10); - if (!Number.isFinite(val)) { - console.error(`Error: ${name} requires a numeric value.`); - process.exit(1); - } - return val; -} - -const cdpPort = parseIntArg('--cdp-port', null); -if (!cdpPort) { - console.error('Error: --cdp-port is required. Instagram needs an authenticated browser session.'); - console.error('Launch Chrome with: google-chrome --remote-debugging-port=9222'); - process.exit(1); -} - -const limit = parseIntArg('--limit', Infinity); -const scrollDelay = parseIntArg('--delay', 2000); - -mkdirSync('output', { recursive: true }); - -function sleep(ms) { - return new Promise(r => setTimeout(r, ms)); -} - -// Classify an Instagram post by its type -function classifyPost(node) { - if (node.__typename === 'GraphSidecar' || node.edge_sidecar_to_children) return 'carousel'; - if (node.__typename === 'GraphVideo' || node.is_video) return 'video'; - return 'photo'; -} - -// Extract post metadata from a GraphQL edge node -function extractPostMeta(node) { - const caption = node.edge_media_to_caption?.edges?.[0]?.node?.text || ''; - return { - id: node.id, - shortcode: node.shortcode, - type: classifyPost(node), - timestamp: node.taken_at_timestamp, - date: node.taken_at_timestamp ? new Date(node.taken_at_timestamp * 1000).toISOString() : null, - caption, - displayUrl: node.display_url, - thumbnailUrl: node.thumbnail_src || node.thumbnail_resources?.[0]?.src, - dimensions: node.dimensions || null, - isVideo: !!node.is_video, - videoUrl: node.video_url || null, - accessibilityCaption: node.accessibility_caption || null, - locationName: node.location?.name || null, - locationId: node.location?.id || null, - likes: node.edge_media_preview_like?.count ?? node.edge_liked_by?.count ?? null, - comments: node.edge_media_to_comment?.count ?? node.edge_media_preview_comment?.count ?? null, - // For carousels, note how many slides - carouselCount: node.edge_sidecar_to_children?.edges?.length || null, - url: `https://www.instagram.com/p/${node.shortcode}/`, - }; -} - -async function main() { - console.log(`Discovering Instagram posts for: ${username}`); - console.log(`Connecting to browser on CDP port ${cdpPort}...`); - - const browser = await chromium.connectOverCDP(`http://127.0.0.1:${cdpPort}`); - const context = browser.contexts()[0] || await browser.newContext(); - const page = await context.newPage(); - - const posts = new Map(); // shortcode → post data (dedupes) - let profileData = null; - let hasMore = true; - let endCursor = null; - let paginationRequests = 0; - - // Intercept GraphQL responses to capture post data - page.on('response', async (response) => { - const url = response.url(); - if (!url.includes('/graphql/query') && !url.includes('/api/v1/')) return; - - const ct = response.headers()['content-type'] || ''; - if (!ct.includes('application/json') && !ct.includes('text/javascript')) return; - - try { - const body = await response.json(); - - // Profile page initial load — data is in entry_data or in the graphql response - const userData = body?.data?.user || - body?.graphql?.user || - body?.data?.xdt_api__v1__feed__user_timeline_graphql_connection; - - if (userData) { - // Extract profile info on first encounter - if (!profileData && (userData.username || userData.full_name)) { - profileData = { - id: userData.id || userData.pk || null, - username: userData.username || username, - fullName: userData.full_name || '', - biography: userData.biography || userData.bio_text || '', - profilePicUrl: userData.profile_pic_url_hd || userData.profile_pic_url || '', - postCount: userData.edge_owner_to_timeline_media?.count ?? - userData.media_count ?? null, - followerCount: userData.edge_followed_by?.count ?? - userData.follower_count ?? null, - followingCount: userData.edge_follow?.count ?? - userData.following_count ?? null, - isPrivate: userData.is_private || false, - isVerified: userData.is_verified || false, - }; - console.log(` Profile: ${profileData.fullName} (@${profileData.username})`); - if (profileData.postCount) { - console.log(` Total posts reported: ${profileData.postCount}`); - } - } - - // Extract posts from the timeline media edges - const timeline = userData.edge_owner_to_timeline_media || - userData.edge_web_feed_timeline; - - if (timeline?.edges) { - for (const edge of timeline.edges) { - const node = edge.node; - if (node?.shortcode && !posts.has(node.shortcode)) { - posts.set(node.shortcode, extractPostMeta(node)); - } - } - // Track pagination cursor - if (timeline.page_info) { - hasMore = timeline.page_info.has_next_page; - endCursor = timeline.page_info.end_cursor; - } - paginationRequests++; - console.log(` Captured ${posts.size} posts (page ${paginationRequests})...`); - } - - // Handle the newer API format (xdt_api__v1__feed) - if (userData.edges) { - for (const edge of userData.edges) { - const node = edge.node; - if (node?.code && !posts.has(node.code)) { - posts.set(node.code, { - id: node.pk || node.id, - shortcode: node.code, - type: node.carousel_media_count ? 'carousel' : node.video_versions ? 'video' : 'photo', - timestamp: node.taken_at, - date: node.taken_at ? new Date(node.taken_at * 1000).toISOString() : null, - caption: node.caption?.text || '', - displayUrl: node.image_versions2?.candidates?.[0]?.url || '', - thumbnailUrl: node.image_versions2?.candidates?.slice(-1)?.[0]?.url || '', - dimensions: node.original_width && node.original_height - ? { width: node.original_width, height: node.original_height } : null, - isVideo: !!node.video_versions, - videoUrl: node.video_versions?.[0]?.url || null, - accessibilityCaption: node.accessibility_caption || null, - locationName: node.location?.name || null, - locationId: node.location?.pk || null, - likes: node.like_count ?? null, - comments: node.comment_count ?? null, - carouselCount: node.carousel_media_count || null, - url: `https://www.instagram.com/p/${node.code}/`, - }); - } - } - if (userData.page_info) { - hasMore = userData.page_info.has_next_page; - endCursor = userData.page_info.end_cursor; - } - paginationRequests++; - console.log(` Captured ${posts.size} posts (page ${paginationRequests})...`); - } - } - } catch { - // Not all responses are JSON — that's fine - } - }); - - // Navigate to the profile - const profileUrl = `https://www.instagram.com/${username}/`; - console.log(`\nNavigating to ${profileUrl}`); - try { - await page.goto(profileUrl, { waitUntil: 'domcontentloaded', timeout: 60000 }); - // Wait for the profile content to appear (response interceptor captures GraphQL data) - await sleep(3000); - } catch (e) { - if (posts.size > 0) { - console.log(` Navigation timeout, but ${posts.size} posts already captured — continuing`); - } else { - console.error(` Navigation failed: ${e.message}`); - await browser.disconnect(); - process.exit(1); - } - } - - // Check if we're logged in by looking for login prompts - const loginPrompt = await page.$('input[name="username"]'); - if (loginPrompt) { - console.error('\nError: Not logged into Instagram in this browser session.'); - console.error('Please log in to Instagram in your browser first, then re-run.'); - await browser.close(); - process.exit(1); - } - - // Check for private profile - const privateMsg = await page.$('text=This account is private'); - if (privateMsg) { - console.warn('\nWarning: This is a private profile. You can only extract your own posts or accounts you follow.'); - } - - // Also try to capture data from the page's window globals - const windowData = await page.evaluate(() => { - const result = {}; - - // Check specific known globals (avoid enumerating all of window) - for (const key of ['__additionalDataLoaded', '__NEXT_DATA__', '_sharedData']) { - if (window[key]) { - try { result[key] = window[key]; } catch {} - } - } - - // Check for require'd modules (older Instagram) - try { - if (window._sharedData?.entry_data?.ProfilePage?.[0]?.graphql?.user) { - result.profilePageData = window._sharedData.entry_data.ProfilePage[0].graphql.user; - } - } catch {} - - return result; - }); - - // Extract any posts from window globals - const globalUser = windowData?.profilePageData; - if (globalUser?.edge_owner_to_timeline_media?.edges) { - for (const edge of globalUser.edge_owner_to_timeline_media.edges) { - const node = edge.node; - if (node?.shortcode && !posts.has(node.shortcode)) { - posts.set(node.shortcode, extractPostMeta(node)); - } - } - if (!profileData && globalUser.username) { - profileData = { - username: globalUser.username, - fullName: globalUser.full_name || '', - biography: globalUser.biography || '', - profilePicUrl: globalUser.profile_pic_url_hd || '', - postCount: globalUser.edge_owner_to_timeline_media?.count ?? null, - followerCount: globalUser.edge_followed_by?.count ?? null, - followingCount: globalUser.edge_follow?.count ?? null, - isPrivate: globalUser.is_private || false, - isVerified: globalUser.is_verified || false, - }; - } - console.log(` Extracted ${posts.size} posts from page data`); - } - - // Phase 1: Scroll to load a couple pages and capture the GraphQL patterns - console.log('\nScrolling to load more posts...'); - let scrollAttempts = 0; - let lastPostCount = posts.size; - let noNewPostsStreak = 0; - let currentDelay = scrollDelay; - - while (posts.size < limit && scrollAttempts < 100) { - const scrollStart = Date.now(); - await page.evaluate(() => window.scrollTo(0, document.documentElement.scrollHeight)); - await sleep(currentDelay); - scrollAttempts++; - - if (posts.size === lastPostCount) { - noNewPostsStreak++; - // Back off when we're not getting new posts — Instagram may be throttling - currentDelay = Math.min(currentDelay * 1.5, scrollDelay * 4); - if (noNewPostsStreak >= 4) break; - } else { - noNewPostsStreak = 0; - // New posts arrived — ease back toward base delay - currentDelay = Math.max(scrollDelay, currentDelay * 0.8); - lastPostCount = posts.size; - console.log(` Scroll ${scrollAttempts}: ${posts.size} posts`); - } - } - - // Phase 2: If we have a cursor and need more posts, use direct GraphQL requests - // executed in the page context (inherits cookies and CSRF tokens) - if (hasMore && endCursor && posts.size < limit) { - console.log(`\nUsing direct GraphQL pagination (cursor available)...`); - - // First, extract the user ID from what we've already captured - const userId = profileData?.id || await page.evaluate((uname) => { - // Try to find the user ID from page source or existing data - const bodyText = document.body.innerHTML; - const idMatch = bodyText.match(/"profilePage_(\d+)"/); - return idMatch ? idMatch[1] : null; - }, username); - - if (!userId) { - // Get user ID via the profile page's metadata - const userIdFromMeta = await page.evaluate(() => { - // Instagram embeds the user ID in various places - const scripts = document.querySelectorAll('script'); - for (const s of scripts) { - const match = s.textContent.match(/"user_id":"(\d+)"/); - if (match) return match[1]; - } - // Also check meta tags - const instagramUrl = document.querySelector('meta[property="al:android:url"]')?.content; - if (instagramUrl) { - const m = instagramUrl.match(/(\d+)/); - if (m) return m[1]; - } - return null; - }); - - if (userIdFromMeta) { - if (!profileData) profileData = {}; - profileData.id = userIdFromMeta; - } - } - - const resolvedUserId = profileData?.id || userId; - - if (resolvedUserId) { - console.log(` User ID: ${resolvedUserId}`); - let cursor = endCursor; - let apiPage = 0; - let apiErrors = 0; - - while (hasMore && cursor && posts.size < limit && apiErrors < 3) { - apiPage++; - try { - // Execute the GraphQL query from within the page context - // This inherits all cookies, CSRF tokens, and headers - const result = await page.evaluate(async ({ userId, after }) => { - // Instagram uses a specific query hash for user media pagination - // We'll try the documented endpoint first - const variables = JSON.stringify({ - id: userId, - first: 12, - after: after, - }); - - // Try the newer API endpoint first - const url = `https://www.instagram.com/graphql/query/?query_hash=472f257a40c653c64c666ce877d59d2b&variables=${encodeURIComponent(variables)}`; - - const res = await fetch(url, { - headers: { - 'X-Requested-With': 'XMLHttpRequest', - 'Accept': '*/*', - }, - credentials: 'include', - }); - - if (!res.ok) return { error: `HTTP ${res.status}` }; - const text = await res.text(); - try { - return JSON.parse(text); - } catch { - return { error: `Not JSON (starts with: ${text.slice(0, 50)})` }; - } - }, { userId: resolvedUserId, after: cursor }); - - if (result.error) { - console.log(` API error: ${result.error}`); - // Rate limited or challenged — wait and retry once - apiErrors++; - console.log(` Waiting 10s and retrying (${apiErrors}/3)...`); - await sleep(10000); - continue; - } - - const media = result?.data?.user?.edge_owner_to_timeline_media; - if (media?.edges) { - for (const edge of media.edges) { - const node = edge.node; - if (node?.shortcode && !posts.has(node.shortcode)) { - posts.set(node.shortcode, extractPostMeta(node)); - } - } - hasMore = media.page_info?.has_next_page ?? false; - cursor = media.page_info?.end_cursor ?? null; - console.log(` API page ${apiPage}: ${posts.size} total posts`); - } else { - // Response format might have changed — try to find data elsewhere - console.log(` Unexpected response format on page ${apiPage}`); - break; - } - - // Rate limit: be polite — API calls need more delay than scrolling - await sleep(Math.max(scrollDelay, 3000)); - } catch (e) { - console.log(` API pagination error: ${e.message}`); - break; - } - } - } else { - console.log(' Could not determine user ID — falling back to scroll only'); - // Continue scrolling as fallback - while (posts.size < limit && scrollAttempts < 200) { - await page.evaluate(() => window.scrollTo(0, document.documentElement.scrollHeight)); - await sleep(scrollDelay); - scrollAttempts++; - if (posts.size > lastPostCount) { - lastPostCount = posts.size; - noNewPostsStreak = 0; - if (scrollAttempts % 5 === 0) console.log(` Scroll ${scrollAttempts}: ${posts.size} posts`); - } else { - noNewPostsStreak++; - if (noNewPostsStreak >= 8) break; - } - } - } - } - - await page.close(); - await browser.disconnect(); - - // Build inventory - const allPosts = [...posts.values()].sort((a, b) => (b.timestamp || 0) - (a.timestamp || 0)); - - const counts = { photo: 0, video: 0, carousel: 0 }; - for (const post of allPosts) { - counts[post.type] = (counts[post.type] || 0) + 1; - } - - const inventory = { - platform: 'instagram', - username, - profile: profileData, - discoveredAt: new Date().toISOString(), - counts, - urls: allPosts.map(p => ({ - url: p.url, - type: p.type, - id: p.id, - shortcode: p.shortcode, - })), - posts: allPosts, - }; - - writeFileSync('output/inventory.json', JSON.stringify(inventory, null, 2)); - - console.log('\nInventory summary:'); - for (const [type, count] of Object.entries(counts)) { - console.log(` ${type}: ${count}`); - } - console.log(`\nTotal: ${allPosts.length} posts discovered`); - if (profileData?.postCount) { - const pct = Math.round((allPosts.length / profileData.postCount) * 100); - console.log(`Coverage: ${allPosts.length}/${profileData.postCount} (${pct}%)`); - } - console.log('Written to output/inventory.json'); - console.log('\nReview this inventory before running extract.js'); -} - -main().catch(e => { console.error(e); process.exit(1); }); diff --git a/scripts/instagram/extract.js b/scripts/instagram/extract.js deleted file mode 100644 index 5a8e9984..00000000 --- a/scripts/instagram/extract.js +++ /dev/null @@ -1,487 +0,0 @@ -#!/usr/bin/env node -/** - * extract.js — Step 2: Extract full content from Instagram posts - * - * Takes the inventory from discover.js and visits each post individually - * to get full-resolution images, carousel slides, video URLs, and comments. - * - * Usage: - * node scripts/instagram/extract.js --cdp-port 9222 - * node scripts/instagram/extract.js --cdp-port 9222 --limit 10 - * node scripts/instagram/extract.js --cdp-port 9222 --skip-media - * - * Options: - * --cdp-port CDP port of your running browser (required) - * --delay Delay between posts (default: 1500) - * --limit Only process first N posts (for testing) - * --skip-media Extract metadata only, don't download images/videos - * --url-list Use inventory from discover.js (default: output/inventory.json) - * - * Output: - * output/pages/.json — extracted post data - * output/media/ — downloaded images and videos - * output/extraction-log.json — summary of what was extracted - */ - -import { chromium } from 'playwright'; -import { writeFileSync, mkdirSync, readFileSync, existsSync, createWriteStream } from 'fs'; -import { basename } from 'path'; -import https from 'https'; -import http from 'http'; - -const args = process.argv.slice(2); -const username = args.find(a => !a.startsWith('--')); -if (!username) { - console.error('Usage: node scripts/instagram/extract.js --cdp-port '); - process.exit(1); -} - -function parseIntArg(name, fallback) { - const idx = args.indexOf(name); - if (idx === -1) return fallback; - const val = parseInt(args[idx + 1], 10); - if (!Number.isFinite(val)) { - console.error(`Error: ${name} requires a numeric value.`); - process.exit(1); - } - return val; -} - -const cdpPort = parseIntArg('--cdp-port', null); -if (!cdpPort) { - console.error('Error: --cdp-port is required.'); - process.exit(1); -} - -const delay = parseIntArg('--delay', 1500); -const limit = parseIntArg('--limit', Infinity); -const skipMedia = args.includes('--skip-media'); -const urlListArg = args.indexOf('--url-list'); -const urlListFile = urlListArg !== -1 ? args[urlListArg + 1] : 'output/inventory.json'; - -mkdirSync('output/pages', { recursive: true }); -mkdirSync('output/media', { recursive: true }); - -function sleep(ms) { - return new Promise(r => setTimeout(r, ms)); -} - -function downloadFile(url, destPath, redirectsLeft = 5) { - return new Promise((resolve, reject) => { - if (redirectsLeft <= 0) return reject(new Error('Too many redirects')); - if (!url.startsWith('https://')) return reject(new Error(`Refused non-HTTPS URL: ${url}`)); - - const file = createWriteStream(destPath); - file.on('error', reject); - - const req = https.get(url, { headers: { 'User-Agent': 'Mozilla/5.0' }, timeout: 30000 }, res => { - if ([301, 302, 303, 307, 308].includes(res.statusCode)) { - res.resume(); // consume response to free socket - file.close(); - const location = res.headers.location; - if (!location?.startsWith('https://')) return reject(new Error(`Redirect to non-HTTPS: ${location}`)); - return downloadFile(location, destPath, redirectsLeft - 1).then(resolve).catch(reject); - } - if (res.statusCode !== 200) { - res.resume(); - file.close(); - return reject(new Error(`HTTP ${res.statusCode}`)); - } - res.pipe(file); - file.on('finish', () => file.close(resolve)); - }); - req.on('error', reject); - req.on('timeout', () => { req.destroy(); reject(new Error('Download timeout')); }); - }); -} - -// Sanitize shortcode for safe use as filename -function safeShortcode(sc) { - return sc.replace(/[^a-zA-Z0-9_-]/g, ''); -} - -// Generate a sane filename from an Instagram CDN URL -function mediaFilename(url, shortcode, index) { - const ext = url.match(/\.(jpg|jpeg|png|webp|mp4|mov)/i)?.[1] || 'jpg'; - return `${safeShortcode(shortcode)}_${index}.${ext.toLowerCase()}`; -} - -async function extractPostData(page, shortcode, isCarousel = false, carouselCount = 10) { - const postUrl = `https://www.instagram.com/p/${shortcode}/`; - const captured = { apiCalls: [], globals: null }; - - // Intercept API responses for this post - const responseHandler = async (response) => { - const url = response.url(); - if (!url.includes('/graphql/query') && !url.includes('/api/v1/media/')) return; - - const ct = response.headers()['content-type'] || ''; - if (!ct.includes('application/json') && !ct.includes('text/javascript')) return; - - try { - const body = await response.json(); - captured.apiCalls.push({ url, data: body }); - } catch {} - }; - - page.on('response', responseHandler); - - try { - await page.goto(postUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }); - // Wait for the main post image or GraphQL response to land - await page.waitForSelector('article img[src*="cdninstagram.com"]', { timeout: 10000 }).catch(() => {}); - } catch (e) { - console.error(` Navigation failed: ${e.message}`); - } - - page.off('response', responseHandler); - - // Extract data from the page — Instagram embeds post data in script tags and window objects - captured.globals = await page.evaluate(() => { - const result = {}; - - // JSON-LD (Instagram provides this for public posts) - result.jsonLd = Array.from( - document.querySelectorAll('script[type="application/ld+json"]') - ).map(s => { - try { return JSON.parse(s.textContent); } catch { return null; } - }).filter(Boolean); - - // Check specific known globals (avoid enumerating all of window) - for (const key of ['__additionalDataLoaded', '__NEXT_DATA__', '_sharedData']) { - if (window[key]) { - try { result[key] = window[key]; } catch {} - } - } - - // Meta tags — Instagram sets good OG tags - result.meta = { - title: document.title, - description: document.querySelector('meta[name="description"]')?.content, - ogTitle: document.querySelector('meta[property="og:title"]')?.content, - ogDescription: document.querySelector('meta[property="og:description"]')?.content, - ogImage: document.querySelector('meta[property="og:image"]')?.content, - ogType: document.querySelector('meta[property="og:type"]')?.content, - }; - - return result; - }); - - // Try to find the post's media from the captured API calls - let postDetail = null; - for (const call of captured.apiCalls) { - const data = call.data; - - // Look for the post in various response shapes - const media = data?.data?.xdt_shortcode_media || - data?.graphql?.shortcode_media || - data?.data?.shortcode_media || - data?.items?.[0]; - - if (media && (media.shortcode === shortcode || media.code === shortcode)) { - postDetail = media; - break; - } - } - - // For carousel posts: click through the carousel arrows to capture each slide's - // full-res image. Instagram's carousel arrows are button[aria-label="Next"] and - // are NOT inside the article element — they're in a parent container. - const carouselSlides = []; - if (isCarousel) { - // Strategy: Instagram supports ?img_index=N to load a specific carousel slide. - // Navigate to each slide directly and grab the main post image. - const expectedSlides = carouselCount || 10; - - // Strategy: navigate to each ?img_index=N, collect ALL
  • images - // from every page load, then deduplicate by Instagram media ID - // (the numeric prefix in the CDN URL). Instagram keeps 3
  • elements - // in the DOM (previous, current, next), so we see overlap between - // adjacent slides — deduplication handles this cleanly. - const seenMediaIds = new Set(); - - const getAllSlideImages = async () => { - return page.evaluate(() => { - const results = []; - for (const img of document.querySelectorAll('li img')) { - const src = img.src || ''; - if (!src.includes('cdninstagram.com/v/t51.')) continue; - if (img.alt?.includes('User avatar')) continue; - const rect = img.getBoundingClientRect(); - if (rect.width < 300) continue; - // Extract the Instagram media ID (unique per photo) - const idMatch = src.match(/\/(\d{5,})_/); - results.push({ src, mediaId: idMatch?.[1] || null }); - } - // Also check for video - const video = document.querySelector('li video[src], li video source, video[src], video source'); - const videoUrl = video?.src || video?.querySelector?.('source')?.src || null; - if (videoUrl) results.push({ src: null, videoUrl, mediaId: 'video_' + Date.now() }); - return results; - }); - }; - - for (let slideIdx = 1; slideIdx <= expectedSlides; slideIdx++) { - try { - const slideUrl = `https://www.instagram.com/p/${shortcode}/?img_index=${slideIdx}`; - await page.goto(slideUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }).catch(() => {}); - await page.waitForSelector('li img[src*="cdninstagram.com"]', { timeout: 8000 }).catch(() => {}); - await sleep(800); - - const images = await getAllSlideImages(); - for (const img of images) { - if (!img.mediaId || seenMediaIds.has(img.mediaId)) continue; - seenMediaIds.add(img.mediaId); - carouselSlides.push({ - type: img.videoUrl ? 'video' : 'photo', - displayUrl: img.src || null, - videoUrl: img.videoUrl || null, - }); - } - } catch { - break; - } - } - - if (carouselSlides.length > 1) { - console.log(` Carousel: ${carouselSlides.length} slides captured`); - } - } - - return { captured, postDetail, carouselSlides }; -} - -function buildPostOutput(shortcode, inventoryPost, postDetail, globals, carouselSlides = []) { - const output = { - shortcode, - sourceUrl: `https://www.instagram.com/p/${shortcode}/`, - extractedAt: new Date().toISOString(), - // Start with inventory data as baseline - ...inventoryPost, - // Enrich with detail data if we got it - media: [], - tags: [], - mentionedUsers: [], - }; - - if (postDetail) { - // Full caption (inventory might have truncated it) - output.caption = postDetail.edge_media_to_caption?.edges?.[0]?.node?.text || - postDetail.caption?.text || - output.caption; - - // Location details - if (postDetail.location) { - output.location = { - name: postDetail.location.name, - id: postDetail.location.id || postDetail.location.pk, - slug: postDetail.location.slug || null, - lat: postDetail.location.lat || null, - lng: postDetail.location.lng || null, - address: (() => { - try { return postDetail.location.address_json ? JSON.parse(postDetail.location.address_json) : null; } - catch { return null; } - })(), - }; - } - - // Tagged users - const taggedEdges = postDetail.edge_media_to_tagged_user?.edges || []; - output.mentionedUsers = taggedEdges.map(e => ({ - username: e.node?.user?.username, - fullName: e.node?.user?.full_name, - x: e.node?.x, - y: e.node?.y, - })); - - // Extract all media items (handles single posts, carousels, and videos) - if (postDetail.edge_sidecar_to_children?.edges) { - // Carousel post - for (const edge of postDetail.edge_sidecar_to_children.edges) { - const child = edge.node; - output.media.push({ - type: child.is_video ? 'video' : 'photo', - displayUrl: child.display_url || child.display_resources?.slice(-1)?.[0]?.src, - videoUrl: child.video_url || null, - dimensions: child.dimensions || null, - accessibilityCaption: child.accessibility_caption || null, - }); - } - } else if (postDetail.carousel_media) { - // Newer API format for carousels - for (const child of postDetail.carousel_media) { - output.media.push({ - type: child.video_versions ? 'video' : 'photo', - displayUrl: child.image_versions2?.candidates?.[0]?.url, - videoUrl: child.video_versions?.[0]?.url || null, - dimensions: child.original_width && child.original_height - ? { width: child.original_width, height: child.original_height } : null, - accessibilityCaption: child.accessibility_caption || null, - }); - } - } else { - // Single photo or video - output.media.push({ - type: postDetail.is_video ? 'video' : 'photo', - displayUrl: postDetail.display_url || - postDetail.image_versions2?.candidates?.[0]?.url, - videoUrl: postDetail.video_url || - postDetail.video_versions?.[0]?.url || null, - dimensions: postDetail.dimensions || - (postDetail.original_width && postDetail.original_height - ? { width: postDetail.original_width, height: postDetail.original_height } : null), - accessibilityCaption: postDetail.accessibility_caption || null, - }); - } - } else { - // No detail data — fall back to what discover gave us - if (inventoryPost?.displayUrl) { - output.media.push({ - type: inventoryPost.isVideo ? 'video' : 'photo', - displayUrl: inventoryPost.displayUrl, - videoUrl: inventoryPost.videoUrl || null, - dimensions: inventoryPost.dimensions || null, - accessibilityCaption: inventoryPost.accessibilityCaption || null, - }); - } - } - - // For carousels: if we only got 1 media item from the API but we have - // carousel slides from clicking through the UI, use those instead - if (carouselSlides.length > 1 && output.media.length <= 1) { - output.media = carouselSlides.map(slide => ({ - ...slide, - dimensions: null, - accessibilityCaption: null, - })); - } - - // Extract hashtags and @mentions from caption - if (output.caption) { - output.tags = [...output.caption.matchAll(/#(\w+)/g)].map(m => m[1]); - const mentions = [...output.caption.matchAll(/@(\w+)/g)].map(m => m[1]); - // Merge with tagged users - for (const mention of mentions) { - if (!output.mentionedUsers.find(u => u.username === mention)) { - output.mentionedUsers.push({ username: mention }); - } - } - } - - // Add OG image as fallback if we have no media - if (output.media.length === 0 && globals?.meta?.ogImage) { - output.media.push({ - type: 'photo', - displayUrl: globals.meta.ogImage, - videoUrl: null, - dimensions: null, - accessibilityCaption: null, - }); - } - - return output; -} - -async function main() { - console.log(`Extracting Instagram posts for: ${username}`); - - // Load inventory - if (!existsSync(urlListFile)) { - console.error(`Inventory file not found: ${urlListFile}`); - console.error('Run discover.js first.'); - process.exit(1); - } - - const inventory = JSON.parse(readFileSync(urlListFile, 'utf8')); - const postsToProcess = (inventory.posts || []).slice(0, limit); - console.log(`Processing ${postsToProcess.length} posts from inventory...\n`); - - // Connect to browser - console.log(`Connecting to browser on CDP port ${cdpPort}...`); - const browser = await chromium.connectOverCDP(`http://127.0.0.1:${cdpPort}`); - const context = browser.contexts()[0] || await browser.newContext(); - const page = await context.newPage(); - console.log('Connected.\n'); - - const log = { processed: [], failed: [], mediaDownloaded: [] }; - const allMediaUrls = []; // { url, filename } pairs - - for (let i = 0; i < postsToProcess.length; i++) { - const inventoryPost = postsToProcess[i]; - const shortcode = inventoryPost.shortcode; - console.log(`[${i + 1}/${postsToProcess.length}] ${inventoryPost.url || shortcode}`); - - try { - const isCarousel = inventoryPost.type === 'carousel'; - const { captured, postDetail, carouselSlides } = await extractPostData( - page, shortcode, isCarousel, inventoryPost.carouselCount - ); - const output = buildPostOutput(shortcode, inventoryPost, postDetail, captured.globals, carouselSlides); - const safeCode = safeShortcode(shortcode); - - // Queue media for download and set local file paths before writing - for (let j = 0; j < output.media.length; j++) { - const item = output.media[j]; - const downloadUrl = item.videoUrl || item.displayUrl; - if (downloadUrl) { - const filename = mediaFilename(downloadUrl, shortcode, j); - allMediaUrls.push({ url: downloadUrl, filename }); - item.localFile = `output/media/${filename}`; - } - } - - writeFileSync(`output/pages/${safeCode}.json`, JSON.stringify(output, null, 2)); - - log.processed.push({ url: output.sourceUrl, shortcode }); - console.log(` Media items: ${output.media.length}, Tags: ${output.tags.length}`); - } catch (e) { - console.error(` FAILED: ${e.message}`); - log.failed.push({ shortcode, error: e.message }); - } - - if (i < postsToProcess.length - 1) await sleep(delay); - } - - await page.close(); - // Disconnect from the CDP session without closing the user's browser - if (typeof browser.disconnect === 'function') { - await browser.disconnect(); - } - - // Download all media (parallel with concurrency limit) - if (!skipMedia && allMediaUrls.length > 0) { - const CONCURRENCY = 8; - console.log(`\nDownloading ${allMediaUrls.length} media files (${CONCURRENCY} concurrent)...`); - let idx = 0; - - async function downloadWorker() { - while (idx < allMediaUrls.length) { - const i = idx++; - const { url, filename } = allMediaUrls[i]; - const dest = `output/media/${filename}`; - try { - await downloadFile(url, dest); - log.mediaDownloaded.push({ url, file: dest }); - process.stdout.write('.'); - } catch (e) { - log.failed.push({ url, error: `Media download: ${e.message}` }); - process.stdout.write('x'); - } - } - } - - await Promise.all(Array.from({ length: CONCURRENCY }, () => downloadWorker())); - console.log(''); - } else if (skipMedia) { - console.log('\nSkipping media download (--skip-media)'); - } - - writeFileSync('output/extraction-log.json', JSON.stringify(log, null, 2)); - console.log(`\nDone.`); - console.log(` Posts extracted: ${log.processed.length}`); - console.log(` Media downloaded: ${log.mediaDownloaded.length}`); - console.log(` Failures: ${log.failed.length}`); - if (log.failed.length) console.log(' See output/extraction-log.json for details'); -} - -main().catch(e => { console.error(e); process.exit(1); }); diff --git a/scripts/instagram/import.js b/scripts/instagram/import.js deleted file mode 100644 index 4e5e2955..00000000 --- a/scripts/instagram/import.js +++ /dev/null @@ -1,496 +0,0 @@ -#!/usr/bin/env node -/** - * import.js — Step 3: Import Instagram content to WordPress.com - * - * Reads output/ from extract.js and publishes to WordPress.com via XML-RPC. - * Import order: media → posts - * - * Uses XML-RPC (wp.uploadFile, wp.newPost) because WordPress.com's REST API - * does not support write operations with application passwords. - * - * Usage: - * node scripts/import.js --site mysite.wordpress.com --user your-wpcom-user --token APP_PASSWORD - * node scripts/import.js --site mysite.wordpress.com --user your-wpcom-user --token APP_PASSWORD --dry-run - * - * Options: - * --site WordPress.com site domain (e.g. mysite.wordpress.com) - * --user WordPress.com username that owns the application password - * --token Application password from wordpress.com/me/security/application-passwords - * --dry-run Show what would be imported without actually doing it - * --only Only import 'media', 'pages', or 'posts' - * - * Getting your application password: - * 1. Go to wordpress.com/me/security/application-passwords - * 2. Create a new application password - * 3. Copy the password and pass it as --token - */ - -import { readFileSync, readdirSync, existsSync } from 'fs'; -import { basename } from 'path'; - -const args = process.argv.slice(2); -function getArg(name) { - const i = args.indexOf(name); - return i !== -1 ? args[i + 1] : null; -} - -const site = getArg('--site'); -const token = getArg('--token'); -const user = getArg('--user'); -const dryRun = args.includes('--dry-run'); -const only = getArg('--only'); -const postType = getArg('--post-type'); // e.g. 'photo' for a custom post type - -if (!site || !token || !user) { - console.error('Usage: node scripts/import.js --site --user --token '); - console.error(' Get your app password at: wordpress.com/me/security/application-passwords'); - process.exit(1); -} - -const xmlRpcUrl = `https://${site}/xmlrpc.php`; -const restApiBase = `https://public-api.wordpress.com/rest/v1.1/sites/${site}`; - -// ─── XML-RPC helpers ──────────────────────────────────────── - -function escapeXml(value) { - return String(value) - .replaceAll('&', '&') - .replaceAll('<', '<') - .replaceAll('>', '>') - .replaceAll('"', '"') - .replaceAll("'", '''); -} - -function xmlValue(value) { - if (value == null) return ''; - if (Buffer.isBuffer(value)) return `${value.toString('base64')}`; - if (value instanceof Date) { - // XML-RPC dateTime.iso8601 must NOT include timezone suffix - const iso = value.toISOString().replace(/[-:]/g, '').replace(/\.\d{3}Z$/, ''); - return `${iso}`; - } - if (typeof value === 'boolean') return `${value ? 1 : 0}`; - if (typeof value === 'number') return Number.isInteger(value) ? `${value}` : `${value}`; - if (Array.isArray(value)) { - return `${value.map(item => `${xmlValue(item)}`).join('')}`; - } - if (typeof value === 'object') { - return `${Object.entries(value) - .filter(([, item]) => item !== undefined) - .map(([key, item]) => `${escapeXml(key)}${xmlValue(item)}`) - .join('')}`; - } - return `${escapeXml(value)}`; -} - -function parseXmlRpcResponse(xml) { - const faultMatch = xml.match(/[\s\S]*?faultString<\/name>\s*([\s\S]*?)<\/string><\/value>[\s\S]*?<\/fault>/); - if (faultMatch) throw new Error(faultMatch[1]); - - const struct = {}; - const namedStringRegex = /([^<]+)<\/name>\s*([\s\S]*?)<\/string><\/value>/g; - const namedIntRegex = /([^<]+)<\/name>\s*<(?:int|i4)>([\s\S]*?)<\/(?:int|i4)><\/value>/g; - let member; - while ((member = namedStringRegex.exec(xml))) struct[member[1]] = member[2]; - while ((member = namedIntRegex.exec(xml))) struct[member[1]] = Number(member[2]); - if (Object.keys(struct).length) return struct; - - const stringMatch = xml.match(/([\s\S]*?)<\/string>/); - if (stringMatch) return stringMatch[1]; - const intMatch = xml.match(/<(?:int|i4)>([\s\S]*?)<\/(?:int|i4)>/); - if (intMatch) return Number(intMatch[1]); - - return null; -} - -async function xmlRpcCall(methodName, params) { - const body = `${methodName}${params - .map(param => `${xmlValue(param)}`) - .join('')}`; - - const res = await fetch(xmlRpcUrl, { - method: 'POST', - headers: { 'Content-Type': 'text/xml' }, - body, - }); - const text = await res.text(); - if (!res.ok) throw new Error(`${methodName} → ${res.status}: ${text}`); - return parseXmlRpcResponse(text); -} - -let cachedBlogId = null; -async function getBlogId() { - if (cachedBlogId) return cachedBlogId; - const res = await fetch(restApiBase); - const data = await res.json().catch(() => ({})); - if (!data.ID) throw new Error(`Could not determine site ID for ${site}`); - cachedBlogId = data.ID; - return cachedBlogId; -} - -function guessMimeType(filename) { - const ext = filename.split('.').pop().toLowerCase(); - const types = { jpg: 'image/jpeg', jpeg: 'image/jpeg', png: 'image/png', gif: 'image/gif', webp: 'image/webp', mp4: 'video/mp4', mov: 'video/quicktime' }; - return types[ext] || 'application/octet-stream'; -} - -// Extract clean text content from accessibility tree nodes -function buildContentFromAccessibility(nodes) { - if (!nodes?.length) return ''; - const blocks = []; - for (const node of nodes) { - if (!node.name) continue; - if (node.role === 'heading') { - // Guess heading level from name length (crude but works as fallback) - blocks.push(`

    ${node.name}

    `); - } else if (['paragraph', 'StaticText', 'article', 'section'].includes(node.role)) { - blocks.push(`

    ${node.name}

    `); - } else if (node.role === 'img' && node.description) { - blocks.push(``); - } - } - return blocks.join('\n'); -} - -// Extract the best available content from a page JSON file -function extractContent(pageData) { - // Instagram posts — caption is the content, images are media attachments - if (pageData.platform === 'instagram' || pageData.shortcode) { - return buildInstagramContent(pageData); - } - - // Priority: Wix blog API response > JSON-LD > accessibility tree - for (const call of pageData.apiCalls || []) { - // Blog post body is typically in post.content or post.richContent - const body = call.data?.post?.content?.plainText || - call.data?.post?.richContent || - call.data?.content?.plainText; - if (body) return typeof body === 'string' ? `

    ${body}

    ` : JSON.stringify(body); - } - - // JSON-LD article body - const article = pageData.globals?.jsonLd?.find(j => j['@type'] === 'Article' || j['@type'] === 'BlogPosting'); - if (article?.articleBody) return `

    ${article.articleBody}

    `; - - // Fallback to accessibility tree - return buildContentFromAccessibility(pageData.accessibility); -} - -// Build WordPress block content from an Instagram post -function buildInstagramContent(pageData) { - const blocks = []; - const media = pageData.media || []; - const imageMedia = media.filter(m => m.type !== 'video' || !m.videoUrl); - const videoMedia = media.filter(m => m.type === 'video' && m.videoUrl); - - // Use a gallery block for carousels (multiple images), single image block otherwise - if (imageMedia.length > 1) { - const galleryImages = imageMedia.map(item => { - const src = item.localFile || item.displayUrl; - if (!src) return ''; - const alt = item.accessibilityCaption || ''; - return `\n
    ${alt.replace(/
    \n`; - }).filter(Boolean); - blocks.push(`\n
    \n${galleryImages.join('\n')}\n
    \n`); - } else if (imageMedia.length === 1) { - const item = imageMedia[0]; - const src = item.localFile || item.displayUrl; - if (src) { - const alt = item.accessibilityCaption || pageData.caption?.slice(0, 125) || ''; - blocks.push(`\n
    ${alt.replace(/
    \n`); - } - } - - // Videos as separate blocks (can't go in gallery) - for (const item of videoMedia) { - blocks.push(`\n
    \n`); - } - - // Caption as a paragraph - if (pageData.caption) { - // Convert @mentions and #hashtags to links - let caption = pageData.caption - .replace(/@(\w+)/g, '@$1') - .replace(/#(\w+)/g, '#$1'); - blocks.push(`\n

    ${caption}

    \n`); - } - - // Link to original Instagram post - if (pageData.shortcode) { - const igUrl = `https://www.instagram.com/p/${pageData.shortcode}/`; - blocks.push(`\n

    Originally posted on Instagram

    \n`); - } - - return blocks.join('\n\n'); -} - -function extractMeta(pageData) { - // Instagram posts - if (pageData.platform === 'instagram' || pageData.shortcode) { - const caption = pageData.caption || ''; - // Title: first line of caption, or first 60 chars, or shortcode - const title = caption.split('\n')[0]?.slice(0, 80) || `Instagram ${pageData.shortcode}`; - return { - title, - description: caption.slice(0, 300), - featuredImageUrl: pageData.media?.[0]?.displayUrl || null, - publishDate: pageData.date || null, - modifiedDate: null, - slug: `ig-${pageData.shortcode}`, - }; - } - - const meta = pageData.globals?.meta || {}; - const jsonLd = pageData.globals?.jsonLd || []; - const article = jsonLd.find(j => ['Article', 'BlogPosting', 'WebPage'].includes(j['@type'])); - - return { - title: meta.ogTitle || meta.title || article?.headline || pageData.slug, - description: meta.description || meta.ogDescription || article?.description || '', - featuredImageUrl: meta.ogImage || article?.image?.url || null, - publishDate: article?.datePublished || null, - modifiedDate: article?.dateModified || null, - slug: pageData.slug, - }; -} - -async function uploadMedia(filePath, filename) { - if (dryRun) { - console.log(` [dry-run] Would upload: ${filename}`); - return { id: 0, source_url: `https://example.com/wp-content/uploads/${filename}` }; - } - - const fileBuffer = readFileSync(filePath); - const blogId = await getBlogId(); - const result = await xmlRpcCall('wp.uploadFile', [ - blogId, - user, - token, - { - name: filename, - type: guessMimeType(filename), - bits: fileBuffer, - overwrite: true, - }, - ]); - - return { - id: result.id || 0, - source_url: result.url, - }; -} - -async function importMedia() { - if (!existsSync('output/media')) { console.log('No media folder found.'); return {}; } - - const files = readdirSync('output/media'); - console.log(`\nUploading ${files.length} media files...`); - - const mediaMap = {}; // original filename → { url, id } - for (const file of files) { - process.stdout.write(` ${file}... `); - try { - const result = await uploadMedia(`output/media/${file}`, file); - mediaMap[file] = { url: result.source_url, id: result.id }; - console.log(`✓ ${result.source_url}`); - } catch (e) { - console.log(`✗ ${e.message}`); - } - } - return mediaMap; -} - -async function importPage(pageData, mediaMap) { - const meta = extractMeta(pageData); - let content = extractContent(pageData); - - for (const [filename, media] of Object.entries(mediaMap)) { - const url = typeof media === 'string' ? media : media.url; - content = content.replaceAll(filename, url); - } - - if (dryRun) { - console.log(` [dry-run] Would create page: ${meta.title} (${meta.slug})`); - return { id: 0, link: '#' }; - } - - const blogId = await getBlogId(); - const id = await xmlRpcCall('wp.newPost', [ - blogId, user, token, - { - post_type: 'page', - post_status: 'draft', - post_title: meta.title, - post_content: content, - post_excerpt: meta.description, - wp_slug: meta.slug, - }, - ]); - - return { id, link: `https://${site}/wp-admin/post.php?post=${id}&action=edit` }; -} - -async function importPost(pageData, mediaMap) { - const meta = extractMeta(pageData); - let content = extractContent(pageData); - - // Replace local file paths with uploaded WordPress URLs - for (const [filename, media] of Object.entries(mediaMap)) { - const url = typeof media === 'string' ? media : media.url; - content = content.replaceAll(filename, url); - } - - if (dryRun) { - console.log(` [dry-run] Would create post: ${meta.title} (${meta.slug})`); - return { id: 0, link: '#' }; - } - - const blogId = await getBlogId(); - - // Format date as "YYYY-MM-DD HH:MM:SS" string — WordPress.com ignores - // dateTime.iso8601 typed values but parses string dates correctly - let postDate; - if (meta.publishDate) { - const d = new Date(meta.publishDate); - postDate = `${d.getUTCFullYear()}-${String(d.getUTCMonth()+1).padStart(2,'0')}-${String(d.getUTCDate()).padStart(2,'0')} ${String(d.getUTCHours()).padStart(2,'0')}:${String(d.getUTCMinutes()).padStart(2,'0')}:${String(d.getUTCSeconds()).padStart(2,'0')}`; - } - - // Find featured image: first media item's WordPress media ID - let featuredImageId; - if (pageData.media?.[0]?.localFile) { - const firstMediaFile = basename(pageData.media[0].localFile); - const mediaEntry = mediaMap[firstMediaFile]; - if (mediaEntry?.id) featuredImageId = mediaEntry.id; - } - - const postData = { - post_type: postType || 'post', - post_status: 'publish', - post_title: meta.title, - post_content: content, - post_excerpt: meta.description, - wp_slug: meta.slug, - post_date: postDate, - post_thumbnail: featuredImageId || undefined, - }; - - const id = await xmlRpcCall('wp.newPost', [blogId, user, token, postData]); - - return { id, link: `https://${site}/wp-admin/post.php?post=${id}&action=edit` }; -} - -async function main() { - if (dryRun) console.log('[DRY RUN — no changes will be made]\n'); - - if (!existsSync('output/pages')) { - console.error('No output/pages directory found. Run extract.js first.'); - process.exit(1); - } - - const pageFiles = readdirSync('output/pages').filter(f => f.endsWith('.json')); - console.log(`Found ${pageFiles.length} extracted pages`); - - // Detect if this is an Instagram import - let isInstagram = false; - if (existsSync('output/inventory.json')) { - const inventory = JSON.parse(readFileSync('output/inventory.json', 'utf8')); - isInstagram = inventory.platform === 'instagram'; - } - - // Determine content type from inventory if available - let typeMap = {}; - if (existsSync('output/inventory.json')) { - const inventory = JSON.parse(readFileSync('output/inventory.json', 'utf8')); - for (const item of inventory.urls) { - if (isInstagram) { - // Instagram uses shortcodes as filenames - typeMap[item.shortcode] = item.type || 'photo'; - } else { - const slug = new URL(item.url).pathname.replace(/^\//, '').replace(/\//g, '--') || 'homepage'; - typeMap[slug] = item.type; - } - } - } - - const urlMap = []; // old URL → new WP URL, for redirect map - - // Step 1: Upload media - let mediaMap = {}; - if (!only || only === 'media') { - mediaMap = await importMedia(); - } - - // Instagram: all items are posts (not pages) - if (isInstagram) { - console.log(`\nImporting ${pageFiles.length} Instagram posts${postType ? ` as "${postType}"` : ''}...`); - for (const file of pageFiles) { - const pageData = JSON.parse(readFileSync(`output/pages/${file}`, 'utf8')); - const shortcode = file.replace('.json', ''); - process.stdout.write(` ${shortcode}... `); - try { - const result = await importPost(pageData, mediaMap); - console.log(`✓ ${result.link}`); - if (pageData.sourceUrl) urlMap.push({ old: pageData.sourceUrl, new: result.link }); - } catch (e) { - console.log(`✗ ${e.message}`); - } - } - } else { - // Step 2: Import pages - const pages = pageFiles.filter(f => { - const slug = f.replace('.json', ''); - return !typeMap[slug] || typeMap[slug] === 'page' || typeMap[slug] === 'homepage'; - }); - - if (!only || only === 'pages') { - console.log(`\nImporting ${pages.length} pages...`); - for (const file of pages) { - const pageData = JSON.parse(readFileSync(`output/pages/${file}`, 'utf8')); - const slug = file.replace('.json', ''); - process.stdout.write(` ${slug}... `); - try { - const result = await importPage(pageData, mediaMap); - console.log(`✓ ${result.link}`); - if (pageData.sourceUrl) urlMap.push({ old: pageData.sourceUrl, new: result.link }); - } catch (e) { - console.log(`✗ ${e.message}`); - } - } - } - - // Step 3: Import posts - const posts = pageFiles.filter(f => { - const slug = f.replace('.json', ''); - return typeMap[slug] === 'blog-post'; - }); - - if (!only || only === 'posts') { - console.log(`\nImporting ${posts.length} blog posts...`); - for (const file of posts) { - const pageData = JSON.parse(readFileSync(`output/pages/${file}`, 'utf8')); - const slug = file.replace('.json', ''); - process.stdout.write(` ${slug}... `); - try { - const result = await importPost(pageData, mediaMap); - console.log(`✓ ${result.link}`); - if (pageData.sourceUrl) urlMap.push({ old: pageData.sourceUrl, new: result.link }); - } catch (e) { - console.log(`✗ ${e.message}`); - } - } - } - } - - // Output redirect map - if (urlMap.length) { - const { writeFileSync } = await import('fs'); - writeFileSync('output/redirect-map.json', JSON.stringify(urlMap, null, 2)); - console.log(`\nRedirect map written to output/redirect-map.json`); - console.log('Use this to set up 301 redirects from your old Wix URLs to WordPress.'); - } - - console.log('\nImport complete. All content created as drafts — review in WordPress admin before publishing.'); - console.log(`https://${site}/wp-admin/`); -} - -main().catch(e => { console.error(e); process.exit(1); }); diff --git a/src/adapters/instagram.ts b/src/adapters/instagram.ts new file mode 100644 index 00000000..d97564a8 --- /dev/null +++ b/src/adapters/instagram.ts @@ -0,0 +1,774 @@ +import type { PlatformAdapter } from '../types.js'; +import type { WxrBuilder } from '../lib/extraction/wxr-builder.js'; +import type { ExtractionLog } from '../lib/extraction/extraction-log.js'; +import type { Server } from '@modelcontextprotocol/sdk/server/index.js'; +import { downloadMedia } from '../lib/extraction/media.js'; +import { getPlaywright, sleep } from './shared.js'; +import { mkdirSync } from 'fs'; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface InstagramAdapterOpts extends Record { + cdpPort?: number; + delay?: number; + limit?: number; + resume?: boolean; + dryRun?: boolean; + verbose?: boolean; + outputDir?: string; +} + +export type InstagramPostType = 'photo' | 'video' | 'carousel'; + +export interface InstagramProfile { + id?: string; + username: string; + fullName: string; + biography: string; + profilePicUrl: string; + postCount: number | null; + followerCount: number | null; + followingCount: number | null; + isPrivate: boolean; + isVerified: boolean; +} + +export interface InstagramPost { + id?: string; + shortcode: string; + type: InstagramPostType; + /** Unix seconds */ + timestamp: number | null; + /** ISO date string */ + date: string | null; + caption: string; + displayUrl: string | null; + thumbnailUrl?: string | null; + dimensions?: { width: number; height: number } | null; + isVideo?: boolean; + videoUrl?: string | null; + accessibilityCaption?: string | null; + locationName?: string | null; + locationId?: string | null; + likes?: number | null; + comments?: number | null; + carouselCount?: number | null; + url: string; +} + +export interface InstagramInventoryUrl { + url: string; + type: InstagramPostType; + id?: string; + shortcode: string; +} + +export interface InstagramInventory { + platform: 'instagram'; + username: string; + profile: InstagramProfile | null; + discoveredAt: string; + counts: Record; + urls: InstagramInventoryUrl[]; + posts: InstagramPost[]; +} + +// --------------------------------------------------------------------------- +// Helpers — URL and caption parsing +// --------------------------------------------------------------------------- + +/** Extract the username from an Instagram profile URL or a bare `@username` / `username`. */ +export function parseInstagramUsername(input: string): string { + const trimmed = input.trim().replace(/^@/, ''); + if (!trimmed) return ''; + try { + const u = new URL(trimmed.includes('://') ? trimmed : `https://${trimmed}`); + if (/instagram\.com$/i.test(u.hostname) || /\.instagram\.com$/i.test(u.hostname)) { + const first = u.pathname.replace(/^\//, '').split('/')[0]; + return first || ''; + } + return trimmed; + } catch { + return trimmed; + } +} + +function classifyInstagramPost(node: Record): InstagramPostType { + if (node.__typename === 'GraphSidecar' || node.edge_sidecar_to_children) return 'carousel'; + if (node.__typename === 'GraphVideo' || node.is_video) return 'video'; + return 'photo'; +} + +interface GraphqlEdgeNode { + id?: string; + shortcode?: string; + __typename?: string; + edge_sidecar_to_children?: { edges?: unknown[] }; + is_video?: boolean; + video_url?: string; + taken_at_timestamp?: number; + display_url?: string; + thumbnail_src?: string; + thumbnail_resources?: Array<{ src: string }>; + dimensions?: { width: number; height: number }; + accessibility_caption?: string; + location?: { name?: string; id?: string; pk?: string }; + edge_media_to_caption?: { edges?: Array<{ node?: { text?: string } }> }; + edge_media_preview_like?: { count?: number }; + edge_liked_by?: { count?: number }; + edge_media_to_comment?: { count?: number }; + edge_media_preview_comment?: { count?: number }; +} + +export function extractPostMeta(node: GraphqlEdgeNode): InstagramPost { + const caption = node.edge_media_to_caption?.edges?.[0]?.node?.text || ''; + const ts = node.taken_at_timestamp ?? null; + return { + id: node.id, + shortcode: node.shortcode || '', + type: classifyInstagramPost(node as unknown as Record), + timestamp: ts, + date: ts ? new Date(ts * 1000).toISOString() : null, + caption, + displayUrl: node.display_url || null, + thumbnailUrl: node.thumbnail_src || node.thumbnail_resources?.[0]?.src || null, + dimensions: node.dimensions || null, + isVideo: !!node.is_video, + videoUrl: node.video_url || null, + accessibilityCaption: node.accessibility_caption || null, + locationName: node.location?.name || null, + locationId: node.location?.id || node.location?.pk || null, + likes: node.edge_media_preview_like?.count ?? node.edge_liked_by?.count ?? null, + comments: node.edge_media_to_comment?.count ?? node.edge_media_preview_comment?.count ?? null, + carouselCount: node.edge_sidecar_to_children?.edges?.length || null, + url: `https://www.instagram.com/p/${node.shortcode}/`, + }; +} + +// --------------------------------------------------------------------------- +// Caption → Gutenberg content +// --------------------------------------------------------------------------- + +function escapeHtml(str: string): string { + return str + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} + +/** Linkify #hashtags and @mentions in a caption, preserving line breaks. */ +export function captionToHtml(caption: string): string { + if (!caption) return ''; + const lines = caption.split(/\r?\n/); + const html = lines.map((line) => { + const escaped = escapeHtml(line); + const withLinks = escaped + .replace(/#(\w+)/g, '#$1') + .replace(/@(\w+)/g, '@$1'); + return withLinks; + }).join('
    '); + return html; +} + +export function extractHashtags(caption: string): string[] { + if (!caption) return []; + return [...caption.matchAll(/#(\w+)/g)].map((m) => m[1]); +} + +export interface RenderedMedia { + /** URL used as the `src` in the block — typically the local media URL placeholder or original URL. */ + url: string; + alt?: string; + width?: number; + height?: number; + id?: number; +} + +/** + * Build Gutenberg block markup for an Instagram post. + * - Photo: `wp:image` block + * - Video: `wp:video` block + * - Carousel: `wp:gallery` block wrapping `wp:image` children + */ +export function buildInstagramPostContent(post: { + type: InstagramPostType; + caption: string; + sourceUrl: string; + media: RenderedMedia[]; + locationName?: string | null; +}): string { + const parts: string[] = []; + + const renderImageBlock = (m: RenderedMedia): string => { + const altAttr = m.alt ? ` alt="${escapeHtml(m.alt)}"` : ' alt=""'; + const idComment = m.id ? `{"id":${m.id}}` : '{}'; + return ( + `\n` + + `
    \n` + + `` + ); + }; + + const renderVideoBlock = (m: RenderedMedia): string => { + return ( + `\n` + + `
    \n` + + `` + ); + }; + + if (post.type === 'carousel' && post.media.length > 1) { + const ids = post.media.filter((m) => m.id).map((m) => m.id); + const galleryAttrs = ids.length ? `{"linkTo":"none","ids":[${ids.join(',')}]}` : `{"linkTo":"none"}`; + const inner = post.media + .map((m) => (m.url && /\.(mp4|mov|webm)$/i.test(m.url) ? renderVideoBlock(m) : renderImageBlock(m))) + .join('\n'); + parts.push( + `\n` + + `
    \n${inner}\n
    \n` + + `` + ); + } else if (post.media.length > 0) { + const first = post.media[0]; + if (post.type === 'video' || /\.(mp4|mov|webm)$/i.test(first.url)) { + parts.push(renderVideoBlock(first)); + } else { + parts.push(renderImageBlock(first)); + } + } + + if (post.caption) { + parts.push(`\n

    ${captionToHtml(post.caption)}

    \n`); + } + + if (post.locationName) { + parts.push( + `\n` + + `

    ${escapeHtml(post.locationName)}

    \n` + + `` + ); + } + + parts.push( + `\n` + + `

    View on Instagram

    \n` + + `` + ); + + return parts.join('\n\n'); +} + +/** Derive a post title from the first line / first N chars of a caption. */ +export function titleFromCaption(caption: string, shortcode: string): string { + if (!caption) return `Instagram post ${shortcode}`; + const firstLine = caption.split(/\r?\n/)[0].trim(); + if (!firstLine) return `Instagram post ${shortcode}`; + if (firstLine.length <= 80) return firstLine; + return firstLine.slice(0, 77).trimEnd() + '…'; +} + +// --------------------------------------------------------------------------- +// Playwright discovery — scrolls profile, intercepts GraphQL +// --------------------------------------------------------------------------- + +interface PwPage { + goto(url: string, opts: Record): Promise; + evaluate(fn: (...a: unknown[]) => T, arg?: unknown): Promise; + waitForSelector(sel: string, opts?: Record): Promise; + on(ev: string, handler: (resp: unknown) => void): void; + off(ev: string, handler: (resp: unknown) => void): void; + close(): Promise; + $(sel: string): Promise; + url(): string; +} + +async function openPage(cdpPort: number): Promise<{ browser: { close(): Promise; disconnect?(): Promise; contexts(): Array<{ newPage(): Promise; pages?(): PwPage[] }> }; page: PwPage }> { + const pw = await getPlaywright(); + const browser = await pw.chromium.connectOverCDP(`http://127.0.0.1:${cdpPort}`); + const context = browser.contexts()[0] || (await (browser as unknown as { newContext(): Promise<{ newPage(): Promise }> }).newContext()); + const page = (await context.newPage()) as unknown as PwPage; + return { browser: browser as unknown as { close(): Promise; disconnect?(): Promise; contexts(): Array<{ newPage(): Promise }> }, page }; +} + +async function discoverProfile( + username: string, + cdpPort: number, + limit: number, + scrollDelay: number, + sendLog: (msg: string) => void +): Promise<{ profile: InstagramProfile | null; posts: InstagramPost[] }> { + const { browser, page } = await openPage(cdpPort); + + const posts = new Map(); + let profile: InstagramProfile | null = null; + let hasMore = true; + let endCursor: string | null = null; + + const responseHandler = async (response: unknown) => { + const resp = response as { url(): string; headers(): Record; json(): Promise }; + const respUrl = resp.url(); + if (!respUrl.includes('/graphql/query') && !respUrl.includes('/api/v1/')) return; + const ct = resp.headers()['content-type'] || ''; + if (!ct.includes('application/json') && !ct.includes('text/javascript')) return; + try { + const body = (await resp.json()) as Record; + const userData = + (body?.data as Record | undefined)?.user || + (body?.graphql as Record | undefined)?.user || + (body?.data as Record | undefined)?.xdt_api__v1__feed__user_timeline_graphql_connection; + if (!userData) return; + const u = userData as Record; + + if (!profile && (u.username || u.full_name)) { + profile = { + id: (u.id as string) || (u.pk as string) || undefined, + username: (u.username as string) || username, + fullName: (u.full_name as string) || '', + biography: (u.biography as string) || (u.bio_text as string) || '', + profilePicUrl: (u.profile_pic_url_hd as string) || (u.profile_pic_url as string) || '', + postCount: (u.edge_owner_to_timeline_media as { count?: number } | undefined)?.count ?? (u.media_count as number) ?? null, + followerCount: (u.edge_followed_by as { count?: number } | undefined)?.count ?? (u.follower_count as number) ?? null, + followingCount: (u.edge_follow as { count?: number } | undefined)?.count ?? (u.following_count as number) ?? null, + isPrivate: !!u.is_private, + isVerified: !!u.is_verified, + }; + } + + const timeline = (u.edge_owner_to_timeline_media as { edges?: Array<{ node?: GraphqlEdgeNode }>; page_info?: { has_next_page?: boolean; end_cursor?: string } } | undefined) + || (u.edge_web_feed_timeline as { edges?: Array<{ node?: GraphqlEdgeNode }>; page_info?: { has_next_page?: boolean; end_cursor?: string } } | undefined); + + if (timeline?.edges) { + for (const edge of timeline.edges) { + const node = edge.node; + if (node?.shortcode && !posts.has(node.shortcode)) { + posts.set(node.shortcode, extractPostMeta(node)); + } + } + if (timeline.page_info) { + hasMore = !!timeline.page_info.has_next_page; + endCursor = timeline.page_info.end_cursor || null; + } + } + } catch { + // Ignore non-JSON responses + } + }; + + page.on('response', responseHandler); + + try { + const profileUrl = `https://www.instagram.com/${username}/`; + sendLog(`Navigating to ${profileUrl}`); + try { + await page.goto(profileUrl, { waitUntil: 'domcontentloaded', timeout: 60000 }); + await sleep(3000); + } catch (e) { + if (posts.size === 0) throw e; + } + + const loginPrompt = await page.$('input[name="username"]'); + if (loginPrompt) { + throw new Error( + 'Not logged into Instagram in this browser session. ' + + 'Log in to Instagram in the Chrome window connected via CDP and re-run.' + ); + } + + sendLog('Scrolling to load posts…'); + let scrollAttempts = 0; + let lastCount = posts.size; + let noNewStreak = 0; + let currentDelay = scrollDelay; + + while (posts.size < limit && scrollAttempts < 100) { + await page.evaluate(() => window.scrollTo(0, document.documentElement.scrollHeight)); + await sleep(currentDelay); + scrollAttempts++; + if (posts.size === lastCount) { + noNewStreak++; + currentDelay = Math.min(currentDelay * 1.5, scrollDelay * 4); + if (noNewStreak >= 4) break; + } else { + noNewStreak = 0; + currentDelay = Math.max(scrollDelay, currentDelay * 0.8); + lastCount = posts.size; + sendLog(` scroll ${scrollAttempts}: ${posts.size} posts`); + } + if (!hasMore) break; + } + + // Suppress "declared but unused" — endCursor is kept as state for future direct GraphQL pagination. + void endCursor; + } finally { + page.off('response', responseHandler); + try { await page.close(); } catch { /* ignore */ } + try { await (browser.disconnect ? browser.disconnect() : browser.close()); } catch { /* ignore */ } + } + + const allPosts = [...posts.values()].sort((a, b) => (b.timestamp || 0) - (a.timestamp || 0)); + return { profile, posts: allPosts }; +} + +// --------------------------------------------------------------------------- +// Per-post extraction via CDP — visits each post, collects media URLs +// --------------------------------------------------------------------------- + +interface ExtractedInstagramPost { + post: InstagramPost; + media: Array<{ + type: 'photo' | 'video'; + sourceUrl: string; + accessibilityCaption?: string | null; + width?: number; + height?: number; + }>; +} + +async function extractSinglePost(page: PwPage, inventoryPost: InstagramPost): Promise { + const shortcode = inventoryPost.shortcode; + const postUrl = inventoryPost.url || `https://www.instagram.com/p/${shortcode}/`; + + const apiCalls: Record[] = []; + const responseHandler = async (response: unknown) => { + const resp = response as { url(): string; headers(): Record; json(): Promise }; + const respUrl = resp.url(); + if (!respUrl.includes('/graphql/query') && !respUrl.includes('/api/v1/media/')) return; + const ct = resp.headers()['content-type'] || ''; + if (!ct.includes('application/json') && !ct.includes('text/javascript')) return; + try { apiCalls.push((await resp.json()) as Record); } catch { /* ignore */ } + }; + + page.on('response', responseHandler); + try { + await page.goto(postUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }); + await page.waitForSelector('article img[src*="cdninstagram.com"]', { timeout: 10000 }).catch(() => {}); + } finally { + page.off('response', responseHandler); + } + + // Find post detail in the intercepted API calls + let detail: Record | null = null; + for (const data of apiCalls) { + const d = data as Record; + const media = + (d?.data as Record | undefined)?.xdt_shortcode_media || + (d?.graphql as Record | undefined)?.shortcode_media || + ((d?.items as Array> | undefined)?.[0]); + if (media && ((media as Record).shortcode === shortcode || (media as Record).code === shortcode)) { + detail = media as Record; + break; + } + } + + const media: ExtractedInstagramPost['media'] = []; + + if (detail) { + const d = detail as Record; + const sidecar = (d.edge_sidecar_to_children as { edges?: Array<{ node?: Record }> } | undefined)?.edges; + if (sidecar) { + for (const edge of sidecar) { + const child = edge.node as Record; + media.push({ + type: child.is_video ? 'video' : 'photo', + sourceUrl: (child.video_url as string) || (child.display_url as string) || '', + accessibilityCaption: (child.accessibility_caption as string) || null, + width: (child.dimensions as { width?: number } | undefined)?.width, + height: (child.dimensions as { height?: number } | undefined)?.height, + }); + } + } else { + media.push({ + type: d.is_video ? 'video' : 'photo', + sourceUrl: (d.video_url as string) || (d.display_url as string) || '', + accessibilityCaption: (d.accessibility_caption as string) || null, + width: (d.dimensions as { width?: number } | undefined)?.width, + height: (d.dimensions as { height?: number } | undefined)?.height, + }); + } + } + + // Carousel slide sweep via ?img_index=N — fallback when API detail didn't expose children + if (inventoryPost.type === 'carousel' && media.length <= 1) { + const expected = inventoryPost.carouselCount || 10; + const seenIds = new Set(); + for (let slide = 1; slide <= expected; slide++) { + try { + const slideUrl = `${postUrl}?img_index=${slide}`; + await page.goto(slideUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }); + await page.waitForSelector('li img[src*="cdninstagram.com"]', { timeout: 8000 }).catch(() => {}); + await sleep(600); + const imgs = (await page.evaluate(() => { + const results: Array<{ src: string; alt: string; mediaId: string | null }> = []; + for (const img of Array.from(document.querySelectorAll('li img'))) { + const src = (img as HTMLImageElement).src || ''; + if (!src.includes('cdninstagram.com/v/t51.')) continue; + const alt = (img as HTMLImageElement).alt || ''; + if (alt.includes('User avatar')) continue; + const rect = (img as HTMLImageElement).getBoundingClientRect(); + if (rect.width < 300) continue; + const idMatch = src.match(/\/(\d{5,})_/); + results.push({ src, alt, mediaId: idMatch?.[1] || null }); + } + return results; + })) as Array<{ src: string; alt: string; mediaId: string | null }>; + + for (const img of imgs) { + if (!img.mediaId || seenIds.has(img.mediaId)) continue; + seenIds.add(img.mediaId); + media.push({ type: 'photo', sourceUrl: img.src, accessibilityCaption: img.alt }); + } + } catch { + break; + } + } + } + + // Last-resort fallback: use display URL from inventory + if (media.length === 0 && inventoryPost.displayUrl) { + media.push({ + type: inventoryPost.isVideo ? 'video' : 'photo', + sourceUrl: inventoryPost.videoUrl || inventoryPost.displayUrl, + accessibilityCaption: inventoryPost.accessibilityCaption || null, + }); + } + + return { post: inventoryPost, media }; +} + +// --------------------------------------------------------------------------- +// The adapter +// --------------------------------------------------------------------------- + +export const instagramAdapter: PlatformAdapter = { + id: 'instagram', + + detect(url: string): boolean { + return /(^|[/.])instagram\.com/i.test(url); + }, + + async discover(url: string, opts: Record): Promise { + const igOpts = opts as InstagramAdapterOpts; + const username = parseInstagramUsername(url); + if (!username) throw new Error(`Could not parse Instagram username from: ${url}`); + if (!igOpts.cdpPort) { + throw new Error( + 'Instagram discovery requires --cdp-port. ' + + 'Launch Chrome with --remote-debugging-port= and log in to Instagram first.' + ); + } + + const scrollDelay = igOpts.delay ?? 2000; + const limit = igOpts.limit ?? Infinity; + const sendLog = (msg: string) => { if (igOpts.verbose) console.log(msg); }; + + const { profile, posts } = await discoverProfile(username, igOpts.cdpPort, limit, scrollDelay, sendLog); + + const counts: Record = { photo: 0, video: 0, carousel: 0 }; + for (const p of posts) counts[p.type] = (counts[p.type] || 0) + 1; + + return { + platform: 'instagram', + username, + profile, + discoveredAt: new Date().toISOString(), + counts, + urls: posts.map((p) => ({ url: p.url, type: p.type, id: p.id, shortcode: p.shortcode })), + posts, + }; + }, + + async extract( + inventory: unknown, + wxr: WxrBuilder, + opts: Record, + context: { log: ExtractionLog; server: Server } + ): Promise<{ + pagesExtracted: number; + postsExtracted: number; + failed: number; + mediaCollected: number; + }> { + const inv = inventory as InstagramInventory; + const igOpts = opts as InstagramAdapterOpts; + const delayMs = igOpts.delay ?? 1500; + const outputDir = igOpts.outputDir || ''; + const mediaDir = outputDir ? `${outputDir}/media` : null; + if (mediaDir) mkdirSync(mediaDir, { recursive: true }); + + if (!igOpts.cdpPort && !igOpts.dryRun) { + throw new Error('Instagram extraction requires --cdp-port (authenticated browser session).'); + } + + const sendLog = (message: string) => { + try { context.server?.sendLoggingMessage?.({ level: 'info', data: message }); } catch { /* ignore */ } + }; + + // Register a single author based on the profile + const authorLogin = (inv.profile?.username || inv.username).toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '') || 'instagram'; + wxr.addAuthor({ + login: authorLogin, + displayName: inv.profile?.fullName || inv.profile?.username || inv.username, + }); + + const processed = igOpts.resume ? context.log.getProcessedUrls() : new Set(); + let postsToProcess = inv.posts.filter((p) => !processed.has(p.url)); + if (igOpts.dryRun) postsToProcess = postsToProcess.slice(0, 3); + + let postsExtracted = 0; + let failed = 0; + const seenMediaNames = new Map(); + const seenMediaHashes = new Map(); + const downloadedUrls = new Set(); + const seenTagSlugs = new Set(); + + // Browser session — only opened if we actually have posts to fetch + let browserSession: { page: PwPage; close: () => Promise } | null = null; + const getPage = async (): Promise => { + if (!browserSession) { + const { browser, page } = await openPage(igOpts.cdpPort as number); + browserSession = { + page, + close: async () => { + try { await page.close(); } catch { /* ignore */ } + try { await (browser.disconnect ? browser.disconnect() : browser.close()); } catch { /* ignore */ } + }, + }; + } + return browserSession.page; + }; + + try { + for (let i = 0; i < postsToProcess.length; i++) { + const inventoryPost = postsToProcess[i]; + const startMs = Date.now(); + sendLog(`[${i + 1}/${postsToProcess.length}] ${inventoryPost.url}`); + + try { + let extracted: ExtractedInstagramPost; + if (igOpts.dryRun) { + // Dry run: don't hit the browser, build content from inventory-only data + extracted = { + post: inventoryPost, + media: inventoryPost.displayUrl + ? [{ type: inventoryPost.isVideo ? 'video' : 'photo', sourceUrl: inventoryPost.videoUrl || inventoryPost.displayUrl }] + : [], + }; + } else { + extracted = await extractSinglePost(await getPage(), inventoryPost); + } + + // Download media for the post and build RenderedMedia list + const rendered: RenderedMedia[] = []; + let featuredMediaId: number | undefined; + + for (const m of extracted.media) { + if (!m.sourceUrl) continue; + let url = m.sourceUrl; + let mediaId: number | undefined; + + if (!igOpts.dryRun && mediaDir && !downloadedUrls.has(m.sourceUrl)) { + downloadedUrls.add(m.sourceUrl); + const result = await downloadMedia(m.sourceUrl, mediaDir, seenMediaNames, seenMediaHashes); + context.log.logMedia({ + url: m.sourceUrl, + localPath: result.localPath, + error: result.error, + }); + if (!result.error && result.localPath) { + mediaId = wxr.addMedia({ + url: m.sourceUrl, + localPath: result.localPath, + title: result.filename || '', + altText: m.accessibilityCaption || '', + }); + if (wxr.isStreaming) { + wxr.flushItem(wxr.items[wxr.items.length - 1]); + } + url = m.sourceUrl; // keep original URL in content; WP importer rewrites via media map + if (!featuredMediaId) featuredMediaId = mediaId; + } + } + + rendered.push({ + url, + alt: m.accessibilityCaption || '', + width: m.width, + height: m.height, + id: mediaId, + }); + } + + // Hashtags → tags + const hashtags = extractHashtags(inventoryPost.caption); + for (const tag of hashtags) { + const slug = tag.toLowerCase(); + if (seenTagSlugs.has(slug)) continue; + seenTagSlugs.add(slug); + wxr.addTag({ slug, name: tag }); + } + + const content = buildInstagramPostContent({ + type: inventoryPost.type, + caption: inventoryPost.caption, + sourceUrl: inventoryPost.url, + media: rendered, + locationName: inventoryPost.locationName, + }); + + const title = titleFromCaption(inventoryPost.caption, inventoryPost.shortcode); + const excerpt = inventoryPost.caption + ? inventoryPost.caption.split(/\r?\n/)[0].slice(0, 200) + : ''; + + wxr.addPost({ + title, + slug: inventoryPost.shortcode.toLowerCase(), + content, + excerpt, + date: inventoryPost.date || new Date().toISOString(), + sourceUrl: inventoryPost.url, + featuredMediaId, + tags: hashtags, + author: authorLogin, + }); + if (wxr.isStreaming) { + wxr.flushItem(wxr.items[wxr.items.length - 1]); + } + + // Redirect from Instagram path to new slug + try { + const originalPath = new URL(inventoryPost.url).pathname; + wxr.addRedirect({ from: originalPath, to: `/${inventoryPost.shortcode.toLowerCase()}` }); + } catch { /* ignore */ } + + postsExtracted++; + context.log.logProcessed({ + url: inventoryPost.url, + slug: inventoryPost.shortcode, + durationMs: Date.now() - startMs, + qualityScore: extracted.media.length > 0 ? 'high' : 'low', + }); + } catch (err) { + failed++; + const msg = err instanceof Error ? err.message : String(err); + context.log.logFailed({ url: inventoryPost.url, error: msg }); + sendLog(` FAILED: ${msg}`); + } + + if (i < postsToProcess.length - 1 && delayMs > 0) await sleep(delayMs); + } + } finally { + if (browserSession) await browserSession.close(); + } + + return { + pagesExtracted: 0, + postsExtracted, + failed, + mediaCollected: downloadedUrls.size, + }; + }, +}; diff --git a/src/lib/extraction/detect-platform.ts b/src/lib/extraction/detect-platform.ts index f9c2c589..a4c5311b 100644 --- a/src/lib/extraction/detect-platform.ts +++ b/src/lib/extraction/detect-platform.ts @@ -31,6 +31,7 @@ const URL_PATTERNS: UrlPattern[] = [ { pattern: /squarespace\.com/i, platform: 'squarespace' }, { pattern: /webflow\.io|webflow\.com/i, platform: 'webflow' }, { pattern: /myshopify\.com|shopify\.com/i, platform: 'shopify' }, + { pattern: /(^|[/.])instagram\.com/i, platform: 'instagram' }, ]; const HTTP_SIGNALS: HttpSignal[] = [ diff --git a/src/mcp-server.ts b/src/mcp-server.ts index 8d1cf302..36a9d15e 100644 --- a/src/mcp-server.ts +++ b/src/mcp-server.ts @@ -18,7 +18,8 @@ import { wixAdapter } from './adapters/wix.js'; import { squarespaceAdapter } from './adapters/squarespace.js'; import { webflowAdapter } from './adapters/webflow.js'; import { shopifyAdapter } from './adapters/shopify.js'; -const adapters: PlatformAdapter[] = [wixAdapter, squarespaceAdapter, webflowAdapter, shopifyAdapter]; +import { instagramAdapter } from './adapters/instagram.js'; +const adapters: PlatformAdapter[] = [wixAdapter, squarespaceAdapter, webflowAdapter, shopifyAdapter, instagramAdapter]; function findAdapter(platform: string): PlatformAdapter | null { return adapters.find((a) => a.id === platform) || null; diff --git a/test/adapters/instagram.test.ts b/test/adapters/instagram.test.ts new file mode 100644 index 00000000..98e0af9b --- /dev/null +++ b/test/adapters/instagram.test.ts @@ -0,0 +1,209 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { readFileSync, existsSync, mkdtempSync, rmSync } from 'fs'; +import { join } from 'path'; +import { tmpdir } from 'os'; +import { + instagramAdapter, + parseInstagramUsername, + captionToHtml, + extractHashtags, + buildInstagramPostContent, + titleFromCaption, + extractPostMeta, + type InstagramInventory, +} from '../../src/adapters/instagram.js'; +import { WxrBuilder } from '../../src/lib/extraction/wxr-builder.js'; +import { ExtractionLog } from '../../src/lib/extraction/extraction-log.js'; + +describe('instagramAdapter', () => { + it('has id "instagram"', () => { + expect(instagramAdapter.id).toBe('instagram'); + }); + + it('detects instagram URLs', () => { + expect(instagramAdapter.detect('https://www.instagram.com/foo')).toBe(true); + expect(instagramAdapter.detect('https://instagram.com/foo')).toBe(true); + expect(instagramAdapter.detect('https://www.example.com')).toBe(false); + expect(instagramAdapter.detect('https://mysite.squarespace.com')).toBe(false); + }); + + it('discover requires cdpPort', async () => { + await expect(instagramAdapter.discover('https://www.instagram.com/foo', {})).rejects.toThrow(/cdp-port/i); + }); +}); + +describe('parseInstagramUsername', () => { + it('extracts username from full URL', () => { + expect(parseInstagramUsername('https://www.instagram.com/mountaincoffee/')).toBe('mountaincoffee'); + }); + + it('strips @ from a bare handle', () => { + expect(parseInstagramUsername('@mountaincoffee')).toBe('mountaincoffee'); + }); + + it('passes through a bare username', () => { + expect(parseInstagramUsername('mountaincoffee')).toBe('mountaincoffee'); + }); +}); + +describe('captionToHtml', () => { + it('linkifies hashtags and mentions', () => { + const html = captionToHtml('Loving #coffee from @baristadaily'); + expect(html).toContain('#coffee'); + expect(html).toContain('@baristadaily'); + }); + + it('escapes HTML in captions', () => { + expect(captionToHtml('a < b & "c"')).toContain('<'); + expect(captionToHtml('a < b & "c"')).toContain('&'); + }); + + it('preserves line breaks as
    ', () => { + expect(captionToHtml('line1\nline2')).toContain('
    '); + }); +}); + +describe('extractHashtags', () => { + it('returns hashtags without the #', () => { + expect(extractHashtags('one #two #three')).toEqual(['two', 'three']); + }); + + it('returns empty for no caption', () => { + expect(extractHashtags('')).toEqual([]); + }); +}); + +describe('titleFromCaption', () => { + it('uses the first line', () => { + expect(titleFromCaption('Hello world\nmore', 'ABC')).toBe('Hello world'); + }); + + it('truncates long captions', () => { + const long = 'a'.repeat(200); + const t = titleFromCaption(long, 'ABC'); + expect(t.length).toBeLessThanOrEqual(80); + expect(t.endsWith('…')).toBe(true); + }); + + it('falls back to shortcode if caption is empty', () => { + expect(titleFromCaption('', 'ABC')).toBe('Instagram post ABC'); + }); +}); + +describe('buildInstagramPostContent', () => { + it('renders a single image as wp:image', () => { + const html = buildInstagramPostContent({ + type: 'photo', + caption: 'Hello', + sourceUrl: 'https://www.instagram.com/p/ABC/', + media: [{ url: 'https://example.com/photo.jpg', alt: 'A photo' }], + }); + expect(html).toContain('\n
    ${alt.replace(/
    \n`; - }).filter(Boolean); - blocks.push(`\n
    \n${galleryImages.join('\n')}\n
    \n`); - } else if (imageMedia.length === 1) { - const item = imageMedia[0]; - const src = item.localFile || item.displayUrl; - if (src) { - const alt = item.accessibilityCaption || pageData.caption?.slice(0, 125) || ''; - blocks.push(`\n
    ${alt.replace(/
    \n`); - } - } - - for (const item of videoMedia) { - blocks.push(`\n
    \n`); - } - - if (pageData.caption) { - let caption = pageData.caption - .replace(/@(\w+)/g, '@$1') - .replace(/#(\w+)/g, '#$1'); - blocks.push(`\n

    ${caption}

    \n`); - } - - if (pageData.shortcode) { - const igUrl = `https://www.instagram.com/p/${pageData.shortcode}/`; - blocks.push(`\n

    Originally posted on Instagram

    \n`); - } - - return blocks.join('\n\n'); -} - -function escapeXml(value) { - return String(value) - .replaceAll('&', '&') - .replaceAll('<', '<') - .replaceAll('>', '>') - .replaceAll('"', '"') - .replaceAll("'", '''); -} - -function xmlValue(value) { - if (value == null) return ''; - if (Buffer.isBuffer(value)) return `${value.toString('base64')}`; - if (value instanceof Date) { - const iso = value.toISOString().replace(/[-:]/g, '').replace(/\.\d{3}Z$/, ''); - return `${iso}`; - } - if (typeof value === 'boolean') return `${value ? 1 : 0}`; - if (typeof value === 'number') return Number.isInteger(value) ? `${value}` : `${value}`; - if (Array.isArray(value)) { - return `${value.map(item => `${xmlValue(item)}`).join('')}`; - } - if (typeof value === 'object') { - return `${Object.entries(value) - .filter(([, item]) => item !== undefined) - .map(([key, item]) => `${escapeXml(key)}${xmlValue(item)}`) - .join('')}`; - } - return `${escapeXml(value)}`; -} - -function extractPostMeta(node) { - const caption = node.edge_media_to_caption?.edges?.[0]?.node?.text || ''; - return { - id: node.id, - shortcode: node.shortcode, - type: node.edge_sidecar_to_children ? 'carousel' : node.is_video ? 'video' : 'photo', - timestamp: node.taken_at_timestamp, - date: node.taken_at_timestamp ? new Date(node.taken_at_timestamp * 1000).toISOString() : null, - caption, - displayUrl: node.display_url, - isVideo: !!node.is_video, - videoUrl: node.video_url || null, - accessibilityCaption: node.accessibility_caption || null, - locationName: node.location?.name || null, - likes: node.edge_media_preview_like?.count ?? null, - comments: node.edge_media_to_comment?.count ?? null, - carouselCount: node.edge_sidecar_to_children?.edges?.length || null, - url: `https://www.instagram.com/p/${node.shortcode}/`, - }; -} - -// ─── Tests ────────────────────────────────────────────────── - -describe('extractMeta', () => { - it('extracts title from first line of caption', () => { - const meta = extractMeta(singlePhotoPost); - assert.equal(meta.title, 'Hello @world! #sunset #photography'); - }); - - it('truncates long titles to 80 chars', () => { - const post = { ...singlePhotoPost, caption: 'A'.repeat(100) + '\nsecond line' }; - const meta = extractMeta(post); - assert.equal(meta.title.length, 80); - }); - - it('falls back to shortcode when no caption', () => { - const meta = extractMeta(noCaptionPost); - assert.equal(meta.title, 'Instagram NOCAP'); - }); - - it('generates ig- prefixed slug', () => { - const meta = extractMeta(singlePhotoPost); - assert.equal(meta.slug, 'ig-ABC123'); - }); - - it('preserves original publish date', () => { - const meta = extractMeta(singlePhotoPost); - assert.equal(meta.publishDate, '2023-06-15T14:30:00.000Z'); - }); - - it('sets featured image URL from first media item', () => { - const meta = extractMeta(singlePhotoPost); - assert.ok(meta.featuredImageUrl.includes('cdninstagram.com')); - }); -}); - -describe('buildInstagramContent', () => { - it('produces wp:image block for single photo', () => { - const content = buildInstagramContent(singlePhotoPost); - assert.ok(content.includes('')); - assert.ok(!content.includes('')); - assert.ok(content.includes('video.mp4')); - }); - - it('handles posts with no caption', () => { - const content = buildInstagramContent(noCaptionPost); - // Should have image but no caption paragraph (empty caption) - assert.ok(content.includes('')); - // Source link should still be present - assert.ok(content.includes('instagram-source')); - }); - - it('uses accessibility caption as alt text', () => { - const content = buildInstagramContent(singlePhotoPost); - assert.ok(content.includes('alt="A sunset over the ocean"')); - }); -}); - -describe('xmlValue', () => { - it('encodes strings with XML escaping', () => { - assert.equal(xmlValue('hello & '), 'hello & <world>'); - }); - - it('encodes integers', () => { - assert.equal(xmlValue(42), '42'); - }); - - it('encodes booleans', () => { - assert.equal(xmlValue(true), '1'); - assert.equal(xmlValue(false), '0'); - }); - - it('encodes dates WITHOUT trailing Z', () => { - const d = new Date('2020-03-15T12:00:00.000Z'); - const result = xmlValue(d); - assert.ok(result.includes('20200315T120000')); - assert.ok(!result.includes('Z')); - }); - - it('encodes buffers as base64', () => { - const buf = Buffer.from('hello'); - assert.equal(xmlValue(buf), 'aGVsbG8='); - }); - - it('encodes null as nil', () => { - assert.equal(xmlValue(null), ''); - }); - - it('encodes objects as structs', () => { - const result = xmlValue({ name: 'test', count: 5 }); - assert.ok(result.includes('')); - assert.ok(result.includes('name')); - assert.ok(result.includes('test')); - assert.ok(result.includes('5')); - }); - - it('skips undefined values in structs', () => { - const result = xmlValue({ name: 'test', missing: undefined }); - assert.ok(!result.includes('missing')); - }); - - it('encodes arrays', () => { - const result = xmlValue([1, 'two']); - assert.ok(result.includes('')); - assert.ok(result.includes('1')); - assert.ok(result.includes('two')); - }); -}); - -describe('extractPostMeta (discover)', () => { - it('classifies photo posts', () => { - const meta = extractPostMeta({ - id: '1', shortcode: 'TEST', display_url: 'http://example.com/img.jpg', - taken_at_timestamp: 1686830000, - edge_media_to_caption: { edges: [{ node: { text: 'Hello' } }] }, - }); - assert.equal(meta.type, 'photo'); - assert.equal(meta.caption, 'Hello'); - }); - - it('classifies carousel posts', () => { - const meta = extractPostMeta({ - id: '2', shortcode: 'CAR', display_url: 'http://example.com/img.jpg', - taken_at_timestamp: 1686830000, - edge_sidecar_to_children: { edges: [{ node: {} }, { node: {} }] }, - edge_media_to_caption: { edges: [] }, - }); - assert.equal(meta.type, 'carousel'); - assert.equal(meta.carouselCount, 2); - }); - - it('classifies video posts', () => { - const meta = extractPostMeta({ - id: '3', shortcode: 'VID', display_url: 'http://example.com/img.jpg', - taken_at_timestamp: 1686830000, is_video: true, - video_url: 'http://example.com/video.mp4', - edge_media_to_caption: { edges: [] }, - }); - assert.equal(meta.type, 'video'); - assert.ok(meta.isVideo); - assert.equal(meta.videoUrl, 'http://example.com/video.mp4'); - }); - - it('converts timestamp to ISO date', () => { - const meta = extractPostMeta({ - id: '4', shortcode: 'DATE', display_url: 'http://example.com/img.jpg', - taken_at_timestamp: 1686830000, - edge_media_to_caption: { edges: [] }, - }); - assert.ok(meta.date.startsWith('2023-06-15')); - }); - - it('extracts location name', () => { - const meta = extractPostMeta({ - id: '5', shortcode: 'LOC', display_url: 'http://example.com/img.jpg', - taken_at_timestamp: 1686830000, - location: { name: 'Central Park', id: '123' }, - edge_media_to_caption: { edges: [] }, - }); - assert.equal(meta.locationName, 'Central Park'); - }); - - it('generates correct Instagram URL', () => { - const meta = extractPostMeta({ - id: '6', shortcode: 'URL_TEST', display_url: 'http://example.com/img.jpg', - taken_at_timestamp: 1686830000, - edge_media_to_caption: { edges: [] }, - }); - assert.equal(meta.url, 'https://www.instagram.com/p/URL_TEST/'); - }); -}); - -describe('date formatting for WordPress', () => { - it('formats date as YYYY-MM-DD HH:MM:SS string', () => { - const d = new Date('2023-06-15T14:30:00.000Z'); - const formatted = `${d.getUTCFullYear()}-${String(d.getUTCMonth()+1).padStart(2,'0')}-${String(d.getUTCDate()).padStart(2,'0')} ${String(d.getUTCHours()).padStart(2,'0')}:${String(d.getUTCMinutes()).padStart(2,'0')}:${String(d.getUTCSeconds()).padStart(2,'0')}`; - assert.equal(formatted, '2023-06-15 14:30:00'); - }); - - it('pads single-digit months and days', () => { - const d = new Date('2023-01-05T03:09:07.000Z'); - const formatted = `${d.getUTCFullYear()}-${String(d.getUTCMonth()+1).padStart(2,'0')}-${String(d.getUTCDate()).padStart(2,'0')} ${String(d.getUTCHours()).padStart(2,'0')}:${String(d.getUTCMinutes()).padStart(2,'0')}:${String(d.getUTCSeconds()).padStart(2,'0')}`; - assert.equal(formatted, '2023-01-05 03:09:07'); - }); -}); - -describe('carousel deduplication logic', () => { - it('deduplicates by Instagram media ID', () => { - // Simulate what the extractor does: collect images across slides, - // deduplicate by the numeric media ID prefix in the CDN URL - const allImages = [ - { src: 'https://cdn.com/v/t51.29350-15/111_222.jpg', mediaId: '111' }, - { src: 'https://cdn.com/v/t51.82787-15/111_222.jpg', mediaId: '111' }, // same photo, different CDN path - { src: 'https://cdn.com/v/t51.82787-15/333_444.jpg', mediaId: '333' }, - { src: 'https://cdn.com/v/t51.82787-15/555_666.jpg', mediaId: '555' }, - ]; - - const seen = new Set(); - const deduped = []; - for (const img of allImages) { - if (!img.mediaId || seen.has(img.mediaId)) continue; - seen.add(img.mediaId); - deduped.push(img); - } - - assert.equal(deduped.length, 3); - assert.deepEqual(deduped.map(d => d.mediaId), ['111', '333', '555']); - }); -}); From 6ade650b1dcac530def386397bddf181f4083482 Mon Sep 17 00:00:00 2001 From: Matthew Batchelder Date: Sun, 12 Apr 2026 08:24:54 -0700 Subject: [PATCH 3/3] Address review feedback: drop Instagram from legacy CLI, harden username parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - cli.js: revert PR #4's Instagram branch — it referenced scripts/instagram/ scripts that no longer exist after restructuring. The legacy CLI is for legacy script entry points only; Instagram is reachable via the adapter. - src/adapters/instagram.ts: parseInstagramUsername now strips path segments from non-URL inputs so 'foo/bar' cannot be smuggled into a navigation URL. - DISCOVERIES.md: rewrite the 'How it works' paragraph for the adapter pipeline instead of the deleted scripts/instagram/ pipeline. --- DISCOVERIES.md | 2 +- cli.js | 33 ++++++--------------------------- src/adapters/instagram.ts | 6 ++++-- 3 files changed, 11 insertions(+), 30 deletions(-) diff --git a/DISCOVERIES.md b/DISCOVERIES.md index 10f7e102..ba8f13e3 100644 --- a/DISCOVERIES.md +++ b/DISCOVERIES.md @@ -26,7 +26,7 @@ Instagram is a React app that communicates via GraphQL queries to `https://www.i ### How it works -Three-step pipeline: discover (scroll + intercept GraphQL) → extract (visit each post, use `?img_index=N` for carousels, download media) → import (XML-RPC `wp.uploadFile` for media, `wp.newPost` with `post_thumbnail` for featured images and gallery blocks for carousels). +Implemented in `src/adapters/instagram.ts`: `discover()` scrolls the profile and intercepts GraphQL responses to build a post inventory; `extract()` visits each post (walking `?img_index=N` for carousel slides, deduped by Instagram media ID), downloads media via the shared `downloadMedia` helper, and emits `wp:image` / `wp:video` / `wp:gallery` Gutenberg blocks into the WXR. Hashtags become WordPress tags; `@mentions` and `#hashtags` in captions are linkified. The WXR is then imported via the generic WP importer in `src/lib/import/`. ### Why it's better than the previous approach diff --git a/cli.js b/cli.js index 4dfb17bc..8d1d768e 100644 --- a/cli.js +++ b/cli.js @@ -267,7 +267,6 @@ function getBrowserUserAgent(browser) { function detectPlatform(url) { const lower = url.toLowerCase(); - if (lower.includes('instagram.com')) return 'instagram'; if (lower.includes('wix.com') || lower.includes('wixsite.com')) return 'wix'; if (lower.includes('squarespace.com')) return 'squarespace'; if (lower.includes('webflow.io') || lower.includes('webflow.com')) return 'webflow'; @@ -406,7 +405,6 @@ async function main() { heading('Login Detection'); const platforms = [ - { name: 'Instagram', domain: '.instagram.com' }, { name: 'Wix', domain: '.wix.com' }, { name: 'Squarespace', domain: '.squarespace.com' }, { name: 'Webflow', domain: '.webflow.com' }, @@ -457,13 +455,12 @@ async function main() { ok(`Detected platform: ${BOLD}${detectedPlatform}${RESET}`); } else { const platChoice = await askChoice('Which platform is this site on?', [ - { label: 'Instagram', value: 'instagram' }, { label: 'Wix', value: 'wix' }, { label: 'Squarespace', value: 'squarespace' }, { label: 'Webflow', value: 'webflow' }, { label: 'Other / not sure', value: 'unknown' }, ]); - detectedPlatform = platChoice.value; + // Use first choice as default if detection fails } const activePlatform = detectedPlatform !== 'unknown' ? detectedPlatform : 'wix'; @@ -524,31 +521,13 @@ async function main() { // ── Step 4: Run discovery ── heading('Step 1: Discovering Site Content'); + log(`Scanning ${siteUrl} for all pages, posts, and media...\n`); mkdirSync('output', { recursive: true }); - // Instagram uses the browser's own UA via CDP — user-agent flag is only for Wix/other platforms - const uaArgs = (userAgent && activePlatform !== 'instagram') ? ['--user-agent', userAgent] : []; + const uaArgs = userAgent ? ['--user-agent', userAgent] : []; const cdpArgs = cdpPort ? ['--cdp-port', String(cdpPort)] : []; - - // Instagram uses a username, not a site URL - let discoverTarget = siteUrl; - if (activePlatform === 'instagram') { - // Extract username from URL or use as-is - const igMatch = siteUrl.match(/instagram\.com\/([^/?]+)/); - discoverTarget = igMatch ? igMatch[1] : siteUrl.replace(/^https?:\/\//, '').replace(/\/$/, ''); - log(`Discovering posts for @${discoverTarget}...\n`); - if (!cdpPort) { - fail('Instagram requires a CDP connection to an authenticated browser.'); - fail('Launch Chrome with: google-chrome --remote-debugging-port=9222'); - rl.close(); - return; - } - } else { - log(`Scanning ${siteUrl} for all pages, posts, and media...\n`); - } - - const discoverResult = await runScript(`scripts/${activePlatform}/discover.js`, [discoverTarget, ...uaArgs, ...cdpArgs]); + const discoverResult = await runScript(`scripts/${activePlatform}/discover.js`, [siteUrl, ...uaArgs, ...cdpArgs]); if (discoverResult.code !== 0) { fail('Discovery failed. See output above.'); const retry = await ask('Try again? (y/n)'); @@ -579,10 +558,10 @@ async function main() { // ── Step 5: Extract content ── heading('Step 2: Extracting Content'); - log(`Extracting all content from ${activePlatform === 'instagram' ? '@' + discoverTarget : siteUrl}...\n`); + log(`Extracting all pages, posts, and media from ${siteUrl}...\n`); const extractResult = await runScript(`scripts/${activePlatform}/extract.js`, [ - activePlatform === 'instagram' ? discoverTarget : siteUrl, + siteUrl, '--url-list', 'output/inventory.json', ...uaArgs, ...cdpArgs diff --git a/src/adapters/instagram.ts b/src/adapters/instagram.ts index d97564a8..4133d3d1 100644 --- a/src/adapters/instagram.ts +++ b/src/adapters/instagram.ts @@ -89,9 +89,11 @@ export function parseInstagramUsername(input: string): string { const first = u.pathname.replace(/^\//, '').split('/')[0]; return first || ''; } - return trimmed; + // Bare input (not an Instagram URL): take the first path segment so + // `foo/bar` cannot smuggle a path into a navigation URL. + return trimmed.split('/')[0]; } catch { - return trimmed; + return trimmed.split('/')[0]; } }