From aba77b8379eb0335f90de8b2ee3da43f879d8378 Mon Sep 17 00:00:00 2001 From: Joe Boydston Date: Thu, 2 Apr 2026 22:29:22 -0600 Subject: [PATCH] Add Substack migration support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New platform extractor for Substack publications. Uses Substack's undocumented public API for content discovery and extraction — no browser needed. Supports dual-source extraction: API for metadata and free content, CSV export for full paid post content. Images are unwrapped from Substack's CDN proxy URLs to download originals. Also adds --self-hosted flag to import.js for non-WordPress.com sites, fixes media upload Content-Type handling, and improves image URL replacement to handle CDN wrapper URLs. Tested end-to-end against coloradomedia.substack.com → Atomic staging site. Co-Authored-By: Claude Opus 4.6 (1M context) --- AGENTS.md | 73 +++++- DISCOVERIES.md | 31 +++ README.md | 30 +++ package.json | 2 +- prompts/substack.md | 84 +++++++ scripts/import.js | 99 ++++++-- scripts/substack/discover.js | 191 +++++++++++++++ scripts/substack/extract.js | 453 +++++++++++++++++++++++++++++++++++ 8 files changed, 938 insertions(+), 25 deletions(-) create mode 100644 prompts/substack.md create mode 100644 scripts/substack/discover.js create mode 100644 scripts/substack/extract.js diff --git a/AGENTS.md b/AGENTS.md index c8cf3d6b..4f18ff3f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,7 +11,7 @@ This file is the entry point for any AI agent using this repository. Read it bef 3. **User prompts** — non-technical users paste these into their AI to drive the whole migration 4. **A living playbook** — this repo improves itself through AI-contributed discoveries -Currently supports: **Wix**. Squarespace, Webflow, and Shopify are planned. +Currently supports: **Wix** and **Substack**. Squarespace, Webflow, and Shopify are planned. ## If you're helping a user migrate from Wix @@ -96,6 +96,57 @@ This approach works for any JavaScript-heavy platform, not just Wix. --- +## If you're helping a user migrate from Substack + +### Step 1 — Understand the publication + +```bash +node scripts/substack/discover.js +``` + +This queries Substack's public API to list all posts, categorize them (free/paid/podcast), and writes `output/inventory.json`. No browser needed. + +### Step 2 — Extract all content + +```bash +node scripts/substack/extract.js +``` + +This fetches each post via the API. For **paid posts**, the API only returns the free preview. To get full paid content: + +1. Ask the user to export from Substack: Settings > Exports > Create new export +2. They'll get a ZIP with a CSV file +3. Unzip it and pass the CSV: + +```bash +node scripts/substack/extract.js --csv-export posts.csv +``` + +The extractor merges API metadata (rich) with CSV content (complete), giving you the best of both. + +**If scripts aren't available**, you can extract directly: +1. Fetch the archive: `/api/v1/archive?sort=new&limit=50&offset=0` +2. For each post: `/api/v1/posts/` — returns full JSON with `body_html` +3. Download images by unwrapping CDN URLs: `substackcdn.com/image/fetch/.../https://substack-post-media.s3.amazonaws.com/...` — the real URL is embedded after the transform params + +### Step 3 — Import to WordPress.com + +Same as Wix — the import script is platform-agnostic: + +```bash +node scripts/import.js --site --token +``` + +### Step 4 — Handle Substack-specific concerns + +After import, help the user with: +- **Subscribers**: Export from Substack (Settings > Exports), import into their email service or WordPress.com newsletter +- **Paid subscriptions**: Complex — involves Stripe migration. Discuss options before acting. +- **Redirects**: If they have a custom domain, set up redirects from `/p/` to new WordPress paths. If on `*.substack.com`, they can't redirect — suggest a pinned farewell post. +- **Podcast hosting**: Audio files need to be re-hosted if they have podcast episodes. + +--- + ## Platform-specific barriers ### Wix @@ -109,6 +160,20 @@ This approach works for any JavaScript-heavy platform, not just Wix. | Anti-bot blocking | Use real browser via Playwright (not fetch/curl); add delays | | Dynamic pages (CMS collections) | Query Wix's `/_api/wix-data-server/` endpoints directly | +### Substack + +| Problem | Solution | +|---|---| +| Paid content truncated in API | Use Substack's CSV export (`--csv-export`) for full body HTML | +| No official API | Undocumented `/api/v1/posts` and `/api/v1/archive` endpoints are public | +| Images behind CDN wrapper | Unwrap `substackcdn.com/image/fetch/.../` URLs to get original S3 paths | +| URL structure (`/p/slug`) | Generate redirect map; set WordPress permalinks to match if possible | +| Subtitles not in WordPress | Add as styled first paragraph or use a subtitle plugin | +| Subscriber migration | Separate CSV export; requires email service setup | +| Paid subscriber migration | Stripe account transfer needed; no automated path | +| Comments not exportable | Not included in any export; consider a scraping approach if critical | +| Notes/threads | Newer feature; may not export cleanly via API | + --- ## How to contribute improvements back @@ -149,11 +214,15 @@ data-liberation-agent/ ├── DISCOVERIES.md ← log of community-contributed findings ├── package.json ├── prompts/ -│ └── wix.md ← what users paste into their AI for a Wix migration +│ ├── wix.md ← what users paste into their AI for a Wix migration +│ └── substack.md ← what users paste into their AI for a Substack migration ├── scripts/ │ ├── wix/ │ │ ├── discover.js ← inventory the Wix site (sitemap + categorization) │ │ └── extract.js ← extract all content via network interception +│ ├── substack/ +│ │ ├── discover.js ← inventory the Substack publication via API +│ │ └── extract.js ← extract content via API + CSV export │ └── import.js ← publish to WordPress.com via REST API (platform-agnostic) ├── examples/ │ ├── wix-api-blog-post.json ← example of Wix internal API response diff --git a/DISCOVERIES.md b/DISCOVERIES.md index 3ee5e33b..56803fc1 100644 --- a/DISCOVERIES.md +++ b/DISCOVERIES.md @@ -6,6 +6,37 @@ AI agents: when you contribute an improvement, add an entry here. See [CONTRIBUT --- +## 2026-04-02 — Substack public API for content extraction + +**Found by:** Claude + human contributor (Joe Boydston, Newspack team) +**During:** Adding Substack as a new migration platform +**Type:** API endpoint | new platform + +### What I found + +Substack has undocumented but publicly accessible API endpoints that return rich JSON for any publication without authentication: + +- `/api/v1/archive?sort=new&limit=50&offset=0` — paginated list of all posts with metadata +- `/api/v1/posts/` — full post data including `body_html`, cover image, subtitle, audience tier, word count, reactions, and author bylines + +Images are served through a CDN wrapper at `substackcdn.com/image/fetch/w_XXXX,.../https://substack-post-media.s3.amazonaws.com/...`. The original full-resolution URL is embedded in the CDN path after the transform parameters. + +### How it works + +The discover script paginates through `/api/v1/archive` to build a complete inventory. The extract script then fetches each post individually via `/api/v1/posts/` for rich metadata and full HTML body. For paid posts (where the API only returns free preview content), the extractor can merge in full HTML from Substack's official CSV export via `--csv-export`. + +No browser or authentication needed — this runs entirely via `fetch()`. + +### Why it's better than the previous approach + +Unlike Wix (which requires Playwright to intercept internal API calls during page load), Substack's public API means: +- No browser dependency — extraction is fast and lightweight +- Clean, structured JSON with semantic fields +- Dual-source strategy (API + CSV) ensures complete content including paid posts +- Rate limiting is lenient with 500ms delays between requests + +--- + ## 2026-03-31 — Wix Dashboard API reverse engineering via CDP **Found by:** Claude + human contributor (live probing against Brave browser) diff --git a/README.md b/README.md index d7d4d225..3ea11d20 100644 --- a/README.md +++ b/README.md @@ -17,10 +17,30 @@ This repo gives people a prompt they can paste into any AI assistant (Claude, Ch | Platform | Status | Prompt | |---|---|---| | **Wix** | Ready | [`prompts/wix.md`](./prompts/wix.md) | +| **Substack** | Ready | [`prompts/substack.md`](./prompts/substack.md) | | Squarespace | Planned | — | | Webflow | Planned | — | | Shopify (blog/pages) | Planned | — | +## Quick start (Substack) + +```bash +# 1. Install dependencies +npm install + +# 2. Discover all content on your Substack +node scripts/substack/discover.js https://yourpub.substack.com + +# 3. Extract all content (uses Substack's public API) +node scripts/substack/extract.js https://yourpub.substack.com + +# 4. For paid posts, export from Substack first, then: +node scripts/substack/extract.js https://yourpub.substack.com --csv-export posts.csv + +# 5. Import to WordPress.com +node scripts/import.js --site your-wp-site --token YOUR_APP_PASSWORD +``` + ## Quick start (Wix) ```bash @@ -58,6 +78,16 @@ This means the playbook gets smarter with every migration. - [ ] Wix Stores / WooCommerce migration - [ ] Wix Bookings migration +### Substack +- [x] Public API extraction (free content) +- [x] CSV export support (paid content) +- [x] Image CDN URL unwrapping +- [x] WordPress.com REST API import +- [x] Migration prompt for non-technical users +- [ ] Podcast episode audio migration +- [ ] Subscriber list import +- [ ] Paid subscription migration (Stripe) + ### General - [x] WordPress.com REST API import script - [ ] WordPress Studio local-first workflow diff --git a/package.json b/package.json index bb627e1b..8b903ead 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "data-liberation-agent", "version": "0.1.0", - "description": "AI-assisted migration from Wix to WordPress.com", + "description": "AI-assisted migration from closed platforms to WordPress.com", "type": "module", "scripts": { "discover": "node scripts/discover.js", diff --git a/prompts/substack.md b/prompts/substack.md new file mode 100644 index 00000000..59d63e96 --- /dev/null +++ b/prompts/substack.md @@ -0,0 +1,84 @@ +# Substack to WordPress.com Migration Prompt + +Copy everything below this line and paste it into your AI assistant (Claude, ChatGPT, Gemini, etc.). + +--- + +I want to migrate my publication from Substack to WordPress.com. My Substack URL is: **[PASTE YOUR SUBSTACK URL HERE]** + +I have (or will create) a WordPress.com account. Please help me migrate using the playbook at https://github.com/Automattic/data-liberation-agent — read AGENTS.md first for full instructions. + +Here's what I need you to do: + +## Step 1: Inventory my publication + +- Use the Substack API to list all my posts: fetch `[MY SUBSTACK URL]/api/v1/archive?sort=new&limit=50&offset=0` (paginate through all results) +- Categorize each post: free, paid, podcast, thread, page +- Note my publication name, description, and about page +- Count total posts, paid vs. free breakdown, and any podcast episodes +- Show me the inventory and wait for my approval before proceeding + +## Step 2: Export and extract content + +**For free content:** +- Fetch each post's full content via the API: `[MY SUBSTACK URL]/api/v1/posts/[SLUG]` +- This gives you the full HTML body, metadata, cover images, and dates + +**For paid content (if I have any):** +- Ask me to go to Substack Settings > Exports > Create new export +- I'll download the ZIP and give you the CSV file +- Use the CSV's `body_html` column for paid post content — the API only gives the free preview + +**For all content:** +- Download every image — Substack wraps images through `substackcdn.com/image/fetch/...`. Extract the original URL from inside the CDN wrapper to get full-resolution images +- Preserve for each post: title, subtitle, URL slug, publish date, categories/sections, cover image, audience (free/paid), word count + +## Step 3: Set up WordPress.com + +I need to create/have a WordPress.com site. Help me: +- Recommend a theme that works well for newsletters/blogs +- Create categories based on any Substack sections I have +- Configure basic settings: site title, tagline, permalink structure matching `/p/[slug]` if possible (for easier redirects) + +For connecting to WordPress.com, I can either: +- Enable MCP at wordpress.com/me/mcp and connect you directly +- Generate an Application Password at wordpress.com/me/security/application-passwords + +Tell me which you need. + +## Step 4: Publish everything + +In this order: +1. Upload all images to the WordPress media library (needed first to get new URLs) +2. Create all posts with correct dates, featured images, and content +3. Rewrite all internal links and image `src` attributes from Substack URLs to new WordPress URLs +4. Set up navigation and homepage + +**Special handling needed:** +- **Subtitles**: Substack has a subtitle field — add it as the first line of the post in `` tags, or use a subtitle plugin if available +- **Paid content**: If I want to gate content on WordPress, ask me which membership/paywall plugin to use (WooCommerce Memberships, Restrict Content Pro, etc.) and which posts should stay gated +- **Podcast episodes**: If I have podcast content, check if the audio files are downloadable from Substack and re-host them. Set up a podcasting plugin if needed. + +## Step 5: Handle subscribers + +- Ask me to export my subscriber list from Substack (Settings > Exports) +- Help me choose an email service (Mailchimp, ConvertKit, ActiveCampaign, etc.) or the built-in WordPress.com newsletter feature +- Import the subscriber list + +**Note about paid subscribers**: If I have paid subscribers through Substack, this is complex — I may need to: +- Set up a new payment system (Stripe on WordPress) +- Communicate with subscribers about the move +- Plan a transition period + +Tell me what my options are before doing anything. + +## Step 6: Verify and redirect + +When done: +- Give me a URL mapping table: old Substack URL → new WordPress URL (for setting up redirects) +- Check that no images still point to `substackcdn.com` +- If I use a custom domain on Substack, help me set up 301 redirects from `/p/[slug]` to the new WordPress paths +- If I'm on `[name].substack.com`, note that I can't set up redirects — but I can add a pinned post on Substack linking to my new site +- List anything that needs manual attention + +Work methodically — do one step at a time, show me progress, and wait for my go-ahead before moving to the next step. If you hit something unexpected, tell me what you found rather than guessing. diff --git a/scripts/import.js b/scripts/import.js index 868c89a1..500dd448 100644 --- a/scripts/import.js +++ b/scripts/import.js @@ -8,17 +8,19 @@ * Usage: * node scripts/import.js --site mysite.wordpress.com --token APP_PASSWORD * node scripts/import.js --site mysite.wordpress.com --token APP_PASSWORD --dry-run + * node scripts/import.js --site mysite.com --user admin --token APP_PASSWORD --self-hosted * * Options: - * --site WordPress.com site domain (e.g. mysite.wordpress.com) - * --token Application password from wordpress.com/me/security/application-passwords + * --site WordPress site domain (e.g. mysite.wordpress.com) + * --token Application password + * --user WordPress username (required for --self-hosted, default: "wix-escape" for WP.com) + * --self-hosted Use direct WP REST API instead of WordPress.com proxy * --dry-run Show what would be imported without actually doing it * --only Only import 'media', 'pages', or 'posts' * * Getting your application password: - * 1. Go to wordpress.com/me/security/application-passwords - * 2. Create a new application password named "wix-escape" - * 3. Copy the password and pass it as --token + * WordPress.com: wordpress.com/me/security/application-passwords + * Self-hosted/Atomic: WP Admin > Users > Profile > Application Passwords */ import { readFileSync, readdirSync, existsSync } from 'fs'; @@ -33,17 +35,28 @@ function getArg(name) { const site = getArg('--site'); const token = getArg('--token'); +const user = getArg('--user'); +const selfHosted = args.includes('--self-hosted'); const dryRun = args.includes('--dry-run'); const only = getArg('--only'); if (!site || !token) { console.error('Usage: node scripts/import.js --site --token '); - console.error(' Get your app password at: wordpress.com/me/security/application-passwords'); + console.error(' WordPress.com: node scripts/import.js --site mysite.wordpress.com --token APP_PASSWORD'); + console.error(' Self-hosted: node scripts/import.js --site mysite.com --user admin --token APP_PASSWORD --self-hosted'); process.exit(1); } -const apiBase = `https://public-api.wordpress.com/wp/v2/sites/${site}`; -const authHeader = `Basic ${Buffer.from(`wix-escape:${token}`).toString('base64')}`; +if (selfHosted && !user) { + console.error('--self-hosted requires --user '); + process.exit(1); +} + +const apiBase = selfHosted + ? `https://${site}/wp-json/wp/v2` + : `https://public-api.wordpress.com/wp/v2/sites/${site}`; +const authUser = user || 'wix-escape'; +const authHeader = `Basic ${Buffer.from(`${authUser}:${token}`).toString('base64')}`; async function wpRequest(method, endpoint, body, isFormData = false) { const headers = { Authorization: authHeader }; @@ -81,6 +94,9 @@ function buildContentFromAccessibility(nodes) { // Extract the best available content from a page JSON file function extractContent(pageData) { + // Platform-specific content field (e.g. Substack stores HTML directly) + if (pageData.content?.html) return pageData.content.html; + // Priority: Wix blog API response > JSON-LD > accessibility tree for (const call of pageData.apiCalls || []) { // Blog post body is typically in post.content or post.richContent @@ -119,16 +135,20 @@ async function uploadMedia(filePath, filename) { return { id: 0, source_url: `https://example.com/wp-content/uploads/${filename}` }; } - const FormData = (await import('node:buffer')).default; - // Use fetch with FormData - const formData = new globalThis.FormData(); const fileBuffer = readFileSync(filePath); - const blob = new Blob([fileBuffer]); - formData.append('file', blob, filename); + const ext = filename.split('.').pop().toLowerCase(); + const mimeTypes = { + jpg: 'image/jpeg', jpeg: 'image/jpeg', png: 'image/png', + gif: 'image/gif', webp: 'image/webp', svg: 'image/svg+xml', + }; const res = await fetch(`${apiBase}/media`, { method: 'POST', - headers: { Authorization: authHeader, 'Content-Disposition': `attachment; filename="${filename}"` }, + headers: { + Authorization: authHeader, + 'Content-Disposition': `attachment; filename="${filename}"`, + 'Content-Type': mimeTypes[ext] || 'application/octet-stream', + }, body: fileBuffer, }); @@ -159,14 +179,50 @@ async function importMedia() { return mediaMap; } +// Replace image URLs in content with uploaded WordPress media URLs +function replaceImageUrls(content, mediaMap) { + let result = content; + + // First pass: replace full CDN wrapper URLs (must run before filename replacement) + // CDN format: https://substackcdn.com/image/fetch//https%3A%2F%2Fsubstack-post-media.../ + result = result.replace( + /https:\/\/substackcdn\.com\/image\/fetch\/[^"'\s>]+/g, + (cdnUrl) => { + for (const [filename, wpUrl] of Object.entries(mediaMap)) { + if (cdnUrl.includes(filename) || cdnUrl.includes(encodeURIComponent(filename))) { + return wpUrl; + } + } + return cdnUrl; + } + ); + + // Second pass: replace direct S3 URLs + result = result.replace( + /https:\/\/substack-post-media\.s3\.amazonaws\.com\/[^"'\s>]+/g, + (s3Url) => { + for (const [filename, wpUrl] of Object.entries(mediaMap)) { + if (s3Url.includes(filename)) { + return wpUrl; + } + } + return s3Url; + } + ); + + // Third pass: replace remaining bare filenames (Wix-style and other platforms) + for (const [filename, wpUrl] of Object.entries(mediaMap)) { + result = result.replaceAll(filename, wpUrl); + } + + return result; +} + async function importPage(pageData, mediaMap) { const meta = extractMeta(pageData); let content = extractContent(pageData); - // Replace any Wix image URLs in content with uploaded WP URLs - for (const [filename, wpUrl] of Object.entries(mediaMap)) { - content = content.replaceAll(filename, wpUrl); - } + content = replaceImageUrls(content, mediaMap); const body = { title: meta.title, @@ -188,9 +244,7 @@ async function importPost(pageData, mediaMap) { const meta = extractMeta(pageData); let content = extractContent(pageData); - for (const [filename, wpUrl] of Object.entries(mediaMap)) { - content = content.replaceAll(filename, wpUrl); - } + content = replaceImageUrls(content, mediaMap); const body = { title: meta.title, @@ -264,7 +318,8 @@ async function main() { // Step 3: Import posts const posts = pageFiles.filter(f => { const slug = f.replace('.json', ''); - return typeMap[slug] === 'blog-post'; + const type = typeMap[slug]; + return type === 'blog-post' || type === 'post' || type === 'paid-post' || type === 'podcast' || type === 'thread' || type === 'video'; }); if (!only || only === 'posts') { diff --git a/scripts/substack/discover.js b/scripts/substack/discover.js new file mode 100644 index 00000000..554a1211 --- /dev/null +++ b/scripts/substack/discover.js @@ -0,0 +1,191 @@ +#!/usr/bin/env node +/** + * discover.js — Step 1: Inventory a Substack publication + * + * Uses Substack's public API and sitemap to find all content. + * No browser or authentication needed — Substack's API is publicly accessible. + * + * Usage: + * node scripts/substack/discover.js https://yourpub.substack.com + * node scripts/substack/discover.js https://customdomain.com + * + * Output: + * output/inventory.json — categorized list of all content URLs + */ + +import { writeFileSync, mkdirSync } from 'fs'; + +const args = process.argv.slice(2); +const siteUrl = args.find(a => a.startsWith('http')); +if (!siteUrl) { + console.error('Usage: node scripts/substack/discover.js '); + process.exit(1); +} + +const base = siteUrl.replace(/\/$/, ''); +mkdirSync('output', { recursive: true }); + +function sleep(ms) { + return new Promise(r => setTimeout(r, ms)); +} + +// Classify a Substack URL into a content type +function classify(url, apiPost) { + if (apiPost) { + if (apiPost.type === 'podcast') return 'podcast'; + if (apiPost.type === 'thread') return 'thread'; + if (apiPost.type === 'video') return 'video'; + if (apiPost.audience !== 'everyone') return 'paid-post'; + return 'post'; + } + const path = new URL(url).pathname.toLowerCase(); + if (path.startsWith('/p/')) return 'post'; + if (path === '/about' || path === '/about/') return 'page'; + if (path === '/archive' || path.startsWith('/archive')) return 'archive'; + if (path === '/' || path === '') return 'homepage'; + if (path === '/podcast' || path.startsWith('/podcast')) return 'podcast'; + return 'page'; +} + +// Fetch all posts via Substack's undocumented public API +async function fetchAllPosts() { + const posts = []; + let offset = 0; + const limit = 50; + + while (true) { + const url = `${base}/api/v1/archive?sort=new&limit=${limit}&offset=${offset}`; + console.log(` Fetching posts (offset ${offset})...`); + + try { + const res = await fetch(url); + if (!res.ok) { + console.error(` API returned ${res.status} at offset ${offset}`); + break; + } + const batch = await res.json(); + if (!Array.isArray(batch) || batch.length === 0) break; + posts.push(...batch); + offset += limit; + if (batch.length < limit) break; + await sleep(500); + } catch (e) { + console.error(` API fetch failed: ${e.message}`); + break; + } + } + + return posts; +} + +// Fetch sitemap for any additional URLs +async function fetchSitemap() { + const urls = []; + try { + const res = await fetch(`${base}/sitemap.xml`); + if (!res.ok) return urls; + const text = await res.text(); + const locs = [...text.matchAll(/([^<]+)<\/loc>/g)].map(m => m[1].trim()); + for (const loc of locs) { + if (loc.endsWith('.xml')) { + // Recurse into sitemap indexes + try { + const subRes = await fetch(loc); + if (subRes.ok) { + const subText = await subRes.text(); + const subLocs = [...subText.matchAll(/([^<]+)<\/loc>/g)].map(m => m[1].trim()); + urls.push(...subLocs.filter(l => !l.endsWith('.xml'))); + } + } catch {} + } else { + urls.push(loc); + } + } + } catch (e) { + console.error(` Sitemap fetch failed: ${e.message}`); + } + return urls; +} + +async function main() { + console.log(`Discovering: ${base}`); + + // Fetch posts via API + console.log('\nFetching posts via API...'); + const apiPosts = await fetchAllPosts(); + console.log(` Found ${apiPosts.length} posts via API`); + + // Fetch sitemap for additional URLs + console.log('\nFetching sitemap...'); + const sitemapUrls = await fetchSitemap(); + console.log(` Found ${sitemapUrls.length} URLs in sitemap`); + + // Build inventory from API posts (primary source) + const inventory = { + siteUrl: base, + discoveredAt: new Date().toISOString(), + platform: 'substack', + navigation: [ + { text: 'Home', href: base }, + { text: 'Archive', href: `${base}/archive` }, + { text: 'About', href: `${base}/about` }, + ], + counts: {}, + urls: [], + apiPostCount: apiPosts.length, + }; + + const seen = new Set(); + + // Add API posts with rich metadata + for (const post of apiPosts) { + const url = post.canonical_url || `${base}/p/${post.slug}`; + if (seen.has(url)) continue; + seen.add(url); + + const type = classify(url, post); + inventory.urls.push({ + url, + type, + slug: post.slug, + title: post.title, + subtitle: post.subtitle || null, + date: post.post_date, + audience: post.audience, + wordCount: post.word_count || null, + hasPodcast: !!post.podcast_url, + }); + inventory.counts[type] = (inventory.counts[type] || 0) + 1; + } + + // Add any sitemap URLs not already covered + for (const url of sitemapUrls) { + if (seen.has(url)) continue; + seen.add(url); + const type = classify(url); + if (type === 'archive' || type === 'homepage') continue; + inventory.urls.push({ url, type }); + inventory.counts[type] = (inventory.counts[type] || 0) + 1; + } + + writeFileSync('output/inventory.json', JSON.stringify(inventory, null, 2)); + + console.log('\nInventory summary:'); + for (const [type, count] of Object.entries(inventory.counts)) { + console.log(` ${type}: ${count}`); + } + console.log(`\nTotal: ${inventory.urls.length} URLs`); + console.log('Written to output/inventory.json'); + console.log('\nReview this inventory before running extract.js'); + + // Warn about paid content + const paidCount = inventory.counts['paid-post'] || 0; + if (paidCount > 0) { + console.log(`\nNote: ${paidCount} paid posts detected. The public API only returns`); + console.log('free preview content for these. To get full paid content, use the'); + console.log('Substack export (Settings > Exports) and pass the ZIP to extract.js'); + console.log('with --csv-export.'); + } +} + +main().catch(e => { console.error(e); process.exit(1); }); diff --git a/scripts/substack/extract.js b/scripts/substack/extract.js new file mode 100644 index 00000000..937160bb --- /dev/null +++ b/scripts/substack/extract.js @@ -0,0 +1,453 @@ +#!/usr/bin/env node +/** + * extract.js — Step 2: Extract all content from a Substack publication + * + * Two extraction modes: + * 1. API mode (default): Fetches each post via Substack's public API. + * Fast, no browser needed, but paid posts are truncated. + * 2. CSV mode (--csv-export): Reads from Substack's ZIP/CSV export. + * Gets full content for paid posts but no API metadata. + * + * Best results: run both. API for metadata + free posts, CSV for paid content. + * + * Usage: + * node scripts/substack/extract.js https://yourpub.substack.com + * node scripts/substack/extract.js https://yourpub.substack.com --csv-export export.zip + * node scripts/substack/extract.js https://yourpub.substack.com --csv-export posts.csv + * node scripts/substack/extract.js https://yourpub.substack.com --delay 1000 --limit 10 + * + * Options: + * --csv-export Path to Substack export ZIP or CSV file + * --delay Delay between API requests (default: 500) + * --limit Only process first N posts (for testing) + * --url-list Use URL list from discover.js output + * + * Output: + * output/pages/.json — extracted post data (matches import.js format) + * output/media/ — downloaded images + * output/extraction-log.json — summary of what was extracted + */ + +import { writeFileSync, mkdirSync, readFileSync, existsSync, createWriteStream } from 'fs'; +import { basename } from 'path'; +import https from 'https'; +import http from 'http'; + +const args = process.argv.slice(2); +const siteUrl = args.find(a => a.startsWith('http')); +if (!siteUrl) { + console.error('Usage: node scripts/substack/extract.js [options]'); + process.exit(1); +} + +const base = siteUrl.replace(/\/$/, ''); +const delay = parseInt(args[args.indexOf('--delay') + 1] || '500'); +const limitArg = args.indexOf('--limit'); +const limit = limitArg !== -1 ? parseInt(args[limitArg + 1]) : Infinity; +const csvArg = args.indexOf('--csv-export'); +const csvFile = csvArg !== -1 ? args[csvArg + 1] : null; +const urlListArg = args.indexOf('--url-list'); +const urlListFile = urlListArg !== -1 ? args[urlListArg + 1] : null; + +mkdirSync('output/pages', { recursive: true }); +mkdirSync('output/media', { recursive: true }); + +function sleep(ms) { + return new Promise(r => setTimeout(r, ms)); +} + +function slugify(url) { + return new URL(url).pathname.replace(/^\//, '').replace(/\//g, '--') || 'homepage'; +} + +async function downloadFile(url, destPath) { + return new Promise((resolve, reject) => { + const proto = url.startsWith('https') ? https : http; + const file = createWriteStream(destPath); + proto.get(url, res => { + if (res.statusCode === 301 || res.statusCode === 302) { + file.close(); + return downloadFile(res.headers.location, destPath).then(resolve).catch(reject); + } + if (res.statusCode !== 200) { + file.close(); + return reject(new Error(`HTTP ${res.statusCode}`)); + } + res.pipe(file); + file.on('finish', () => file.close(resolve)); + }).on('error', reject); + }); +} + +// Extract original image URL from Substack's CDN wrapper +// substackcdn.com/image/fetch/w_1234,h_567,.../https://substack-post-media.s3.amazonaws.com/... +function unwrapImageUrl(cdnUrl) { + const match = cdnUrl.match(/substackcdn\.com\/image\/fetch\/[^/]+\/(https?:\/\/.+)/); + if (match) return decodeURIComponent(match[1]); + return cdnUrl; +} + +// Find all image URLs in HTML content or JSON data +function extractImageUrls(data) { + const urls = new Set(); + const str = typeof data === 'string' ? data : JSON.stringify(data); + + // Substack CDN images — stop at quotes, whitespace, parentheses, commas, or escaped quotes + const cdnMatches = str.match(/https:\/\/substackcdn\.com\/image\/fetch\/[^"'\s),\\]+\.(png|jpg|jpeg|gif|webp|svg)(\?[^"'\s),\\]*)?/gi) || []; + for (const url of cdnMatches) { + urls.add(url.replace(/\\u002F/g, '/')); + } + + // Direct S3 images + const s3Matches = str.match(/https:\/\/substack-post-media\.s3\.amazonaws\.com\/[^"'\s),\\]+\.(png|jpg|jpeg|gif|webp|svg)(\?[^"'\s),\\]*)?/gi) || []; + for (const url of s3Matches) { + urls.add(url); + } + + // Bucketeer (older Substack images) + const bucketMatches = str.match(/https:\/\/bucketeer-[^"'\s),\\]+\.(png|jpg|jpeg|gif|webp|svg)(\?[^"'\s),\\]*)?/gi) || []; + for (const url of bucketMatches) { + urls.add(url); + } + + return [...urls]; +} + +// Parse Substack's CSV export +function parseCSV(csvContent) { + const lines = csvContent.split('\n'); + if (lines.length < 2) return []; + + // Parse header + const headers = parseCSVLine(lines[0]); + const posts = []; + + let currentLine = ''; + for (let i = 1; i < lines.length; i++) { + currentLine += (currentLine ? '\n' : '') + lines[i]; + // Count unescaped quotes — if odd, the line continues + const quoteCount = (currentLine.match(/"/g) || []).length; + if (quoteCount % 2 !== 0) continue; + + const values = parseCSVLine(currentLine); + if (values.length >= headers.length) { + const post = {}; + for (let j = 0; j < headers.length; j++) { + post[headers[j]] = values[j] || ''; + } + posts.push(post); + } + currentLine = ''; + } + + return posts; +} + +function parseCSVLine(line) { + const values = []; + let current = ''; + let inQuotes = false; + + for (let i = 0; i < line.length; i++) { + const char = line[i]; + if (char === '"') { + if (inQuotes && line[i + 1] === '"') { + current += '"'; + i++; + } else { + inQuotes = !inQuotes; + } + } else if (char === ',' && !inQuotes) { + values.push(current); + current = ''; + } else { + current += char; + } + } + values.push(current); + return values; +} + +// Extract ZIP file to get CSV (basic ZIP handling for single-file ZIPs) +async function loadCSVExport(filePath) { + const content = readFileSync(filePath); + + // Check if it's a ZIP + if (content[0] === 0x50 && content[1] === 0x4B) { + console.error('ZIP files must be extracted first. Unzip and pass the CSV path:'); + console.error(` unzip ${filePath} -d substack-export/`); + console.error(' node scripts/substack/extract.js --csv-export substack-export/posts.csv'); + process.exit(1); + } + + return parseCSV(content.toString('utf8')); +} + +// Fetch a single post via Substack's public API +async function fetchPost(slug) { + const url = `${base}/api/v1/posts/${slug}`; + const res = await fetch(url); + if (!res.ok) throw new Error(`API returned ${res.status}`); + return res.json(); +} + +// Convert a Substack API post to the output format expected by import.js +function apiPostToPageData(post) { + const url = post.canonical_url || `${base}/p/${post.slug}`; + return { + sourceUrl: url, + slug: `p--${post.slug}`, + extractedAt: new Date().toISOString(), + platform: 'substack', + apiCalls: [{ url: `${base}/api/v1/posts/${post.slug}`, data: post }], + globals: { + jsonLd: [{ + '@type': post.type === 'podcast' ? 'PodcastEpisode' : 'BlogPosting', + headline: post.title, + description: post.subtitle || post.description || '', + datePublished: post.post_date, + dateModified: post.post_date, + articleBody: post.body_text || '', + image: post.cover_image ? { url: post.cover_image } : undefined, + author: post.publishedBylines?.map(b => ({ + '@type': 'Person', + name: b.name, + })) || [], + }], + meta: { + title: post.title, + description: post.subtitle || post.description || '', + ogTitle: post.title, + ogDescription: post.subtitle || '', + ogImage: post.cover_image || null, + canonical: url, + }, + }, + content: { + html: post.body_html || '', + subtitle: post.subtitle || null, + audience: post.audience, + type: post.type, + wordCount: post.word_count || null, + podcastUrl: post.podcast_url || null, + likes: post.reactions?.['❤'] || post.reaction_count || 0, + commentCount: post.comment_count || 0, + }, + accessibility: null, + }; +} + +// Convert a CSV row to the output format expected by import.js +function csvPostToPageData(row) { + const url = row.canonical_url || `${base}/p/${row.slug}`; + return { + sourceUrl: url, + slug: `p--${row.slug}`, + extractedAt: new Date().toISOString(), + platform: 'substack', + apiCalls: [], + globals: { + jsonLd: [{ + '@type': 'BlogPosting', + headline: row.title, + description: row.subtitle || '', + datePublished: row.post_date, + dateModified: row.post_date, + articleBody: '', + author: [], + }], + meta: { + title: row.title, + description: row.subtitle || '', + ogTitle: row.title, + ogDescription: row.subtitle || '', + ogImage: null, + canonical: url, + }, + }, + content: { + html: row.body_html || '', + subtitle: row.subtitle || null, + audience: row.audience || 'everyone', + type: row.type || 'newsletter', + wordCount: parseInt(row.word_count) || null, + podcastUrl: row.podcast_url || null, + likes: parseInt(row.reaction_count) || 0, + commentCount: parseInt(row.comment_count) || 0, + }, + accessibility: null, + }; +} + +async function main() { + console.log(`Extracting: ${base}`); + + const log = { processed: [], failed: [], mediaDownloaded: [], skippedPaid: [] }; + const allImageUrls = new Set(); + + // Load posts to process + let posts = []; + + if (urlListFile) { + const inventory = JSON.parse(readFileSync(urlListFile, 'utf8')); + posts = inventory.urls + .filter(u => ['post', 'paid-post', 'podcast', 'thread', 'video'].includes(u.type)) + .map(u => ({ slug: u.slug || u.url.split('/p/')[1], url: u.url, ...u })); + } else { + // Fetch archive to get all slugs + console.log('\nFetching post list...'); + let offset = 0; + while (true) { + const url = `${base}/api/v1/archive?sort=new&limit=50&offset=${offset}`; + try { + const res = await fetch(url); + if (!res.ok) break; + const batch = await res.json(); + if (!Array.isArray(batch) || batch.length === 0) break; + posts.push(...batch); + offset += 50; + if (batch.length < 50) break; + await sleep(300); + } catch { break; } + } + console.log(` Found ${posts.length} posts`); + } + + posts = posts.slice(0, limit); + + // Load CSV export if provided (for paid content) + let csvPosts = {}; + if (csvFile) { + console.log(`\nLoading CSV export from ${csvFile}...`); + const rows = await loadCSVExport(csvFile); + for (const row of rows) { + if (row.slug) csvPosts[row.slug] = row; + } + console.log(` Loaded ${Object.keys(csvPosts).length} posts from CSV`); + } + + // Extract each post + console.log(`\nProcessing ${posts.length} posts...\n`); + + for (let i = 0; i < posts.length; i++) { + const post = posts[i]; + const slug = post.slug; + console.log(`[${i + 1}/${posts.length}] ${slug}`); + + try { + let pageData; + + // Try API first for rich metadata + try { + const apiPost = await fetchPost(slug); + pageData = apiPostToPageData(apiPost); + + // If this is paid content and we have the CSV, use CSV for full HTML + if (apiPost.audience !== 'everyone' && csvPosts[slug]) { + console.log(' Using CSV for full paid content'); + pageData.content.html = csvPosts[slug].body_html || pageData.content.html; + } else if (apiPost.audience !== 'everyone' && !csvPosts[slug]) { + console.log(' Paid post — content may be truncated (use --csv-export for full content)'); + log.skippedPaid.push({ slug, title: apiPost.title }); + } + } catch (apiErr) { + // Fall back to CSV if API fails + if (csvPosts[slug]) { + console.log(` API failed (${apiErr.message}), using CSV`); + pageData = csvPostToPageData(csvPosts[slug]); + } else { + throw apiErr; + } + } + + writeFileSync(`output/pages/${pageData.slug}.json`, JSON.stringify(pageData, null, 2)); + + // Collect image URLs + for (const imgUrl of extractImageUrls(pageData)) { + allImageUrls.add(imgUrl); + } + + log.processed.push({ url: pageData.sourceUrl, slug: pageData.slug }); + console.log(` OK (${pageData.content.wordCount || '?'} words, ${pageData.content.audience})`); + } catch (e) { + console.error(` FAILED: ${e.message}`); + log.failed.push({ slug, error: e.message }); + } + + if (i < posts.length - 1) await sleep(delay); + } + + // Also process any CSV-only posts not in the API results + if (csvFile) { + const apiSlugs = new Set(posts.map(p => p.slug)); + const csvOnly = Object.entries(csvPosts).filter(([slug]) => !apiSlugs.has(slug)); + if (csvOnly.length > 0) { + console.log(`\nProcessing ${csvOnly.length} CSV-only posts...`); + for (const [slug, row] of csvOnly) { + if (row.is_published === 'false') continue; + const pageData = csvPostToPageData(row); + writeFileSync(`output/pages/${pageData.slug}.json`, JSON.stringify(pageData, null, 2)); + for (const imgUrl of extractImageUrls(pageData)) { + allImageUrls.add(imgUrl); + } + log.processed.push({ url: pageData.sourceUrl, slug: pageData.slug }); + } + } + } + + // Download all media + console.log(`\nDownloading ${allImageUrls.size} media files...`); + for (const imgUrl of allImageUrls) { + // Unwrap CDN URL to get original + const originalUrl = unwrapImageUrl(imgUrl); + let urlPath; + try { + urlPath = new URL(originalUrl).pathname; + } catch { + urlPath = originalUrl.split('?')[0]; + } + let filename = basename(urlPath) || `image-${Date.now()}.jpg`; + // Sanitize filename — truncate if too long, remove query artifacts + filename = filename.split('?')[0].split('&')[0]; + if (filename.length > 200) { + const ext = filename.match(/\.(png|jpg|jpeg|gif|webp|svg)$/i)?.[0] || '.jpg'; + filename = filename.slice(0, 190) + ext; + } + const dest = `output/media/${filename}`; + + // Skip if already downloaded + if (existsSync(dest)) { + process.stdout.write('s'); + continue; + } + + try { + await downloadFile(originalUrl, dest); + log.mediaDownloaded.push({ url: imgUrl, originalUrl, file: dest }); + process.stdout.write('.'); + } catch (e) { + // Try the CDN URL if original fails + try { + await downloadFile(imgUrl, dest); + log.mediaDownloaded.push({ url: imgUrl, file: dest }); + process.stdout.write('.'); + } catch (e2) { + log.failed.push({ url: imgUrl, error: `Media download: ${e2.message}` }); + process.stdout.write('x'); + } + } + } + + writeFileSync('output/extraction-log.json', JSON.stringify(log, null, 2)); + console.log(`\n\nDone.`); + console.log(` Posts extracted: ${log.processed.length}`); + console.log(` Media downloaded: ${log.mediaDownloaded.length}`); + console.log(` Failures: ${log.failed.length}`); + if (log.skippedPaid.length) { + console.log(` Paid posts with truncated content: ${log.skippedPaid.length}`); + console.log(' Use --csv-export with your Substack export to get full paid content'); + } + if (log.failed.length) console.log(' See output/extraction-log.json for details'); +} + +main().catch(e => { console.error(e); process.exit(1); });