Ethan Catzel Kindle Highlights Scraper
import { promises as fs } from "fs";
import path from "path";
import { chromium } from "playwright-extra";
import stealthPlugin from "puppeteer-extra-plugin-stealth";
import { DateTime } from "luxon";
// Add stealth plugin to avoid detection
chromium.use(stealthPlugin());
// Helper function to add random delays
const delay = (min: number, max: number) =>
new Promise(resolve => setTimeout(resolve, Math.random() * (max - min) + min));
async function main(numberOfBooks: number) {
console.log("Starting Kindle highlights scraper...");
const browser = await chromium.launch({
headless: true, // Run in headless mode for speed
args: [
'--disable-blink-features=AutomationControlled',
'--disable-dev-shm-usage',
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-accelerated-2d-canvas',
'--disable-gpu'
]
});
const context = await browser.newContext({
viewport: { width: 1920, height: 1080 },
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
locale: 'en-US',
timezoneId: 'America/New_York'
});
const page = await context.newPage();
try {
// Go to login page with random delay
console.log("Navigating to Amazon Kindle...");
await page.goto("https://read.amazon.com/landing", {
waitUntil: "domcontentloaded"
});
await delay(500, 1000);
// Click sign in button
console.log("Clicking sign in button...");
await page.click("#top-sign-in-btn");
await delay(300, 600);
// Type in email with human-like typing speed
console.log("Entering email...");
await page.fill('input[name="email"]', process.env.AMAZON_EMAIL!, {
force: false
});
await delay(100, 300);
// Check if we need to click continue button first
const continueButton = await page.locator("#continue").count();
if (continueButton > 0) {
console.log("Clicking continue button...");
await page.click("#continue");
await delay(200, 400);
}
// Type in password
console.log("Entering password...");
await page.fill('input[name="password"]', process.env.AMAZON_PASSWORD!, {
force: false
});
await delay(100, 300);
// Login - click the sign in button
console.log("Clicking sign in button...");
await page.click('input[type="submit"]');
// Wait for navigation with timeout and check for CAPTCHA
console.log("Waiting for login to complete...");
try {
await page.waitForURL(/^https:\/\/read\.amazon\.com\/(notebook|kp\/notebook|kindle-library)/, {
timeout: 30000
});
} catch (error) {
// Check if we're on a CAPTCHA/verification page
const currentUrl = page.url();
if (currentUrl.includes('/ap/cvf/') || currentUrl.includes('captcha') || currentUrl.includes('verify')) {
console.error("\n⚠️ Amazon is requesting additional verification (CAPTCHA/2FA).");
console.error("This typically happens when:");
console.error(" - Logging in from a new location/device");
console.error(" - Too many automated attempts");
console.error("\nPossible solutions:");
console.error(" 1. Log in manually through a regular browser first");
console.error(" 2. Complete the verification in the opened browser window");
console.error(" 3. Try again later");
console.error("\nCurrent URL:", currentUrl);
// Keep browser open for manual intervention
console.log("\n💡 Browser will stay open for 2 minutes for manual intervention...");
await delay(120000, 120000);
throw new Error("Verification required - please complete manually");
}
throw error;
}
// Add delay before navigating to notebook
await delay(500, 1000);
// Go to the notes and highlights page
console.log("Navigating to highlights page...");
await page.goto("https://read.amazon.com/notebook", {
waitUntil: "networkidle"
});
// Wait for books to load
console.log("Waiting for books to load...");
await page.waitForSelector(".kp-notebook-library-each-book", { timeout: 15000 });
await delay(200, 400);
let books = await page.evaluate(() => {
let books = [];
for (const bookElement of document.querySelectorAll(
".kp-notebook-library-each-book",
)) {
const bookId = bookElement.id;
const titleAuthorPair = bookElement.querySelectorAll<HTMLElement>(
".kp-notebook-searchable",
);
const title = titleAuthorPair[0].textContent!.split(":")[0].trim();
const author = titleAuthorPair[1].textContent!.replace("By: ", "").trim();
const lastAccessedElement = bookElement.querySelector<HTMLInputElement>(
`#kp-notebook-annotated-date-${bookId}`,
);
const lastAccessed = lastAccessedElement ? lastAccessedElement.value : "";
books.push({
asin: bookId,
title,
author,
lastAccessed,
url: `https://amazon.com/dp/${bookId}`,
});
}
return books;
});
console.log(`Found ${books.length} books with highlights`);
// Go to each book's highlights page
for (const [index, book] of books.slice(0, numberOfBooks).entries()) {
console.log(`\nProcessing book ${index + 1}/${Math.min(numberOfBooks, books.length)}: ${book.title}`);
await page.goto(
`https://read.amazon.com/notebook?asin=${book.asin}&contentLimitState=&token=`,
{ waitUntil: "networkidle" }
);
await delay(300, 600);
let highlights: string[] = [];
let pageNumber = 1;
while (true) {
// Wait for highlights to load
await page.waitForSelector("#highlight", { timeout: 5000 }).catch(() => {});
const pageHighlights = await page.evaluate(() => {
return Array.from(document.querySelectorAll("#highlight")).map(
(el: any) => {
let text = el.textContent.trim();
return text.charAt(0).toUpperCase() + text.slice(1);
},
);
});
console.log(` - Found ${pageHighlights.length} highlights on page ${pageNumber}`);
highlights = highlights.concat(pageHighlights);
const hasNextPage = await page.evaluate(() => {
const nextButton = document.querySelector<HTMLButtonElement>(
"#kp-notebook-annotations-next",
);
return nextButton && !nextButton.disabled;
});
if (!hasNextPage) {
break;
}
// Add delay before clicking next
await delay(200, 400);
await page.click("#kp-notebook-annotations-next");
await page.waitForLoadState("networkidle");
pageNumber++;
}
console.log(` - Total highlights collected: ${highlights.length}`);
// Create a `.md` file with the contents
const fileName = `./src/content/kindle-highlights/${book.title
.toLowerCase()
.replace(/[^\w\s]/g, "")
.replace(/\s+/g, "-")}.md`;
// Parse the date - if it fails for any reason, use epoch date (1970-01-01)
let formattedDate = "1970-01-01";
if (book.lastAccessed) {
try {
const parsed = DateTime.fromFormat(book.lastAccessed, "EEEE, MMMM d, yyyy");
if (parsed.isValid) {
formattedDate = parsed.toFormat("yyyy-MM-dd");
}
} catch {
// Date parsing failed, keep as epoch date
}
}
const contents = `---
title: ${book.title}
author: ${book.author}
lastAccessed: ${formattedDate}
url: ${book.url}
---
${highlights.join("\n\n")}
`;
await fs.mkdir(path.dirname(fileName), { recursive: true });
await fs.writeFile(fileName, contents);
console.log(` - Saved to: ${fileName}`);
// Add delay between books
if (index < books.slice(0, numberOfBooks).length - 1) {
await delay(500, 1000);
}
}
console.log("\n✅ Scraping completed successfully!");
} catch (error) {
console.error("\n❌ Error during scraping:", error);
throw error;
} finally {
await browser.close();
}
}
const args = process.argv.slice(2);
if (args.length !== 1) {
console.error("Please provide exactly one integer argument greater than 1.");
process.exit(1);
}
const input = parseInt(args[0], 10);
if (isNaN(input) || input <= 0 || !Number.isInteger(input)) {
console.error("The argument must be an integer greater than 1.");
process.exit(1);
}
main(input).catch((error) => {
console.error("Fatal error:", error);
process.exit(1);
});