Extract text and annotations from Shamela book pages

src/defuddle.ts

@@ -938,6 +938,8 @@ export class Defuddle {
 			// Use site-specific extractor first, if there is one
 			const url = options.url || this.doc.URL;
 			const extractorOpts: ExtractorOptions = {
+				// Preserve a caller's selection, independently of retry overrides.
+				contentSelector: this.options.contentSelector,
 				includeReplies: options.includeReplies as ExtractorOptions['includeReplies'],
 				language: options.language,
 				fetch: options.fetch,

src/extractor-registry.ts

@@ -27,6 +27,7 @@ import { LeetCodeExtractor } from './extractors/leetcode';
 import { LwnExtractor } from './extractors/lwn';
 import { MastodonExtractor } from './extractors/mastodon';
 import { GmailExtractor } from './extractors/gmail';
+import { ShamelaExtractor } from './extractors/shamela';
 
 type ExtractorConstructor = new (document: Document, url: string, schemaOrgData?: any, options?: ExtractorOptions) => BaseExtractor;
 
@@ -39,6 +40,11 @@ export class ExtractorRegistry {
 	private static mappings: ExtractorMapping[] = [];
 
 	static initialize() {
+		this.register({
+			patterns: ['shamela.ws'],
+			extractor: ShamelaExtractor
+		});
+
 		// Register all extractors with their URL patterns
 		// X Article extractor must be registered BEFORE Twitter to take priority
 		// DOM-based canExtract() determines if page has article content

src/extractors/_base.ts

@@ -1,6 +1,7 @@
 import { ExtractorResult } from '../types/extractors';
 
 export interface ExtractorOptions {
+	contentSelector?: string;
 	includeReplies?: boolean | 'extractors';
 	language?: string;
 	fetch?: typeof globalThis.fetch;
@@ -51,4 +52,4 @@ export abstract class BaseExtractor {
 	async extractAsync(): Promise<ExtractorResult> {
 		return this.extract();
 	}
-} 
+}

src/extractors/shamela.ts

@@ -0,0 +1,19 @@
+import { BaseExtractor } from './_base';
+import { ExtractorResult } from '../types/extractors';
+
+export class ShamelaExtractor extends BaseExtractor {
+	canExtract(): boolean {
+		if (this.options.contentSelector) return false;
+		const url = new URL(this.url);
+		if (!['shamela.ws', 'www.shamela.ws'].includes(url.hostname) ||
+			!/^\/book\/\d+\/\d+\/?$/.test(url.pathname)) return false;
+		const roots = this.document.querySelectorAll('.nass');
+		return roots.length === 1 && !!roots[0].textContent?.trim();
+	}
+
+	extract(): ExtractorResult {
+		// Book text and annotations share this root; #content belongs to search.
+		// Reuse the pipeline for standardization, URL resolution and sanitization.
+		return { content: '', contentHtml: '', contentSelector: '.nass' };
+	}
+}

tests/expected/issues--386-hidden-dialog-entry.md

@@ -0,0 +1,14 @@
+```json
+{
+  "title": "Example book",
+  "author": "",
+  "site": "shamela.ws",
+  "published": ""
+}
+```
+
+CHAPTER\_SENTINEL This chapter considers how careful observation contributes to a reliable understanding of a subject. Readers compare accounts, examine the supporting evidence, and separate a description from an interpretation. A useful explanation states its assumptions clearly and connects each conclusion with the observation that supports it. These practices help readers identify which parts of an argument are established and which parts require further investigation.
+
+The next section develops the same method through a practical example. Several independent observations may agree while differing in small details. Those differences should be examined before a conclusion is selected. Repeating the observation under another condition helps distinguish a consistent result from an accidental one. Recording the procedure also allows another reader to inspect the reasoning and repeat the work without relying on the author's memory.
+
+A final comparison brings the evidence together. Each source contributes something specific, and no single account should replace all the others merely because it is longer. Readers can ask whether an explanation covers the observations, whether its assumptions remain appropriate, and whether another interpretation accounts for the same facts. The purpose of this chapter is to show how those questions lead to a clearer and more reliable understanding of the material.

tests/expected/issues--386-short-book-page.md

@@ -0,0 +1,10 @@
+```json
+{
+  "title": "Example book",
+  "author": "",
+  "site": "shamela.ws",
+  "published": ""
+}
+```
+
+CHAPTER\_SENTINEL Readers compare observations carefully and record their methods so another reader can inspect the evidence and repeat the work.

tests/extractor-xss.test.ts

@@ -1,6 +1,7 @@
 import { describe, test, expect } from 'vitest';
 import { Defuddle } from '../src/node';
 import { parseLinkedomHTML } from '../src/utils/linkedom-compat';
+import { parseDocument } from './helpers';
 import { ConversationExtractor } from '../src/extractors/_conversation';
 import { C2WikiExtractor } from '../src/extractors/c2-wiki';
 import type { ConversationMessage, ConversationMetadata, Footnote } from '../src/types/extractors';
@@ -46,6 +47,25 @@ function assertNoExecutableAttributes(html: string) {
 }
 
 describe('Extractor output XSS sanitization (GHSA-jg4p-g6xj-4qmf)', () => {
+	test.each([{}, {
+		removeExactSelectors: false, removePartialSelectors: false,
+		removeHiddenElements: false, removeLowScoring: false,
+		removeSmallImages: false, removeContentPatterns: false, standardize: false,
+	}])('sanitizes Shamela book attributes with pipeline options %j', async options => {
+		const url = 'https://shamela.ws/book/0000000/0000000';
+		const html = `<html><body><div class="nass" onclick="alert(1)"><p>Safe chapter text.
+			<a href="javascript:alert(1)">Unsafe link</a><img src="data:text/html,bad" onerror="alert(1)">
+			<iframe srcdoc="bad" src="data:image/png,bad" onload="alert(1)"></iframe></p></div></body></html>`;
+		const result = await Defuddle(parseDocument(html, url), url, { ...options, useAsync: false });
+		expect(result.extractorType).toBe('shamela');
+		expect(result.content).toContain('Safe chapter text.');
+		assertNoExecutableAttributes(result.content);
+		const output = parseDocument(`<html><body>${result.content}</body></html>`, url);
+		expect(output.querySelector('[srcdoc]')).toBeNull();
+		expect(output.querySelector('img')?.getAttribute('src')).toBeNull();
+		expect(output.querySelector('iframe')?.getAttribute('src')).toBeNull();
+	});
+
 	test('does not emit an event handler attribute from X header image alt', async () => {
 		// alt contains a double-quote that, unescaped, would close the attribute
 		// and turn the rest into a real onerror handler.

tests/fixtures/issues--386-hidden-dialog-entry.html

@@ -0,0 +1,18 @@
+<!-- {"url":"https://shamela.ws/book/0000000/0000000"} -->
+<!doctype html>
+<html lang="en">
+<head><title>Example book</title></head>
+<body>
+<div class="nass">
+<p>CHAPTER_SENTINEL This chapter considers how careful observation contributes to a reliable understanding of a subject. Readers compare accounts, examine the supporting evidence, and separate a description from an interpretation. A useful explanation states its assumptions clearly and connects each conclusion with the observation that supports it. These practices help readers identify which parts of an argument are established and which parts require further investigation.</p>
+<p>The next section develops the same method through a practical example. Several independent observations may agree while differing in small details. Those differences should be examined before a conclusion is selected. Repeating the observation under another condition helps distinguish a consistent result from an accidental one. Recording the procedure also allows another reader to inspect the reasoning and repeat the work without relying on the author's memory.</p>
+<p>A final comparison brings the evidence together. Each source contributes something specific, and no single account should replace all the others merely because it is longer. Readers can ask whether an explanation covers the observations, whether its assumptions remain appropriate, and whether another interpretation accounts for the same facts. The purpose of this chapter is to show how those questions lead to a clearer and more reliable understanding of the material.</p>
+</div>
+<div role="dialog" aria-hidden="true">
+<input type="text">
+<div id="content">
+<p>SEARCH_SENTINEL Use this panel to select a collection and search the available volumes. Enter a phrase, choose the fields that should be included, and select a matching mode. Results can be limited to a specific part of the collection. These instructions describe search controls rather than the chapter being read. Close the panel to return to the current page, or adjust the search settings before requesting another set of results.</p>
+</div>
+</div>
+</body>
+</html>

tests/fixtures/issues--386-short-book-page.html

@@ -0,0 +1,16 @@
+<!-- {"url":"https://shamela.ws/book/0000000/0000001"} -->
+<!doctype html>
+<html lang="en">
+<head><title>Example book</title></head>
+<body>
+<div class="nass">
+<p>CHAPTER_SENTINEL Readers compare observations carefully and record their methods so another reader can inspect the evidence and repeat the work.</p>
+</div>
+<div role="dialog" aria-hidden="true">
+<input type="text">
+<div id="content">
+<p>SEARCH_SENTINEL Use this panel to select a collection and search the available volumes. Enter a phrase, choose the fields that should be included, and select a matching mode. Results can be limited to a specific part of the collection. These instructions describe search controls rather than the chapter being read. Close the panel to return to the current page, or adjust the search settings before requesting another set of results.</p>
+</div>
+</div>
+</body>
+</html>

tests/shamela.test.ts

@@ -0,0 +1,81 @@
+import { describe, expect, test } from 'vitest';
+import { Defuddle } from '../src/node';
+import { parseDocument } from './helpers';
+
+const URL = 'https://shamela.ws/book/0000000/0000000';
+const CHAPTER = 'CHAPTER_SENTINEL Readers compare observations carefully and record their methods so another reader can inspect the evidence and repeat the work. ';
+const SEARCH = 'SEARCH_SENTINEL Choose a collection and enter a phrase to search its volumes. Select a matching mode and review the available search settings. ';
+const DIALOG = `<div role="dialog" aria-hidden="true"><input type="text"><div id="content">${SEARCH.repeat(5)}</div></div>`;
+const page = (body: string) => `<html><head><title>Example book</title></head><body>${body}</body></html>`;
+
+async function parse(html: string, url = URL, contentSelector?: string) {
+	return Defuddle(parseDocument(html, url), url, { useAsync: false, separateMarkdown: true, contentSelector });
+}
+
+describe('Shamela book pages', () => {
+	test.each([20, 40, 49, 50, 51, 60, 202])('keeps a %i-word chapter through retries', async words => {
+		const chapter = 'CHAPTER_SENTINEL ' + Array(words - 1).fill('chapter').join(' ');
+		const result = await parse(page(`${DIALOG}<div class="nass"><p>${chapter}</p></div>`));
+		expect(result.wordCount).toBe(words);
+		expect(result.content).toContain('CHAPTER_SENTINEL');
+		expect(result.contentMarkdown).toContain('CHAPTER\\_SENTINEL');
+		expect(result.content).not.toContain('SEARCH_SENTINEL');
+		expect(result.extractorType).toBe('shamela');
+	});
+
+	test.each([
+		'<div role="dialog" aria-hidden="true">',
+		'<div hidden><div role="dialog" aria-hidden="true">',
+		'<div role="dialog" aria-hidden="false">',
+	])('keeps the book root separate from overlay %s', async start => {
+		const end = start.includes('<div hidden>') ? '</div></div>' : '</div>';
+		const result = await parse(page(`${start}<div id="content">${SEARCH.repeat(5)}</div>${end}<div class="nass"><p>${CHAPTER}</p></div>`));
+		expect(result.content).toContain('CHAPTER_SENTINEL');
+		expect(result.content).not.toContain('SEARCH_SENTINEL');
+	});
+
+	test.each(['hidden', 'aria-hidden="true"'])('recovers the book inside nested %s ancestors', async hidden => {
+		const result = await parse(page(`${DIALOG}<div ${hidden}><div ${hidden}><div class="nass"><p>${CHAPTER}</p></div></div></div>`));
+		expect(result.content).toContain('CHAPTER_SENTINEL');
+		expect(result.content).not.toContain('SEARCH_SENTINEL');
+	});
+
+	test('does not confuse duplicate content IDs with the book root', async () => {
+		const result = await parse(page(`${DIALOG}<div class="nass" id="content"><p>${CHAPTER}</p></div>`));
+		expect(result.content).toContain('CHAPTER_SENTINEL');
+		expect(result.content).not.toContain('SEARCH_SENTINEL');
+	});
+
+	test('respects a caller-selected overlay', async () => {
+		const result = await parse(page(`${DIALOG}<div class="nass"><p>${CHAPTER}</p></div>`), URL, '#content');
+		expect(result.content).toContain('SEARCH_SENTINEL');
+		expect(result.content).not.toContain('CHAPTER_SENTINEL');
+		expect(result.extractorType).not.toBe('shamela');
+	});
+
+	test.each([
+		'https://shamela.ws.evil.example/book/1/2',
+		'https://example.org/?url=https://shamela.ws/book/1/2',
+		'https://shamela.ws/search?q=chapter',
+		'https://shamela.ws/book/1',
+	])('leaves other pages to automatic extraction: %s', async url => {
+		const result = await parse(page(`<div class="nass">Unrelated sidebar.</div><article><p>${CHAPTER.repeat(5)}</p></article>`), url);
+		expect(result.content).toContain('CHAPTER_SENTINEL');
+		expect(result.extractorType).not.toBe('shamela');
+	});
+
+	test.each(['', '<div class="nass"></div>', '<div class="nass">First copy.</div><div class="nass">Second copy.</div>'])('falls back when the book root is absent, empty, or ambiguous: %s', async roots => {
+		const result = await parse(page(`${roots}<article><p>${CHAPTER.repeat(5)}</p></article>`));
+		expect(result.content).toContain('CHAPTER_SENTINEL');
+		expect(result.extractorType).not.toBe('shamela');
+	});
+
+	test.each([
+		`<div role="dialog" aria-hidden="true"><p>${CHAPTER.repeat(5)}</p></div>`,
+		`<div aria-hidden="true"><div role="dialog"><p>${CHAPTER.repeat(5)}</p></div></div>`,
+		`<details><summary>Read chapter</summary><p>${CHAPTER.repeat(5)}</p></details>`,
+	])('preserves generic hidden/expandable article recovery %#', async body => {
+		const result = await parse(page(`${body}<div id="fps">FPS: --</div>`), 'https://library.example.org/book/1/2');
+		expect(result.content).toContain('CHAPTER_SENTINEL');
+	});
+});