Merge pull request #1919 from openzim/namespace-blacklist

Don't mirror 'Story' namespace
openzim · Oct 4, 2023 · fc79938 · fc79938
2 parents 190cd26 + 4b29977
commit fc79938
Show file tree

Hide file tree

Showing 3 changed files with 45 additions and 12 deletions.
diff --git a/src/MediaWiki.ts b/src/MediaWiki.ts
@@ -12,6 +12,7 @@ import ApiURLDirector from './util/builders/url/api.director.js'
 import DesktopURLDirector from './util/builders/url/desktop.director.js'
 import VisualEditorURLDirector from './util/builders/url/visual-editor.director.js'
 import { checkApiAvailability } from './util/mw-api.js'
+import { BLACKLISTED_NS } from './util/const.js'
 
 export interface QueryOpts {
   action: string
@@ -34,7 +35,6 @@ class MediaWiki {
   }
 
   public metaData: MWMetaData
-  public _base: string
   public baseUrl: URL
   public getCategories: boolean
   public namespaces: MWNamespaces = {}
@@ -227,18 +227,22 @@ class MediaWiki {
         const num = entry.id
         const allowedSubpages = 'subpages' in entry
         const isContent = type === 'namespaces' ? !!(entry.content || util.contains(addNamespaces, num)) : !!(entry.content !== undefined || util.contains(addNamespaces, num))
+        const isBlacklisted = BLACKLISTED_NS.includes(name)
         const canonical = entry.canonical ? entry.canonical : ''
         const details = { num, allowedSubpages, isContent }
+
         /* Namespaces in local language */
         this.namespaces[util.lcFirst(name)] = details
         this.namespaces[util.ucFirst(name)] = details
+
         /* Namespaces in English (if available) */
         if (canonical) {
           this.namespaces[util.lcFirst(canonical)] = details
           this.namespaces[util.ucFirst(canonical)] = details
         }
+
         /* Is content to mirror */
-        if (isContent) {
+        if (isContent && !isBlacklisted) {
           this.namespacesToMirror.push(name)
         }
       })

diff --git a/src/util/const.ts b/src/util/const.ts
@@ -19,3 +19,4 @@ export const LOAD_PHP = /script.src = ".*load\.php.*";/
 export const RULE_TO_REDIRECT = /window\.top !== window\.self/
 export const WEBP_HANDLER_URL = 'https://gist.githubusercontent.com/rgaudin/60bb9cc6f187add506584258028b8ee1/raw/9d575b8e25d67eed2a9c9a91d3e053a0062d2fc7/web-handler.js'
 export const MAX_FILE_DOWNLOAD_RETRIES = 5
+export const BLACKLISTED_NS = ['Story'] // 'Story' Wikipedia namespace is content, but not indgestable by Parsoid https://github.com/openzim/mwoffliner/issues/1853
diff --git a/test/unit/mwApi.test.ts b/test/unit/mwApi.test.ts
@@ -9,26 +9,32 @@ import { jest } from '@jest/globals'
 
 jest.setTimeout(10000)
 
-describe('mwApi', () => {
-  beforeAll(startRedis)
-  afterAll(stopRedis)
+beforeAll(async () => {
+  MediaWiki.reset()
+  await startRedis()
+})
+afterAll(stopRedis)
+
+const initMW = async (downloader: Downloader) => {
+  await MediaWiki.getMwMetaData(downloader)
+  await MediaWiki.hasCoordinates(downloader)
+  await MediaWiki.hasWikimediaDesktopRestApi()
+  await MediaWiki.hasVisualEditorApi()
 
+  await MediaWiki.getNamespaces([], downloader)
+}
+
+describe('mwApi', () => {
   let downloader: Downloader
 
   beforeEach(async () => {
     await RedisStore.articleDetailXId.flush()
 
     MediaWiki.base = 'https://en.wikipedia.org'
     MediaWiki.getCategories = true
-
     downloader = new Downloader({ uaString: `${config.userAgent} (contact@kiwix.org)`, speed: 1, reqTimeout: 1000 * 60, webp: false, optimisationCacheUrl: '' })
 
-    await MediaWiki.getMwMetaData(downloader)
-    await MediaWiki.hasCoordinates(downloader)
-    await MediaWiki.hasWikimediaDesktopRestApi()
-    await MediaWiki.hasVisualEditorApi()
-
-    await MediaWiki.getNamespaces([], downloader)
+    await initMW(downloader)
   })
 
   test('MWApi Article Ids', async () => {
@@ -116,3 +122,25 @@ describe('mwApi', () => {
     expect(interWikiTitle).toBeNull()
   })
 })
+
+describe('Test blacklisted NSs', () => {
+  let downloader: Downloader
+
+  beforeEach(async () => {
+    await RedisStore.articleDetailXId.flush()
+
+    MediaWiki.base = 'https://id.wikipedia.org'
+    MediaWiki.getCategories = true
+
+    downloader = new Downloader({ uaString: `${config.userAgent} (contact@kiwix.org)`, speed: 1, reqTimeout: 1000 * 60, webp: false, optimisationCacheUrl: '' })
+
+    await initMW(downloader)
+  })
+
+  test('Prevent blacklisted namespaces to mirroring', async () => {
+    const aIds = ['Story:Satelit_Oberon', 'London']
+    await getArticleIds(downloader, 'Main_Page', aIds)
+
+    expect(MediaWiki.namespacesToMirror).not.toContain('Story')
+  })
+})