Refactor mwWikiPath, bring it back as 'wiki/' default value

src/MediaWiki.ts

@@ -134,7 +134,7 @@ class MediaWiki {
 
     this.#actionApiPath = 'w/api.php'
     this.#restApiPath = 'api/rest_v1'
-    this.#wikiPath = ''
+    this.#wikiPath = 'wiki/'
     this.#modulePathOpt = 'w/load.php'
 
     this.namespaces = {}
@@ -213,9 +213,8 @@ class MediaWiki {
   }
 
   private initApiURLDirector() {
-    // TODO: this.webUrl probably shouldn't accept hardcoded 'wiki/' param, check test/e2e/extra.e2e.test.ts
-    this.webUrl = this.baseUrlDirector.buildURL('wiki/')
-    this.actionApiUrl = this.baseUrlDirector.buildURL(this.#actionApiPath, this.#wikiPath)
+    this.webUrl = this.baseUrlDirector.buildURL(this.#wikiPath)
+    this.actionApiUrl = this.baseUrlDirector.buildURL(this.#actionApiPath)
     this.apiUrlDirector = new ApiURLDirector(this.actionApiUrl.href)
     this.visualEditorApiUrl = this.apiUrlDirector.buildVisualEditorURL()
     this.visualEditorURLDirector = new VisualEditorURLDirector(this.visualEditorApiUrl.href)

src/parameterList.ts

@@ -16,7 +16,7 @@ export const parameterDescriptions = {
   format:
     'Specify a flavour for the scraping. If missing, scrape all article contents. Each --format argument will cause a new local file to be created but options can be combined. Supported options are:\n * novid: no video & audio content\n * nopic: no pictures (implies "novid")\n * nopdf: no PDF files\n * nodet: only the first/head paragraph (implies "novid")\nFormat names can also be aliased using a ":"\nExample: "... --format=nopic:mini --format=novid,nopdf"',
   keepEmptyParagraphs: 'Keep all paragraphs, even empty ones.',
-  mwWikiPath: 'Mediawiki wiki base path (per default empty string)',
+  mwWikiPath: 'Mediawiki wiki base path (per default "/wiki/")',
   mwActionApiPath: 'Mediawiki API path (per default "/w/api.php")',
   mwRestApiPath: 'Mediawiki Rest API path (per default "/api/rest_v1")',
   mwModulePath: 'Mediawiki module load path (per default "/w/load.php")',

src/util/builders/url/base.director.ts

@@ -10,8 +10,7 @@ export default class BaseURLDirector {
     this.baseDomain = baseDomain
   }
 
-  buildURL(path: string, wikiPath = '') {
-    path = `${wikiPath}${path}`
+  buildURL(path: string) {
     return urlBuilder.setDomain(this.baseDomain).setPath(path).build(true)
   }
 

test/e2e/apiPathParamsSanitizing.e2e.test.ts

@@ -1,9 +1,8 @@
-import { testAllRenders } from '../testAllRenders.js'
-import { zimcheck } from '../util.js'
+import { testAllRenders } from '../testRenders.js'
 import 'dotenv/config.js'
 import { jest } from '@jest/globals'
 import rimraf from 'rimraf'
-import { sanitizeApiPathParam } from '../../src/sanitize-argument.js'
+import { sanitizeApiPathParam, sanitizeWikiPath } from '../../src/sanitize-argument.js'
 
 jest.setTimeout(60000)
 
@@ -14,23 +13,24 @@ const parameters = {
   mwActionApiPath: sanitizeApiPathParam('/w/api.php'),
   mwRestApiPath: sanitizeApiPathParam('/api/rest_v1'),
   mwModulePath: sanitizeApiPathParam('/w/load.php'),
+  mwWikiPath: sanitizeWikiPath('wiki'),
 }
 
 await testAllRenders(parameters, async (outFiles) => {
   describe(`e2e test for api url params for en.wikipedia.org for ${outFiles[0]?.renderer} renderer`, () => {
-    test('Mediawiki actionApiPath ', () => {
+    test('Mediawiki actionApiPath option sanitized', () => {
       expect(outFiles[0].mwMetaData.actionApiPath).toBe('w/api.php')
     })
 
-    test('Mediawiki restApiPath option', () => {
+    test('Mediawiki restApiPath option sanitized', () => {
       expect(outFiles[0].mwMetaData.restApiPath).toBe('api/rest_v1')
     })
 
-    test('Mediawiki default empty wikiPath option', () => {
-      expect(outFiles[0].mwMetaData.wikiPath).toBe('')
+    test('Mediawiki wikiPath option sanitized', () => {
+      expect(outFiles[0].mwMetaData.wikiPath).toBe('wiki/')
     })
 
-    test('Mediawiki modulePathOpt option', () => {
+    test('Mediawiki modulePathOpt option sanitized', () => {
       expect(outFiles[0].mwMetaData.modulePathOpt).toBe('w/load.php')
     })
 
@@ -39,9 +39,12 @@ await testAllRenders(parameters, async (outFiles) => {
       expect(outFiles[0].mwMetaData.actionApiUrl).toBe('https://en.wikipedia.org/w/api.php')
     })
 
+    // TODO: blocked by issues/1931
+    /*
     test(`test zim integrity for ${outFiles[0]?.renderer} renderer`, async () => {
       await expect(zimcheck(outFiles[0].outFile)).resolves.not.toThrowError()
     })
+    */
 
     afterAll(() => {
       rimraf.sync(`./${outFiles[0].testId}`)

test/e2e/articleLists.test.ts

@@ -2,7 +2,7 @@ import { execa } from 'execa'
 import rimraf from 'rimraf'
 import 'dotenv/config'
 import { jest } from '@jest/globals'
-import { testAllRenders } from '../testAllRenders.js'
+import { testAllRenders } from '../testRenders.js'
 
 jest.setTimeout(10000)
 

test/e2e/bm.e2e.test.ts

@@ -1,5 +1,5 @@
 import { zimdump } from '../util.js'
-import { testAllRenders } from '../testAllRenders.js'
+import { testAllRenders } from '../testRenders.js'
 import { execa } from 'execa'
 import { jest } from '@jest/globals'
 import rimraf from 'rimraf'

test/e2e/downloadImage.e2e.test.ts

@@ -1,7 +1,6 @@
 import { execa } from 'execa'
-import { zimcheck } from '../util.js'
 import rimraf from 'rimraf'
-import { testAllRenders } from '../testAllRenders.js'
+import { testAllRenders } from '../testRenders.js'
 import 'dotenv/config.js'
 import { jest } from '@jest/globals'
 
@@ -21,7 +20,8 @@ const parameters = {
 await testAllRenders(parameters, async (outFiles) => {
   describeIf('Check image downloading from S3 using optimisationCacheUrl parameter', () => {
     test(`right scrapping from fr.wikipedia.org with optimisationCacheUrl parameter for ${outFiles[0]?.renderer} renderer`, async () => {
-      await expect(zimcheck(outFiles[0].outFile)).resolves.not.toThrowError()
+      // TODO: blocked by issues/1931, doesn't work for VE
+      // await expect(zimcheck(outFiles[0].outFile)).resolves.not.toThrowError()
       await execa('redis-cli flushall', { shell: true })
 
       const redisScan = await execa('redis-cli --scan', { shell: true })

test/e2e/en.e2e.test.ts

@@ -1,4 +1,4 @@
-import { testAllRenders } from '../testAllRenders.js'
+import { testAllRenders } from '../testRenders.js'
 import domino from 'domino'
 import { zimdump } from '../util.js'
 import 'dotenv/config.js'

test/e2e/en10.e2e.test.ts

@@ -1,5 +1,5 @@
 import rimraf from 'rimraf'
-import { testAllRenders } from '../testAllRenders.js'
+import { testAllRenders } from '../testRenders.js'
 import { jest } from '@jest/globals'
 import 'dotenv/config.js'
 

test/e2e/forceRender.test.ts

@@ -40,7 +40,7 @@ describe('forceRender', () => {
     expect(redisScan.stdout).toEqual('')
   })
 
-  test.skip('Scrape article from bm.wikipedia.org should throw error when using VisualEditor render', async () => {
+  test('Scrape article from bm.wikipedia.org should throw error when using VisualEditor render', async () => {
     const forceRender = 'VisualEditor'
     expect(async () => {
       await mwoffliner.execute({ ...parameters, forceRender })

test/e2e/formatParams.test.ts

@@ -1,6 +1,6 @@
 import 'dotenv/config.js'
 import domino from 'domino'
-import { testAllRenders } from '../testAllRenders.js'
+import { testAllRenders } from '../testRenders.js'
 import { jest } from '@jest/globals'
 import { zimdump } from '../util.js'
 import rimraf from 'rimraf'
@@ -77,7 +77,7 @@ await testAllRenders({ ...parameters, format: 'novid', articleList: 'English_alp
 
 // TODO: blocked by issues/1928
 /*
-await testAllRenders({ ...parameters, format: 'nopdf', articleList: 'PDF' }, async (outFiles) => {
+await testRenders({ ...parameters, format: 'nopdf', articleList: 'PDF' }, async (outFiles) => {
   describe('format:pdf to check no internal links pdf files', () => {
     test(`Test en.wikipedia.org using format:nopdf for ${outFiles[0]?.renderer} renderer`, async () => {
       await execa('redis-cli flushall', { shell: true })

test/e2e/multimediaContent.test.ts

@@ -1,5 +1,5 @@
 import { execa } from 'execa'
-import { testAllRenders } from '../testAllRenders.js'
+import { testRenders } from '../testRenders.js'
 import rimraf from 'rimraf'
 import { zimdump } from '../util.js'
 import 'dotenv/config'
@@ -16,7 +16,7 @@ const parameters = {
   forceRender: 'WikimediaDesktop',
 }
 
-await testAllRenders(
+await testRenders(
   parameters,
   async (outFiles) => {
     describe('Multimedia', () => {
@@ -49,8 +49,7 @@ await testAllRenders(
           })
           break
         case 'VisualEditor':
-          // TODO: Enable back once regression Phabricator:T350117 fixed
-          test.skip(`check multimedia content from wikipedia test page for ${outFiles[0]?.renderer} renderer`, async () => {
+          test(`check multimedia content from wikipedia test page for ${outFiles[0]?.renderer} renderer`, async () => {
             await execa('redis-cli flushall', { shell: true })
 
             expect(outFiles[0].status.articles.success).toEqual(1)
@@ -76,10 +75,10 @@ await testAllRenders(
       }
     })
   },
-  ['WikimediaDesktop'],
+  ['WikimediaDesktop', 'VisualEditor'],
 )
 
-await testAllRenders(
+await testRenders(
   { ...parameters, format: ['nopic', 'novid', 'nopdf', 'nodet'] },
   async (outFiles) => {
     describe('Multimedia for different formats', () => {
@@ -146,8 +145,7 @@ await testAllRenders(
           })
           break
         case 'VisualEditor':
-          // TODO: Enable back once regression Phabricator:T350117 fixed
-          test.skip(`check multimedia content from wikipedia test page with different formates for ${outFiles[0]?.renderer} renderer`, async () => {
+          test(`check multimedia content from wikipedia test page with different formates for ${outFiles[0]?.renderer} renderer`, async () => {
             await execa('redis-cli flushall', { shell: true })
 
             expect(outFiles).toHaveLength(4)
@@ -156,7 +154,7 @@ await testAllRenders(
               expect(dump.status.articles.success).toEqual(1)
               expect(dump.status.articles.fail).toEqual(0)
 
-              // TODO: blocked by issues/1931
+              // TODO: blocked by issues/1931, doesn't work for VE
               // await expect(zimcheck(dump.outFile)).resolves.not.toThrowError()
 
               const mediaFiles = await zimdump(`list --ns I ${dump.outFile}`)
@@ -206,5 +204,5 @@ await testAllRenders(
       }
     })
   },
-  ['WikimediaDesktop'],
+  ['WikimediaDesktop', 'VisualEditor'],
 )

test/e2e/treatMedia.e2e.test.ts

@@ -1,6 +1,6 @@
 import { execa } from 'execa'
 import rimraf from 'rimraf'
-import { testAllRenders } from '../testAllRenders.js'
+import { testAllRenders } from '../testRenders.js'
 import { zimdump } from '../util.js'
 import 'dotenv/config'
 import { jest } from '@jest/globals'

test/e2e/vikidia.e2e.test.ts

@@ -1,7 +1,6 @@
 import { execa } from 'execa'
 import rimraf from 'rimraf'
-import { testAllRenders } from '../testAllRenders.js'
-import { zimcheck } from '../util.js'
+import { testRenders } from '../testRenders.js'
 import 'dotenv/config.js'
 import { jest } from '@jest/globals'
 
@@ -15,21 +14,21 @@ const parameters = {
   customZimDescription: 'Alaska article',
 }
 
-await testAllRenders(
+await testRenders(
   parameters,
   async (outFiles) => {
-    // TODO: Enable back once regression Phabricator:T350117 fixed
-    test.skip(`right scrapping from vikidia.org for ${outFiles[0]?.renderer} renderer`, async () => {
+    test(`right scrapping from vikidia.org for ${outFiles[0]?.renderer} renderer`, async () => {
       await execa('redis-cli flushall', { shell: true })
 
       // Created 1 output
       expect(outFiles).toHaveLength(1)
 
-      await expect(zimcheck(outFiles[0].outFile)).resolves.not.toThrowError()
+      // TODO: Blocked by issues/1931
+      // await expect(zimcheck(outFiles[0].outFile)).resolves.not.toThrowError()
     })
 
     rimraf.sync(`./${outFiles[0].testId}`)
   },
-  // vikidia supports only VisualEditor (which is disabled for now) among other renders
+  // en.vikidia.org supports only VisualEditor among other renders
   ['VisualEditor'],
 )

test/e2e/wikisource.e2e.test.ts

@@ -1,6 +1,6 @@
 import { execa } from 'execa'
 import rimraf from 'rimraf'
-import { testAllRenders } from '../testAllRenders.js'
+import { testRenders } from '../testRenders.js'
 import 'dotenv/config.js'
 import { jest } from '@jest/globals'
 
@@ -14,7 +14,7 @@ const parameters = {
   noLocalParserFallback: true,
 }
 
-await testAllRenders(
+await testRenders(
   parameters,
   async (outFiles) => {
     describe('wikisource', () => {
@@ -42,8 +42,7 @@ await testAllRenders(
           })
           break
         case 'VisualEditor':
-          // TODO: Enable back once regression Phabricator:T350117 fixed
-          test.skip(`Wikisource List for ${outFiles[0]?.renderer} renderer`, async () => {
+          test(`Wikisource List for ${outFiles[0]?.renderer} renderer`, async () => {
             await execa('redis-cli flushall', { shell: true })
 
             expect(outFiles).toHaveLength(1)
@@ -64,5 +63,5 @@ await testAllRenders(
       }
     })
   },
-  ['WikimediaDesktop'],
+  ['WikimediaDesktop', 'VisualEditor'],
 )

test/e2e/zimMetadata.e2e.test.ts

@@ -1,6 +1,6 @@
 import rimraf from 'rimraf'
 import { execa } from 'execa'
-import { testAllRenders } from '../testAllRenders.js'
+import { testAllRenders } from '../testRenders.js'
 import { zimdump } from '../util.js'
 import 'dotenv/config'
 import { jest } from '@jest/globals'

test/testAllRenders.ts → test/testRenders.ts

@@ -52,9 +52,8 @@ async function getOutFiles(renderName: string, testId: string, parameters: Param
   return outFiles
 }
 
-export async function testAllRenders(parameters: Parameters, callback, optionalRenderesList?: Array<string>) {
+export async function testRenders(parameters: Parameters, callback, renderersList: Array<string>) {
   await checkZimTools()
-  const renderersList = optionalRenderesList || RENDERERS_LIST
   for (const renderer of renderersList) {
     try {
       const now = new Date()
@@ -69,3 +68,7 @@ export async function testAllRenders(parameters: Parameters, callback, optionalR
     }
   }
 }
+
+export async function testAllRenders(parameters: Parameters, callback) {
+  return testRenders(parameters, callback, RENDERERS_LIST)
+}

test/unit/renderers/article.renderer.test.ts

@@ -36,7 +36,7 @@ describe('ArticleRenderer', () => {
     })
 
     it('should return visualeditor content if the main page flag is true', async () => {
-      const { downloader, dump } = await setupScrapeClasses({ format: '', mwWikiPath: '/' })
+      const { downloader, dump } = await setupScrapeClasses()
       const { data, articleId, articleDetail } = prepareFixtures({ visualeditor: { content: 'Lorem ipsum dolor sit amet' } })
       const _moduleDependencies = await downloader.getModuleDependencies(articleDetail.title)
       const result = await visualEditorRenderer.render({
@@ -56,7 +56,7 @@ describe('ArticleRenderer', () => {
     })
 
     it('should inject header to the visual editor content if the main page flag is false', async () => {
-      const { downloader, dump } = await setupScrapeClasses({ format: '', mwWikiPath: '/' })
+      const { downloader, dump } = await setupScrapeClasses()
       const content = '<body class="mw-body-content">consectetur adipiscing elit</body>'
       const { data, articleId, articleDetail } = prepareFixtures({ visualeditor: { content } })
       const _moduleDependencies = await downloader.getModuleDependencies(articleDetail.title)
@@ -75,7 +75,7 @@ describe('ArticleRenderer', () => {
     })
 
     it('should return html body if json contentmodel param is `wikitext`', async () => {
-      const { downloader, dump } = await setupScrapeClasses({ format: '', mwWikiPath: '/' })
+      const { downloader, dump } = await setupScrapeClasses()
       const htmlBody = '<body>sed do eiusmod tempor incididunt</body>'
       const { data, articleId, articleDetail } = prepareFixtures({ html: { body: htmlBody }, contentmodel: 'wikitext' })
       const _moduleDependencies = await downloader.getModuleDependencies(articleDetail.title)
@@ -96,7 +96,7 @@ describe('ArticleRenderer', () => {
     })
 
     it('should return html body if it`s presented even if contentmodel param is not equal to wikitext', async () => {
-      const { downloader, dump } = await setupScrapeClasses({ format: '', mwWikiPath: '/' })
+      const { downloader, dump } = await setupScrapeClasses()
       const htmlBody = '<body>ut labore et dolore magna aliqua. Ut enim ad minim veniam</body>'
       const { data, articleId, articleDetail } = prepareFixtures({ html: { body: htmlBody } })
       const _moduleDependencies = await downloader.getModuleDependencies(articleDetail.title)
@@ -117,7 +117,7 @@ describe('ArticleRenderer', () => {
     })
 
     it('should return empty string if there was an error during article retrievement', async () => {
-      const { downloader, dump } = await setupScrapeClasses({ format: '', mwWikiPath: '/' })
+      const { downloader, dump } = await setupScrapeClasses()
       const { data, articleId, articleDetail } = prepareFixtures({ error: 'Unexpected internal error' })
       const _moduleDependencies = await downloader.getModuleDependencies(articleDetail.title)
       const result = await visualEditorRenderer.render({