diff --git a/src/Downloader.ts b/src/Downloader.ts index afb05357..d470d66b 100644 --- a/src/Downloader.ts +++ b/src/Downloader.ts @@ -52,6 +52,7 @@ interface DownloaderOpts { s3?: S3 webp: boolean backoffOptions?: BackoffOptions + mwWikiPath?: string } interface BackoffOptions { @@ -107,7 +108,7 @@ class Downloader { this.optimisationCacheUrl = optimisationCacheUrl this.webp = webp this.s3 = s3 - this.apiUrlDirector = new ApiURLDirector(MediaWiki.apiUrl.href) + this.apiUrlDirector = new ApiURLDirector(MediaWiki.actionApiUrl.href) this.backoffOptions = { strategy: new backoff.ExponentialStrategy(), @@ -620,7 +621,7 @@ class Downloader { } private async getSubCategories(articleId: string, continueStr = ''): Promise> { - const apiUrlDirector = new ApiURLDirector(MediaWiki.apiUrl.href) + const apiUrlDirector = new ApiURLDirector(MediaWiki.actionApiUrl.href) const { query, continue: cont } = await this.getJSON(apiUrlDirector.buildSubCategoriesURL(articleId, continueStr)) const items = query.categorymembers.filter((a: any) => a && a.title) @@ -651,7 +652,7 @@ class Downloader { let jsDependenciesList: string[] = [] let styleDependenciesList: string[] = [] - const apiUrlDirector = new ApiURLDirector(MediaWiki.apiUrl.href) + const apiUrlDirector = new ApiURLDirector(MediaWiki.actionApiUrl.href) const articleApiUrl = apiUrlDirector.buildArticleApiURL(title) diff --git a/src/MediaWiki.ts b/src/MediaWiki.ts index 60faa856..36684b5c 100644 --- a/src/MediaWiki.ts +++ b/src/MediaWiki.ts @@ -44,20 +44,21 @@ class MediaWiki { public queryOpts: QueryOpts #wikiPath: string - #apiPath: string + #actionApiPath: string + #restApiPath: string + #modulePathOpt: string #username: string #password: string - #apiActionPath: string #domain: string private apiUrlDirector: ApiURLDirector + private baseUrlDirector: BaseURLDirector private wikimediaDesktopUrlDirector: WikimediaDesktopURLDirector private wikimediaMobileUrlDirector: WikimediaMobileURLDirector - private VisualEditorURLDirector: VisualEditorURLDirector + private visualEditorURLDirector: VisualEditorURLDirector public visualEditorApiUrl: URL - public apiUrl: URL + public actionApiUrl: URL public modulePath: string // only for reading - public _modulePathOpt: string // only for whiting to generate modulePath public mobileModulePath: string public webUrl: URL public WikimediaDesktopApiUrl: URL @@ -76,12 +77,18 @@ class MediaWiki { this.#password = value } - set apiActionPath(value: string) { - this.#apiActionPath = value + set actionApiPath(value: string) { + if (value) { + this.#actionApiPath = value + this.initApiURLDirector() + } } - set apiPath(value: string) { - this.#apiPath = value + set restApiPath(value: string) { + if (value) { + this.#restApiPath = value + this.initApiURLDirector() + } } set domain(value: string) { @@ -89,16 +96,34 @@ class MediaWiki { } set wikiPath(value: string) { - this.#wikiPath = value + if (value) { + this.#wikiPath = value + this.initApiURLDirector() + } } set base(value: string) { - this.baseUrl = basicURLDirector.buildMediawikiBaseURL(value) - this.initMWApis() + if (value) { + this.baseUrl = basicURLDirector.buildMediawikiBaseURL(value) + this.baseUrlDirector = new BaseURLDirector(this.baseUrl.href) + this.initMWApis() + this.initApiURLDirector() + } } set modulePathOpt(value: string) { - this._modulePathOpt = value + if (value) { + this.#modulePathOpt = value + if (this.baseUrlDirector) { + this.modulePath = this.baseUrlDirector.buildModuleURL(this.#modulePathOpt) + } else { + logger.error('Base url director should be specified first') + } + } else { + if (this.baseUrlDirector) { + this.modulePath = this.baseUrlDirector.buildModuleURL(this.#modulePathOpt) + } + } } private initializeMediaWikiDefaults(): void { @@ -107,11 +132,13 @@ class MediaWiki { this.#password = '' this.getCategories = false + this.#actionApiPath = 'w/api.php' + this.#restApiPath = 'api/rest_v1' + this.#wikiPath = 'wiki/' + this.#modulePathOpt = 'w/load.php' + this.namespaces = {} this.namespacesToMirror = [] - - this.#apiActionPath = 'w/api.php' - this.#wikiPath = 'wiki/' this.apiCheckArticleId = 'MediaWiki:Sidebar' this.queryOpts = { @@ -152,7 +179,7 @@ class MediaWiki { public async hasVisualEditorApi(): Promise { if (this.#hasVisualEditorApi === null) { - this.#hasVisualEditorApi = await checkApiAvailability(this.VisualEditorURLDirector.buildArticleURL(this.apiCheckArticleId)) + this.#hasVisualEditorApi = await checkApiAvailability(this.visualEditorURLDirector.buildArticleURL(this.apiCheckArticleId)) return this.#hasVisualEditorApi } return this.#hasVisualEditorApi @@ -178,23 +205,24 @@ class MediaWiki { } private initMWApis() { - const baseUrlDirector = new BaseURLDirector(this.baseUrl.href) - this.webUrl = baseUrlDirector.buildURL(this.#wikiPath) - this.apiUrl = baseUrlDirector.buildURL(this.#apiActionPath) - this.apiUrlDirector = new ApiURLDirector(this.apiUrl.href) - this.visualEditorApiUrl = this.apiUrlDirector.buildVisualEditorURL() - this.WikimediaDesktopApiUrl = baseUrlDirector.buildWikimediaDesktopApiUrl(this.#apiPath) - this.WikimediaMobileApiUrl = baseUrlDirector.buildWikimediaMobileApiUrl(this.#apiPath) - this.modulePath = baseUrlDirector.buildModuleURL(this._modulePathOpt) - this.mobileModulePath = baseUrlDirector.buildMobileModuleURL() + this.WikimediaDesktopApiUrl = this.baseUrlDirector.buildWikimediaDesktopApiUrl(this.#restApiPath) + this.WikimediaMobileApiUrl = this.baseUrlDirector.buildWikimediaMobileApiUrl(this.#restApiPath) + this.mobileModulePath = this.baseUrlDirector.buildMobileModuleURL() this.wikimediaDesktopUrlDirector = new WikimediaDesktopURLDirector(this.WikimediaDesktopApiUrl.href) this.wikimediaMobileUrlDirector = new WikimediaMobileURLDirector(this.WikimediaMobileApiUrl.href) - this.VisualEditorURLDirector = new VisualEditorURLDirector(this.visualEditorApiUrl.href) + } + + private initApiURLDirector() { + this.webUrl = this.baseUrlDirector.buildURL(this.#wikiPath) + this.actionApiUrl = this.baseUrlDirector.buildURL(this.#actionApiPath) + this.apiUrlDirector = new ApiURLDirector(this.actionApiUrl.href) + this.visualEditorApiUrl = this.apiUrlDirector.buildVisualEditorURL() + this.visualEditorURLDirector = new VisualEditorURLDirector(this.visualEditorApiUrl.href) } public async login(downloader: Downloader) { if (this.#username && this.#password) { - let url = this.apiUrl.href + '?' + let url = this.actionApiUrl.href + '?' // Add domain if configured if (this.#domain) { @@ -205,7 +233,7 @@ class MediaWiki { const { content, responseHeaders } = await downloader.downloadContent(url + 'action=query&meta=tokens&type=login&format=json&formatversion=2') // Logging in - await axios(this.apiUrl.href, { + await axios(this.actionApiUrl.href, { data: qs.stringify({ action: 'login', format: 'json', @@ -413,13 +441,15 @@ class MediaWiki { const mwMetaData: MWMetaData = { webUrl: this.webUrl.href, - apiUrl: this.apiUrl.href, + actionApiUrl: this.actionApiUrl.href, + modulePathOpt: this.#modulePathOpt, modulePath: this.modulePath, mobileModulePath: this.mobileModulePath, webUrlPath: this.webUrl.pathname, wikiPath: this.#wikiPath, baseUrl: this.baseUrl.href, - apiActionPath: this.#apiActionPath, + actionApiPath: this.#actionApiPath, + restApiPath: this.#restApiPath, domain: this.#domain, textDir: textDir as TextDirection, diff --git a/src/mwoffliner.lib.ts b/src/mwoffliner.lib.ts index 53f21463..b0f8a9de 100644 --- a/src/mwoffliner.lib.ts +++ b/src/mwoffliner.lib.ts @@ -76,7 +76,7 @@ async function execute(argv: any) { mwUrl, mwWikiPath, mwActionApiPath, - mwApiPath, + mwRestApiPath, mwModulePath, mwDomain, mwUsername, @@ -158,13 +158,13 @@ async function execute(argv: any) { /* Wikipedia/... URL; Normalize by adding trailing / as necessary */ MediaWiki.base = mwUrl MediaWiki.getCategories = !!argv.getCategories - MediaWiki.apiActionPath = mwActionApiPath - MediaWiki.apiPath = mwApiPath + MediaWiki.wikiPath = mwWikiPath + MediaWiki.actionApiPath = mwActionApiPath + MediaWiki.restApiPath = mwRestApiPath MediaWiki.modulePathOpt = mwModulePath MediaWiki.domain = mwDomain MediaWiki.password = mwPassword MediaWiki.username = mwUsername - MediaWiki.wikiPath = mwWikiPath /* Download helpers; TODO: Merge with something else / expand this. */ const downloader = new Downloader({ @@ -513,7 +513,7 @@ async function execute(argv: any) { } } - const apiUrlDirector = new ApiURLDirector(MediaWiki.apiUrl.href) + const apiUrlDirector = new ApiURLDirector(MediaWiki.actionApiUrl.href) const body = await downloader.getJSON(apiUrlDirector.buildSiteInfoURL()) diff --git a/src/parameterList.ts b/src/parameterList.ts index eee869e4..3d9e54ac 100644 --- a/src/parameterList.ts +++ b/src/parameterList.ts @@ -17,8 +17,8 @@ export const parameterDescriptions = { 'Specify a flavour for the scraping. If missing, scrape all article contents. Each --format argument will cause a new local file to be created but options can be combined. Supported options are:\n * novid: no video & audio content\n * nopic: no pictures (implies "novid")\n * nopdf: no PDF files\n * nodet: only the first/head paragraph (implies "novid")\nFormat names can also be aliased using a ":"\nExample: "... --format=nopic:mini --format=novid,nopdf"', keepEmptyParagraphs: 'Keep all paragraphs, even empty ones.', mwWikiPath: 'Mediawiki wiki base path (per default "/wiki/")', - mwActionApiPath: 'Mediawiki action API path (per default "/w/api.php")', - mwApiPath: 'Mediawiki Rest API path (per default "/api/rest_v1")', + mwActionApiPath: 'Mediawiki API path (per default "/w/api.php")', + mwRestApiPath: 'Mediawiki Rest API path (per default "/api/rest_v1")', mwModulePath: 'Mediawiki module load path (per default "/w/load.php")', mwDomain: 'Mediawiki user domain (thought for private wikis)', mwUsername: 'Mediawiki username (thought for private wikis)', diff --git a/src/renderers/renderer.builder.ts b/src/renderers/renderer.builder.ts index cffb1cab..27f7d6e4 100644 --- a/src/renderers/renderer.builder.ts +++ b/src/renderers/renderer.builder.ts @@ -47,30 +47,35 @@ export class RendererBuilder { } case 'specific': // renderName argument is required for 'specific' mode - switch (renderName) { - case 'WikimediaDesktop': - if (hasWikimediaDesktopApi) { - return new WikimediaDesktopRenderer() - } - logger.error('Cannot create an instance of WikimediaDesktop renderer.') - process.exit(1) - case 'VisualEditor': - if (hasVisualEditorApi) { - return new VisualEditorRenderer() - } - logger.error('Cannot create an instance of VisualEditor renderer.') - process.exit(1) - case 'WikimediaMobile': - if (hasWikimediaMobileApi) { - return new WikimediaMobileRenderer() - } - logger.error('No available mobile renderer.') - process.exit(1) - default: - throw new Error(`Unknown renderName for specific mode: ${renderName}`) - } + return this.handleSpecificRender(renderName, hasVisualEditorApi, hasWikimediaDesktopApi, hasWikimediaMobileApi) default: throw new Error(`Unknown render: ${renderType}`) } } + + private handleSpecificRender(renderName: string, hasVisualEditorApi: boolean, hasWikimediaDesktopApi: boolean, hasWikimediaMobileApi: boolean) { + // renderName argument is required for 'specific' mode + switch (renderName) { + case 'WikimediaDesktop': + if (hasWikimediaDesktopApi) { + return new WikimediaDesktopRenderer() + } + logger.error('Cannot create an instance of WikimediaDesktop renderer.') + process.exit(1) + case 'VisualEditor': + if (hasVisualEditorApi) { + return new VisualEditorRenderer() + } + logger.error('Cannot create an instance of VisualEditor renderer.') + process.exit(1) + case 'WikimediaMobile': + if (hasWikimediaMobileApi) { + return new WikimediaMobileRenderer() + } + logger.error('No available mobile renderer.') + process.exit(1) + default: + throw new Error(`Unknown renderName for specific mode: ${renderName}`) + } + } } diff --git a/src/sanitize-argument.ts b/src/sanitize-argument.ts index 41d3cef4..1fcb793e 100644 --- a/src/sanitize-argument.ts +++ b/src/sanitize-argument.ts @@ -31,6 +31,10 @@ export async function sanitize_all(argv: any) { customZimLongDescription, customZimDescription, forceRender, + mwWikiPath, + mwActionApiPath, + mwRestApiPath, + mwModulePath, } = argv sanitizeDoubleUsedParameters(argv) @@ -78,6 +82,18 @@ export async function sanitize_all(argv: any) { throw err }) + // sanitizing mwWikiPath + sanitizeWikiPath(mwWikiPath) + + // sanitizing mwRestApiPath + sanitizeApiPathParam(mwRestApiPath) + + // sanitizing mwActionApiPath + sanitizeApiPathParam(mwActionApiPath) + + // sanitizing mwModulePath + sanitizeApiPathParam(mwModulePath) + // sanitize Custom Main Page if (argv.customMainPage) { argv.customMainPage = argv.customMainPage.replace(/ /g, '_') @@ -103,6 +119,30 @@ export async function sanitize_all(argv: any) { } } +export function sanitizeWikiPath(mwWikiPath = '') { + mwWikiPath = sanitizeApiPathParam(mwWikiPath) + + // Make sure wikiPath always has forward slash at the end for the correct URL building + if (!mwWikiPath?.endsWith('/')) { + mwWikiPath += '/' + } + + return mwWikiPath +} + +export function sanitizeApiPathParam(apiPathParam: string) { + if (!apiPathParam) { + return + } + + // No api params should start from forward slash + if (apiPathParam.startsWith('/')) { + apiPathParam = apiPathParam.slice(1) + } + + return apiPathParam +} + export function sanitizeStringMaxLength(text: string, key: string, length: number) { if (text && text.length > length) { throw new Error(`${key} should be less than ${length} characters.`) diff --git a/src/types.d.ts b/src/types.d.ts index 1170a392..54aeb661 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -159,12 +159,14 @@ interface MWMetaData { baseUrl: string wikiPath: string - apiActionPath: string + actionApiPath: string + restApiPath: string domain: string webUrl: string - apiUrl: string + actionApiUrl: string webUrlPath: string modulePath: string + modulePathOpt: string mobileModulePath: string } @@ -179,8 +181,7 @@ interface MWNamespaces { interface MWConfig { base: string wikiPath?: string - apiActionPath?: string - apiPath?: string + actionApiPath?: string domain?: string username?: string password?: string diff --git a/src/util/builders/url/base.director.ts b/src/util/builders/url/base.director.ts index 6006441b..c0791cc2 100644 --- a/src/util/builders/url/base.director.ts +++ b/src/util/builders/url/base.director.ts @@ -24,14 +24,14 @@ export default class BaseURLDirector { buildWikimediaDesktopApiUrl(path?: string) { return urlBuilder .setDomain(this.baseDomain) - .setPath(path ?? 'api/rest_v1/page/html') + .setPath(path ? `${path}/page/html` : 'api/rest_v1/page/html') .build(true, '/') } buildWikimediaMobileApiUrl(path?: string) { return urlBuilder .setDomain(this.baseDomain) - .setPath(path ?? 'api/rest_v1/page/mobile-html') + .setPath(path ? `${path}/page/mobile-html` : 'api/rest_v1/page/mobile-html') .build(true, '/') } diff --git a/test/e2e/apiPathParamsSanitizing.e2e.test.ts b/test/e2e/apiPathParamsSanitizing.e2e.test.ts new file mode 100644 index 00000000..46bd1b11 --- /dev/null +++ b/test/e2e/apiPathParamsSanitizing.e2e.test.ts @@ -0,0 +1,53 @@ +import { testAllRenders } from '../testRenders.js' +import 'dotenv/config.js' +import { jest } from '@jest/globals' +import rimraf from 'rimraf' +import { sanitizeApiPathParam, sanitizeWikiPath } from '../../src/sanitize-argument.js' + +jest.setTimeout(60000) + +const parameters = { + mwUrl: 'https://en.wikipedia.org', + articleList: 'BMW', + adminEmail: 'test@kiwix.org', + mwActionApiPath: sanitizeApiPathParam('/w/api.php'), + mwRestApiPath: sanitizeApiPathParam('/api/rest_v1'), + mwModulePath: sanitizeApiPathParam('/w/load.php'), + mwWikiPath: sanitizeWikiPath('wiki'), +} + +await testAllRenders(parameters, async (outFiles) => { + describe(`e2e test for api url params for en.wikipedia.org for ${outFiles[0]?.renderer} renderer`, () => { + test('Mediawiki actionApiPath option sanitized', () => { + expect(outFiles[0].mwMetaData.actionApiPath).toBe('w/api.php') + }) + + test('Mediawiki restApiPath option sanitized', () => { + expect(outFiles[0].mwMetaData.restApiPath).toBe('api/rest_v1') + }) + + test('Mediawiki wikiPath option sanitized', () => { + expect(outFiles[0].mwMetaData.wikiPath).toBe('wiki/') + }) + + test('Mediawiki modulePathOpt option sanitized', () => { + expect(outFiles[0].mwMetaData.modulePathOpt).toBe('w/load.php') + }) + + test('Mediawiki modulePath and actionApiUrl options', () => { + expect(outFiles[0].mwMetaData.modulePath).toBe('https://en.wikipedia.org/w/load.php?') + expect(outFiles[0].mwMetaData.actionApiUrl).toBe('https://en.wikipedia.org/w/api.php') + }) + + // TODO: blocked by issues/1931 + /* + test(`test zim integrity for ${outFiles[0]?.renderer} renderer`, async () => { + await expect(zimcheck(outFiles[0].outFile)).resolves.not.toThrowError() + }) + */ + + afterAll(() => { + rimraf.sync(`./${outFiles[0].testId}`) + }) + }) +}) diff --git a/test/e2e/articleLists.test.ts b/test/e2e/articleLists.test.ts index 8719c2b3..38166a22 100644 --- a/test/e2e/articleLists.test.ts +++ b/test/e2e/articleLists.test.ts @@ -2,7 +2,7 @@ import { execa } from 'execa' import rimraf from 'rimraf' import 'dotenv/config' import { jest } from '@jest/globals' -import { testAllRenders } from '../testAllRenders.js' +import { testAllRenders } from '../testRenders.js' jest.setTimeout(10000) @@ -20,8 +20,6 @@ const parameters = { await testAllRenders(parameters, async (outFiles) => { describe('articleList', () => { - const now = new Date() - const testId = `mwo-test-${+now}` const listMinusIgnore = 2 test(`articleList and articleListIgnore check using ${outFiles[0].renderer} renderer`, async () => { @@ -49,10 +47,12 @@ await testAllRenders(parameters, async (outFiles) => { } */ - rimraf.sync(`./${testId}`) const redisScan = await execa('redis-cli --scan', { shell: true }) // Redis has been cleared expect(redisScan.stdout).toEqual('') }) + afterAll(() => { + rimraf.sync(`./${outFiles[0].testId}`) + }) }) }) diff --git a/test/e2e/bm.e2e.test.ts b/test/e2e/bm.e2e.test.ts index 377b635e..d89a3cdd 100644 --- a/test/e2e/bm.e2e.test.ts +++ b/test/e2e/bm.e2e.test.ts @@ -1,6 +1,5 @@ -import * as mwoffliner from '../../src/mwoffliner.lib.js' import { zimdump } from '../util.js' -import { testAllRenders } from '../testAllRenders.js' +import { testAllRenders } from '../testRenders.js' import { execa } from 'execa' import { jest } from '@jest/globals' import rimraf from 'rimraf' @@ -16,43 +15,46 @@ const parameters = { } await testAllRenders(parameters, async (outFiles) => { - describe('e2e test for bm.wikipedia.org', () => { - // TODO: blocked by issues/1931 - /* + // TODO: blocked by issues/1931 + /* test(`test zim integrity for ${outFiles[0]?.renderer} renderer`, async () => { await expect(zimcheck(outFiles[0].outFile)).resolves.not.toThrowError() }) - */ - - test(`Simple articleList for ${outFiles[0]?.renderer} renderer`, async () => { - // Created 1 output - expect(outFiles).toHaveLength(1) - - for (const dump of outFiles) { - if (dump.nopic) { - // nopic has enough files - expect(dump.status.files.success).toBeGreaterThan(14) - // nopic has enough redirects - expect(dump.status.redirects.written).toBeGreaterThan(170) - // nopic has enough articles - expect(dump.status.articles.success).toBeGreaterThan(700) - } + */ + + test(`Simple articleList for ${outFiles[0]?.renderer} renderer for bm.wikipedia.org`, async () => { + await execa('redis-cli flushall', { shell: true }) + // Created 1 output + expect(outFiles).toHaveLength(1) + + for (const dump of outFiles) { + if (dump.nopic) { + // nopic has enough files + expect(dump.status.files.success).toBeGreaterThan(14) + // nopic has enough redirects + expect(dump.status.redirects.written).toBeGreaterThan(170) + // nopic has enough articles + expect(dump.status.articles.success).toBeGreaterThan(700) } - }) + } + }) - test(`Articles with "Discussion" namespace for ${outFiles[0]?.renderer} renderer`, async () => { - await execa('redis-cli flushall', { shell: true }) - const outFiles = await mwoffliner.execute({ ...parameters, addNamespaces: 1 }) + afterAll(() => { + rimraf.sync(`./${outFiles[0].testId}`) + }) +}) - // Created 1 output - expect(outFiles).toHaveLength(1) - const discussionArticlesStr = await zimdump(`list --ns A/Discussion ${outFiles[0].outFile}`) - const discussionArticlesList = discussionArticlesStr.match(/Discussion:/g) - expect(discussionArticlesList.length).toBeGreaterThan(30) - }) +await testAllRenders({ ...parameters, addNamespaces: 1 }, async (outFiles) => { + test(`Articles with "Discussion" namespace for ${outFiles[0]?.renderer} renderer for bm.wikipedia.org`, async () => { + await execa('redis-cli flushall', { shell: true }) - afterAll(() => { - rimraf.sync(`./${outFiles[0].testId}`) - }) + // Created 1 output + expect(outFiles).toHaveLength(1) + const discussionArticlesStr = await zimdump(`list --ns A/Discussion ${outFiles[0].outFile}`) + const discussionArticlesList = discussionArticlesStr.match(/Discussion:/g) + expect(discussionArticlesList.length).toBeGreaterThan(30) + }) + afterAll(() => { + rimraf.sync(`./${outFiles[0].testId}`) }) }) diff --git a/test/e2e/downloadImage.e2e.test.ts b/test/e2e/downloadImage.e2e.test.ts index 77cdbba0..4c93482c 100644 --- a/test/e2e/downloadImage.e2e.test.ts +++ b/test/e2e/downloadImage.e2e.test.ts @@ -1,7 +1,6 @@ import { execa } from 'execa' -import { zimcheck } from '../util.js' import rimraf from 'rimraf' -import { testAllRenders } from '../testAllRenders.js' +import { testAllRenders } from '../testRenders.js' import 'dotenv/config.js' import { jest } from '@jest/globals' @@ -21,7 +20,8 @@ const parameters = { await testAllRenders(parameters, async (outFiles) => { describeIf('Check image downloading from S3 using optimisationCacheUrl parameter', () => { test(`right scrapping from fr.wikipedia.org with optimisationCacheUrl parameter for ${outFiles[0]?.renderer} renderer`, async () => { - await expect(zimcheck(outFiles[0].outFile)).resolves.not.toThrowError() + // TODO: blocked by issues/1931, doesn't work for VE + // await expect(zimcheck(outFiles[0].outFile)).resolves.not.toThrowError() await execa('redis-cli flushall', { shell: true }) const redisScan = await execa('redis-cli --scan', { shell: true }) diff --git a/test/e2e/en.e2e.test.ts b/test/e2e/en.e2e.test.ts index 437d660e..40f010c2 100644 --- a/test/e2e/en.e2e.test.ts +++ b/test/e2e/en.e2e.test.ts @@ -1,4 +1,4 @@ -import { testAllRenders } from '../testAllRenders.js' +import { testAllRenders } from '../testRenders.js' import domino from 'domino' import { zimdump } from '../util.js' import 'dotenv/config.js' diff --git a/test/e2e/en10.e2e.test.ts b/test/e2e/en10.e2e.test.ts index 13610e95..fb5dfd09 100644 --- a/test/e2e/en10.e2e.test.ts +++ b/test/e2e/en10.e2e.test.ts @@ -1,5 +1,5 @@ import rimraf from 'rimraf' -import { testAllRenders } from '../testAllRenders.js' +import { testAllRenders } from '../testRenders.js' import { jest } from '@jest/globals' import 'dotenv/config.js' diff --git a/test/e2e/extra.e2e.test.ts b/test/e2e/extra.e2e.test.ts index f49d3b57..4d8dd12c 100644 --- a/test/e2e/extra.e2e.test.ts +++ b/test/e2e/extra.e2e.test.ts @@ -36,7 +36,7 @@ describe('Extra', () => { outputDirectory: testId, redis: process.env.REDIS, format: ['nopic'], - forceRender: renderer, + forceRender: 'WikimediaDesktop', }) // Created 1 outputs diff --git a/test/e2e/forceRender.test.ts b/test/e2e/forceRender.test.ts index edac673b..21b710f6 100644 --- a/test/e2e/forceRender.test.ts +++ b/test/e2e/forceRender.test.ts @@ -17,6 +17,7 @@ describe('forceRender', () => { redis: process.env.REDIS, format: ['nopic'], articleList: 'France', + mwActionApiPath: 'w/api.php', } beforeAll(async () => { diff --git a/test/e2e/formatParams.test.ts b/test/e2e/formatParams.test.ts index d9435ab2..a2bc4981 100644 --- a/test/e2e/formatParams.test.ts +++ b/test/e2e/formatParams.test.ts @@ -1,8 +1,10 @@ import 'dotenv/config.js' import domino from 'domino' -import { testAllRenders } from '../testAllRenders.js' +import { testAllRenders } from '../testRenders.js' import { jest } from '@jest/globals' import { zimdump } from '../util.js' +import rimraf from 'rimraf' +import { execa } from 'execa' jest.setTimeout(200000) @@ -15,12 +17,14 @@ const parameters = { await testAllRenders({ ...parameters, format: 'nopic', articleList: 'BMW' }, async (outFiles) => { describe('format:nopic', () => { test(`Test en.wikipedia.org using format:nopic for ${outFiles[0]?.renderer} renderer`, async () => { + await execa('redis-cli flushall', { shell: true }) const articleFromDump = await zimdump(`show --url A/BMW ${outFiles[0].outFile}`) const articleDoc = domino.createDocument(articleFromDump) const imgElements = Array.from(articleDoc.querySelectorAll('img')) expect(imgElements).toHaveLength(0) + rimraf.sync(`./${outFiles[0].testId}`) }) }) }) @@ -28,6 +32,7 @@ await testAllRenders({ ...parameters, format: 'nopic', articleList: 'BMW' }, asy await testAllRenders({ ...parameters, format: 'nodet', articleList: 'BMW' }, async (outFiles) => { describe('format:nodet', () => { test(`Test en.wikipedia.org using format:nodet for ${outFiles[0]?.renderer} renderer`, async () => { + await execa('redis-cli flushall', { shell: true }) const articleFromDump = await zimdump(`show --url A/BMW ${outFiles[0].outFile}`) const articleDoc = domino.createDocument(articleFromDump) @@ -35,6 +40,7 @@ await testAllRenders({ ...parameters, format: 'nodet', articleList: 'BMW' }, asy expect(sectionsElements).toHaveLength(1) expect(sectionsElements[0].getAttribute('data-mw-section-id')).toEqual('0') + rimraf.sync(`./${outFiles[0].testId}`) }) }) }) @@ -42,12 +48,14 @@ await testAllRenders({ ...parameters, format: 'nodet', articleList: 'BMW' }, asy await testAllRenders({ ...parameters, format: 'novid', articleList: 'Animation' }, async (outFiles) => { describe('format:novid to check no video tags', () => { test(`Test en.wikipedia.org using format:novid for ${outFiles[0]?.renderer} renderer (no video)`, async () => { + await execa('redis-cli flushall', { shell: true }) const articleFromDump = await zimdump(`show --url A/Animation ${outFiles[0].outFile}`) const articleDoc = domino.createDocument(articleFromDump) const audioElements = Array.from(articleDoc.querySelectorAll('audio')) expect(audioElements).toHaveLength(0) + rimraf.sync(`./${outFiles[0].testId}`) }) }) }) @@ -55,24 +63,30 @@ await testAllRenders({ ...parameters, format: 'novid', articleList: 'Animation' await testAllRenders({ ...parameters, format: 'novid', articleList: 'English_alphabet' }, async (outFiles) => { describe('format:novid to check no audio tags', () => { test(`Test en.wikipedia.org using format:novid for ${outFiles[0]?.renderer} renderer (no audio)`, async () => { + await execa('redis-cli flushall', { shell: true }) const articleFromDump = await zimdump(`show --url A/English_alphabet ${outFiles[0].outFile}`) const articleDoc = domino.createDocument(articleFromDump) const videoElements = Array.from(articleDoc.querySelectorAll('video')) expect(videoElements).toHaveLength(0) + rimraf.sync(`./${outFiles[0].testId}`) }) }) }) -await testAllRenders({ ...parameters, format: 'nopdf', articleList: 'PDF' }, async (outFiles) => { +// TODO: blocked by issues/1928 +/* +await testRenders({ ...parameters, format: 'nopdf', articleList: 'PDF' }, async (outFiles) => { describe('format:pdf to check no internal links pdf files', () => { - test.skip(`Test en.wikipedia.org using format:nopdf for ${outFiles[0]?.renderer} renderer`, async () => { + test(`Test en.wikipedia.org using format:nopdf for ${outFiles[0]?.renderer} renderer`, async () => { + await execa('redis-cli flushall', { shell: true }) const articleFromDump = await zimdump(`show --url A/PDF ${outFiles[0].outFile}`) const articleDoc = domino.createDocument(articleFromDump) - // TODO: blocked by issues/1928 // eslint-disable-next-line @typescript-eslint/no-unused-vars const anchorElements = Array.from(articleDoc.querySelectorAll('a')) + rimraf.sync(`./${outFiles[0].testId}`) }) }) }) +*/ diff --git a/test/e2e/multimediaContent.test.ts b/test/e2e/multimediaContent.test.ts index edbc8efb..83f67a2e 100644 --- a/test/e2e/multimediaContent.test.ts +++ b/test/e2e/multimediaContent.test.ts @@ -1,7 +1,7 @@ import { execa } from 'execa' -import { testAllRenders } from '../testAllRenders.js' +import { testRenders } from '../testRenders.js' import rimraf from 'rimraf' -import { zimcheck, zimdump } from '../util.js' +import { zimdump } from '../util.js' import 'dotenv/config' import { jest } from '@jest/globals' @@ -16,79 +16,49 @@ const parameters = { forceRender: 'WikimediaDesktop', } -await testAllRenders(parameters, async (outFiles) => { - describe('Multimedia', () => { - // TODO: blocked by issues/1925 - if (outFiles[0].renderer !== 'WikimediaMobile') { - test(`check multimedia content from wikipedia test page for ${outFiles[0]?.renderer} renderer`, async () => { - await execa('redis-cli flushall', { shell: true }) - - expect(outFiles[0].status.articles.success).toEqual(1) - expect(outFiles[0].status.articles.fail).toEqual(0) - const mediaFiles = await zimdump(`list --ns I ${outFiles[0].outFile}`) - - expect(mediaFiles.split('\n').sort()).toEqual( - [ - 'I/Kiwix_-_WikiArabia_Cairo_2017.pdf', - 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm', - 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg', - 'I/Kiwix_icon.svg.png', - 'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg', - 'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg', - 'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg', - ].sort(), - ) - }) - afterAll(() => { - rimraf.sync(`./${outFiles[0].testId}`) - }) - } - }) -}) - -await testAllRenders({ ...parameters, format: ['nopic', 'novid', 'nopdf', 'nodet'] }, async (outFiles) => { - describe('Multimedia for different formats', () => { - // TODO: blocked by issues/1925 - if (outFiles[0].renderer !== 'WikimediaMobile') { - test(`check multimedia content from wikipedia test page with different formates for ${outFiles[0]?.renderer} renderer`, async () => { - await execa('redis-cli flushall', { shell: true }) - - expect(outFiles).toHaveLength(4) - for (const dump of outFiles) { - expect(dump.status.articles.success).toEqual(1) - expect(dump.status.articles.fail).toEqual(0) - - await expect(zimcheck(dump.outFile)).resolves.not.toThrowError() - - const mediaFiles = await zimdump(`list --ns I ${dump.outFile}`) - if (dump.nopic) { - expect(mediaFiles.split('\n').sort()).toEqual( - [ - 'I/Kiwix_-_WikiArabia_Cairo_2017.pdf', - // 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm', // these files were omitted by nopic parameter - // 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg', - // 'I/Kiwix_icon.svg.png', - // 'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg', - // 'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg', - // 'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg', - ].sort(), - ) - } else if (dump.novid) { +await testRenders( + parameters, + async (outFiles) => { + describe('Multimedia', () => { + switch (outFiles[0].renderer) { + // TODO: blocked by issues/1925 + case 'WikimediaMobile': + break + case 'WikimediaDesktop': + test(`check multimedia content from wikipedia test page for ${outFiles[0]?.renderer} renderer`, async () => { + await execa('redis-cli flushall', { shell: true }) + + expect(outFiles[0].status.articles.success).toEqual(1) + expect(outFiles[0].status.articles.fail).toEqual(0) + const mediaFiles = await zimdump(`list --ns I ${outFiles[0].outFile}`) + expect(mediaFiles.split('\n').sort()).toEqual( [ 'I/Kiwix_-_WikiArabia_Cairo_2017.pdf', - // 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm', // these files were omitted by novid parameter - // 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg', + 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm', + 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg', 'I/Kiwix_icon.svg.png', - // 'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg', + 'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg', 'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg', 'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg', ].sort(), ) - } else if (dump.nopdf) { + }) + afterAll(() => { + rimraf.sync(`./${outFiles[0].testId}`) + }) + break + case 'VisualEditor': + test(`check multimedia content from wikipedia test page for ${outFiles[0]?.renderer} renderer`, async () => { + await execa('redis-cli flushall', { shell: true }) + + expect(outFiles[0].status.articles.success).toEqual(1) + expect(outFiles[0].status.articles.fail).toEqual(0) + const mediaFiles = await zimdump(`list --ns I ${outFiles[0].outFile}`) + expect(mediaFiles.split('\n').sort()).toEqual( [ - // 'I/Kiwix_-_WikiArabia_Cairo_2017.pdf', // this file was omitted by nopdf parameter + 'I/Kiwix_-_WikiArabia_Cairo_2017.pdf', 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm', 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg', 'I/Kiwix_icon.svg.png', @@ -97,12 +67,142 @@ await testAllRenders({ ...parameters, format: ['nopic', 'novid', 'nopdf', 'nodet 'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg', ].sort(), ) - } - } - }) - afterAll(() => { - rimraf.sync(`./${outFiles[0].testId}`) - }) - } - }) -}) + }) + afterAll(() => { + rimraf.sync(`./${outFiles[0].testId}`) + }) + break + } + }) + }, + ['WikimediaDesktop', 'VisualEditor'], +) + +await testRenders( + { ...parameters, format: ['nopic', 'novid', 'nopdf', 'nodet'] }, + async (outFiles) => { + describe('Multimedia for different formats', () => { + // TODO: blocked by issues/1925 + switch (outFiles[0].renderer) { + // TODO: blocked by issues/1925 + case 'WikimediaMobile': + break + case 'WikimediaDesktop': + test(`check multimedia content from wikipedia test page with different formates for ${outFiles[0]?.renderer} renderer`, async () => { + await execa('redis-cli flushall', { shell: true }) + + expect(outFiles).toHaveLength(4) + + for (const dump of outFiles) { + expect(dump.status.articles.success).toEqual(1) + expect(dump.status.articles.fail).toEqual(0) + + // TODO: blocked by issues/1931 + // await expect(zimcheck(dump.outFile)).resolves.not.toThrowError() + + const mediaFiles = await zimdump(`list --ns I ${dump.outFile}`) + if (dump.nopic) { + expect(mediaFiles.split('\n').sort()).toEqual( + [ + 'I/Kiwix_-_WikiArabia_Cairo_2017.pdf', + // 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm', // these files were omitted by nopic parameter + // 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg', + // 'I/Kiwix_icon.svg.png', + // 'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg', + // 'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg', + // 'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg', + ].sort(), + ) + } else if (dump.novid) { + expect(mediaFiles.split('\n').sort()).toEqual( + [ + 'I/Kiwix_-_WikiArabia_Cairo_2017.pdf', + // 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm', // these files were omitted by novid parameter + // 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg', + 'I/Kiwix_icon.svg.png', + // 'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg', + 'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg', + 'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg', + ].sort(), + ) + } else if (dump.nopdf) { + expect(mediaFiles.split('\n').sort()).toEqual( + [ + // 'I/Kiwix_-_WikiArabia_Cairo_2017.pdf', // this file was omitted by nopdf parameter + 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm', + 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg', + 'I/Kiwix_icon.svg.png', + 'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg', + 'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg', + 'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg', + ].sort(), + ) + } + } + }) + afterAll(() => { + rimraf.sync(`./${outFiles[0].testId}`) + }) + break + case 'VisualEditor': + test(`check multimedia content from wikipedia test page with different formates for ${outFiles[0]?.renderer} renderer`, async () => { + await execa('redis-cli flushall', { shell: true }) + + expect(outFiles).toHaveLength(4) + + for (const dump of outFiles) { + expect(dump.status.articles.success).toEqual(1) + expect(dump.status.articles.fail).toEqual(0) + + // TODO: blocked by issues/1931, doesn't work for VE + // await expect(zimcheck(dump.outFile)).resolves.not.toThrowError() + + const mediaFiles = await zimdump(`list --ns I ${dump.outFile}`) + if (dump.nopic) { + expect(mediaFiles.split('\n').sort()).toEqual( + [ + 'I/Kiwix_-_WikiArabia_Cairo_2017.pdf', + // 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm', // these files were omitted by nopic parameter + // 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg', + // 'I/Kiwix_icon.svg.png', + // 'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg', + // 'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg', + // 'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg', + ].sort(), + ) + } else if (dump.novid) { + expect(mediaFiles.split('\n').sort()).toEqual( + [ + 'I/Kiwix_-_WikiArabia_Cairo_2017.pdf', + // 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm', // these files were omitted by novid parameter + // 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg', + 'I/Kiwix_icon.svg.png', + // 'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg', + 'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg', + 'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg', + ].sort(), + ) + } else if (dump.nopdf) { + expect(mediaFiles.split('\n').sort()).toEqual( + [ + // 'I/Kiwix_-_WikiArabia_Cairo_2017.pdf', // this file was omitted by nopdf parameter + 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm', + 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg', + 'I/Kiwix_icon.svg.png', + 'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg', + 'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg', + 'I/page1-640px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg', + ].sort(), + ) + } + } + }) + afterAll(() => { + rimraf.sync(`./${outFiles[0].testId}`) + }) + break + } + }) + }, + ['WikimediaDesktop', 'VisualEditor'], +) diff --git a/test/e2e/treatMedia.e2e.test.ts b/test/e2e/treatMedia.e2e.test.ts index 475290c3..9ff18b3c 100644 --- a/test/e2e/treatMedia.e2e.test.ts +++ b/test/e2e/treatMedia.e2e.test.ts @@ -1,6 +1,6 @@ import { execa } from 'execa' import rimraf from 'rimraf' -import { testAllRenders } from '../testAllRenders.js' +import { testAllRenders } from '../testRenders.js' import { zimdump } from '../util.js' import 'dotenv/config' import { jest } from '@jest/globals' diff --git a/test/e2e/vikidia.e2e.test.ts b/test/e2e/vikidia.e2e.test.ts index f0d01530..b79dd609 100644 --- a/test/e2e/vikidia.e2e.test.ts +++ b/test/e2e/vikidia.e2e.test.ts @@ -1,7 +1,6 @@ import { execa } from 'execa' import rimraf from 'rimraf' -import { testAllRenders } from '../testAllRenders.js' -import { zimcheck } from '../util.js' +import { testRenders } from '../testRenders.js' import 'dotenv/config.js' import { jest } from '@jest/globals' @@ -15,19 +14,21 @@ const parameters = { customZimDescription: 'Alaska article', } -await testAllRenders(parameters, async (outFiles) => { - describe('vikidia', () => { +await testRenders( + parameters, + async (outFiles) => { test(`right scrapping from vikidia.org for ${outFiles[0]?.renderer} renderer`, async () => { await execa('redis-cli flushall', { shell: true }) // Created 1 output expect(outFiles).toHaveLength(1) - await expect(zimcheck(outFiles[0].outFile)).resolves.not.toThrowError() + // TODO: Blocked by issues/1931 + // await expect(zimcheck(outFiles[0].outFile)).resolves.not.toThrowError() }) - afterAll(() => { - rimraf.sync(`./${outFiles[0].testId}`) - }) - }) -}) + rimraf.sync(`./${outFiles[0].testId}`) + }, + // en.vikidia.org supports only VisualEditor among other renders + ['VisualEditor'], +) diff --git a/test/e2e/wikisource.e2e.test.ts b/test/e2e/wikisource.e2e.test.ts index 53f1c25c..f5d63344 100644 --- a/test/e2e/wikisource.e2e.test.ts +++ b/test/e2e/wikisource.e2e.test.ts @@ -1,6 +1,6 @@ import { execa } from 'execa' import rimraf from 'rimraf' -import { testAllRenders } from '../testAllRenders.js' +import { testRenders } from '../testRenders.js' import 'dotenv/config.js' import { jest } from '@jest/globals' @@ -14,27 +14,54 @@ const parameters = { noLocalParserFallback: true, } -await testAllRenders(parameters, async (outFiles) => { - describe('vikidia', () => { - test(`Wikisource List for ${outFiles[0]?.renderer} renderer`, async () => { - await execa('redis-cli flushall', { shell: true }) - - expect(outFiles).toHaveLength(1) - - for (const dump of outFiles) { - if (dump.nopic) { - // nopic has enough files - expect(dump.status.files.success).toBeGreaterThanOrEqual(20) - // nopic has enough redirects - expect(dump.status.redirects.written).toBeGreaterThanOrEqual(16) - // nopic has enough articles - expect(dump.status.articles.success).toBeGreaterThanOrEqual(61) - } - } - }) +await testRenders( + parameters, + async (outFiles) => { + describe('wikisource', () => { + switch (outFiles[0].renderer) { + case 'WikimediaDesktop': + test(`Wikisource List for ${outFiles[0]?.renderer} renderer`, async () => { + await execa('redis-cli flushall', { shell: true }) + + expect(outFiles).toHaveLength(1) + + for (const dump of outFiles) { + if (dump.nopic) { + // nopic has enough files + expect(dump.status.files.success).toBeGreaterThanOrEqual(20) + // nopic has enough redirects + expect(dump.status.redirects.written).toBeGreaterThanOrEqual(16) + // nopic has enough articles + expect(dump.status.articles.success).toBeGreaterThanOrEqual(61) + } + } + }) - afterAll(() => { - rimraf.sync(`./${outFiles[0].testId}`) + afterAll(() => { + rimraf.sync(`./${outFiles[0].testId}`) + }) + break + case 'VisualEditor': + test(`Wikisource List for ${outFiles[0]?.renderer} renderer`, async () => { + await execa('redis-cli flushall', { shell: true }) + + expect(outFiles).toHaveLength(1) + + for (const dump of outFiles) { + if (dump.nopic) { + // nopic has enough files + expect(dump.status.files.success).toBeGreaterThanOrEqual(20) + // nopic has enough redirects + expect(dump.status.redirects.written).toBeGreaterThanOrEqual(16) + // nopic has enough articles + expect(dump.status.articles.success).toBeGreaterThanOrEqual(61) + } + } + }) + rimraf.sync(`./${outFiles[0].testId}`) + break + } }) - }) -}) + }, + ['WikimediaDesktop', 'VisualEditor'], +) diff --git a/test/e2e/zimMetadata.e2e.test.ts b/test/e2e/zimMetadata.e2e.test.ts index c7beedab..6f753626 100644 --- a/test/e2e/zimMetadata.e2e.test.ts +++ b/test/e2e/zimMetadata.e2e.test.ts @@ -1,6 +1,6 @@ import rimraf from 'rimraf' import { execa } from 'execa' -import { testAllRenders } from '../testAllRenders.js' +import { testAllRenders } from '../testRenders.js' import { zimdump } from '../util.js' import 'dotenv/config' import { jest } from '@jest/globals' diff --git a/test/testAllRenders.ts b/test/testRenders.ts similarity index 65% rename from test/testAllRenders.ts rename to test/testRenders.ts index 0b575383..68062359 100644 --- a/test/testAllRenders.ts +++ b/test/testRenders.ts @@ -7,12 +7,17 @@ import { zimcheckAvailable, zimdumpAvailable } from './util.js' interface Parameters { mwUrl: string adminEmail: string + outputDirectory?: string + addNamespaces?: number articleList?: string articleListToIgnore?: string redis?: string format?: string | string[] noLocalParserFallback?: boolean forceRender?: string + mwActionApiPath?: string + mwRestApiPath?: string + mwModulePath?: string } /* @@ -42,19 +47,28 @@ async function checkZimTools() { async function getOutFiles(renderName: string, testId: string, parameters: Parameters): Promise { await execa('redis-cli flushall', { shell: true }) - const outFiles = await mwoffliner.execute(parameters) + const outFiles = await mwoffliner.execute({ ...parameters, outputDirectory: testId, forceRender: renderName }) return outFiles } -export async function testAllRenders(parameters: Parameters, callback) { +export async function testRenders(parameters: Parameters, callback, renderersList: Array) { await checkZimTools() - for (const renderer of RENDERERS_LIST) { - const now = new Date() - const testId = `mwo-test-${+now}` - const outFiles = await getOutFiles(renderer, testId, parameters) - outFiles[0].testId = testId - outFiles[0].renderer = renderer - await callback(outFiles) + for (const renderer of renderersList) { + try { + const now = new Date() + const testId = `mwo-test-${+now}` + const outFiles = await getOutFiles(renderer, testId, parameters) + outFiles[0].testId = testId + outFiles[0].renderer = renderer + await callback(outFiles) + } catch (err) { + logger.error(err.message) + return + } } } + +export async function testAllRenders(parameters: Parameters, callback) { + return testRenders(parameters, callback, RENDERERS_LIST) +} diff --git a/test/unit/builders/url/base.director.test.ts b/test/unit/builders/url/base.director.test.ts index f679d2f9..ac96230b 100644 --- a/test/unit/builders/url/base.director.test.ts +++ b/test/unit/builders/url/base.director.test.ts @@ -9,6 +9,12 @@ describe('BaseURLDirector', () => { expect(url.href).toBe('https://en.m.wikipedia.com/v1/test/api') }) + + it('should return URL object with mwActionApiPath param', () => { + const url = baseUrlDirector.buildURL('api.php') + + expect(url.href).toBe('https://en.m.wikipedia.com/api.php') + }) }) describe('buildWikimediaApiURL', () => { @@ -26,8 +32,8 @@ describe('BaseURLDirector', () => { }) describe('buildWikimediaMobileApiUrl', () => { - it('should return mobile rest URL with provided path and trailing char', () => { - const url = baseUrlDirector.buildWikimediaMobileApiUrl('api/rest_v2/page/mobile-html') + it('should automatically return mobile rest URL with provided api rest path', () => { + const url = baseUrlDirector.buildWikimediaMobileApiUrl('api/rest_v2') expect(url.href).toBe('https://en.m.wikipedia.com/api/rest_v2/page/mobile-html/') }) @@ -40,8 +46,8 @@ describe('BaseURLDirector', () => { }) describe('buildWikimediaDesktopApiUrl', () => { - it('should return a desktop URL with provided path and trailing char', () => { - const url = baseUrlDirector.buildWikimediaDesktopApiUrl('api/rest_v2/page/html') + it('should automatically return a desktop URL with provided api rest path', () => { + const url = baseUrlDirector.buildWikimediaDesktopApiUrl('api/rest_v2') expect(url.href).toBe('https://en.m.wikipedia.com/api/rest_v2/page/html/') }) diff --git a/test/unit/renderers/article.renderer.test.ts b/test/unit/renderers/article.renderer.test.ts index 2a616440..d8c5c57a 100644 --- a/test/unit/renderers/article.renderer.test.ts +++ b/test/unit/renderers/article.renderer.test.ts @@ -36,7 +36,7 @@ describe('ArticleRenderer', () => { }) it('should return visualeditor content if the main page flag is true', async () => { - const { downloader, dump } = await setupScrapeClasses({ format: '' }) + const { downloader, dump } = await setupScrapeClasses() const { data, articleId, articleDetail } = prepareFixtures({ visualeditor: { content: 'Lorem ipsum dolor sit amet' } }) const _moduleDependencies = await downloader.getModuleDependencies(articleDetail.title) const result = await visualEditorRenderer.render({ @@ -56,7 +56,7 @@ describe('ArticleRenderer', () => { }) it('should inject header to the visual editor content if the main page flag is false', async () => { - const { downloader, dump } = await setupScrapeClasses({ format: '' }) + const { downloader, dump } = await setupScrapeClasses() const content = 'consectetur adipiscing elit' const { data, articleId, articleDetail } = prepareFixtures({ visualeditor: { content } }) const _moduleDependencies = await downloader.getModuleDependencies(articleDetail.title) @@ -75,7 +75,7 @@ describe('ArticleRenderer', () => { }) it('should return html body if json contentmodel param is `wikitext`', async () => { - const { downloader, dump } = await setupScrapeClasses({ format: '' }) + const { downloader, dump } = await setupScrapeClasses() const htmlBody = 'sed do eiusmod tempor incididunt' const { data, articleId, articleDetail } = prepareFixtures({ html: { body: htmlBody }, contentmodel: 'wikitext' }) const _moduleDependencies = await downloader.getModuleDependencies(articleDetail.title) @@ -96,7 +96,7 @@ describe('ArticleRenderer', () => { }) it('should return html body if it`s presented even if contentmodel param is not equal to wikitext', async () => { - const { downloader, dump } = await setupScrapeClasses({ format: '' }) + const { downloader, dump } = await setupScrapeClasses() const htmlBody = 'ut labore et dolore magna aliqua. Ut enim ad minim veniam' const { data, articleId, articleDetail } = prepareFixtures({ html: { body: htmlBody } }) const _moduleDependencies = await downloader.getModuleDependencies(articleDetail.title) @@ -117,7 +117,7 @@ describe('ArticleRenderer', () => { }) it('should return empty string if there was an error during article retrievement', async () => { - const { downloader, dump } = await setupScrapeClasses({ format: '' }) + const { downloader, dump } = await setupScrapeClasses() const { data, articleId, articleDetail } = prepareFixtures({ error: 'Unexpected internal error' }) const _moduleDependencies = await downloader.getModuleDependencies(articleDetail.title) const result = await visualEditorRenderer.render({ diff --git a/test/unit/renderers/renderer.builder.test.ts b/test/unit/renderers/renderer.builder.test.ts index 9a6687ee..25e6b7db 100644 --- a/test/unit/renderers/renderer.builder.test.ts +++ b/test/unit/renderers/renderer.builder.test.ts @@ -1,9 +1,9 @@ import { jest } from '@jest/globals' import { setupScrapeClasses } from '../../util.js' import { RendererBuilder } from '../../../src/renderers/renderer.builder.js' -import { VisualEditorRenderer } from '../../../src/renderers/visual-editor.renderer.js' import { RendererBuilderOptions } from '../../../src/renderers/abstract.renderer.js' import { WikimediaDesktopRenderer } from '../../../src/renderers/wikimedia-desktop.renderer.js' +import { VisualEditorRenderer } from '../../../src/renderers/visual-editor.renderer.js' jest.setTimeout(10000) diff --git a/test/unit/treatments/media.treatment.test.ts b/test/unit/treatments/media.treatment.test.ts index 9976c10b..bc6eb5d4 100644 --- a/test/unit/treatments/media.treatment.test.ts +++ b/test/unit/treatments/media.treatment.test.ts @@ -29,7 +29,7 @@ describe('MediaTreatment', () => { describe('treatSubtitle', () => { test('treat one subtitle', async () => { - const { dump } = await setupScrapeClasses({ format: '' }) + const { dump } = await setupScrapeClasses() // Wikicode is taken from article "Mechanical energy" which has a video with subtitle const wikicode = diff --git a/test/unit/urlRewriting.test.ts b/test/unit/urlRewriting.test.ts index 6b945ab6..60c384a6 100644 --- a/test/unit/urlRewriting.test.ts +++ b/test/unit/urlRewriting.test.ts @@ -88,6 +88,7 @@ describe('Styles', () => { expect($extHttpsNoRel.getAttribute('href')).toEqual('https://google.com') await rewriteUrl(parentArticleId, dump, $wikiLink) + // wikiLink is still a link with simple parent id expect($wikiLink.nodeName).toEqual('A') // wikiLink HREF is correct with simple parent id diff --git a/test/unit/webpAndRedirection.test.ts b/test/unit/webpAndRedirection.test.ts index 4a094a76..fc9816ea 100644 --- a/test/unit/webpAndRedirection.test.ts +++ b/test/unit/webpAndRedirection.test.ts @@ -53,7 +53,6 @@ Real-time computer graphics` expect(isWebpCandidateImageUrl('../I/osm-intl%2C9%2C52.2789%2C8.0431%2C300x300.jpg')).toBeTruthy() // detecting webp URL having jpeg at last expect(isWebpCandidateImageUrl('../I/osm-intl%2C9%2C52.2789%2C8.0431%2C300x300.jpeg')).toBeTruthy() - // passed test for png expect(await isWebpPresent('I/Animexample3edit.png.webp', zimFile)).toBeTruthy() // passed test for jpg