Skip to content

Commit

Permalink
Merge pull request #1939 from openzim/I1935-rollback-mw-api-configs
Browse files Browse the repository at this point in the history 
Refactor mwApiPath (to mwActionApiPath), mwRestApiPath, mwModulePath and mwWikiPath parameters + fix regression of test render template
  • Loading branch information
@kelson42
kelson42 authored Nov 13, 2023
2 parents c4c2dde + d3b0125 commit 99da4fd
Show file tree
Hide file tree
Showing 29 changed files with 541 additions and 246 deletions.
7 changes: 4 additions & 3 deletions src/Downloader.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ interface DownloaderOpts {
s3?: S3
webp: boolean
backoffOptions?: BackoffOptions
mwWikiPath?: string
}

interface BackoffOptions {
Expand Down Expand Up @@ -107,7 +108,7 @@ class Downloader {
this.optimisationCacheUrl = optimisationCacheUrl
this.webp = webp
this.s3 = s3
this.apiUrlDirector = new ApiURLDirector(MediaWiki.apiUrl.href)
this.apiUrlDirector = new ApiURLDirector(MediaWiki.actionApiUrl.href)

this.backoffOptions = {
strategy: new backoff.ExponentialStrategy(),
Expand Down Expand Up @@ -620,7 +621,7 @@ class Downloader {
}

private async getSubCategories(articleId: string, continueStr = ''): Promise<Array<{ pageid: number; ns: number; title: string }>> {
const apiUrlDirector = new ApiURLDirector(MediaWiki.apiUrl.href)
const apiUrlDirector = new ApiURLDirector(MediaWiki.actionApiUrl.href)

const { query, continue: cont } = await this.getJSON<any>(apiUrlDirector.buildSubCategoriesURL(articleId, continueStr))
const items = query.categorymembers.filter((a: any) => a && a.title)
Expand Down Expand Up @@ -651,7 +652,7 @@ class Downloader {
let jsDependenciesList: string[] = []
let styleDependenciesList: string[] = []

const apiUrlDirector = new ApiURLDirector(MediaWiki.apiUrl.href)
const apiUrlDirector = new ApiURLDirector(MediaWiki.actionApiUrl.href)

const articleApiUrl = apiUrlDirector.buildArticleApiURL(title)

Expand Down
92 changes: 61 additions & 31 deletions src/MediaWiki.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,20 +44,21 @@ class MediaWiki {
public queryOpts: QueryOpts

#wikiPath: string
#apiPath: string
#actionApiPath: string
#restApiPath: string
#modulePathOpt: string
#username: string
#password: string
#apiActionPath: string
#domain: string
private apiUrlDirector: ApiURLDirector
private baseUrlDirector: BaseURLDirector
private wikimediaDesktopUrlDirector: WikimediaDesktopURLDirector
private wikimediaMobileUrlDirector: WikimediaMobileURLDirector
private VisualEditorURLDirector: VisualEditorURLDirector
private visualEditorURLDirector: VisualEditorURLDirector

public visualEditorApiUrl: URL
public apiUrl: URL
public actionApiUrl: URL
public modulePath: string // only for reading
public _modulePathOpt: string // only for whiting to generate modulePath
public mobileModulePath: string
public webUrl: URL
public WikimediaDesktopApiUrl: URL
Expand All @@ -76,29 +77,53 @@ class MediaWiki {
this.#password = value
}

set apiActionPath(value: string) {
this.#apiActionPath = value
set actionApiPath(value: string) {
if (value) {
this.#actionApiPath = value
this.initApiURLDirector()
}
}

set apiPath(value: string) {
this.#apiPath = value
set restApiPath(value: string) {
if (value) {
this.#restApiPath = value
this.initApiURLDirector()
}
}

set domain(value: string) {
this.#domain = value
}

set wikiPath(value: string) {
this.#wikiPath = value
if (value) {
this.#wikiPath = value
this.initApiURLDirector()
}
}

set base(value: string) {
this.baseUrl = basicURLDirector.buildMediawikiBaseURL(value)
this.initMWApis()
if (value) {
this.baseUrl = basicURLDirector.buildMediawikiBaseURL(value)
this.baseUrlDirector = new BaseURLDirector(this.baseUrl.href)
this.initMWApis()
this.initApiURLDirector()
}
}

set modulePathOpt(value: string) {
this._modulePathOpt = value
if (value) {
this.#modulePathOpt = value
if (this.baseUrlDirector) {
this.modulePath = this.baseUrlDirector.buildModuleURL(this.#modulePathOpt)
} else {
logger.error('Base url director should be specified first')
}
} else {
if (this.baseUrlDirector) {
this.modulePath = this.baseUrlDirector.buildModuleURL(this.#modulePathOpt)
}
}
}

private initializeMediaWikiDefaults(): void {
Expand All @@ -107,11 +132,13 @@ class MediaWiki {
this.#password = ''
this.getCategories = false

this.#actionApiPath = 'w/api.php'
this.#restApiPath = 'api/rest_v1'
this.#wikiPath = 'wiki/'
this.#modulePathOpt = 'w/load.php'

this.namespaces = {}
this.namespacesToMirror = []

this.#apiActionPath = 'w/api.php'
this.#wikiPath = 'wiki/'
this.apiCheckArticleId = 'MediaWiki:Sidebar'

this.queryOpts = {
Expand Down Expand Up @@ -152,7 +179,7 @@ class MediaWiki {

public async hasVisualEditorApi(): Promise<boolean> {
if (this.#hasVisualEditorApi === null) {
this.#hasVisualEditorApi = await checkApiAvailability(this.VisualEditorURLDirector.buildArticleURL(this.apiCheckArticleId))
this.#hasVisualEditorApi = await checkApiAvailability(this.visualEditorURLDirector.buildArticleURL(this.apiCheckArticleId))
return this.#hasVisualEditorApi
}
return this.#hasVisualEditorApi
Expand All @@ -178,23 +205,24 @@ class MediaWiki {
}

private initMWApis() {
const baseUrlDirector = new BaseURLDirector(this.baseUrl.href)
this.webUrl = baseUrlDirector.buildURL(this.#wikiPath)
this.apiUrl = baseUrlDirector.buildURL(this.#apiActionPath)
this.apiUrlDirector = new ApiURLDirector(this.apiUrl.href)
this.visualEditorApiUrl = this.apiUrlDirector.buildVisualEditorURL()
this.WikimediaDesktopApiUrl = baseUrlDirector.buildWikimediaDesktopApiUrl(this.#apiPath)
this.WikimediaMobileApiUrl = baseUrlDirector.buildWikimediaMobileApiUrl(this.#apiPath)
this.modulePath = baseUrlDirector.buildModuleURL(this._modulePathOpt)
this.mobileModulePath = baseUrlDirector.buildMobileModuleURL()
this.WikimediaDesktopApiUrl = this.baseUrlDirector.buildWikimediaDesktopApiUrl(this.#restApiPath)
this.WikimediaMobileApiUrl = this.baseUrlDirector.buildWikimediaMobileApiUrl(this.#restApiPath)
this.mobileModulePath = this.baseUrlDirector.buildMobileModuleURL()
this.wikimediaDesktopUrlDirector = new WikimediaDesktopURLDirector(this.WikimediaDesktopApiUrl.href)
this.wikimediaMobileUrlDirector = new WikimediaMobileURLDirector(this.WikimediaMobileApiUrl.href)
this.VisualEditorURLDirector = new VisualEditorURLDirector(this.visualEditorApiUrl.href)
}

private initApiURLDirector() {
this.webUrl = this.baseUrlDirector.buildURL(this.#wikiPath)
this.actionApiUrl = this.baseUrlDirector.buildURL(this.#actionApiPath)
this.apiUrlDirector = new ApiURLDirector(this.actionApiUrl.href)
this.visualEditorApiUrl = this.apiUrlDirector.buildVisualEditorURL()
this.visualEditorURLDirector = new VisualEditorURLDirector(this.visualEditorApiUrl.href)
}

public async login(downloader: Downloader) {
if (this.#username && this.#password) {
let url = this.apiUrl.href + '?'
let url = this.actionApiUrl.href + '?'

// Add domain if configured
if (this.#domain) {
Expand All @@ -205,7 +233,7 @@ class MediaWiki {
const { content, responseHeaders } = await downloader.downloadContent(url + 'action=query&meta=tokens&type=login&format=json&formatversion=2')

// Logging in
await axios(this.apiUrl.href, {
await axios(this.actionApiUrl.href, {
data: qs.stringify({
action: 'login',
format: 'json',
Expand Down Expand Up @@ -413,13 +441,15 @@ class MediaWiki {

const mwMetaData: MWMetaData = {
webUrl: this.webUrl.href,
apiUrl: this.apiUrl.href,
actionApiUrl: this.actionApiUrl.href,
modulePathOpt: this.#modulePathOpt,
modulePath: this.modulePath,
mobileModulePath: this.mobileModulePath,
webUrlPath: this.webUrl.pathname,
wikiPath: this.#wikiPath,
baseUrl: this.baseUrl.href,
apiActionPath: this.#apiActionPath,
actionApiPath: this.#actionApiPath,
restApiPath: this.#restApiPath,
domain: this.#domain,

textDir: textDir as TextDirection,
Expand Down
10 changes: 5 additions & 5 deletions src/mwoffliner.lib.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ async function execute(argv: any) {
mwUrl,
mwWikiPath,
mwActionApiPath,
mwApiPath,
mwRestApiPath,
mwModulePath,
mwDomain,
mwUsername,
Expand Down Expand Up @@ -158,13 +158,13 @@ async function execute(argv: any) {
/* Wikipedia/... URL; Normalize by adding trailing / as necessary */
MediaWiki.base = mwUrl
MediaWiki.getCategories = !!argv.getCategories
MediaWiki.apiActionPath = mwActionApiPath
MediaWiki.apiPath = mwApiPath
MediaWiki.wikiPath = mwWikiPath
MediaWiki.actionApiPath = mwActionApiPath
MediaWiki.restApiPath = mwRestApiPath
MediaWiki.modulePathOpt = mwModulePath
MediaWiki.domain = mwDomain
MediaWiki.password = mwPassword
MediaWiki.username = mwUsername
MediaWiki.wikiPath = mwWikiPath

/* Download helpers; TODO: Merge with something else / expand this. */
const downloader = new Downloader({
Expand Down Expand Up @@ -513,7 +513,7 @@ async function execute(argv: any) {
}
}

const apiUrlDirector = new ApiURLDirector(MediaWiki.apiUrl.href)
const apiUrlDirector = new ApiURLDirector(MediaWiki.actionApiUrl.href)

const body = await downloader.getJSON<any>(apiUrlDirector.buildSiteInfoURL())

Expand Down
4 changes: 2 additions & 2 deletions src/parameterList.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ export const parameterDescriptions = {
'Specify a flavour for the scraping. If missing, scrape all article contents. Each --format argument will cause a new local file to be created but options can be combined. Supported options are:\n * novid: no video & audio content\n * nopic: no pictures (implies "novid")\n * nopdf: no PDF files\n * nodet: only the first/head paragraph (implies "novid")\nFormat names can also be aliased using a ":"\nExample: "... --format=nopic:mini --format=novid,nopdf"',
keepEmptyParagraphs: 'Keep all paragraphs, even empty ones.',
mwWikiPath: 'Mediawiki wiki base path (per default "/wiki/")',
mwActionApiPath: 'Mediawiki action API path (per default "/w/api.php")',
mwApiPath: 'Mediawiki Rest API path (per default "/api/rest_v1")',
mwActionApiPath: 'Mediawiki API path (per default "/w/api.php")',
mwRestApiPath: 'Mediawiki Rest API path (per default "/api/rest_v1")',
mwModulePath: 'Mediawiki module load path (per default "/w/load.php")',
mwDomain: 'Mediawiki user domain (thought for private wikis)',
mwUsername: 'Mediawiki username (thought for private wikis)',
Expand Down
49 changes: 27 additions & 22 deletions src/renderers/renderer.builder.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,30 +47,35 @@ export class RendererBuilder {
}
case 'specific':
// renderName argument is required for 'specific' mode
switch (renderName) {
case 'WikimediaDesktop':
if (hasWikimediaDesktopApi) {
return new WikimediaDesktopRenderer()
}
logger.error('Cannot create an instance of WikimediaDesktop renderer.')
process.exit(1)
case 'VisualEditor':
if (hasVisualEditorApi) {
return new VisualEditorRenderer()
}
logger.error('Cannot create an instance of VisualEditor renderer.')
process.exit(1)
case 'WikimediaMobile':
if (hasWikimediaMobileApi) {
return new WikimediaMobileRenderer()
}
logger.error('No available mobile renderer.')
process.exit(1)
default:
throw new Error(`Unknown renderName for specific mode: ${renderName}`)
}
return this.handleSpecificRender(renderName, hasVisualEditorApi, hasWikimediaDesktopApi, hasWikimediaMobileApi)
default:
throw new Error(`Unknown render: ${renderType}`)
}
}

private handleSpecificRender(renderName: string, hasVisualEditorApi: boolean, hasWikimediaDesktopApi: boolean, hasWikimediaMobileApi: boolean) {
// renderName argument is required for 'specific' mode
switch (renderName) {
case 'WikimediaDesktop':
if (hasWikimediaDesktopApi) {
return new WikimediaDesktopRenderer()
}
logger.error('Cannot create an instance of WikimediaDesktop renderer.')
process.exit(1)
case 'VisualEditor':
if (hasVisualEditorApi) {
return new VisualEditorRenderer()
}
logger.error('Cannot create an instance of VisualEditor renderer.')
process.exit(1)
case 'WikimediaMobile':
if (hasWikimediaMobileApi) {
return new WikimediaMobileRenderer()
}
logger.error('No available mobile renderer.')
process.exit(1)
default:
throw new Error(`Unknown renderName for specific mode: ${renderName}`)
}
}
}
40 changes: 40 additions & 0 deletions src/sanitize-argument.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ export async function sanitize_all(argv: any) {
customZimLongDescription,
customZimDescription,
forceRender,
mwWikiPath,
mwActionApiPath,
mwRestApiPath,
mwModulePath,
} = argv

sanitizeDoubleUsedParameters(argv)
Expand Down Expand Up @@ -78,6 +82,18 @@ export async function sanitize_all(argv: any) {
throw err
})

// sanitizing mwWikiPath
sanitizeWikiPath(mwWikiPath)

// sanitizing mwRestApiPath
sanitizeApiPathParam(mwRestApiPath)

// sanitizing mwActionApiPath
sanitizeApiPathParam(mwActionApiPath)

// sanitizing mwModulePath
sanitizeApiPathParam(mwModulePath)

// sanitize Custom Main Page
if (argv.customMainPage) {
argv.customMainPage = argv.customMainPage.replace(/ /g, '_')
Expand All @@ -103,6 +119,30 @@ export async function sanitize_all(argv: any) {
}
}

export function sanitizeWikiPath(mwWikiPath = '') {
mwWikiPath = sanitizeApiPathParam(mwWikiPath)

// Make sure wikiPath always has forward slash at the end for the correct URL building
if (!mwWikiPath?.endsWith('/')) {
mwWikiPath += '/'
}

return mwWikiPath
}

export function sanitizeApiPathParam(apiPathParam: string) {
if (!apiPathParam) {
return
}

// No api params should start from forward slash
if (apiPathParam.startsWith('/')) {
apiPathParam = apiPathParam.slice(1)
}

return apiPathParam
}

export function sanitizeStringMaxLength(text: string, key: string, length: number) {
if (text && text.length > length) {
throw new Error(`${key} should be less than ${length} characters.`)
Expand Down
Loading

0 comments on commit 99da4fd

Please sign in to comment.