Skip to content

Commit

Permalink
cherry-pick(#11006): add browser like UA to browser fetcher (#11395)
Browse files Browse the repository at this point in the history 
Drive-by: unify all Playwright user agents across the board.

Co-authored-by: Andrey Lushnikov <lushnikov@chromium.org>

Co-authored-by: Max Schmitt <max@schmitt.mx>
  • Loading branch information
@aslushnikov @mxschmitt
aslushnikov and mxschmitt authored Jan 14, 2022
1 parent 12edb70 commit dd02142
Show file tree
Hide file tree
Showing 5 changed files with 99 additions and 21 deletions.
4 changes: 2 additions & 2 deletions packages/playwright-core/src/server/fetch.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import zlib from 'zlib';
import { HTTPCredentials } from '../../types/types';
import * as channels from '../protocol/channels';
import { TimeoutSettings } from '../utils/timeoutSettings';
import { assert, createGuid, getPlaywrightVersion, monotonicTime } from '../utils/utils';
import { assert, createGuid, getUserAgent, monotonicTime } from '../utils/utils';
import { BrowserContext } from './browserContext';
import { CookieStore, domainMatches } from './cookieStore';
import { MultipartFormData } from './formData';
Expand Down Expand Up @@ -457,7 +457,7 @@ export class GlobalAPIRequestContext extends APIRequestContext {
}
this._options = {
baseURL: options.baseURL,
userAgent: options.userAgent || `Playwright/${getPlaywrightVersion()}`,
userAgent: options.userAgent || getUserAgent(),
extraHTTPHeaders: options.extraHTTPHeaders,
ignoreHTTPSErrors: !!options.ignoreHTTPSErrors,
httpCredentials: options.httpCredentials,
Expand Down
5 changes: 3 additions & 2 deletions packages/playwright-core/src/utils/browserFetcher.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import extract from 'extract-zip';
import fs from 'fs';
import os from 'os';
import path from 'path';
import { existsAsync, download } from './utils';
import { existsAsync, download, getUserAgent } from './utils';
import { debugLogger } from './debugLogger';

export async function downloadBrowserWithProgressBar(title: string, browserDirectory: string, executablePath: string, downloadURL: string, downloadFileName: string): Promise<boolean> {
Expand All @@ -35,7 +35,8 @@ export async function downloadBrowserWithProgressBar(title: string, browserDirec
try {
await download(url, zipPath, {
progressBarName,
log: debugLogger.log.bind(debugLogger, 'install')
log: debugLogger.log.bind(debugLogger, 'install'),
userAgent: getUserAgent(),
});
debugLogger.log('install', `extracting archive`);
debugLogger.log('install', `-- zip: ${zipPath}`);
Expand Down
11 changes: 8 additions & 3 deletions packages/playwright-core/src/utils/ubuntuVersion.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ function getUbuntuVersionSyncInternal(): string {
}
}

function parseUbuntuVersion(osReleaseText: string): string {
export function parseOSReleaseText(osReleaseText: string): Map<string, string> {
const fields = new Map();
for (const line of osReleaseText.split('\n')) {
const tokens = line.split('=');
Expand All @@ -72,11 +72,16 @@ function parseUbuntuVersion(osReleaseText: string): string {
continue;
fields.set(name.toLowerCase(), value);
}
return fields;
}

function parseUbuntuVersion(osReleaseText: string): string {
const fields = parseOSReleaseText(osReleaseText);
// For Linux mint
if (fields.get('distrib_id') && fields.get('distrib_id').toLowerCase() === 'ubuntu')
if (fields.get('distrib_id') && fields.get('distrib_id')?.toLowerCase() === 'ubuntu')
return fields.get('distrib_release') || '';

if (!fields.get('name') || fields.get('name').toLowerCase() !== 'ubuntu')
if (!fields.get('name') || fields.get('name')?.toLowerCase() !== 'ubuntu')
return '';
return fields.get('version_id') || '';
}
85 changes: 73 additions & 12 deletions packages/playwright-core/src/utils/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,10 @@ import * as crypto from 'crypto';
import os from 'os';
import http from 'http';
import https from 'https';
import { spawn, SpawnOptions } from 'child_process';
import { spawn, SpawnOptions, execSync } from 'child_process';
import { getProxyForUrl } from 'proxy-from-env';
import * as URL from 'url';
import { getUbuntuVersionSync } from './ubuntuVersion';
import { getUbuntuVersionSync, parseOSReleaseText } from './ubuntuVersion';
import { NameValue } from '../protocol/channels';
import ProgressBar from 'progress';

Expand Down Expand Up @@ -115,8 +115,13 @@ export function fetchData(params: HTTPRequestParams, onError?: (params: HTTPRequ

type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void;
type DownloadFileLogger = (message: string) => void;
type DownloadFileOptions = {
progressCallback?: OnProgressCallback,
log?: DownloadFileLogger,
userAgent?: string
};

function downloadFile(url: string, destinationPath: string, options: {progressCallback?: OnProgressCallback, log?: DownloadFileLogger} = {}): Promise<{error: any}> {
function downloadFile(url: string, destinationPath: string, options: DownloadFileOptions = {}): Promise<{error: any}> {
const {
progressCallback,
log = () => {},
Expand All @@ -130,7 +135,12 @@ function downloadFile(url: string, destinationPath: string, options: {progressCa

const promise: Promise<{error: any}> = new Promise(x => { fulfill = x; });

httpRequest({ url }, response => {
httpRequest({
url,
headers: options.userAgent ? {
'User-Agent': options.userAgent,
} : undefined,
}, response => {
log(`-- response status code: ${response.statusCode}`);
if (response.statusCode !== 200) {
const error = new Error(`Download failed: server returned code ${response.statusCode}. URL: ${url}`);
Expand All @@ -156,23 +166,27 @@ function downloadFile(url: string, destinationPath: string, options: {progressCa
}
}

type DownloadOptions = {
progressBarName?: string,
retryCount?: number
log?: DownloadFileLogger
userAgent?: string
};

export async function download(
url: string,
destination: string,
options: {
progressBarName?: string,
retryCount?: number
log?: DownloadFileLogger
} = {}
options: DownloadOptions = {}
) {
const { progressBarName = 'file', retryCount = 3, log = () => {} } = options;
const { progressBarName = 'file', retryCount = 3, log = () => {}, userAgent } = options;
for (let attempt = 1; attempt <= retryCount; ++attempt) {
log(
`downloading ${progressBarName} - attempt #${attempt}`
);
const { error } = await downloadFile(url, destination, {
progressCallback: getDownloadProgress(progressBarName),
log,
userAgent,
});
if (!error) {
log(`SUCCESS downloading ${progressBarName}`);
Expand Down Expand Up @@ -421,8 +435,55 @@ export function canAccessFile(file: string) {
}
}

export function getUserAgent() {
return `Playwright/${getPlaywrightVersion()} (${os.arch()}/${os.platform()}/${os.release()})`;
let cachedUserAgent: string | undefined;
export function getUserAgent(): string {
if (cachedUserAgent)
return cachedUserAgent;
try {
cachedUserAgent = determineUserAgent();
} catch (e) {
cachedUserAgent = 'Playwright/unknown';
}
return cachedUserAgent;
}

function determineUserAgent(): string {
let osIdentifier = 'unknown';
let osVersion = 'unknown';
if (process.platform === 'win32') {
const version = os.release().split('.');
osIdentifier = 'windows';
osVersion = `${version[0]}.${version[1]}`;
} else if (process.platform === 'darwin') {
const version = execSync('sw_vers -productVersion').toString().trim().split('.');
osIdentifier = 'macOS';
osVersion = `${version[0]}.${version[1]}`;
} else if (process.platform === 'linux') {
try {
// List of /etc/os-release values for different distributions could be
// found here: https://gist.github.com/aslushnikov/8ceddb8288e4cf9db3039c02e0f4fb75
const osReleaseText = fs.readFileSync('/etc/os-release', 'utf8');
const fields = parseOSReleaseText(osReleaseText);
osIdentifier = fields.get('id') || 'unknown';
osVersion = fields.get('version_id') || 'unknown';
} catch (e) {
// Linux distribution without /etc/os-release.
// Default to linux/unknown.
osIdentifier = 'linux';
}
}

let langName = 'unknown';
let langVersion = 'unknown';
if (!process.env.PW_CLI_TARGET_LANG) {
langName = 'node';
langVersion = process.version.substring(1).split('.').slice(0, 2).join('.');
} else if (['node', 'python', 'java', 'csharp'].includes(process.env.PW_CLI_TARGET_LANG)) {
langName = process.env.PW_CLI_TARGET_LANG;
langVersion = process.env.PW_CLI_TARGET_LANG_VERSION ?? 'unknown';
}

return `Playwright/${getPlaywrightVersion()} (${os.arch()}; ${osIdentifier} ${osVersion}) ${langName}/${langVersion}`;
}

export function getPlaywrightVersion(majorMinorOnly = false) {
Expand Down
15 changes: 13 additions & 2 deletions tests/global-fetch.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/

import http from 'http';
import os from 'os';
import * as util from 'util';
import { getPlaywrightVersion } from 'playwright-core/lib/utils/utils';
import { expect, playwrightTest as it } from './config/browserTest';
Expand Down Expand Up @@ -177,13 +178,23 @@ it('should resolve url relative to gobal baseURL option', async ({ playwright, s
expect(response.url()).toBe(server.EMPTY_PAGE);
});

it('should set playwright as user-agent', async ({ playwright, server }) => {
it('should set playwright as user-agent', async ({ playwright, server, isWindows, isLinux, isMac }) => {
const request = await playwright.request.newContext();
const [serverRequest] = await Promise.all([
server.waitForRequest('/empty.html'),
request.get(server.EMPTY_PAGE)
]);
expect(serverRequest.headers['user-agent']).toBe('Playwright/' + getPlaywrightVersion());
const userAgentMasked = serverRequest.headers['user-agent']
.replace(os.arch(), '<ARCH>')
.replace(getPlaywrightVersion(), 'X.X.X')
.replace(/\d+/g, 'X');

if (isWindows)
expect(userAgentMasked).toBe('Playwright/X.X.X (<ARCH>; windows X.X) node/X.X');
else if (isLinux)
expect(userAgentMasked).toBe('Playwright/X.X.X (<ARCH>; ubuntu X.X) node/X.X');
else if (isMac)
expect(userAgentMasked).toBe('Playwright/X.X.X (<ARCH>; macOS X.X) node/X.X');
});

it('should be able to construct with context options', async ({ playwright, browserType, server }) => {
Expand Down

0 comments on commit dd02142

Please sign in to comment.