Add support for request options

.github/workflows/build.yml

@@ -5,15 +5,15 @@ on: [push]
 jobs:
   build:
     name: "Deno tests and build npm files"
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
 
       - name: Setup Deno
-        uses: denoland/setup-deno@v1.1.1
+        uses: denoland/setup-deno@v2
         with:
-          deno-version: v1.x
+          deno-version: v2.x
 
       - name: Check formatting
         run: deno fmt --check
@@ -27,10 +27,10 @@ jobs:
         run: deno task test:ci
 
       - name: Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
-          node-version: '18.x'  # Build files using a fixed node version
-          registry-url: 'https://registry.npmjs.org'
+          node-version: "22.x" # Build files using a fixed node version
+          registry-url: "https://registry.npmjs.org"
 
       - name: Build npm files
         run: deno task npm
@@ -39,7 +39,7 @@ jobs:
         run: zip npm.zip ./npm -r
 
       - name: Upload build files for smoke tests
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: npm
           path: npm.zip
@@ -48,10 +48,24 @@ jobs:
   smoke-tests-commonjs:
     name: "Smoke tests (CommonJS)"
     needs: build
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [7.x, 8.x, 9.x, 10.x, 11.x, 12.x, 13.x, 14.x, 15.x, 16.x, 17.x, 18.x, 19.x]
+        node-version: [
+          7.x,
+          8.x,
+          9.x,
+          10.x,
+          11.x,
+          12.x,
+          13.x,
+          14.x,
+          15.x,
+          16.x,
+          17.x,
+          18.x,
+          19.x,
+        ]
         include:
           - command: test
           - command: test:use-openssl-ca
@@ -63,16 +77,16 @@ jobs:
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
-          registry-url: 'https://registry.npmjs.org'
+          registry-url: "https://registry.npmjs.org"
 
       - name: Download build files
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: npm
 
@@ -90,22 +104,22 @@ jobs:
   smoke-tests-esm:
     name: "Smoke tests (ESM)"
     needs: build
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         node-version: [14.x, 15.x, 16.x, 17.x, 18.x, 19.x]
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
-          registry-url: 'https://registry.npmjs.org'
+          registry-url: "https://registry.npmjs.org"
 
       - name: Download build files
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: npm
 

.github/workflows/release.yml

@@ -5,19 +5,19 @@ on:
     branches:
       - master
     paths:
-      - 'version.ts'
+      - "version.ts"
 
 jobs:
   release:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Deno
-        uses: denoland/setup-deno@v1.1.1
+        uses: denoland/setup-deno@v2
         with:
-          deno-version: v1.x
+          deno-version: v2.x
 
       - name: Check formatting
         run: deno fmt --check
@@ -31,10 +31,10 @@ jobs:
         run: deno task test:ci
 
       - name: Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
-          node-version: '18.x'
-          registry-url: 'https://registry.npmjs.org'
+          node-version: "22.x"
+          registry-url: "https://registry.npmjs.org"
 
       - name: Build npm files
         run: deno task npm

README.md

@@ -114,6 +114,39 @@ await getJson({ engine: "google", q: "coffee" }); // uses the API key defined in
 await getJson({ engine: "google", api_key: API_KEY_2, q: "coffee" }); // API_KEY_2 will be used
 ```
 
+### Using a Proxy
+
+You can use a proxy by passing `requestOptions` with an `HttpsProxyAgent`
+instance. This can be done either globally through the config object or
+per-request in the parameters.
+
+First, install the required package:
+
+```bash
+npm install https-proxy-agent
+```
+
+Then use it in your code:
+
+```js
+import { config, getJson } from "serpapi";
+import { HttpsProxyAgent } from "https-proxy-agent";
+
+// Global configuration
+config.requestOptions = {
+  agent: new HttpsProxyAgent("http://proxy-server:port"),
+};
+
+// Or per-request configuration
+await getJson({
+  engine: "google",
+  q: "coffee",
+  requestOptions: {
+    agent: new HttpsProxyAgent("http://proxy-server:port"),
+  },
+});
+```
+
 ## Pagination
 
 Built-in pagination is not supported. Please refer to our pagination examples

deno.json

@@ -8,9 +8,7 @@
     "npm": "deno run -A scripts/build_npm.ts"
   },
   "fmt": {
-    "files": {
-      "exclude": ["npm/", "examples/node", "smoke_tests/"]
-    }
+    "exclude": ["npm/", "examples/node", "smoke_tests/"]
   },
   "lint": {
     "files": {

examples/node/js_node_14_up/basic_example.js

@@ -7,6 +7,7 @@
  */
 
 import * as Dotenv from "dotenv";
+import process from "process";
 import { config, getJson } from "serpapi";
 
 Dotenv.config();

examples/node/js_node_14_up/pagination_example.js

@@ -7,6 +7,7 @@
  */
 
 import * as Dotenv from "dotenv";
+import process from "process";
 import { config, getJson } from "serpapi";
 
 Dotenv.config();

examples/node/js_node_7_up/basic_example.js

@@ -3,6 +3,7 @@
  */
 
 const Dotenv = require("dotenv");
+const process = require("process");
 const { config, getJson } = require("serpapi");
 
 Dotenv.config();

examples/node/js_node_7_up/pagination_example.js

@@ -6,6 +6,7 @@ const Dotenv = require("dotenv");
 const { config, getJson } = require("serpapi");
 const url = require("url");
 const qs = require("querystring");
+const process = require("process");
 
 Dotenv.config();
 config.api_key = process.env.API_KEY;

scripts/build_npm.ts

@@ -1,4 +1,4 @@
-import { build, emptyDir } from "https://deno.land/x/dnt@0.37.0/mod.ts";
+import { build, emptyDir } from "https://deno.land/x/dnt@0.40.0/mod.ts";
 import { version } from "../version.ts";
 
 await emptyDir("./npm");

smoke_tests/commonjs/commonjs.js

@@ -3,6 +3,7 @@
  */
 
 const Dotenv = require("dotenv");
+const process = require("process");
 const {
   config,
   getJson,

smoke_tests/esm/esm.js

@@ -7,6 +7,7 @@
  */
 
 import Dotenv from "dotenv";
+import process from "process";
 import {
   config,
   getAccount,

src/config.ts

@@ -1,7 +1,11 @@
+import http from "node:http";
+
 export type Config = {
   api_key: string | null;
   timeout: number;
+  requestOptions?: http.RequestOptions;
 };
+
 export const config: Config = {
   api_key: null,
   timeout: 60000,

src/utils.ts

@@ -1,7 +1,10 @@
 import { version } from "../version.ts";
 import https from "node:https";
+import http from "node:http";
 import qs from "node:querystring";
 import { RequestTimeoutError } from "./errors.ts";
+import { config } from "./config.ts";
+import process from "node:process";
 
 /**
  * This `_internals` object is needed to support stubbing/spying of
@@ -12,12 +15,15 @@ import { RequestTimeoutError } from "./errors.ts";
  */
 export const _internals = {
   execute: execute,
-  getBaseUrl: getBaseUrl,
+  getHostnameAndPort: getHostnameAndPort,
 };
 
 /** Facilitates stubbing in tests, e.g. localhost as the base url */
-function getBaseUrl() {
-  return "https://serpapi.com";
+function getHostnameAndPort() {
+  return {
+    hostname: "serpapi.com",
+    port: 443,
+  };
 }
 
 export function getSource() {
@@ -27,9 +33,7 @@ export function getSource() {
     if (denoVersion) {
       return `deno@${denoVersion},${moduleSource}`;
     }
-    // @ts-ignore: scope of nodejs
   } else if (typeof process == "object") {
-    // @ts-ignore: scope of nodejs
     const nodeVersion = process.versions?.node;
     if (nodeVersion) {
       return `nodejs@${nodeVersion},${moduleSource}`;
@@ -38,40 +42,56 @@ export function getSource() {
   return `nodejs,${moduleSource}`;
 }
 
-export function buildUrl(
+export function buildRequestOptions(
   path: string,
   parameters: qs.ParsedUrlQueryInput,
-): string {
+): http.RequestOptions {
   const clonedParams = { ...parameters };
   for (const k in clonedParams) {
-    if (clonedParams[k] === undefined) {
+    if (
+      k === "requestOptions" ||
+      k === "timeout" ||
+      clonedParams[k] === undefined
+    ) {
       delete clonedParams[k];
     }
   }
-  return `${_internals.getBaseUrl()}${path}?${qs.stringify(clonedParams)}`;
+  const basicOptions = {
+    ..._internals.getHostnameAndPort(),
+    path: `${path}?${qs.stringify(clonedParams)}`,
+    method: "GET",
+  };
+
+  return {
+    ...config.requestOptions,
+    ...(parameters.requestOptions as http.RequestOptions),
+    ...basicOptions,
+  };
 }
 
 export function execute(
   path: string,
   parameters: qs.ParsedUrlQueryInput,
   timeout: number,
 ): Promise<string> {
-  const url = buildUrl(path, {
+  const options = buildRequestOptions(path, {
     ...parameters,
     source: getSource(),
   });
+
   return new Promise((resolve, reject) => {
     let timer: number;
-    const req = https.get(url, (resp) => {
+
+    const handleResponse = (resp: http.IncomingMessage) => {
       resp.setEncoding("utf8");
       let data = "";
 
-      // A chunk of data has been recieved.
+      // A chunk of data has been received
       resp.on("data", (chunk) => {
         data += chunk;
       });
 
-      // The whole response has been received. Print out the result.
+      // The whole response has been received
       resp.on("end", () => {
         try {
           if (resp.statusCode == 200) {
@@ -85,10 +105,14 @@ export function execute(
           if (timer) clearTimeout(timer);
         }
       });
-    }).on("error", (err) => {
+    };
+
+    const handleError = (err: Error) => {
       reject(err);
       if (timer) clearTimeout(timer);
-    });
+    };
+
+    const req = https.get(options, handleResponse).on("error", handleError);
 
     if (timeout > 0) {
       timer = setTimeout(() => {