Skip to content

Commit f56e69c

Browse files
GranitosaurusGranitosaurus
committed
update example for each platform
1 parent 41ba5a2 commit f56e69c

File tree

12 files changed

+594
-19
lines changed

12 files changed

+594
-19
lines changed

‎examples/README.md

Lines changed: 6 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,9 @@
11
# Scrapfly Typescript SDK Examples
22

3-
This directory contains commonly used examples for the Scrapfly Typescript SDK which is available in Typescript runtimes (bun, deno) as well as javascript ones like Nodejs.
3+
In this directory, examples are provided by JavaScript runtime platform as Scrapfly's Typescript SDK supports multiple options:
44

5-
You can use `node` to run the `.js` examples:
6-
7-
```
8-
node examples/basic-get.js
9-
```
10-
11-
Or compile `.ts` examples to `.js`:
12-
13-
```
14-
tsc examples/scrape/basic-get.ts -o examples/basic-get.js
15-
node examples/scrape/basic-get.js
16-
```
17-
18-
Or run typescript directly through runtimes like `.ts`:
19-
20-
```
21-
bun examples/scrape/basic-get.ts
22-
```
5+
- [node_commonjs](./node_commonjs) - Examples for NodeJS with CommonJS module system.
6+
- [node_esm](./node_esm) - Examples for NodeJS with ECMAScript Modules (ESM) module system.
7+
- [bun](./bun) - Examples for Bun Typescript runtime.
8+
- [deno](./deno) - Examples for Deno Typescript runtime.
9+
- [serverless: cloudflare workers, aws lambda, supabase](./cloudflareworker_awslambda_supabase) - Examples for serverless platforms like Cloudflare Workers, AWS Lambda and Supabase.

‎examples/bun/README.md

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Scrapfly SDK with Bun
2+
3+
Bun is a modern javascript runtime that can execute both javascript and typescript without transpilation.
4+
5+
These examples demonstrate Typescript SDK usage with Bun and for that install the SDK using jsr.io which distributes Typescript files:
6+
7+
```
8+
$ bunx jsr add @scrapfly/scrapfly-sdk
9+
```
10+
11+
Then see `bun_examples.ts` for examples and each example can be run by specifying the example function name:
12+
13+
```bash
14+
$ bun run bun_examples.ts <example_name> <scrapfly_api_key>
15+
# for example
16+
$ bun run bun_examples.ts basicGet scp-test-123
17+
```

‎examples/bun/bun_examples.ts

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
import { ScrapflyClient, ScrapeConfig, ScreenshotConfig, ExtractionConfig } from 'scrapfly-sdk';
2+
3+
4+
/* To start, you can always get your account information using the .account() method
5+
*/
6+
export async function getAccount(apiKey: string) {
7+
const client = new ScrapflyClient({ key: apiKey});
8+
const account = await client.account();
9+
console.log('account');
10+
console.log(account);
11+
}
12+
13+
/* For a basic scrape the only required parameter is the URL
14+
*/
15+
export async function basicGet(apiKey: string) {
16+
const client = new ScrapflyClient({ key: apiKey});
17+
18+
let scrape_result = await client.scrape(
19+
new ScrapeConfig({
20+
url: 'https://httpbin.dev/html',
21+
// Anti Scraping Protection bypass - enable this when scraping protected targets
22+
asp: true,
23+
// server side cache - great for repeated requests
24+
cache: true,
25+
cache_ttl: 3600, // in seconds
26+
// cache_clear: true, // you can always clear the cache explicitly!
27+
}),
28+
);
29+
30+
// the scrape_result.result contains all result details
31+
console.log("web log url:"); // you can check web UI for request details:
32+
console.log(scrape_result.result.log_url);
33+
34+
console.log("page content:");
35+
console.log(scrape_result.result.content);
36+
37+
console.log("response headers:");
38+
console.log(scrape_result.result.response_headers);
39+
40+
console.log("response cookies:");
41+
console.log(scrape_result.result.cookies);
42+
}
43+
44+
/* Enabling js_render enabled scrapfly cloud browsers and enables
45+
* a bunch of other features like browser control, js execution, screenshots, etc.
46+
*/
47+
export async function JSRender(apiKey: string) {
48+
const client = new ScrapflyClient({ key: apiKey});
49+
50+
let scrape_result = await client.scrape(
51+
new ScrapeConfig({
52+
url: 'https://web-scraping.dev/product/1',
53+
// enable browsers:
54+
render_js: true,
55+
// this enables more options
56+
// you can wait for some element to appear:
57+
wait_for_selector: '.review',
58+
// you can wait explicitly for N seconds
59+
rendering_wait: 3000, // 3 seconds
60+
// you can control the browser through scenarios:
61+
// https://scrapfly.io/docs/scrape-api/javascript-scenario
62+
js_scenario: [
63+
{ click: { selector: '#load-more-reviews' }},
64+
{ wait: 2000},
65+
],
66+
// or even run any custom JS code!
67+
js: 'return document.querySelector(".review").innerText',
68+
}),
69+
);
70+
71+
// the scrape_result.result contains all result details:
72+
console.log("web log url:"); // you can check web UI for request details:
73+
console.log(scrape_result.result.log_url);
74+
75+
console.log("page content:");
76+
console.log(scrape_result.result.content.substring(0, 1000) + '...');
77+
78+
console.log("browser data capture");
79+
console.log(scrape_result.result.browser_data);
80+
}
81+
82+
83+
// CLI entry point
84+
async function main(): Promise<void> {
85+
if (process.argv.length < 4) {
86+
console.log(
87+
`Usage: bun run bun_examples.ts <functionName> <apiKey>\n` +
88+
`getAccount - Get account information\n` +
89+
`basicGet - Basic scrape\n` +
90+
`JSRender - Scrape with JS rendering\n`
91+
);
92+
return;
93+
}
94+
const args = process.argv.slice(2);
95+
const functionName = args[0];
96+
const apiKey = args[1];
97+
98+
// Dynamically import the current module
99+
const module = await import('./bun_examples.ts');
100+
101+
if (module[functionName]) {
102+
(module[functionName] as Function)(apiKey);
103+
} else {
104+
console.log(`Function ${functionName} not found.`);
105+
}
106+
}
107+
108+
// Check if the script is being run directly
109+
if (import.meta.url === `file://${process.argv[1]}`) {
110+
main();
111+
}

‎examples/cloudflareworker_awslambda_supabase/.denoflare

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"$schema": "https://raw.githubusercontent.com/skymethod/denoflare/v0.6.0/common/config.schema.json",
3+
"scripts": {
4+
"example": {
5+
"path": "index.ts",
6+
"localPort": 3030
7+
}
8+
},
9+
"profiles": {
10+
}
11+
}

‎examples/cloudflareworker_awslambda_supabase/README.md

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Scrapfly SDK with Cloudflare Workers or AWS Lambda or Supabase
2+
3+
There are many serverless platforms that can run Javascript or Typescript code which fit well wtih Scrapfly.io. This directory contains examples for Cloudflare Workers, AWS Lambda and Supabase.
4+
5+
The easiest way is to use [Denoflare](https://denoflare.dev/) which allows to easily develop, test and deploy Typescript workers to Cloudflare Workers, AWS Lambda or Supabase.
6+
7+
To start, install denoflare:
8+
9+
```bash
10+
deno install --unstable-worker-options --allow-read --allow-net --allow-env --allow-run --name denoflare --force https://github.com/skymethod/denoflare/blob/master/cli/cli.ts
11+
```
12+
13+
Then see `index.ts` for your worker code and configure the `key` with your Scrapfly API key.
14+
15+
To test your worker locally use the `serve` command:
16+
17+
```bash
18+
denoflare serve example --bundle backend=esbuild
19+
```
20+
21+
Finally, to push your worker use one of the `push-` commands:
22+
23+
```bash
24+
# for cloudflare
25+
denoflare push-deploy --bundle backend=esbuild
26+
# for aws lambda
27+
denoflare push-lambda --bundle backend=esbuild
28+
# for supabase
29+
denoflare push-supabase --bundle backend=esbuild
30+
```
31+
32+
for more see https://denoflare.dev/

‎examples/cloudflareworker_awslambda_supabase/index.ts

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import { ScrapflyClient, ScrapeConfig, ScreenshotConfig, ExtractionConfig } from 'jsr:@scrapfly/scrapfly-sdk';
2+
3+
const key = 'YOUR SCRAPFLY KEY';
4+
const client = new ScrapflyClient({ key });
5+
6+
export default {
7+
async fetch(request: Request, env: any) {
8+
try {
9+
const result = await client.scrape(
10+
new ScrapeConfig({
11+
url: 'https://web-scraping.dev/product/1',
12+
}),
13+
);
14+
const data = {
15+
"url": "https://web-scraping.dev/product/1",
16+
"price": result.selector(".product-price").text(),
17+
"title": result.selector(".product-title").text(),
18+
"description": result.selector(".product-description").text(),
19+
};
20+
return new Response(JSON.stringify(data), {
21+
headers: {
22+
"content-type": "application/json",
23+
}
24+
})
25+
} catch (e) {
26+
return new Response(JSON.stringify({"error": e.message}), {
27+
headers: {
28+
"content-type": "application/json",
29+
}
30+
})
31+
}
32+
},
33+
}

‎examples/deno/README.md

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Scrapfly SDK with Deno
2+
3+
Deno is a modern and secure runtime for JavaScript and TypeScript that uses V8 and is built in Rust.
4+
5+
6+
These examples demonstrate Typescript SDK usage with Deno and for that install the SDK using jsr.io which distributes Typescript files:
7+
8+
```bash
9+
deno add jsr:@scrapfly/scrapfly-sdk
10+
```
11+
12+
Then see `deno_examples.ts` for examples and each example can be run by specifying the example function name:
13+
14+
```bash
15+
$ deno run --allow-net -A deno_examples.ts <example_name> <scrapfly_api_key>
16+
# for example
17+
$ deno run --allow-net -A deno_examples.ts basicGet scp-test-123
18+
```
19+

‎examples/deno/deno_examples.ts

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
import { ScrapflyClient, ScrapeConfig, ScreenshotConfig, ExtractionConfig } from 'jsr:@scrapfly/scrapfly-sdk';
2+
3+
4+
/* To start, you can always get your account information using the .account() method
5+
*/
6+
export async function getAccount(apiKey: string) {
7+
const client = new ScrapflyClient({ key: apiKey});
8+
const account = await client.account();
9+
console.log('account');
10+
console.log(account);
11+
}
12+
13+
/* For a basic scrape the only required parameter is the URL
14+
*/
15+
export async function basicGet(apiKey: string) {
16+
const client = new ScrapflyClient({ key: apiKey});
17+
18+
let scrape_result = await client.scrape(
19+
new ScrapeConfig({
20+
url: 'https://httpbin.dev/html',
21+
// Anti Scraping Protection bypass - enable this when scraping protected targets
22+
asp: true,
23+
// server side cache - great for repeated requests
24+
cache: true,
25+
cache_ttl: 3600, // in seconds
26+
// cache_clear: true, // you can always clear the cache explicitly!
27+
}),
28+
);
29+
30+
// the scrape_result.result contains all result details
31+
console.log("web log url:"); // you can check web UI for request details:
32+
console.log(scrape_result.result.log_url);
33+
34+
console.log("page content:");
35+
console.log(scrape_result.result.content);
36+
37+
console.log("response headers:");
38+
console.log(scrape_result.result.response_headers);
39+
40+
console.log("response cookies:");
41+
console.log(scrape_result.result.cookies);
42+
}
43+
44+
/* Enabling js_render enabled scrapfly cloud browsers and enables
45+
* a bunch of other features like browser control, js execution, screenshots, etc.
46+
*/
47+
export async function JSRender(apiKey: string) {
48+
const client = new ScrapflyClient({ key: apiKey});
49+
50+
let scrape_result = await client.scrape(
51+
new ScrapeConfig({
52+
url: 'https://web-scraping.dev/product/1',
53+
// enable browsers:
54+
render_js: true,
55+
// this enables more options
56+
// you can wait for some element to appear:
57+
wait_for_selector: '.review',
58+
// you can wait explicitly for N seconds
59+
rendering_wait: 3000, // 3 seconds
60+
// you can control the browser through scenarios:
61+
// https://scrapfly.io/docs/scrape-api/javascript-scenario
62+
js_scenario: [
63+
{ click: { selector: '#load-more-reviews' }},
64+
{ wait: 2000},
65+
],
66+
// or even run any custom JS code!
67+
js: 'return document.querySelector(".review").innerText',
68+
}),
69+
);
70+
71+
// the scrape_result.result contains all result details:
72+
console.log("web log url:"); // you can check web UI for request details:
73+
console.log(scrape_result.result.log_url);
74+
75+
console.log("page content:");
76+
console.log(scrape_result.result.content.substring(0, 1000) + '...');
77+
78+
console.log("browser data capture");
79+
console.log(scrape_result.result.browser_data);
80+
}
81+
82+
// CLI entry point
83+
async function main(): Promise<void> {
84+
if (Deno.args.length < 2) {
85+
console.log(
86+
`Usage: deno run --allow-net deno_examples.ts <functionName> <apiKey>\n` +
87+
`getAccount - Get account information\n` +
88+
`basicGet - Basic scrape\n` +
89+
`JSRender - Scrape with JS rendering\n`
90+
);
91+
return;
92+
}
93+
const [functionName, apiKey] = Deno.args;
94+
95+
// Dynamically import the current module
96+
const module = await import('./deno_examples.ts');
97+
98+
if (module[functionName]) {
99+
(module[functionName] as Function)(apiKey);
100+
} else {
101+
console.log(`Function ${functionName} not found.`);
102+
}
103+
}
104+
105+
// Check if the script is being run directly
106+
if (import.meta.main) {
107+
main();
108+
}

‎examples/node_commonjs/README.md

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Scrapfly SDK with CommonJS (NodeJS)
2+
3+
CommonJS is a NodeJS module system that allows you to import modules using `require` function.
4+
5+
The examples in `commonjs_examples.cjs` demonstrate how to use Scrapfly SDK with CommonJS. To start, init a project and install the SDK using npm:
6+
7+
```bash
8+
$ npm init -y
9+
$ npm install scrapfly-sdk
10+
```
11+
12+
Then see `commonjs_examples.cjs` for examples and each example can be run by specifying the example function name:
13+
14+
```bash
15+
$ node commonjs_examples.cjs <example_name> <scrapfly_api_key>
16+
# for example
17+
$ node commonjs_examples.cjs basicGet scp-test-123
18+
```

0 commit comments

Comments
 (0)