Crawlee + Cheerio

1// Apify SDK - toolkit for building Apify Actors (Read more at https://docs.apify.com/sdk/js/)
2import { Actor } from 'apify';
3// Crawlee - web scraping and browser automation library (Read more at https://crawlee.dev)
4import { CheerioCrawler, Dataset } from 'crawlee';
5// this is ESM project, and as such, it requires you to specify extensions in your relative imports
6// read more about this here: https://nodejs.org/docs/latest-v18.x/api/esm.html#mandatory-file-extensions
7// note that we need to use `.js` even when inside TS files
8// import { router } from './routes.js';
9
10interface Input {
11    startUrls: string[];
12    maxRequestsPerCrawl: number;
13}
14
15// The init() call configures the Actor for its environment. It's recommended to start every Actor with an init()
16await Actor.init();
17
18// Structure of input is defined in input_schema.json
19const {
20    startUrls = ['https://crawlee.dev'],
21    maxRequestsPerCrawl = 100,
22} = await Actor.getInput<Input>() ?? {} as Input;
23
24const proxyConfiguration = await Actor.createProxyConfiguration();
25
26const crawler = new CheerioCrawler({
27    proxyConfiguration,
28    maxRequestsPerCrawl,
29    requestHandler: async ({ enqueueLinks, request, $, log }) => {
30        log.info('enqueueing new URLs');
31        await enqueueLinks();
32
33        // Extract title from the page.
34        const title = $('title').text();
35        log.info(`${title}`, { url: request.loadedUrl });
36
37        // Save url and title to Dataset - a table-like storage.
38        await Dataset.pushData({ url: request.loadedUrl, title });
39    },
40});
41
42await crawler.run(startUrls);
43
44// Gracefully exit the Actor process. It's recommended to quit all Actors with an exit()
45await Actor.exit();