feat(web-scraper): add full support for Playwright user content extractors

Author: azasypkin

.nvmrc

@@ -1 +1 @@
-20
+22

components/web_scraper/package.json

@@ -7,7 +7,7 @@
   },
   "version": "0.0.1",
   "engines": {
-    "node": "20.x"
+    "node": "22.x"
   },
   "type": "module",
   "main": "dist/src/index.js",
@@ -17,8 +17,8 @@
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
     "watch": "NODE_OPTIONS=--import=./register.js RETRACK_WEB_SCRAPER_ENV_PATH=../../.env nodemon src/index.ts",
-    "test": "NODE_OPTIONS=--import=./register.js NODE_NO_WARNINGS=1 node --test ./src/tests.ts",
-    "test:watch": "NODE_OPTIONS=--import=./register.js NODE_NO_WARNINGS=1 node --test --watch ./src/tests.ts"
+    "test": "NODE_OPTIONS=--import=./register.js NODE_NO_WARNINGS=1 node --experimental-test-module-mocks --test ./src/tests.ts",
+    "test:watch": "NODE_OPTIONS=--import=./register.js NODE_NO_WARNINGS=1 node --experimental-test-module-mocks --test --watch ./src/tests.ts"
   },
   "dependencies": {
     "@fastify/compress": "^7.0.3",
@@ -27,14 +27,15 @@
     "fastify": "^4.28.1",
     "js-beautify": "^1.15.1",
     "node-cache": "^5.1.2",
-    "playwright": "1.45.1"
+    "playwright-core": "1.45.1"
   },
   "devDependencies": {
     "@eslint/eslintrc": "^3.1.0",
-    "@eslint/js": "^9.9.0",
+    "@eslint/js": "^9.9.1",
     "@types/js-beautify": "^1.14.3",
-    "@types/node": "^22.2.0",
-    "eslint": "^9.9.0",
+    "@types/node": "^22.5.1",
+    "@types/ws": "^8.5.12",
+    "eslint": "^9.9.1",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-prettier": "^5.2.1",
     "globals": "^15.9.0",
@@ -43,6 +44,7 @@
     "prettier": "^3.3.3",
     "ts-node": "^10.9.2",
     "typescript": "^5.5.4",
-    "typescript-eslint": "^8.1.0"
+    "typescript-eslint": "^8.3.0",
+    "ws": "^8.18.0"
   }
 }

components/web_scraper/src/api/api_result.ts

@@ -2,29 +2,24 @@ import { mock } from 'node:test';
 
 import { fastify } from 'fastify';
 import NodeCache from 'node-cache';
-import type { Browser } from 'playwright';
 
 import type { Config } from '../config.js';
 import { configure } from '../config.js';
-import type { BrowserInfo } from '../index.js';
-import { createBrowserMock } from '../mocks.js';
+import type { BrowserEndpoint } from '../utilities/browser.js';
 
 interface MockOptions {
-  browser?: Browser;
-  browserInfo?: BrowserInfo;
+  browserEndpoint?: BrowserEndpoint;
   config?: Config;
 }
 
 export function createMock({
-  browser = createBrowserMock() as unknown as Browser,
   config = configure(),
-  browserInfo = { running: false, contexts: [] },
+  browserEndpoint = { url: 'ws://localhost:3000', protocol: 'playwright' },
 }: MockOptions = {}) {
   return {
     server: fastify({ logger: { level: 'warn' } }),
     cache: new NodeCache({ stdTTL: 0 }),
     config,
-    acquireBrowser: mock.fn(() => Promise.resolve(browser)),
-    browserInfo: mock.fn(() => browserInfo),
+    getBrowserEndpoint: mock.fn(() => Promise.resolve(browserEndpoint)),
   };
 }

components/web_scraper/src/api/api_route_params.mocks.ts

@@ -1,14 +1,12 @@
 import type { FastifyInstance } from 'fastify';
 import type NodeCache from 'node-cache';
-import type { Browser } from 'playwright';
 
 import type { Config } from '../config.js';
-import type { BrowserInfo } from '../index.js';
+import type { BrowserEndpoint } from '../utilities/browser.js';
 
 export interface ApiRouteParams {
   server: FastifyInstance;
   cache: NodeCache;
   config: Config;
-  acquireBrowser: () => Promise<Browser>;
-  browserInfo: () => BrowserInfo;
+  getBrowserEndpoint: (options?: { launchServer?: boolean }) => Promise<BrowserEndpoint>;
 }

components/web_scraper/src/api/api_route_params.ts

@@ -1,37 +1,4 @@
-import type { FastifyBaseLogger } from 'fastify';
-import type { Browser } from 'playwright';
-
 export class Diagnostics {
-  public static async screenshot(log: FastifyBaseLogger, browser: Browser) {
-    log.info('Capturing screenshots...');
-    if (!browser.isConnected()) {
-      log.error('Browser is not connected, bailing out...');
-      return;
-    }
-
-    try {
-      const pages = browser.contexts().flatMap((context) => context.pages());
-      log.debug(`Retrieved ${pages.length} pages.`);
-      for (const page of pages) {
-        if (page.isClosed()) {
-          log.debug(`Page is closed: ${page.url()}.`);
-        } else if (page.url() === 'about:blank') {
-          log.debug(`Skipping page: ${page.url()}.`);
-        } else {
-          log.info(`Making screenshot ${page.url()}.`);
-          log.error(
-            {
-              screenshot: Buffer.from((await page.screenshot({ fullPage: true })).toString('base64'), 'base64'),
-            },
-            `Screenshot is made ${page.url()}.`,
-          );
-        }
-      }
-    } catch (err) {
-      log.error('Failed to capture screenshots', err);
-    }
-  }
-
   public static errorMessage(err: unknown): string {
     if (typeof err === 'string') {
       return err;

components/web_scraper/src/api/diagnostics.ts

@@ -10,19 +10,14 @@ await test('[/api/status] can successfully create route', () => {
 
 await test('[/api/status] returns version from the config', async () => {
   const configMock = { version: '1.0.0-rc.100', browserTTLSec: 1, cacheTTLSec: 2, port: 3 };
-  const browserInfoMock = {
-    running: true,
-    name: 'chromium',
-    version: '1.0.0',
-    contexts: [{ pages: ['https://retrack.dev'] }],
-  };
-  const response = await registerStatusGetRoutes(
-    createMock({ config: configMock, browserInfo: browserInfoMock }),
-  ).inject({
+  const response = await registerStatusGetRoutes(createMock({ config: configMock })).inject({
     method: 'GET',
     url: '/api/status',
   });
 
-  assert.strictEqual(response.body, JSON.stringify({ version: configMock.version, browser: browserInfoMock }));
+  assert.strictEqual(
+    response.body,
+    JSON.stringify({ version: configMock.version, browser: { protocol: 'playwright', url: 'ws://localhost:3000' } }),
+  );
   assert.strictEqual(response.statusCode, 200);
 });

components/web_scraper/src/api/status/get.test.ts

@@ -1,6 +1,6 @@
 import type { ApiRouteParams } from '../api_route_params.js';
 
-export function registerStatusGetRoutes({ server, config, browserInfo }: ApiRouteParams) {
+export function registerStatusGetRoutes({ server, config, getBrowserEndpoint }: ApiRouteParams) {
   // Register a route that returns the status of the Web Scraper component.
   return server.get(
     '/api/status',
@@ -14,24 +14,19 @@ export function registerStatusGetRoutes({ server, config, browserInfo }: ApiRout
               browser: {
                 type: 'object',
                 properties: {
-                  running: { type: 'boolean' },
-                  name: { type: 'string', nullable: true },
-                  version: { type: 'string', nullable: true },
-                  contexts: {
-                    type: 'array',
-                    items: { type: 'object', properties: { pages: { type: 'array', items: { type: 'string' } } } },
-                  },
+                  protocol: { type: 'string' },
+                  url: { type: 'string', nullable: true },
                 },
               },
             },
           },
         },
       },
     },
-    () => {
+    async () => {
       return {
         version: config.version,
-        browser: browserInfo(),
+        browser: await getBrowserEndpoint({ launchServer: false }),
       };
     },
   );

components/web_scraper/src/api/status/get.ts

@@ -1,4 +1,36 @@
 /**
- * Default timeout for the page load, in ms.
+ * Default timeout for the user scenario, in ms.
  */
-export const DEFAULT_TIMEOUT_MS = 10000;
+export const DEFAULT_USER_SCRIPT_TIMEOUT_MS = 30000;
+
+// Every user scenario is represented as an ES module that is prefixed with this string.
+export const USER_MODULE_PREFIX = 'data:text/javascript,void("retrack");';
+
+/**
+ * Represents the type of message that can be sent from the worker to the main thread.
+ */
+export enum WorkerMessageType {
+  LOG = 'log',
+  RESULT = 'error',
+}
+
+/**
+ * Represents a log message that can be sent from the worker to the main thread.
+ */
+export interface WorkerLogMessage {
+  type: WorkerMessageType.LOG;
+  message: string;
+  level?: string;
+  screenshots?: Map<string, Uint8Array>;
+  args?: ReadonlyArray<object>;
+}
+
+/**
+ * Represents a result message that can be sent from the worker to the main thread.
+ */
+export interface WorkerResultMessage {
+  type: WorkerMessageType.RESULT;
+  content: { type: WorkerStringResultType; value: string };
+}
+
+export type WorkerStringResultType = 'html' | 'text' | 'json';