feat(web-scraper): add support for saving screenshots to a disk

Author: azasypkin

.gitignore

@@ -8,6 +8,7 @@ dist/
 .env
 
 components/retrack-types/Cargo.lock
+components/retrack-web-scraper/.screenshots
 
 # Local Retrack configuration file
 retrack.toml

components/retrack-web-scraper/src/api/web_page/constants.ts

@@ -51,4 +51,6 @@ export interface WorkerData {
   userAgent?: string;
   // Whether to ignore HTTPS errors when sending network requests.
   ignoreHTTPSErrors?: boolean;
+  // Path to a folder where to save screenshots.
+  screenshotsPath?: string;
 }

components/retrack-web-scraper/src/api/web_page/execute.ts

@@ -45,7 +45,7 @@ interface RequestBodyType {
   ignoreHTTPSErrors?: boolean;
 }
 
-export function registerExecuteRoutes({ server, getBrowserEndpoint }: ApiRouteParams) {
+export function registerExecuteRoutes({ config, server, getBrowserEndpoint }: ApiRouteParams) {
   return server.post<{ Body: RequestBodyType }>(
     '/api/web_page/execute',
     {
@@ -75,6 +75,7 @@ export function registerExecuteRoutes({ server, getBrowserEndpoint }: ApiRoutePa
         previousContent: request.body.previousContent,
         userAgent: request.body.userAgent,
         ignoreHTTPSErrors: request.body.ignoreHTTPSErrors,
+        screenshotsPath: config.browserScreenshotsPath,
       };
 
       try {
@@ -103,7 +104,7 @@ export function registerExecuteRoutes({ server, getBrowserEndpoint }: ApiRoutePa
               if (message.level === 'error') {
                 workerLog.error(message.message, message.args);
                 for (const [url, screenshot] of message.screenshots ?? []) {
-                  workerLog.error({ screenshot }, `Screenshot for ${url}.`);
+                  workerLog.error({ screenshot: Buffer.from(screenshot).toString('base64') }, `Screenshot for ${url}.`);
                 }
               } else {
                 workerLog.info(message.message, message.args);

components/retrack-web-scraper/src/api/web_page/worker.ts

@@ -14,7 +14,8 @@ if (!parentPort) {
 }
 
 // Load the extractor script as an ES module.
-const { endpoint, extractor, tags, previousContent, userAgent, ignoreHTTPSErrors } = workerData as WorkerData;
+const { endpoint, extractor, tags, previousContent, userAgent, ignoreHTTPSErrors, screenshotsPath } =
+  workerData as WorkerData;
 
 // SECURITY: Basic prototype pollution protection against the most common vectors until we can use Playwright with
 // `--frozen-intrinsics`. It DOES NOT protect against all prototype pollution vectors.
@@ -112,7 +113,18 @@ try {
       pages.map((page) => ({ url: page.url() })),
       new Map(
         await Promise.all(
-          pages.map(async (page) => [page.url(), await page.screenshot({ fullPage: true })] as [string, Uint8Array]),
+          pages.map(
+            async (page) =>
+              [
+                page.url(),
+                await page.screenshot({
+                  fullPage: true,
+                  path: screenshotsPath
+                    ? `${screenshotsPath}/screenshot_${encodeURIComponent(page.url())}_${Date.now()}.png`
+                    : undefined,
+                }),
+              ] as [string, Uint8Array],
+          ),
         ),
       ),
     );

components/retrack-web-scraper/src/config.ts

@@ -7,6 +7,7 @@ export interface Config {
   port: number;
   cacheTTLSec: number;
   browserTTLSec: number;
+  browserScreenshotsPath?: string;
   userAgent?: string;
 }
 
@@ -18,6 +19,7 @@ export function configure(): Config {
     port: +(process.env.RETRACK_WEB_SCRAPER_PORT ?? 0) || 7272,
     cacheTTLSec: +(process.env.RETRACK_WEB_SCRAPER_CACHE_TTL_SEC ?? 0) || 20 * 60,
     browserTTLSec: +(process.env.RETRACK_WEB_SCRAPER_BROWSER_TTL_SEC ?? 0) || 10 * 60,
+    browserScreenshotsPath: process.env.RETRACK_WEB_SCRAPER_BROWSER_SCREENSHOTS_PATH,
     userAgent: process.env.RETRACK_WEB_SCRAPER_USER_AGENT,
   };
 }

components/retrack-web-scraper/src/index.ts

@@ -19,7 +19,10 @@ const server = fastify({
       ? { level: process.env.RETRACK_WEB_SCRAPER_LOG_LEVEL ?? 'debug' }
       : {
           level: process.env.RETRACK_WEB_SCRAPER_LOG_LEVEL ?? 'debug',
-          transport: { target: 'pino-pretty', options: { translateTime: 'HH:MM:ss Z', ignore: 'pid,hostname' } },
+          transport: {
+            target: 'pino-pretty',
+            options: { translateTime: 'HH:MM:ss Z', ignore: 'pid,hostname,screenshot' },
+          },
         },
 }).addHook('onClose', () => stopBrowserServer());