Web Automation & Data Scraping with Puppeteer: Headless Chrome, Stealth & PDF Generation

import puppeteer from 'puppeteer-extra';
import StealthPlugin from 'puppeteer-extra-plugin-stealth';

// Apply stealth plugin to bypass Cloudflare / Bot detection fingerprints
puppeteer.use(StealthPlugin());

export async function scrapeCompetitorPricing(targetUrl: string) {
  const browser = await puppeteer.launch({
    headless: true,
    args: [
      '--no-sandbox',
      '--disable-setuid-sandbox',
      '--disable-dev-shm-usage', // Prevents memory crashes in Docker
      '--disable-accelerated-2d-canvas',
      '--disable-gpu',
    ],
  });

  const page = await browser.newPage();

  // Set realistic viewport & user agent
  await page.setViewport({ width: 1920, height: 1080 });
  await page.setUserAgent(
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36'
  );

  // Block unnecessary assets (images, stylesheets, fonts) to accelerate scraping by 4x
  await page.setRequestInterception(true);
  page.on('request', (req) => {
    if (['image', 'stylesheet', 'font', 'media'].includes(req.resourceType())) {
      req.abort();
    } else {
      req.continue();
    }
  });

  try {
    await page.goto(targetUrl, { waitUntil: 'networkidle2', timeout: 30000 });

    // Wait for the dynamic price element
    await page.waitForSelector('[data-testid="product-price"]', { timeout: 5000 });

    const products = await page.evaluate(() => {
      const items = Array.from(document.querySelectorAll('.product-card'));
      return items.map((el) => ({
        title: el.querySelector('h2')?.textContent?.trim() || '',
        price: el.querySelector('.price')?.textContent?.trim() || '',
      }));
    });

    return products;
  } finally {
    await browser.close();
  }
}

import puppeteer from 'puppeteer';

export async function generateInvoicePdf(htmlContent: string): Promise<Buffer> {
  const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] });
  const page = await browser.newPage();

  await page.setContent(htmlContent, { waitUntil: 'networkidle0' });

  const pdfBuffer = await page.pdf({
    format: 'A4',
    printBackground: true,
    margin: {
      top: '20mm',
      bottom: '20mm',
      left: '15mm',
      right: '15mm',
    },
  });

  await browser.close();
  return Buffer.from(pdfBuffer);
}