Puppeteer on Kubernetes throws errors: "Navigation frame was detached", "Requesting main frame too early"

Question

I am trying to run a nodejs based Docker container on a k8s cluster. The code refuses to run, and is continuously getting errors:

Navigation frame was detached
Requesting main frame too early

I have cut down to a minimal code that should do some work:

const puppeteer = require('puppeteer');
const os = require('os');

function delay(time) {
    return new Promise(resolve => setTimeout(resolve, time));
}

const platform = os.platform();

(async () => {
    console.log('started')
    let browserConfig = {}
    if (platform === 'linux') {
        browserConfig = {
            executablePath: '/usr/bin/google-chrome',
            headless: true,
            args:['--no-sandbox','--disable-web-security','--disable-features=IsolateOrigins,site-per-process','--disable-gpu', '--disable-dev-shm-usage']
        }
    }
    else {
        browserConfig = {
            headless: true,
            args:['--no-sandbox','--disable-web-security','--disable-features=IsolateOrigins,site-per-process','--disable-gpu', '--disable-dev-shm-usage']
        }
    }
    console.log('create browser')
    const browser = await puppeteer.launch(browserConfig)
    console.log('create page')
    const page = await browser.newPage()
    await page.setViewport({ width: 1920, height: 926 })

    console.log('browsing to url')
    await page.goto("https://www.example.com", {
            waitUntil: 'load',
            timeout: 3000000
        })
    console.log('waiting')
    await delay(5000)
    console.log('get content')
    const s = await page.content();
    console.log('content', s);
    console.log('close browser')
    await browser.close()
    console.log('finished')
})();

That code is running on the local nodejs CLI, and also when packed to a Docker image using this Dockerfile:

# Install dependencies only when needed
FROM node:20.11.1-slim AS deps
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true

RUN apt-get update && \
    apt-get install -y libc6 && \
    apt-get install -y git && \
    rm -rf /var/lib/apt/lists/*

WORKDIR /app

COPY package.json package-lock.json ./
RUN npm ci --legacy-peer-deps

# Rebuild the source code only when needed
FROM node:20.11.1-slim AS builder
WORKDIR /app

COPY . .
COPY --from=deps /app/node_modules ./node_modules
ARG NODE_ENV=production
RUN echo ${NODE_ENV}
# RUN NODE_ENV=${NODE_ENV} npm run build

# Production image, copy all the files and run next
FROM node:20.11.1-slim AS runner
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true

WORKDIR /app

RUN apt-get update && apt-get install -y gnupg wget && \
  wget --quiet --output-document=- https://dl-ssl.google.com/linux/linux_signing_key.pub | gpg --dearmor > /etc/apt/trusted.gpg.d/google-archive.gpg && \
  echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list && \
  apt-get update && \
  apt-get install -y google-chrome-stable --no-install-recommends && \
  rm -rf /var/lib/apt/lists/*

COPY --from=builder /app/node_modules ./node_modules
COPY --from=builder /app/package.json ./package.json
COPY --from=builder /app/app.js ./app.js

# Expose
EXPOSE 3000

# CMD ["node", "app.js"]
CMD node app.js

Running this image on local Docker is doing the job. However when I try to deploy it to a Kubernetes cluster, it fails with these errors.

k8s error messages

This is very frustrating :( any help will be much appreciated.

Answer 1

Eventually after long hours of testing vs. the Kubernetes cluster, and with the help of my devops engineer, I solved this issue. Seems that it was similar to the problem described on This answer (which I tried to use before but did not succeed to find the proper place to implement). It seems that the k8s cluster behaved differently when instantiating new pages. Adding more delay solved that issue.

function delay(time) {
    return new Promise(resolve => setTimeout(resolve, time));
}

// ...

const browser = await puppeteer.launch(browserConfig)
await delay(1000)   // Wait a bit due to race condition issues on k8s (the pod crashed).
const page = await browser.newPage()
await delay(1000)  // <- Added here
await page.setViewport({ width: 1920, height: 926 })
await delay(2000) // <- Added here too

// ... 

await page.goto("https://www.example.com", {
                    waitUntil: ['domcontentloaded', 'networkidle2'],
                    timeout: 3000000
                })
await delay(1000)