31:T4dc,# [puppeteer-ws] Headless Browser + Puppeteer API ***This notebook brings the power of [Puppeteer](https://pptr.dev/) to your browser.*** You can write any automations available with [Puppeteer](https://pptr.dev/) directly in this notebook. List of possibilities (from the [Puppeteer](https://pptr.dev/) site): - Generate screenshots and PDFs of pages. - Crawl a SPA (Single-Page Application) and generate pre-rendered content (i.e., "SSR" (Server-Side Rendering)). - Automate form submission, UI testing, keyboard input, etc. - Create an automated testing environment using the latest JavaScript and browser features. - Capture a timeline trace of your site to help diagnose performance issues. - Test Chrome Extensions. How it works? This notebook connects to an external Chrome application in headless mode and pilots it to execute automation scripts. > **Note:** *We developed the [WebRun DevTools](/@kotelnikov/webrun-devtools?collection=@kotelnikov/webrun-devtools) Chrome Extension which allows you to already write automation scripts. Now we are working on adopting it to bring the whole Puppeteer API to all JavaScript environments without launching external processes, NodeJS or anything like that! We appreciate your feedback.* 32:T624,{ // Launch the browser and open a new blank page const browser = await newBrowser(wsUrl); try { const page = await browser.newPage(); // Navigate the page to a URL await page.goto("https://developer.chrome.com/"); // Set screen size await page.setViewport({ width: 1080, height: 1024 }); // Locate the full title with a unique string const textSelector = await page.waitForSelector("h3"); const fullTitle = await textSelector?.evaluate((el) => el.textContent); const screenshots = []; screenshots.push(await makeScreenshot(page)); // await page.click(".button.dcc-filled-button"); // await Promises.delay(1000); // // Type into search box await page.focus(".devsite-searchbox input"); await page.type(".devsite-searchbox input", "getting"); await page.keyboard.press(" "); await page.keyboard.press("Enter"); await Promises.delay(3000); screenshots.push(await makeScreenshot(page)); return htl.html`

${fullTitle}

${screenshots}

`; } finally { await browser.close(); } async function makeScreenshot(page) { const imgUrl = await page.screenshot({ type: "png", encoding: "base64", fullPage: true }); return htl.html`

`; } }33:T1402,connectToCdpBrowser = { const { CdpBrowser } = await import( "https://esm.sh/puppeteer-core@21.5.2/lib/esm/puppeteer/cdp/Browser.js" ); const { CdpPage } = await import( "https://esm.sh/puppeteer-core@21.5.2/lib/esm/puppeteer/cdp/Page.js" ); const { Connection } = await import( "https://esm.sh/puppeteer-core@21.5.2/lib/esm/puppeteer/cdp/Connection.js" ); const { debug } = await import( "https://esm.sh/puppeteer-core@21.5.2/lib/esm/puppeteer/common/Debug.js" ); /** * @internal */ const debugError = debug("puppeteer:error"); const DEFAULT_VIEWPORT = Object.freeze({ width: 800, height: 600 }); class BrowserInBrowser extends CdpBrowser { constructor({ product, connection, contextIds, ignoreHTTPSErrors = false, defaultViewport = DEFAULT_VIEWPORT, process, closeCallback, targetFilterCallback, isPageTargetCallback, waitForInitiallyDiscoveredTargets }) { super( product, connection, contextIds, (ignoreHTTPSErrors = false), defaultViewport, process, closeCallback, targetFilterCallback, isPageTargetCallback, waitForInitiallyDiscoveredTargets ); this._defaultViewport = defaultViewport; this._ignoreHTTPSErrors = ignoreHTTPSErrors; this._updateTarget = this._updateTarget.bind(this); this.on("targetcreated", this._updateTarget); } async dispose() { this.off("targetcreated", this._updateTarget); return await super.dispose(); } _updateTarget(target) { // packages/puppeteer-core/src/cdp/Target.ts#241 // override async page(): Promise { ... } target.page = async function () { if (!this.pagePromise) { const session = this._session(); this.pagePromise = ( session ? Promise.resolve(session) : this._sessionFactory()(/* isAutoAttachEmulated=*/ false) ).then(async (client) => { const ignoreHTTPSErrors = false; // const defaultViewport = { height: 600, width: 800 }; const page = await CdpPage._create( client, this, this._ignoreHTTPSErrors, this._defaultViewport ); page.pdf = async function (options = {}) { // client.send("Performance.enable"); const { landscape, displayHeaderFooter, headerTemplate, footerTemplate, printBackground, scale, width: paperWidth, height: paperHeight, margin, pageRanges, preferCSSPageSize, omitBackground, timeout: ms, tagged: generateTaggedPDF } = this._getPDFOptions(options); // if (omitBackground) { // await this.#emulationManager.setTransparentBackgroundColor(); // } const printCommandPromise = client.send("Page.printToPDF", { // transferMode: 'ReturnAsStream', landscape, displayHeaderFooter, headerTemplate, footerTemplate, printBackground, scale, paperWidth, paperHeight, marginTop: margin.top, marginBottom: margin.bottom, marginLeft: margin.left, marginRight: margin.right, pageRanges, preferCSSPageSize, generateTaggedPDF }); const { data } = await printCommandPromise; return atob(data); }; return page; }); } return (await this.pagePromise) ?? null; }; } } async function connectToCdpBrowser(options) { const { transport, ignoreHTTPSErrors = false, defaultViewport = DEFAULT_VIEWPORT, targetFilter, _isPageTarget: isPageTarget } = options; const { slowMo = 0, protocolTimeout } = options; const connection = new Connection("", transport, slowMo, protocolTimeout); const version = await connection.send("Browser.getVersion"); const product = version.product.toLowerCase().includes("firefox") ? "firefox" : "chrome"; const { browserContextIds } = await connection.send( "Target.getBrowserContexts" ); const browser = new BrowserInBrowser({ product: product || "chrome", connection, contextIds: browserContextIds, ignoreHTTPSErrors, defaultViewport, closeCallback: () => { // return connection.send("Browser.close").catch(debugError); }, targetFilterCallback: targetFilter, isPageTargetCallback: isPageTarget, waitForInitiallyDiscoveredTargets: false }); await browser._attach(); return browser; } return connectToCdpBrowser; }34:[] 2f:{"title":"[puppeteer-ws] Headless Browser + Puppeteer API","cells":[{"id":0,"value":"$31","pinned":false,"hidden":false,"mode":"md"},{"id":365,"value":"toc({\n headers: \"h2,h3,h4\"\n // title: \"\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":317,"value":"## Getting Started\n\n**Step 1:** Launch Chrome in \"headless\" mode:\n```sh\n> google-chrome --headless=new --remote-debugging-port=9222 --remote-allow-origins=\"*\" about:empty\n```\nIn the output of this command, you will find a browser automation URL. It will be something like:\n\n`ws://127.0.0.1:9222/devtools/browser/968c405e-f4bb-47a7-9a41-dfd424bf9797`\n\n**Step 2:** Copy the WebSocket URL in the form below:","pinned":false,"hidden":false,"mode":"md"},{"id":257,"value":"wsInputForm = Inputs.form({\n description: htl.html`

WebSocket URL for Chrome Browser:

`,\n url: Inputs.text({\n value: \"\",\n label: \"Browser WebSockets URL:\"\n }),\n submit: Inputs.button(\"Go!\")\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":259,"value":"wsUrl = listenForm(wsInputForm, {\n validate : (config) => !!config.url,\n value : config => config.url\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":416,"value":"***Note: You can track automatically presence of a headless browser on a specific port using the WebRunDevTools HTTP service. In this case the step can be completely skipped.***","pinned":false,"hidden":false,"mode":"md"},{"id":331,"value":"After that, this browser will establish a WebSockets connection with your headless instance, and you will be able to write automation scripts. Check examples below to see how it works.","pinned":false,"hidden":false,"mode":"md"},{"id":353,"value":"## Examples","pinned":false,"hidden":false,"mode":"md"},{"id":355,"value":"### Example 1: Automation and Screenshots\n\nIn this example we open the https://developer.chrome.com/ site, enter some keywords in the search bar and generate a screenshot of search results.","pinned":false,"hidden":false,"mode":"md"},{"id":225,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":358,"value":"### Example 2: PDF Generation\n\nThis example shows how to generate and visualize a PDF document directly in this notebook. ","pinned":false,"hidden":false,"mode":"md"},{"id":103,"value":"{\n const browser = await newBrowser(wsUrl);\n try {\n const page = await browser.newPage();\n try {\n await page.goto(\"https://pptr.dev/\");\n const pdfData = await page.pdf();\n const dataUrl = `data:application/pdf;base64,${btoa(pdfData)}`;\n return await pdfView(dataUrl);\n } finally {\n await page.close();\n }\n } finally {\n await browser.close();\n }\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":204,"value":"import { pdfView } from \"@saneef/pdf-view\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":400,"value":"{\n // Launch the browser and open a new blank page\n const browser = await newBrowser(wsUrl);\n try {\n const page = await browser.newPage();\n\n // Navigate the page to a URL\n await page.goto(\"https://pptr.dev\");\n\n // Set screen size\n await page.setViewport({ width: 1080, height: 1024 });\n\n await Promises.delay(3000);\n\n const screenshot = await makeScreenshot(page);\n\n return htl.html`

\n${screenshot}\n

`;\n } finally {\n await browser.close();\n }\n\n async function makeScreenshot(page) {\n const imgUrl = await page.screenshot({\n type: \"png\",\n encoding: \"base64\",\n fullPage: true\n });\n return htl.html`

`;\n }\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":312,"value":"## Implementation","pinned":false,"hidden":false,"mode":"md"},{"id":98,"value":"class BrowserWebSocketTransport {\n static create(url) {\n return new Promise((resolve, reject) => {\n const ws = new WebSocket(url);\n\n ws.addEventListener(\"open\", () => {\n return resolve(new BrowserWebSocketTransport(ws));\n });\n ws.addEventListener(\"error\", reject);\n });\n }\n\n #ws;\n onmessage;\n onclose;\n\n constructor(ws) {\n this.#ws = ws;\n this.#ws.addEventListener(\"message\", (event) => {\n console.log(\"[recieve]\", JSON.parse(event.data));\n if (this.onmessage) {\n this.onmessage.call(null, event.data);\n }\n });\n this.#ws.addEventListener(\"close\", () => {\n if (this.onclose) {\n this.onclose.call(null);\n }\n });\n // Silently ignore all errors - we don't know what to do with them.\n this.#ws.addEventListener(\"error\", () => {});\n }\n\n send(message) {\n const info = JSON.parse(message);\n console.log(\"[send]\", info);\n this.#ws.send(message);\n }\n\n close() {\n this.#ws.close();\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":80,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":335,"value":"## Utilities","pinned":false,"hidden":false,"mode":"md"},{"id":264,"value":"async function* listenForm(form, {\n validate = (d) => true,\n value = (d) => d,\n submit = (d) => d?.submit || 0\n}) {\n let prevSubmit = 0;\n for await (const d of Generators.input(form)) {\n const newSubmit = submit(d);\n if (!d || newSubmit === prevSubmit) continue;\n prevSubmit = newSubmit;\n if (validate(d)) yield await value(d);\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":213,"value":"async function newBrowser(wsUrl) {\n const transport = await BrowserWebSocketTransport.create(wsUrl);\n const browser = await connectToCdpBrowser({\n transport\n });\n return browser;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":363,"value":"import { toc } from \"@nebrius/indented-toc\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":314,"value":"## References\n\nSome references used in preparation of this notebook:\n* [GitHub devtools-protocol](https://github.com/chromedevtools/devtools-protocol) - Official DevTools Protocol repository\n* https://raw.githubusercontent.com/ChromeDevTools/devtools-protocol/master/json/browser_protocol.json\n* https://github.com/cyrus-and/chrome-remote-interface/\n* https://github.com/cyrus-and/chrome-remote-interface/blob/master/lib/protocol.json","pinned":false,"hidden":false,"mode":"md"}],"files":"$W34","stdlib":"1","titling":"auto"}