30:T93a,md`# Browser Automation with Puppeteer ![](${await FileAttachment("puppeteer.png").url()}) [Puppeteer by Google](https://github.com/puppeteer/puppeteer) is a headless Chrome automation library. Now you can do it in the browser without installing or running anything! By using the work of [pupeteer-web](https://www.npmjs.com/package/puppeteer-web) (thanks [@entrptaher](https://github.com/entrptaher)!) we are able to make a connection with a remote pupeteer instance hosted by [@endpointservices](https://observablehq.com/@endpointservices) (in the serverless cell [binary](https://github.com/endpointservices/serverlesscells)) You need to login to use the service. We use IndeWeb login, so you can use either 1. An [IndieWeb Identity URL](https://indieweb.org/IndieAuth) of your personal homepage. 2. a Github profile URL. 3. an Observable profile URL, if it includes a link to a Github profile URL or some other authentication method. Alternatively, you can use a demo identity to login and try without any prior setup: ~~~ https://webcode.run/notebooks/@endpointservices/identity/deploys/demo/mods/T ~~~ There is no billing for this ATM. But if it is popular I would adopt some usage based pricing in the order of $0.0001 per second. ## What can I do? [puppeteer docs](https://github.com/puppeteer/puppeteer#what-can-i-do) Most things that you can do manually in the browser can be done using Puppeteer! Here are a few examples to get you started: - Collect data from other websites - Generate screenshots and PDFs of pages. - Crawl a SPA (Single-Page Application) and generate pre-rendered content (i.e. "SSR" (Server-Side Rendering)). - Automate form submission, UI testing, keyboard input, etc. - Create an up-to-date, automated testing environment. Run your tests directly in the latest version of Chrome using the latest JavaScript and browser features. - Capture a timeline trace of your site to help diagnose performance issues. ### Quickstart ~~~js import {viewof connect, viewof user} from '@endpointservices/puppeteer' ~~~ An example of a 3rd party notebook using puppeteer can be found [here](https://observablehq.com/@tdlgkjfhdljovtttqrzu/puppeteer-example) `31:[{"id":"d388b40ec6f0de2f7bc7298b068e8a0e84b4db2a6f31d6e150901f185e40f04c3bd38d06b75ada5932ece93d6e1af1fed8a9a6369d0682ae790944f964e9d924","href":"https://static.observableusercontent.com/files/d388b40ec6f0de2f7bc7298b068e8a0e84b4db2a6f31d6e150901f185e40f04c3bd38d06b75ada5932ece93d6e1af1fed8a9a6369d0682ae790944f964e9d924","name":"puppeteer-web-0.0.3.js","type":"application/javascript","lastModified":1622661947034.231,"size":692355},{"id":"243b533b7711810576f16d3718a988783a7b4cfc059c2eda243baf41d26762d9f128c746a5a8c2b181c7b7895506df8de83dd693447949a579543c0a91e969e4","href":"https://static.observableusercontent.com/files/243b533b7711810576f16d3718a988783a7b4cfc059c2eda243baf41d26762d9f128c746a5a8c2b181c7b7895506df8de83dd693447949a579543c0a91e969e4","name":"puppeteer.png","type":"image/png","lastModified":1623052726986.842,"size":153538}] 2e:{"title":"Browser Automation with Puppeteer","cells":[{"id":0,"value":"$30","pinned":false,"hidden":false,"mode":"ojs"},{"id":64,"value":"viewof user","pinned":false,"hidden":false,"mode":"ojs"},{"id":123,"value":"viewof connect = {\n const ui = md`\n

[puppeteer](https://observablehq.com/@endpointservices/puppeteer) connection settings

\naccount ${user.uid}\n

\n\n `\n\n ui.value = async function connect() {\n const url = `wss://webcode.run/puppeteer?token=${await user.getIdToken()}`\n return await puppeteer_web.connect({\n browserWSEndpoint: url,\n ignoreHTTPSErrors: true\n });\n }\n return ui;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":225,"value":"md`### Taking a screenshot`","pinned":false,"hidden":false,"mode":"ojs"},{"id":16,"value":"screenshot = {\n runDemos;\n const browser = await connect();\n const page = await browser.newPage();\n await page.goto(\"https://news.ycombinator.com\");\n const buffer = await page.screenshot();\n\n const imgBlob = new Blob([buffer], { type: \"image/png\" });\n const img = htl.html`

`;\n\n browser.close(); // Usage based pricing! Close when not in use!!!\n return img;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":229,"value":"md`### Scraping the DOM`","pinned":false,"hidden":false,"mode":"ojs"},{"id":232,"value":"scrapeDOM = {\n runDemos;\n const browser = await connect();\n const page = await browser.newPage();\n await page.goto(\"https://news.ycombinator.com\");\n await page.waitForSelector(\".titlelink\");\n // Get the link to all the required books\n let links = await page.$$eval(\".titlelink\", (links) =>\n links.map((link, i) => ({\n rank: i + 1,\n href: link.href,\n title: link.innerHTML\n }))\n );\n\n browser.close(); // Usage based pricing! Close when not in use!!!\n return Inputs.table(links, {\n columns: [\"rank\", \"title\", \"href\"],\n layout: \"auto\"\n });\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":259,"value":"md`## Performance Tracing`","pinned":false,"hidden":false,"mode":"ojs"},{"id":303,"value":"Plot.plot({\n y: {\n axis: null\n },\n marks: [\n Plot.barX(completeEvents, {\n x1: \"ts\",\n x2: \"te\",\n y: \"depth\",\n fill: \"#000\",\n })\n ]\n})\n","pinned":true,"hidden":false,"mode":"ojs"},{"id":262,"value":"traceData = {\n runDemos\n const browser = await connect()\n const page = await browser.newPage();\n // Drag and drop this JSON file to the DevTools Performance panel!\n await page.tracing.start({categories: ['blink', 'cc', 'netlog', 'v8', 'sequence_manager', 'toplevel']});\n await page.goto('https://observablehq.com');\n const buffer = await page.tracing.stop();\n await browser.close();\n return buffer\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":396,"value":"htl.html`\n\n\n download profile data\n

Its compatible with timeline-viewer\n`","pinned":true,"hidden":false,"mode":"ojs"},{"id":348,"value":"// Trace format: https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview\ncompleteEvents = {\n let depth = 0;\n return JSON.parse(traceData).traceEvents.reduce(\n (acc, evt) => {\n if (evt.ph === 'B') {\n depth++;\n } else if (evt.ph === 'E') {\n depth--;\n } else if (evt.ph === 'X') {\n acc.push({\n ...evt,\n te: evt.ts + evt.dur,\n depth: depth\n })\n }\n return acc;\n }\n , [])\n\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":201,"value":"runDemos = htl.html``.href.startsWith('https://observablehq.com/@endpointservices/puppeteer') || invalidation","pinned":false,"hidden":false,"mode":"ojs"},{"id":5,"value":"puppeteer_web = require(await FileAttachment(\"puppeteer-web-0.0.3.js\").url())","pinned":false,"hidden":false,"mode":"ojs"},{"id":120,"value":"import {view} from '@tomlarkworthy/view'","pinned":false,"hidden":false,"mode":"ojs"},{"id":418,"value":"import { viewof user } from \"@endpointservices/login-with-comment\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":438,"value":"import { footer } from \"@endpointservices/footer-with-backups\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":441,"value":"footer","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W31","stdlib":"1","titling":"auto"}