From 49a2dfb4cba8def96600893bc6539fbade649346 Mon Sep 17 00:00:00 2001 From: uki00a Date: Sun, 31 Jul 2022 12:52:51 +0900 Subject: [PATCH 1/6] BREAKING: Migrate to deno-puppeteer --- carol.ts | 38 +++++++++++++++++++++++++------------- deps.ts | 17 +++++++++-------- http_request.ts | 29 +++++++++++++++-------------- puppeteer.ts | 26 ++------------------------ types.ts | 8 ++++---- util.ts | 6 ++++++ 6 files changed, 61 insertions(+), 63 deletions(-) diff --git a/carol.ts b/carol.ts index 56b3eae..9445c2c 100644 --- a/carol.ts +++ b/carol.ts @@ -69,6 +69,14 @@ interface ExtendedSelf extends Self { receivedFromParent(message: Messages.Any): void; } +// deno-lint-ignore no-explicit-any +type PageFunctionArgs = any[]; + +interface PageFunction { + // deno-lint-ignore no-explicit-any + (...args: PageFunctionArgs): any; +} + interface Carol { loadParams(): Promise; } @@ -222,8 +230,11 @@ class Application extends EventEmitter implements types.Application { ); } - // deno-lint-ignore no-explicit-any, ban-types - evaluate(pageFunction: Function | string, ...args: unknown[]): Promise { + // deno-lint-ignore no-explicit-any + evaluate( + pageFunction: PageFunction | string, + ...args: PageFunctionArgs + ): Promise { return this.mainWindow().evaluate(pageFunction, ...args); } @@ -322,7 +333,7 @@ class Window extends EventEmitter implements types.Window { private loadParams_!: unknown[]; private domContentLoadedCallback_?: () => void; - private windowId_: unknown; + private windowId_!: number; private interceptionInitialized_ = false; private hostHandle_: HandleProxy; private receivedFromChild_!: (message: Messages.Any) => void; @@ -352,7 +363,7 @@ class Window extends EventEmitter implements types.Window { */ async init_(): Promise { this.logger_.debug("[app] Configuring window"); - const targetId = this.page_.target()._targetInfo.targetId; + const targetId = this.page_.target()._targetId; assert(this.options_.bgcolor); const bgcolor = Color.parse(this.options_.bgcolor); assert(bgcolor, "Invalid bgcolor"); @@ -394,9 +405,8 @@ class Window extends EventEmitter implements types.Window { } async evaluate( - // deno-lint-ignore ban-types - pageFunction: string | Function, - ...args: unknown[] + pageFunction: string | PageFunction, + ...args: PageFunctionArgs // deno-lint-ignore no-explicit-any ): Promise { try { @@ -444,7 +454,7 @@ class Window extends EventEmitter implements types.Window { // Await here to process exceptions. await this.page_.goto( new URL(this.loadURI_, "https://domain/").toString(), - { timeout: 0, waitFor: "domcontentloaded" }, + { timeout: 0, waitUntil: "domcontentloaded" }, ); // Available in Chrome M73+. this.session_.send("Page.resetNavigationHistory").catch((_e) => {}); @@ -453,7 +463,7 @@ class Window extends EventEmitter implements types.Window { return result; } - initBounds_(result: { windowId: unknown }) { + initBounds_(result: { windowId: number }) { this.windowId_ = result.windowId; return this.setBounds( { @@ -664,7 +674,7 @@ class Window extends EventEmitter implements types.Window { } async fullscreen(): Promise { - const bounds = { windowState: "fullscreen" }; + const bounds = { windowState: "fullscreen" } as const; await this.app_.session_.send( "Browser.setWindowBounds", { windowId: this.windowId_, bounds }, @@ -672,7 +682,7 @@ class Window extends EventEmitter implements types.Window { } async minimize(): Promise { - const bounds = { windowState: "minimized" }; + const bounds = { windowState: "minimized" } as const; await this.app_.session_.send( "Browser.setWindowBounds", { windowId: this.windowId_, bounds }, @@ -680,7 +690,7 @@ class Window extends EventEmitter implements types.Window { } async maximize(): Promise { - const bounds = { windowState: "maximized" }; + const bounds = { windowState: "maximized" } as const; await this.app_.session_.send( "Browser.setWindowBounds", { windowId: this.windowId_, bounds }, @@ -833,9 +843,11 @@ class HostWindow { "Runtime.evaluate", { expression }, ); + const { objectId } = result; + assert(objectId != null, "objectId must be set"); return this.window_.session_.send( "DOM.getFileInfo", - { objectId: result.objectId }, + { objectId }, ); } } diff --git a/deps.ts b/deps.ts index a5e80f3..b77fe57 100644 --- a/deps.ts +++ b/deps.ts @@ -27,11 +27,12 @@ export { fail, } from "https://deno.land/std@0.149.0/testing/asserts.ts"; -export { default as puppeteer } from "https://unpkg.com/puppeteer@13.3.2/lib/esm/puppeteer/web.js"; -export { EventEmitter } from "https://unpkg.com/puppeteer@13.3.2/lib/esm/puppeteer/common/EventEmitter.js"; -export { BrowserWebSocketTransport } from "https://unpkg.com/puppeteer@13.3.2/lib/esm/puppeteer/common/BrowserWebSocketTransport.js"; - -export type { Browser } from "https://unpkg.com/puppeteer@13.3.2/lib/esm/puppeteer/common/Browser.js"; -export type { Target } from "https://unpkg.com/puppeteer@13.3.2/lib/esm/puppeteer/common/Target.js"; -export type { CDPSession } from "https://unpkg.com/puppeteer@13.3.2/lib/esm/puppeteer/common/Connection.js"; -export type { Page } from "https://unpkg.com/puppeteer@13.3.2/lib/esm/puppeteer/common/Page.js"; +export type { + Browser, + CDPSession, + default as puppeteer, + Page, + Protocol, + Target, +} from "https://deno.land/x/puppeteer@14.1.1/mod.ts"; +export { EventEmitter } from "https://deno.land/x/puppeteer@14.1.1/vendor/puppeteer-core/puppeteer/common/EventEmitter.js"; diff --git a/http_request.ts b/http_request.ts index a01694e..eba3b9f 100644 --- a/http_request.ts +++ b/http_request.ts @@ -18,8 +18,13 @@ */ import { concat, encode, encodeToBase64 } from "./deps.ts"; -import type { CDPSession } from "./deps.ts"; +import type { CDPSession, Protocol } from "./deps.ts"; import type * as types from "./types.ts"; +import { assert } from "./util.ts"; + +type ContinueInterceptedRequestRequest = Partial< + Protocol.Network.ContinueInterceptedRequestRequest +>; const statusTexts = { "100": "Continue", @@ -87,7 +92,8 @@ const statusTexts = { export interface HttpRequestParams { request: Request; resourceType: string; - interceptionId: number; + interceptionId: + Protocol.Network.ContinueInterceptedRequestRequest["interceptionId"]; } interface Request { @@ -96,15 +102,6 @@ interface Request { headers: Record; } -interface ResolveParams { - url?: string; - method?: string; - headers?: Record; - interceptionId?: number; - rawResponse?: string; - errorReason?: string; -} - export class HttpRequest implements types.HttpRequest { done_ = false; @@ -150,7 +147,7 @@ export class HttpRequest implements types.HttpRequest { deferToBrowser(overrides?: types.Overrides) { this.logger_.debug("[server] deferToBrowser", this.url()); - const params = {} as ResolveParams; + const params = {} as ContinueInterceptedRequestRequest; if (overrides && overrides.url) params.url = overrides.url; if (overrides && overrides.method) params.method = overrides.method; if (overrides && overrides.headers) params.headers = overrides.headers; @@ -207,11 +204,15 @@ export class HttpRequest implements types.HttpRequest { this.resolve_({}); } - resolve_(params: ResolveParams): Promise { + resolve_(params: ContinueInterceptedRequestRequest): Promise { this.logger_.debug("[server] resolve", this.url()); if (this.done_) throw new Error("Already resolved given request"); params.interceptionId = this.params_.interceptionId; this.done_ = true; - return this.session_.send("Network.continueInterceptedRequest", params); + assert(params.interceptionId != null); + return this.session_.send( + "Network.continueInterceptedRequest", + params as Protocol.Network.ContinueInterceptedRequestRequest, + ); } } diff --git a/puppeteer.ts b/puppeteer.ts index c8c883c..a406e49 100644 --- a/puppeteer.ts +++ b/puppeteer.ts @@ -16,13 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { - BrowserWebSocketTransport, - BufReader, - decode, - puppeteer, - resolve, -} from "./deps.ts"; +import { BufReader, decode, puppeteer, resolve } from "./deps.ts"; import type { Browser, Target } from "./deps.ts"; import type { AppOptions } from "./types.ts"; import { getLocalDataDir } from "./util.ts"; @@ -32,21 +26,6 @@ interface LaunchResult { chromeProcess: Deno.Process; } -class ExtendedBrowserWebSocketTransport extends BrowserWebSocketTransport { - #ws: WebSocket; - - constructor(ws: WebSocket) { - super(ws); - this.#ws = ws; - } - - close(): void { - if (this.#ws.readyState === this.#ws.OPEN) { - super.close(); - } - } -} - export async function launch( executablePath: string, headless: boolean, @@ -58,10 +37,9 @@ export async function launch( stderr: "piped", }); const wsEndpoint = await waitForWSEndpoint(chromeProcess.stderr); - const transport = await ExtendedBrowserWebSocketTransport.create(wsEndpoint); const browser = await puppeteer.connect({ ignoreHTTPSErrors: true, - transport, + browserWSEndpoint: wsEndpoint, }); await browser.waitForTarget((t: Target) => t.type() === "page"); return { diff --git a/types.ts b/types.ts index 89a0e58..f54589f 100644 --- a/types.ts +++ b/types.ts @@ -220,10 +220,10 @@ export interface Window { } export interface Bounds { - left: number; - top: number; - width: number; - height: number; + left?: number; + top?: number; + width?: number; + height?: number; } export interface HttpHandler { diff --git a/util.ts b/util.ts index c00e6ad..bf32c3a 100644 --- a/util.ts +++ b/util.ts @@ -1,5 +1,11 @@ import { dirname, fromFileUrl, join } from "./deps.ts"; +export function assert(expr: unknown, msg = ""): asserts expr { + if (!expr) { + throw new Error("assertion failed: " + msg); + } +} + export function getLocalDataDir(): string { const __dirname = import.meta.url.startsWith("file://") ? dirname(fromFileUrl(import.meta.url)) From 43fa4dbdb40e81ba946c59598e3e3929d20a43de Mon Sep 17 00:00:00 2001 From: uki00a Date: Sun, 31 Jul 2022 13:07:19 +0900 Subject: [PATCH 2/6] chore: Run `deno lint` --- carol.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carol.ts b/carol.ts index 9445c2c..46b1828 100644 --- a/carol.ts +++ b/carol.ts @@ -230,10 +230,10 @@ class Application extends EventEmitter implements types.Application { ); } - // deno-lint-ignore no-explicit-any evaluate( pageFunction: PageFunction | string, ...args: PageFunctionArgs + // deno-lint-ignore no-explicit-any ): Promise { return this.mainWindow().evaluate(pageFunction, ...args); } From 3044163c41316bd3c94598cb56e64333cde398ad Mon Sep 17 00:00:00 2001 From: uki00a Date: Sun, 31 Jul 2022 13:09:00 +0900 Subject: [PATCH 3/6] fix: deps.ts --- deps.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deps.ts b/deps.ts index b77fe57..e6bb3ea 100644 --- a/deps.ts +++ b/deps.ts @@ -27,10 +27,12 @@ export { fail, } from "https://deno.land/std@0.149.0/testing/asserts.ts"; +export { + default as puppeteer, +} from "https://deno.land/x/puppeteer@14.1.1/mod.ts"; export type { Browser, CDPSession, - default as puppeteer, Page, Protocol, Target, From 2e3abb8ce39d1d33e3152af6a7e71ee1fd7f760f Mon Sep 17 00:00:00 2001 From: uki00a Date: Sun, 31 Jul 2022 15:02:56 +0900 Subject: [PATCH 4/6] fix: revert ExtendedBrowserWebSocketTransport --- deps.ts | 1 + puppeteer.ts | 26 ++++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/deps.ts b/deps.ts index e6bb3ea..c34f5a6 100644 --- a/deps.ts +++ b/deps.ts @@ -38,3 +38,4 @@ export type { Target, } from "https://deno.land/x/puppeteer@14.1.1/mod.ts"; export { EventEmitter } from "https://deno.land/x/puppeteer@14.1.1/vendor/puppeteer-core/puppeteer/common/EventEmitter.js"; +export { BrowserWebSocketTransport } from "https://deno.land/x/puppeteer@14.1.1/vendor/puppeteer-core/puppeteer/common/BrowserWebSocketTransport.js"; diff --git a/puppeteer.ts b/puppeteer.ts index a406e49..25f72bc 100644 --- a/puppeteer.ts +++ b/puppeteer.ts @@ -16,11 +16,32 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { BufReader, decode, puppeteer, resolve } from "./deps.ts"; +import { + BrowserWebSocketTransport, + BufReader, + decode, + puppeteer, + resolve, +} from "./deps.ts"; import type { Browser, Target } from "./deps.ts"; import type { AppOptions } from "./types.ts"; import { getLocalDataDir } from "./util.ts"; +class ExtendedBrowserWebSocketTransport extends BrowserWebSocketTransport { + #ws: WebSocket; + + constructor(ws: WebSocket) { + super(ws); + this.#ws = ws; + } + + async close(): Promise { + if (this.#ws.readyState === this.#ws.OPEN) { + await super.close(); + } + } +} + interface LaunchResult { browser: Browser; chromeProcess: Deno.Process; @@ -37,9 +58,10 @@ export async function launch( stderr: "piped", }); const wsEndpoint = await waitForWSEndpoint(chromeProcess.stderr); + const transport = await ExtendedBrowserWebSocketTransport.create(wsEndpoint); const browser = await puppeteer.connect({ ignoreHTTPSErrors: true, - browserWSEndpoint: wsEndpoint, + transport, }); await browser.waitForTarget((t: Target) => t.type() === "page"); return { From 5c84a8f072662ccea3f7d60518536ff367cbb931 Mon Sep 17 00:00:00 2001 From: uki00a Date: Sun, 31 Jul 2022 15:06:01 +0900 Subject: [PATCH 5/6] ci: Enable CAROL_DEBUG --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e884a6..23c8447 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,5 +36,5 @@ jobs: run: chrome --version - name: Run tests run: | - deno task test + CAROL_DEBUG=1 deno task test From 7f288d1fc0561f0ac98da3b5ac2d3c8eb0d1684d Mon Sep 17 00:00:00 2001 From: uki00a Date: Sun, 31 Jul 2022 17:47:02 +0900 Subject: [PATCH 6/6] fix: stop the process when the connection to CDP fails --- puppeteer.ts | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/puppeteer.ts b/puppeteer.ts index 25f72bc..6573e1c 100644 --- a/puppeteer.ts +++ b/puppeteer.ts @@ -58,16 +58,24 @@ export async function launch( stderr: "piped", }); const wsEndpoint = await waitForWSEndpoint(chromeProcess.stderr); - const transport = await ExtendedBrowserWebSocketTransport.create(wsEndpoint); - const browser = await puppeteer.connect({ - ignoreHTTPSErrors: true, - transport, - }); - await browser.waitForTarget((t: Target) => t.type() === "page"); - return { - browser, - chromeProcess, - }; + try { + const transport = await ExtendedBrowserWebSocketTransport.create( + wsEndpoint, + ); + const browser = await puppeteer.connect({ + ignoreHTTPSErrors: true, + transport, + }); + await browser.waitForTarget((t: Target) => t.type() === "page"); + return { + browser, + chromeProcess, + }; + } catch (error) { + console.error(error); + chromeProcess.close(); + throw error; + } } function prepareChromeArgs(