From 6ee3ae8c05cd1e564389ccf8cab6730caf121af7 Mon Sep 17 00:00:00 2001 From: kimdonghyun Date: Wed, 30 Jul 2025 05:23:13 +0900 Subject: [PATCH 1/2] =?UTF-8?q?fix:=20brower=20=EC=84=A4=EC=A0=95=20?= =?UTF-8?q?=EC=A4=91=20=EC=B5=9C=EC=86=8C=20=EC=84=A4=EC=A0=95=EB=A7=8C=20?= =?UTF-8?q?=EC=82=AC=EC=9A=A9=20=EB=B0=8F=20=EA=B3=A0=EC=96=91=EC=8B=9C?= =?UTF-8?q?=EC=9B=90=EB=8B=B9=EC=A2=85=ED=95=A9=EC=82=AC=ED=9A=8C=EB=B3=B5?= =?UTF-8?q?=EC=A7=80=EA=B4=80=EC=97=90=EC=84=9C=20=EB=B0=9C=EC=83=9D?= =?UTF-8?q?=ED=95=98=EB=8A=94=20timeout=20=EB=AC=B8=EC=A0=9C=EC=9B=90?= =?UTF-8?q?=EC=9D=B8=20=EB=B3=B5=EC=A7=80=EA=B4=80=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/aws/lambda/CrawlingService.ts | 32 +++++++-------- src/entity/job/JobRegistry.ts | 2 + ...14\353\263\265\354\247\200\352\264\200.ts" | 41 +++++++++++++++++++ 3 files changed, 59 insertions(+), 16 deletions(-) create mode 100644 "src/entity/job/implement/gyeonggi/goyang/\352\263\240\354\226\221\354\213\234\354\233\220\353\213\271\354\242\205\355\225\251\354\202\254\355\232\214\353\263\265\354\247\200\352\264\200.ts" diff --git a/src/aws/lambda/CrawlingService.ts b/src/aws/lambda/CrawlingService.ts index 79631ed..37688d1 100644 --- a/src/aws/lambda/CrawlingService.ts +++ b/src/aws/lambda/CrawlingService.ts @@ -131,22 +131,22 @@ export class CrawlingService { executablePath: await chromiumBinary.executablePath(), args: [ ...chromiumBinary.args, - '--no-sandbox', - '--disable-dev-shm-usage', - '--disable-gpu', - '--disable-features=VizDisplayCompositor', - '--disable-background-timer-throttling', - '--disable-backgrounding-occluded-windows', - '--disable-renderer-backgrounding', - '--disable-web-security', - '--single-process', - '--disable-setuid-sandbox', - '--no-zygote', - '--disable-accelerated-2d-canvas', - '--no-first-run', - '--no-default-browser-check', - '--disable-extensions', - '--disable-plugins', + // '--no-sandbox', + // '--disable-dev-shm-usage', + // '--disable-gpu', + // '--disable-features=VizDisplayCompositor', + // '--disable-background-timer-throttling', + // '--disable-backgrounding-occluded-windows', + // '--disable-renderer-backgrounding', + // '--disable-web-security', + // '--single-process', + // '--disable-setuid-sandbox', + // '--no-zygote', + // '--disable-accelerated-2d-canvas', + // '--no-first-run', + // '--no-default-browser-check', + // '--disable-extensions', + // '--disable-plugins', ], }); console.log('Browser initialized successfully'); diff --git a/src/entity/job/JobRegistry.ts b/src/entity/job/JobRegistry.ts index 0fa1ef5..d534f54 100644 --- a/src/entity/job/JobRegistry.ts +++ b/src/entity/job/JobRegistry.ts @@ -19,6 +19,7 @@ import { 서울시사회복지사협회 } from './implement/seoul/서울시사 import { 거모종합사회복지관 } from './implement/gyeonggi/siheung/거모종합사회복지관'; import { 고강종합사회복지관 } from './implement/gyeonggi/bucheon/고강종합사회복지관'; import { 고산종합사회복지관 } from './implement/gyeonggi/uijeongbu/고산종합사회복지관'; +import { 고양시원당종합사회복지관 } from './implement/gyeonggi/goyang/고양시원당종합사회복지관'; /** * 모든 크롤링 Job들을 등록하고 관리하는 Registry 클래스 @@ -45,6 +46,7 @@ export class JobRegistry { new 거모종합사회복지관(), new 고강종합사회복지관(), new 고산종합사회복지관(), + new 고양시원당종합사회복지관(), ]; static getAllJobs(): Job[] { diff --git "a/src/entity/job/implement/gyeonggi/goyang/\352\263\240\354\226\221\354\213\234\354\233\220\353\213\271\354\242\205\355\225\251\354\202\254\355\232\214\353\263\265\354\247\200\352\264\200.ts" "b/src/entity/job/implement/gyeonggi/goyang/\352\263\240\354\226\221\354\213\234\354\233\220\353\213\271\354\242\205\355\225\251\354\202\254\355\232\214\353\263\265\354\247\200\352\264\200.ts" new file mode 100644 index 0000000..2f99da7 --- /dev/null +++ "b/src/entity/job/implement/gyeonggi/goyang/\352\263\240\354\226\221\354\213\234\354\233\220\353\213\271\354\242\205\355\225\251\354\202\254\355\232\214\353\263\265\354\247\200\352\264\200.ts" @@ -0,0 +1,41 @@ +import { Locator } from "playwright-core"; +import { Category } from "../../../../Category"; +import { Optimize, Optimizer } from "../../../../Optimize"; +import { SimpleTemplateStep } from "../../../../step/SimpleTemplateStep"; +import { AbstractJob } from "../../../AbstractJob"; +import { parseKoreaDate } from "../../../../../utils/DateUtils"; + +export class 고양시원당종합사회복지관 extends AbstractJob { + constructor() { + super('고양시원당종합사회복지관', 'https://wdss.or.kr', [new 공지사항()]) + } + + registerOptimizer(optimizer: Optimizer) { + optimizer.register(Optimize.JS); + } +} + +class 공지사항 extends SimpleTemplateStep { + constructor() { + super( + 'https://wdss.or.kr/wondang/sub4_1.do', + '.board-01.board-list tbody tr:not(:first-child)', + Category.NOTICE + ); + } + + async select(card: Locator, baseUrl: string): Promise { + const titleElement = card.locator('.board-title a'); + const link = await titleElement.getAttribute('href'); + const title = (await titleElement.textContent()).trim(); + + const dateStr = (await card.locator('td:nth-child(4)').textContent()).trim(); + const createdAt = parseKoreaDate(dateStr, '-'); + + return { + title: title, + createdAt: createdAt, + link: link + } + } +} From 4ef1bb34141f394a1446ffb6a5850a7f26314cbd Mon Sep 17 00:00:00 2001 From: kimdonghyun Date: Wed, 30 Jul 2025 05:29:02 +0900 Subject: [PATCH 2/2] =?UTF-8?q?chore:=20=EC=A3=BC=EC=84=9D=20=EC=BD=94?= =?UTF-8?q?=EB=93=9C=20=EC=82=AD=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/aws/lambda/CrawlingService.ts | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/src/aws/lambda/CrawlingService.ts b/src/aws/lambda/CrawlingService.ts index 37688d1..17a0dbd 100644 --- a/src/aws/lambda/CrawlingService.ts +++ b/src/aws/lambda/CrawlingService.ts @@ -130,23 +130,7 @@ export class CrawlingService { headless: true, executablePath: await chromiumBinary.executablePath(), args: [ - ...chromiumBinary.args, - // '--no-sandbox', - // '--disable-dev-shm-usage', - // '--disable-gpu', - // '--disable-features=VizDisplayCompositor', - // '--disable-background-timer-throttling', - // '--disable-backgrounding-occluded-windows', - // '--disable-renderer-backgrounding', - // '--disable-web-security', - // '--single-process', - // '--disable-setuid-sandbox', - // '--no-zygote', - // '--disable-accelerated-2d-canvas', - // '--no-first-run', - // '--no-default-browser-check', - // '--disable-extensions', - // '--disable-plugins', + ...chromiumBinary.args ], }); console.log('Browser initialized successfully');