diff --git a/.github/workflows/update-ky-youtube.yml b/.github/workflows/update-ky-youtube.yml new file mode 100644 index 0000000..971759a --- /dev/null +++ b/.github/workflows/update-ky-youtube.yml @@ -0,0 +1,62 @@ +name: Update ky by Youtube + +on: + schedule: + - cron: "0 14 * * *" # 한국 시간 23:00 실행 (UTC+9 → UTC 14:00) + workflow_dispatch: + +permissions: + contents: write # push 권한을 위해 필요 + +jobs: + run-npm-task: + runs-on: ubuntu-latest + + steps: + - name: Checkout branch + uses: actions/checkout@v4 + with: + ref: feat/songUpdate + persist-credentials: false # 수동 인증으로 푸시 제어 + + - name: Use Node.js 18 + uses: actions/setup-node@v4 + with: + node-version: "18" + + - name: Install pnpm + uses: pnpm/action-setup@v2 + with: + version: 9 + run_install: false + + - name: Install dependencies + working-directory: packages/crawling + run: pnpm install + + - name: Create .env file + working-directory: packages/crawling + run: | + echo "SUPABASE_URL=${{ secrets.SUPABASE_URL }}" >> .env + echo "SUPABASE_KEY=${{ secrets.SUPABASE_KEY }}" >> .env + + - name: run update script - packages/crawling/crawlYoutube.ts + working-directory: packages/crawling + run: pnpm run ky-youtube + + - name: Commit and push changes to feat/songUpdate branch + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + git checkout feat/songUpdate + + git add . + if git diff --cached --quiet; then + echo "✅ No changes to commit" + else + git commit -m "chore: update crawled TJ song data [skip ci]" + git push origin feat/songUpdate + fi + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/apps/web/src/hooks/useSearchSong.ts b/apps/web/src/hooks/useSearchSong.ts index 9613556..386f895 100644 --- a/apps/web/src/hooks/useSearchSong.ts +++ b/apps/web/src/hooks/useSearchSong.ts @@ -25,10 +25,10 @@ export default function useSearchSong() { const [saveModalType, setSaveModalType] = useState(''); const [selectedSaveSong, setSelectedSaveSong] = useState(null); // const { data: searchResults, isLoading } = useSearchSongQuery(query, searchType, isAuthenticated); - const { mutate: toggleToSing } = useToggleToSingMutation(); - const { mutate: toggleLike } = useToggleLikeMutation(); - const { mutate: postSong } = useSaveMutation(); - const { mutate: moveSong } = useMoveSaveSongMutation(); + const { mutate: toggleToSing, isPending: isToggleToSingPending } = useToggleToSingMutation(); + const { mutate: toggleLike, isPending: isToggleLikePending } = useToggleLikeMutation(); + const { mutate: postSong, isPending: isPostSongPending } = useSaveMutation(); + const { mutate: moveSong, isPending: isMoveSongPending } = useMoveSaveSongMutation(); const { data: searchResults, @@ -56,6 +56,11 @@ export default function useSearchSong() { toast.error('로그인이 필요해요.'); return; } + + if (isToggleToSingPending) { + toast.error('요청 중입니다. 잠시 후 다시 시도해주세요.'); + return; + } toggleToSing({ songId, method, query, searchType }); }; @@ -64,6 +69,11 @@ export default function useSearchSong() { toast.error('로그인이 필요해요.'); return; } + + if (isToggleLikePending) { + toast.error('요청 중입니다. 잠시 후 다시 시도해주세요.'); + return; + } toggleLike({ songId, method, query, searchType }); }; @@ -72,15 +82,25 @@ export default function useSearchSong() { toast.error('로그인이 필요해요.'); return; } + setSelectedSaveSong(song); setSaveModalType(method === 'POST' ? 'POST' : 'PATCH'); }; const postSaveSong = async (songId: string, folderName: string) => { + if (isPostSongPending) { + toast.error('요청 중입니다. 잠시 후 다시 시도해주세요.'); + return; + } postSong({ songId, folderName, query, searchType }); }; const patchSaveSong = async (songId: string, folderId: string) => { + if (isMoveSongPending) { + toast.error('요청 중입니다. 잠시 후 다시 시도해주세요.'); + return; + } + moveSong({ songIdArray: [songId], folderId }); }; diff --git a/apps/web/src/queries/tosingSongQuery.ts b/apps/web/src/queries/tosingSongQuery.ts index 0dc054f..fe6c70f 100644 --- a/apps/web/src/queries/tosingSongQuery.ts +++ b/apps/web/src/queries/tosingSongQuery.ts @@ -102,35 +102,6 @@ export function useDeleteToSingSongMutation() { }); } -// 여러 곡 부를 노래 삭제 - 미사용? - -// export function useDeleteToSingSongArrayMutation() { -// const queryClient = useQueryClient(); - -// return useMutation({ -// mutationFn: (songIds: string[]) => deleteToSingSongArray({ songIds }), -// onMutate: async (songIds: string[]) => { -// queryClient.cancelQueries({ queryKey: ['toSingSong'] }); -// const prev = queryClient.getQueryData(['toSingSong']); -// queryClient.setQueryData(['toSingSong'], (old: ToSingSong[]) => -// old.filter(song => !songIds.includes(song.songs.id)), -// ); -// return { prev }; -// }, -// onError: (error, variables, context) => { -// console.error('error', error); -// alert(error.message ?? 'DELETE 실패'); -// queryClient.setQueryData(['toSingSong'], context?.prev); -// }, -// onSettled: () => { -// queryClient.invalidateQueries({ queryKey: ['toSingSong'] }); -// queryClient.invalidateQueries({ queryKey: ['likeSong'] }); -// queryClient.invalidateQueries({ queryKey: ['saveSongFolder'] }); -// queryClient.invalidateQueries({ queryKey: ['recentSingLog'] }); -// }, -// }); -// } - // 🎵 부를 노래 순서 변경 export function usePatchToSingSongMutation() { const queryClient = useQueryClient(); diff --git a/packages/crawling/package.json b/packages/crawling/package.json index 92b0417..51aa2e2 100644 --- a/packages/crawling/package.json +++ b/packages/crawling/package.json @@ -9,7 +9,7 @@ "scripts": { "ky-open": "tsx src/findKYByOpen.ts", "ky-youtube": "tsx src/crawling/crawlYoutube.ts", - "ky-valid": "tsx src/crawling/crawlYoutubeValid.ts", + "ky-youtube-ubuntu": "tsx src/crawling/crawlYoutubeUbuntu.ts", "ky-update": "pnpm run ky-youtube & pnpm run ky-valid", "trans": "tsx src/postTransDictionary.ts", "recent-tj": "tsx src/crawling/crawlRecentTJ.ts", diff --git a/packages/crawling/src/assets/crawlKYYoutubeFailedList.txt b/packages/crawling/src/assets/crawlKYYoutubeFailedList.txt index f39258d..32cc8e9 100644 --- a/packages/crawling/src/assets/crawlKYYoutubeFailedList.txt +++ b/packages/crawling/src/assets/crawlKYYoutubeFailedList.txt @@ -25805,3 +25805,4 @@ Nightwalker-텐(TEN) My Love Mine All Mine-Mitski MONSTERS(최강야구OST)-이원석 Lose Control-Teddy Swims +Fruit Fly-Leah Dou,검정치마 diff --git a/packages/crawling/src/crawling/crawlRecentTJ.ts b/packages/crawling/src/crawling/crawlRecentTJ.ts index a173c7c..8df2fa3 100644 --- a/packages/crawling/src/crawling/crawlRecentTJ.ts +++ b/packages/crawling/src/crawling/crawlRecentTJ.ts @@ -15,6 +15,7 @@ dotenv.config(); // action 우분투 환경에서의 호환을 위해 추가 const browser = await puppeteer.launch({ + headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox'], }); diff --git a/packages/crawling/src/crawling/crawlYoutube.ts b/packages/crawling/src/crawling/crawlYoutube.ts index 3387d8e..80eee5c 100644 --- a/packages/crawling/src/crawling/crawlYoutube.ts +++ b/packages/crawling/src/crawling/crawlYoutube.ts @@ -15,7 +15,11 @@ import { isValidKYExistNumber } from './isValidKYExistNumber'; // youtube에서 KY 노래방 번호 크롤링 // crawlYoutubeValid에서 진행하는 실제 사이트 검증도 포함 -const browser = await puppeteer.launch(); +// action 우분투 환경에서의 호환을 위해 추가 +const browser = await puppeteer.launch({ + headless: true, +}); + const page = await browser.newPage(); const baseUrl = 'https://www.youtube.com/@KARAOKEKY/search'; @@ -62,17 +66,22 @@ const data = await getSongsKyNullDB(); const failedSongs = loadCrawlYoutubeFailedKYSongs(); console.log('getSongsKyNullDB : ', data.length); +console.log(failedSongs.size); let index = 0; for (const song of data) { + // 테스트를 위해 100회 반복 후 종료시키기 + if (index >= 100) { + break; + } + const query = song.title + '-' + song.artist; if (failedSongs.has(query)) { + console.log('failedSongs has : ', query); continue; } - console.log(song.title, ' - ', song.artist); - let resultKyNum = null; try { resultKyNum = await scrapeSongNumber(query); @@ -97,6 +106,7 @@ for (const song of data) { } else saveCrawlYoutubeFailedKYSongs(song.title, song.artist); index++; + console.log(query); console.log('scrapeSongNumber : ', index); } diff --git a/packages/crawling/src/crawling/crawlYoutubeUbuntu.ts b/packages/crawling/src/crawling/crawlYoutubeUbuntu.ts new file mode 100644 index 0000000..4e7c223 --- /dev/null +++ b/packages/crawling/src/crawling/crawlYoutubeUbuntu.ts @@ -0,0 +1,127 @@ +import * as cheerio from 'cheerio'; +import puppeteer from 'puppeteer'; + +import { getSongsKyNullDB } from '@/supabase/getDB'; +import { updateSongsKyDB } from '@/supabase/updateDB'; +import { Song } from '@/types'; +import { + loadCrawlYoutubeFailedKYSongs, + saveCrawlYoutubeFailedKYSongs, + updateDataLog, +} from '@/utils/logData'; + +import { isValidKYExistNumber } from './isValidKYExistNumber'; + +// youtube에서 KY 노래방 번호 크롤링 +// crawlYoutubeValid에서 진행하는 실제 사이트 검증도 포함 + +// action 우분투 환경에서의 호환을 위해 추가 +// const browser = await puppeteer.launch({ +// headless: true, +// executablePath: '/usr/bin/chromium-browser', // 또는 "/usr/bin/chromium" +// args: [ +// '--no-sandbox', +// '--disable-setuid-sandbox', +// '--disable-dev-shm-usage', // 리눅스 메모리 제한 대응 +// '--disable-gpu', +// '--disable-infobars', +// '--single-process', +// '--window-size=1920,1080', +// ], +// }); + +const browser = await puppeteer.launch({ + headless: true, + args: ['--no-sandbox', '--disable-setuid-sandbox'], +}); + +const page = await browser.newPage(); + +const baseUrl = 'https://www.youtube.com/@KARAOKEKY/search'; + +const scrapeSongNumber = async (query: string) => { + const searchUrl = `${baseUrl}?query=${encodeURIComponent(query)}`; + + // page.goto의 waitUntil 문제였음! + await page.goto(searchUrl, { + waitUntil: 'networkidle2', + timeout: 0, + }); + + const html = await page.content(); + const $ = cheerio.load(html); + + // id contents 의 첫번째 ytd-item-section-renderer 찾기 + // const firstItem = $("#contents ytd-item-section-renderer").first(); + + const firstItem = $('ytd-video-renderer').first(); + + // yt-formatted-string 찾기 + const title = firstItem.find('yt-formatted-string').first().text().trim(); + + const karaokeNumber = extractKaraokeNumber(title); + + return karaokeNumber; +}; + +const extractKaraokeNumber = (title: string) => { + // KY. 찾고 ) 가 올때까지 찾기 + const matchResult = title.match(/KY\.\s*(\d{2,5})\)/); + const karaokeNumber = matchResult ? matchResult[1] : null; + return karaokeNumber; +}; + +const updateData = async (data: Song) => { + const result = await updateSongsKyDB(data); + updateDataLog(result.success, 'crawlYoutubeSuccess.txt'); + updateDataLog(result.failed, 'crawlYoutubeFailed.txt'); +}; + +const data = await getSongsKyNullDB(); +const failedSongs = loadCrawlYoutubeFailedKYSongs(); + +console.log('getSongsKyNullDB : ', data.length); +let index = 0; + +for (const song of data) { + // 테스트를 위해 100회 반복 후 종료시키기 + if (index >= 100) { + break; + } + + const query = song.title + '-' + song.artist; + + if (failedSongs.has(query)) { + continue; + } + + console.log(song.title, ' - ', song.artist); + + let resultKyNum = null; + try { + resultKyNum = await scrapeSongNumber(query); + } catch (error) { + continue; + } + + if (resultKyNum) { + let isValid = true; + try { + isValid = await isValidKYExistNumber(page, resultKyNum, song.title, song.artist); + } catch (error) { + continue; + } + + if (!isValid) { + saveCrawlYoutubeFailedKYSongs(song.title, song.artist); + continue; + } else { + await updateData({ ...song, num_ky: resultKyNum }); + } + } else saveCrawlYoutubeFailedKYSongs(song.title, song.artist); + + index++; + console.log('scrapeSongNumber : ', index); +} + +browser.close();