code.py

from playwright.sync_api import sync_playwright
import time
import random
import os


def scrape_dunnes_persistent(url):

    with sync_playwright() as p:
        # Use persistent context to save cookies and session
        user_data_dir = os.path.join(os.getcwd(), 'browser_data')
        os.makedirs(user_data_dir, exist_ok=True)
        
        browser = p.chromium.launch_persistent_context(
            user_data_dir=user_data_dir,
            channel="chromium", 
            headless=True,
            viewport={'width': 1920, 'height': 1080},
            user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            locale='en-IE',
            timezone_id='Europe/Dublin',
            geolocation={'longitude': -6.2603, 'latitude': 53.3498},
            permissions=['geolocation'],
            color_scheme='light',
            args=[
                '--disable-blink-features=AutomationControlled',
                '--disable-dev-shm-usage',
            ],
            extra_http_headers={
                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
                'Accept-Language': 'en-IE,en;q=0.9',
                'Accept-Encoding': 'gzip, deflate, br',
                'DNT': '1',
                'Connection': 'keep-alive',
                'Upgrade-Insecure-Requests': '1',
            }
        )
        
        # Enhanced stealth script
        browser.add_init_script("""
            Object.defineProperty(navigator, 'webdriver', {
                get: () => undefined
            });
            
            window.chrome = {
                runtime: {},
                loadTimes: function() {},
                csi: function() {},
                app: {}
            };
            
            Object.defineProperty(navigator, 'languages', {
                get: () => ['en-IE', 'en']
            });
            
            Object.defineProperty(navigator, 'platform', {
                get: () => 'MacIntel'
            });
            
            Object.defineProperty(navigator, 'hardwareConcurrency', {
                get: () => 8
            });
            
            Object.defineProperty(navigator, 'deviceMemory', {
                get: () => 8
            });
        """)
        
        # Get the first page (persistent context creates one automatically)
        pages = browser.pages
        if pages:
            page = pages[0]
        else:
            page = browser.new_page()
        
        try:
            print("Step 1: Establishing session with homepage (Persistent Context)...")
            print("Note: This will save cookies for future runs")
            
            page.goto("https://www.dunnesstoresgrocery.com", wait_until='domcontentloaded', timeout=30000)
            time.sleep(random.uniform(5, 8))
            
            # Check for error
            if 'failed some security checks' in page.content():
                print("Challenge detected, waiting...")
                time.sleep(20)
                page.reload(wait_until='networkidle', timeout=60000)
                time.sleep(random.uniform(3, 5))
            
            print(f"Step 2: Navigating to product page: {url}")
            page.goto(url, wait_until='domcontentloaded', timeout=60000)
            
            time.sleep(random.uniform(5, 8))
            
            # Check for error
            content_check = page.content()
            if 'failed some security checks' in content_check or 'Error</title>' in content_check:
                print("Challenge detected, waiting longer...")
                time.sleep(25)
                page.reload(wait_until='networkidle', timeout=60000)
                time.sleep(random.uniform(5, 8))
                content_check = page.content()
            
            # Human-like behavior
            for _ in range(random.randint(3, 5)):
                page.mouse.move(random.randint(100, 1800), random.randint(100, 900))
                time.sleep(random.uniform(1, 2))
            
            for i in range(random.randint(2, 4)):
                scroll_amount = random.randint(300, 600) * (i + 1)
                page.evaluate(f"window.scrollTo(0, {scroll_amount})")
                time.sleep(random.uniform(2, 3))
            
            time.sleep(random.uniform(3, 5))
            
            content = page.content()
            print(f"Successfully fetched {len(content)} characters")
            
            if 'failed some security checks' in content or 'Error</title>' in content:
                print("⚠ Warning: Page may contain error message")
            else:
                print("✓ Session saved! Next run should be faster.")
            
            return content
            
        except Exception as e:
            print(f"Error: {e}")
            try:
                page.screenshot(path='error_persistent.png')
                print("Screenshot saved as error_persistent.png")
            except:
                pass
            return None
        finally:
            browser.close()


if __name__ == "__main__":
    url = "https://www.dunnesstoresgrocery.com/sm/delivery/rsid/258/categories/drinks/mixers-id-51042?f=Breadcrumb%3Agrocery%2Fdrinks%2Fmixers"
    result = scrape_dunnes_persistent(url)
    if result:
        print("✓ Success!")
        with open('dunnes_product_persistent.html', 'w', encoding='utf-8') as f:
            f.write(result)
        print("HTML saved to dunnes_product_persistent.html")
    else:
        print("✗ Failed")