scraper.py

"""
Finn.no scraper using ScrapingAnt API.
"""

import time
import re
from typing import List, Optional

import requests
from bs4 import BeautifulSoup

from config import (
    SCRAPINGANT_API_URL,
    SELECTORS,
    DEFAULT_DELAY,
)
from models import CarListing, ListingCollection
from utils import (
    build_search_url,
    parse_listing_text,
    extract_listing_id,
    make_absolute_url,
    clean_text
)


class FinnScraper:
    """Scraper for Finn.no car listings."""

    def __init__(self, api_key: str, delay: float = DEFAULT_DELAY):
        """
        Initialize the scraper.

        Args:
            api_key: ScrapingAnt API key
            delay: Delay between requests in seconds
        """
        self.api_key = api_key
        self.delay = delay
        self.collection = ListingCollection()

    def _make_request(self, url: str) -> Optional[str]:
        """Make a request using ScrapingAnt API."""
        params = {
            "url": url,
            "x-api-key": self.api_key,
            "browser": "true"
        }

        try:
            response = requests.get(SCRAPINGANT_API_URL, params=params, timeout=120)
            response.raise_for_status()
            return response.text
        except requests.RequestException as e:
            print(f"Request failed for {url}: {e}")
            return None

    def _parse_listings(self, html: str) -> List[CarListing]:
        """Parse listings from HTML content."""
        soup = BeautifulSoup(html, 'html.parser')
        listings = []

        # Find all article elements
        articles = soup.select(SELECTORS["listing_article"])

        for article in articles:
            try:
                # Find the main link to listing details
                link = article.select_one(SELECTORS["listing_link"])
                if not link:
                    continue

                href = link.get("href", "")
                if "/mobility/item/" not in href:
                    continue

                # Extract listing URL and ID
                listing_url = make_absolute_url(href)
                listing_id = extract_listing_id(href)

                if not listing_id:
                    continue

                # Get all text content from the article
                article_text = article.get_text(separator="\n")

                # Parse the text to extract fields
                parsed = parse_listing_text(article_text)

                # Get image
                img = article.select_one("img")
                image_url = img.get("src", "") if img else ""

                listing = CarListing(
                    title=parsed.get("title", ""),
                    price=parsed.get("price", ""),
                    year=parsed.get("year", ""),
                    mileage=parsed.get("mileage", ""),
                    fuel_type=parsed.get("fuel_type", ""),
                    transmission=parsed.get("transmission", ""),
                    location=parsed.get("location", ""),
                    seller_name=parsed.get("seller_name", ""),
                    seller_type=parsed.get("seller_type", ""),
                    listing_id=listing_id,
                    listing_url=listing_url,
                    image_url=image_url,
                    time_posted=parsed.get("time_posted", "")
                )

                listings.append(listing)

            except Exception as e:
                print(f"Error parsing listing: {e}")
                continue

        return listings

    def scrape(
        self,
        max_pages: int = 2,
        min_price: Optional[int] = None,
        max_price: Optional[int] = None,
        min_year: Optional[int] = None,
        max_year: Optional[int] = None,
        fuel_type: Optional[str] = None
    ) -> int:
        """
        Scrape car listings.

        Args:
            max_pages: Maximum number of pages to scrape
            min_price: Minimum price filter (NOK)
            max_price: Maximum price filter (NOK)
            min_year: Minimum registration year filter
            max_year: Maximum registration year filter
            fuel_type: Fuel type filter

        Returns:
            Number of listings scraped
        """
        total_scraped = 0

        for page in range(1, max_pages + 1):
            url = build_search_url(
                page=page,
                min_price=min_price,
                max_price=max_price,
                min_year=min_year,
                max_year=max_year,
                fuel_type=fuel_type
            )

            print(f"Scraping page {page}: {url}")

            html = self._make_request(url)
            if not html:
                print(f"Failed to fetch page {page}")
                continue

            listings = self._parse_listings(html)
            added = self.collection.add_many(listings)

            print(f"Page {page}: scraped {len(listings)} listings, {added} new")
            total_scraped += added

            # Check if we got listings
            if len(listings) == 0:
                print("No more listings found, stopping pagination")
                break

            # Delay between requests
            if page < max_pages:
                time.sleep(self.delay)

        return total_scraped

    def get_results(self) -> ListingCollection:
        """Get the collection of scraped listings."""
        return self.collection

    def export_csv(self, filepath: str) -> None:
        """Export results to CSV."""
        self.collection.to_csv(filepath)

    def export_json(self, filepath: str) -> None:
        """Export results to JSON."""
        self.collection.to_json(filepath)