#!/usr/bin/env python3
"""
Avito Product Data Fetcher

Extracts:
- Product images
- Product name, price
- Full product description
- Location, seller info

Uses Playwright for scraping.
"""

import json
import re
import time
from typing import Dict, List, Optional
from pathlib import Path


def extract_avito_id(url: str) -> Optional[str]:
    """Extract product ID from Avito URL.

    Example: https://www.avito.ru/ryazan/sport_i_otdyh/konki_progulochnye_bauer_2518117887
    Returns: 2518117887
    """
    # Strip query parameters
    url_path = url.split('?')[0].rstrip('/')
    match = re.search(r'_(\d+)$', url_path)
    return match.group(1) if match else None


def is_avito_url(url: str) -> bool:
    """Check if URL is an Avito product page."""
    url_path = url.split('?')[0].rstrip('/')
    return 'avito.ru/' in url and re.search(r'_\d+$', url_path) is not None


COOKIES_FILE = Path(__file__).parent / "avito_cookies.json"


def load_cookies() -> List[Dict]:
    """Load cookies from file."""
    if COOKIES_FILE.exists():
        with open(COOKIES_FILE, 'r') as f:
            return json.load(f)
    return []


def fetch_avito_product(url: str, headless: bool = True, proxy: str = None) -> Optional[Dict]:
    """
    Fetch Avito product data using Playwright.

    Args:
        url: Avito product URL
        headless: Run browser in headless mode
        proxy: SOCKS5 proxy URL (e.g., "socks5://PROXY_HOST:1080")

    Returns:
        Product data dict with name, images, price, description
    """
    import traceback
    import logging
    logger = logging.getLogger(__name__)

    logger.info(f"fetch_avito_product called with url={url[:50]}...")

    try:
        from playwright.sync_api import sync_playwright
        logger.info("Playwright imported successfully")
    except ImportError as e:
        logger.error(f"Playwright import failed: {e}")
        print("Playwright not installed. Install with: pip install playwright && playwright install chromium")
        return None

    product_id = extract_avito_id(url)
    if not product_id:
        logger.error(f"Could not extract Avito ID from URL: {url}")
        print(f"Could not extract Avito ID from URL: {url}")
        return None

    logger.info(f"Product ID extracted: {product_id}")

    # Wrap everything in try-except to catch any errors
    try:
        result = _fetch_avito_product_impl(url, product_id, headless, proxy)
        logger.info(f"_fetch_avito_product_impl returned: {type(result)}")
        return result
    except Exception as e:
        logger.error(f"Avito fetch error: {e}")
        logger.error(traceback.format_exc())
        print(f"Avito fetch error: {e}")
        traceback.print_exc()
        return None


def _fetch_avito_product_impl(url: str, product_id: str, headless: bool, proxy: str) -> Optional[Dict]:
    """Internal implementation of Avito fetch."""
    from playwright.sync_api import sync_playwright

    # Default: no proxy (try direct first)
    # Set AVITO_PROXY env var to use proxy, e.g., "socks5://PROXY_HOST:1080"
    if proxy is None:
        import os
        proxy = os.environ.get("AVITO_PROXY", "")

    with sync_playwright() as p:
        # Launch browser with stealth options
        launch_args = {
            "headless": headless,
            "args": [
                '--disable-blink-features=AutomationControlled',
                '--no-sandbox',
            ]
        }

        # Add proxy if specified
        if proxy:
            launch_args["proxy"] = {"server": proxy}
            print(f"  Using proxy: {proxy}")

        # Try Firefox first (less detectable), fallback to Chromium
        try:
            browser = p.firefox.launch(**{k: v for k, v in launch_args.items() if k != 'args'})
            browser_type = "firefox"
        except Exception:
            browser = p.chromium.launch(**launch_args)
            browser_type = "chromium"

        print(f"  Using browser: {browser_type}")

        context = browser.new_context(
            viewport={"width": 1920, "height": 1080},
            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:122.0) Gecko/20100101 Firefox/122.0" if browser_type == "firefox" else "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
            locale="ru-RU",
            timezone_id="Europe/Moscow",
            geolocation={"latitude": 55.7558, "longitude": 37.6173},
            permissions=["geolocation"],
        )

        # Load saved cookies to bypass captcha
        cookies = load_cookies()
        if cookies:
            context.add_cookies(cookies)
            print(f"  Loaded {len(cookies)} cookies")

        page = context.new_page()

        # Better stealth - hide automation signals
        page.add_init_script("""
            // Hide webdriver
            Object.defineProperty(navigator, 'webdriver', {get: () => undefined});

            // Hide automation flags
            window.chrome = { runtime: {} };

            // Fix permissions
            const originalQuery = window.navigator.permissions.query;
            window.navigator.permissions.query = (parameters) => (
                parameters.name === 'notifications' ?
                    Promise.resolve({ state: Notification.permission }) :
                    originalQuery(parameters)
            );

            // Hide plugins length
            Object.defineProperty(navigator, 'plugins', {
                get: () => [1, 2, 3, 4, 5]
            });

            // Hide languages
            Object.defineProperty(navigator, 'languages', {
                get: () => ['ru-RU', 'ru', 'en-US', 'en']
            });
        """)

        try:
            # Navigate to product page
            print(f"Fetching Avito: {url}")
            page.goto(url, wait_until="domcontentloaded", timeout=30000)
            time.sleep(5)  # Wait longer for JS to load

            # Try multiple selectors for main content
            selectors_to_try = [
                'h1[itemprop="name"]',
                'h1[data-marker="item-view/title-info"]',
                'h1.title-info-title-text',
                '[data-marker="item-view/title"]',
                'h1',
            ]

            found = False
            for selector in selectors_to_try:
                try:
                    page.wait_for_selector(selector, timeout=5000)
                    found = True
                    print(f"  Found content with selector: {selector}")
                    break
                except:
                    continue

            if not found:
                # Check for IP block / captcha page
                if page.query_selector('button:has-text("Продолжить")'):
                    print("  Detected IP block page, clicking 'Продолжить'...")
                    page.click('button:has-text("Продолжить")')
                    time.sleep(5)

                    # Check again for content
                    for selector in selectors_to_try:
                        try:
                            page.wait_for_selector(selector, timeout=5000)
                            found = True
                            print(f"  Found content after captcha: {selector}")
                            break
                        except:
                            continue

            if not found:
                # Take screenshot for debugging
                page.screenshot(path="/tmp/avito_debug.png")
                print("  Could not find main content. Screenshot saved to /tmp/avito_debug.png")

            # Scroll to trigger lazy loading
            page.evaluate("window.scrollTo(0, 500)")
            time.sleep(1)

            # Extract product data
            product_data = page.evaluate(r"""() => {
                const result = {};

                // Extract name from h1
                const h1 = document.querySelector('h1');
                result.name = h1 ? h1.textContent.trim() : '';

                // Extract price
                const priceEl = document.querySelector('[itemprop="price"]') ||
                               document.querySelector('[data-marker="item-view/item-price"]') ||
                               document.querySelector('.style-item-price-text-_w822');
                result.price = priceEl ? priceEl.textContent.trim() : '';

                // Extract description
                const descEl = document.querySelector('[itemprop="description"]') ||
                              document.querySelector('[data-marker="item-view/item-description"]');
                result.description = descEl ? descEl.textContent.trim() : '';

                // Extract location
                const locationEl = document.querySelector('[data-marker="item-view/item-address"]') ||
                                  document.querySelector('.style-item-address__string-wt61A');
                result.location = locationEl ? locationEl.textContent.trim() : '';

                // Extract seller name
                const sellerEl = document.querySelector('[data-marker="seller-info/name"]') ||
                                document.querySelector('.style-seller-info-name-lsJsT a');
                result.seller = sellerEl ? sellerEl.textContent.trim() : '';

                // Extract category from breadcrumbs
                const breadcrumbs = document.querySelectorAll('[data-marker="breadcrumb-item"] span, .breadcrumbs-link-text');
                result.category = Array.from(breadcrumbs).map(el => el.textContent.trim()).join(' > ');

                // Extract images
                const images = [];

                // PRIORITY 1: Check for data attributes with full-size URLs (for lightbox/zoom)
                const imageWrappers = document.querySelectorAll(
                    '[data-marker="image-frame/image-wrapper"],' +
                    '[data-marker="image-frame"],' +
                    '[class*="gallery-img-frame"],' +
                    '[class*="image-frame"]'
                );

                imageWrappers.forEach(wrapper => {
                    // Check all data-* attributes for URLs
                    for (let attr of wrapper.attributes) {
                        if (attr.name.startsWith('data-') && attr.value) {
                            try {
                                // Try parsing as JSON (might be a data object)
                                const data = JSON.parse(attr.value);
                                if (data.url || data.src || data.image) {
                                    const url = data.url || data.src || data.image;
                                    if (url && url.includes('avito') && !images.includes(url)) {
                                        images.push(url.split('?')[0]);
                                    }
                                }
                            } catch {
                                // Try as plain URL
                                if (attr.value.startsWith('http') && attr.value.includes('avito') && !images.includes(attr.value)) {
                                    images.push(attr.value.split('?')[0]);
                                }
                            }
                        }
                    }
                });

                // PRIORITY 2: Get gallery images from <img> tags (fallback)
                const galleryImages = document.querySelectorAll(
                    '[data-marker="image-frame/image-wrapper"] img, ' +
                    '.gallery-img-frame img, ' +
                    '.images-preview-previewImageWrapper img, ' +
                    '[class*="image-frame"] img'
                );

                galleryImages.forEach(img => {
                    let src = img.src || img.getAttribute('data-src');
                    if (src && !src.includes('placeholder') && !images.includes(src)) {
                        // Convert thumbnails to full size (handles both .75x55. and /75x55/ formats)
                        src = src.replace(/([\.\/])(\d+x\d+)([\.\/])/, '$1' + '1280x960' + '$3');
                        // Strip ?cqp= parameter which causes thumbnails on img.avito.st CDN
                        src = src.split('?')[0];
                        images.push(src);
                    }
                });

                // Also try srcset for high-res images
                const srcsetImages = document.querySelectorAll('img[srcset]');
                srcsetImages.forEach(img => {
                    const srcset = img.getAttribute('srcset');
                    if (srcset) {
                        const matches = srcset.match(/https?:\/\/[^\s,]+/g);
                        if (matches) {
                            matches.forEach(src => {
                                if (!src.includes('placeholder') && !images.includes(src)) {
                                    // Strip ?cqp= parameter
                                    src = src.split('?')[0];
                                    images.push(src);
                                }
                            });
                        }
                    }
                });

                result.images = images;

                return result;
            }""")

            # Also try to get images from page scripts (Avito often has image data in JSON)
            script_images = page.evaluate(r"""() => {
                const images = [];
                const scripts = document.querySelectorAll('script');
                scripts.forEach(script => {
                    const text = script.textContent;
                    if (text && text.includes('"images"')) {
                        // Try to find image URLs in JSON
                        const matches = text.match(/"https?:\/\/[^"]*\\.(?:jpg|jpeg|png|webp)[^"]*"/gi);
                        if (matches) {
                            matches.forEach(m => {
                                const url = m.replace(/"/g, '');
                                if (!url.includes('placeholder') && !images.includes(url)) {
                                    images.push(url);
                                }
                            });
                        }
                    }
                });
                return images;
            }""")

            # Merge images
            all_images = list(set(product_data.get('images', []) + script_images))

            # Collect ALL possible URL variations (we'll filter by size after downloading)
            all_variants = []
            for img in all_images:
                if img and 'avito' in img and not 'placeholder' in img:
                    # Add MULTIPLE variants of each URL to maximize chances of getting full-size

                    # Variant 1: Original URL without query params
                    base_url = img.split('?')[0]
                    all_variants.append(base_url)

                    # Variant 2: Try with dimension transformation (for old avatars.avito.ru CDN)
                    transformed = re.sub(r'([\.\/])(\d+x\d+)([\.\/])', r'\g<1>1280x960\g<3>', base_url)
                    if transformed != base_url:
                        all_variants.append(transformed)

                    # Variant 3: Try different CDN subdomains (20, 40, 50, 00)
                    if 'img.avito.st' in base_url:
                        for cdn_num in ['00', '20', '40', '50']:
                            variant = re.sub(r'\d+\.img\.avito\.st', f'{cdn_num}.img.avito.st', base_url)
                            all_variants.append(variant)

            # Deduplicate while preserving order
            clean_images = []
            seen = set()
            for img in all_variants:
                if img not in seen:
                    seen.add(img)
                    clean_images.append(img)

            result = {
                "product_id": product_id,
                "name": product_data.get("name", ""),
                "url": url,
                "price": product_data.get("price", ""),
                "description": product_data.get("description", ""),
                "location": product_data.get("location", ""),
                "seller": product_data.get("seller", ""),
                "category": product_data.get("category", ""),
                "images": clean_images[:10],  # Limit to 10 images
                "source": "avito",
                # Avito doesn't have these, but include for compatibility
                "rating": None,
                "reviews_count": 0,
                "reviews": [],
                "brand": "",
            }

            print(f"  Found: {result['name'][:50]}...")
            print(f"  Images: {len(result['images'])}")

            return result

        except Exception as e:
            print(f"Error fetching Avito product: {e}")
            import traceback
            traceback.print_exc()
            return None

        finally:
            browser.close()


def download_avito_images(product_data: Dict, output_dir: Path) -> List[Path]:
    """Download product images to output directory.

    Strategy: Download ALL image candidates, then keep only the largest ones
    to filter out thumbnails automatically.
    """
    import httpx
    import os

    output_dir = Path(output_dir)
    output_dir.mkdir(parents=True, exist_ok=True)

    # Create temp directory for all downloads
    temp_dir = output_dir / "temp_all"
    temp_dir.mkdir(exist_ok=True)

    headers = {
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
        "Referer": "https://www.avito.ru/"
    }

    # Download ALL candidates
    candidates = []
    for i, img_url in enumerate(product_data.get("images", []), 1):
        try:
            # Determine extension
            ext = "jpg"
            if ".png" in img_url:
                ext = "png"
            elif ".webp" in img_url:
                ext = "webp"

            temp_path = temp_dir / f"candidate_{i}.{ext}"

            response = httpx.get(img_url, headers=headers, timeout=30, follow_redirects=True)
            if response.status_code == 200 and len(response.content) > 0:
                with open(temp_path, "wb") as f:
                    f.write(response.content)

                file_size = os.path.getsize(temp_path)
                candidates.append({
                    'path': temp_path,
                    'size': file_size,
                    'url': img_url
                })
                print(f"  Downloaded candidate {i}: {file_size/1024:.1f} KB")
            else:
                print(f"  Failed candidate {i}: HTTP {response.status_code}")

        except Exception as e:
            print(f"  Error downloading candidate {i}: {e}")

    # Sort by size (largest first)
    candidates.sort(key=lambda x: x['size'], reverse=True)

    # Filter: Keep files larger than 15KB OR top 7 largest
    MIN_SIZE = 15 * 1024  # 15KB threshold
    MAX_IMAGES = 7

    kept = []
    for candidate in candidates:
        if candidate['size'] >= MIN_SIZE or len(kept) < MAX_IMAGES:
            kept.append(candidate)
            if len(kept) >= MAX_IMAGES and candidate['size'] >= MIN_SIZE:
                break

    print(f"  Keeping {len(kept)} largest images (out of {len(candidates)} candidates)")

    # Move kept files to final location
    downloaded = []
    for i, candidate in enumerate(kept, 1):
        ext = candidate['path'].suffix
        final_path = output_dir / f"original_{i}{ext}"

        # Move from temp to final location
        candidate['path'].rename(final_path)
        downloaded.append(final_path)
        print(f"  Kept: {final_path.name} ({candidate['size']/1024:.1f} KB)")

    # Clean up temp directory
    try:
        import shutil
        shutil.rmtree(temp_dir)
    except:
        pass

    return downloaded


if __name__ == "__main__":
    import sys

    if len(sys.argv) < 2:
        print("Usage: python avito_fetcher.py <avito_url>")
        print("Example: python avito_fetcher.py https://www.avito.ru/ryazan/sport_i_otdyh/konki_progulochnye_bauer_2518117887")
        sys.exit(1)

    url = sys.argv[1]

    if not is_avito_url(url):
        print(f"Not a valid Avito URL: {url}")
        sys.exit(1)

    result = fetch_avito_product(url)

    if result:
        print("\n=== Product Data ===")
        print(json.dumps(result, indent=2, ensure_ascii=False))
    else:
        print("Failed to fetch product data")
        sys.exit(1)
