import logging from datetime import datetime, timedelta, timezone from repositories.competitor_repository import competitor_repository from repositories.product_repository import product_repository logger = logging.getLogger(__name__) # Default staleness threshold — configurable here only, per CLAUDE.md §7. CACHE_STALENESS_HOURS = 24 def needs_refresh(competitor): """ Decides whether a competitor needs a live re-scrape (REFRESH path) or can be served from PocketBase (FAST path). Data flow: competitor record → change_detected flag set? → True (refresh) → last_scraped missing or older than CACHE_STALENESS_HOURS? → True (refresh) → otherwise → False (fast path is safe) """ if competitor.get('change_detected'): return True last_scraped = competitor.get('last_scraped') if not last_scraped: return True try: scraped_at = datetime.fromisoformat(last_scraped.replace('Z', '+00:00')) except (ValueError, AttributeError): return True age = datetime.now(timezone.utc) - scraped_at return age > timedelta(hours=CACHE_STALENESS_HOURS) def analyse_from_cache(tenant_id, competitor_id, context): """ FAST path analysis — uses cached PocketBase data instead of a live scrape. The LLM receives already-structured JSON and generates the analysis narrative, comments, and relevancy score only — it does NOT re-extract fields from raw page content. Data flow: competitor_id → PocketBase products (latest) → structured JSON + client trip context (destination/duration/style) → LLM prompt (analysis only, no extraction, via core.extractor) → narrative + comments + relevancy score returned """ products = product_repository.get_latest_for_competitor(competitor_id, limit=1) if not products: raise ValueError('No cached product data available for fast path') latest = products[0] # core.extractor is a Phase 2 module — imported lazily so this # service is importable before the scraping engine refactor lands. from core.extractor import extract_with_openrouter prompt = ( 'You are a travel industry analyst. Given this already-extracted ' 'competitor trip data, write a short comparative analysis against ' f"the client's product context: {context}.\n\nCompetitor data:\n{latest}" ) return extract_with_openrouter(prompt, content='') def run_competitor_analysis(tenant_id, competitor, context): """ Decides the fast vs refresh path for one competitor and executes it. Called per-competitor by jobs.run_research_job — failures here are caught by the caller so one competitor's failure never aborts the whole batch (CLAUDE.md §18 resilience rule). Data flow: competitor + context → needs_refresh() decision → FAST: analyse_from_cache() (seconds) → REFRESH: core.scraper.scrape() → core.extractor.extract_with_openrouter() → PocketBase products/price_history updated → competitor.change_detected reset, last_scraped bumped → analysis narrative generated from the fresh data → result dict returned """ if not needs_refresh(competitor): logger.info(f'{competitor["name"]} — fast path (cache fresh)') return {'path': 'fast', 'result': analyse_from_cache(tenant_id, competitor['id'], context)} logger.info(f'{competitor["name"]} — refresh path (stale or changed)') # Phase 2 modules — imported lazily, same reasoning as above. from core.scraper import scrape from core.extractor import extract_with_openrouter from industries.adventure_travel.prompt import build_prompt page_data = scrape(competitor['website'], country=competitor.get('primary_market')) if not page_data.get('success'): raise ValueError(page_data.get('text', 'Scrape failed')) prompt = build_prompt( competitor['name'], page_data, context.get('product_name'), context.get('destination'), context.get('duration'), context.get('travel_style'), is_ngs=context.get('tab_type') == 'NGS' ) extracted = extract_with_openrouter(prompt, content=page_data['text']) product = product_repository.create({ 'tenant_id': tenant_id, 'competitor_id': competitor['id'], 'trip_name': extracted.get('tripName'), 'url': page_data['url'], 'duration_days': extracted.get('duration'), 'majority_price_usd': extracted.get('majorityPrice'), 'price_low_usd': extracted.get('priceLow'), 'price_high_usd': extracted.get('priceHigh'), 'comments': extracted.get('comments'), 'relevancy': extracted.get('relevancy'), 'is_active': True, }) competitor_repository.update(competitor['id'], { 'change_detected': False, 'last_scraped': datetime.now(timezone.utc).isoformat(), }) return {'path': 'refresh', 'result': extracted, 'product': product}