import logging from datetime import datetime, timedelta, timezone from repositories.competitor_repository import competitor_repository from repositories.product_repository import product_repository from repositories.price_history_repository import price_history_repository from repositories.scrape_repository import scrape_repository from repositories.ngs_meta_repository import ngs_meta_repository from services import alert_service from services.url_utils import is_generic_homepage logger = logging.getLogger(__name__) # A price move smaller than this is normal noise, not alert-worthy — # matches the >5% threshold in CLAUDE.md's n8n price-change-alert spec. PRICE_CHANGE_ALERT_THRESHOLD_PERCENT = 5 def needs_refresh(competitor): """ Decides whether a competitor needs a live re-scrape (REFRESH path) or can be served from PocketBase (FAST path). Data flow: competitor record → change_detected flag set? → True (refresh) → last_scraped missing or older than CACHE_STALENESS_HOURS? → True (refresh) → otherwise → False (fast path is safe) """ if competitor.get('change_detected'): return True last_scraped = competitor.get('last_scraped') if not last_scraped: return True try: scraped_at = datetime.fromisoformat(last_scraped.replace('Z', '+00:00')) except (ValueError, AttributeError): return True # CACHE_STALENESS_HOURS' canonical, adjustable home is core/scraper.py # per CLAUDE.md §7 ("in core/scraper.py — adjust here only") — imported # lazily here for the same reason every other core.* reference in this # file is lazy: this module must stay importable independent of Phase 2. from core.scraper import CACHE_STALENESS_HOURS age = datetime.now(timezone.utc) - scraped_at return age > timedelta(hours=CACHE_STALENESS_HOURS) def analyse_from_cache(tenant_id, competitor_id, context): """ FAST path analysis — uses cached PocketBase data instead of a live scrape. The LLM's only job is to write a fresh comparative narrative against the client's *current* context and reassess relevancy — it does NOT re-extract trip/pricing fields from raw page content, since those already exist on the cached `products` record. The structured fields (tripName, price, duration, etc.) returned here come directly from that cached record, not from the LLM call. An earlier version returned the LLM's raw output as the entire result — since the prompt only ever asked for a narrative, with no schema, the model would invent its own ad-hoc JSON shape (bare `{"analysis": "..."}`, or with extra invented keys) that never matched what ResultsTable.vue actually renders (item.result.tripName, .majorityPrice/.gbpPrice, .duration, .comments) — the dashboard showed a "complete" job with an empty-looking results table because none of those fields were ever present. Now the LLM output is merged into (not used as) the returned dict. Data flow: competitor_id → PocketBase products (latest) → structured fields mapped from the cached record (trip_name → tripName, majority_price_usd → majorityPrice, etc.) → client trip context (destination/duration/style) → LLM prompt constrained to {comments, relevancy} only (via core.extractor) → cached structured fields + fresh comments/relevancy merged and returned """ products = product_repository.get_latest_for_competitor(competitor_id, limit=1) if not products: raise ValueError('No cached product data available for fast path') latest = products[0] # core.extractor is a Phase 2 module — imported lazily so this # service is importable before the scraping engine refactor lands. from core.extractor import extract_with_openrouter prompt = ( 'You are a travel industry analyst. Given this already-extracted ' 'competitor trip data, assess it against ' f"the client's product context: {context}.\n\nCompetitor data:\n{latest}\n\n" 'Return ONLY a valid JSON object:\n' '{\n' ' "comments": "short comparative analysis: key differences, strengths, weaknesses, pricing position",\n' ' "relevancy": relevancy score 1-5 as integer\n' '}' ) narrative = extract_with_openrouter(prompt, content='') result = { 'link': latest.get('url'), 'tripName': latest.get('trip_name'), 'tripCode': latest.get('trip_code'), 'serviceLevel': latest.get('service_level'), 'groupSize': latest.get('group_size'), 'duration': latest.get('duration_days'), 'meals': latest.get('meals'), 'startLocation': latest.get('start_location'), 'endLocation': latest.get('end_location'), 'departures': latest.get('departures'), 'seasonality': latest.get('seasonality'), 'startDays': latest.get('start_days'), 'targetAudience': latest.get('target_audience'), 'hotels': latest.get('hotels'), 'activities': latest.get('activities'), 'majorityPrice': latest.get('majority_price_usd'), 'priceLow': latest.get('price_low_usd'), 'priceHigh': latest.get('price_high_usd'), 'dateUsed': latest.get('date_used'), 'audPrice': latest.get('aud_price'), 'cadPrice': latest.get('cad_price'), 'eurPrice': latest.get('eur_price'), 'gbpPrice': latest.get('gbp_price'), 'comments': narrative.get('comments', ''), 'relevancy': narrative.get('relevancy', latest.get('relevancy')), } # NGS fields live on a separate products_ngs_meta record (see # _save_ngs_meta()) — only look it up when this run is actually NGS, # to avoid an extra PocketBase round trip on the common Standard-tab # fast path. if context.get('tab_type') == 'NGS': ngs_meta = ngs_meta_repository.get_by_product_id(latest['id']) if ngs_meta: result['exclusiveAccess'] = ngs_meta.get('exclusive_access') result['groupLeader'] = ngs_meta.get('group_leader') result['sustainability'] = ngs_meta.get('sustainability') return result def detect_change(competitor_id, new_hash): """ Compares new_hash against the last content_hash recorded for this competitor. On a diff: sets competitor.change_detected = true and change_detected_at = now. Replaces Firecrawl Monitor's webhook role (CLAUDE.md §4/§10) — the daily/on-demand scrape itself is now the only place a change is ever detected. Data flow: competitor_id + new_hash → scrape_repository.get_last_hash_for_url() → no prior hash: True (first scrape — always process) → hash differs: competitor_repository.update() sets change flags → True → hash matches: no update → False """ last_hash = scrape_repository.get_last_hash_for_url(competitor_id) if last_hash is None: return True if last_hash != new_hash: competitor_repository.update(competitor_id, { 'change_detected': True, 'change_detected_at': datetime.now(timezone.utc).isoformat(), }) return True return False def run_competitor_analysis(tenant_id, competitor, context): """ Decides the fast vs refresh path for one competitor and executes it. Called per-competitor by jobs.run_research_job — failures here are caught by the caller so one competitor's failure never aborts the whole batch (CLAUDE.md §18 resilience rule). context['force_refresh'] (PM-initiated, via TripIntentForm.vue's collapsed "Advanced options" toggle) bypasses BOTH cache-shortcut points below — the initial needs_refresh() fast-path gate, and the post-scrape "content unchanged, reuse cached narrative" shortcut — so a forced run always ends up calling extract_with_openrouter() for real, never analyse_from_cache(). The content-hash bookkeeping itself (scrape_runs row, change_detected clearing) still runs normally even when forced, so future non-forced runs keep an accurate hash history. Data flow: competitor + context → needs_refresh() decision (or force_refresh) → FAST: analyse_from_cache() (seconds) → REFRESH: core.scraper.scrape() → compute_content_hash() → detect_change() against the last recorded hash → unchanged (and not force_refresh): change_detected cleared (confirmed stable since last look), skip LLM extraction, reuse cached narrative, bump last_scraped only (no wasted extraction cost) → changed (or force_refresh): core.extractor.extract_with_openrouter() → PocketBase products/price_history updated → last_scraped bumped, change_detected left True (stays visible until the next scrape confirms no further change) → result dict returned """ force_refresh = context.get('force_refresh', False) if not needs_refresh(competitor) and not force_refresh: logger.info(f'{competitor["name"]} — fast path (cache fresh)') fast_result = analyse_from_cache(tenant_id, competitor['id'], context) return { 'path': 'fast', 'result': fast_result, 'is_generic_page': is_generic_homepage(fast_result.get('link')), } logger.info(f'{competitor["name"]} — refresh path (force_refresh)' if force_refresh else f'{competitor["name"]} — refresh path (stale or changed)') # Phase 2 modules — imported lazily, same reasoning as above. from core.scraper import scrape, compute_content_hash from core.extractor import extract_with_openrouter from industries.adventure_travel.prompt import build_prompt # context['confirmed_url'] is the page confirmed at UrlConfirmation.vue # (a Firecrawl /map match or a manual swap) — falls back to the # competitor's stored root domain only when nothing was confirmed for # this run (e.g. the daily cron path, which has no trip-intent context # at all). Regression fix: this used to always scrape # competitor['website'] unconditionally, silently discarding whatever # trip-specific URL was actually matched/confirmed for this run. target_url = context.get('confirmed_url') or competitor['website'] # Computed once here and reused by both possible refresh-path returns # below — a meta-fact about which page was actually scraped this run, # not an extracted trip field, so it's a top-level sibling of `result`/ # `path`/`product` in the returned dict rather than nested inside `result`. is_generic = is_generic_homepage(target_url) page_data = scrape(target_url, country=competitor.get('primary_market')) if not page_data.get('success'): raise ValueError(page_data.get('text', 'Scrape failed')) content_hash = compute_content_hash(page_data['text']) changed = detect_change(competitor['id'], content_hash) # Every refresh scrape gets its own scrape_runs record purely to # persist content_hash for the next comparison — the outer batch # job's scrape_runs row (created before run_research_job) tracks # overall job status, not per-competitor hash history. scrape_repository.create({ 'tenant_id': tenant_id, 'competitor_id': competitor['id'], 'triggered_by': 'cron', 'status': 'complete', 'content_hash': content_hash, 'content_length': len(page_data['text']), 'hash_algorithm': 'sha256', 'competitors_total': 1, 'competitors_done': 1, 'results': {}, 'completed_at': datetime.now(timezone.utc).isoformat(), }) if not changed and not force_refresh: # Confirmed no further change since the last scrape — this is # the moment change_detected gets cleared. If it was never true # to begin with, this is a harmless no-op. logger.info(f'{competitor["name"]} — content unchanged, skipping LLM extraction') competitor_repository.update(competitor['id'], { 'change_detected': False, 'last_scraped': datetime.now(timezone.utc).isoformat(), }) return { 'path': 'refresh', 'result': analyse_from_cache(tenant_id, competitor['id'], context), 'unchanged': True, 'is_generic_page': is_generic, } if not changed: # force_refresh bypasses the "reuse cache" shortcut, but the hash # comparison itself is still accurate — content really is # unchanged — so change_detected is still cleared here. Only the # decision to skip real extraction is bypassed; falls through to # a real extraction below instead of returning early. logger.info(f'{competitor["name"]} — unchanged but force_refresh requested, running real extraction anyway') competitor_repository.update(competitor['id'], {'change_detected': False}) prompt = build_prompt( competitor['name'], page_data, context.get('product_name'), context.get('destination'), context.get('duration'), context.get('travel_style'), company_name=context.get('company_name'), is_ngs=context.get('tab_type') == 'NGS' ) extracted = extract_with_openrouter(prompt, content=page_data['text']) product = _save_scrape_result(tenant_id, competitor, extracted, page_data) # change_detected is deliberately left as detect_change() set it # (True) rather than reset here — detection and extraction now # happen in the same scrape, so there's no separate "handled" event # to reset it on. It stays visible (CompetitorsPage's "⚠ Changed" # badge, Apps Script sidebar status, tomorrow's n8n cron filter) # until the *next* scrape confirms no further change and clears it # in the branch above — at worst one extra hash-only confirmation # scrape per genuine change, no LLM cost since it lands on the # unchanged branch. competitor_repository.update(competitor['id'], { 'last_scraped': datetime.now(timezone.utc).isoformat(), }) return {'path': 'refresh', 'result': extracted, 'product': product, 'is_generic_page': is_generic} def _save_scrape_result(tenant_id, competitor, extracted, page_data): """ Persists one competitor's freshly-extracted trip data, deciding between "this is a new product" and "this is an update to a product we already track" by matching on (competitor_id, trip_name) — the same dedup key CLAUDE.md's is_new detection and price-history diffing rely on (§15/§19 Phase 5). Data flow: extracted fields → product_repository.get_by_competitor_and_name() → no match: create product with is_new=True → alert_service.create_alert('new_product') → match found: compare majority_price_usd → changed: price_history record written, product updated, alert_service.create_alert('price_change') if the move exceeds PRICE_CHANGE_ALERT_THRESHOLD_PERCENT → unchanged: product's last_seen/fields refreshed, no alert → product record returned either way """ now = datetime.now(timezone.utc).isoformat() trip_name = extracted.get('tripName') new_price = extracted.get('majorityPrice') common_fields = { 'trip_name': trip_name, 'url': page_data['url'], 'destination': extracted.get('destination'), 'duration_days': extracted.get('duration'), 'majority_price_usd': new_price, 'price_low_usd': extracted.get('priceLow'), 'price_high_usd': extracted.get('priceHigh'), 'aud_price': extracted.get('audPrice'), 'cad_price': extracted.get('cadPrice'), 'eur_price': extracted.get('eurPrice'), 'gbp_price': extracted.get('gbpPrice'), 'date_used': extracted.get('dateUsed'), 'comments': extracted.get('comments'), 'relevancy': extracted.get('relevancy'), # These all already have PocketBase columns and were previously # extracted by the LLM (industries/adventure_travel/prompt.py's # build_prompt() JSON schema) but silently dropped on write — the # dashboard's transposed results grid (ResultsTable.vue / # resultFields.js) needs every one of these to mirror the legacy # Sheet's row layout. 'trip_code': extracted.get('tripCode'), 'service_level': extracted.get('serviceLevel'), 'group_size': extracted.get('groupSize'), 'meals': extracted.get('meals'), 'start_location': extracted.get('startLocation'), 'end_location': extracted.get('endLocation'), 'departures': extracted.get('departures'), 'seasonality': extracted.get('seasonality'), 'start_days': extracted.get('startDays'), 'target_audience': extracted.get('targetAudience'), 'hotels': extracted.get('hotels'), 'activities': extracted.get('activities'), 'last_seen': now, 'is_active': True, } existing = product_repository.get_by_competitor_and_name(competitor['id'], trip_name) if not existing: product = product_repository.create({ 'tenant_id': tenant_id, 'competitor_id': competitor['id'], 'is_new': True, 'first_seen': now, **common_fields, }) alert_service.create_alert( tenant_id, competitor['id'], 'new_product', f"{competitor['name']} added a new product: {trip_name or 'Unnamed trip'}", product_id=product['id'], ) _save_ngs_meta(product['id'], extracted) return product product = product_repository.update(existing['id'], {**common_fields, 'is_new': False}) old_price = existing.get('majority_price_usd') if old_price is not None and new_price is not None and old_price != new_price: change_amount = new_price - old_price change_percent = round((change_amount / old_price) * 100, 1) if old_price else None price_history_repository.create({ 'tenant_id': tenant_id, 'product_id': product['id'], 'majority_price_usd': new_price, 'price_low_usd': extracted.get('priceLow'), 'price_high_usd': extracted.get('priceHigh'), 'aud_price': extracted.get('audPrice'), 'cad_price': extracted.get('cadPrice'), 'eur_price': extracted.get('eurPrice'), 'gbp_price': extracted.get('gbpPrice'), 'date_used': extracted.get('dateUsed'), 'change_field': 'majorityPrice', 'change_amount': change_amount, 'change_percent': change_percent, }) if change_percent is not None and abs(change_percent) > PRICE_CHANGE_ALERT_THRESHOLD_PERCENT: direction = 'dropped' if change_percent < 0 else 'raised' alert_service.create_alert( tenant_id, competitor['id'], 'price_change', f"{competitor['name']} {direction} {trip_name or 'a product'} {change_percent}%", product_id=product['id'], change_amount=change_amount, change_percent=change_percent, ) _save_ngs_meta(product['id'], extracted) return product def _save_ngs_meta(product_id, extracted): """ Upserts NGS-only extraction fields (exclusiveAccess/groupLeader/ sustainability) into products_ngs_meta, keyed on product_id. Only fires when build_prompt() actually asked for these fields (is_ngs=True in run_competitor_analysis()) — inferred here from their presence in `extracted` rather than threading a tab_type/context param through _save_scrape_result()'s already-long signature, since a Standard-tab extraction never has these keys at all. Silently no-ops for a Standard-tab run. """ if not any(extracted.get(k) is not None for k in ('exclusiveAccess', 'groupLeader', 'sustainability')): return fields = { 'product_id': product_id, 'exclusive_access': extracted.get('exclusiveAccess'), 'group_leader': extracted.get('groupLeader'), 'sustainability': extracted.get('sustainability'), } existing = ngs_meta_repository.get_by_product_id(product_id) if existing: ngs_meta_repository.update(existing['id'], fields) else: ngs_meta_repository.create(fields)