import os import logging import numpy as np from repositories.client_trips_repository import client_trips_repository logger = logging.getLogger(__name__) OPENROUTER_API_KEY = os.getenv('OPENROUTER_API_KEY') EMBEDDING_MODEL = 'openai/text-embedding-3-small' SIMILARITY_THRESHOLD = 0.75 def save_client_trip(tenant_id, destination, duration, travel_style, product_name=None, url=None, extracted=None, price_usd=None, activities=None): """ Upserts a client_trips record from a PM's trip-intent description (CLAUDE.md §8's TripIntentForm — destination/duration/travel_style, optionally a product name). Every /research submission calls this, which is the only place client_trips ever gets populated — without it, find_comparable_trips() has nothing on the client side to match competitor products against. url/extracted/price_usd/activities are optional and only ever passed by scrape_own_trip() below, once the tenant's own trip page has actually been scraped and extracted — the original typed-values-only call site in api.py's /research route passes none of these, so its behavior is unchanged. Dedupes on (tenant_id, trip_name) so re-running analysis for the same product doesn't create duplicate rows — it just keeps the existing one current. Data flow: destination/duration/travel_style/product_name (+ optional url/extracted/price_usd/activities from a real scrape) → trip_name resolved (product_name if given, else a destination+style fallback) → client_trips_repository.get_by_tenant_and_name() → found: update in place → not found: create → client_trips record returned """ trip_name = product_name or f'{destination} {travel_style}'.strip() payload = { 'tenant_id': tenant_id, 'trip_name': trip_name, 'destination': destination, 'duration_days': duration, } if url is not None: payload['url'] = url if extracted is not None: payload['extracted'] = extracted if price_usd is not None: payload['price_usd'] = price_usd if activities is not None: payload['activities'] = activities existing = client_trips_repository.get_by_tenant_and_name(tenant_id, trip_name) if existing: # A stale embedding would silently never get regenerated — # /internal/embed-products only embeds trips missing a vector — # so clear it whenever the text embed_trip() hashes actually changed. if existing.get('destination') != destination or existing.get('duration_days') != duration: payload['embedding'] = None return client_trips_repository.update(existing['id'], payload) return client_trips_repository.create(payload) def scrape_own_trip(tenant_id, website_url, context): """ Scrapes and extracts the tenant's OWN confirmed/matched trip page — the same Firecrawl-matched-or-manually-swapped URL pattern already used for competitors, pointed at the tenant's own site instead. Gives build_prompt()'s own_trip param and client_trips real ground-truth data instead of only the PM's typed destination/duration/style. Reuses the same CACHE_STALENESS_HOURS freshness window as competitors (core/scraper.py) so the tenant's own site isn't rescraped on every single analysis run — only when the cached client_trips row is stale or context['force_refresh'] is set, both exactly mirroring analysis_service.needs_refresh()'s competitor logic. Never raises — a side-effect failure here (site blocked, proxy error, extraction failure) must not fail the whole batch (CLAUDE.md §18). Logs and returns None on any failure, which build_prompt() treats identically to a tenant who never confirmed an own-trip URL. The returned dict always carries a `cached` bool (True when served from client_trips.extracted without a live scrape, False when just freshly extracted) — surfaced to the dashboard via jobs.py's results['own_trip'] so ResultsTable.vue can show the same "from PocketBase vs new scrape" pill on the own-trip column as every competitor column. `link` is always overwritten with the real website_url actually scraped, rather than trusting the LLM's own echoed `"link"` field verbatim — same reasoning as analysis_service._save_scrape_result() using page_data['url'], not extracted.get('link'), for a competitor's stored product URL. Data flow: website_url + context → trip_name resolved exactly as save_client_trip() does (product_name, else destination+style) → existing client_trips row fresh (and not force_refresh)? → return its cached `extracted` blob (cached=True) → otherwise: core.scraper.scrape() → build_own_trip_prompt() → core.extractor.extract_with_openrouter() → link corrected to the real website_url → save_client_trip() with url/extracted/price_usd/activities → extracted dict returned (cached=False) """ from datetime import datetime, timedelta, timezone trip_name = context.get('product_name') or f"{context.get('destination')} {context.get('travel_style')}".strip() existing = client_trips_repository.get_by_tenant_and_name(tenant_id, trip_name) force_refresh = context.get('force_refresh', False) if existing and existing.get('extracted') and not force_refresh: last_scraped = existing.get('last_scraped') if last_scraped: try: from core.scraper import CACHE_STALENESS_HOURS scraped_at = datetime.fromisoformat(last_scraped.replace('Z', '+00:00')) if datetime.now(timezone.utc) - scraped_at <= timedelta(hours=CACHE_STALENESS_HOURS): logger.info(f'Own trip for tenant {tenant_id} — cache fresh, reusing extracted data') return {**existing['extracted'], 'cached': True} except (ValueError, AttributeError): pass # unparseable timestamp — fall through and rescrape try: from core.scraper import scrape from core.extractor import extract_with_openrouter from industries.adventure_travel.prompt import build_own_trip_prompt page_data = scrape(website_url) if not page_data.get('success'): logger.warning(f'Own-trip scrape failed for tenant {tenant_id}: {page_data.get("text")}') return None prompt = build_own_trip_prompt( page_data, context.get('product_name'), context.get('destination'), context.get('duration'), context.get('travel_style'), context.get('company_name'), is_ngs=context.get('tab_type') == 'NGS', ) extracted = extract_with_openrouter(prompt, content=page_data['text']) extracted['link'] = website_url saved = save_client_trip( tenant_id, context.get('destination'), context.get('duration'), context.get('travel_style'), product_name=context.get('product_name'), url=website_url, extracted=extracted, price_usd=extracted.get('majorityPrice'), activities=extracted.get('activities'), ) client_trips_repository.update(saved['id'], {'last_scraped': datetime.now(timezone.utc).isoformat()}) return {**extracted, 'cached': False} except Exception as e: logger.error(f'scrape_own_trip failed for tenant {tenant_id}: {str(e)}') return None def embed_trip(trip): """ Generates a semantic embedding for a trip using OpenRouter's text-embedding-3-small, combining trip name, destination, duration, and activities into a single text representation first. Data flow: trip dict → formatted text string → OpenRouter embeddings API → vector list returned for storage in products.embedding / client_trips.embedding """ # Imported lazily so this module has no hard dependency on the # `openai` package at import time for callers that only need # find_comparable_trips() (pure math, no API calls). from openai import OpenAI text = ( f"{trip.get('trip_name', '')}\n" f"Destination: {trip.get('destination', '')}\n" f"Duration: {trip.get('duration_days', '')} days\n" f"Activities: {trip.get('activities', '')}\n" f"Start: {trip.get('start_location', '')}" ) client = OpenAI(api_key=OPENROUTER_API_KEY, base_url='https://openrouter.ai/api/v1') response = client.embeddings.create(model=EMBEDDING_MODEL, input=text) return response.data[0].embedding def _cosine_similarity(a, b): """Standard cosine similarity between two equal-length embedding vectors.""" a, b = np.array(a), np.array(b) denom = (np.linalg.norm(a) * np.linalg.norm(b)) if denom == 0: return 0.0 return float(np.dot(a, b) / denom) def find_comparable_trips(client_trips, competitor_trips, threshold=SIMILARITY_THRESHOLD): """ Calculates cosine similarity between every client trip and every competitor trip, returning matches above threshold sorted by similarity score descending. Data flow: client_trips + competitor_trips (each with .embedding) → pairwise cosine similarity → filter by threshold → sort by score descending → list of match dicts returned """ matches = [] for client_trip in client_trips: client_embedding = client_trip.get('embedding') if not client_embedding: continue for competitor_trip in competitor_trips: competitor_embedding = competitor_trip.get('embedding') if not competitor_embedding: continue score = _cosine_similarity(client_embedding, competitor_embedding) if score >= threshold: matches.append({ 'client_product': client_trip.get('trip_name'), 'competitor_product': competitor_trip.get('id'), 'similarity_score': round(score, 4), 'price_difference': _safe_diff( competitor_trip.get('majority_price_usd'), client_trip.get('price_usd') ), 'duration_difference': _safe_diff( competitor_trip.get('duration_days'), client_trip.get('duration_days') ), }) matches.sort(key=lambda m: m['similarity_score'], reverse=True) return matches def _safe_diff(a, b): """Returns a - b if both are present, else None — keeps the diff fields nullable per schema.""" if a is None or b is None: return None return a - b