128 lines
4.9 KiB
Python
128 lines
4.9 KiB
Python
import logging
|
|
from datetime import datetime, timedelta, timezone
|
|
from repositories.competitor_repository import competitor_repository
|
|
from repositories.product_repository import product_repository
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Default staleness threshold — configurable here only, per CLAUDE.md §7.
|
|
CACHE_STALENESS_HOURS = 24
|
|
|
|
|
|
def needs_refresh(competitor):
|
|
"""
|
|
Decides whether a competitor needs a live re-scrape (REFRESH path)
|
|
or can be served from PocketBase (FAST path).
|
|
|
|
Data flow:
|
|
competitor record →
|
|
change_detected flag set? → True (refresh) →
|
|
last_scraped missing or older than CACHE_STALENESS_HOURS? → True (refresh) →
|
|
otherwise → False (fast path is safe)
|
|
"""
|
|
if competitor.get('change_detected'):
|
|
return True
|
|
|
|
last_scraped = competitor.get('last_scraped')
|
|
if not last_scraped:
|
|
return True
|
|
|
|
try:
|
|
scraped_at = datetime.fromisoformat(last_scraped.replace('Z', '+00:00'))
|
|
except (ValueError, AttributeError):
|
|
return True
|
|
|
|
age = datetime.now(timezone.utc) - scraped_at
|
|
return age > timedelta(hours=CACHE_STALENESS_HOURS)
|
|
|
|
|
|
def analyse_from_cache(tenant_id, competitor_id, context):
|
|
"""
|
|
FAST path analysis — uses cached PocketBase data instead of a live
|
|
scrape. The LLM receives already-structured JSON and generates the
|
|
analysis narrative, comments, and relevancy score only — it does
|
|
NOT re-extract fields from raw page content.
|
|
|
|
Data flow:
|
|
competitor_id → PocketBase products (latest) →
|
|
structured JSON + client trip context (destination/duration/style) →
|
|
LLM prompt (analysis only, no extraction, via core.extractor) →
|
|
narrative + comments + relevancy score returned
|
|
"""
|
|
products = product_repository.get_latest_for_competitor(competitor_id, limit=1)
|
|
if not products:
|
|
raise ValueError('No cached product data available for fast path')
|
|
|
|
latest = products[0]
|
|
|
|
# core.extractor is a Phase 2 module — imported lazily so this
|
|
# service is importable before the scraping engine refactor lands.
|
|
from core.extractor import extract_with_openrouter
|
|
|
|
prompt = (
|
|
'You are a travel industry analyst. Given this already-extracted '
|
|
'competitor trip data, write a short comparative analysis against '
|
|
f"the client's product context: {context}.\n\nCompetitor data:\n{latest}"
|
|
)
|
|
return extract_with_openrouter(prompt, content='')
|
|
|
|
|
|
def run_competitor_analysis(tenant_id, competitor, context):
|
|
"""
|
|
Decides the fast vs refresh path for one competitor and executes it.
|
|
Called per-competitor by jobs.run_research_job — failures here are
|
|
caught by the caller so one competitor's failure never aborts the
|
|
whole batch (CLAUDE.md §18 resilience rule).
|
|
|
|
Data flow:
|
|
competitor + context → needs_refresh() decision →
|
|
FAST: analyse_from_cache() (seconds) →
|
|
REFRESH: core.scraper.scrape() → core.extractor.extract_with_openrouter()
|
|
→ PocketBase products/price_history updated →
|
|
competitor.change_detected reset, last_scraped bumped →
|
|
analysis narrative generated from the fresh data →
|
|
result dict returned
|
|
"""
|
|
if not needs_refresh(competitor):
|
|
logger.info(f'{competitor["name"]} — fast path (cache fresh)')
|
|
return {'path': 'fast', 'result': analyse_from_cache(tenant_id, competitor['id'], context)}
|
|
|
|
logger.info(f'{competitor["name"]} — refresh path (stale or changed)')
|
|
# Phase 2 modules — imported lazily, same reasoning as above.
|
|
from core.scraper import scrape
|
|
from core.extractor import extract_with_openrouter
|
|
from industries.adventure_travel.prompt import build_prompt
|
|
|
|
page_data = scrape(competitor['website'], country=competitor.get('primary_market'))
|
|
if not page_data.get('success'):
|
|
raise ValueError(page_data.get('text', 'Scrape failed'))
|
|
|
|
prompt = build_prompt(
|
|
competitor['name'], page_data,
|
|
context.get('product_name'), context.get('destination'),
|
|
context.get('duration'), context.get('travel_style'),
|
|
is_ngs=context.get('tab_type') == 'NGS'
|
|
)
|
|
extracted = extract_with_openrouter(prompt, content=page_data['text'])
|
|
|
|
product = product_repository.create({
|
|
'tenant_id': tenant_id,
|
|
'competitor_id': competitor['id'],
|
|
'trip_name': extracted.get('tripName'),
|
|
'url': page_data['url'],
|
|
'duration_days': extracted.get('duration'),
|
|
'majority_price_usd': extracted.get('majorityPrice'),
|
|
'price_low_usd': extracted.get('priceLow'),
|
|
'price_high_usd': extracted.get('priceHigh'),
|
|
'comments': extracted.get('comments'),
|
|
'relevancy': extracted.get('relevancy'),
|
|
'is_active': True,
|
|
})
|
|
|
|
competitor_repository.update(competitor['id'], {
|
|
'change_detected': False,
|
|
'last_scraped': datetime.now(timezone.utc).isoformat(),
|
|
})
|
|
|
|
return {'path': 'refresh', 'result': extracted, 'product': product}
|