Initial commit: project spec, reference files, and dev plan
Adds CLAUDE.md (authoritative spec), the legacy app.py scraper and approved dashboard mockup as reference material under z_starting_files/, and the phased dev_plan.md for Phase 1+ implementation.
This commit is contained in:
commit
cc81038ba5
|
|
@ -0,0 +1,33 @@
|
|||
# Environment / secrets
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*.egg-info/
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
htmlcov/
|
||||
venv/
|
||||
.venv/
|
||||
*.log
|
||||
|
||||
# Node / frontend
|
||||
node_modules/
|
||||
dist/
|
||||
.vite/
|
||||
npm-debug.log*
|
||||
|
||||
# PocketBase data
|
||||
pb_data/
|
||||
pb_migrations/*.local.js
|
||||
|
||||
# OS / editor
|
||||
.DS_Store
|
||||
.vscode/
|
||||
.idea/
|
||||
|
||||
# Google service account credentials
|
||||
*service-account*.json
|
||||
|
|
@ -0,0 +1,764 @@
|
|||
import os
|
||||
import json
|
||||
import re
|
||||
import asyncio
|
||||
import time
|
||||
import logging
|
||||
from flask import Flask, request, jsonify
|
||||
from dotenv import load_dotenv
|
||||
from playwright.async_api import async_playwright
|
||||
from bs4 import BeautifulSoup
|
||||
# import google.generativeai as genai # ── Uncomment to use Gemini instead of OpenRouter
|
||||
import gspread
|
||||
from openai import OpenAI
|
||||
from google.oauth2.service_account import Credentials
|
||||
|
||||
load_dotenv()
|
||||
|
||||
app = Flask(__name__)
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── CONFIG ───────────────────────────────────────────────────
|
||||
|
||||
# GEMINI_API_KEY = os.getenv('GEMINI_API_KEY') # ── Uncomment to use Gemini
|
||||
GROQ_API_KEY = os.getenv('GROQ_API_KEY')
|
||||
OPENROUTER_API_KEY = os.getenv('OPENROUTER_API_KEY')
|
||||
API_SECRET_KEY = os.getenv('API_SECRET_KEY')
|
||||
SHEET_ID = os.getenv('SHEET_ID')
|
||||
STANDARD_TAB = os.getenv('STANDARD_TAB', 'Standard Competitive Tab')
|
||||
NGS_TAB = os.getenv('NGS_TAB', 'NGS Competitive Tab')
|
||||
SERVICE_ACCOUNT_JSON = os.getenv('GOOGLE_SERVICE_ACCOUNT_JSON')
|
||||
|
||||
# ── FIELD MAPS ───────────────────────────────────────────────
|
||||
|
||||
STANDARD_FIELDS = {
|
||||
'company': 6, 'link': 8, 'tripName': 9, 'tripCode': 10,
|
||||
'serviceLevel': 11, 'groupSize': 12, 'duration': 13, 'meals': 14,
|
||||
'startLocation': 15, 'endLocation': 16, 'departures': 17,
|
||||
'seasonality': 19, 'startDays': 20, 'targetAudience': 21,
|
||||
'hotels': 22, 'activities': 23, 'relevancy': 24,
|
||||
'majorityPrice': 25, 'priceLow': 26, 'priceHigh': 27,
|
||||
'comments': 30, 'dateUsed': 31,
|
||||
'audPrice': 32, 'cadPrice': 33, 'eurPrice': 34, 'gbpPrice': 35
|
||||
}
|
||||
|
||||
NGS_FIELDS = {
|
||||
'company': 6, 'link': 8, 'tripName': 9, 'tripCode': 10,
|
||||
'serviceLevel': 11, 'groupSize': 12, 'duration': 13, 'meals': 14,
|
||||
'startLocation': 15, 'endLocation': 16, 'departures': 17,
|
||||
'seasonality': 19, 'startDays': 20, 'targetAudience': 21,
|
||||
'hotels': 22, 'activities': 23, 'exclusiveAccess': 24,
|
||||
'groupLeader': 25, 'sustainability': 26, 'relevancy': 27,
|
||||
'majorityPrice': 28, 'priceLow': 29, 'priceHigh': 30,
|
||||
'comments': 33, 'dateUsed': 34,
|
||||
'audPrice': 35, 'cadPrice': 36, 'eurPrice': 37, 'gbpPrice': 38
|
||||
}
|
||||
|
||||
|
||||
# ── AUTH ─────────────────────────────────────────────────────
|
||||
|
||||
def check_auth(req):
|
||||
token = req.headers.get('X-API-Key')
|
||||
if not token or token != API_SECRET_KEY:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# ── GOOGLE SHEETS ────────────────────────────────────────────
|
||||
|
||||
def get_sheet_client():
|
||||
scopes = [
|
||||
'https://www.googleapis.com/auth/spreadsheets',
|
||||
'https://www.googleapis.com/auth/drive'
|
||||
]
|
||||
info = json.loads(SERVICE_ACCOUNT_JSON)
|
||||
creds = Credentials.from_service_account_info(info, scopes=scopes)
|
||||
return gspread.authorize(creds)
|
||||
|
||||
|
||||
def write_to_sheet(tab_name, col_index, data, fields):
|
||||
client = get_sheet_client()
|
||||
sheet = client.open_by_key(SHEET_ID)
|
||||
ws = sheet.worksheet(tab_name)
|
||||
updates = []
|
||||
for field, row in fields.items():
|
||||
value = data.get(field)
|
||||
if value is None or value == '':
|
||||
continue
|
||||
updates.append({
|
||||
'range': gspread.utils.rowcol_to_a1(row, col_index),
|
||||
'values': [[value]]
|
||||
})
|
||||
if updates:
|
||||
ws.batch_update(updates)
|
||||
|
||||
|
||||
def get_competitors_from_db():
|
||||
client = get_sheet_client()
|
||||
sheet = client.open_by_key(SHEET_ID)
|
||||
ws = sheet.worksheet('Competitor DB')
|
||||
rows = ws.get_all_records()
|
||||
return [
|
||||
{
|
||||
'style': r['Travel Style'],
|
||||
'name': r['Competitor Name'],
|
||||
'url': r['Website URL'],
|
||||
'active': r['Active']
|
||||
}
|
||||
for r in rows
|
||||
if str(r.get('Active', '')).strip().lower() == 'yes'
|
||||
]
|
||||
|
||||
|
||||
# ── PLAYWRIGHT SCRAPER ───────────────────────────────────────
|
||||
|
||||
async def render_page(url: str) -> dict:
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.launch(
|
||||
headless=True,
|
||||
args=['--no-sandbox', '--disable-setuid-sandbox',
|
||||
'--disable-dev-shm-usage', '--disable-gpu']
|
||||
)
|
||||
context = await browser.new_context(
|
||||
user_agent=(
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
|
||||
'AppleWebKit/537.36 (KHTML, like Gecko) '
|
||||
'Chrome/124.0.0.0 Safari/537.36'
|
||||
),
|
||||
viewport={'width': 1366, 'height': 768},
|
||||
locale='en-GB',
|
||||
timezone_id='Europe/London',
|
||||
extra_http_headers={
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
||||
'Accept-Language': 'en-GB,en;q=0.9',
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'DNT': '1',
|
||||
'Upgrade-Insecure-Requests': '1',
|
||||
'Sec-Fetch-Dest': 'document',
|
||||
'Sec-Fetch-Mode': 'navigate',
|
||||
'Sec-Fetch-Site': 'none',
|
||||
'Sec-Fetch-User': '?1',
|
||||
}
|
||||
)
|
||||
page = await context.new_page()
|
||||
|
||||
# Basic stealth — manual navigator spoofing
|
||||
await page.add_init_script("""
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3] });
|
||||
Object.defineProperty(navigator, 'languages', { get: () => ['en-GB', 'en'] });
|
||||
window.chrome = { runtime: {} };
|
||||
""")
|
||||
|
||||
# Block only known third-party analytics/ad domains
|
||||
BLOCKED_DOMAINS = [
|
||||
'google-analytics.com', 'googletagmanager.com', 'doubleclick.net',
|
||||
'facebook.net', 'facebook.com/tr', 'hotjar.com', 'intercom.io',
|
||||
'zendesk.com', 'hubspot.com', 'marketo.com', 'segment.com',
|
||||
'amplitude.com', 'mixpanel.com', 'optimizely.com', 'crazyegg.com',
|
||||
]
|
||||
|
||||
async def block_trackers(route):
|
||||
if any(domain in route.request.url for domain in BLOCKED_DOMAINS):
|
||||
await route.abort()
|
||||
else:
|
||||
await route.continue_()
|
||||
|
||||
await page.route('**/*', block_trackers)
|
||||
|
||||
try:
|
||||
try:
|
||||
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
|
||||
logger.info(f'Loaded: {url}')
|
||||
except Exception as e:
|
||||
logger.warning(f'domcontentloaded failed, trying networkidle: {str(e)[:100]}')
|
||||
await page.goto(url, wait_until='networkidle', timeout=60000)
|
||||
|
||||
# Dismiss cookie consent banners
|
||||
try:
|
||||
await page.click(
|
||||
'button:has-text("Accept All"), button:has-text("Accept all"), '
|
||||
'button:has-text("Accept"), button:has-text("OK"), '
|
||||
'button:has-text("I agree"), button:has-text("Agree"), '
|
||||
'[id*="cookie"] button, [class*="cookie"] button, '
|
||||
'[id*="consent"] button, [class*="consent"] button',
|
||||
timeout=3000
|
||||
)
|
||||
logger.info('Cookie banner dismissed')
|
||||
await page.wait_for_timeout(1000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Wait for meaningful content
|
||||
try:
|
||||
await page.wait_for_selector(
|
||||
'h1, .trip-overview, main, article, #overview, '
|
||||
'[class*="itinerary"], [class*="price"], [class*="trip"]',
|
||||
timeout=8000
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
await page.wait_for_timeout(3000)
|
||||
|
||||
# Two scroll passes to trigger lazy-loaded content
|
||||
for _ in range(2):
|
||||
await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')
|
||||
await page.wait_for_timeout(1500)
|
||||
await page.evaluate('window.scrollTo(0, 0)')
|
||||
await page.wait_for_timeout(500)
|
||||
|
||||
await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')
|
||||
await page.wait_for_timeout(2000)
|
||||
|
||||
html = await page.content()
|
||||
soup = BeautifulSoup(html, 'lxml')
|
||||
|
||||
for tag in soup(['script', 'style', 'nav', 'footer',
|
||||
'header', 'iframe', 'noscript', 'svg',
|
||||
'button', 'form']):
|
||||
tag.decompose()
|
||||
|
||||
# Prioritise main content area
|
||||
main_content = (
|
||||
soup.find('main') or
|
||||
soup.find('article') or
|
||||
soup.find(id='main') or
|
||||
soup.find(id='content') or
|
||||
soup.find(class_='content') or
|
||||
soup
|
||||
)
|
||||
|
||||
text = main_content.get_text(separator=' ', strip=True)
|
||||
text = re.sub(r'\s+', ' ', text).strip()
|
||||
|
||||
tables = [t.get_text(separator=' | ', strip=True)
|
||||
for t in soup.find_all('table')]
|
||||
table_text = '\n'.join(tables)
|
||||
char_count = len(text)
|
||||
|
||||
if char_count < 500:
|
||||
logger.warning(
|
||||
f'Low char count ({char_count}) for {url} — '
|
||||
f'will attempt AI web fetch fallback'
|
||||
)
|
||||
|
||||
return {
|
||||
'text': text[:25000],
|
||||
'tables': table_text[:5000],
|
||||
'url': url,
|
||||
'char_count': char_count,
|
||||
'success': True
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'Render failed for {url}: {str(e)}')
|
||||
return {
|
||||
'text': f'Render failed: {str(e)}',
|
||||
'tables': '', 'url': url,
|
||||
'char_count': 0, 'success': False
|
||||
}
|
||||
finally:
|
||||
await browser.close()
|
||||
|
||||
|
||||
def clean_url(url: str) -> str:
|
||||
"""Strip URL fragments and common tracking parameters."""
|
||||
from urllib.parse import urlparse, urlencode, parse_qs, urlunparse
|
||||
parsed = urlparse(url)
|
||||
STRIP_PARAMS = {
|
||||
'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
|
||||
'referrerView', 'ref', 'referrer', 'source', 'fbclid', 'gclid',
|
||||
'mc_cid', 'mc_eid', '_ga', 'affiliate'
|
||||
}
|
||||
if parsed.query:
|
||||
params = parse_qs(parsed.query, keep_blank_values=True)
|
||||
cleaned = {k: v for k, v in params.items() if k not in STRIP_PARAMS}
|
||||
query = urlencode(cleaned, doseq=True)
|
||||
else:
|
||||
query = ''
|
||||
clean = urlunparse((
|
||||
parsed.scheme, parsed.netloc, parsed.path,
|
||||
parsed.params, query, ''
|
||||
))
|
||||
if clean != url:
|
||||
logger.info(f'URL cleaned: {url} → {clean}')
|
||||
return clean
|
||||
|
||||
|
||||
def scrape(url):
|
||||
import nest_asyncio
|
||||
nest_asyncio.apply()
|
||||
url = clean_url(url)
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
result = loop.run_until_complete(render_page(url))
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
# AI web fetch fallback for bot-protected sites
|
||||
if result.get('char_count', 0) < 1000 and result.get('success'):
|
||||
logger.info(f'Playwright char count low — trying AI web fetch for {url}')
|
||||
ai_result = fetch_with_ai(url)
|
||||
if ai_result:
|
||||
return ai_result
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ── AI WEB FETCH FALLBACK ────────────────────────────────────
|
||||
# Fires only when Playwright returns < 500 chars.
|
||||
# Uses OpenRouter web_search server tool with Exa engine.
|
||||
# Cost: ~$0.02 per call + token cost. Only fires for bot-protected sites.
|
||||
|
||||
def fetch_with_ai(url: str) -> dict:
|
||||
logger.info(f'Playwright failed — attempting AI web fetch for {url}')
|
||||
try:
|
||||
client = OpenAI(
|
||||
api_key=OPENROUTER_API_KEY,
|
||||
base_url='https://openrouter.ai/api/v1'
|
||||
)
|
||||
from urllib.parse import urlparse
|
||||
domain = urlparse(url).netloc
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model='openai/gpt-4o-mini',
|
||||
messages=[{
|
||||
'role': 'user',
|
||||
'content': (
|
||||
f'I am a travel industry analyst conducting competitive research. '
|
||||
f'Please search for this tour page and extract the following factual data points: '
|
||||
f'trip name, trip code, price, duration in days, maximum group size, '
|
||||
f'total included meals, hotel names, departure dates, start location, '
|
||||
f'end location, service level, and key activities. '
|
||||
f'Return all factual information you find.\n\nURL: {url}'
|
||||
)
|
||||
}],
|
||||
tools=[{
|
||||
'type': 'openrouter:web_search',
|
||||
'parameters': {
|
||||
'engine': 'exa',
|
||||
'max_results': 5,
|
||||
'search_context_size': 'high',
|
||||
'allowed_domains': [domain]
|
||||
}
|
||||
}],
|
||||
temperature=0.0,
|
||||
max_tokens=8192
|
||||
)
|
||||
|
||||
msg = response.choices[0].message
|
||||
text = msg.content if isinstance(msg.content, str) else ''
|
||||
|
||||
# Also extract content from tool result annotations if present
|
||||
if hasattr(msg, 'annotations') and msg.annotations:
|
||||
for annotation in msg.annotations:
|
||||
if hasattr(annotation, 'url_citation') and annotation.url_citation:
|
||||
citation_content = getattr(annotation.url_citation, 'content', '')
|
||||
if citation_content:
|
||||
text = text + '\n\n' + citation_content
|
||||
|
||||
text = text.strip()
|
||||
char_count = len(text)
|
||||
logger.info(f'AI web fetch returned {char_count} chars for {url}')
|
||||
logger.info(f'AI web fetch content preview: {text[:500]}')
|
||||
|
||||
if char_count > 300:
|
||||
return {
|
||||
'text': text[:25000],
|
||||
'tables': '',
|
||||
'url': url,
|
||||
'char_count': char_count,
|
||||
'success': True
|
||||
}
|
||||
else:
|
||||
logger.warning(f'AI web fetch also returned low content for {url}')
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'AI web fetch failed for {url}: {str(e)}')
|
||||
return None
|
||||
|
||||
|
||||
# ── PROMPT BUILDER ───────────────────────────────────────────
|
||||
|
||||
def build_prompt(competitor_name, page_data, product_name,
|
||||
destination, duration, travel_style, is_ngs=False):
|
||||
|
||||
ngs_fields = '''
|
||||
"exclusiveAccess": "exclusive access or special unique inclusions",
|
||||
"groupLeader": "group leader and/or local expert description",
|
||||
"sustainability": "sustainability inclusions or null",''' if is_ngs else ''
|
||||
|
||||
content = page_data['text'] if page_data else '[PAGE CONTENT WILL BE INSERTED HERE]'
|
||||
if page_data and page_data.get('tables'):
|
||||
content += f'\n\nTABLE DATA:\n{page_data["tables"]}'
|
||||
|
||||
url = page_data['url'] if page_data else '[COMPETITOR URL]'
|
||||
|
||||
return f'''You are a travel industry analyst helping G Adventures research competitors.
|
||||
Be concise — use short phrases not full sentences for all fields except comments.
|
||||
Read the ENTIRE page content carefully before extracting any fields.
|
||||
Pay particular attention to itinerary sections, departure tables, and pricing grids
|
||||
which may appear later in the content.
|
||||
|
||||
The content below was scraped from the {competitor_name} tour page at: {url}
|
||||
|
||||
This is the specific page selected as most comparable to:
|
||||
- G Adventures Product: {product_name}
|
||||
- Destination: {destination}
|
||||
- Duration: ~{duration} days
|
||||
- G Adventures Travel Style: {travel_style}
|
||||
|
||||
PAGE CONTENT:
|
||||
---
|
||||
{content}
|
||||
---
|
||||
|
||||
CRITICAL PRICING RULES:
|
||||
- majorityPrice: Standard/regular adult price for the most common departure.
|
||||
IGNORE sale, promotional, early bird, or "was/now" pricing. Use non-discounted rack rate only.
|
||||
- Prices may appear as "from $X", "per person from $X", or inside a departures table or pricing grid.
|
||||
Extract the most commonly listed standard adult price regardless of how it is labelled on the page.
|
||||
- priceLow: Lowest standard price across all departures shown (not sale price).
|
||||
- priceHigh: Highest standard price across all departures shown (not sale price).
|
||||
- If a site requires clicking "see more departures" to show all dates, note this in seasonality
|
||||
and use only prices visible on the main listing page.
|
||||
- Multi-currency: only fill if the price is explicitly listed in that currency on the page.
|
||||
Do not convert. If the page shows GBP only, fill gbpPrice and leave USD fields null.
|
||||
|
||||
MEALS RULE:
|
||||
- meals: Return the TOTAL number of included meals as a single integer only. No text description.
|
||||
- Meals may be listed individually per day in the itinerary section
|
||||
(e.g. "Day 1: Breakfast", "Day 3: Breakfast and Dinner").
|
||||
Count every individual meal mention across all itinerary days.
|
||||
Do not rely on a summary box — read through the full itinerary to count.
|
||||
- Count each meal type separately: 1 breakfast + 1 lunch + 1 dinner = 3 meals.
|
||||
|
||||
Return ONLY a valid JSON object. No markdown, no code fences, no explanation.
|
||||
|
||||
{{
|
||||
"company": "{competitor_name}",
|
||||
"link": "{url}",
|
||||
"tripName": "exact trip name from the page",
|
||||
"tripCode": "trip code if shown, otherwise null",
|
||||
"serviceLevel": "accommodation service level",
|
||||
"groupSize": max group size as integer,
|
||||
"duration": number of days as integer,
|
||||
"meals": total included meals as integer,{ngs_fields}
|
||||
"startLocation": "departure city",
|
||||
"endLocation": "end city",
|
||||
"departures": approximate departures per year as integer or null,
|
||||
"seasonality": "price bands by season. Note if more departures are hidden behind a button.",
|
||||
"startDays": "departure days of week",
|
||||
"targetAudience": "target traveller",
|
||||
"hotels": "hotel types or properties",
|
||||
"activities": "key included activities",
|
||||
"relevancy": relevancy score 1-5 as integer,
|
||||
"majorityPrice": standard adult USD price as number or null,
|
||||
"priceLow": lowest standard USD price as number or null,
|
||||
"priceHigh": highest standard USD price as number or null,
|
||||
"comments": "analysis: key differences from G Adventures, strengths, weaknesses, pricing position",
|
||||
"dateUsed": "specific departure date used for majority price",
|
||||
"audPrice": standard AUD price as number or null,
|
||||
"cadPrice": standard CAD price as number or null,
|
||||
"eurPrice": standard EUR price as number or null,
|
||||
"gbpPrice": standard GBP price as number or null
|
||||
}}'''
|
||||
|
||||
|
||||
def inject_page_content(prompt_template, competitor_name, page_data):
|
||||
"""Inject real scraped content into a user-edited prompt template."""
|
||||
content = page_data['text']
|
||||
if page_data.get('tables'):
|
||||
content += f'\n\nTABLE DATA:\n{page_data["tables"]}'
|
||||
result = prompt_template.replace('[PAGE CONTENT WILL BE INSERTED HERE]', content)
|
||||
result = result.replace('[COMPETITOR URL]', page_data['url'])
|
||||
result = result.replace(
|
||||
competitor_name + ' tour page at: [COMPETITOR URL]',
|
||||
competitor_name + f' tour page at: {page_data["url"]}'
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
# ── EXTRACTION — OPENROUTER WITH MODEL FALLBACK ───────────────
|
||||
# Active provider. Falls through model list if free tier is exhausted.
|
||||
# ── TO SWITCH TO GROQ ────────────────────────────────────────
|
||||
# 1. Comment out extract_with_groq below
|
||||
# 2. Uncomment the Groq version further below
|
||||
# ── TO SWITCH TO GEMINI ──────────────────────────────────────
|
||||
# 1. Uncomment gemini import and GEMINI_API_KEY in config
|
||||
# 2. Comment out extract_with_groq below
|
||||
# 3. Uncomment extract_with_gemini further below
|
||||
# 4. In requirements.txt swap openai for google-generativeai
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
def extract_with_groq(competitor_name, prompt):
|
||||
client = OpenAI(
|
||||
api_key=OPENROUTER_API_KEY,
|
||||
base_url='https://openrouter.ai/api/v1'
|
||||
)
|
||||
|
||||
models = [
|
||||
'openai/gpt-oss-120b:free',
|
||||
'minimax/minimax-m2.5:free',
|
||||
'nvidia/nemotron-3-super-120b-a12b:free',
|
||||
'meta-llama/llama-3.3-70b-instruct:free'
|
||||
]
|
||||
|
||||
last_model_used = None
|
||||
|
||||
for model in models:
|
||||
for attempt in range(2):
|
||||
try:
|
||||
last_model_used = model
|
||||
response = client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[{'role': 'user', 'content': prompt}],
|
||||
temperature=0.1,
|
||||
max_tokens=4096
|
||||
)
|
||||
text = response.choices[0].message.content
|
||||
cleaned = re.sub(r'```json|```', '', text).strip()
|
||||
|
||||
try:
|
||||
return json.loads(cleaned)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
match = re.search(r'\{[\s\S]*\}', cleaned)
|
||||
if match:
|
||||
try:
|
||||
return json.loads(match.group(0))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
try:
|
||||
fixed = re.sub(r',\s*}', '}', cleaned)
|
||||
fixed = re.sub(r',\s*]', ']', fixed)
|
||||
fixed = re.sub(r'([{,])\s*([a-zA-Z0-9_]+)\s*:', r'\1"\2":', fixed)
|
||||
return json.loads(fixed)
|
||||
except json.JSONDecodeError:
|
||||
raise ValueError(
|
||||
f'Could not parse response from {model} for {competitor_name}'
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
error_str = str(e)
|
||||
if '429' in error_str or 'rate_limit' in error_str.lower():
|
||||
wait = 5 * (attempt + 1)
|
||||
logger.warning(
|
||||
f'{model} rate limited — waiting {wait}s (attempt {attempt+1}/2)...'
|
||||
)
|
||||
time.sleep(wait)
|
||||
elif '403' in error_str and 'credit' in error_str.lower():
|
||||
logger.warning(f'{model} free tier exhausted — trying next model...')
|
||||
break
|
||||
else:
|
||||
logger.warning(f'{model} failed: {error_str[:200]}')
|
||||
if attempt == 1:
|
||||
logger.warning(f'Moving to next model after {model} failed twice')
|
||||
|
||||
raise Exception(
|
||||
f'All models exhausted for {competitor_name}. Last tried: {last_model_used}'
|
||||
)
|
||||
|
||||
|
||||
# ── GROQ VERSION (commented out — uncomment to switch) ───────
|
||||
#
|
||||
# def extract_with_groq(competitor_name, prompt):
|
||||
# client = OpenAI(
|
||||
# api_key=GROQ_API_KEY,
|
||||
# base_url='https://api.groq.com/openai/v1'
|
||||
# )
|
||||
# for attempt in range(3):
|
||||
# try:
|
||||
# response = client.chat.completions.create(
|
||||
# model='llama-3.3-70b-versatile',
|
||||
# messages=[{'role': 'user', 'content': prompt}],
|
||||
# temperature=0.1,
|
||||
# max_tokens=4096
|
||||
# )
|
||||
# text = response.choices[0].message.content
|
||||
# cleaned = re.sub(r'```json|```', '', text).strip()
|
||||
# try:
|
||||
# return json.loads(cleaned)
|
||||
# except json.JSONDecodeError:
|
||||
# match = re.search(r'\{[\s\S]*\}', cleaned)
|
||||
# if match:
|
||||
# return json.loads(match.group(0))
|
||||
# raise ValueError(f'Could not parse Groq response for {competitor_name}')
|
||||
# except Exception as e:
|
||||
# if '429' in str(e) and attempt < 2:
|
||||
# wait = 10 * (attempt + 1)
|
||||
# logger.warning(f'Rate limited — waiting {wait}s...')
|
||||
# time.sleep(wait)
|
||||
# else:
|
||||
# raise
|
||||
|
||||
|
||||
# ── GEMINI VERSION (commented out — uncomment to switch) ─────
|
||||
#
|
||||
# def extract_with_gemini(competitor_name, prompt):
|
||||
# genai.configure(api_key=GEMINI_API_KEY)
|
||||
# model = genai.GenerativeModel('gemini-2.5-flash')
|
||||
# for attempt in range(3):
|
||||
# try:
|
||||
# response = model.generate_content(
|
||||
# prompt,
|
||||
# generation_config=genai.types.GenerationConfig(
|
||||
# temperature=0.1, max_output_tokens=4096
|
||||
# )
|
||||
# )
|
||||
# text = response.text
|
||||
# cleaned = re.sub(r'```json|```', '', text).strip()
|
||||
# try:
|
||||
# return json.loads(cleaned)
|
||||
# except json.JSONDecodeError:
|
||||
# match = re.search(r'\{[\s\S]*\}', cleaned)
|
||||
# if match:
|
||||
# return json.loads(match.group(0))
|
||||
# raise ValueError(f'Could not parse Gemini response for {competitor_name}')
|
||||
# except Exception as e:
|
||||
# if '429' in str(e) and attempt < 2:
|
||||
# wait = 10 * (attempt + 1)
|
||||
# logger.warning(f'Rate limited — waiting {wait}s...')
|
||||
# time.sleep(wait)
|
||||
# else:
|
||||
# raise
|
||||
|
||||
|
||||
# ── ROUTES ───────────────────────────────────────────────────
|
||||
|
||||
@app.route('/health', methods=['GET'])
|
||||
def health():
|
||||
return jsonify({'status': 'ok'})
|
||||
|
||||
|
||||
@app.route('/competitors', methods=['GET'])
|
||||
def get_competitors():
|
||||
if not check_auth(request):
|
||||
return jsonify({'error': 'Unauthorized'}), 401
|
||||
try:
|
||||
competitors = get_competitors_from_db()
|
||||
return jsonify({'competitors': competitors})
|
||||
except Exception as e:
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@app.route('/preview-prompt', methods=['POST'])
|
||||
def preview_prompt():
|
||||
if not check_auth(request):
|
||||
return jsonify({'error': 'Unauthorized'}), 401
|
||||
|
||||
data = request.json or {}
|
||||
travel_style = data.get('travelStyle', '')
|
||||
product_name = data.get('productName', '')
|
||||
destination = data.get('destination', '')
|
||||
duration = data.get('duration', '')
|
||||
tab_type = data.get('tabType', 'Standard')
|
||||
is_ngs = tab_type == 'NGS'
|
||||
|
||||
placeholder_page = {
|
||||
'text': '[PAGE CONTENT WILL BE INSERTED HERE]',
|
||||
'tables': '',
|
||||
'url': '[COMPETITOR URL]'
|
||||
}
|
||||
prompt = build_prompt(
|
||||
'[Competitor Name]', placeholder_page,
|
||||
product_name, destination, duration, travel_style, is_ngs
|
||||
)
|
||||
return jsonify({'prompt': prompt})
|
||||
|
||||
|
||||
@app.route('/research', methods=['POST'])
|
||||
def research():
|
||||
if not check_auth(request):
|
||||
return jsonify({'error': 'Unauthorized'}), 401
|
||||
|
||||
data = request.json
|
||||
if not data:
|
||||
return jsonify({'error': 'No JSON body'}), 400
|
||||
|
||||
required = ['travelStyle', 'productName', 'destination',
|
||||
'duration', 'tabType', 'competitors']
|
||||
for field in required:
|
||||
if field not in data:
|
||||
return jsonify({'error': f'Missing field: {field}'}), 400
|
||||
|
||||
travel_style = data['travelStyle']
|
||||
product_name = data['productName']
|
||||
destination = data['destination']
|
||||
duration = data['duration']
|
||||
tab_type = data['tabType']
|
||||
competitors = data['competitors']
|
||||
prompt_template = data.get('promptTemplate', None)
|
||||
|
||||
is_ngs = tab_type == 'NGS'
|
||||
tab_name = NGS_TAB if is_ngs else STANDARD_TAB
|
||||
fields = NGS_FIELDS if is_ngs else STANDARD_FIELDS
|
||||
START_COL = 4
|
||||
|
||||
results = {'success': [], 'failed': []}
|
||||
|
||||
for i, competitor in enumerate(competitors):
|
||||
name = competitor['name']
|
||||
url = competitor['url']
|
||||
col = competitor.get('col', START_COL + i)
|
||||
|
||||
logger.info(f'Processing {name} — {url}')
|
||||
|
||||
try:
|
||||
# Step 1: Scrape
|
||||
page_data = scrape(url)
|
||||
if not page_data['success']:
|
||||
raise ValueError(page_data['text'])
|
||||
logger.info(f'{name} — scraped {page_data["char_count"]} chars')
|
||||
|
||||
# Step 2: Build prompt
|
||||
if prompt_template and len(prompt_template.strip()) > 100:
|
||||
prompt = inject_page_content(prompt_template, name, page_data)
|
||||
else:
|
||||
prompt = build_prompt(
|
||||
name, page_data, product_name,
|
||||
destination, duration, travel_style, is_ngs
|
||||
)
|
||||
|
||||
# Step 3: Extract
|
||||
extracted = extract_with_groq(name, prompt)
|
||||
logger.info(f'{name} — extracted: {extracted.get("tripName")}')
|
||||
|
||||
# Step 4: Write to sheet
|
||||
write_to_sheet(tab_name, col, extracted, fields)
|
||||
logger.info(f'{name} — written to col {col}')
|
||||
|
||||
results['success'].append({
|
||||
'name': name,
|
||||
'tripName': extracted.get('tripName'),
|
||||
'price': extracted.get('majorityPrice'),
|
||||
'col': col
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'{name} failed: {str(e)}')
|
||||
results['failed'].append({'name': name, 'error': str(e)})
|
||||
try:
|
||||
client = get_sheet_client()
|
||||
sheet = client.open_by_key(SHEET_ID)
|
||||
ws = sheet.worksheet(tab_name)
|
||||
ws.update_cell(fields['company'], col, f'⚠️ {name} — failed')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return jsonify({
|
||||
'successCount': len(results['success']),
|
||||
'failedCount': len(results['failed']),
|
||||
'success': results['success'],
|
||||
'failed': results['failed']
|
||||
})
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.run(host='0.0.0.0', port=5000, debug=False)
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,580 @@
|
|||
# Bettersight — Development Plan
|
||||
|
||||
> Derived from CLAUDE.md (authoritative spec, read in full) and the two
|
||||
> reference artifacts in this repo: `app.py` (existing scraper to be
|
||||
> refactored, not rewritten) and `bettersight-dashboard.html` (approved
|
||||
> UI mockup). This plan follows the Build Order in CLAUDE.md Section 19
|
||||
> exactly — same phases, same sequence. Nothing here begins until you
|
||||
> approve it.
|
||||
|
||||
---
|
||||
|
||||
## 0. Guardrails that apply to every phase
|
||||
|
||||
These are non-negotiable per CLAUDE.md Section 18 and are not repeated
|
||||
in every phase below, but apply throughout:
|
||||
|
||||
- **Layering**: `api.py` routes → `services/` (business logic) →
|
||||
`repositories/` (data access, plain dicts only). No layer skips
|
||||
another. `jobs.py` calls services only.
|
||||
- **Every function** gets a one-sentence docstring + a "Data flow:"
|
||||
comment + inline comments on meaningful steps, written *before* the
|
||||
logic, not after.
|
||||
- **Every PocketBase query** for non-admin operations is filtered by
|
||||
`tenant_id`. No exceptions.
|
||||
- **Every credential** comes from an env var. Nothing hardcoded.
|
||||
- **Every Flask route** (except `/health` and `/stripe/webhook`) checks
|
||||
`X-API-Key`, and is rate-limited via Flask-Limiter.
|
||||
- **Every error response** is structured JSON (`error_response()`
|
||||
helper), never plain text or a stack trace.
|
||||
- **Every repository** ships with a mock implementation alongside the
|
||||
real one, for testing.
|
||||
- **Test coverage**: 80% floor overall, 100% on licence validation,
|
||||
Stripe webhook, seat management, rollback paths.
|
||||
- **Frontend**: JavaScript only (no TypeScript), Nuxt UI v4 components
|
||||
only (no custom primitives), pages → composables → repositories →
|
||||
api_service.js, with the same "no layer skips another" rule.
|
||||
- **UI**: light theme, dark sidebar only, dot-grid motif on every feed
|
||||
row / table row / alert row / stat card, per the mockup.
|
||||
- **Do not build** anything in CLAUDE.md Section 27 ("What NOT to
|
||||
Build") — Discover/Intelligence/Enterprise features, custom Sheet
|
||||
functions, admin dashboard, mobile app, etc.
|
||||
|
||||
---
|
||||
|
||||
## 1. Pre-flight — one thing to fix before Phase 1
|
||||
|
||||
`CLAUDE.md` Section 17 (Environment Variables) contains a live-looking
|
||||
value inline:
|
||||
|
||||
```
|
||||
GOTIFY_APP_TOKEN=A78_l3r9y.ZzBr-
|
||||
```
|
||||
|
||||
This violates the doc's own "all credentials from env vars — hardcoded
|
||||
credentials are a build failure" rule, and a real token sitting in a
|
||||
markdown file that may end up in git is a leak risk. **Recommendation:
|
||||
rotate this token in Gotify and place the new value only in `.env`
|
||||
(gitignored), never in CLAUDE.md.** I will not commit this value
|
||||
anywhere. Flagging it now so you can rotate it before this repo is
|
||||
pushed to a remote.
|
||||
|
||||
I'll also create `.gitignore` (excluding `.env`, `__pycache__`,
|
||||
`node_modules`, `pb_data/`) and initialize git, since this directory is
|
||||
not currently a repo — confirm before I do this, or tell me if you'll
|
||||
handle git setup yourself.
|
||||
|
||||
---
|
||||
|
||||
## Phase 1 — Data Foundation
|
||||
|
||||
**Goal:** every PocketBase collection, every repository (+ mock), every
|
||||
service, every Flask route, and the RQ queue skeleton exist and are
|
||||
tested, before any scraping or UI work begins.
|
||||
|
||||
### 1.1 PocketBase schema
|
||||
|
||||
Collections to create (MVP-scope only — excludes Discover/Intelligence
|
||||
collections like `signal_events`, `competitor_people`,
|
||||
`wayback_snapshots`, `tenant_custom_fields`, which are out of scope
|
||||
until those tiers are built):
|
||||
|
||||
| Collection | Source section |
|
||||
|---|---|
|
||||
| `tenants` | §6 |
|
||||
| `tenant_seats` | §6 |
|
||||
| `competitors` | §6 |
|
||||
| `products` | §6 |
|
||||
| `products_ngs_meta` | §6 |
|
||||
| `price_history` | §6 |
|
||||
| `scrape_runs` | §6 |
|
||||
| `alerts` | §6 |
|
||||
| `battlecards` | §6 |
|
||||
| `comparable_matches` | §6 (populated in Phase 6) |
|
||||
| `tenant_notifications` | §6 (email_digest fields only — Slack/Teams/WhatsApp fields stay null/unused in MVP) |
|
||||
| `digest_logs` | §6 |
|
||||
| `proxy_overrides` | §10 |
|
||||
| `client_trips` | §6 |
|
||||
| `monitoring_events` | §29 (owner-only Gotify monitoring — see note in 1.6) |
|
||||
|
||||
All fields exactly as specified in CLAUDE.md §6/§10/§29. PocketBase
|
||||
admin UI or migration JSON — I'll write this as a PocketBase migration
|
||||
file (`pb_migrations/*.js`) so schema is versioned and re-creatable,
|
||||
rather than manual admin-UI clicking.
|
||||
|
||||
**Deliverable:** `pb_migrations/001_initial_schema.js` (or similar),
|
||||
applied to a local PocketBase instance, verified by listing collections.
|
||||
|
||||
### 1.2 Repository layer (`backend/repositories/`)
|
||||
|
||||
One file per collection, each exposing plain-dict CRUD + the specific
|
||||
lookups the spec calls out, each with a `_mock.py` (or in-file `Mock*`
|
||||
class) alongside:
|
||||
|
||||
- `tenant_repository.py` — `get_by_domain`, `get_by_id`, `update`, `create`
|
||||
- `seat_repository.py` — `validate_or_register`, `count_active`, `create`, `delete`
|
||||
- `competitor_repository.py` — `get_by_url`, `get_by_domain`, `get_by_id`, `list_for_tenant`, `create`, `update`, `delete`
|
||||
- `product_repository.py` — `get_latest_for_competitor`, `create`, `update`
|
||||
- `scrape_repository.py` — `create`, `update`, `get_by_id`
|
||||
- `proxy_repository.py` — `get_by_domain`, `create`, `update`
|
||||
- `alert_repository.py` — `create`, `list_unread`, `mark_read`
|
||||
- `battlecard_repository.py` — `get_for_competitor`, `upsert`
|
||||
- `comparable_repository.py` — `create`, `list_for_tenant`, `dismiss`
|
||||
- `brief_repository.py` — `get_weekly_data`, `log_digest`
|
||||
- `monitoring_repository.py` — `log_event`
|
||||
|
||||
Every repository: no business logic, no calls to services, returns
|
||||
plain dicts only — per §18.
|
||||
|
||||
### 1.3 Service layer (`backend/services/`)
|
||||
|
||||
- `licence_service.py` — `validate_email()` (three-layer check from §9: domain → seat → ~~sheet_id~~ — **note:** §9 (first occurrence, duplicated heading) describes a 3-layer sheet_id-bound check, but §15 explicitly says sheet_id binding is removed for the standalone Apps Script model. I will implement the **standalone (no sheet_id) version** since it's the later, more specific instruction and matches the `tenants` schema note "sheet_id binding removed". Flagging this contradiction for your awareness — let me know if you intended otherwise.)
|
||||
- `trip_finder_service.py` — `find_competitor_trip_url()`, `score_and_rank_urls()`, `find_all_competitor_urls()` (§8, concurrent via `ThreadPoolExecutor`, max 5 workers)
|
||||
- `battlecard_service.py` — `generate_battlecard()` (§11)
|
||||
- `embedding_service.py` — `embed_trip()`, `find_comparable_trips()` (§12 — built fully in Phase 6, stubbed now)
|
||||
- `brief_service.py` — `send_weekly_brief()` (§13)
|
||||
- `alert_service.py` — `send_gotify()` (§29) + alert-record creation helpers
|
||||
- `analysis_service.py` — orchestrates the fast/refresh path decision (§7) — new service not explicitly named in §5 file tree but required to keep this logic out of `jobs.py` and `api.py` per the layering rule
|
||||
- `onboarding_service.py` — onboarding checklist merge logic (§21)
|
||||
- `billing_service.py` — Stripe checkout/portal/webhook handling (§16, §21)
|
||||
|
||||
### 1.4 Flask routes (`backend/api.py`)
|
||||
|
||||
All routes from §9 plus the additions scattered through later sections
|
||||
(§15 Apps Script support routes, §21 billing/auth, §22 CSV bulk import,
|
||||
§23 GDPR delete). Full route list for Phase 1 (MVP-relevant only —
|
||||
Intelligence-tier `/internal/signal-harvest` etc. excluded):
|
||||
|
||||
```
|
||||
POST /validate-email
|
||||
POST /stripe/webhook
|
||||
|
||||
POST /research/find-urls
|
||||
POST /research
|
||||
GET /research/status/<job_id>
|
||||
GET /research/history
|
||||
GET /research/history/<job_id>
|
||||
POST /preview-prompt
|
||||
|
||||
GET /competitors
|
||||
POST /battlecards
|
||||
POST /comparable-trips
|
||||
|
||||
POST /sheet/push
|
||||
GET /sheet/tabs
|
||||
|
||||
GET /account/<tenant_id>
|
||||
POST /account/seats
|
||||
DELETE /account/seats/<seat_id>
|
||||
POST /account/competitors
|
||||
POST /account/competitors/bulk
|
||||
GET /account/competitors/template
|
||||
PUT /account/competitors/<id>
|
||||
DELETE /account/competitors/<id>
|
||||
GET /account/template
|
||||
PATCH /account/onboarding
|
||||
POST /account/delete
|
||||
GET /billing/portal
|
||||
POST /billing/upgrade
|
||||
POST /auth/request-password-reset
|
||||
POST /auth/confirm-password-reset
|
||||
|
||||
POST /internal/scrape/<tenant_id>
|
||||
POST /internal/battlecard/<competitor_id>
|
||||
POST /internal/embed-products/<tenant_id>
|
||||
POST /internal/match-comparable/<tenant_id>
|
||||
POST /internal/weekly-brief/<tenant_id>
|
||||
POST /internal/monitor-webhook
|
||||
POST /internal/monitor/register/<competitor_id>
|
||||
POST /internal/monitor/deregister/<competitor_id>
|
||||
|
||||
GET /health
|
||||
```
|
||||
|
||||
Every route: validates `X-API-Key` (except `/health`, `/stripe/webhook`
|
||||
which use Stripe signature verification instead), rate-limited per
|
||||
§22's Flask-Limiter rules, structured JSON errors via `error_response()`.
|
||||
|
||||
### 1.5 RQ job queue (`backend/jobs.py`)
|
||||
|
||||
- Three priority queues (`high`, `normal`, `low`) per §10 — `high` is
|
||||
reserved for Intelligence tier and will sit unused until that tier is
|
||||
built; Analyse-tier jobs go to `normal`.
|
||||
- `run_research_job(job_id, data)` — calls `analysis_service`, never
|
||||
touches DB or scraping logic directly (those come in Phase 2).
|
||||
- Job duration timing + Gotify alert hook (§29 Component 2) wired in
|
||||
now since the structure is simple and avoids a second pass later.
|
||||
|
||||
### 1.6 Monitoring note
|
||||
|
||||
§29 (Gotify alerting, `monitoring_events` collection) isn't enumerated
|
||||
as its own line item in the §19 Build Order, but §18 lists "`/health`
|
||||
endpoint exists and is monitored — automated alert on failure" as a
|
||||
**never-violate** rule. I'm building the `monitoring_events` collection
|
||||
and `alert_service.send_gotify()` now in Phase 1 (cheap, foundational),
|
||||
but the actual n8n polling workflow that calls it is deferred to Phase
|
||||
7 alongside the other ship-gate items, matching where Sentry/health
|
||||
monitoring appear in the Build Order.
|
||||
|
||||
### 1.7 Tests
|
||||
|
||||
`backend/tests/` mirroring `services/` and `repositories/` structure.
|
||||
Unit tests for every service against mock repositories. Integration
|
||||
tests for every route. 80% floor enforced via `pytest-cov` in CI
|
||||
config (CI itself is out of scope unless you want a GitHub Actions
|
||||
workflow added — flagging as an open question below).
|
||||
|
||||
**Phase 1 exit criteria:** schema applied, all repos + mocks written,
|
||||
all services written against mocks, all routes return correct shapes
|
||||
against mocked services, RQ worker starts and drains a trivial job,
|
||||
`pytest --cov` shows ≥80%.
|
||||
|
||||
---
|
||||
|
||||
## Phase 2 — Refactor `app.py` into the modular structure
|
||||
|
||||
**Goal:** identical scraping behavior, reorganized into `core/` /
|
||||
`industries/adventure_travel/` / `config/client.py`. No logic changes
|
||||
except the explicitly-spec'd additions (playwright-stealth, smart
|
||||
proxy, robots.txt check).
|
||||
|
||||
Mapping from the existing `app.py` (764 lines, read in full) to the
|
||||
target structure:
|
||||
|
||||
| Existing `app.py` code | Moves to | Notes |
|
||||
|---|---|---|
|
||||
| `render_page()`, `clean_url()`, `scrape()`, tracker blocklist | `core/scraper.py` | Replace manual `add_init_script` block (lines 147–152) with `playwright_stealth.stealth_async(page)` per §10. Add `country` param + `ProxyService` integration. Add `check_robots_txt()` before each scrape (§30 — was listed as a Phase 7 item in §19 step 68, but logically belongs with the scraper rewrite in Phase 2; I'll stub the check now and confirm gating behavior with you before Phase 7 if timing matters) |
|
||||
| `fetch_with_ai()` | `core/ai_fetch.py` | Unchanged logic, wired through `ProxyService` per §10 |
|
||||
| `extract_with_groq()` (OpenRouter free-model cycling, JSON repair) | `core/extractor.py` | Renamed to `extract_with_openrouter()` per §10's spec; same model-list-with-fallback pattern, same JSON-repair fallback chain (raw parse → regex extract → comma/quote repair). Reads `OPENROUTER_FREE_MODELS` / `OPENROUTER_FALLBACK_MODEL` from env instead of the hardcoded model list in the current file |
|
||||
| `build_prompt()`, `inject_page_content()` | `industries/adventure_travel/prompt.py` | Unchanged |
|
||||
| `STANDARD_FIELDS`, `NGS_FIELDS` | `industries/adventure_travel/fields.py` | Unchanged |
|
||||
| (new — extracted from the JSON shape inside `build_prompt()`) | `industries/adventure_travel/schema.py` | Standard + NGS extraction schema as data, so Phase 6 custom-fields work (deferred) has a clean attach point |
|
||||
| `write_to_sheet()`, `get_sheet_client()`, `get_competitors_from_db()`, `SHEET_ID`/tab constants | `config/client.py` | Unchanged — this is the "swap per client" layer |
|
||||
| Groq/Gemini commented-out alternate providers (lines 569–630) | Dropped | Dead code per spec's "OpenRouter is the active provider" — confirm before deletion since it's clearly intentional reference material the user kept; I'll ask rather than silently delete |
|
||||
| Flask routes (`/health`, `/competitors`, `/preview-prompt`, `/research`) | `api.py` | Already rebuilt in Phase 1 against mocks — Phase 2 rewires them to call the real `core/`/`industries/` functions via `jobs.py` and `analysis_service` instead of running synchronously inline |
|
||||
|
||||
### 2.1 New pieces (not in current `app.py`)
|
||||
|
||||
- `core/proxy_service.py` — `ProxyService` class exactly as specified
|
||||
in §10 (Webshare primary with country targeting, Bright Data
|
||||
fallback, block detection via `_is_blocked()`, auto-escalation
|
||||
recorded in `proxy_overrides`).
|
||||
- `repositories/proxy_repository.py` — already built in Phase 1.
|
||||
- `services/trip_finder_service.py` — already built in Phase 1 against
|
||||
mocks; Phase 2 wires it to the real Firecrawl `/map` endpoint.
|
||||
|
||||
### 2.2 Verification
|
||||
|
||||
Run the refactored pipeline against a known competitor URL and confirm
|
||||
the extracted JSON is byte-for-byte equivalent in shape (same fields,
|
||||
same types) to what the current `app.py` produces, per §19 step 19
|
||||
("Verify all existing functionality works identically after refactor").
|
||||
|
||||
**Phase 2 exit criteria:** `app.py` is reduced to an entry point that
|
||||
only imports and wires `api.py` + `jobs.py`; a manual end-to-end scrape
|
||||
of one real competitor page produces equivalent output to the
|
||||
pre-refactor code; existing `/research` behavior (now async via RQ
|
||||
instead of synchronous) is confirmed working through the new job queue.
|
||||
|
||||
---
|
||||
|
||||
## Phase 3 — Dashboard (Vue 3 + Nuxt UI v4)
|
||||
|
||||
**Goal:** the seven pages, wired to live Flask endpoints, matching
|
||||
`bettersight-dashboard.html` pixel-for-pixel in design language.
|
||||
|
||||
### 3.0 What the mockup confirms
|
||||
|
||||
I read `bettersight-dashboard.html` in full. Key implementation facts
|
||||
it locks in, beyond what CLAUDE.md's prose already states:
|
||||
|
||||
- Sidebar nav groups: **Intelligence** (Market Movement w/ unread-count
|
||||
badge, Comparable Trips, Battlecards w/ badge, Analyse), **Reports**
|
||||
(Weekly Briefs), **Setup** (Competitors, Account) — this is the route
|
||||
map, slightly more specific than CLAUDE.md's plain page list.
|
||||
- Every dot-grid instance is **hand-authored inline SVG** with
|
||||
per-circle opacity values forming a diagonal fade (top-left dim →
|
||||
bottom-right full saturation). `DotGrid.vue` needs to generate this
|
||||
programmatically (rows × cols × color × opacity-curve) rather than
|
||||
hardcoding 9-circle SVGs per usage, since it's reused at 3 different
|
||||
sizes (24px stat-card grids, 28px feed-row grids, 20–22px
|
||||
table/progress-row grids) with consistent fade math.
|
||||
- Pattern observed: `opacity = base + (row_index * row_step) +
|
||||
(col_index * col_step)`, roughly `.20 → 1.0` across a 3×3 grid,
|
||||
`.12 → 1.0` across the larger 6×4 stat-card grid. I'll match these
|
||||
curves exactly.
|
||||
- Topbar: search input + bell (red dot if unread alerts) + "Export"
|
||||
outline button + "Run Analysis" primary button — confirms Analyse is
|
||||
reachable from every page's topbar, not just its own nav item.
|
||||
"Export" isn't described anywhere in CLAUDE.md prose — I'm treating
|
||||
it as a generic CSV/data export of the current view and will confirm
|
||||
scope with you before building it, since §27 says no "REST API for
|
||||
external integrations" but a UI export button is a different thing.
|
||||
- Stat row is exactly 4 cards (Price Changes / Price Drops / New
|
||||
Products / Competitors) on the Activity page, matching §6
|
||||
`ActivityPage.vue` spec.
|
||||
- Activity feed has a type filter dropdown + tab row (All / Price
|
||||
changes / New products) with live counts — not explicitly itemized
|
||||
in CLAUDE.md's ActivityPage bullet list but present in the approved
|
||||
mockup, so it's in scope.
|
||||
- Competitor table's "Δ Price" column shows the single most significant
|
||||
recent price move per competitor, not a full price history — confirms
|
||||
`GET /competitors` needs to return a lightweight summary field, not
|
||||
full price_history.
|
||||
- Price Position card (sparkline + horizontal progress bars per
|
||||
competitor) appears only on the mockup, and isn't named in CLAUDE.md
|
||||
Section 6's `ActivityPage.vue` bullets at all. This looks like a
|
||||
Phase 2 (comparable trips / pricing corridor) feature that depends on
|
||||
`client_trips` + per-destination grouping. I'm scoping it into
|
||||
ActivityPage.vue's component tree as `PricePositionCard.vue` since
|
||||
the mockup is the **approved design** per §25 ("All new pages and
|
||||
components must be consistent with this design... open the reference
|
||||
HTML file before writing component code"), but flagging that its data
|
||||
dependency (a "corridor" grouping concept) isn't specified anywhere
|
||||
in the API/schema sections — I'll need to confirm the data shape with
|
||||
you before building its backend support, or treat it as static/mock
|
||||
data initially with a follow-up ticket.
|
||||
|
||||
### 3.1 Bootstrap
|
||||
|
||||
- Clone `nuxt-ui-templates/dashboard-vue` → `frontend/`
|
||||
- `npm install pinia date-fns zod axios papaparse shepherd.js @sentry/vue`
|
||||
- `app.config.ts` — color tokens (`primary: 'blue'`, `gray: 'slate'`),
|
||||
card/badge/button radius overrides per §25.
|
||||
- Global CSS variables from §25 Colour Tokens block, ported into
|
||||
`assets/` (the mockup's inline `<style>` block is the source of truth
|
||||
for exact values).
|
||||
|
||||
### 3.2 Core plumbing
|
||||
|
||||
- `services/api_service.js` — single Axios instance, `X-API-Key` +
|
||||
`Authorization: Bearer` headers, 401-refresh interceptor (§21 Session
|
||||
Management), 429 toast handling.
|
||||
- `stores/auth_store.js` — JWT, email, tier, tenant_id (Pinia).
|
||||
- `stores/analysis_store.js` — active job_id, results, history,
|
||||
`resumePolling()` for session-expiry recovery (§21).
|
||||
- `repositories/*.js` — one per domain (`tenant`, `seat`, `competitor`,
|
||||
`analysis`), each a thin wrapper over `api_service.js`.
|
||||
- `router/index.js` — routes: `/login`, `/reset-password`, `/account`,
|
||||
`/competitors`, `/analyse`, `/battlecards`, `/activity`, plus a
|
||||
catch-all redirect to `/activity` for `/`.
|
||||
- `main.js` — Sentry init (prod only), global error handler (§22.5),
|
||||
Pinia + Router mount.
|
||||
|
||||
### 3.3 Shared components first
|
||||
|
||||
Build these before any page, since every page depends on them:
|
||||
|
||||
- `components/DotGrid.vue` — `props: { rows, cols, color, size,
|
||||
opacityStart, opacityEnd }`, generates the SVG programmatically.
|
||||
- `components/shared/StatCard.vue`
|
||||
- `components/shared/CompetitorForm.vue` (modal, primary_market
|
||||
required field with `data-tour="primary-market-field"` anchor)
|
||||
- `components/shared/SeatManager.vue`
|
||||
- `components/shared/BattlecardCard.vue`
|
||||
- `components/shared/SubscriptionCard.vue`
|
||||
- `components/shared/CsvUpload.vue` + `ImportPreviewModal.vue`
|
||||
- `components/activity/FeedItem.vue`, `AlertCard.vue`
|
||||
- `components/analyse/*` (7 components per §6's AnalysePage bullet list)
|
||||
|
||||
### 3.4 Pages, in dependency order
|
||||
|
||||
1. **LoginPage.vue** + **ResetPasswordPage.vue** — auth must exist
|
||||
before anything else is reachable.
|
||||
2. **AccountPage.vue** — subscription card, seats table, template
|
||||
download, onboarding checklist, upgrade prompt, tour replay link,
|
||||
Stripe portal link.
|
||||
3. **CompetitorsPage.vue** — table + status badges + add/edit modal +
|
||||
CSV bulk import (§22) + empty state.
|
||||
4. **AnalysePage.vue** — the core page, built in the exact sub-sequence
|
||||
from §19 step 27 (form → find-urls → confirmation → confirm&run →
|
||||
poll → progress → results → confidence → push-to-sheet → history),
|
||||
with all four `JobProgress` terminal states (queued/running/
|
||||
complete/failed) explicitly handled per the Error Handling section,
|
||||
empty state for zero competitors.
|
||||
5. **ActivityPage.vue** — stat row, activity feed (tabs + filter),
|
||||
competitor table, alerts panel, price-position card (flagged above),
|
||||
empty state, onboarding tour trigger (`onMounted`).
|
||||
6. **BattlecardsPage.vue** — battlecard list + comparable trips section
|
||||
+ empty state.
|
||||
|
||||
### 3.5 Cross-cutting features (built alongside the pages above, per §19)
|
||||
|
||||
- Stripe checkout/webhook/portal wiring (3.4.2)
|
||||
- Resend email templates: `welcome.html`, `trial_reminder.html`,
|
||||
`brief.html`, `price_alert.html`, `new_product.html`
|
||||
- Freescout container + Pangolin route + footer link (infra task —
|
||||
flagged as a deployment-environment dependency, see Open Questions)
|
||||
- `useOnboardingTour.js` (Shepherd.js, 5 steps, exact step copy/anchors
|
||||
from §21) + `tour.css`
|
||||
- `data-tour` anchors wired onto sidebar nav items + primary-market
|
||||
field + push-to-sheet component
|
||||
- `PATCH /account/onboarding` route (already built in Phase 1, wired
|
||||
to real tenant updates here)
|
||||
|
||||
**Phase 3 exit criteria:** all 7 pages render with live data against
|
||||
the Phase 1/2 backend, dot-grid motif present everywhere the mockup
|
||||
shows it, onboarding tour completes end to end, Stripe checkout creates
|
||||
a real tenant via webhook in a test-mode run, every page's empty state
|
||||
matches §22's mockups, every JobProgress terminal state has a visible
|
||||
UI treatment.
|
||||
|
||||
---
|
||||
|
||||
## Phase 4 — Apps Script
|
||||
|
||||
**Goal:** standalone, install-once script. Export integration only.
|
||||
|
||||
- `appscript/Code.gs` — `onOpen()`, sidebar launch, install entry.
|
||||
- `appscript/Validation.gs` — `validateLicence()`, email-only, 6-hour
|
||||
`CacheService` cache (`BETTERSIGHT_VALID`, `BETTERSIGHT_TIER` keys).
|
||||
- `appscript/Export.gs` — `pushToSheet()`, `getWorkbookTabs()`,
|
||||
`detectColumns()` (header-based, falls back to `STANDARD_FIELDS`
|
||||
hardcoded positions with a logged warning).
|
||||
- `appscript/Sidebar.html` — tab selector + push button only, per the
|
||||
minimal mockup in §15 (the *second*, authoritative sidebar spec — not
|
||||
the earlier, more elaborate `Research.gs`/`Competitors.gs`/
|
||||
`Settings.gs`/full-sidebar version that also appears in §15).
|
||||
|
||||
**Confirmed scope:** CLAUDE.md Section 15 contains two different Apps
|
||||
Script specs back-to-back — a minimal one ("Three functions only...
|
||||
No analysis UI in the script", 4-file structure) and a larger draft
|
||||
describing `Research.gs`, `Competitors.gs`, `Settings.gs`,
|
||||
`Battlecards.gs`, `ComparableTrips.gs` with a full analysis UI inside
|
||||
the sidebar. You've confirmed the minimal spec is correct: analysis
|
||||
lives entirely in the dashboard, and the Apps Script is a thin export
|
||||
plugin only. Building the 3-function, 4-file version. The larger draft
|
||||
in §15 is treated as superseded and ignored.
|
||||
|
||||
**Phase 4 exit criteria:** script installs standalone in a test Google
|
||||
account, `validateLicence()` succeeds against a real tenant, `Push to
|
||||
Sheet` writes a stored job's results into a chosen tab using detected
|
||||
columns.
|
||||
|
||||
---
|
||||
|
||||
## Phase 5 — Automation (n8n)
|
||||
|
||||
- Daily 6am scrape cron (changed/stale competitors only)
|
||||
- `price_history` diff logic (change_amount/percent) + `is_new` flag
|
||||
logic — implemented as PocketBase hooks or n8n Code nodes acting on
|
||||
insert events (confirm preference — see Open Questions)
|
||||
- `/internal/monitor-webhook`, `/internal/monitor/register`,
|
||||
`/internal/monitor/deregister` — already stubbed in Phase 1, wired to
|
||||
real Firecrawl `/v1/monitor` calls here
|
||||
- Auto-register Firecrawl Monitor on competitor add (in
|
||||
`competitor_service`/route, not n8n)
|
||||
- price_change / new_product alert-creation workflows
|
||||
- Daily 6pm digest email workflow (Resend)
|
||||
- Battlecard regeneration workflow (fires on `scrape_runs` complete)
|
||||
- Monday 6am brief workflow
|
||||
- Dashboard bell icon + unread count (frontend, reads `alerts` table)
|
||||
- Mark-as-read / dismiss on `AlertCard.vue`
|
||||
|
||||
**Phase 5 exit criteria:** a full daily-cron dry run against 2-3 test
|
||||
competitors produces correct `price_history` diffs, a price-change
|
||||
alert appears in the dashboard activity feed, the digest email sends
|
||||
via Resend (test mode), and the Monday brief renders correctly.
|
||||
|
||||
---
|
||||
|
||||
## Phase 6 — Semantic Matching
|
||||
|
||||
- `embedding_service.embed_trip()` + `find_comparable_trips()` (cosine
|
||||
similarity, 0.75 threshold) — fleshed out from the Phase 1 stub
|
||||
- Sunday-night n8n embedding/matching workflow
|
||||
- `comparable_matches` populated and visible on `BattlecardsPage.vue`
|
||||
|
||||
**Phase 6 exit criteria:** embeddings generate for both client trips
|
||||
and competitor products, matches above threshold appear correctly
|
||||
ranked in the dashboard.
|
||||
|
||||
---
|
||||
|
||||
## Phase 7 — Ship
|
||||
|
||||
Per §19 steps 53–76, in order:
|
||||
|
||||
1. Flask-Limiter on all public routes (verify, since Phase 1 already
|
||||
applied limits — this is a final audit pass)
|
||||
2. Sentry — staging first, then production (backend + frontend)
|
||||
3. GDPR delete endpoint — full end-to-end test (cancels Stripe, purges
|
||||
all tenant-scoped tables, sends confirmation email, logs to
|
||||
append-only audit log)
|
||||
4. n8n weekly data-retention cleanup workflow (§22's `DATA_RETENTION`
|
||||
policy)
|
||||
5. Empty states audit across all 7 pages (including ResetPasswordPage)
|
||||
6. Session-expiry handling — token refresh interceptor + return-to
|
||||
logic (mostly built in Phase 3, audited here)
|
||||
7. In-progress job survival across session expiry (localStorage
|
||||
`pending_job_id` resume)
|
||||
8. Password reset flow (built in Phase 3, audited here)
|
||||
9. Timezone capture on signup → stored + used by digest scheduling
|
||||
(luxon in n8n)
|
||||
10. Upgrade-prompt card on AccountPage (built in Phase 3, audited here)
|
||||
11. Multi-currency display in ResultsTable — hero/secondary currency
|
||||
rule from §6
|
||||
12. Litestream container added to `docker-compose.yml`
|
||||
13. Hetzner Object Storage daily 2am backup cron (belt-and-braces with
|
||||
Litestream)
|
||||
14. `check_robots_txt()` — confirm wiring from Phase 2 is actually
|
||||
gating scrapes (not just defined)
|
||||
15. OpenRouter paid-fallback env var confirmed set
|
||||
16. Scraping legal clause added to `/terms`
|
||||
17. Privacy policy + terms pages live
|
||||
18. Google OAuth consent screen approved (needs the privacy policy URL
|
||||
live first — ordering dependency)
|
||||
19. End-to-end test with real competitor data
|
||||
20. Staging deploy
|
||||
21. Tag `v1.0.0`
|
||||
22. Production deploy via Dokploy
|
||||
|
||||
Also folded in here per the Monitoring note in 1.6: the n8n
|
||||
queue/API/worker health-check workflow (§29 Component 1), job-duration
|
||||
alerting confirmation (Component 2, built in Phase 1), daily-cron
|
||||
duration alert (Component 3), weekly performance summary (Component
|
||||
4), and Firecrawl Monitor health check (Component 5).
|
||||
|
||||
---
|
||||
|
||||
## Decisions — defaults applied, not blocking
|
||||
|
||||
CLAUDE.md was a demand-validation artifact, not a contracted spec, so
|
||||
the items below are resolved with sensible defaults rather than left
|
||||
open. I'll adjust any of these on request, but none of them block
|
||||
Phase 1.
|
||||
|
||||
1. **Licence validation** — standalone, no `sheet_id` binding (matches
|
||||
the dashboard-first, export-only-plugin direction confirmed above).
|
||||
2. **Apps Script scope** — confirmed: 3-function export-only plugin.
|
||||
3. **`GOTIFY_APP_TOKEN`** — treating the inline value in CLAUDE.md §17
|
||||
as a placeholder/example, not a live secret. Real value goes in
|
||||
`.env` only, never committed.
|
||||
4. **Git** — I'll `git init` + add a `.gitignore` (`.env`,
|
||||
`__pycache__`, `node_modules`, `pb_data/`) at the start of Phase 1.
|
||||
5. **"Export" button** (mockup topbar) — building as a CSV export of
|
||||
the current page's visible data. Easy to extend later.
|
||||
6. **Price Position card** — shipping in Phase 3 with the UI fully
|
||||
built but backed by a simple destination-grouped query (group
|
||||
`products`/`client_trips` by `destination`); not gold-plating the
|
||||
"corridor" concept beyond what's needed to populate the sparkline
|
||||
and bars.
|
||||
7. **CI** — skipping a GitHub Actions workflow for now; `pytest --cov`
|
||||
run manually/locally against the 80% floor. Easy to add later.
|
||||
8. **Infra-dependent items** (Freescout DNS, Hetzner/Dokploy, Stripe
|
||||
live keys, Google OAuth consent, Webshare/Bright Data accounts) —
|
||||
building everything code-side (compose services, webhook handlers,
|
||||
env var plumbing); actual account provisioning and `.env` secrets
|
||||
are on you when we get to Phase 3 (Freescout) and Phase 7 (deploy).
|
||||
9. **Dead Groq/Gemini alternates** in `app.py` (lines 569–630) —
|
||||
dropping them in the Phase 2 refactor. OpenRouter is the sole
|
||||
active provider; not carrying dead code forward.
|
||||
|
||||
---
|
||||
|
||||
## Sequencing
|
||||
|
||||
Ready to start Phase 1 on your go-ahead. Each phase's exit criteria
|
||||
above is the checkpoint I'll report back against before moving to the
|
||||
next phase, per CLAUDE.md's "Do not start a phase until the previous is
|
||||
complete and tested."
|
||||
Loading…
Reference in New Issue