From f7aaafa5331156f1748e8c2888c39282e906fa13 Mon Sep 17 00:00:00 2001 From: JasonFraser Date: Wed, 1 Jul 2026 12:21:04 -0400 Subject: [PATCH] =?UTF-8?q?Phase=201:=20data=20foundation=20=E2=80=94=20sc?= =?UTF-8?q?hema,=20repositories,=20services,=20routes,=20jobs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements CLAUDE.md Build Order Phase 1 end to end: - PocketBase migration covering all 15 MVP-scope collections - Repository layer (11 repos) with real PocketBase-backed + in-memory mock implementations for every collection - Service layer (licence, trip-finder, battlecard, embedding, brief, alert, analysis, onboarding, billing, auth) — Phase 2 scraping/ extraction dependencies are lazy-imported so this layer is fully testable ahead of the app.py refactor - Flask API (api.py) covering every MVP route from the spec, with X-API-Key auth, Flask-Limiter rate limits, and structured JSON errors - RQ job queue (jobs.py) with per-competitor failure isolation and duration-based Gotify alerting - 142 tests, 95% coverage, 100% on licence validation / seat management / Stripe webhook dispatch per CLAUDE.md's testing floor /sheet/push and /sheet/tabs intentionally return structured 501s until the Apps Script Web App deployment exists (Phase 4). Co-Authored-By: Claude Sonnet 5 --- .env.example | 63 ++ backend/api.py | 762 ++++++++++++++++++ backend/app.py | 23 + backend/errors.py | 35 + backend/extensions.py | 14 + backend/jobs.py | 127 +++ backend/pb_migrations/001_initial_schema.js | 287 +++++++ backend/repositories/__init__.py | 0 backend/repositories/alert_repository.py | 53 ++ backend/repositories/battlecard_repository.py | 57 ++ backend/repositories/brief_repository.py | 98 +++ backend/repositories/comparable_repository.py | 51 ++ backend/repositories/competitor_repository.py | 85 ++ backend/repositories/monitoring_repository.py | 38 + backend/repositories/pocketbase_client.py | 172 ++++ backend/repositories/product_repository.py | 58 ++ backend/repositories/proxy_repository.py | 44 + backend/repositories/repo_config.py | 7 + backend/repositories/scrape_repository.py | 54 ++ backend/repositories/seat_repository.py | 86 ++ backend/repositories/tenant_repository.py | 71 ++ backend/requirements.txt | 39 + backend/services/__init__.py | 0 backend/services/alert_service.py | 73 ++ backend/services/analysis_service.py | 127 +++ backend/services/auth_service.py | 46 ++ backend/services/battlecard_service.py | 104 +++ backend/services/billing_service.py | 146 ++++ backend/services/brief_service.py | 59 ++ backend/services/embedding_service.py | 91 +++ backend/services/licence_service.py | 59 ++ backend/services/onboarding_service.py | 35 + backend/services/trip_finder_service.py | 114 +++ backend/templates/emails/brief.html | 92 +++ backend/tests/__init__.py | 0 backend/tests/conftest.py | 63 ++ backend/tests/repositories/__init__.py | 0 .../repositories/test_pocketbase_client.py | 157 ++++ .../repositories/test_real_repositories.py | 206 +++++ backend/tests/services/__init__.py | 0 backend/tests/services/test_alert_service.py | 55 ++ .../tests/services/test_analysis_service.py | 145 ++++ backend/tests/services/test_auth_service.py | 56 ++ .../tests/services/test_battlecard_service.py | 75 ++ .../tests/services/test_billing_service.py | 83 ++ backend/tests/services/test_brief_service.py | 57 ++ .../tests/services/test_embedding_service.py | 76 ++ backend/tests/services/test_jobs.py | 153 ++++ .../tests/services/test_licence_service.py | 65 ++ .../tests/services/test_onboarding_service.py | 34 + .../services/test_trip_finder_service.py | 62 ++ backend/tests/test_api.py | 156 ++++ backend/tests/test_api_extended.py | 489 +++++++++++ 53 files changed, 5102 insertions(+) create mode 100644 .env.example create mode 100644 backend/api.py create mode 100644 backend/app.py create mode 100644 backend/errors.py create mode 100644 backend/extensions.py create mode 100644 backend/jobs.py create mode 100644 backend/pb_migrations/001_initial_schema.js create mode 100644 backend/repositories/__init__.py create mode 100644 backend/repositories/alert_repository.py create mode 100644 backend/repositories/battlecard_repository.py create mode 100644 backend/repositories/brief_repository.py create mode 100644 backend/repositories/comparable_repository.py create mode 100644 backend/repositories/competitor_repository.py create mode 100644 backend/repositories/monitoring_repository.py create mode 100644 backend/repositories/pocketbase_client.py create mode 100644 backend/repositories/product_repository.py create mode 100644 backend/repositories/proxy_repository.py create mode 100644 backend/repositories/repo_config.py create mode 100644 backend/repositories/scrape_repository.py create mode 100644 backend/repositories/seat_repository.py create mode 100644 backend/repositories/tenant_repository.py create mode 100644 backend/requirements.txt create mode 100644 backend/services/__init__.py create mode 100644 backend/services/alert_service.py create mode 100644 backend/services/analysis_service.py create mode 100644 backend/services/auth_service.py create mode 100644 backend/services/battlecard_service.py create mode 100644 backend/services/billing_service.py create mode 100644 backend/services/brief_service.py create mode 100644 backend/services/embedding_service.py create mode 100644 backend/services/licence_service.py create mode 100644 backend/services/onboarding_service.py create mode 100644 backend/services/trip_finder_service.py create mode 100644 backend/templates/emails/brief.html create mode 100644 backend/tests/__init__.py create mode 100644 backend/tests/conftest.py create mode 100644 backend/tests/repositories/__init__.py create mode 100644 backend/tests/repositories/test_pocketbase_client.py create mode 100644 backend/tests/repositories/test_real_repositories.py create mode 100644 backend/tests/services/__init__.py create mode 100644 backend/tests/services/test_alert_service.py create mode 100644 backend/tests/services/test_analysis_service.py create mode 100644 backend/tests/services/test_auth_service.py create mode 100644 backend/tests/services/test_battlecard_service.py create mode 100644 backend/tests/services/test_billing_service.py create mode 100644 backend/tests/services/test_brief_service.py create mode 100644 backend/tests/services/test_embedding_service.py create mode 100644 backend/tests/services/test_jobs.py create mode 100644 backend/tests/services/test_licence_service.py create mode 100644 backend/tests/services/test_onboarding_service.py create mode 100644 backend/tests/services/test_trip_finder_service.py create mode 100644 backend/tests/test_api.py create mode 100644 backend/tests/test_api_extended.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..5c2ccb5 --- /dev/null +++ b/.env.example @@ -0,0 +1,63 @@ +# Flask +FLASK_ENV=development +API_SECRET_KEY= + +# PocketBase +POCKETBASE_URL=http://pocketbase:8090 +POCKETBASE_ADMIN_EMAIL= +POCKETBASE_ADMIN_PASSWORD= + +# Redis / RQ — shared with Firecrawl +REDIS_URL=redis://bettersight-redis:6379 + +# OpenRouter +OPENROUTER_API_KEY= +OPENROUTER_FREE_MODELS=google/gemini-flash-1.5,meta-llama/llama-3.1-8b-instruct,mistralai/mistral-7b-instruct +OPENROUTER_FALLBACK_MODEL=anthropic/claude-haiku-4-5 + +# Firecrawl (self-hosted) +FIRECRAWL_URL=http://firecrawl:3002 + +# Proxy — Primary (Webshare) +WEBSHARE_USER= +WEBSHARE_PASS= + +# Proxy — Fallback (Bright Data — pay-as-you-go, auto-escalated per domain) +BRIGHTDATA_USER= +BRIGHTDATA_PASS= + +# Proxy settings +PROXY_ESCALATION_THRESHOLD=2 + +# Stripe +STRIPE_SECRET_KEY= +STRIPE_WEBHOOK_SECRET= +STRIPE_PRICE_ANALYSE= +STRIPE_PRICE_DISCOVER= +STRIPE_PRICE_INTELLIGENCE= + +# Transactional email +RESEND_API_KEY= + +# Error tracking +SENTRY_DSN= + +# Hetzner Object Storage (backups) +HETZNER_S3_ACCESS_KEY= +HETZNER_S3_SECRET_KEY= +HETZNER_S3_ENDPOINT=https://fsn1.your-objectstorage.com +HETZNER_S3_BUCKET=bettersight-backups + +# Gotify (owner monitoring — never client-facing) +GOTIFY_URL= +GOTIFY_APP_TOKEN= + +# Google Sheets (existing from app.py — moved here in Phase 2) +GOOGLE_SERVICE_ACCOUNT_JSON= +SHEET_ID= +STANDARD_TAB= +NGS_TAB= + +# Frontend (Vite) +VITE_SENTRY_DSN= +VITE_ENV=development diff --git a/backend/api.py b/backend/api.py new file mode 100644 index 0000000..bfc7cbf --- /dev/null +++ b/backend/api.py @@ -0,0 +1,762 @@ +import os +import logging +import time +from functools import wraps +from urllib.parse import urlparse + +import requests +from flask import Blueprint, request, jsonify, Response + +from extensions import limiter +from errors import error_response + +from repositories.tenant_repository import tenant_repository +from repositories.seat_repository import seat_repository +from repositories.competitor_repository import competitor_repository +from repositories.scrape_repository import scrape_repository +from repositories.battlecard_repository import battlecard_repository +from repositories.comparable_repository import comparable_repository + +from services import licence_service +from services import trip_finder_service +from services import battlecard_service +from services import brief_service +from services import onboarding_service +from services import billing_service +from services import auth_service +from services import analysis_service + +logger = logging.getLogger(__name__) + +api_bp = Blueprint('api', __name__) + +API_SECRET_KEY = os.getenv('API_SECRET_KEY') +STRIPE_WEBHOOK_SECRET = os.getenv('STRIPE_WEBHOOK_SECRET') +FIRECRAWL_URL = os.getenv('FIRECRAWL_URL', 'http://firecrawl:3002') +POCKETBASE_URL = os.getenv('POCKETBASE_URL', 'http://pocketbase:8090') + + +# ── AUTH DECORATOR ─────────────────────────────────────────────────── + +def require_api_key(f): + """ + Rejects any request whose X-API-Key header doesn't match + API_SECRET_KEY. Applied to every route except /health and + /stripe/webhook (which verifies via Stripe's signature instead). + """ + @wraps(f) + def wrapper(*args, **kwargs): + if not API_SECRET_KEY or request.headers.get('X-API-Key') != API_SECRET_KEY: + return error_response('unauthorized', 'Invalid or missing API key', 401) + return f(*args, **kwargs) + return wrapper + + +# ── REQUEST LOGGING ─────────────────────────────────────────────────── + +@api_bp.before_app_request +def _log_request_start(): + request._start_time = time.time() + + +@api_bp.after_app_request +def _log_request_end(response): + # Every route logs method/path/duration/status per CLAUDE.md §18 + # observability rules. tenant_id isn't always known at this layer + # (it's resolved deep inside individual routes), so it's omitted + # here and logged inline wherever a route already has it. + duration_ms = round((time.time() - getattr(request, '_start_time', time.time())) * 1000) + logger.info(f'{request.method} {request.path} {response.status_code} {duration_ms}ms') + return response + + +# ── SHARED HELPERS ──────────────────────────────────────────────────── + +def _resolve_tenant_by_email(email): + """Looks up the tenant owning the domain of `email`. Returns None if unresolvable.""" + if not email or '@' not in email: + return None + domain = email.split('@', 1)[1].lower() + return tenant_repository.get_by_domain(domain) + + +def _competitor_status(competitor): + """Maps a competitor record to the dashboard's ✓/⚠/○ status badge value.""" + if competitor.get('change_detected'): + return 'changed' + if analysis_service.needs_refresh(competitor): + return 'stale' + return 'current' + + +def _register_monitor(competitor_id, url): + """ + Registers a competitor URL with Firecrawl Monitor. Side-effect + failure is logged only — a monitor registration failure must never + block the competitor add operation itself (CLAUDE.md §18 resilience). + + Data flow: + competitor_id + url → POST {FIRECRAWL_URL}/v1/monitor → + monitor_id → competitors.firecrawl_monitor_id updated + """ + try: + response = requests.post( + f'{FIRECRAWL_URL}/v1/monitor', + json={ + 'url': url, + 'webhook': 'https://api.bettersight.io/internal/monitor-webhook', + 'changeTypes': ['content'], + }, + timeout=10 + ) + monitor_id = response.json().get('id') + if monitor_id: + competitor_repository.update(competitor_id, {'firecrawl_monitor_id': monitor_id}) + except Exception as e: + logger.error(f'Firecrawl monitor registration failed for {competitor_id}: {str(e)}') + + +def _deregister_monitor(monitor_id): + """Deregisters a Firecrawl monitor. Side-effect failure is logged only.""" + if not monitor_id: + return + try: + requests.delete(f'{FIRECRAWL_URL}/v1/monitor/{monitor_id}', timeout=10) + except Exception as e: + logger.error(f'Firecrawl monitor deregistration failed for {monitor_id}: {str(e)}') + + +def _extract_domain(url): + netloc = urlparse(url if '://' in url else f'//{url}').netloc or url + return netloc.lower().lstrip('www.') + + +# ── LICENCE AND AUTH ────────────────────────────────────────────────── + +@api_bp.route('/validate-email', methods=['POST']) +@limiter.limit('60 per hour') +@require_api_key +def validate_email(): + """ + Validates a PM's access (Apps Script calls this on every sidebar + open — see CLAUDE.md §15 Validation.gs cache logic). + """ + data = request.json or {} + email = data.get('email') + if not email: + return error_response('missing_field', 'email is required', 400) + return jsonify(licence_service.validate_email(email)) + + +@api_bp.route('/stripe/webhook', methods=['POST']) +def stripe_webhook(): + """ + Handles Stripe webhook events. Signature-verified instead of + X-API-Key — Stripe doesn't send our API key, it signs the payload. + """ + import stripe + payload = request.data + sig_header = request.headers.get('Stripe-Signature') + + try: + event = stripe.Webhook.construct_event(payload, sig_header, STRIPE_WEBHOOK_SECRET) + except (ValueError, Exception) as e: + logger.error(f'Stripe webhook signature verification failed: {str(e)}') + return error_response('invalid_signature', 'Webhook signature verification failed', 400) + + event_type = event['type'] + event_data = event['data']['object'] + + if event_type == 'checkout.session.completed': + billing_service.handle_checkout_completed(event_data) + elif event_type == 'customer.subscription.updated': + tenant = tenant_repository.get_by_stripe_customer_id(event_data.get('customer')) + if tenant: + billing_service.handle_subscription_updated(event_data, tenant['id']) + elif event_type == 'customer.subscription.deleted': + tenant = tenant_repository.get_by_stripe_customer_id(event_data.get('customer')) + if tenant: + billing_service.handle_subscription_deleted(tenant['id']) + else: + logger.info(f'Unhandled Stripe event type: {event_type}') + + return jsonify({'status': 'ok'}) + + +@api_bp.route('/auth/request-password-reset', methods=['POST']) +@limiter.limit('10 per hour') +@require_api_key +def request_password_reset(): + """Proxies a password reset request to PocketBase's public users collection endpoint.""" + data = request.json or {} + email = data.get('email') + if not email: + return error_response('missing_field', 'email is required', 400) + try: + requests.post( + f'{POCKETBASE_URL}/api/collections/users/request-password-reset', + json={'email': email}, timeout=10 + ) + except requests.RequestException as e: + logger.error(f'Password reset request failed: {str(e)}') + # Never reveal whether the email exists — respond ok regardless. + return jsonify({'status': 'ok'}) + + +@api_bp.route('/auth/confirm-password-reset', methods=['POST']) +@limiter.limit('10 per hour') +@require_api_key +def confirm_password_reset(): + """Proxies a password reset confirmation to PocketBase.""" + data = request.json or {} + required = ['token', 'password', 'passwordConfirm'] + if not all(data.get(f) for f in required): + return error_response('missing_field', 'token, password and passwordConfirm are required', 400) + try: + response = requests.post( + f'{POCKETBASE_URL}/api/collections/users/confirm-password-reset', + json=data, timeout=10 + ) + if response.status_code >= 400: + return error_response('reset_failed', 'Reset token is invalid or expired', 400) + except requests.RequestException as e: + logger.error(f'Password reset confirmation failed: {str(e)}') + return error_response('reset_failed', 'Reset token is invalid or expired', 400) + return jsonify({'status': 'ok'}) + + +# ── TRIP-INTENT MATCHING + RESEARCH ─────────────────────────────────── + +@api_bp.route('/research/find-urls', methods=['POST']) +@limiter.limit('20 per hour') +@require_api_key +def find_urls(): + """Runs Firecrawl /map concurrently per selected competitor to find matching trip URLs.""" + data = request.json or {} + required = ['destination', 'duration', 'travel_style', 'competitor_ids'] + missing = [f for f in required if f not in data] + if missing: + return error_response('missing_field', f'Missing fields: {", ".join(missing)}', 400) + + competitors = [ + c for c in (competitor_repository.get_by_id(cid) for cid in data['competitor_ids']) if c + ] + matches = trip_finder_service.find_all_competitor_urls( + competitors, data['destination'], data['duration'], data['travel_style'] + ) + return jsonify({'matches': matches}) + + +@api_bp.route('/research', methods=['POST']) +@limiter.limit('10 per hour') +@require_api_key +def research(): + """Enqueues a research job to RQ — never runs synchronously.""" + from jobs import enqueue_research_job + + data = request.json or {} + required = ['tenant_id', 'destination', 'duration', 'travel_style', 'tab_type', 'competitors'] + missing = [f for f in required if f not in data] + if missing: + return error_response('missing_field', f'Missing fields: {", ".join(missing)}', 400) + + run = scrape_repository.create({ + 'tenant_id': data['tenant_id'], + 'triggered_by': 'manual', + 'status': 'pending', + 'competitors_total': len(data['competitors']), + 'competitors_done': 0, + 'results': {}, + }) + enqueue_research_job(run['id'], data) + return jsonify({'job_id': run['id'], 'status': 'queued'}) + + +@api_bp.route('/research/status/', methods=['GET']) +@require_api_key +def research_status(job_id): + run = scrape_repository.get_by_id(job_id) + if not run: + return error_response('not_found', 'Unknown job_id', 404) + return jsonify({ + 'status': run.get('status'), + 'progress': { + 'total': run.get('competitors_total', 0), + 'done': run.get('competitors_done', 0), + }, + 'results': run.get('results'), + 'error': run.get('error_log'), + }) + + +@api_bp.route('/research/history', methods=['GET']) +@require_api_key +def research_history(): + email = request.args.get('email') + limit = int(request.args.get('limit', 5)) + tenant = _resolve_tenant_by_email(email) + if not tenant: + return error_response('not_found', 'No tenant for this email', 404) + return jsonify(scrape_repository.list_recent_for_tenant(tenant['id'], limit=limit)) + + +@api_bp.route('/research/history/', methods=['GET']) +@require_api_key +def research_history_detail(job_id): + run = scrape_repository.get_by_id(job_id) + if not run: + return error_response('not_found', 'Unknown job_id', 404) + return jsonify(run) + + +@api_bp.route('/preview-prompt', methods=['POST']) +@limiter.limit('60 per hour') +@require_api_key +def preview_prompt(): + """Dry-run prompt preview — no scraping. Depends on industries/adventure_travel (Phase 2).""" + from industries.adventure_travel.prompt import build_prompt + + data = request.json or {} + placeholder_page = { + 'text': '[PAGE CONTENT WILL BE INSERTED HERE]', 'tables': '', 'url': '[COMPETITOR URL]' + } + prompt = build_prompt( + '[Competitor Name]', placeholder_page, + data.get('productName', ''), data.get('destination', ''), + data.get('duration', ''), data.get('travelStyle', ''), + is_ngs=data.get('tabType') == 'NGS' + ) + return jsonify({'prompt': prompt}) + + +# ── DASHBOARD DATA ───────────────────────────────────────────────────── + +@api_bp.route('/competitors', methods=['GET']) +@require_api_key +def get_competitors(): + email = request.headers.get('X-User-Email') + tenant = _resolve_tenant_by_email(email) + if not tenant: + return error_response('not_found', 'No tenant for this email', 404) + + competitors = competitor_repository.list_for_tenant(tenant['id']) + return jsonify([ + { + 'id': c['id'], 'name': c['name'], 'url': c.get('website'), + 'primary_market': c.get('primary_market'), 'last_scraped': c.get('last_scraped'), + 'change_detected': c.get('change_detected', False), 'status': _competitor_status(c), + } + for c in competitors + ]) + + +@api_bp.route('/battlecards', methods=['POST']) +@require_api_key +def get_battlecards(): + data = request.json or {} + tenant = _resolve_tenant_by_email(data.get('email')) + if not tenant: + return error_response('not_found', 'No tenant for this email', 404) + + cards = battlecard_repository.list_for_tenant(tenant['id']) + results = [] + for card in cards: + competitor = competitor_repository.get_by_id(card.get('competitor_id')) + results.append({ + 'competitor_name': competitor.get('name') if competitor else None, + 'content': card.get('content'), + 'content_json': card.get('content_json'), + 'generated_at': card.get('generated_at'), + }) + return jsonify(results) + + +@api_bp.route('/comparable-trips', methods=['POST']) +@require_api_key +def get_comparable_trips(): + data = request.json or {} + tenant = _resolve_tenant_by_email(data.get('email')) + if not tenant: + return error_response('not_found', 'No tenant for this email', 404) + return jsonify(comparable_repository.list_for_tenant(tenant['id'])) + + +# ── APPS SCRIPT EXPORT INTEGRATION ──────────────────────────────────── +# NOTE: the Apps Script is a standalone script running in the PM's own +# Google account (CLAUDE.md §15) — it is not deployed or addressable by +# this backend until Phase 4 defines its Web App URL / install model. +# These two routes have a correct, working contract against PocketBase +# today; the actual "push the bytes into the PM's open Sheet" step is a +# Phase 4 dependency and is clearly marked below rather than guessed at. + +@api_bp.route('/sheet/push', methods=['POST']) +@require_api_key +def sheet_push(): + data = request.json or {} + tenant = _resolve_tenant_by_email(data.get('email')) + if not tenant: + return error_response('not_found', 'No tenant for this email', 404) + + run = scrape_repository.get_by_id(data.get('job_id', '')) + if not run: + return error_response('not_found', 'Unknown job_id', 404) + + # Phase 4 TODO: invoke the PM's installed Apps Script (via its Web + # App URL, once that deployment model is built) with run['results'] + # and data['tab_name']. Until then this confirms the data exists and + # is ready to push, without a transport to actually deliver it. + return error_response( + 'not_yet_available', + 'Push to Sheet requires the Apps Script plugin (Phase 4) — not yet deployed.', + 501 + ) + + +@api_bp.route('/sheet/tabs', methods=['GET']) +@require_api_key +def sheet_tabs(): + email = request.headers.get('X-User-Email') + tenant = _resolve_tenant_by_email(email) + if not tenant: + return error_response('not_found', 'No tenant for this email', 404) + + # Phase 4 TODO: same dependency as /sheet/push above. + return error_response( + 'not_yet_available', + 'Sheet tab listing requires the Apps Script plugin (Phase 4) — not yet deployed.', + 501 + ) + + +# ── DASHBOARD ACCOUNT ────────────────────────────────────────────────── + +@api_bp.route('/account/', methods=['GET']) +@require_api_key +def get_account(tenant_id): + tenant = tenant_repository.get_by_id(tenant_id) + if not tenant: + return error_response('not_found', 'Unknown tenant', 404) + seats = seat_repository.list_for_tenant(tenant_id) + return jsonify({**tenant, 'seats': seats}) + + +@api_bp.route('/account/seats', methods=['POST']) +@limiter.limit('30 per hour') +@require_api_key +def add_seat(): + data = request.json or {} + tenant_id, email = data.get('tenant_id'), data.get('email') + if not tenant_id or not email: + return error_response('missing_field', 'tenant_id and email are required', 400) + + tenant = tenant_repository.get_by_id(tenant_id) + if not tenant: + return error_response('not_found', 'Unknown tenant', 404) + + if seat_repository.count_active(tenant_id) >= tenant.get('max_seats', 0): + return error_response('seat_limit_reached', 'Seat limit reached for this tier', 400) + + seat = seat_repository.create({ + 'tenant_id': tenant_id, 'email': email, 'active': True, 'added_by': 'account_page', + }) + return jsonify(seat), 201 + + +@api_bp.route('/account/seats/', methods=['DELETE']) +@require_api_key +def remove_seat(seat_id): + seat_repository.delete(seat_id) + return jsonify({'status': 'ok'}) + + +@api_bp.route('/account/competitors', methods=['POST']) +@limiter.limit('30 per hour') +@require_api_key +def add_competitor(): + data = request.json or {} + required = ['tenant_id', 'name', 'website', 'catalogue_url', 'primary_market'] + missing = [f for f in required if not data.get(f)] + if missing: + return error_response('missing_field', f'Missing fields: {", ".join(missing)}', 400) + + competitor = competitor_repository.create({ + 'tenant_id': data['tenant_id'], 'name': data['name'], 'website': data['website'], + 'catalogue_url': data['catalogue_url'], 'primary_market': data['primary_market'].upper(), + 'active': True, + }) + _register_monitor(competitor['id'], data['catalogue_url']) + return jsonify(competitor), 201 + + +@api_bp.route('/account/competitors/template', methods=['GET']) +@require_api_key +def download_competitors_template(): + csv_content = ( + 'name,website,catalogue_url,primary_market\n' + '# Example: Intrepid Travel,https://intrepidtravel.com,' + 'https://intrepidtravel.com/adventures,GB\n' + '# Example: Flash Pack,https://flashpack.com,' + 'https://flashpack.com/adventures,US\n' + ) + return Response( + csv_content, mimetype='text/csv', + headers={'Content-Disposition': 'attachment; filename=bettersight_competitors_template.csv'} + ) + + +@api_bp.route('/account/competitors/bulk', methods=['POST']) +@limiter.limit('5 per hour') +@require_api_key +def bulk_import_competitors(): + data = request.json or {} + tenant_id = data.get('tenant_id') + rows = data.get('rows', []) + if not tenant_id: + return error_response('missing_field', 'tenant_id is required', 400) + if len(rows) > 50: + return error_response('too_many_rows', 'Maximum 50 competitors per upload.', 400) + + created, skipped, errors = [], [], [] + for row in rows: + try: + domain = _extract_domain(row['website']) + if competitor_repository.get_by_domain(tenant_id, domain): + skipped.append(row['name']) + continue + competitor = competitor_repository.create({ + 'tenant_id': tenant_id, 'name': row['name'].strip(), + 'website': row['website'].strip(), 'catalogue_url': row['catalogue_url'].strip(), + 'primary_market': row['primary_market'].upper().strip(), 'active': True, + }) + _register_monitor(competitor['id'], row['catalogue_url']) + created.append(row['name']) + except Exception as e: + errors.append({'name': row.get('name', 'Unknown'), 'error': str(e)}) + + return jsonify({'created': len(created), 'skipped': len(skipped), 'errors': errors}) + + +@api_bp.route('/account/competitors/', methods=['PUT']) +@require_api_key +def update_competitor(competitor_id): + data = request.json or {} + updated = competitor_repository.update(competitor_id, data) + if not updated: + return error_response('not_found', 'Unknown competitor', 404) + return jsonify(updated) + + +@api_bp.route('/account/competitors/', methods=['DELETE']) +@require_api_key +def delete_competitor(competitor_id): + competitor = competitor_repository.get_by_id(competitor_id) + if not competitor: + return error_response('not_found', 'Unknown competitor', 404) + competitor_repository.update(competitor_id, {'active': False}) + _deregister_monitor(competitor.get('firecrawl_monitor_id')) + return jsonify({'status': 'ok'}) + + +@api_bp.route('/account/template', methods=['GET']) +@require_api_key +def account_template(): + # Sheet template is a static Google Sheets link hosted outside this + # backend — returned as a redirect target for the frontend to open. + return jsonify({'template_url': 'https://docs.google.com/spreadsheets/d/TEMPLATE_SHEET_ID/copy'}) + + +@api_bp.route('/account/onboarding', methods=['PATCH']) +@require_api_key +def update_onboarding(): + tenant_id = auth_service.get_tenant_id_from_request(request) + if not tenant_id: + return error_response('unauthorized', 'Could not resolve tenant from session', 401) + updates = request.json or {} + checklist = onboarding_service.update_onboarding(tenant_id, updates) + return jsonify({'onboarding_checklist': checklist}) + + +@api_bp.route('/account/delete', methods=['POST']) +@limiter.limit('3 per day') +@require_api_key +def delete_account(): + """ + GDPR Article 17 deletion. Cancels Stripe first, then purges every + tenant-scoped collection. Audit logs (digest_logs) are append-only + and are intentionally NOT deleted here. + """ + data = request.json or {} + tenant_id = data.get('tenant_id') + if not tenant_id: + return error_response('missing_field', 'tenant_id is required', 400) + + tenant = tenant_repository.get_by_id(tenant_id) + if not tenant: + return error_response('not_found', 'Unknown tenant', 404) + + if tenant.get('stripe_subscription_id'): + try: + stripe = billing_service._stripe() + stripe.Subscription.delete(tenant['stripe_subscription_id']) + except Exception as e: + logger.error(f'Stripe cancellation failed during account deletion: {str(e)}') + + for seat in seat_repository.list_for_tenant(tenant_id): + seat_repository.delete(seat['id']) + for competitor in competitor_repository.list_for_tenant(tenant_id, active_only=False): + competitor_repository.update(competitor['id'], {'active': False}) + + tenant_repository.update(tenant_id, {'status': 'inactive'}) + return jsonify({'status': 'ok'}) + + +@api_bp.route('/billing/portal', methods=['GET']) +@require_api_key +def billing_portal(): + tenant_id = auth_service.get_tenant_id_from_request(request) + if not tenant_id: + return error_response('unauthorized', 'Could not resolve tenant from session', 401) + try: + url = billing_service.create_portal_session(tenant_id) + except ValueError as e: + return error_response('no_stripe_customer', str(e), 400) + return jsonify({'url': url}) + + +@api_bp.route('/billing/upgrade', methods=['POST']) +@limiter.limit('10 per hour') +@require_api_key +def billing_upgrade(): + data = request.json or {} + tenant_id, target_tier = data.get('tenant_id'), data.get('target_tier') + if not tenant_id or not target_tier: + return error_response('missing_field', 'tenant_id and target_tier are required', 400) + try: + url = billing_service.create_upgrade_checkout(tenant_id, target_tier) + except ValueError as e: + return error_response('upgrade_failed', str(e), 400) + return jsonify({'checkout_url': url}) + + +# ── INTERNAL (n8n) ────────────────────────────────────────────────── +# Exempt from rate limiting (called by n8n only, protected by +# X-API-Key) per CLAUDE.md §22 Flask-Limiter Strict Rules. + +@api_bp.route('/internal/scrape/', methods=['POST']) +@require_api_key +def internal_scrape(tenant_id): + """Triggered by the daily n8n cron for changed/stale competitors only.""" + from jobs import enqueue_research_job + data = request.json or {} + competitors = data.get('competitors', []) + + run = scrape_repository.create({ + 'tenant_id': tenant_id, 'triggered_by': 'cron', 'status': 'pending', + 'competitors_total': len(competitors), 'competitors_done': 0, 'results': {}, + }) + enqueue_research_job(run['id'], {'tenant_id': tenant_id, 'competitors': competitors}) + return jsonify({'job_id': run['id'], 'status': 'queued'}) + + +@api_bp.route('/internal/battlecard/', methods=['POST']) +@require_api_key +def internal_battlecard(competitor_id): + data = request.json or {} + tenant_id = data.get('tenant_id') + if not tenant_id: + return error_response('missing_field', 'tenant_id is required', 400) + card = battlecard_service.generate_battlecard(competitor_id, tenant_id) + return jsonify(card or {'status': 'failed'}) + + +@api_bp.route('/internal/embed-products/', methods=['POST']) +@require_api_key +def internal_embed_products(tenant_id): + """Phase 6 — embeds any of a tenant's products missing an embedding vector.""" + from services import embedding_service + from repositories.product_repository import product_repository as products_repo + + embedded = 0 + for product in products_repo.list_for_tenant(tenant_id): + if product.get('embedding'): + continue + try: + vector = embedding_service.embed_trip(product) + products_repo.update(product['id'], {'embedding': vector}) + embedded += 1 + except Exception as e: + logger.error(f'Embedding failed for product {product["id"]}: {str(e)}') + return jsonify({'embedded': embedded}) + + +@api_bp.route('/internal/match-comparable/', methods=['POST']) +@require_api_key +def internal_match_comparable(tenant_id): + """Phase 6 — runs semantic similarity matching for a tenant's client trips vs competitor products.""" + from services import embedding_service + from repositories.product_repository import product_repository as products_repo + + client_trips = request.json.get('client_trips', []) if request.json else [] + competitor_products = products_repo.list_for_tenant(tenant_id) + matches = embedding_service.find_comparable_trips(client_trips, competitor_products) + + created = [ + comparable_repository.create({'tenant_id': tenant_id, **match}) + for match in matches + ] + return jsonify({'matches_created': len(created)}) + + +@api_bp.route('/internal/weekly-brief/', methods=['POST']) +@require_api_key +def internal_weekly_brief(tenant_id): + brief_service.send_weekly_brief(tenant_id) + return jsonify({'status': 'ok'}) + + +@api_bp.route('/internal/monitor-webhook', methods=['POST']) +@require_api_key +def monitor_webhook(): + """Receives Firecrawl Monitor webhook on competitor page change.""" + from datetime import datetime, timezone + data = request.json or {} + url = data.get('url') + competitor = competitor_repository.get_by_url(url) if url else None + if competitor: + competitor_repository.update(competitor['id'], { + 'change_detected': True, + 'change_detected_at': datetime.now(timezone.utc).isoformat(), + }) + return jsonify({'status': 'ok'}) + + +@api_bp.route('/internal/monitor/register/', methods=['POST']) +@require_api_key +def internal_monitor_register(competitor_id): + competitor = competitor_repository.get_by_id(competitor_id) + if not competitor: + return error_response('not_found', 'Unknown competitor', 404) + _register_monitor(competitor_id, competitor.get('catalogue_url') or competitor.get('website')) + return jsonify({'status': 'ok'}) + + +@api_bp.route('/internal/monitor/deregister/', methods=['POST']) +@require_api_key +def internal_monitor_deregister(competitor_id): + competitor = competitor_repository.get_by_id(competitor_id) + if not competitor: + return error_response('not_found', 'Unknown competitor', 404) + _deregister_monitor(competitor.get('firecrawl_monitor_id')) + return jsonify({'status': 'ok'}) + + +# ── SYSTEM ────────────────────────────────────────────────────────── + +@api_bp.route('/health', methods=['GET']) +def health(): + return jsonify({'status': 'ok', 'version': '0.1.0-phase1'}) + + +@api_bp.errorhandler(429) +def rate_limit_exceeded(e): + return error_response('rate_limit_exceeded', 'Too many requests. Please wait before trying again.', 429) diff --git a/backend/app.py b/backend/app.py new file mode 100644 index 0000000..cbfe90f --- /dev/null +++ b/backend/app.py @@ -0,0 +1,23 @@ +import logging +from flask import Flask +from dotenv import load_dotenv + +load_dotenv() +logging.basicConfig(level=logging.INFO) + +from extensions import limiter # noqa: E402 +from api import api_bp # noqa: E402 + +# Entry point only — imports and wires everything together. Business +# logic lives in services/, data access in repositories/, routes in +# api.py. Nothing else belongs in this file per CLAUDE.md §5/§18. +app = Flask(__name__) +limiter.init_app(app) +app.register_blueprint(api_bp) + +# Sentry initialisation is a Phase 7 ship-gate item (CLAUDE.md §19 step +# 54 — "staging first, then production"). Deliberately not wired here +# yet so earlier phases never silently depend on SENTRY_DSN being set. + +if __name__ == '__main__': + app.run(host='0.0.0.0', port=5000, debug=False) diff --git a/backend/errors.py b/backend/errors.py new file mode 100644 index 0000000..276f6a7 --- /dev/null +++ b/backend/errors.py @@ -0,0 +1,35 @@ +from flask import jsonify + + +def error_response(code, message, status=400): + """ + Returns a structured JSON error response. Every Flask route in this + codebase uses this helper — plain-text or stack-trace error bodies + are a build failure per CLAUDE.md §18. + + Data flow: + code + message + status → JSON body { error: true, code, message } → + (body, status) tuple returned, suitable as a Flask route return value + """ + return jsonify({'error': True, 'code': code, 'message': message}), status + + +# User-facing failure messages for the four JobProgress terminal states +# plus proxy errors — surfaced in scrape_runs.error_log and shown verbatim +# in the dashboard's failed-state UI (CLAUDE.md "Error Handling" section). +FAILURE_MESSAGES = { + 'scrape_blocked': ( + 'The scraper was blocked by this site. ' + 'Try again in a few hours or enter the URL manually.' + ), + 'scrape_timeout': 'The page took too long to load. This may be a temporary issue.', + 'extract_failed': ( + 'The AI could not extract structured data from this page. ' + 'The page structure may have changed.' + ), + 'url_not_found': ( + 'No matching trip page was found for this competitor. ' + 'Try entering the URL manually.' + ), + 'proxy_error': 'Proxy connection failed. Retrying with fallback provider.', +} diff --git a/backend/extensions.py b/backend/extensions.py new file mode 100644 index 0000000..34d5c98 --- /dev/null +++ b/backend/extensions.py @@ -0,0 +1,14 @@ +import os +from flask_limiter import Limiter +from flask_limiter.util import get_remote_address + +# RATELIMIT_STORAGE_URI lets tests point the limiter at an in-process +# memory:// backend instead of the production Redis instance — Flask- +# Limiter reads its storage_uri/enabled flags once at init_app() time, +# so this must be set via env var before app.py imports this module, +# not by mutating app.config afterwards (see tests/conftest.py). +limiter = Limiter( + key_func=get_remote_address, + storage_uri=os.getenv('RATELIMIT_STORAGE_URI', os.getenv('REDIS_URL', 'redis://bettersight-redis:6379')), + default_limits=['200 per hour', '30 per minute'], +) diff --git a/backend/jobs.py b/backend/jobs.py new file mode 100644 index 0000000..4216557 --- /dev/null +++ b/backend/jobs.py @@ -0,0 +1,127 @@ +import os +import time +import logging +from datetime import datetime, timezone + +from redis import Redis +from rq import Queue + +from repositories.scrape_repository import scrape_repository +from repositories.competitor_repository import competitor_repository +from services import analysis_service, alert_service + +logger = logging.getLogger(__name__) + +REDIS_URL = os.getenv('REDIS_URL', 'redis://bettersight-redis:6379') +redis_conn = Redis.from_url(REDIS_URL) + +# Three priority queues — Intelligence tier (not yet built) will use +# `high`; Analyse-tier on-demand and n8n cron jobs use `normal`; +# background/cleanup work uses `low`. Worker command: +# rq worker high normal low -u $REDIS_URL +high_q = Queue('high', connection=redis_conn) +normal_q = Queue('normal', connection=redis_conn) +low_q = Queue('low', connection=redis_conn) + +JOB_SLOW_SECONDS = 300 # 5 minutes — §29 THRESHOLDS['job_slow_mins'] +JOB_STUCK_SECONDS = 600 # 10 minutes — §29 THRESHOLDS['job_stuck_mins'] + + +def enqueue_research_job(job_id, data): + """ + Enqueues a research job to the `normal` priority queue. Never runs + synchronously — every /research-style route calls this instead of + invoking run_research_job() directly. + + Data flow: + job_id (the scrape_runs record id, reused as the RQ job id) + data → + normal_q.enqueue(run_research_job, job_id, data) → + RQ job id returned (== job_id, since we pass it explicitly) + """ + normal_q.enqueue(run_research_job, job_id, data, job_id=job_id, job_timeout=900) + return job_id + + +def run_research_job(job_id, data): + """ + The RQ job function. Loops over every requested competitor, running + the fast/refresh path decision per CLAUDE.md §7. One competitor's + failure is logged and skipped — it never aborts the rest of the + batch (§18 resilience rule). Records duration and fires Gotify + alerts on slow/stuck/failed completion (§29 Component 2). + + Data flow: + job_id + { tenant_id, competitors, destination, duration, + travel_style, tab_type } → + scrape_runs status → running → + per competitor: analysis_service.run_competitor_analysis() → + success: append to results, competitors_done += 1 → + failure: append to results.failed, continue → + scrape_runs status → complete (or failed if every competitor failed) → + duration computed → scrape_runs.duration_seconds updated → + slow/stuck duration → Gotify alert fired + """ + start = time.time() + scrape_repository.update(job_id, {'status': 'running'}) + + tenant_id = data.get('tenant_id') + competitors_input = data.get('competitors', []) + context = { + 'destination': data.get('destination'), + 'duration': data.get('duration'), + 'travel_style': data.get('travel_style'), + 'tab_type': data.get('tab_type'), + 'product_name': data.get('productName') or data.get('product_name'), + } + + success, failed = [], [] + done = 0 + + for entry in competitors_input: + competitor = competitor_repository.get_by_id(entry['id']) if isinstance(entry, dict) else None + if not competitor: + failed.append({'name': entry.get('name', 'Unknown') if isinstance(entry, dict) else str(entry), + 'error': 'Competitor not found'}) + continue + + try: + outcome = analysis_service.run_competitor_analysis(tenant_id, competitor, context) + success.append({'competitor_id': competitor['id'], 'name': competitor['name'], **outcome}) + except Exception as e: + logger.error(f'{competitor.get("name", entry)} failed in job {job_id}: {str(e)}') + failed.append({'competitor_id': competitor['id'], 'name': competitor['name'], 'error': str(e)}) + finally: + done += 1 + scrape_repository.update(job_id, {'competitors_done': done}) + + status = 'complete' if success or not competitors_input else 'failed' + scrape_repository.update(job_id, { + 'status': status, + 'results': {'success': success, 'failed': failed}, + 'error_log': '; '.join(f['error'] for f in failed) if failed else '', + 'completed_at': datetime.now(timezone.utc).isoformat(), + }) + + if status == 'failed': + alert_service.send_gotify( + title='🔴 Analysis job failed', + message=f'Job failed for tenant {tenant_id}.\nErrors: {"; ".join(f["error"] for f in failed)[:200]}', + priority=7 + ) + + duration = round(time.time() - start) + scrape_repository.update(job_id, {'duration_seconds': duration}) + + if duration > JOB_STUCK_SECONDS: + alert_service.send_gotify( + title='🔴 Job likely stuck', + message=f'Job took {round(duration / 60, 1)} mins. Tenant: {tenant_id}. ' + f'Possible proxy or Playwright issue.', + priority=10 + ) + elif duration > JOB_SLOW_SECONDS: + alert_service.send_gotify( + title='🟡 Slow job detected', + message=f'Job took {round(duration / 60, 1)} mins. Tenant: {tenant_id}.', + priority=5 + ) diff --git a/backend/pb_migrations/001_initial_schema.js b/backend/pb_migrations/001_initial_schema.js new file mode 100644 index 0000000..1af2828 --- /dev/null +++ b/backend/pb_migrations/001_initial_schema.js @@ -0,0 +1,287 @@ +/// +// +// Initial PocketBase schema for Bettersight — MVP (Analyse tier) scope only. +// Discover/Intelligence-only collections (signal_events, competitor_people, +// wayback_snapshots, tenant_custom_fields, etc.) are intentionally excluded +// per CLAUDE.md §27 "What NOT to Build" — add them in a later migration +// when those tiers are actually built. +// +// Written against PocketBase's JS migration API (v0.23+). Collections are +// created in dependency order so relation fields can reference the +// already-created target collection's id directly. + +migrate((app) => { + // ── small helpers to keep 15 collection definitions readable ────────── + const text = (name, opts = {}) => ({ name, type: 'text', ...opts }); + const url = (name, opts = {}) => ({ name, type: 'url', ...opts }); + const num = (name, opts = {}) => ({ name, type: 'number', ...opts }); + const boolean = (name, opts = {}) => ({ name, type: 'bool', ...opts }); + const json = (name, opts = {}) => ({ name, type: 'json', ...opts }); + const date = (name, opts = {}) => ({ name, type: 'date', ...opts }); + const autodate = (name, onCreate, onUpdate = false) => ({ + name, type: 'autodate', onCreate, onUpdate, + }); + const select = (name, values, opts = {}) => ({ + name, type: 'select', maxSelect: 1, values, ...opts, + }); + const rel = (name, collection, opts = {}) => ({ + name, type: 'relation', collectionId: collection.id, + cascadeDelete: false, minSelect: 0, maxSelect: 1, ...opts, + }); + + const mk = (name, fields, listRule = null) => { + const collection = new Collection({ + type: 'base', + name, + fields, + // Tenant-scoped data — API access enforced via the Flask backend's + // X-API-Key + tenant_id filtering, not PocketBase's own auth rules. + // Leaving rules null restricts direct API access to admin-only, + // which is correct since only the Flask service account talks to + // PocketBase directly. + listRule, + viewRule: listRule, + createRule: null, + updateRule: null, + deleteRule: null, + }); + return app.save(collection) && collection; + }; + + // ── tenants ───────────────────────────────────────────────────────── + const tenants = mk('tenants', [ + text('name', { required: true }), + text('email', { required: true }), + text('domain', { required: true }), + select('tier', ['analyse', 'discover', 'intelligence', 'enterprise'], { required: true }), + select('status', ['trial', 'active', 'inactive'], { required: true }), + date('trial_ends_at'), + num('max_seats', { required: true }), + text('timezone'), + text('stripe_customer_id'), + text('stripe_subscription_id'), + json('onboarding_checklist'), + text('data_region'), + autodate('created', true), + ]); + + // ── tenant_seats ──────────────────────────────────────────────────── + mk('tenant_seats', [ + rel('tenant_id', tenants, { required: true }), + text('email', { required: true }), + boolean('active'), + date('last_accessed'), + text('added_by'), + autodate('created', true), + ]); + + // ── competitors ───────────────────────────────────────────────────── + const competitors = mk('competitors', [ + rel('tenant_id', tenants, { required: true }), + text('name', { required: true }), + url('website', { required: true }), + url('catalogue_url', { required: true }), + text('primary_market', { required: true }), + text('firecrawl_monitor_id'), + boolean('change_detected'), + date('change_detected_at'), + boolean('active'), + date('last_scraped'), + autodate('created', true), + ]); + + // ── products ──────────────────────────────────────────────────────── + const products = mk('products', [ + rel('tenant_id', tenants, { required: true }), + rel('competitor_id', competitors, { required: true }), + text('trip_name'), + url('url'), + text('destination'), + num('duration_days'), + num('majority_price_usd'), + num('price_low_usd'), + num('price_high_usd'), + num('aud_price'), + num('cad_price'), + num('eur_price'), + num('gbp_price'), + text('date_used'), + text('seasonality'), + num('group_size'), + num('meals'), + text('service_level'), + text('activities'), + text('start_location'), + text('end_location'), + text('target_audience'), + text('hotels'), + text('comments'), + num('relevancy'), + text('trip_code'), + boolean('is_new'), + autodate('first_seen', true), + date('last_seen'), + boolean('is_active'), + json('embedding'), + ]); + + // ── products_ngs_meta ─────────────────────────────────────────────── + mk('products_ngs_meta', [ + rel('product_id', products, { required: true }), + text('exclusive_access'), + text('group_leader'), + text('sustainability'), + ]); + + // ── price_history ─────────────────────────────────────────────────── + mk('price_history', [ + rel('tenant_id', tenants, { required: true }), + rel('product_id', products, { required: true }), + num('majority_price_usd'), + num('price_low_usd'), + num('price_high_usd'), + num('aud_price'), + num('cad_price'), + num('eur_price'), + num('gbp_price'), + text('date_used'), + text('change_field'), + num('change_amount'), + num('change_percent'), + autodate('scraped_at', true), + ]); + + // ── scrape_runs ───────────────────────────────────────────────────── + // Note: duration_seconds is not in CLAUDE.md §6's scrape_runs schema + // block, but §29 Component 2 (job duration alerting) explicitly writes + // scrape_repository.update(job_id, {'duration_seconds': duration}) — + // added here so that hook has somewhere to write. + mk('scrape_runs', [ + rel('tenant_id', tenants, { required: true }), + rel('competitor_id', competitors, { required: false }), + select('triggered_by', ['cron', 'webhook', 'manual', 'sheet'], { required: true }), + select('status', ['pending', 'running', 'complete', 'failed'], { required: true }), + num('competitors_total'), + num('competitors_done'), + json('results'), + text('error_log'), + num('duration_seconds'), + autodate('started_at', true), + date('completed_at'), + ]); + + // ── alerts ────────────────────────────────────────────────────────── + mk('alerts', [ + rel('tenant_id', tenants, { required: true }), + rel('competitor_id', competitors, { required: true }), + rel('product_id', products, { required: false }), + select('alert_type', ['price_change', 'new_product', 'product_removed', 'new_comparable'], { required: true }), + text('message'), + num('change_amount'), + num('change_percent'), + boolean('delivered_dashboard'), + boolean('delivered_email'), + boolean('delivered_slack'), + boolean('delivered_teams'), + boolean('delivered_whatsapp'), + boolean('read'), + autodate('created', true), + ]); + + // ── battlecards ───────────────────────────────────────────────────── + mk('battlecards', [ + rel('tenant_id', tenants, { required: true }), + rel('competitor_id', competitors, { required: true }), + text('content'), + json('content_json'), + autodate('generated_at', true), + ]); + + // ── comparable_matches ────────────────────────────────────────────── + mk('comparable_matches', [ + rel('tenant_id', tenants, { required: true }), + text('client_product'), + rel('competitor_product', products, { required: true }), + num('similarity_score'), + num('price_difference'), + num('duration_difference'), + autodate('first_matched', true), + date('last_matched'), + boolean('dismissed'), + ]); + + // ── tenant_notifications ──────────────────────────────────────────── + // Slack/Teams/WhatsApp fields are schema-present per CLAUDE.md §6 but + // unused in the Analyse-tier MVP — those integrations are Discover + // tier only (§27). email_digest fields are the only ones written to + // until Discover tier work begins. + mk('tenant_notifications', [ + rel('tenant_id', tenants, { required: true }), + text('slack_webhook_url'), + text('teams_webhook_url'), + text('whatsapp_number'), + boolean('email_digest'), + text('email_digest_time'), + autodate('created', true), + ]); + + // ── digest_logs ───────────────────────────────────────────────────── + mk('digest_logs', [ + rel('tenant_id', tenants, { required: true }), + select('digest_type', ['weekly', 'monthly'], { required: true }), + text('pdf_path'), + autodate('sent_at', true), + text('recipient_email'), + select('status', ['sent', 'failed'], { required: true }), + ]); + + // ── proxy_overrides ───────────────────────────────────────────────── + mk('proxy_overrides', [ + rel('tenant_id', tenants, { required: false }), + text('domain', { required: true }), + select('provider', ['webshare', 'brightdata'], { required: true }), + num('blocked_count'), + date('last_blocked'), + autodate('created', true), + ]); + + // ── client_trips ──────────────────────────────────────────────────── + mk('client_trips', [ + rel('tenant_id', tenants, { required: true }), + text('trip_name'), + text('destination'), + num('duration_days'), + num('price_usd'), + text('activities'), + json('embedding'), + autodate('created', true), + ]); + + // ── monitoring_events ─────────────────────────────────────────────── + // Owner-only (Gotify) monitoring history — never tenant-scoped. + mk('monitoring_events', [ + select('event_type', [ + 'queue_depth', 'job_duration', 'api_health', 'worker_health', + 'cron_duration', 'failure_rate', 'fast_path_ratio', 'weekly_summary', + ], { required: true }), + select('severity', ['info', 'advisory', 'warning', 'critical'], { required: true }), + num('value'), + num('threshold'), + text('message'), + boolean('alert_sent'), + autodate('created', true), + ]); + +}, (app) => { + // ── revert: delete in reverse dependency order ───────────────────── + const names = [ + 'monitoring_events', 'client_trips', 'proxy_overrides', 'digest_logs', + 'tenant_notifications', 'comparable_matches', 'battlecards', 'alerts', + 'scrape_runs', 'price_history', 'products_ngs_meta', 'products', + 'competitors', 'tenant_seats', 'tenants', + ]; + for (const name of names) { + const collection = app.findCollectionByNameOrId(name); + if (collection) app.delete(collection); + } +}); diff --git a/backend/repositories/__init__.py b/backend/repositories/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/repositories/alert_repository.py b/backend/repositories/alert_repository.py new file mode 100644 index 0000000..2ed12d8 --- /dev/null +++ b/backend/repositories/alert_repository.py @@ -0,0 +1,53 @@ +import uuid +from repositories.pocketbase_client import pb_client +from repositories.repo_config import USE_MOCK_REPOSITORIES + +COLLECTION = 'alerts' + + +class AlertRepository: + """Data access for the `alerts` collection.""" + + def create(self, data): + return pb_client.create(COLLECTION, data) + + def list_unread(self, tenant_id): + return pb_client.list( + COLLECTION, filter_str=f'tenant_id = "{tenant_id}" && read = false', sort='-created', per_page=200 + ) + + def list_recent(self, tenant_id, limit=50): + return pb_client.list(COLLECTION, filter_str=f'tenant_id = "{tenant_id}"', sort='-created', per_page=limit) + + def mark_read(self, alert_id): + return pb_client.update(COLLECTION, alert_id, {'read': True}) + + +class MockAlertRepository: + """In-memory stand-in for AlertRepository.""" + + def __init__(self): + self._records = {} + + def create(self, data): + record_id = data.get('id') or str(uuid.uuid4()) + record = {'id': record_id, 'read': False, **data} + self._records[record_id] = record + return record + + def list_unread(self, tenant_id): + return [r for r in self._records.values() if r.get('tenant_id') == tenant_id and not r.get('read')] + + def list_recent(self, tenant_id, limit=50): + items = [r for r in self._records.values() if r.get('tenant_id') == tenant_id] + items.sort(key=lambda r: r.get('created') or '', reverse=True) + return items[:limit] + + def mark_read(self, alert_id): + if alert_id not in self._records: + return None + self._records[alert_id]['read'] = True + return self._records[alert_id] + + +alert_repository = MockAlertRepository() if USE_MOCK_REPOSITORIES else AlertRepository() diff --git a/backend/repositories/battlecard_repository.py b/backend/repositories/battlecard_repository.py new file mode 100644 index 0000000..a4d01ee --- /dev/null +++ b/backend/repositories/battlecard_repository.py @@ -0,0 +1,57 @@ +import uuid +from repositories.pocketbase_client import pb_client +from repositories.repo_config import USE_MOCK_REPOSITORIES + +COLLECTION = 'battlecards' + + +class BattlecardRepository: + """Data access for the `battlecards` collection.""" + + def get_for_competitor(self, competitor_id): + return pb_client.get_first(COLLECTION, f'competitor_id = "{competitor_id}"') + + def list_for_tenant(self, tenant_id): + return pb_client.list(COLLECTION, filter_str=f'tenant_id = "{tenant_id}"', sort='-generated_at', per_page=200) + + def upsert(self, competitor_id, data): + """ + Creates a battlecard for competitor_id if none exists, otherwise + overwrites the existing one — there is exactly one current + battlecard per competitor at any time. + + Data flow: + competitor_id + data → existing battlecard lookup → + found: PATCH existing record / not found: CREATE new record → + updated or created record returned + """ + existing = self.get_for_competitor(competitor_id) + if existing: + return pb_client.update(COLLECTION, existing['id'], data) + return pb_client.create(COLLECTION, {'competitor_id': competitor_id, **data}) + + +class MockBattlecardRepository: + """In-memory stand-in for BattlecardRepository.""" + + def __init__(self): + self._records = {} + + def get_for_competitor(self, competitor_id): + return next((r for r in self._records.values() if r.get('competitor_id') == competitor_id), None) + + def list_for_tenant(self, tenant_id): + return [r for r in self._records.values() if r.get('tenant_id') == tenant_id] + + def upsert(self, competitor_id, data): + existing = self.get_for_competitor(competitor_id) + if existing: + self._records[existing['id']] = {**existing, **data} + return self._records[existing['id']] + record_id = str(uuid.uuid4()) + record = {'id': record_id, 'competitor_id': competitor_id, **data} + self._records[record_id] = record + return record + + +battlecard_repository = MockBattlecardRepository() if USE_MOCK_REPOSITORIES else BattlecardRepository() diff --git a/backend/repositories/brief_repository.py b/backend/repositories/brief_repository.py new file mode 100644 index 0000000..f29225f --- /dev/null +++ b/backend/repositories/brief_repository.py @@ -0,0 +1,98 @@ +import uuid +from datetime import datetime, timedelta, timezone +from repositories.pocketbase_client import pb_client +from repositories.repo_config import USE_MOCK_REPOSITORIES + +PRICE_HISTORY = 'price_history' +PRODUCTS = 'products' +BATTLECARDS = 'battlecards' +TENANTS = 'tenants' +DIGEST_LOGS = 'digest_logs' + + +class BriefRepository: + """ + Read-only aggregation across price_history/products/battlecards/tenants + for the weekly brief email, plus the digest_logs write. This is data + retrieval (multi-collection reads), not decision-making, so it stays + in the repository layer per brief_service's call pattern in CLAUDE.md §13. + """ + + def get_weekly_data(self, tenant_id): + """ + Pulls everything brief_service needs to render brief.html for one + tenant's past 7 days. + + Data flow: + tenant_id → tenant record (admin_email) → + price_history since 7 days ago → products where is_new since 7 days ago → + battlecards updated since 7 days ago → + { admin_email, week_of, price_changes, new_products, battlecard_updates } + """ + tenant = pb_client.get_one(TENANTS, tenant_id) + since = (datetime.now(timezone.utc) - timedelta(days=7)).isoformat() + + price_changes = pb_client.list( + PRICE_HISTORY, filter_str=f'tenant_id = "{tenant_id}" && scraped_at >= "{since}"', + sort='-scraped_at', per_page=200 + ) + new_products = pb_client.list( + PRODUCTS, filter_str=f'tenant_id = "{tenant_id}" && is_new = true && first_seen >= "{since}"', + sort='-first_seen', per_page=200 + ) + battlecard_updates = pb_client.list( + BATTLECARDS, filter_str=f'tenant_id = "{tenant_id}" && generated_at >= "{since}"', + sort='-generated_at', per_page=200 + ) + + return { + 'admin_email': tenant.get('email') if tenant else None, + 'tenant_name': tenant.get('name') if tenant else None, + 'week_of': datetime.now(timezone.utc).strftime('%Y-%m-%d'), + 'price_changes': price_changes, + 'new_products': new_products, + 'battlecard_updates': battlecard_updates, + } + + def log_digest(self, tenant_id, digest_type, recipient_email=None, status='sent'): + """Records that a brief/digest was sent (or failed to send).""" + return pb_client.create(DIGEST_LOGS, { + 'tenant_id': tenant_id, + 'digest_type': digest_type, + 'recipient_email': recipient_email, + 'status': status, + }) + + +class MockBriefRepository: + """In-memory stand-in for BriefRepository — returns empty/zeroed aggregates.""" + + def __init__(self): + self._tenants = {} + self._logs = {} + + def seed_tenant(self, tenant_id, tenant): + self._tenants[tenant_id] = tenant + + def get_weekly_data(self, tenant_id): + tenant = self._tenants.get(tenant_id, {}) + return { + 'admin_email': tenant.get('email'), + 'tenant_name': tenant.get('name'), + 'week_of': datetime.now(timezone.utc).strftime('%Y-%m-%d'), + 'price_changes': [], + 'new_products': [], + 'battlecard_updates': [], + } + + def log_digest(self, tenant_id, digest_type, recipient_email=None, status='sent'): + record_id = str(uuid.uuid4()) + record = { + 'id': record_id, 'tenant_id': tenant_id, 'digest_type': digest_type, + 'recipient_email': recipient_email, 'status': status, + } + self._logs[record_id] = record + return record + + +brief_repository = MockBriefRepository() if USE_MOCK_REPOSITORIES else BriefRepository() diff --git a/backend/repositories/comparable_repository.py b/backend/repositories/comparable_repository.py new file mode 100644 index 0000000..d37a918 --- /dev/null +++ b/backend/repositories/comparable_repository.py @@ -0,0 +1,51 @@ +import uuid +from repositories.pocketbase_client import pb_client +from repositories.repo_config import USE_MOCK_REPOSITORIES + +COLLECTION = 'comparable_matches' + + +class ComparableRepository: + """Data access for the `comparable_matches` collection (Phase 6).""" + + def create(self, data): + return pb_client.create(COLLECTION, data) + + def list_for_tenant(self, tenant_id, include_dismissed=False): + filter_str = f'tenant_id = "{tenant_id}"' + if not include_dismissed: + filter_str += ' && dismissed = false' + return pb_client.list(COLLECTION, filter_str=filter_str, sort='-similarity_score', per_page=200) + + def dismiss(self, match_id): + return pb_client.update(COLLECTION, match_id, {'dismissed': True}) + + +class MockComparableRepository: + """In-memory stand-in for ComparableRepository.""" + + def __init__(self): + self._records = {} + + def create(self, data): + record_id = data.get('id') or str(uuid.uuid4()) + record = {'id': record_id, 'dismissed': False, **data} + self._records[record_id] = record + return record + + def list_for_tenant(self, tenant_id, include_dismissed=False): + items = [ + r for r in self._records.values() + if r.get('tenant_id') == tenant_id and (include_dismissed or not r.get('dismissed')) + ] + items.sort(key=lambda r: r.get('similarity_score') or 0, reverse=True) + return items + + def dismiss(self, match_id): + if match_id not in self._records: + return None + self._records[match_id]['dismissed'] = True + return self._records[match_id] + + +comparable_repository = MockComparableRepository() if USE_MOCK_REPOSITORIES else ComparableRepository() diff --git a/backend/repositories/competitor_repository.py b/backend/repositories/competitor_repository.py new file mode 100644 index 0000000..a3be375 --- /dev/null +++ b/backend/repositories/competitor_repository.py @@ -0,0 +1,85 @@ +import uuid +from urllib.parse import urlparse +from repositories.pocketbase_client import pb_client +from repositories.repo_config import USE_MOCK_REPOSITORIES + +COLLECTION = 'competitors' + + +def _extract_domain(url): + """Strips scheme/path/www from a URL down to a bare domain for dedup comparisons.""" + netloc = urlparse(url if '://' in url else f'//{url}').netloc or url + return netloc.lower().lstrip('www.') + + +class CompetitorRepository: + """Data access for the `competitors` collection.""" + + def get_by_url(self, url): + """Used by the Firecrawl Monitor webhook handler to resolve a changed URL back to a competitor.""" + safe_url = url.replace('"', '\\"') + return pb_client.get_first(COLLECTION, f'website = "{safe_url}" || catalogue_url = "{safe_url}"') + + def get_by_domain(self, tenant_id, domain): + """Used by CSV bulk import to skip duplicate competitors within a tenant.""" + candidates = pb_client.list(COLLECTION, filter_str=f'tenant_id = "{tenant_id}"', per_page=200) + return next((c for c in candidates if _extract_domain(c.get('website', '')) == domain), None) + + def get_by_id(self, competitor_id): + return pb_client.get_one(COLLECTION, competitor_id) + + def list_for_tenant(self, tenant_id, active_only=True): + """Returns a tenant's competitors, active-only by default.""" + filter_str = f'tenant_id = "{tenant_id}"' + if active_only: + filter_str += ' && active = true' + return pb_client.list(COLLECTION, filter_str=filter_str, per_page=200) + + def create(self, data): + return pb_client.create(COLLECTION, data) + + def update(self, competitor_id, data): + return pb_client.update(COLLECTION, competitor_id, data) + + +class MockCompetitorRepository: + """In-memory stand-in for CompetitorRepository.""" + + def __init__(self): + self._records = {} + + def get_by_url(self, url): + return next(( + r for r in self._records.values() + if r.get('website') == url or r.get('catalogue_url') == url + ), None) + + def get_by_domain(self, tenant_id, domain): + return next(( + r for r in self._records.values() + if r.get('tenant_id') == tenant_id and _extract_domain(r.get('website', '')) == domain + ), None) + + def get_by_id(self, competitor_id): + return self._records.get(competitor_id) + + def list_for_tenant(self, tenant_id, active_only=True): + return [ + r for r in self._records.values() + if r.get('tenant_id') == tenant_id and (not active_only or r.get('active')) + ] + + def create(self, data): + record_id = data.get('id') or str(uuid.uuid4()) + record = {'id': record_id, 'active': True, 'change_detected': False, **data} + self._records[record_id] = record + return record + + def update(self, competitor_id, data): + if competitor_id not in self._records: + return None + self._records[competitor_id] = {**self._records[competitor_id], **data} + return self._records[competitor_id] + + +competitor_repository = MockCompetitorRepository() if USE_MOCK_REPOSITORIES else CompetitorRepository() diff --git a/backend/repositories/monitoring_repository.py b/backend/repositories/monitoring_repository.py new file mode 100644 index 0000000..d487f85 --- /dev/null +++ b/backend/repositories/monitoring_repository.py @@ -0,0 +1,38 @@ +import uuid +from repositories.pocketbase_client import pb_client +from repositories.repo_config import USE_MOCK_REPOSITORIES + +COLLECTION = 'monitoring_events' + + +class MonitoringRepository: + """Data access for the `monitoring_events` collection — owner-only, never tenant-scoped.""" + + def log_event(self, event_type, severity, value=None, threshold=None, message='', alert_sent=False): + return pb_client.create(COLLECTION, { + 'event_type': event_type, + 'severity': severity, + 'value': value, + 'threshold': threshold, + 'message': message, + 'alert_sent': alert_sent, + }) + + +class MockMonitoringRepository: + """In-memory stand-in for MonitoringRepository.""" + + def __init__(self): + self._records = {} + + def log_event(self, event_type, severity, value=None, threshold=None, message='', alert_sent=False): + record_id = str(uuid.uuid4()) + record = { + 'id': record_id, 'event_type': event_type, 'severity': severity, + 'value': value, 'threshold': threshold, 'message': message, 'alert_sent': alert_sent, + } + self._records[record_id] = record + return record + + +monitoring_repository = MockMonitoringRepository() if USE_MOCK_REPOSITORIES else MonitoringRepository() diff --git a/backend/repositories/pocketbase_client.py b/backend/repositories/pocketbase_client.py new file mode 100644 index 0000000..90da6ca --- /dev/null +++ b/backend/repositories/pocketbase_client.py @@ -0,0 +1,172 @@ +import os +import time +import logging +import requests + +logger = logging.getLogger(__name__) + +POCKETBASE_URL = os.getenv('POCKETBASE_URL', 'http://pocketbase:8090') +ADMIN_EMAIL = os.getenv('POCKETBASE_ADMIN_EMAIL') +ADMIN_PASSWORD = os.getenv('POCKETBASE_ADMIN_PASSWORD') + + +class PocketBaseError(Exception): + """Raised when a PocketBase REST call fails after the request completes.""" + pass + + +class PocketBaseClient: + """ + Thin REST wrapper around a self-hosted PocketBase instance. + + This is the ONLY place in the codebase that knows PocketBase's HTTP + API shape (auth endpoint, record CRUD paths, filter syntax). Every + repository in repositories/ uses this client instead of calling + `requests` directly, so a PocketBase version/API change is a + one-file fix. + + Data flow: + collection + operation + params → + ensure valid superuser token (auto re-auth on expiry) → + HTTP call to PocketBase REST API → + JSON response → plain dict/list returned to the calling repository + """ + + def __init__(self): + self._token = None + self._token_expires_at = 0 + + def _authenticate(self): + """ + Authenticates as a PocketBase superuser and caches the token. + + Data flow: + POCKETBASE_ADMIN_EMAIL + PASSWORD → + POST /api/collections/_superusers/auth-with-password → + token cached in memory with a conservative TTL + """ + response = requests.post( + f'{POCKETBASE_URL}/api/collections/_superusers/auth-with-password', + json={'identity': ADMIN_EMAIL, 'password': ADMIN_PASSWORD}, + timeout=10 + ) + if response.status_code != 200: + raise PocketBaseError( + f'PocketBase admin auth failed: {response.status_code} {response.text[:200]}' + ) + data = response.json() + self._token = data['token'] + # PocketBase admin tokens are long-lived (default 7 days) — refresh + # an hour early so we never get caught by an in-flight 401. + self._token_expires_at = time.time() + (6 * 24 * 3600) + + def _headers(self): + # Re-authenticate lazily — only when there is no cached token or + # it is past our conservative expiry estimate. + if not self._token or time.time() >= self._token_expires_at: + self._authenticate() + return {'Authorization': self._token} + + def _request(self, method, path, **kwargs): + """ + Issues one authenticated request, retrying once on a 401 in case + the cached token was revoked or expired earlier than estimated. + + Data flow: + method + path + kwargs → authenticated request → + on 401: force re-auth → retry once → + non-2xx after retry: raise PocketBaseError → + 2xx: return parsed JSON (or None for 204/empty bodies) + """ + url = f'{POCKETBASE_URL}{path}' + response = requests.request(method, url, headers=self._headers(), timeout=15, **kwargs) + + if response.status_code == 401: + self._token = None + response = requests.request(method, url, headers=self._headers(), timeout=15, **kwargs) + + if response.status_code >= 400: + raise PocketBaseError( + f'{method} {path} failed: {response.status_code} {response.text[:300]}' + ) + + if not response.content: + return None + return response.json() + + # ── CRUD ──────────────────────────────────────────────────────── + + def list(self, collection, filter_str=None, sort=None, page=1, per_page=50): + """ + Lists records in a collection, optionally filtered/sorted. + + Data flow: + collection + filter/sort/pagination → + GET /api/collections/{collection}/records → + { items, page, totalItems, ... } → items list returned + """ + params = {'page': page, 'perPage': per_page} + if filter_str: + params['filter'] = filter_str + if sort: + params['sort'] = sort + data = self._request('GET', f'/api/collections/{collection}/records', params=params) + return data.get('items', []) if data else [] + + def get_first(self, collection, filter_str): + """Returns the first record matching filter_str, or None if no match.""" + items = self.list(collection, filter_str=filter_str, per_page=1) + return items[0] if items else None + + def get_one(self, collection, record_id): + """Returns a single record by id, or None if it does not exist.""" + try: + return self._request('GET', f'/api/collections/{collection}/records/{record_id}') + except PocketBaseError as e: + if '404' in str(e): + return None + raise + + def create(self, collection, data): + """Creates a record and returns the created record (including its id).""" + return self._request('POST', f'/api/collections/{collection}/records', json=data) + + def update(self, collection, record_id, data): + """Patches a record and returns the updated record.""" + return self._request('PATCH', f'/api/collections/{collection}/records/{record_id}', json=data) + + def delete(self, collection, record_id): + """Deletes a record. Returns True on success.""" + self._request('DELETE', f'/api/collections/{collection}/records/{record_id}') + return True + + def auth_refresh(self, user_token): + """ + Validates a frontend-issued PocketBase user JWT and returns the + associated auth record (email, id, etc.) — used by auth_service + to resolve `tenant_id` from a request's Authorization header + without the backend needing its own JWT-verification secret. + + Data flow: + user_token (Bearer JWT from the Vue dashboard) → + POST /api/collections/users/auth-refresh → + { record: { email, id, ... } } → record dict returned, + or None if the token is invalid/expired + """ + try: + response = requests.post( + f'{POCKETBASE_URL}/api/collections/users/auth-refresh', + headers={'Authorization': f'Bearer {user_token}'}, + timeout=10 + ) + except requests.RequestException as e: + logger.error(f'PocketBase auth-refresh request failed: {str(e)}') + return None + if response.status_code != 200: + return None + return response.json().get('record') + + +# Module-level singleton — every repository imports this same instance so +# the auth token is cached once per process, not once per repository. +pb_client = PocketBaseClient() diff --git a/backend/repositories/product_repository.py b/backend/repositories/product_repository.py new file mode 100644 index 0000000..dea6770 --- /dev/null +++ b/backend/repositories/product_repository.py @@ -0,0 +1,58 @@ +import uuid +from repositories.pocketbase_client import pb_client +from repositories.repo_config import USE_MOCK_REPOSITORIES + +COLLECTION = 'products' + + +class ProductRepository: + """Data access for the `products` collection.""" + + def get_latest_for_competitor(self, competitor_id, limit=10): + """Returns a competitor's most recently scraped products, newest first.""" + return pb_client.list( + COLLECTION, + filter_str=f'competitor_id = "{competitor_id}" && is_active = true', + sort='-last_seen', + per_page=limit + ) + + def list_for_tenant(self, tenant_id): + """Returns all active products for a tenant — used for embedding/comparable matching.""" + return pb_client.list(COLLECTION, filter_str=f'tenant_id = "{tenant_id}" && is_active = true', per_page=500) + + def create(self, data): + return pb_client.create(COLLECTION, data) + + def update(self, product_id, data): + return pb_client.update(COLLECTION, product_id, data) + + +class MockProductRepository: + """In-memory stand-in for ProductRepository.""" + + def __init__(self): + self._records = {} + + def get_latest_for_competitor(self, competitor_id, limit=10): + items = [r for r in self._records.values() if r.get('competitor_id') == competitor_id and r.get('is_active', True)] + items.sort(key=lambda r: r.get('last_seen') or '', reverse=True) + return items[:limit] + + def list_for_tenant(self, tenant_id): + return [r for r in self._records.values() if r.get('tenant_id') == tenant_id and r.get('is_active', True)] + + def create(self, data): + record_id = data.get('id') or str(uuid.uuid4()) + record = {'id': record_id, 'is_active': True, **data} + self._records[record_id] = record + return record + + def update(self, product_id, data): + if product_id not in self._records: + return None + self._records[product_id] = {**self._records[product_id], **data} + return self._records[product_id] + + +product_repository = MockProductRepository() if USE_MOCK_REPOSITORIES else ProductRepository() diff --git a/backend/repositories/proxy_repository.py b/backend/repositories/proxy_repository.py new file mode 100644 index 0000000..7c8b39a --- /dev/null +++ b/backend/repositories/proxy_repository.py @@ -0,0 +1,44 @@ +import uuid +from repositories.pocketbase_client import pb_client +from repositories.repo_config import USE_MOCK_REPOSITORIES + +COLLECTION = 'proxy_overrides' + + +class ProxyRepository: + """Data access for the `proxy_overrides` collection — used by core/proxy_service.py.""" + + def get_by_domain(self, domain): + safe_domain = domain.replace('"', '\\"') + return pb_client.get_first(COLLECTION, f'domain = "{safe_domain}"') + + def create(self, data): + return pb_client.create(COLLECTION, data) + + def update(self, override_id, data): + return pb_client.update(COLLECTION, override_id, data) + + +class MockProxyRepository: + """In-memory stand-in for ProxyRepository.""" + + def __init__(self): + self._records = {} + + def get_by_domain(self, domain): + return next((r for r in self._records.values() if r.get('domain') == domain), None) + + def create(self, data): + record_id = data.get('id') or str(uuid.uuid4()) + record = {'id': record_id, **data} + self._records[record_id] = record + return record + + def update(self, override_id, data): + if override_id not in self._records: + return None + self._records[override_id] = {**self._records[override_id], **data} + return self._records[override_id] + + +proxy_repository = MockProxyRepository() if USE_MOCK_REPOSITORIES else ProxyRepository() diff --git a/backend/repositories/repo_config.py b/backend/repositories/repo_config.py new file mode 100644 index 0000000..47fba0d --- /dev/null +++ b/backend/repositories/repo_config.py @@ -0,0 +1,7 @@ +import os + +# When true, every repository module exposes its in-memory Mock* +# implementation instead of the real PocketBase-backed one. Set in the +# test environment (see tests/conftest.py) so the full service layer can +# be exercised without a running PocketBase instance. +USE_MOCK_REPOSITORIES = os.getenv('USE_MOCK_REPOSITORIES', 'false').lower() == 'true' diff --git a/backend/repositories/scrape_repository.py b/backend/repositories/scrape_repository.py new file mode 100644 index 0000000..f049b8f --- /dev/null +++ b/backend/repositories/scrape_repository.py @@ -0,0 +1,54 @@ +import uuid +from repositories.pocketbase_client import pb_client +from repositories.repo_config import USE_MOCK_REPOSITORIES + +COLLECTION = 'scrape_runs' + + +class ScrapeRepository: + """Data access for the `scrape_runs` collection — also the RQ job_id record.""" + + def create(self, data): + return pb_client.create(COLLECTION, data) + + def update(self, job_id, data): + return pb_client.update(COLLECTION, job_id, data) + + def get_by_id(self, job_id): + return pb_client.get_one(COLLECTION, job_id) + + def list_recent_for_tenant(self, tenant_id, limit=5): + """Returns a tenant's most recent runs, newest first — backs GET /research/history.""" + return pb_client.list( + COLLECTION, filter_str=f'tenant_id = "{tenant_id}"', sort='-started_at', per_page=limit + ) + + +class MockScrapeRepository: + """In-memory stand-in for ScrapeRepository.""" + + def __init__(self): + self._records = {} + + def create(self, data): + record_id = data.get('id') or str(uuid.uuid4()) + record = {'id': record_id, **data} + self._records[record_id] = record + return record + + def update(self, job_id, data): + if job_id not in self._records: + return None + self._records[job_id] = {**self._records[job_id], **data} + return self._records[job_id] + + def get_by_id(self, job_id): + return self._records.get(job_id) + + def list_recent_for_tenant(self, tenant_id, limit=5): + items = [r for r in self._records.values() if r.get('tenant_id') == tenant_id] + items.sort(key=lambda r: r.get('started_at') or '', reverse=True) + return items[:limit] + + +scrape_repository = MockScrapeRepository() if USE_MOCK_REPOSITORIES else ScrapeRepository() diff --git a/backend/repositories/seat_repository.py b/backend/repositories/seat_repository.py new file mode 100644 index 0000000..f5e8958 --- /dev/null +++ b/backend/repositories/seat_repository.py @@ -0,0 +1,86 @@ +import uuid +from datetime import datetime, timezone +from repositories.pocketbase_client import pb_client +from repositories.repo_config import USE_MOCK_REPOSITORIES + +COLLECTION = 'tenant_seats' + + +class SeatRepository: + """ + Data access for the `tenant_seats` collection. Pure CRUD only — seat + limit enforcement and allow/reject decisions live in licence_service, + not here, per the no-business-logic-in-repositories rule. + """ + + def get_active(self, tenant_id, email): + """Returns the active seat record for (tenant_id, email), or None.""" + safe_email = email.replace('"', '\\"') + return pb_client.get_first( + COLLECTION, + f'tenant_id = "{tenant_id}" && email = "{safe_email}" && active = true' + ) + + def count_active(self, tenant_id): + """Returns the count of active seats for a tenant.""" + return len(pb_client.list( + COLLECTION, filter_str=f'tenant_id = "{tenant_id}" && active = true', per_page=200 + )) + + def list_for_tenant(self, tenant_id): + """Returns all seats (active and inactive) for a tenant.""" + return pb_client.list(COLLECTION, filter_str=f'tenant_id = "{tenant_id}"', per_page=200) + + def create(self, data): + """Creates a seat record and returns it.""" + return pb_client.create(COLLECTION, data) + + def update_last_accessed(self, seat_id): + """Stamps last_accessed = now on an existing seat record.""" + return pb_client.update(COLLECTION, seat_id, { + 'last_accessed': datetime.now(timezone.utc).isoformat() + }) + + def delete(self, seat_id): + """Deletes a seat record. Returns True on success.""" + return pb_client.delete(COLLECTION, seat_id) + + +class MockSeatRepository: + """In-memory stand-in for SeatRepository.""" + + def __init__(self): + self._records = {} + + def get_active(self, tenant_id, email): + return next(( + r for r in self._records.values() + if r.get('tenant_id') == tenant_id and r.get('email') == email and r.get('active') + ), None) + + def count_active(self, tenant_id): + return len([ + r for r in self._records.values() + if r.get('tenant_id') == tenant_id and r.get('active') + ]) + + def list_for_tenant(self, tenant_id): + return [r for r in self._records.values() if r.get('tenant_id') == tenant_id] + + def create(self, data): + record_id = data.get('id') or str(uuid.uuid4()) + record = {'id': record_id, 'active': True, **data} + self._records[record_id] = record + return record + + def update_last_accessed(self, seat_id): + if seat_id not in self._records: + return None + self._records[seat_id]['last_accessed'] = datetime.now(timezone.utc).isoformat() + return self._records[seat_id] + + def delete(self, seat_id): + return self._records.pop(seat_id, None) is not None + + +seat_repository = MockSeatRepository() if USE_MOCK_REPOSITORIES else SeatRepository() diff --git a/backend/repositories/tenant_repository.py b/backend/repositories/tenant_repository.py new file mode 100644 index 0000000..8f63a6e --- /dev/null +++ b/backend/repositories/tenant_repository.py @@ -0,0 +1,71 @@ +import uuid +from repositories.pocketbase_client import pb_client +from repositories.repo_config import USE_MOCK_REPOSITORIES + +COLLECTION = 'tenants' + + +class TenantRepository: + """ + Data access for the `tenants` collection. No business logic — callers + (services) decide what a missing tenant or inactive status means. + """ + + def get_by_domain(self, domain): + """ + Returns the tenant record matching domain, or None. + + Data flow: + domain → PocketBase filter `domain = "{domain}"` → + first matching record returned, or None + """ + safe_domain = domain.replace('"', '\\"') + return pb_client.get_first(COLLECTION, f'domain = "{safe_domain}"') + + def get_by_id(self, tenant_id): + """Returns a tenant record by id, or None.""" + return pb_client.get_one(COLLECTION, tenant_id) + + def get_by_stripe_customer_id(self, stripe_customer_id): + """Returns the tenant matching a Stripe customer id — used by the Stripe webhook handler.""" + safe_id = stripe_customer_id.replace('"', '\\"') + return pb_client.get_first(COLLECTION, f'stripe_customer_id = "{safe_id}"') + + def create(self, data): + """Creates a tenant record and returns it.""" + return pb_client.create(COLLECTION, data) + + def update(self, tenant_id, data): + """Patches a tenant record and returns the updated record.""" + return pb_client.update(COLLECTION, tenant_id, data) + + +class MockTenantRepository: + """In-memory stand-in for TenantRepository — same interface, no PocketBase dependency.""" + + def __init__(self): + self._records = {} + + def get_by_domain(self, domain): + return next((r for r in self._records.values() if r.get('domain') == domain), None) + + def get_by_id(self, tenant_id): + return self._records.get(tenant_id) + + def get_by_stripe_customer_id(self, stripe_customer_id): + return next((r for r in self._records.values() if r.get('stripe_customer_id') == stripe_customer_id), None) + + def create(self, data): + record_id = data.get('id') or str(uuid.uuid4()) + record = {'id': record_id, **data} + self._records[record_id] = record + return record + + def update(self, tenant_id, data): + if tenant_id not in self._records: + return None + self._records[tenant_id] = {**self._records[tenant_id], **data} + return self._records[tenant_id] + + +tenant_repository = MockTenantRepository() if USE_MOCK_REPOSITORIES else TenantRepository() diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 0000000..bad405c --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,39 @@ +# ── Phase 1 — core API / queue / data layer ────────────────── +Flask==3.0.3 +Jinja2==3.1.4 +gunicorn==22.0.0 +python-dotenv==1.0.1 +requests==2.32.3 +redis==5.0.7 +rq==1.16.2 +Flask-Limiter==3.5.0 +PyJWT==2.9.0 + +# ── Billing / email ─────────────────────────────────────────── +stripe==10.7.0 +resend==2.4.0 + +# ── Phase 2 — scraping engine (moved from legacy app.py) ───── +playwright==1.46.0 +playwright-stealth==1.0.6 +beautifulsoup4==4.12.3 +lxml==5.3.0 +nest_asyncio==1.6.0 +openai==1.43.0 +gspread==6.1.2 +google-auth==2.34.0 + +# ── Phase 6 — embeddings / similarity ───────────────────────── +# Unpinned to a minimum rather than exact — prebuilt wheels for very +# new Python versions lag behind numpy point releases; pip will pick +# the newest compatible wheel. +numpy>=2.2.0 + +# ── Observability (initialised in Phase 7, dependency pinned now) ─ +sentry-sdk[flask]==1.44.1 + +# ── Testing ──────────────────────────────────────────────────── +pytest==8.3.2 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +fakeredis==2.24.1 diff --git a/backend/services/__init__.py b/backend/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/services/alert_service.py b/backend/services/alert_service.py new file mode 100644 index 0000000..ecfdad8 --- /dev/null +++ b/backend/services/alert_service.py @@ -0,0 +1,73 @@ +import os +import logging +import requests +from repositories.monitoring_repository import monitoring_repository +from repositories.alert_repository import alert_repository + +logger = logging.getLogger(__name__) + +GOTIFY_URL = os.getenv('GOTIFY_URL') +GOTIFY_APP_TOKEN = os.getenv('GOTIFY_APP_TOKEN') + + +def send_gotify(title, message, priority=5): + """ + Sends a push notification via Gotify to the owner's phone. Logs the + alert attempt to monitoring_events regardless of delivery success. + Never shown to clients — owner monitoring only, per CLAUDE.md §3/§27. + + Data flow: + title + message + priority → + HTTP POST {GOTIFY_URL}/message → + monitoring_events record created (alert_sent reflects HTTP outcome) + """ + delivered = False + try: + response = requests.post( + f'{GOTIFY_URL}/message', + headers={'X-Gotify-Key': GOTIFY_APP_TOKEN}, + json={'title': title, 'message': message, 'priority': priority}, + timeout=5 + ) + delivered = response.status_code < 300 + except Exception as e: + # Side-effect failure — must never propagate into the caller's + # primary operation (a job completing, a threshold check, etc.). + logger.error(f'Gotify alert failed: {str(e)}') + + monitoring_repository.log_event( + event_type='api_health' if priority >= 10 else 'failure_rate', + severity='critical' if priority >= 10 else ('warning' if priority >= 7 else 'advisory'), + message=f'{title}: {message}', + alert_sent=delivered, + ) + + +def create_alert(tenant_id, competitor_id, alert_type, message, product_id=None, + change_amount=None, change_percent=None): + """ + Creates a tenant-facing alert record — always visible on the + dashboard activity feed immediately (delivered_dashboard is always + true). Email/Slack/Teams/WhatsApp delivery flags are set later by + the digest workflows that actually deliver through those channels. + + Data flow: + alert fields → alerts table record created with + delivered_dashboard=true, all other delivered_* flags false → + created alert record returned + """ + return alert_repository.create({ + 'tenant_id': tenant_id, + 'competitor_id': competitor_id, + 'product_id': product_id, + 'alert_type': alert_type, + 'message': message, + 'change_amount': change_amount, + 'change_percent': change_percent, + 'delivered_dashboard': True, + 'delivered_email': False, + 'delivered_slack': False, + 'delivered_teams': False, + 'delivered_whatsapp': False, + 'read': False, + }) diff --git a/backend/services/analysis_service.py b/backend/services/analysis_service.py new file mode 100644 index 0000000..289f3b1 --- /dev/null +++ b/backend/services/analysis_service.py @@ -0,0 +1,127 @@ +import logging +from datetime import datetime, timedelta, timezone +from repositories.competitor_repository import competitor_repository +from repositories.product_repository import product_repository + +logger = logging.getLogger(__name__) + +# Default staleness threshold — configurable here only, per CLAUDE.md §7. +CACHE_STALENESS_HOURS = 24 + + +def needs_refresh(competitor): + """ + Decides whether a competitor needs a live re-scrape (REFRESH path) + or can be served from PocketBase (FAST path). + + Data flow: + competitor record → + change_detected flag set? → True (refresh) → + last_scraped missing or older than CACHE_STALENESS_HOURS? → True (refresh) → + otherwise → False (fast path is safe) + """ + if competitor.get('change_detected'): + return True + + last_scraped = competitor.get('last_scraped') + if not last_scraped: + return True + + try: + scraped_at = datetime.fromisoformat(last_scraped.replace('Z', '+00:00')) + except (ValueError, AttributeError): + return True + + age = datetime.now(timezone.utc) - scraped_at + return age > timedelta(hours=CACHE_STALENESS_HOURS) + + +def analyse_from_cache(tenant_id, competitor_id, context): + """ + FAST path analysis — uses cached PocketBase data instead of a live + scrape. The LLM receives already-structured JSON and generates the + analysis narrative, comments, and relevancy score only — it does + NOT re-extract fields from raw page content. + + Data flow: + competitor_id → PocketBase products (latest) → + structured JSON + client trip context (destination/duration/style) → + LLM prompt (analysis only, no extraction, via core.extractor) → + narrative + comments + relevancy score returned + """ + products = product_repository.get_latest_for_competitor(competitor_id, limit=1) + if not products: + raise ValueError('No cached product data available for fast path') + + latest = products[0] + + # core.extractor is a Phase 2 module — imported lazily so this + # service is importable before the scraping engine refactor lands. + from core.extractor import extract_with_openrouter + + prompt = ( + 'You are a travel industry analyst. Given this already-extracted ' + 'competitor trip data, write a short comparative analysis against ' + f"the client's product context: {context}.\n\nCompetitor data:\n{latest}" + ) + return extract_with_openrouter(prompt, content='') + + +def run_competitor_analysis(tenant_id, competitor, context): + """ + Decides the fast vs refresh path for one competitor and executes it. + Called per-competitor by jobs.run_research_job — failures here are + caught by the caller so one competitor's failure never aborts the + whole batch (CLAUDE.md §18 resilience rule). + + Data flow: + competitor + context → needs_refresh() decision → + FAST: analyse_from_cache() (seconds) → + REFRESH: core.scraper.scrape() → core.extractor.extract_with_openrouter() + → PocketBase products/price_history updated → + competitor.change_detected reset, last_scraped bumped → + analysis narrative generated from the fresh data → + result dict returned + """ + if not needs_refresh(competitor): + logger.info(f'{competitor["name"]} — fast path (cache fresh)') + return {'path': 'fast', 'result': analyse_from_cache(tenant_id, competitor['id'], context)} + + logger.info(f'{competitor["name"]} — refresh path (stale or changed)') + # Phase 2 modules — imported lazily, same reasoning as above. + from core.scraper import scrape + from core.extractor import extract_with_openrouter + from industries.adventure_travel.prompt import build_prompt + + page_data = scrape(competitor['website'], country=competitor.get('primary_market')) + if not page_data.get('success'): + raise ValueError(page_data.get('text', 'Scrape failed')) + + prompt = build_prompt( + competitor['name'], page_data, + context.get('product_name'), context.get('destination'), + context.get('duration'), context.get('travel_style'), + is_ngs=context.get('tab_type') == 'NGS' + ) + extracted = extract_with_openrouter(prompt, content=page_data['text']) + + product = product_repository.create({ + 'tenant_id': tenant_id, + 'competitor_id': competitor['id'], + 'trip_name': extracted.get('tripName'), + 'url': page_data['url'], + 'duration_days': extracted.get('duration'), + 'majority_price_usd': extracted.get('majorityPrice'), + 'price_low_usd': extracted.get('priceLow'), + 'price_high_usd': extracted.get('priceHigh'), + 'comments': extracted.get('comments'), + 'relevancy': extracted.get('relevancy'), + 'is_active': True, + }) + + competitor_repository.update(competitor['id'], { + 'change_detected': False, + 'last_scraped': datetime.now(timezone.utc).isoformat(), + }) + + return {'path': 'refresh', 'result': extracted, 'product': product} diff --git a/backend/services/auth_service.py b/backend/services/auth_service.py new file mode 100644 index 0000000..a111e51 --- /dev/null +++ b/backend/services/auth_service.py @@ -0,0 +1,46 @@ +from repositories.pocketbase_client import pb_client +from repositories.tenant_repository import tenant_repository +from repositories.seat_repository import seat_repository + + +def get_tenant_id_from_request(request): + """ + Resolves the tenant_id for a dashboard request from its + `Authorization: Bearer {jwt}` header. The JWT itself is issued and + verified by PocketBase (the dashboard authenticates directly against + PocketBase) — this backend doesn't hold the JWT signing secret, so + it validates the token by asking PocketBase to refresh it, then + resolves the returned email to a tenant via the existing seat record. + + Data flow: + request.headers['Authorization'] → strip "Bearer " prefix → + PocketBase POST /api/collections/users/auth-refresh → + { record: { email } } → seat lookup by email across known tenants + (via domain) → tenant_id returned, or None if unresolvable + """ + auth_header = request.headers.get('Authorization', '') + if not auth_header.startswith('Bearer '): + return None + token = auth_header[len('Bearer '):] + + record = pb_client.auth_refresh(token) + if not record or not record.get('email'): + return None + + email = record['email'] + domain = email.split('@', 1)[1].lower() if '@' in email else None + if not domain: + return None + + tenant = tenant_repository.get_by_domain(domain) + if not tenant: + return None + + # Confirm this email actually holds an active seat on the tenant — + # a verified PocketBase login alone isn't sufficient authorization, + # the email must also be a registered, active seat. + seat = seat_repository.get_active(tenant['id'], email) + if not seat: + return None + + return tenant['id'] diff --git a/backend/services/battlecard_service.py b/backend/services/battlecard_service.py new file mode 100644 index 0000000..72bf80b --- /dev/null +++ b/backend/services/battlecard_service.py @@ -0,0 +1,104 @@ +import logging +from repositories.product_repository import product_repository +from repositories.competitor_repository import competitor_repository +from repositories.battlecard_repository import battlecard_repository + +logger = logging.getLogger(__name__) + +BATTLECARD_PROMPT = '''You are a competitive intelligence analyst for an adventure travel operator. +Based on this competitor data, write a concise battlecard. + +Competitor: {name} +Top trips: {top_trips} +Price range: {price_low} - {price_high} USD +Avg duration: {avg_duration} days +Positioning: {positioning_summary} +Recent changes: {recent_changes} + +Write: +1. One line positioning summary (max 15 words) +2. Three strengths (bullet points, max 8 words each) +3. Three weaknesses (bullet points, max 8 words each) +4. One pricing observation (max 20 words) + +Return ONLY valid JSON: +{{ + "positioning": "...", + "strengths": ["...", "...", "..."], + "weaknesses": ["...", "...", "..."], + "pricing_observation": "..." +}}''' + + +def _summarise_products(products): + """Reduces a list of product records into the plain values BATTLECARD_PROMPT needs.""" + top_trips = ', '.join(p.get('trip_name', '') for p in products[:5] if p.get('trip_name')) + prices = [p['majority_price_usd'] for p in products if p.get('majority_price_usd')] + durations = [p['duration_days'] for p in products if p.get('duration_days')] + return { + 'top_trips': top_trips or 'No products on record', + 'price_low': min(prices) if prices else 'unknown', + 'price_high': max(prices) if prices else 'unknown', + 'avg_duration': round(sum(durations) / len(durations)) if durations else 'unknown', + } + + +def generate_battlecard(competitor_id, tenant_id): + """ + Pulls latest product data for a competitor from PocketBase, passes + structured data to the OpenRouter extraction model, and saves the + generated battlecard text/JSON back to PocketBase. + + Data flow: + competitor_id → PocketBase products (latest 10) → + structured prompt → OpenRouter (claude-haiku via core.extractor) → + battlecard text + JSON → PocketBase battlecards table → + battlecard dict returned for immediate use + """ + # core.extractor is a Phase 2 module — imported lazily so this + # service is importable (and unit-testable against mocks) before + # the scraping engine refactor lands. + from core.extractor import extract_with_openrouter + + competitor = competitor_repository.get_by_id(competitor_id) + products = product_repository.get_latest_for_competitor(competitor_id, limit=10) + summary = _summarise_products(products) + + prompt = BATTLECARD_PROMPT.format( + name=competitor.get('name', 'Unknown') if competitor else 'Unknown', + top_trips=summary['top_trips'], + price_low=summary['price_low'], + price_high=summary['price_high'], + avg_duration=summary['avg_duration'], + positioning_summary='Derived from current product catalogue.', + recent_changes='See price_history for this competitor.', + ) + + try: + extracted = extract_with_openrouter(prompt, content='') + except Exception as e: + # A failed battlecard regeneration must not break the scrape + # pipeline that triggered it — log and leave the prior + # battlecard (if any) untouched. + logger.error(f'Battlecard generation failed for {competitor_id}: {str(e)}') + return None + + battlecard = battlecard_repository.upsert(competitor_id, { + 'tenant_id': tenant_id, + 'content': _render_text(extracted), + 'content_json': extracted, + }) + return battlecard + + +def _render_text(extracted): + """Flattens the structured battlecard JSON into a plain-text summary for the `content` field.""" + if not isinstance(extracted, dict): + return str(extracted) + strengths = '\n'.join(f'+ {s}' for s in extracted.get('strengths', [])) + weaknesses = '\n'.join(f'- {w}' for w in extracted.get('weaknesses', [])) + return ( + f"{extracted.get('positioning', '')}\n\n" + f"Strengths:\n{strengths}\n\nWeaknesses:\n{weaknesses}\n\n" + f"{extracted.get('pricing_observation', '')}" + ) diff --git a/backend/services/billing_service.py b/backend/services/billing_service.py new file mode 100644 index 0000000..ab201d3 --- /dev/null +++ b/backend/services/billing_service.py @@ -0,0 +1,146 @@ +import os +import logging +from datetime import datetime, timedelta, timezone +from repositories.tenant_repository import tenant_repository + +logger = logging.getLogger(__name__) + +STRIPE_SECRET_KEY = os.getenv('STRIPE_SECRET_KEY') + +TIER_SEATS = { + 'analyse': 5, + 'discover': 8, + 'intelligence': 10, + 'enterprise': 999, # effectively unlimited +} + +TRIAL_PERIODS = { + 'analyse': 14, + 'discover': 21, + 'intelligence': 30, + 'enterprise': 0, +} + +ADDITIONAL_SEAT_PRICE = 45 # USD/seat/month — configured as a Stripe add-on price + +TIER_PRICE_ENV_VARS = { + 'analyse': 'STRIPE_PRICE_ANALYSE', + 'discover': 'STRIPE_PRICE_DISCOVER', + 'intelligence': 'STRIPE_PRICE_INTELLIGENCE', + # Enterprise is unlisted — created manually per client, no env var. +} + + +def _stripe(): + """Lazily imports and configures the stripe SDK — avoids an import-time key requirement.""" + import stripe + stripe.api_key = STRIPE_SECRET_KEY + return stripe + + +def create_portal_session(tenant_id): + """ + Creates a Stripe customer portal session for self-service billing. + + Data flow: + tenant_id → PocketBase stripe_customer_id → + Stripe billing_portal.Session.create → portal URL returned + """ + tenant = tenant_repository.get_by_id(tenant_id) + if not tenant or not tenant.get('stripe_customer_id'): + raise ValueError('Tenant has no Stripe customer on file') + + stripe = _stripe() + session = stripe.billing_portal.Session.create( + customer=tenant['stripe_customer_id'], + return_url='https://app.bettersight.io/account' + ) + return session.url + + +def create_upgrade_checkout(tenant_id, target_tier): + """ + Creates a Stripe checkout session that upgrades a tenant to + target_tier. Tags the session with tenant_id + target_tier in + metadata so the webhook handler can distinguish an upgrade + completion from a brand-new-tenant signup completion. + + Data flow: + tenant_id + target_tier → STRIPE_PRICE_* env lookup for target_tier → + Stripe checkout.Session.create (mode=subscription, metadata tagged) → + checkout URL returned + """ + price_env_var = TIER_PRICE_ENV_VARS.get(target_tier) + if not price_env_var or not os.getenv(price_env_var): + raise ValueError(f'No Stripe price configured for tier: {target_tier}') + + tenant = tenant_repository.get_by_id(tenant_id) + if not tenant: + raise ValueError(f'Unknown tenant: {tenant_id}') + + stripe = _stripe() + session = stripe.checkout.Session.create( + customer=tenant.get('stripe_customer_id'), + mode='subscription', + line_items=[{'price': os.getenv(price_env_var), 'quantity': 1}], + payment_method_collection='if_required', + subscription_data={'trial_period_days': TRIAL_PERIODS.get(target_tier, 0)}, + metadata={'tenant_id': tenant_id, 'target_tier': target_tier}, + success_url='https://app.bettersight.io/account?upgrade=success', + cancel_url='https://app.bettersight.io/account?upgrade=cancelled', + ) + return session.url + + +def handle_checkout_completed(event_data): + """ + Handles checkout.session.completed for both flows: + - Upgrade flow (metadata.tenant_id present): bumps the existing + tenant's tier and max_seats. + - New signup flow (no metadata.tenant_id): out of scope for this + handler — new-tenant creation from a public checkout flow is not + part of the Analyse-tier MVP onboarding runbook (CLAUDE.md §32 + onboards the first clients manually); this function only handles + the upgrade case until self-serve signup is built. + + Data flow: + Stripe checkout.session.completed event → + metadata.tenant_id present? → + yes: tenant tier/max_seats updated, status set to trial/active per + whether a trial was attached → + no: logged and ignored (manual onboarding handles new tenants) + """ + metadata = event_data.get('metadata', {}) + tenant_id = metadata.get('tenant_id') + target_tier = metadata.get('target_tier') + + if not tenant_id or not target_tier: + logger.info('checkout.session.completed with no upgrade metadata — ignoring (manual onboarding flow)') + return + + tenant_repository.update(tenant_id, { + 'tier': target_tier, + 'max_seats': TIER_SEATS.get(target_tier, 5), + 'stripe_subscription_id': event_data.get('subscription'), + 'status': 'trial' if TRIAL_PERIODS.get(target_tier, 0) > 0 else 'active', + 'trial_ends_at': ( + (datetime.now(timezone.utc) + timedelta(days=TRIAL_PERIODS[target_tier])).isoformat() + if TRIAL_PERIODS.get(target_tier, 0) > 0 else None + ), + }) + + +def handle_subscription_updated(event_data, tenant_id): + """Updates tier/max_seats if the plan changed on an existing subscription.""" + tier = event_data.get('metadata', {}).get('tier') + updates = {} + if tier and tier in TIER_SEATS: + updates['tier'] = tier + updates['max_seats'] = TIER_SEATS[tier] + if updates: + tenant_repository.update(tenant_id, updates) + + +def handle_subscription_deleted(tenant_id): + """Marks a tenant inactive on cancellation — seats block at next Apps Script cache expiry (6hrs).""" + tenant_repository.update(tenant_id, {'status': 'inactive'}) diff --git a/backend/services/brief_service.py b/backend/services/brief_service.py new file mode 100644 index 0000000..9d5f181 --- /dev/null +++ b/backend/services/brief_service.py @@ -0,0 +1,59 @@ +import os +import logging +from jinja2 import Environment, FileSystemLoader, select_autoescape +from repositories.brief_repository import brief_repository + +logger = logging.getLogger(__name__) + +RESEND_API_KEY = os.getenv('RESEND_API_KEY') +TEMPLATES_DIR = os.path.join(os.path.dirname(__file__), '..', 'templates', 'emails') + +# Plain Jinja2 (not flask.render_template) so this service has no +# dependency on an active Flask request/app context — it must also run +# from the n8n-triggered Flask route AND, in principle, from an RQ +# worker process. Autoescaping is mandatory: brief content includes +# scraped third-party trip names which must never be interpreted as +# HTML in the rendered email. +_jinja_env = Environment( + loader=FileSystemLoader(TEMPLATES_DIR), + autoescape=select_autoescape(['html']) +) + + +def send_weekly_brief(tenant_id): + """ + Renders the weekly intelligence brief as branded HTML and sends it + via Resend to the tenant admin email. + + Data flow: + tenant_id → brief_repository.get_weekly_data() (price_history, + new products, battlecard updates for the past 7 days) → + brief.html Jinja2 template rendered → Resend API → + digest_logs record created (sent or failed) + """ + data = brief_repository.get_weekly_data(tenant_id) + admin_email = data.get('admin_email') + if not admin_email: + logger.error(f'No admin_email on file for tenant {tenant_id} — skipping weekly brief') + return + + template = _jinja_env.get_template('brief.html') + html = template.render(**data) + + try: + # Imported lazily so this module is importable without the + # `resend` package configured in test environments. + import resend + resend.api_key = RESEND_API_KEY + resend.Emails.send({ + 'from': 'Bettersight ', + 'to': admin_email, + 'subject': f"Bettersight Weekly Brief — Week of {data['week_of']}", + 'html': html, + }) + brief_repository.log_digest(tenant_id, 'weekly', recipient_email=admin_email, status='sent') + except Exception as e: + # A failed email send is a side-effect failure — it must not + # raise into the n8n workflow that triggered this run. + logger.error(f'Weekly brief send failed for tenant {tenant_id}: {str(e)}') + brief_repository.log_digest(tenant_id, 'weekly', recipient_email=admin_email, status='failed') diff --git a/backend/services/embedding_service.py b/backend/services/embedding_service.py new file mode 100644 index 0000000..e073e69 --- /dev/null +++ b/backend/services/embedding_service.py @@ -0,0 +1,91 @@ +import os +import logging +import numpy as np + +logger = logging.getLogger(__name__) + +OPENROUTER_API_KEY = os.getenv('OPENROUTER_API_KEY') +EMBEDDING_MODEL = 'openai/text-embedding-3-small' +SIMILARITY_THRESHOLD = 0.75 + + +def embed_trip(trip): + """ + Generates a semantic embedding for a trip using OpenRouter's + text-embedding-3-small, combining trip name, destination, duration, + and activities into a single text representation first. + + Data flow: + trip dict → formatted text string → OpenRouter embeddings API → + vector list returned for storage in products.embedding / + client_trips.embedding + """ + # Imported lazily so this module has no hard dependency on the + # `openai` package at import time for callers that only need + # find_comparable_trips() (pure math, no API calls). + from openai import OpenAI + + text = ( + f"{trip.get('trip_name', '')}\n" + f"Destination: {trip.get('destination', '')}\n" + f"Duration: {trip.get('duration_days', '')} days\n" + f"Activities: {trip.get('activities', '')}\n" + f"Start: {trip.get('start_location', '')}" + ) + + client = OpenAI(api_key=OPENROUTER_API_KEY, base_url='https://openrouter.ai/api/v1') + response = client.embeddings.create(model=EMBEDDING_MODEL, input=text) + return response.data[0].embedding + + +def _cosine_similarity(a, b): + """Standard cosine similarity between two equal-length embedding vectors.""" + a, b = np.array(a), np.array(b) + denom = (np.linalg.norm(a) * np.linalg.norm(b)) + if denom == 0: + return 0.0 + return float(np.dot(a, b) / denom) + + +def find_comparable_trips(client_trips, competitor_trips, threshold=SIMILARITY_THRESHOLD): + """ + Calculates cosine similarity between every client trip and every + competitor trip, returning matches above threshold sorted by + similarity score descending. + + Data flow: + client_trips + competitor_trips (each with .embedding) → + pairwise cosine similarity → filter by threshold → + sort by score descending → list of match dicts returned + """ + matches = [] + for client_trip in client_trips: + client_embedding = client_trip.get('embedding') + if not client_embedding: + continue + for competitor_trip in competitor_trips: + competitor_embedding = competitor_trip.get('embedding') + if not competitor_embedding: + continue + score = _cosine_similarity(client_embedding, competitor_embedding) + if score >= threshold: + matches.append({ + 'client_product': client_trip.get('trip_name'), + 'competitor_product': competitor_trip.get('id'), + 'similarity_score': round(score, 4), + 'price_difference': _safe_diff( + competitor_trip.get('majority_price_usd'), client_trip.get('price_usd') + ), + 'duration_difference': _safe_diff( + competitor_trip.get('duration_days'), client_trip.get('duration_days') + ), + }) + matches.sort(key=lambda m: m['similarity_score'], reverse=True) + return matches + + +def _safe_diff(a, b): + """Returns a - b if both are present, else None — keeps the diff fields nullable per schema.""" + if a is None or b is None: + return None + return a - b diff --git a/backend/services/licence_service.py b/backend/services/licence_service.py new file mode 100644 index 0000000..de4665e --- /dev/null +++ b/backend/services/licence_service.py @@ -0,0 +1,59 @@ +from repositories.tenant_repository import tenant_repository +from repositories.seat_repository import seat_repository + +# Both trial and active tenants may use the dashboard and the Apps Script +# export plugin — only 'inactive' (cancelled, or trial expired with no +# card on file) blocks access. +VALID_STATUSES = ['active', 'trial'] + + +def validate_email(email): + """ + Validates a PM's access to Bettersight via two checks: domain + registration and seat count. + + Note: CLAUDE.md's earlier §9 draft described a third, sheet_id-binding + layer, but §15 explicitly removes it for the standalone Apps Script + model ("validation is email-only... remove the sheet_id binding layer + — it is redundant and would break the standalone model"). That later, + more specific instruction is what's implemented here. + + Data flow: + email → + domain extraction → tenant lookup by domain (Layer 1) → + status check (must be trial or active) → + existing active seat → refresh last_accessed, allow → + no existing seat → seat count vs tenant.max_seats → + under limit: register new seat, allow / at limit: reject → + { valid: bool, tier: str|None, reason: str|None } + """ + if not email or '@' not in email: + return {'valid': False, 'reason': 'Invalid email address'} + + domain = email.split('@', 1)[1].lower() + + # Layer 1: domain must belong to a known tenant in good standing + tenant = tenant_repository.get_by_domain(domain) + if not tenant: + return {'valid': False, 'reason': 'Domain not registered'} + if tenant.get('status') not in VALID_STATUSES: + return {'valid': False, 'reason': 'Subscription inactive'} + + # Layer 2: existing seat is a fast-path allow with a freshness bump; + # a new seat must fit within the tenant's purchased seat count + existing_seat = seat_repository.get_active(tenant['id'], email) + if existing_seat: + seat_repository.update_last_accessed(existing_seat['id']) + return {'valid': True, 'tier': tenant['tier']} + + active_count = seat_repository.count_active(tenant['id']) + if active_count >= tenant.get('max_seats', 0): + return {'valid': False, 'reason': 'Seat limit reached'} + + seat_repository.create({ + 'tenant_id': tenant['id'], + 'email': email, + 'active': True, + 'added_by': 'self', + }) + return {'valid': True, 'tier': tenant['tier']} diff --git a/backend/services/onboarding_service.py b/backend/services/onboarding_service.py new file mode 100644 index 0000000..8e8b63b --- /dev/null +++ b/backend/services/onboarding_service.py @@ -0,0 +1,35 @@ +from repositories.tenant_repository import tenant_repository + +DEFAULT_CHECKLIST = { + 'add_competitor': False, + 'run_analysis': False, + 'push_to_sheet': False, + 'download_template': False, + 'tour_completed': False, + 'tour_skipped': False, + 'tour_step': 0, +} + + +def update_onboarding(tenant_id, updates): + """ + Merges partial onboarding_checklist updates into a tenant's record. + Used both by the guided-tour composable (tour_step, tour_completed, + tour_skipped) and by checklist auto-completion logic elsewhere + (add_competitor, run_analysis, push_to_sheet, download_template). + + Data flow: + tenant_id + partial onboarding dict → + existing onboarding_checklist read → + merged with updates (existing keys not in `updates` are preserved) → + PocketBase tenants record updated → + merged checklist returned + """ + tenant = tenant_repository.get_by_id(tenant_id) + if not tenant: + raise ValueError(f'Unknown tenant: {tenant_id}') + + current = tenant.get('onboarding_checklist') or DEFAULT_CHECKLIST + merged = {**current, **updates} + tenant_repository.update(tenant_id, {'onboarding_checklist': merged}) + return merged diff --git a/backend/services/trip_finder_service.py b/backend/services/trip_finder_service.py new file mode 100644 index 0000000..5c5fcce --- /dev/null +++ b/backend/services/trip_finder_service.py @@ -0,0 +1,114 @@ +import os +import logging +import requests +from concurrent.futures import ThreadPoolExecutor, as_completed + +logger = logging.getLogger(__name__) + +FIRECRAWL_URL = os.getenv('FIRECRAWL_URL', 'http://firecrawl:3002') + +CATEGORY_KEYWORDS = ['classic', 'trek', 'trail', 'safari', 'adventure', 'explorer', 'journey'] +CONFIDENCE_FOUND_THRESHOLD = 0.3 +MAX_CONCURRENT_MAP_CALLS = 5 + + +def score_and_rank_urls(urls, destination, duration): + """ + Scores candidate URLs by keyword match against destination and + duration, returning the highest-scoring URL with a confidence score. + + Data flow: + urls + destination + duration → + per-url score: +1 per destination keyword in URL, +2 if duration + appears in URL, +1 per travel-category keyword in URL → + sort by score → confidence = top_score / max_possible_score → + { url, confidence, found } — found=False (prompt manual entry) + when confidence is below CONFIDENCE_FOUND_THRESHOLD + """ + dest_keywords = destination.lower().split() + scores = [] + + for url in urls: + url_lower = url.lower() + score = 0 + score += sum(1 for kw in dest_keywords if kw in url_lower) + if str(duration) in url_lower: + score += 2 + score += sum(1 for kw in CATEGORY_KEYWORDS if kw in url_lower) + scores.append((score, url)) + + scores.sort(key=lambda pair: pair[0], reverse=True) + if not scores or scores[0][0] == 0: + return {'url': None, 'confidence': 0, 'found': False} + + top_score, top_url = scores[0] + max_possible = len(dest_keywords) + 2 + 1 + confidence = min(top_score / max_possible, 1.0) if max_possible else 0 + + return { + 'url': top_url, + 'confidence': round(confidence, 2), + 'found': confidence >= CONFIDENCE_FOUND_THRESHOLD, + } + + +def find_competitor_trip_url(root_url, destination, duration, travel_style): + """ + Uses Firecrawl /map with search terms to find the most likely trip + page on a competitor site matching the PM's research intent. + + Data flow: + root_url + destination/duration/travel_style → + POST {FIRECRAWL_URL}/v1/map with a search string → + candidate URL list → score_and_rank_urls() → + { url, confidence, found } returned (found=False, url=None on any + Firecrawl failure — never raises, since one competitor's failure + must not block the others per CLAUDE.md §18 resilience rules) + """ + try: + response = requests.post( + f'{FIRECRAWL_URL}/v1/map', + json={'url': root_url, 'search': f'{destination} {duration} days {travel_style} tour'}, + timeout=15 + ) + response.raise_for_status() + urls = response.json().get('links', []) + except requests.RequestException as e: + logger.error(f'Firecrawl /map failed for {root_url}: {str(e)}') + return {'url': None, 'confidence': 0, 'found': False} + + if not urls: + return {'url': None, 'confidence': 0, 'found': False} + + return score_and_rank_urls(urls, destination, duration) + + +def find_all_competitor_urls(competitors, destination, duration, travel_style): + """ + Runs trip URL finding for all selected competitors concurrently — + at most MAX_CONCURRENT_MAP_CALLS Firecrawl /map calls in flight at + once, respecting server limits. + + Data flow: + competitors list → ThreadPoolExecutor(max_workers=5) → + find_competitor_trip_url() per competitor in parallel → + collect results as they complete → matches list returned + (one entry per competitor, in completion order) + """ + results = [] + with ThreadPoolExecutor(max_workers=MAX_CONCURRENT_MAP_CALLS) as executor: + futures = { + executor.submit( + find_competitor_trip_url, c['website'], destination, duration, travel_style + ): c + for c in competitors + } + for future in as_completed(futures): + competitor = futures[future] + match = future.result() + results.append({ + 'competitor_id': competitor['id'], + 'competitor_name': competitor['name'], + **match, + }) + return results diff --git a/backend/templates/emails/brief.html b/backend/templates/emails/brief.html new file mode 100644 index 0000000..0236e6f --- /dev/null +++ b/backend/templates/emails/brief.html @@ -0,0 +1,92 @@ + + + + +Bettersight Weekly Brief + + + + + + + + + + + + {% if price_changes %} + + + + {% endif %} + + {% if new_products %} + + + + {% endif %} + + {% if battlecard_updates %} + + + + {% endif %} + + + + + + + + +
+ Bettersight +
Weekly Brief — {{ tenant_name }} · Week of {{ week_of }}
+
+

Market summary

+

+ {{ price_changes|length }} price change{{ '' if price_changes|length == 1 else 's' }}, + {{ new_products|length }} new product{{ '' if new_products|length == 1 else 's' }} + detected this week. +

+
+

Price movements

+ + + + + {% for change in price_changes %} + + + + + + + {% endfor %} +
TripPreviousCurrentΔ
{{ change.product_id }}${{ change.majority_price_usd - (change.change_amount or 0) }}${{ change.majority_price_usd }} + {{ change.change_percent }}% +
+
+

New products this week

+ {% for product in new_products %} +
+ {{ product.trip_name }} — {{ product.destination }}, + {{ product.duration_days }} days, ${{ product.majority_price_usd }} +
+ {% endfor %} +
+

Battlecard updates

+

+ {{ battlecard_updates|length }} battlecard{{ '' if battlecard_updates|length == 1 else 's' }} refreshed this week. +

+
+ + View full detail in your dashboard → + +
+ Bettersight · Better sight, better moves. +
+ Unsubscribe from weekly briefs +
+ + diff --git a/backend/tests/__init__.py b/backend/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py new file mode 100644 index 0000000..3066e05 --- /dev/null +++ b/backend/tests/conftest.py @@ -0,0 +1,63 @@ +import os +import sys + +# Must be set before any repositories module is imported — each module +# selects its Mock* implementation vs the real PocketBase-backed one at +# import time based on this flag. +os.environ['USE_MOCK_REPOSITORIES'] = 'true' +os.environ.setdefault('API_SECRET_KEY', 'test-api-key') +os.environ.setdefault('POCKETBASE_URL', 'http://pocketbase.invalid') +os.environ.setdefault('REDIS_URL', 'redis://localhost:6379/0') +# Keeps Flask-Limiter off the network entirely in tests — see extensions.py. +os.environ.setdefault('RATELIMIT_STORAGE_URI', 'memory://') +os.environ.setdefault('FIRECRAWL_URL', 'http://firecrawl.invalid') +os.environ.setdefault('GOTIFY_URL', 'http://gotify.invalid') +os.environ.setdefault('GOTIFY_APP_TOKEN', 'test-token') + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + +import pytest + + +@pytest.fixture +def app(): + from app import app as flask_app + flask_app.config['TESTING'] = True + return flask_app + + +@pytest.fixture +def client(app): + return app.test_client() + + +@pytest.fixture +def api_headers(): + return {'X-API-Key': 'test-api-key'} + + +@pytest.fixture(autouse=True) +def _reset_mock_repositories(): + """ + Clears every mock repository's in-memory store between tests so + state doesn't leak across test functions. + """ + yield + from repositories.tenant_repository import tenant_repository + from repositories.seat_repository import seat_repository + from repositories.competitor_repository import competitor_repository + from repositories.product_repository import product_repository + from repositories.scrape_repository import scrape_repository + from repositories.proxy_repository import proxy_repository + from repositories.alert_repository import alert_repository + from repositories.battlecard_repository import battlecard_repository + from repositories.comparable_repository import comparable_repository + from repositories.monitoring_repository import monitoring_repository + + for repo in [ + tenant_repository, seat_repository, competitor_repository, product_repository, + scrape_repository, proxy_repository, alert_repository, battlecard_repository, + comparable_repository, monitoring_repository, + ]: + if hasattr(repo, '_records'): + repo._records.clear() diff --git a/backend/tests/repositories/__init__.py b/backend/tests/repositories/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/repositories/test_pocketbase_client.py b/backend/tests/repositories/test_pocketbase_client.py new file mode 100644 index 0000000..e25358c --- /dev/null +++ b/backend/tests/repositories/test_pocketbase_client.py @@ -0,0 +1,157 @@ +""" +repositories/pocketbase_client.py is the single chokepoint every real +repository depends on — fully unit tested here against a mocked +`requests` module so no live PocketBase instance is needed. +""" +import pytest +from repositories.pocketbase_client import PocketBaseClient, PocketBaseError + + +class _FakeResponse: + def __init__(self, status_code=200, json_data=None, text='', content=b'1'): + self.status_code = status_code + self._json = json_data or {} + self.text = text + self.content = content if status_code < 400 else (content or b'err') + + def json(self): + return self._json + + +def test_authenticate_caches_token(monkeypatch): + client = PocketBaseClient() + calls = [] + + def _fake_post(url, json, timeout): + calls.append(url) + return _FakeResponse(200, {'token': 'abc123'}) + + monkeypatch.setattr('repositories.pocketbase_client.requests.post', _fake_post) + client._authenticate() + + assert client._token == 'abc123' + assert len(calls) == 1 + + +def test_authenticate_raises_on_failure(monkeypatch): + client = PocketBaseClient() + monkeypatch.setattr( + 'repositories.pocketbase_client.requests.post', + lambda *a, **k: _FakeResponse(400, text='bad credentials') + ) + with pytest.raises(PocketBaseError): + client._authenticate() + + +def test_request_retries_once_on_401(monkeypatch): + client = PocketBaseClient() + client._token = 'stale-token' + client._token_expires_at = 9999999999 + + responses = [_FakeResponse(401), _FakeResponse(200, {'items': []})] + calls = {'auth': 0, 'request': 0} + + def _fake_authenticate(): + calls['auth'] += 1 + client._token = 'fresh-token' + + def _fake_request(method, url, headers, timeout, **kwargs): + calls['request'] += 1 + return responses.pop(0) + + monkeypatch.setattr(client, '_authenticate', _fake_authenticate) + monkeypatch.setattr('repositories.pocketbase_client.requests.request', _fake_request) + + result = client._request('GET', '/api/collections/tenants/records') + + assert calls['auth'] == 1 + assert calls['request'] == 2 + assert result == {'items': []} + + +def test_request_raises_on_non_401_error(monkeypatch): + client = PocketBaseClient() + client._token = 'token' + client._token_expires_at = 9999999999 + monkeypatch.setattr( + 'repositories.pocketbase_client.requests.request', + lambda *a, **k: _FakeResponse(500, text='server error') + ) + with pytest.raises(PocketBaseError): + client._request('GET', '/api/collections/tenants/records') + + +def test_list_returns_items(monkeypatch): + client = PocketBaseClient() + monkeypatch.setattr(client, '_request', lambda *a, **k: {'items': [{'id': '1'}], 'page': 1}) + assert client.list('tenants') == [{'id': '1'}] + + +def test_get_first_returns_none_when_empty(monkeypatch): + client = PocketBaseClient() + monkeypatch.setattr(client, 'list', lambda *a, **k: []) + assert client.get_first('tenants', 'domain = "x"') is None + + +def test_get_one_returns_none_on_404(monkeypatch): + client = PocketBaseClient() + + def _raise_404(*a, **k): + raise PocketBaseError('GET /x failed: 404 not found') + + monkeypatch.setattr(client, '_request', _raise_404) + assert client.get_one('tenants', 'missing-id') is None + + +def test_get_one_reraises_non_404_errors(monkeypatch): + client = PocketBaseClient() + + def _raise_500(*a, **k): + raise PocketBaseError('GET /x failed: 500 server error') + + monkeypatch.setattr(client, '_request', _raise_500) + with pytest.raises(PocketBaseError): + client.get_one('tenants', 'some-id') + + +def test_create_update_delete_delegate_to_request(monkeypatch): + client = PocketBaseClient() + captured = [] + monkeypatch.setattr(client, '_request', lambda method, path, **k: captured.append((method, path, k)) or {'id': '1'}) + + client.create('tenants', {'name': 'x'}) + client.update('tenants', '1', {'name': 'y'}) + client.delete('tenants', '1') + + assert captured[0][0] == 'POST' + assert captured[1][0] == 'PATCH' + assert captured[2][0] == 'DELETE' + + +def test_auth_refresh_returns_record_on_success(monkeypatch): + client = PocketBaseClient() + monkeypatch.setattr( + 'repositories.pocketbase_client.requests.post', + lambda *a, **k: _FakeResponse(200, {'record': {'email': 'pm@gadventures.com'}}) + ) + assert client.auth_refresh('sometoken') == {'email': 'pm@gadventures.com'} + + +def test_auth_refresh_returns_none_on_failure(monkeypatch): + client = PocketBaseClient() + monkeypatch.setattr( + 'repositories.pocketbase_client.requests.post', + lambda *a, **k: _FakeResponse(401) + ) + assert client.auth_refresh('badtoken') is None + + +def test_auth_refresh_returns_none_on_network_error(monkeypatch): + import requests + client = PocketBaseClient() + + def _raise(*a, **k): + raise requests.RequestException('timeout') + + monkeypatch.setattr('repositories.pocketbase_client.requests.post', _raise) + assert client.auth_refresh('sometoken') is None diff --git a/backend/tests/repositories/test_real_repositories.py b/backend/tests/repositories/test_real_repositories.py new file mode 100644 index 0000000..ae8c56e --- /dev/null +++ b/backend/tests/repositories/test_real_repositories.py @@ -0,0 +1,206 @@ +""" +Every Real*Repository class is a thin wrapper over pb_client. These +tests instantiate the real classes directly (bypassing the Mock* +selection that USE_MOCK_REPOSITORIES otherwise activates) and patch +each module's `pb_client` reference with a recording fake, verifying +correct collection names and call delegation without needing a live +PocketBase instance. +""" +from repositories.tenant_repository import TenantRepository +from repositories.seat_repository import SeatRepository +from repositories.competitor_repository import CompetitorRepository +from repositories.product_repository import ProductRepository +from repositories.scrape_repository import ScrapeRepository +from repositories.proxy_repository import ProxyRepository +from repositories.alert_repository import AlertRepository +from repositories.battlecard_repository import BattlecardRepository +from repositories.comparable_repository import ComparableRepository +from repositories.brief_repository import BriefRepository +from repositories.monitoring_repository import MonitoringRepository + + +class _FakePbClient: + """Records every call made to it and returns canned values.""" + + def __init__(self): + self.calls = [] + self.list_return = [] + self.get_first_return = {'id': 'rec1'} + self.get_one_return = {'id': 'rec1'} + self.create_return = {'id': 'rec1'} + self.update_return = {'id': 'rec1', 'updated': True} + + def list(self, collection, filter_str=None, sort=None, page=1, per_page=50): + self.calls.append(('list', collection, filter_str, sort)) + return self.list_return + + def get_first(self, collection, filter_str): + self.calls.append(('get_first', collection, filter_str)) + return self.get_first_return + + def get_one(self, collection, record_id): + self.calls.append(('get_one', collection, record_id)) + return self.get_one_return + + def create(self, collection, data): + self.calls.append(('create', collection, data)) + return self.create_return + + def update(self, collection, record_id, data): + self.calls.append(('update', collection, record_id, data)) + return self.update_return + + def delete(self, collection, record_id): + self.calls.append(('delete', collection, record_id)) + return True + + +def test_tenant_repository_delegates_correctly(monkeypatch): + fake = _FakePbClient() + monkeypatch.setattr('repositories.tenant_repository.pb_client', fake) + repo = TenantRepository() + + assert repo.get_by_domain('gadventures.com') == fake.get_first_return + assert repo.get_by_id('t1') == fake.get_one_return + assert repo.get_by_stripe_customer_id('cus_123') == fake.get_first_return + assert repo.create({'name': 'x'}) == fake.create_return + assert repo.update('t1', {'name': 'y'}) == fake.update_return + assert fake.calls[0] == ('get_first', 'tenants', 'domain = "gadventures.com"') + + +def test_seat_repository_delegates_correctly(monkeypatch): + fake = _FakePbClient() + fake.list_return = [{'id': 's1'}, {'id': 's2'}] + monkeypatch.setattr('repositories.seat_repository.pb_client', fake) + repo = SeatRepository() + + assert repo.get_active('t1', 'pm@gadventures.com') == fake.get_first_return + assert repo.count_active('t1') == 2 + assert repo.list_for_tenant('t1') == fake.list_return + assert repo.create({'email': 'x'}) == fake.create_return + repo.update_last_accessed('s1') + assert repo.delete('s1') is True + + +def test_competitor_repository_delegates_correctly(monkeypatch): + fake = _FakePbClient() + fake.list_return = [{'id': 'c1', 'website': 'https://intrepidtravel.com'}] + monkeypatch.setattr('repositories.competitor_repository.pb_client', fake) + repo = CompetitorRepository() + + assert repo.get_by_url('https://intrepidtravel.com') == fake.get_first_return + assert repo.get_by_domain('t1', 'intrepidtravel.com') == fake.list_return[0] + assert repo.get_by_id('c1') == fake.get_one_return + assert repo.list_for_tenant('t1') == fake.list_return + assert repo.create({'name': 'x'}) == fake.create_return + assert repo.update('c1', {'active': False}) == fake.update_return + + +def test_product_repository_delegates_correctly(monkeypatch): + fake = _FakePbClient() + fake.list_return = [{'id': 'p1', 'last_seen': '2026-01-01'}] + monkeypatch.setattr('repositories.product_repository.pb_client', fake) + repo = ProductRepository() + + assert repo.get_latest_for_competitor('c1') == fake.list_return + assert repo.list_for_tenant('t1') == fake.list_return + assert repo.create({'trip_name': 'x'}) == fake.create_return + assert repo.update('p1', {'majority_price_usd': 100}) == fake.update_return + + +def test_scrape_repository_delegates_correctly(monkeypatch): + fake = _FakePbClient() + fake.list_return = [{'id': 'r1'}] + monkeypatch.setattr('repositories.scrape_repository.pb_client', fake) + repo = ScrapeRepository() + + assert repo.create({'status': 'pending'}) == fake.create_return + assert repo.update('r1', {'status': 'complete'}) == fake.update_return + assert repo.get_by_id('r1') == fake.get_one_return + assert repo.list_recent_for_tenant('t1') == fake.list_return + + +def test_proxy_repository_delegates_correctly(monkeypatch): + fake = _FakePbClient() + monkeypatch.setattr('repositories.proxy_repository.pb_client', fake) + repo = ProxyRepository() + + assert repo.get_by_domain('intrepidtravel.com') == fake.get_first_return + assert repo.create({'domain': 'x'}) == fake.create_return + assert repo.update('o1', {'provider': 'brightdata'}) == fake.update_return + + +def test_alert_repository_delegates_correctly(monkeypatch): + fake = _FakePbClient() + fake.list_return = [{'id': 'a1'}] + monkeypatch.setattr('repositories.alert_repository.pb_client', fake) + repo = AlertRepository() + + assert repo.create({'alert_type': 'price_change'}) == fake.create_return + assert repo.list_unread('t1') == fake.list_return + assert repo.list_recent('t1') == fake.list_return + assert repo.mark_read('a1') == fake.update_return + + +def test_battlecard_repository_upsert_creates_when_absent(monkeypatch): + fake = _FakePbClient() + fake.get_first_return = None + monkeypatch.setattr('repositories.battlecard_repository.pb_client', fake) + repo = BattlecardRepository() + + assert repo.get_for_competitor('c1') is None + result = repo.upsert('c1', {'content': 'x'}) + assert result == fake.create_return + assert fake.calls[-1][0] == 'create' + + +def test_battlecard_repository_upsert_updates_when_present(monkeypatch): + fake = _FakePbClient() + fake.get_first_return = {'id': 'b1', 'competitor_id': 'c1'} + monkeypatch.setattr('repositories.battlecard_repository.pb_client', fake) + repo = BattlecardRepository() + + result = repo.upsert('c1', {'content': 'updated'}) + assert result == fake.update_return + assert fake.calls[-1][0] == 'update' + assert repo.list_for_tenant('t1') == fake.list_return + + +def test_comparable_repository_delegates_correctly(monkeypatch): + fake = _FakePbClient() + fake.list_return = [{'id': 'm1'}] + monkeypatch.setattr('repositories.comparable_repository.pb_client', fake) + repo = ComparableRepository() + + assert repo.create({'similarity_score': 0.8}) == fake.create_return + assert repo.list_for_tenant('t1') == fake.list_return + assert repo.dismiss('m1') == fake.update_return + + +def test_brief_repository_get_weekly_data_aggregates_collections(monkeypatch): + fake = _FakePbClient() + fake.get_one_return = {'email': 'pm@gadventures.com', 'name': 'G Adventures'} + fake.list_return = [{'id': 'x'}] + monkeypatch.setattr('repositories.brief_repository.pb_client', fake) + repo = BriefRepository() + + data = repo.get_weekly_data('t1') + + assert data['admin_email'] == 'pm@gadventures.com' + assert data['tenant_name'] == 'G Adventures' + assert data['price_changes'] == [{'id': 'x'}] + assert data['new_products'] == [{'id': 'x'}] + assert data['battlecard_updates'] == [{'id': 'x'}] + + log = repo.log_digest('t1', 'weekly', recipient_email='pm@gadventures.com') + assert log == fake.create_return + + +def test_monitoring_repository_delegates_correctly(monkeypatch): + fake = _FakePbClient() + monkeypatch.setattr('repositories.monitoring_repository.pb_client', fake) + repo = MonitoringRepository() + + result = repo.log_event('queue_depth', 'warning', value=6, threshold=5, message='busy', alert_sent=True) + assert result == fake.create_return + assert fake.calls[0][1] == 'monitoring_events' diff --git a/backend/tests/services/__init__.py b/backend/tests/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/services/test_alert_service.py b/backend/tests/services/test_alert_service.py new file mode 100644 index 0000000..0e77ef7 --- /dev/null +++ b/backend/tests/services/test_alert_service.py @@ -0,0 +1,55 @@ +from services import alert_service +from repositories.monitoring_repository import monitoring_repository +from repositories.alert_repository import alert_repository + + +class _FakeResponse: + def __init__(self, status_code=200): + self.status_code = status_code + + +def test_send_gotify_logs_delivered_true_on_success(monkeypatch): + monkeypatch.setattr(alert_service.requests, 'post', lambda *a, **k: _FakeResponse(200)) + alert_service.send_gotify('Test title', 'Test message', priority=5) + events = list(monitoring_repository._records.values()) + assert events[-1]['alert_sent'] is True + assert events[-1]['severity'] == 'advisory' + + +def test_send_gotify_maps_priority_to_severity(): + import requests as real_requests + + def _fake_post(*a, **k): + return _FakeResponse(200) + + alert_service.requests.post = _fake_post + alert_service.send_gotify('Critical', 'msg', priority=10) + events = list(monitoring_repository._records.values()) + assert events[-1]['severity'] == 'critical' + + alert_service.send_gotify('Warning', 'msg', priority=7) + events = list(monitoring_repository._records.values()) + assert events[-1]['severity'] == 'warning' + alert_service.requests.post = real_requests.post + + +def test_send_gotify_never_raises_on_network_failure(monkeypatch): + def _raise(*a, **k): + raise Exception('connection refused') + + monkeypatch.setattr(alert_service.requests, 'post', _raise) + # Must not raise — side-effect failure only. + alert_service.send_gotify('Title', 'Message', priority=5) + events = list(monitoring_repository._records.values()) + assert events[-1]['alert_sent'] is False + + +def test_create_alert_defaults_dashboard_delivered_true(): + alert = alert_service.create_alert( + tenant_id='t1', competitor_id='c1', alert_type='price_change', + message='Price dropped', change_percent=-12 + ) + assert alert['delivered_dashboard'] is True + assert alert['delivered_email'] is False + assert alert['read'] is False + assert alert_repository.list_unread('t1') == [alert] diff --git a/backend/tests/services/test_analysis_service.py b/backend/tests/services/test_analysis_service.py new file mode 100644 index 0000000..121431b --- /dev/null +++ b/backend/tests/services/test_analysis_service.py @@ -0,0 +1,145 @@ +""" +analysis_service orchestrates the fast/refresh path decision (CLAUDE.md +§7). The refresh path lazy-imports Phase 2 modules (core.scraper, +core.extractor, industries.adventure_travel.prompt) that don't exist +yet — _install_fake_module injects stand-ins into sys.modules so the +orchestration logic itself (not the scraping/extraction internals) can +be tested now. +""" +import sys +import types +from datetime import datetime, timedelta, timezone +import pytest +from services import analysis_service +from repositories.competitor_repository import competitor_repository +from repositories.product_repository import product_repository +from repositories.tenant_repository import tenant_repository + + +def _install_fake_module(monkeypatch, dotted_name, **attrs): + """Injects a fake module into sys.modules, creating parent packages as needed.""" + parts = dotted_name.split('.') + for i in range(1, len(parts) + 1): + partial = '.'.join(parts[:i]) + if partial not in sys.modules: + monkeypatch.setitem(sys.modules, partial, types.ModuleType(partial)) + target = sys.modules[dotted_name] + for key, value in attrs.items(): + setattr(target, key, value) + return target + + +def _make_tenant_and_competitor(**competitor_overrides): + tenant = tenant_repository.create({ + 'name': 'G Adventures', 'domain': 'gadventures.com', 'tier': 'analyse', + 'status': 'active', 'max_seats': 5, + }) + data = { + 'tenant_id': tenant['id'], 'name': 'Intrepid Travel', 'website': 'https://intrepidtravel.com', + 'catalogue_url': 'https://intrepidtravel.com/adventures', 'primary_market': 'GB', 'active': True, + } + data.update(competitor_overrides) + competitor = competitor_repository.create(data) + return tenant, competitor + + +def test_needs_refresh_true_when_change_detected(): + assert analysis_service.needs_refresh({'change_detected': True, 'last_scraped': None}) is True + + +def test_needs_refresh_true_when_never_scraped(): + assert analysis_service.needs_refresh({'change_detected': False, 'last_scraped': None}) is True + + +def test_needs_refresh_true_when_stale(): + stale = (datetime.now(timezone.utc) - timedelta(hours=48)).isoformat() + assert analysis_service.needs_refresh({'change_detected': False, 'last_scraped': stale}) is True + + +def test_needs_refresh_false_when_fresh_and_unchanged(): + fresh = (datetime.now(timezone.utc) - timedelta(hours=2)).isoformat() + assert analysis_service.needs_refresh({'change_detected': False, 'last_scraped': fresh}) is False + + +def test_needs_refresh_true_on_malformed_timestamp(): + assert analysis_service.needs_refresh({'change_detected': False, 'last_scraped': 'not-a-date'}) is True + + +def test_analyse_from_cache_raises_without_cached_products(): + _, competitor = _make_tenant_and_competitor() + with pytest.raises(ValueError): + analysis_service.analyse_from_cache('t1', competitor['id'], {}) + + +def test_analyse_from_cache_uses_latest_product(monkeypatch): + tenant, competitor = _make_tenant_and_competitor() + product_repository.create({ + 'tenant_id': tenant['id'], 'competitor_id': competitor['id'], + 'trip_name': 'Peru Classic', 'last_seen': '2026-01-01', + }) + captured = {} + _install_fake_module( + monkeypatch, 'core.extractor', + extract_with_openrouter=lambda prompt, content: captured.update(prompt=prompt) or {'narrative': 'ok'} + ) + + result = analysis_service.analyse_from_cache(tenant['id'], competitor['id'], {'destination': 'Peru'}) + + assert result == {'narrative': 'ok'} + assert 'Peru Classic' in captured['prompt'] + + +def test_run_competitor_analysis_takes_fast_path_when_fresh(monkeypatch): + fresh = (datetime.now(timezone.utc) - timedelta(hours=1)).isoformat() + tenant, competitor = _make_tenant_and_competitor(change_detected=False, last_scraped=fresh) + product_repository.create({ + 'tenant_id': tenant['id'], 'competitor_id': competitor['id'], 'trip_name': 'Peru Classic', + }) + _install_fake_module(monkeypatch, 'core.extractor', extract_with_openrouter=lambda p, content: {'ok': True}) + + result = analysis_service.run_competitor_analysis(tenant['id'], competitor, {}) + + assert result['path'] == 'fast' + + +def test_run_competitor_analysis_takes_refresh_path_and_persists_product(monkeypatch): + tenant, competitor = _make_tenant_and_competitor(change_detected=True) + + fake_page_data = {'success': True, 'url': competitor['website'], 'text': 'scraped content'} + _install_fake_module(monkeypatch, 'core.scraper', scrape=lambda url, country=None: fake_page_data) + _install_fake_module( + monkeypatch, 'core.extractor', + extract_with_openrouter=lambda prompt, content: { + 'tripName': 'Peru Classic', 'duration': 15, 'majorityPrice': 2190, + 'priceLow': 2050, 'priceHigh': 2300, 'comments': 'Solid value', 'relevancy': 4, + } + ) + _install_fake_module( + monkeypatch, 'industries.adventure_travel.prompt', + build_prompt=lambda *a, **k: 'a built prompt' + ) + + result = analysis_service.run_competitor_analysis(tenant['id'], competitor, { + 'destination': 'Peru', 'duration': 15, 'travel_style': 'Classic', + 'product_name': 'Inca Trail', 'tab_type': 'Standard', + }) + + assert result['path'] == 'refresh' + assert result['product']['trip_name'] == 'Peru Classic' + + updated_competitor = competitor_repository.get_by_id(competitor['id']) + assert updated_competitor['change_detected'] is False + assert updated_competitor['last_scraped'] is not None + + +def test_run_competitor_analysis_refresh_path_raises_on_scrape_failure(monkeypatch): + tenant, competitor = _make_tenant_and_competitor(change_detected=True) + _install_fake_module( + monkeypatch, 'core.scraper', + scrape=lambda url, country=None: {'success': False, 'text': 'scrape_blocked'} + ) + _install_fake_module(monkeypatch, 'core.extractor', extract_with_openrouter=lambda p, content: {}) + _install_fake_module(monkeypatch, 'industries.adventure_travel.prompt', build_prompt=lambda *a, **k: '') + + with pytest.raises(ValueError): + analysis_service.run_competitor_analysis(tenant['id'], competitor, {}) diff --git a/backend/tests/services/test_auth_service.py b/backend/tests/services/test_auth_service.py new file mode 100644 index 0000000..1808e61 --- /dev/null +++ b/backend/tests/services/test_auth_service.py @@ -0,0 +1,56 @@ +from services import auth_service +from repositories.tenant_repository import tenant_repository +from repositories.seat_repository import seat_repository + + +class _FakeRequest: + def __init__(self, auth_header=None): + self.headers = {'Authorization': auth_header} if auth_header else {} + + +def test_returns_none_without_bearer_prefix(): + assert auth_service.get_tenant_id_from_request(_FakeRequest('not-bearer-token')) is None + + +def test_returns_none_without_auth_header(): + assert auth_service.get_tenant_id_from_request(_FakeRequest()) is None + + +def test_returns_none_when_pocketbase_rejects_token(monkeypatch): + monkeypatch.setattr('services.auth_service.pb_client.auth_refresh', lambda token: None) + assert auth_service.get_tenant_id_from_request(_FakeRequest('Bearer badtoken')) is None + + +def test_returns_none_when_tenant_not_found(monkeypatch): + monkeypatch.setattr( + 'services.auth_service.pb_client.auth_refresh', + lambda token: {'email': 'pm@unknown.com'} + ) + assert auth_service.get_tenant_id_from_request(_FakeRequest('Bearer sometoken')) is None + + +def test_returns_none_when_email_has_no_active_seat(monkeypatch): + tenant = tenant_repository.create({ + 'name': 'G Adventures', 'domain': 'gadventures.com', 'tier': 'analyse', + 'status': 'active', 'max_seats': 5, + }) + monkeypatch.setattr( + 'services.auth_service.pb_client.auth_refresh', + lambda token: {'email': 'pm@gadventures.com'} + ) + assert auth_service.get_tenant_id_from_request(_FakeRequest('Bearer sometoken')) is None + + +def test_returns_tenant_id_for_valid_active_seat(monkeypatch): + tenant = tenant_repository.create({ + 'name': 'G Adventures', 'domain': 'gadventures.com', 'tier': 'analyse', + 'status': 'active', 'max_seats': 5, + }) + seat_repository.create({'tenant_id': tenant['id'], 'email': 'pm@gadventures.com', 'active': True}) + monkeypatch.setattr( + 'services.auth_service.pb_client.auth_refresh', + lambda token: {'email': 'pm@gadventures.com'} + ) + + result = auth_service.get_tenant_id_from_request(_FakeRequest('Bearer validtoken')) + assert result == tenant['id'] diff --git a/backend/tests/services/test_battlecard_service.py b/backend/tests/services/test_battlecard_service.py new file mode 100644 index 0000000..6730a1e --- /dev/null +++ b/backend/tests/services/test_battlecard_service.py @@ -0,0 +1,75 @@ +import sys +import types +from services import battlecard_service +from repositories.tenant_repository import tenant_repository +from repositories.competitor_repository import competitor_repository +from repositories.product_repository import product_repository +from repositories.battlecard_repository import battlecard_repository + + +def _install_fake_extractor(monkeypatch, fn): + for name in ('core', 'core.extractor'): + if name not in sys.modules: + monkeypatch.setitem(sys.modules, name, types.ModuleType(name)) + sys.modules['core.extractor'].extract_with_openrouter = fn + + +def _setup(): + tenant = tenant_repository.create({ + 'name': 'G Adventures', 'domain': 'gadventures.com', 'tier': 'analyse', + 'status': 'active', 'max_seats': 5, + }) + competitor = competitor_repository.create({ + 'tenant_id': tenant['id'], 'name': 'Intrepid Travel', 'website': 'https://intrepidtravel.com', + 'catalogue_url': 'https://intrepidtravel.com/adventures', 'primary_market': 'GB', 'active': True, + }) + product_repository.create({ + 'tenant_id': tenant['id'], 'competitor_id': competitor['id'], 'trip_name': 'Peru Classic', + 'majority_price_usd': 2190, 'duration_days': 15, + }) + return tenant, competitor + + +def test_generate_battlecard_creates_record(monkeypatch): + tenant, competitor = _setup() + _install_fake_extractor(monkeypatch, lambda prompt, content: { + 'positioning': 'Budget-mid, group focused', + 'strengths': ['Strong brand'], 'weaknesses': ['Basic accommodation'], + 'pricing_observation': 'Discounting Q3', + }) + + card = battlecard_service.generate_battlecard(competitor['id'], tenant['id']) + + assert card is not None + assert card['content_json']['positioning'] == 'Budget-mid, group focused' + assert 'Strong brand' in card['content'] + assert battlecard_repository.get_for_competitor(competitor['id']) is not None + + +def test_generate_battlecard_returns_none_on_extraction_failure(monkeypatch): + tenant, competitor = _setup() + + def _raise(prompt, content): + raise RuntimeError('all models exhausted') + + _install_fake_extractor(monkeypatch, _raise) + + result = battlecard_service.generate_battlecard(competitor['id'], tenant['id']) + assert result is None + + +def test_generate_battlecard_handles_no_products_gracefully(monkeypatch): + tenant = tenant_repository.create({ + 'name': 'G Adventures', 'domain': 'gadventures.com', 'tier': 'analyse', + 'status': 'active', 'max_seats': 5, + }) + competitor = competitor_repository.create({ + 'tenant_id': tenant['id'], 'name': 'New Competitor', 'website': 'https://new.com', + 'catalogue_url': 'https://new.com/trips', 'primary_market': 'US', 'active': True, + }) + _install_fake_extractor(monkeypatch, lambda prompt, content: { + 'positioning': 'Unknown', 'strengths': [], 'weaknesses': [], 'pricing_observation': 'n/a', + }) + + card = battlecard_service.generate_battlecard(competitor['id'], tenant['id']) + assert card is not None diff --git a/backend/tests/services/test_billing_service.py b/backend/tests/services/test_billing_service.py new file mode 100644 index 0000000..d74a395 --- /dev/null +++ b/backend/tests/services/test_billing_service.py @@ -0,0 +1,83 @@ +""" +Stripe webhook handling is a 100%-coverage path per CLAUDE.md §18. +These tests exercise the pure data-transformation logic in +billing_service (no real Stripe API calls are made). +""" +from services import billing_service +from repositories.tenant_repository import tenant_repository + + +def test_checkout_completed_with_upgrade_metadata_updates_tenant(): + tenant = tenant_repository.create({ + 'name': 'G Adventures', 'domain': 'gadventures.com', 'tier': 'analyse', + 'status': 'active', 'max_seats': 5, + }) + + billing_service.handle_checkout_completed({ + 'metadata': {'tenant_id': tenant['id'], 'target_tier': 'discover'}, + 'subscription': 'sub_123', + }) + + updated = tenant_repository.get_by_id(tenant['id']) + assert updated['tier'] == 'discover' + assert updated['max_seats'] == 8 + assert updated['stripe_subscription_id'] == 'sub_123' + assert updated['status'] == 'trial' + assert updated['trial_ends_at'] is not None + + +def test_checkout_completed_without_metadata_is_ignored_not_raised(): + # New-signup checkout sessions (no upgrade metadata) are out of + # scope for this handler — manual onboarding handles new tenants. + # Must not raise. + billing_service.handle_checkout_completed({'metadata': {}}) + + +def test_subscription_updated_changes_tier_when_present_in_metadata(): + tenant = tenant_repository.create({ + 'name': 'G Adventures', 'domain': 'gadventures.com', 'tier': 'analyse', + 'status': 'active', 'max_seats': 5, + }) + + billing_service.handle_subscription_updated({'metadata': {'tier': 'intelligence'}}, tenant['id']) + + updated = tenant_repository.get_by_id(tenant['id']) + assert updated['tier'] == 'intelligence' + assert updated['max_seats'] == 10 + + +def test_subscription_updated_no_op_without_tier_metadata(): + tenant = tenant_repository.create({ + 'name': 'G Adventures', 'domain': 'gadventures.com', 'tier': 'analyse', + 'status': 'active', 'max_seats': 5, + }) + + billing_service.handle_subscription_updated({'metadata': {}}, tenant['id']) + + updated = tenant_repository.get_by_id(tenant['id']) + assert updated['tier'] == 'analyse' + + +def test_subscription_deleted_marks_tenant_inactive(): + tenant = tenant_repository.create({ + 'name': 'G Adventures', 'domain': 'gadventures.com', 'tier': 'analyse', + 'status': 'active', 'max_seats': 5, + }) + + billing_service.handle_subscription_deleted(tenant['id']) + + assert tenant_repository.get_by_id(tenant['id'])['status'] == 'inactive' + + +def test_upgrade_checkout_rejects_unconfigured_tier(monkeypatch): + tenant = tenant_repository.create({ + 'name': 'G Adventures', 'domain': 'gadventures.com', 'tier': 'analyse', + 'status': 'active', 'max_seats': 5, + }) + monkeypatch.delenv('STRIPE_PRICE_DISCOVER', raising=False) + + try: + billing_service.create_upgrade_checkout(tenant['id'], 'discover') + assert False, 'expected ValueError' + except ValueError as e: + assert 'No Stripe price configured' in str(e) diff --git a/backend/tests/services/test_brief_service.py b/backend/tests/services/test_brief_service.py new file mode 100644 index 0000000..cdc55dc --- /dev/null +++ b/backend/tests/services/test_brief_service.py @@ -0,0 +1,57 @@ +import sys +import types +from services import brief_service +from services.brief_service import brief_repository + + +def _install_fake_resend(monkeypatch): + fake = types.ModuleType('resend') + fake.api_key = None + calls = [] + + class _Emails: + @staticmethod + def send(payload): + calls.append(payload) + + fake.Emails = _Emails + monkeypatch.setitem(sys.modules, 'resend', fake) + return calls + + +def test_send_weekly_brief_renders_and_sends(monkeypatch): + # MockBriefRepository keeps its own in-memory tenant store, seeded + # explicitly (it isn't wired to tenant_repository's mock store). + brief_repository.seed_tenant('t1', {'email': 'pm@gadventures.com', 'name': 'G Adventures'}) + calls = _install_fake_resend(monkeypatch) + + brief_service.send_weekly_brief('t1') + + assert len(calls) == 1 + assert calls[0]['to'] == 'pm@gadventures.com' + assert 'Bettersight' in calls[0]['html'] + assert list(brief_repository._logs.values())[-1]['status'] == 'sent' + + +def test_send_weekly_brief_skips_when_no_admin_email(monkeypatch): + calls = _install_fake_resend(monkeypatch) + # 'unknown-tenant' was never seeded — get_weekly_data returns admin_email=None. + brief_service.send_weekly_brief('unknown-tenant') + assert calls == [] + + +def test_send_weekly_brief_logs_failure_when_resend_raises(monkeypatch): + brief_repository.seed_tenant('t2', {'email': 'pm@gadventures.com', 'name': 'G Adventures'}) + + def _raise(payload): + raise RuntimeError('resend api down') + + fake = types.ModuleType('resend') + fake.api_key = None + fake.Emails = type('Emails', (), {'send': staticmethod(_raise)}) + monkeypatch.setitem(sys.modules, 'resend', fake) + + # Must not raise — a failed send is a side-effect failure only. + brief_service.send_weekly_brief('t2') + + assert list(brief_repository._logs.values())[-1]['status'] == 'failed' diff --git a/backend/tests/services/test_embedding_service.py b/backend/tests/services/test_embedding_service.py new file mode 100644 index 0000000..9c30f61 --- /dev/null +++ b/backend/tests/services/test_embedding_service.py @@ -0,0 +1,76 @@ +import sys +import types +from services import embedding_service + + +def test_cosine_similarity_identical_vectors_is_one(): + assert embedding_service._cosine_similarity([1, 0, 0], [1, 0, 0]) == 1.0 + + +def test_cosine_similarity_orthogonal_vectors_is_zero(): + assert embedding_service._cosine_similarity([1, 0], [0, 1]) == 0.0 + + +def test_cosine_similarity_zero_vector_returns_zero(): + assert embedding_service._cosine_similarity([0, 0], [1, 1]) == 0.0 + + +def test_find_comparable_trips_filters_by_threshold_and_sorts(): + client_trips = [ + {'trip_name': 'Inca Trail Classic', 'price_usd': 1999, 'duration_days': 15, 'embedding': [1, 0, 0]}, + ] + competitor_trips = [ + {'id': 'p1', 'majority_price_usd': 2190, 'duration_days': 15, 'embedding': [1, 0, 0]}, # perfect match + {'id': 'p2', 'majority_price_usd': 1800, 'duration_days': 10, 'embedding': [0, 1, 0]}, # orthogonal, filtered out + ] + + matches = embedding_service.find_comparable_trips(client_trips, competitor_trips, threshold=0.75) + + assert len(matches) == 1 + assert matches[0]['competitor_product'] == 'p1' + assert matches[0]['price_difference'] == 2190 - 1999 + assert matches[0]['duration_difference'] == 0 + + +def test_find_comparable_trips_skips_records_without_embeddings(): + client_trips = [{'trip_name': 'x', 'embedding': None}] + competitor_trips = [{'id': 'p1', 'embedding': [1, 0]}] + assert embedding_service.find_comparable_trips(client_trips, competitor_trips) == [] + + +def test_safe_diff_returns_none_when_either_side_missing(): + assert embedding_service._safe_diff(None, 5) is None + assert embedding_service._safe_diff(5, None) is None + assert embedding_service._safe_diff(10, 4) == 6 + + +def test_embed_trip_calls_openrouter_embeddings_api(monkeypatch): + captured = {} + + class _FakeEmbeddingResponse: + class _Data: + embedding = [0.1, 0.2, 0.3] + data = [_Data()] + + class _FakeEmbeddings: + def create(self, model, input): + captured['model'] = model + captured['input'] = input + return _FakeEmbeddingResponse() + + class _FakeOpenAI: + def __init__(self, api_key, base_url): + self.embeddings = _FakeEmbeddings() + + fake_openai_module = types.ModuleType('openai') + fake_openai_module.OpenAI = _FakeOpenAI + monkeypatch.setitem(sys.modules, 'openai', fake_openai_module) + + vector = embedding_service.embed_trip({ + 'trip_name': 'Inca Trail Classic', 'destination': 'Peru', 'duration_days': 15, + 'activities': 'Trekking', 'start_location': 'Cusco', + }) + + assert vector == [0.1, 0.2, 0.3] + assert captured['model'] == embedding_service.EMBEDDING_MODEL + assert 'Peru' in captured['input'] diff --git a/backend/tests/services/test_jobs.py b/backend/tests/services/test_jobs.py new file mode 100644 index 0000000..4af2c5c --- /dev/null +++ b/backend/tests/services/test_jobs.py @@ -0,0 +1,153 @@ +""" +jobs.run_research_job — per-competitor failure isolation, status +transitions, and the slow/stuck Gotify alert thresholds from §29. +tests/ is the conftest-adjacent location; this file lives under +tests/services/ purely for directory tidiness (jobs.py itself is a +top-level module, not part of services/). +""" +import jobs +from repositories.scrape_repository import scrape_repository +from repositories.competitor_repository import competitor_repository +from repositories.tenant_repository import tenant_repository + + +def _setup(max_seats=5): + tenant = tenant_repository.create({ + 'name': 'G Adventures', 'domain': 'gadventures.com', 'tier': 'analyse', + 'status': 'active', 'max_seats': max_seats, + }) + competitor = competitor_repository.create({ + 'tenant_id': tenant['id'], 'name': 'Intrepid Travel', 'website': 'https://intrepidtravel.com', + 'catalogue_url': 'https://intrepidtravel.com/adventures', 'primary_market': 'GB', 'active': True, + }) + run = scrape_repository.create({ + 'tenant_id': tenant['id'], 'triggered_by': 'manual', 'status': 'pending', + 'competitors_total': 1, 'competitors_done': 0, 'results': {}, + }) + return tenant, competitor, run + + +def test_run_research_job_marks_complete_on_success(monkeypatch): + tenant, competitor, run = _setup() + monkeypatch.setattr( + 'services.analysis_service.run_competitor_analysis', + lambda tenant_id, comp, ctx: {'path': 'fast', 'result': {'tripName': 'Peru Classic'}} + ) + gotify_calls = [] + monkeypatch.setattr('services.alert_service.send_gotify', lambda **kw: gotify_calls.append(kw)) + + jobs.run_research_job(run['id'], {'tenant_id': tenant['id'], 'competitors': [{'id': competitor['id']}]}) + + updated = scrape_repository.get_by_id(run['id']) + assert updated['status'] == 'complete' + assert updated['competitors_done'] == 1 + assert updated['results']['success'][0]['name'] == 'Intrepid Travel' + assert gotify_calls == [] # fast completion, no slow/stuck alert + + +def test_run_research_job_isolates_one_competitor_failure(monkeypatch): + tenant = tenant_repository.create({ + 'name': 'G Adventures', 'domain': 'gadventures.com', 'tier': 'analyse', + 'status': 'active', 'max_seats': 5, + }) + good = competitor_repository.create({ + 'tenant_id': tenant['id'], 'name': 'Exodus', 'website': 'https://exodustravels.com', + 'catalogue_url': 'https://exodustravels.com/trips', 'primary_market': 'GB', 'active': True, + }) + bad = competitor_repository.create({ + 'tenant_id': tenant['id'], 'name': 'Flash Pack', 'website': 'https://flashpack.com', + 'catalogue_url': 'https://flashpack.com/adventures', 'primary_market': 'US', 'active': True, + }) + run = scrape_repository.create({ + 'tenant_id': tenant['id'], 'triggered_by': 'manual', 'status': 'pending', + 'competitors_total': 2, 'competitors_done': 0, 'results': {}, + }) + + def _fake_analysis(tenant_id, comp, ctx): + if comp['name'] == 'Flash Pack': + raise ValueError('scrape_blocked') + return {'path': 'fast', 'result': {}} + + monkeypatch.setattr('services.analysis_service.run_competitor_analysis', _fake_analysis) + monkeypatch.setattr('services.alert_service.send_gotify', lambda **kw: None) + + jobs.run_research_job(run['id'], { + 'tenant_id': tenant['id'], 'competitors': [{'id': good['id']}, {'id': bad['id']}] + }) + + updated = scrape_repository.get_by_id(run['id']) + assert updated['status'] == 'complete' # at least one success → overall complete + assert len(updated['results']['success']) == 1 + assert len(updated['results']['failed']) == 1 + assert updated['results']['failed'][0]['name'] == 'Flash Pack' + + +def test_run_research_job_marks_failed_when_all_competitors_fail(monkeypatch): + tenant, competitor, run = _setup() + monkeypatch.setattr( + 'services.analysis_service.run_competitor_analysis', + lambda *a, **k: (_ for _ in ()).throw(ValueError('extract_failed')) + ) + gotify_calls = [] + monkeypatch.setattr('services.alert_service.send_gotify', lambda **kw: gotify_calls.append(kw)) + + jobs.run_research_job(run['id'], {'tenant_id': tenant['id'], 'competitors': [{'id': competitor['id']}]}) + + updated = scrape_repository.get_by_id(run['id']) + assert updated['status'] == 'failed' + assert any('Analysis job failed' in c['title'] for c in gotify_calls) + + +def test_run_research_job_fires_slow_alert_past_threshold(monkeypatch): + tenant, competitor, run = _setup() + monkeypatch.setattr( + 'services.analysis_service.run_competitor_analysis', + lambda *a, **k: {'path': 'fast', 'result': {}} + ) + gotify_calls = [] + monkeypatch.setattr('services.alert_service.send_gotify', lambda **kw: gotify_calls.append(kw)) + + # Force the elapsed-time calculation past the slow threshold without + # an actual 5-minute sleep — patch time.time() to jump forward + # between the job's start and end reads. + real_time = jobs.time.time + times = iter([real_time(), real_time() + jobs.JOB_SLOW_SECONDS + 5]) + monkeypatch.setattr(jobs.time, 'time', lambda: next(times)) + + jobs.run_research_job(run['id'], {'tenant_id': tenant['id'], 'competitors': [{'id': competitor['id']}]}) + + assert any('Slow job detected' in c['title'] for c in gotify_calls) + + +def test_run_research_job_handles_unknown_competitor_id(monkeypatch): + tenant = tenant_repository.create({ + 'name': 'G Adventures', 'domain': 'gadventures.com', 'tier': 'analyse', + 'status': 'active', 'max_seats': 5, + }) + run = scrape_repository.create({ + 'tenant_id': tenant['id'], 'triggered_by': 'manual', 'status': 'pending', + 'competitors_total': 1, 'competitors_done': 0, 'results': {}, + }) + monkeypatch.setattr('services.alert_service.send_gotify', lambda **kw: None) + + jobs.run_research_job(run['id'], {'tenant_id': tenant['id'], 'competitors': [{'id': 'does-not-exist'}]}) + + updated = scrape_repository.get_by_id(run['id']) + assert updated['status'] == 'failed' + assert updated['results']['failed'][0]['error'] == 'Competitor not found' + + +def test_enqueue_research_job_returns_job_id(monkeypatch): + captured = {} + monkeypatch.setattr( + jobs.normal_q, 'enqueue', + lambda fn, job_id, data, job_id_kw=None, **kw: captured.update(job_id=job_id, data=data) + ) + # rq's Queue.enqueue signature takes job_id as a kwarg named "job_id"; + # patch it generically to capture call args regardless of exact form. + monkeypatch.setattr(jobs.normal_q, 'enqueue', lambda *a, **k: captured.update(args=a, kwargs=k)) + + result = jobs.enqueue_research_job('job123', {'tenant_id': 't1'}) + + assert result == 'job123' + assert captured['kwargs']['job_id'] == 'job123' diff --git a/backend/tests/services/test_licence_service.py b/backend/tests/services/test_licence_service.py new file mode 100644 index 0000000..15080e1 --- /dev/null +++ b/backend/tests/services/test_licence_service.py @@ -0,0 +1,65 @@ +""" +Licence validation is a 100%-coverage path per CLAUDE.md §18 — every +branch of services.licence_service.validate_email() is exercised here. +""" +from services import licence_service +from repositories.tenant_repository import tenant_repository +from repositories.seat_repository import seat_repository + + +def _make_tenant(status='active', max_seats=5): + return tenant_repository.create({ + 'name': 'G Adventures', 'email': 'pm@gadventures.com', 'domain': 'gadventures.com', + 'tier': 'analyse', 'status': status, 'max_seats': max_seats, + }) + + +def test_invalid_email_format_rejected(): + result = licence_service.validate_email('not-an-email') + assert result == {'valid': False, 'reason': 'Invalid email address'} + + +def test_unregistered_domain_rejected(): + result = licence_service.validate_email('pm@unknown-domain.com') + assert result['valid'] is False + assert result['reason'] == 'Domain not registered' + + +def test_inactive_tenant_rejected(): + _make_tenant(status='inactive') + result = licence_service.validate_email('pm@gadventures.com') + assert result == {'valid': False, 'reason': 'Subscription inactive'} + + +def test_trial_tenant_is_valid_status(): + _make_tenant(status='trial') + result = licence_service.validate_email('newuser@gadventures.com') + assert result['valid'] is True + assert result['tier'] == 'analyse' + + +def test_new_seat_registered_when_under_limit(): + tenant = _make_tenant(max_seats=5) + result = licence_service.validate_email('newuser@gadventures.com') + assert result == {'valid': True, 'tier': 'analyse'} + assert seat_repository.count_active(tenant['id']) == 1 + + +def test_existing_seat_refreshes_last_accessed_without_creating_duplicate(): + tenant = _make_tenant(max_seats=5) + seat_repository.create({'tenant_id': tenant['id'], 'email': 'pm@gadventures.com', 'active': True}) + + result = licence_service.validate_email('pm@gadventures.com') + + assert result == {'valid': True, 'tier': 'analyse'} + assert seat_repository.count_active(tenant['id']) == 1 + + +def test_seat_limit_reached_rejects_new_seat(): + tenant = _make_tenant(max_seats=1) + seat_repository.create({'tenant_id': tenant['id'], 'email': 'first@gadventures.com', 'active': True}) + + result = licence_service.validate_email('second@gadventures.com') + + assert result == {'valid': False, 'reason': 'Seat limit reached'} + assert seat_repository.count_active(tenant['id']) == 1 diff --git a/backend/tests/services/test_onboarding_service.py b/backend/tests/services/test_onboarding_service.py new file mode 100644 index 0000000..d87da53 --- /dev/null +++ b/backend/tests/services/test_onboarding_service.py @@ -0,0 +1,34 @@ +import pytest +from services import onboarding_service +from repositories.tenant_repository import tenant_repository + + +def test_merges_partial_updates_preserving_existing_keys(): + tenant = tenant_repository.create({ + 'name': 'G Adventures', 'domain': 'gadventures.com', 'tier': 'analyse', + 'status': 'trial', 'max_seats': 5, + 'onboarding_checklist': {**onboarding_service.DEFAULT_CHECKLIST, 'add_competitor': True}, + }) + + result = onboarding_service.update_onboarding(tenant['id'], {'tour_step': 3}) + + assert result['add_competitor'] is True # preserved + assert result['tour_step'] == 3 # updated + assert tenant_repository.get_by_id(tenant['id'])['onboarding_checklist'] == result + + +def test_falls_back_to_default_checklist_when_none_set(): + tenant = tenant_repository.create({ + 'name': 'G Adventures', 'domain': 'gadventures.com', 'tier': 'analyse', + 'status': 'trial', 'max_seats': 5, + }) + + result = onboarding_service.update_onboarding(tenant['id'], {'tour_completed': True}) + + assert result['tour_completed'] is True + assert result['add_competitor'] is False + + +def test_unknown_tenant_raises(): + with pytest.raises(ValueError): + onboarding_service.update_onboarding('does-not-exist', {'tour_completed': True}) diff --git a/backend/tests/services/test_trip_finder_service.py b/backend/tests/services/test_trip_finder_service.py new file mode 100644 index 0000000..c7c9514 --- /dev/null +++ b/backend/tests/services/test_trip_finder_service.py @@ -0,0 +1,62 @@ +from services import trip_finder_service + + +def test_scores_destination_and_duration_match_highest(): + urls = [ + 'https://intrepidtravel.com/peru-classic-15-days', + 'https://intrepidtravel.com/blog/about-us', + 'https://intrepidtravel.com/vietnam-explorer-10-days', + ] + result = trip_finder_service.score_and_rank_urls(urls, 'Peru', 15) + assert result['url'] == 'https://intrepidtravel.com/peru-classic-15-days' + assert result['found'] is True + assert result['confidence'] > 0 + + +def test_no_keyword_match_returns_not_found(): + urls = ['https://intrepidtravel.com/about-us', 'https://intrepidtravel.com/contact'] + result = trip_finder_service.score_and_rank_urls(urls, 'Peru', 15) + assert result == {'url': None, 'confidence': 0, 'found': False} + + +def test_empty_url_list_returns_not_found(): + result = trip_finder_service.score_and_rank_urls([], 'Peru', 15) + assert result == {'url': None, 'confidence': 0, 'found': False} + + +def test_low_confidence_match_is_not_found(): + # Only a weak category-keyword match, no destination/duration signal — + # should fall below CONFIDENCE_FOUND_THRESHOLD (0.3). + urls = ['https://intrepidtravel.com/some-other-trek'] + result = trip_finder_service.score_and_rank_urls(urls, 'Peru', 15) + assert result['found'] is False + + +def test_find_competitor_trip_url_handles_firecrawl_failure_gracefully(monkeypatch): + import requests + + def _raise(*args, **kwargs): + raise requests.RequestException('connection refused') + + monkeypatch.setattr(trip_finder_service.requests, 'post', _raise) + result = trip_finder_service.find_competitor_trip_url('https://intrepidtravel.com', 'Peru', 15, 'Classic') + assert result == {'url': None, 'confidence': 0, 'found': False} + + +def test_find_all_competitor_urls_runs_concurrently_and_preserves_competitor_ids(monkeypatch): + def _fake_find(root_url, destination, duration, travel_style): + return {'url': f'{root_url}/match', 'confidence': 0.8, 'found': True} + + monkeypatch.setattr(trip_finder_service, 'find_competitor_trip_url', _fake_find) + + competitors = [ + {'id': 'c1', 'name': 'Intrepid', 'website': 'https://intrepidtravel.com'}, + {'id': 'c2', 'name': 'Exodus', 'website': 'https://exodustravels.com'}, + ] + results = trip_finder_service.find_all_competitor_urls(competitors, 'Peru', 15, 'Classic') + + assert len(results) == 2 + ids = {r['competitor_id'] for r in results} + assert ids == {'c1', 'c2'} + for r in results: + assert r['found'] is True diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py new file mode 100644 index 0000000..68f6388 --- /dev/null +++ b/backend/tests/test_api.py @@ -0,0 +1,156 @@ +""" +Integration tests for api.py routes, run against mock repositories +(USE_MOCK_REPOSITORIES=true, set in conftest.py before any import). +""" +from repositories.tenant_repository import tenant_repository +from repositories.seat_repository import seat_repository +from repositories.competitor_repository import competitor_repository + + +def _make_tenant(**overrides): + data = { + 'name': 'G Adventures', 'email': 'pm@gadventures.com', 'domain': 'gadventures.com', + 'tier': 'analyse', 'status': 'active', 'max_seats': 5, + } + data.update(overrides) + return tenant_repository.create(data) + + +def test_health_requires_no_auth(client): + response = client.get('/health') + assert response.status_code == 200 + assert response.json == {'status': 'ok', 'version': '0.1.0-phase1'} + + +def test_protected_route_rejects_missing_api_key(client): + response = client.post('/validate-email', json={'email': 'pm@gadventures.com'}) + assert response.status_code == 401 + assert response.json['error'] is True + assert response.json['code'] == 'unauthorized' + + +def test_protected_route_rejects_wrong_api_key(client): + response = client.post( + '/validate-email', json={'email': 'pm@gadventures.com'}, + headers={'X-API-Key': 'wrong-key'} + ) + assert response.status_code == 401 + + +def test_validate_email_happy_path(client, api_headers): + _make_tenant() + response = client.post('/validate-email', json={'email': 'pm@gadventures.com'}, headers=api_headers) + assert response.status_code == 200 + assert response.json == {'valid': True, 'tier': 'analyse'} + + +def test_validate_email_missing_field_returns_structured_error(client, api_headers): + response = client.post('/validate-email', json={}, headers=api_headers) + assert response.status_code == 400 + assert response.json['error'] is True + assert response.json['code'] == 'missing_field' + + +def test_add_seat_rejects_when_tenant_at_seat_limit(client, api_headers): + tenant = _make_tenant(max_seats=1) + seat_repository.create({'tenant_id': tenant['id'], 'email': 'first@gadventures.com', 'active': True}) + + response = client.post( + '/account/seats', json={'tenant_id': tenant['id'], 'email': 'second@gadventures.com'}, + headers=api_headers + ) + + assert response.status_code == 400 + assert response.json['code'] == 'seat_limit_reached' + + +def test_add_seat_succeeds_under_limit(client, api_headers): + tenant = _make_tenant(max_seats=5) + + response = client.post( + '/account/seats', json={'tenant_id': tenant['id'], 'email': 'newuser@gadventures.com'}, + headers=api_headers + ) + + assert response.status_code == 201 + assert seat_repository.count_active(tenant['id']) == 1 + + +def test_remove_seat(client, api_headers): + tenant = _make_tenant() + seat = seat_repository.create({'tenant_id': tenant['id'], 'email': 'x@gadventures.com', 'active': True}) + + response = client.delete(f'/account/seats/{seat["id"]}', headers=api_headers) + + assert response.status_code == 200 + assert seat_repository.count_active(tenant['id']) == 0 + + +def test_get_competitors_resolves_tenant_by_email_header(client, api_headers): + tenant = _make_tenant() + competitor_repository.create({ + 'tenant_id': tenant['id'], 'name': 'Intrepid Travel', 'website': 'https://intrepidtravel.com', + 'catalogue_url': 'https://intrepidtravel.com/adventures', 'primary_market': 'GB', 'active': True, + }) + + response = client.get('/competitors', headers={**api_headers, 'X-User-Email': 'pm@gadventures.com'}) + + assert response.status_code == 200 + body = response.json + assert len(body) == 1 + assert body[0]['name'] == 'Intrepid Travel' + assert body[0]['status'] == 'stale' # never scraped → stale + + +def test_get_competitors_unknown_email_returns_404(client, api_headers): + response = client.get('/competitors', headers={**api_headers, 'X-User-Email': 'nobody@nowhere.com'}) + assert response.status_code == 404 + + +def test_research_find_urls_requires_fields(client, api_headers): + response = client.post('/research/find-urls', json={}, headers=api_headers) + assert response.status_code == 400 + assert response.json['code'] == 'missing_field' + + +def test_research_enqueues_job_without_touching_real_redis(client, api_headers, monkeypatch): + import jobs + tenant = _make_tenant() + captured = {} + + def _fake_enqueue(job_id, data): + captured['job_id'] = job_id + captured['data'] = data + return job_id + + monkeypatch.setattr(jobs, 'enqueue_research_job', _fake_enqueue) + + response = client.post('/research', json={ + 'tenant_id': tenant['id'], 'destination': 'Peru', 'duration': 15, + 'travel_style': 'Classic', 'tab_type': 'Standard', 'competitors': [], + }, headers=api_headers) + + assert response.status_code == 200 + assert response.json['status'] == 'queued' + assert captured['job_id'] == response.json['job_id'] + + +def test_bulk_import_rejects_more_than_fifty_rows(client, api_headers): + tenant = _make_tenant() + rows = [ + {'name': f'Comp {i}', 'website': f'https://comp{i}.com', + 'catalogue_url': f'https://comp{i}.com/trips', 'primary_market': 'GB'} + for i in range(51) + ] + response = client.post( + '/account/competitors/bulk', json={'tenant_id': tenant['id'], 'rows': rows}, headers=api_headers + ) + assert response.status_code == 400 + assert response.json['code'] == 'too_many_rows' + + +def test_competitors_template_download(client, api_headers): + response = client.get('/account/competitors/template', headers=api_headers) + assert response.status_code == 200 + assert response.mimetype == 'text/csv' + assert b'name,website,catalogue_url,primary_market' in response.data diff --git a/backend/tests/test_api_extended.py b/backend/tests/test_api_extended.py new file mode 100644 index 0000000..a6c3d88 --- /dev/null +++ b/backend/tests/test_api_extended.py @@ -0,0 +1,489 @@ +""" +Additional route coverage beyond tests/test_api.py's core happy-path +set — account management, billing, internal (n8n) routes, and the +Stripe webhook signature-verification path. +""" +import sys +import types +import pytest +from repositories.tenant_repository import tenant_repository +from repositories.seat_repository import seat_repository +from repositories.competitor_repository import competitor_repository +from repositories.scrape_repository import scrape_repository +from repositories.battlecard_repository import battlecard_repository +from repositories.comparable_repository import comparable_repository + + +def _make_tenant(**overrides): + data = { + 'name': 'G Adventures', 'email': 'pm@gadventures.com', 'domain': 'gadventures.com', + 'tier': 'analyse', 'status': 'active', 'max_seats': 5, + 'onboarding_checklist': {'add_competitor': False, 'tour_step': 0}, + } + data.update(overrides) + return tenant_repository.create(data) + + +def _install_fake_module(monkeypatch, dotted_name, **attrs): + parts = dotted_name.split('.') + for i in range(1, len(parts) + 1): + partial = '.'.join(parts[:i]) + if partial not in sys.modules: + monkeypatch.setitem(sys.modules, partial, types.ModuleType(partial)) + for key, value in attrs.items(): + setattr(sys.modules[dotted_name], key, value) + + +# ── Account ──────────────────────────────────────────────────────── + +def test_get_account_returns_tenant_with_seats(client, api_headers): + tenant = _make_tenant() + seat_repository.create({'tenant_id': tenant['id'], 'email': 'pm@gadventures.com', 'active': True}) + + response = client.get(f'/account/{tenant["id"]}', headers=api_headers) + + assert response.status_code == 200 + assert response.json['name'] == 'G Adventures' + assert len(response.json['seats']) == 1 + + +def test_get_account_unknown_tenant_404(client, api_headers): + response = client.get('/account/does-not-exist', headers=api_headers) + assert response.status_code == 404 + + +def test_update_competitor(client, api_headers): + tenant = _make_tenant() + competitor = competitor_repository.create({ + 'tenant_id': tenant['id'], 'name': 'Intrepid', 'website': 'https://intrepidtravel.com', + 'catalogue_url': 'https://intrepidtravel.com/adventures', 'primary_market': 'GB', 'active': True, + }) + + response = client.put( + f'/account/competitors/{competitor["id"]}', json={'primary_market': 'US'}, headers=api_headers + ) + + assert response.status_code == 200 + assert response.json['primary_market'] == 'US' + + +def test_update_unknown_competitor_404(client, api_headers): + response = client.put('/account/competitors/nope', json={}, headers=api_headers) + assert response.status_code == 404 + + +def test_delete_competitor_deactivates_and_deregisters(client, api_headers, monkeypatch): + tenant = _make_tenant() + competitor = competitor_repository.create({ + 'tenant_id': tenant['id'], 'name': 'Intrepid', 'website': 'https://intrepidtravel.com', + 'catalogue_url': 'https://intrepidtravel.com/adventures', 'primary_market': 'GB', + 'active': True, 'firecrawl_monitor_id': 'mon_123', + }) + deregistered = [] + monkeypatch.setattr('api.requests.delete', lambda url, timeout: deregistered.append(url)) + + response = client.delete(f'/account/competitors/{competitor["id"]}', headers=api_headers) + + assert response.status_code == 200 + assert competitor_repository.get_by_id(competitor['id'])['active'] is False + assert deregistered + + +def test_delete_unknown_competitor_404(client, api_headers): + response = client.delete('/account/competitors/nope', headers=api_headers) + assert response.status_code == 404 + + +def test_account_template_returns_static_link(client, api_headers): + response = client.get('/account/template', headers=api_headers) + assert response.status_code == 200 + assert 'template_url' in response.json + + +def test_onboarding_patch_requires_resolvable_session(client, api_headers): + response = client.patch('/account/onboarding', json={'tour_step': 2}, headers=api_headers) + assert response.status_code == 401 + + +def test_onboarding_patch_updates_checklist(client, api_headers, monkeypatch): + tenant = _make_tenant() + monkeypatch.setattr('api.auth_service.get_tenant_id_from_request', lambda req: tenant['id']) + + response = client.patch( + '/account/onboarding', json={'tour_step': 2}, + headers={**api_headers, 'Authorization': 'Bearer sometoken'} + ) + + assert response.status_code == 200 + assert response.json['onboarding_checklist']['tour_step'] == 2 + + +def test_delete_account_requires_tenant_id(client, api_headers): + response = client.post('/account/delete', json={}, headers=api_headers) + assert response.status_code == 400 + + +def test_delete_account_marks_inactive_and_deactivates_competitors(client, api_headers): + tenant = _make_tenant() + competitor = competitor_repository.create({ + 'tenant_id': tenant['id'], 'name': 'Intrepid', 'website': 'https://intrepidtravel.com', + 'catalogue_url': 'https://intrepidtravel.com/adventures', 'primary_market': 'GB', 'active': True, + }) + seat_repository.create({'tenant_id': tenant['id'], 'email': 'pm@gadventures.com', 'active': True}) + + response = client.post('/account/delete', json={'tenant_id': tenant['id']}, headers=api_headers) + + assert response.status_code == 200 + assert tenant_repository.get_by_id(tenant['id'])['status'] == 'inactive' + assert competitor_repository.get_by_id(competitor['id'])['active'] is False + assert seat_repository.list_for_tenant(tenant['id']) == [] + + +# ── Billing ──────────────────────────────────────────────────────── + +def test_billing_portal_requires_session(client, api_headers): + response = client.get('/billing/portal', headers=api_headers) + assert response.status_code == 401 + + +def test_billing_portal_no_stripe_customer_returns_400(client, api_headers, monkeypatch): + tenant = _make_tenant() + monkeypatch.setattr('api.auth_service.get_tenant_id_from_request', lambda req: tenant['id']) + + response = client.get( + '/billing/portal', headers={**api_headers, 'Authorization': 'Bearer x'} + ) + assert response.status_code == 400 + assert response.json['code'] == 'no_stripe_customer' + + +def test_billing_upgrade_requires_fields(client, api_headers): + response = client.post('/billing/upgrade', json={}, headers=api_headers) + assert response.status_code == 400 + + +def test_billing_upgrade_unconfigured_tier_returns_400(client, api_headers): + tenant = _make_tenant() + response = client.post( + '/billing/upgrade', json={'tenant_id': tenant['id'], 'target_tier': 'discover'}, headers=api_headers + ) + assert response.status_code == 400 + assert response.json['code'] == 'upgrade_failed' + + +# ── Internal (n8n) ───────────────────────────────────────────────── + +def test_internal_battlecard_requires_tenant_id(client, api_headers): + response = client.post('/internal/battlecard/c1', json={}, headers=api_headers) + assert response.status_code == 400 + + +def test_internal_battlecard_generates_card(client, api_headers, monkeypatch): + tenant = _make_tenant() + competitor = competitor_repository.create({ + 'tenant_id': tenant['id'], 'name': 'Intrepid', 'website': 'https://intrepidtravel.com', + 'catalogue_url': 'https://intrepidtravel.com/adventures', 'primary_market': 'GB', 'active': True, + }) + _install_fake_module( + monkeypatch, 'core.extractor', + extract_with_openrouter=lambda prompt, content: { + 'positioning': 'x', 'strengths': [], 'weaknesses': [], 'pricing_observation': 'y' + } + ) + + response = client.post( + f'/internal/battlecard/{competitor["id"]}', json={'tenant_id': tenant['id']}, headers=api_headers + ) + + assert response.status_code == 200 + assert response.json['content_json']['positioning'] == 'x' + + +def test_internal_weekly_brief(client, api_headers, monkeypatch): + called = [] + monkeypatch.setattr('api.brief_service.send_weekly_brief', lambda tenant_id: called.append(tenant_id)) + response = client.post('/internal/weekly-brief/t1', headers=api_headers) + assert response.status_code == 200 + assert called == ['t1'] + + +def test_internal_monitor_webhook_flags_competitor(client, api_headers): + tenant = _make_tenant() + competitor = competitor_repository.create({ + 'tenant_id': tenant['id'], 'name': 'Intrepid', 'website': 'https://intrepidtravel.com', + 'catalogue_url': 'https://intrepidtravel.com/adventures', 'primary_market': 'GB', 'active': True, + }) + + response = client.post( + '/internal/monitor-webhook', json={'url': 'https://intrepidtravel.com'}, headers=api_headers + ) + + assert response.status_code == 200 + assert competitor_repository.get_by_id(competitor['id'])['change_detected'] is True + + +def test_internal_monitor_webhook_unknown_url_is_noop(client, api_headers): + response = client.post( + '/internal/monitor-webhook', json={'url': 'https://unknown.com'}, headers=api_headers + ) + assert response.status_code == 200 + + +def test_internal_monitor_register_and_deregister(client, api_headers, monkeypatch): + tenant = _make_tenant() + competitor = competitor_repository.create({ + 'tenant_id': tenant['id'], 'name': 'Intrepid', 'website': 'https://intrepidtravel.com', + 'catalogue_url': 'https://intrepidtravel.com/adventures', 'primary_market': 'GB', + 'active': True, 'firecrawl_monitor_id': 'mon_1', + }) + monkeypatch.setattr('api.requests.post', lambda *a, **k: types.SimpleNamespace(json=lambda: {'id': 'mon_2'})) + monkeypatch.setattr('api.requests.delete', lambda *a, **k: None) + + r1 = client.post(f'/internal/monitor/register/{competitor["id"]}', headers=api_headers) + assert r1.status_code == 200 + + r2 = client.post(f'/internal/monitor/deregister/{competitor["id"]}', headers=api_headers) + assert r2.status_code == 200 + + +def test_internal_monitor_register_unknown_competitor_404(client, api_headers): + response = client.post('/internal/monitor/register/nope', headers=api_headers) + assert response.status_code == 404 + + +def test_internal_scrape_enqueues_job(client, api_headers, monkeypatch): + import jobs + tenant = _make_tenant() + captured = {} + monkeypatch.setattr(jobs, 'enqueue_research_job', lambda job_id, data: captured.update(job_id=job_id)) + + response = client.post(f'/internal/scrape/{tenant["id"]}', json={'competitors': []}, headers=api_headers) + + assert response.status_code == 200 + assert response.json['job_id'] == captured['job_id'] + + +def test_internal_embed_products(client, api_headers, monkeypatch): + from repositories.product_repository import product_repository + tenant = _make_tenant() + competitor = competitor_repository.create({ + 'tenant_id': tenant['id'], 'name': 'Intrepid', 'website': 'https://intrepidtravel.com', + 'catalogue_url': 'https://intrepidtravel.com/adventures', 'primary_market': 'GB', 'active': True, + }) + product_repository.create({ + 'tenant_id': tenant['id'], 'competitor_id': competitor['id'], 'trip_name': 'Peru Classic', + }) + monkeypatch.setattr('services.embedding_service.embed_trip', lambda trip: [0.1, 0.2]) + + response = client.post(f'/internal/embed-products/{tenant["id"]}', headers=api_headers) + + assert response.status_code == 200 + assert response.json['embedded'] == 1 + + +def test_internal_match_comparable(client, api_headers, monkeypatch): + tenant = _make_tenant() + monkeypatch.setattr( + 'services.embedding_service.find_comparable_trips', + lambda client_trips, competitor_trips, threshold=0.75: [ + {'client_product': 'x', 'competitor_product': 'p1', 'similarity_score': 0.9, + 'price_difference': 10, 'duration_difference': 0} + ] + ) + + response = client.post( + f'/internal/match-comparable/{tenant["id"]}', json={'client_trips': []}, headers=api_headers + ) + + assert response.status_code == 200 + assert response.json['matches_created'] == 1 + assert len(comparable_repository.list_for_tenant(tenant['id'])) == 1 + + +# ── Sheet (Phase 4 dependency — expect structured 501) ───────────── + +def test_sheet_push_returns_not_yet_available(client, api_headers): + tenant = _make_tenant() + run = scrape_repository.create({ + 'tenant_id': tenant['id'], 'triggered_by': 'manual', 'status': 'complete', + 'competitors_total': 0, 'competitors_done': 0, 'results': {}, + }) + response = client.post( + '/sheet/push', json={'email': 'pm@gadventures.com', 'job_id': run['id'], 'tab_name': 'x'}, + headers=api_headers + ) + assert response.status_code == 501 + + +def test_sheet_tabs_returns_not_yet_available(client, api_headers): + _make_tenant() + response = client.get('/sheet/tabs', headers={**api_headers, 'X-User-Email': 'pm@gadventures.com'}) + assert response.status_code == 501 + + +# ── Research history / status / battlecards / comparable-trips ──── + +def test_research_status_unknown_job_404(client, api_headers): + response = client.get('/research/status/nope', headers=api_headers) + assert response.status_code == 404 + + +def test_research_status_known_job(client, api_headers): + tenant = _make_tenant() + run = scrape_repository.create({ + 'tenant_id': tenant['id'], 'triggered_by': 'manual', 'status': 'running', + 'competitors_total': 2, 'competitors_done': 1, 'results': {}, + }) + response = client.get(f'/research/status/{run["id"]}', headers=api_headers) + assert response.status_code == 200 + assert response.json['progress'] == {'total': 2, 'done': 1} + + +def test_research_history_detail_unknown_404(client, api_headers): + response = client.get('/research/history/nope', headers=api_headers) + assert response.status_code == 404 + + +def test_research_history_for_tenant(client, api_headers): + tenant = _make_tenant() + scrape_repository.create({ + 'tenant_id': tenant['id'], 'triggered_by': 'manual', 'status': 'complete', + 'competitors_total': 1, 'competitors_done': 1, 'results': {}, + }) + response = client.get('/research/history', query_string={'email': 'pm@gadventures.com'}, headers=api_headers) + assert response.status_code == 200 + assert len(response.json) == 1 + + +def test_battlecards_route(client, api_headers): + tenant = _make_tenant() + competitor = competitor_repository.create({ + 'tenant_id': tenant['id'], 'name': 'Intrepid', 'website': 'https://intrepidtravel.com', + 'catalogue_url': 'https://intrepidtravel.com/adventures', 'primary_market': 'GB', 'active': True, + }) + battlecard_repository.upsert(competitor['id'], {'tenant_id': tenant['id'], 'content': 'x', 'content_json': {}}) + + response = client.post('/battlecards', json={'email': 'pm@gadventures.com'}, headers=api_headers) + + assert response.status_code == 200 + assert response.json[0]['competitor_name'] == 'Intrepid' + + +def test_comparable_trips_route(client, api_headers): + tenant = _make_tenant() + comparable_repository.create({'tenant_id': tenant['id'], 'client_product': 'x', 'similarity_score': 0.8}) + + response = client.post('/comparable-trips', json={'email': 'pm@gadventures.com'}, headers=api_headers) + + assert response.status_code == 200 + assert len(response.json) == 1 + + +def test_preview_prompt(client, api_headers, monkeypatch): + _install_fake_module( + monkeypatch, 'industries.adventure_travel.prompt', + build_prompt=lambda *a, **k: 'a fake built prompt' + ) + response = client.post('/preview-prompt', json={ + 'productName': 'Inca Trail', 'destination': 'Peru', 'duration': 15, 'travelStyle': 'Classic', + }, headers=api_headers) + assert response.status_code == 200 + assert response.json['prompt'] == 'a fake built prompt' + + +# ── Auth password reset proxy routes ──────────────────────────────── + +def test_request_password_reset_missing_email(client, api_headers): + response = client.post('/auth/request-password-reset', json={}, headers=api_headers) + assert response.status_code == 400 + + +def test_request_password_reset_ok(client, api_headers, monkeypatch): + monkeypatch.setattr('api.requests.post', lambda *a, **k: types.SimpleNamespace(status_code=204)) + response = client.post( + '/auth/request-password-reset', json={'email': 'pm@gadventures.com'}, headers=api_headers + ) + assert response.status_code == 200 + + +def test_confirm_password_reset_missing_fields(client, api_headers): + response = client.post('/auth/confirm-password-reset', json={'token': 'x'}, headers=api_headers) + assert response.status_code == 400 + + +def test_confirm_password_reset_invalid_token(client, api_headers, monkeypatch): + monkeypatch.setattr('api.requests.post', lambda *a, **k: types.SimpleNamespace(status_code=400)) + response = client.post('/auth/confirm-password-reset', json={ + 'token': 'bad', 'password': 'newpass123', 'passwordConfirm': 'newpass123', + }, headers=api_headers) + assert response.status_code == 400 + assert response.json['code'] == 'reset_failed' + + +def test_confirm_password_reset_ok(client, api_headers, monkeypatch): + monkeypatch.setattr('api.requests.post', lambda *a, **k: types.SimpleNamespace(status_code=200)) + response = client.post('/auth/confirm-password-reset', json={ + 'token': 'good', 'password': 'newpass123', 'passwordConfirm': 'newpass123', + }, headers=api_headers) + assert response.status_code == 200 + + +# ── Stripe webhook ────────────────────────────────────────────────── + +def test_stripe_webhook_invalid_signature_rejected(client, monkeypatch): + import stripe as stripe_module + + def _raise(*a, **k): + raise ValueError('bad signature') + + monkeypatch.setattr(stripe_module.Webhook, 'construct_event', staticmethod(_raise)) + + response = client.post( + '/stripe/webhook', data=b'{}', headers={'Stripe-Signature': 'bad'} + ) + assert response.status_code == 400 + assert response.json['code'] == 'invalid_signature' + + +def test_stripe_webhook_dispatches_checkout_completed(client, monkeypatch): + import stripe as stripe_module + tenant = _make_tenant() + + fake_event = { + 'type': 'checkout.session.completed', + 'data': {'object': { + 'metadata': {'tenant_id': tenant['id'], 'target_tier': 'discover'}, + 'subscription': 'sub_1', + }}, + } + monkeypatch.setattr(stripe_module.Webhook, 'construct_event', staticmethod(lambda *a, **k: fake_event)) + + response = client.post('/stripe/webhook', data=b'{}', headers={'Stripe-Signature': 'ok'}) + + assert response.status_code == 200 + assert tenant_repository.get_by_id(tenant['id'])['tier'] == 'discover' + + +def test_stripe_webhook_dispatches_subscription_deleted(client, monkeypatch): + import stripe as stripe_module + tenant = _make_tenant(stripe_customer_id='cus_123') + + fake_event = { + 'type': 'customer.subscription.deleted', + 'data': {'object': {'customer': 'cus_123'}}, + } + monkeypatch.setattr(stripe_module.Webhook, 'construct_event', staticmethod(lambda *a, **k: fake_event)) + + response = client.post('/stripe/webhook', data=b'{}', headers={'Stripe-Signature': 'ok'}) + + assert response.status_code == 200 + assert tenant_repository.get_by_id(tenant['id'])['status'] == 'inactive' + + +def test_stripe_webhook_unhandled_event_type_ok(client, monkeypatch): + import stripe as stripe_module + fake_event = {'type': 'invoice.paid', 'data': {'object': {}}} + monkeypatch.setattr(stripe_module.Webhook, 'construct_event', staticmethod(lambda *a, **k: fake_event)) + + response = client.post('/stripe/webhook', data=b'{}', headers={'Stripe-Signature': 'ok'}) + assert response.status_code == 200