import uuid from repositories.pocketbase_client import pb_client from repositories.repo_config import USE_MOCK_REPOSITORIES COLLECTION = 'scrape_runs' class ScrapeRepository: """Data access for the `scrape_runs` collection — also the RQ job_id record.""" def create(self, data): return pb_client.create(COLLECTION, data) def update(self, job_id, data): return pb_client.update(COLLECTION, job_id, data) def get_by_id(self, job_id): return pb_client.get_one(COLLECTION, job_id) def list_recent_for_tenant(self, tenant_id, limit=5): """ Returns a tenant's most recent BATCH runs, newest first — backs GET /research/history. Filters to triggered_by="manual" — analysis_service.run_competitor_analysis()'s refresh path creates its own extra scrape_runs row PER COMPETITOR purely to persist a content_hash for next time (hardcoded triggered_by='cron', competitors_total=1, results={}), regardless of what triggered the parent job. Without this filter those bookkeeping rows — created within the same batch, so their started_at timestamps sort ahead of or alongside the real job — drowned out the actual batch runs in "last 5 runs", with misleading competitor counts and empty `results` that broke Restore. 'manual' is the only triggered_by value that represents a real, user-visible batch run today; if a genuine scheduled *batch* job (distinct from this per-competitor stub) is ever introduced, this filter needs revisiting. """ return pb_client.list( COLLECTION, filter_str=f'tenant_id = "{tenant_id}" && triggered_by = "manual"', sort='-started_at', per_page=limit ) def get_latest_complete_for_tenant(self, tenant_id): """ Returns the tenant's most recent completed BATCH run, or None — backs GET /research/history/latest (the Apps Script sidebar's "Pull latest results", CLAUDE.md §15). Same triggered_by="manual" filter as list_recent_for_tenant() and for the same reason — the per-competitor content-hash bookkeeping rows are also created with status='complete', so without this filter the Sheet-pull integration could silently pull a 1-competitor bookkeeping stub instead of the PM's actual last analysis. """ items = pb_client.list( COLLECTION, filter_str=f'tenant_id = "{tenant_id}" && triggered_by = "manual" && status = "complete"', sort='-started_at', per_page=1 ) return items[0] if items else None def get_last_hash_for_url(self, competitor_id, url): """ Returns the most recently recorded content_hash for this exact competitor+url combination, or None if this specific page has never been hashed before. Backs detect_change()'s content-hash comparison. Scoped by url as well as competitor_id — CLAUDE.md §8's trip-intent matching means one competitor can have many different confirmed trip URLs scraped over time. Comparing against whichever page happened to be scraped MOST RECENTLY for this competitor (the old, url-blind lookup) meant a fresh scrape of URL A got compared against URL B's hash whenever B was scraped more recently — almost always a mismatch, which kept change_detected stuck True and made the true cache-only fast path unreachable in practice. Confirmed directly against real scrape_runs history before this fix. """ safe_url = url.replace('"', '\\"') items = pb_client.list( COLLECTION, filter_str=f'competitor_id = "{competitor_id}" && scraped_url = "{safe_url}" && content_hash != ""', sort='-started_at', per_page=1 ) return items[0]['content_hash'] if items else None def list_all_for_tenant(self, tenant_id): """Every scrape_runs row for a tenant, no limit — used for GDPR erasure.""" return pb_client.list(COLLECTION, filter_str=f'tenant_id = "{tenant_id}"', per_page=500) def list_older_than(self, cutoff_iso): """Used by the weekly data-cleanup job — CLAUDE.md §22 DATA_RETENTION (90 days for scrape_runs).""" return pb_client.list(COLLECTION, filter_str=f'started_at < "{cutoff_iso}"', per_page=500) def delete(self, job_id): return pb_client.delete(COLLECTION, job_id) class MockScrapeRepository: """In-memory stand-in for ScrapeRepository.""" def __init__(self): self._records = {} # Real PocketBase auto-populates started_at via autodate(true) — # every "most recent first" sort in this class relies on that. # A monotonic counter (rather than datetime.now()) guarantees # correct ordering across records created within the same test, # regardless of clock resolution. self._sequence = 0 def create(self, data): record_id = data.get('id') or str(uuid.uuid4()) self._sequence += 1 record = {'id': record_id, 'started_at': f'{self._sequence:020d}', **data} self._records[record_id] = record return record def update(self, job_id, data): if job_id not in self._records: return None self._records[job_id] = {**self._records[job_id], **data} return self._records[job_id] def get_by_id(self, job_id): return self._records.get(job_id) def list_recent_for_tenant(self, tenant_id, limit=5): items = [ r for r in self._records.values() if r.get('tenant_id') == tenant_id and r.get('triggered_by') == 'manual' ] items.sort(key=lambda r: r.get('started_at') or '', reverse=True) return items[:limit] def get_latest_complete_for_tenant(self, tenant_id): items = [ r for r in self._records.values() if r.get('tenant_id') == tenant_id and r.get('triggered_by') == 'manual' and r.get('status') == 'complete' ] items.sort(key=lambda r: r.get('started_at') or '', reverse=True) return items[0] if items else None def get_last_hash_for_url(self, competitor_id, url): items = [ r for r in self._records.values() if r.get('competitor_id') == competitor_id and r.get('scraped_url') == url and r.get('content_hash') ] items.sort(key=lambda r: r.get('started_at') or '', reverse=True) return items[0]['content_hash'] if items else None def list_all_for_tenant(self, tenant_id): return [r for r in self._records.values() if r.get('tenant_id') == tenant_id] def list_older_than(self, cutoff_iso): return [r for r in self._records.values() if (r.get('started_at') or '') < cutoff_iso] def delete(self, job_id): return self._records.pop(job_id, None) is not None scrape_repository = MockScrapeRepository() if USE_MOCK_REPOSITORIES else ScrapeRepository()