import uuid from repositories.pocketbase_client import pb_client from repositories.repo_config import USE_MOCK_REPOSITORIES COLLECTION = 'scrape_runs' class ScrapeRepository: """Data access for the `scrape_runs` collection — also the RQ job_id record.""" def create(self, data): return pb_client.create(COLLECTION, data) def update(self, job_id, data): return pb_client.update(COLLECTION, job_id, data) def get_by_id(self, job_id): return pb_client.get_one(COLLECTION, job_id) def list_recent_for_tenant(self, tenant_id, limit=5): """ Returns a tenant's most recent BATCH runs, newest first — backs GET /research/history. Filters to triggered_by="manual" — analysis_service.run_competitor_analysis()'s refresh path creates its own extra scrape_runs row PER COMPETITOR purely to persist a content_hash for next time (hardcoded triggered_by='cron', competitors_total=1, results={}), regardless of what triggered the parent job. Without this filter those bookkeeping rows — created within the same batch, so their started_at timestamps sort ahead of or alongside the real job — drowned out the actual batch runs in "last 5 runs", with misleading competitor counts and empty `results` that broke Restore. 'manual' is the only triggered_by value that represents a real, user-visible batch run today; if a genuine scheduled *batch* job (distinct from this per-competitor stub) is ever introduced, this filter needs revisiting. """ return pb_client.list( COLLECTION, filter_str=f'tenant_id = "{tenant_id}" && triggered_by = "manual"', sort='-started_at', per_page=limit ) def get_latest_complete_for_tenant(self, tenant_id): """ Returns the tenant's most recent completed BATCH run, or None — backs GET /research/history/latest (the Apps Script sidebar's "Pull latest results", CLAUDE.md §15). Same triggered_by="manual" filter as list_recent_for_tenant() and for the same reason — the per-competitor content-hash bookkeeping rows are also created with status='complete', so without this filter the Sheet-pull integration could silently pull a 1-competitor bookkeeping stub instead of the PM's actual last analysis. """ items = pb_client.list( COLLECTION, filter_str=f'tenant_id = "{tenant_id}" && triggered_by = "manual" && status = "complete"', sort='-started_at', per_page=1 ) return items[0] if items else None def get_last_hash_for_url(self, competitor_id): """ Returns the most recently recorded content_hash for this competitor's prior scrape, or None if it has never been hashed before. Backs detect_change()'s content-hash comparison. """ items = pb_client.list( COLLECTION, filter_str=f'competitor_id = "{competitor_id}" && content_hash != ""', sort='-started_at', per_page=1 ) return items[0]['content_hash'] if items else None def list_all_for_tenant(self, tenant_id): """Every scrape_runs row for a tenant, no limit — used for GDPR erasure.""" return pb_client.list(COLLECTION, filter_str=f'tenant_id = "{tenant_id}"', per_page=500) def list_older_than(self, cutoff_iso): """Used by the weekly data-cleanup job — CLAUDE.md §22 DATA_RETENTION (90 days for scrape_runs).""" return pb_client.list(COLLECTION, filter_str=f'started_at < "{cutoff_iso}"', per_page=500) def delete(self, job_id): return pb_client.delete(COLLECTION, job_id) class MockScrapeRepository: """In-memory stand-in for ScrapeRepository.""" def __init__(self): self._records = {} # Real PocketBase auto-populates started_at via autodate(true) — # every "most recent first" sort in this class relies on that. # A monotonic counter (rather than datetime.now()) guarantees # correct ordering across records created within the same test, # regardless of clock resolution. self._sequence = 0 def create(self, data): record_id = data.get('id') or str(uuid.uuid4()) self._sequence += 1 record = {'id': record_id, 'started_at': f'{self._sequence:020d}', **data} self._records[record_id] = record return record def update(self, job_id, data): if job_id not in self._records: return None self._records[job_id] = {**self._records[job_id], **data} return self._records[job_id] def get_by_id(self, job_id): return self._records.get(job_id) def list_recent_for_tenant(self, tenant_id, limit=5): items = [ r for r in self._records.values() if r.get('tenant_id') == tenant_id and r.get('triggered_by') == 'manual' ] items.sort(key=lambda r: r.get('started_at') or '', reverse=True) return items[:limit] def get_latest_complete_for_tenant(self, tenant_id): items = [ r for r in self._records.values() if r.get('tenant_id') == tenant_id and r.get('triggered_by') == 'manual' and r.get('status') == 'complete' ] items.sort(key=lambda r: r.get('started_at') or '', reverse=True) return items[0] if items else None def get_last_hash_for_url(self, competitor_id): items = [ r for r in self._records.values() if r.get('competitor_id') == competitor_id and r.get('content_hash') ] items.sort(key=lambda r: r.get('started_at') or '', reverse=True) return items[0]['content_hash'] if items else None def list_all_for_tenant(self, tenant_id): return [r for r in self._records.values() if r.get('tenant_id') == tenant_id] def list_older_than(self, cutoff_iso): return [r for r in self._records.values() if (r.get('started_at') or '') < cutoff_iso] def delete(self, job_id): return self._records.pop(job_id, None) is not None scrape_repository = MockScrapeRepository() if USE_MOCK_REPOSITORIES else ScrapeRepository()