147 lines
6.1 KiB
Python
147 lines
6.1 KiB
Python
import uuid
|
|
from repositories.pocketbase_client import pb_client
|
|
from repositories.repo_config import USE_MOCK_REPOSITORIES
|
|
|
|
COLLECTION = 'scrape_runs'
|
|
|
|
|
|
class ScrapeRepository:
|
|
"""Data access for the `scrape_runs` collection — also the RQ job_id record."""
|
|
|
|
def create(self, data):
|
|
return pb_client.create(COLLECTION, data)
|
|
|
|
def update(self, job_id, data):
|
|
return pb_client.update(COLLECTION, job_id, data)
|
|
|
|
def get_by_id(self, job_id):
|
|
return pb_client.get_one(COLLECTION, job_id)
|
|
|
|
def list_recent_for_tenant(self, tenant_id, limit=5):
|
|
"""
|
|
Returns a tenant's most recent BATCH runs, newest first — backs
|
|
GET /research/history. Filters to triggered_by="manual" —
|
|
analysis_service.run_competitor_analysis()'s refresh path creates
|
|
its own extra scrape_runs row PER COMPETITOR purely to persist a
|
|
content_hash for next time (hardcoded triggered_by='cron',
|
|
competitors_total=1, results={}), regardless of what triggered the
|
|
parent job. Without this filter those bookkeeping rows — created
|
|
within the same batch, so their started_at timestamps sort ahead
|
|
of or alongside the real job — drowned out the actual batch runs
|
|
in "last 5 runs", with misleading competitor counts and empty
|
|
`results` that broke Restore. 'manual' is the only triggered_by
|
|
value that represents a real, user-visible batch run today; if a
|
|
genuine scheduled *batch* job (distinct from this per-competitor
|
|
stub) is ever introduced, this filter needs revisiting.
|
|
"""
|
|
return pb_client.list(
|
|
COLLECTION, filter_str=f'tenant_id = "{tenant_id}" && triggered_by = "manual"',
|
|
sort='-started_at', per_page=limit
|
|
)
|
|
|
|
def get_latest_complete_for_tenant(self, tenant_id):
|
|
"""
|
|
Returns the tenant's most recent completed BATCH run, or None —
|
|
backs GET /research/history/latest (the Apps Script sidebar's
|
|
"Pull latest results", CLAUDE.md §15). Same triggered_by="manual"
|
|
filter as list_recent_for_tenant() and for the same reason — the
|
|
per-competitor content-hash bookkeeping rows are also created with
|
|
status='complete', so without this filter the Sheet-pull
|
|
integration could silently pull a 1-competitor bookkeeping stub
|
|
instead of the PM's actual last analysis.
|
|
"""
|
|
items = pb_client.list(
|
|
COLLECTION, filter_str=f'tenant_id = "{tenant_id}" && triggered_by = "manual" && status = "complete"',
|
|
sort='-started_at', per_page=1
|
|
)
|
|
return items[0] if items else None
|
|
|
|
def get_last_hash_for_url(self, competitor_id):
|
|
"""
|
|
Returns the most recently recorded content_hash for this
|
|
competitor's prior scrape, or None if it has never been hashed
|
|
before. Backs detect_change()'s content-hash comparison.
|
|
"""
|
|
items = pb_client.list(
|
|
COLLECTION,
|
|
filter_str=f'competitor_id = "{competitor_id}" && content_hash != ""',
|
|
sort='-started_at', per_page=1
|
|
)
|
|
return items[0]['content_hash'] if items else None
|
|
|
|
def list_all_for_tenant(self, tenant_id):
|
|
"""Every scrape_runs row for a tenant, no limit — used for GDPR erasure."""
|
|
return pb_client.list(COLLECTION, filter_str=f'tenant_id = "{tenant_id}"', per_page=500)
|
|
|
|
def list_older_than(self, cutoff_iso):
|
|
"""Used by the weekly data-cleanup job — CLAUDE.md §22 DATA_RETENTION (90 days for scrape_runs)."""
|
|
return pb_client.list(COLLECTION, filter_str=f'started_at < "{cutoff_iso}"', per_page=500)
|
|
|
|
def delete(self, job_id):
|
|
return pb_client.delete(COLLECTION, job_id)
|
|
|
|
|
|
class MockScrapeRepository:
|
|
"""In-memory stand-in for ScrapeRepository."""
|
|
|
|
def __init__(self):
|
|
self._records = {}
|
|
# Real PocketBase auto-populates started_at via autodate(true) —
|
|
# every "most recent first" sort in this class relies on that.
|
|
# A monotonic counter (rather than datetime.now()) guarantees
|
|
# correct ordering across records created within the same test,
|
|
# regardless of clock resolution.
|
|
self._sequence = 0
|
|
|
|
def create(self, data):
|
|
record_id = data.get('id') or str(uuid.uuid4())
|
|
self._sequence += 1
|
|
record = {'id': record_id, 'started_at': f'{self._sequence:020d}', **data}
|
|
self._records[record_id] = record
|
|
return record
|
|
|
|
def update(self, job_id, data):
|
|
if job_id not in self._records:
|
|
return None
|
|
self._records[job_id] = {**self._records[job_id], **data}
|
|
return self._records[job_id]
|
|
|
|
def get_by_id(self, job_id):
|
|
return self._records.get(job_id)
|
|
|
|
def list_recent_for_tenant(self, tenant_id, limit=5):
|
|
items = [
|
|
r for r in self._records.values()
|
|
if r.get('tenant_id') == tenant_id and r.get('triggered_by') == 'manual'
|
|
]
|
|
items.sort(key=lambda r: r.get('started_at') or '', reverse=True)
|
|
return items[:limit]
|
|
|
|
def get_latest_complete_for_tenant(self, tenant_id):
|
|
items = [
|
|
r for r in self._records.values()
|
|
if r.get('tenant_id') == tenant_id and r.get('triggered_by') == 'manual' and r.get('status') == 'complete'
|
|
]
|
|
items.sort(key=lambda r: r.get('started_at') or '', reverse=True)
|
|
return items[0] if items else None
|
|
|
|
def get_last_hash_for_url(self, competitor_id):
|
|
items = [
|
|
r for r in self._records.values()
|
|
if r.get('competitor_id') == competitor_id and r.get('content_hash')
|
|
]
|
|
items.sort(key=lambda r: r.get('started_at') or '', reverse=True)
|
|
return items[0]['content_hash'] if items else None
|
|
|
|
def list_all_for_tenant(self, tenant_id):
|
|
return [r for r in self._records.values() if r.get('tenant_id') == tenant_id]
|
|
|
|
def list_older_than(self, cutoff_iso):
|
|
return [r for r in self._records.values() if (r.get('started_at') or '') < cutoff_iso]
|
|
|
|
def delete(self, job_id):
|
|
return self._records.pop(job_id, None) is not None
|
|
|
|
|
|
scrape_repository = MockScrapeRepository() if USE_MOCK_REPOSITORIES else ScrapeRepository()
|