app.bettersight.io/backend/repositories/scrape_repository.py

import uuid
from repositories.pocketbase_client import pb_client
from repositories.repo_config import USE_MOCK_REPOSITORIES

COLLECTION = 'scrape_runs'


class ScrapeRepository:
    """Data access for the `scrape_runs` collection — also the RQ job_id record."""

    def create(self, data):
        return pb_client.create(COLLECTION, data)

    def update(self, job_id, data):
        return pb_client.update(COLLECTION, job_id, data)

    def get_by_id(self, job_id):
        return pb_client.get_one(COLLECTION, job_id)

    def list_recent_for_tenant(self, tenant_id, limit=5):
        """
        Returns a tenant's most recent BATCH runs, newest first — backs
        GET /research/history. Filters to triggered_by="manual" —
        analysis_service.run_competitor_analysis()'s refresh path creates
        its own extra scrape_runs row PER COMPETITOR purely to persist a
        content_hash for next time (hardcoded triggered_by='cron',
        competitors_total=1, results={}), regardless of what triggered the
        parent job. Without this filter those bookkeeping rows — created
        within the same batch, so their started_at timestamps sort ahead
        of or alongside the real job — drowned out the actual batch runs
        in "last 5 runs", with misleading competitor counts and empty
        `results` that broke Restore. 'manual' is the only triggered_by
        value that represents a real, user-visible batch run today; if a
        genuine scheduled *batch* job (distinct from this per-competitor
        stub) is ever introduced, this filter needs revisiting.
        """
        return pb_client.list(
            COLLECTION, filter_str=f'tenant_id = "{tenant_id}" && triggered_by = "manual"',
            sort='-started_at', per_page=limit
        )

    def get_latest_complete_for_tenant(self, tenant_id):
        """
        Returns the tenant's most recent completed BATCH run, or None —
        backs GET /research/history/latest (the Apps Script sidebar's
        "Pull latest results", CLAUDE.md §15). Same triggered_by="manual"
        filter as list_recent_for_tenant() and for the same reason — the
        per-competitor content-hash bookkeeping rows are also created with
        status='complete', so without this filter the Sheet-pull
        integration could silently pull a 1-competitor bookkeeping stub
        instead of the PM's actual last analysis.
        """
        items = pb_client.list(
            COLLECTION, filter_str=f'tenant_id = "{tenant_id}" && triggered_by = "manual" && status = "complete"',
            sort='-started_at', per_page=1
        )
        return items[0] if items else None

    def get_last_hash_for_url(self, competitor_id, url):
        """
        Returns the most recently recorded content_hash for this exact
        competitor+url combination, or None if this specific page has
        never been hashed before. Backs detect_change()'s content-hash
        comparison.

        Scoped by url as well as competitor_id — CLAUDE.md §8's
        trip-intent matching means one competitor can have many
        different confirmed trip URLs scraped over time. Comparing
        against whichever page happened to be scraped MOST RECENTLY for
        this competitor (the old, url-blind lookup) meant a fresh scrape
        of URL A got compared against URL B's hash whenever B was
        scraped more recently — almost always a mismatch, which kept
        change_detected stuck True and made the true cache-only fast
        path unreachable in practice. Confirmed directly against real
        scrape_runs history before this fix.
        """
        safe_url = url.replace('"', '\\"')
        items = pb_client.list(
            COLLECTION,
            filter_str=f'competitor_id = "{competitor_id}" && scraped_url = "{safe_url}" && content_hash != ""',
            sort='-started_at', per_page=1
        )
        return items[0]['content_hash'] if items else None

    def list_all_for_tenant(self, tenant_id):
        """Every scrape_runs row for a tenant, no limit — used for GDPR erasure."""
        return pb_client.list(COLLECTION, filter_str=f'tenant_id = "{tenant_id}"', per_page=500)

    def list_older_than(self, cutoff_iso):
        """Used by the weekly data-cleanup job — CLAUDE.md §22 DATA_RETENTION (90 days for scrape_runs)."""
        return pb_client.list(COLLECTION, filter_str=f'started_at < "{cutoff_iso}"', per_page=500)

    def delete(self, job_id):
        return pb_client.delete(COLLECTION, job_id)


class MockScrapeRepository:
    """In-memory stand-in for ScrapeRepository."""

    def __init__(self):
        self._records = {}
        # Real PocketBase auto-populates started_at via autodate(true) —
        # every "most recent first" sort in this class relies on that.
        # A monotonic counter (rather than datetime.now()) guarantees
        # correct ordering across records created within the same test,
        # regardless of clock resolution.
        self._sequence = 0

    def create(self, data):
        record_id = data.get('id') or str(uuid.uuid4())
        self._sequence += 1
        record = {'id': record_id, 'started_at': f'{self._sequence:020d}', **data}
        self._records[record_id] = record
        return record

    def update(self, job_id, data):
        if job_id not in self._records:
            return None
        self._records[job_id] = {**self._records[job_id], **data}
        return self._records[job_id]

    def get_by_id(self, job_id):
        return self._records.get(job_id)

    def list_recent_for_tenant(self, tenant_id, limit=5):
        items = [
            r for r in self._records.values()
            if r.get('tenant_id') == tenant_id and r.get('triggered_by') == 'manual'
        ]
        items.sort(key=lambda r: r.get('started_at') or '', reverse=True)
        return items[:limit]

    def get_latest_complete_for_tenant(self, tenant_id):
        items = [
            r for r in self._records.values()
            if r.get('tenant_id') == tenant_id and r.get('triggered_by') == 'manual' and r.get('status') == 'complete'
        ]
        items.sort(key=lambda r: r.get('started_at') or '', reverse=True)
        return items[0] if items else None

    def get_last_hash_for_url(self, competitor_id, url):
        items = [
            r for r in self._records.values()
            if r.get('competitor_id') == competitor_id and r.get('scraped_url') == url and r.get('content_hash')
        ]
        items.sort(key=lambda r: r.get('started_at') or '', reverse=True)
        return items[0]['content_hash'] if items else None

    def list_all_for_tenant(self, tenant_id):
        return [r for r in self._records.values() if r.get('tenant_id') == tenant_id]

    def list_older_than(self, cutoff_iso):
        return [r for r in self._records.values() if (r.get('started_at') or '') < cutoff_iso]

    def delete(self, job_id):
        return self._records.pop(job_id, None) is not None


scrape_repository = MockScrapeRepository() if USE_MOCK_REPOSITORIES else ScrapeRepository()