41 lines
1.9 KiB
JavaScript
41 lines
1.9 KiB
JavaScript
/// <reference path="../pb_data/types.d.ts" />
|
|
//
|
|
// Adds `scraped_url` to `scrape_runs` — fixes a real bug where content-hash
|
|
// change detection was scoped only by competitor_id, not by which specific
|
|
// page was actually scraped. Once trip-intent matching (CLAUDE.md §8) let a
|
|
// PM scrape different trip pages for the same competitor across different
|
|
// analysis runs, get_last_hash_for_url(competitor_id) compared each new
|
|
// scrape's hash against whatever page happened to be scraped MOST
|
|
// RECENTLY for that competitor — almost always a different page — so
|
|
// change_detected kept getting set back to True on nearly every run, and
|
|
// needs_refresh() (which gates the fast path on change_detected being
|
|
// False) never let the true cache-only fast path trigger in practice.
|
|
// Confirmed directly against real scrape_runs history: the same
|
|
// competitor's recorded content_hash changed on 5 of 6 consecutive scrapes
|
|
// over one test session, even though re-scraping the exact same URL twice
|
|
// in immediate succession produced an identical hash both times.
|
|
//
|
|
// See services/analysis_service.py's detect_change()/needs_refresh()/
|
|
// run_competitor_analysis() and repositories/scrape_repository.py's
|
|
// get_last_hash_for_url() for the corresponding code fix.
|
|
|
|
migrate((app) => {
|
|
const collection = app.findCollectionByNameOrId('scrape_runs');
|
|
|
|
collection.fields.add(new Field({ type: 'text', name: 'scraped_url' }));
|
|
|
|
collection.indexes = [
|
|
...collection.indexes,
|
|
'CREATE INDEX idx_scrape_runs_url_hash ON scrape_runs (competitor_id, scraped_url, content_hash, started_at)',
|
|
];
|
|
|
|
return app.save(collection);
|
|
}, (app) => {
|
|
const collection = app.findCollectionByNameOrId('scrape_runs');
|
|
|
|
collection.fields.removeByName('scraped_url');
|
|
collection.indexes = collection.indexes.filter((idx) => !idx.includes('idx_scrape_runs_url_hash'));
|
|
|
|
return app.save(collection);
|
|
});
|