762 lines
34 KiB
Python
762 lines
34 KiB
Python
"""
|
|
analysis_service orchestrates the fast/refresh path decision (CLAUDE.md
|
|
§7). It calls into core.scraper / core.extractor /
|
|
industries.adventure_travel.prompt, all of which are now real Phase 2
|
|
modules — _install_fake_module imports the real module (Phase 2
|
|
guarantees it exists on disk) and swaps out just the functions this
|
|
test needs via monkeypatch.setattr (properly reverted after each test).
|
|
"""
|
|
import importlib
|
|
from datetime import datetime, timedelta, timezone
|
|
import pytest
|
|
from services import analysis_service
|
|
from repositories.competitor_repository import competitor_repository
|
|
from repositories.product_repository import product_repository
|
|
from repositories.tenant_repository import tenant_repository
|
|
from repositories.alert_repository import alert_repository
|
|
from repositories.price_history_repository import price_history_repository
|
|
from repositories.scrape_repository import scrape_repository
|
|
from repositories.ngs_meta_repository import ngs_meta_repository
|
|
|
|
|
|
def _install_fake_module(monkeypatch, dotted_name, **attrs):
|
|
"""
|
|
Imports the real module at `dotted_name` (every module this helper
|
|
is ever called with — core.scraper, core.extractor,
|
|
industries.adventure_travel.prompt — is a real, importable Phase 2
|
|
module) and patches the given attributes via monkeypatch.setattr,
|
|
which records the original value and restores it on teardown.
|
|
|
|
A previous version of this helper created a *blank fake* module
|
|
when `dotted_name` wasn't already in sys.modules, rather than
|
|
importing the real one — harmless back when these were Phase 1
|
|
stand-ins for modules that didn't exist yet, but broken now: it
|
|
left monkeypatch.setattr patching an empty module lacking the very
|
|
attribute being set, raising AttributeError. Since Phase 2 these
|
|
modules always exist, always import for real.
|
|
"""
|
|
target = importlib.import_module(dotted_name)
|
|
for key, value in attrs.items():
|
|
monkeypatch.setattr(target, key, value)
|
|
return target
|
|
|
|
|
|
def _make_tenant_and_competitor(**competitor_overrides):
|
|
tenant = tenant_repository.create({
|
|
'name': 'G Adventures', 'domain': 'gadventures.com', 'tier': 'analyse',
|
|
'status': 'active', 'max_seats': 5,
|
|
})
|
|
data = {
|
|
'tenant_id': tenant['id'], 'name': 'Intrepid Travel', 'website': 'https://intrepidtravel.com',
|
|
'catalogue_url': 'https://intrepidtravel.com/adventures', 'primary_market': 'GB', 'active': True,
|
|
}
|
|
data.update(competitor_overrides)
|
|
competitor = competitor_repository.create(data)
|
|
return tenant, competitor
|
|
|
|
|
|
def test_needs_refresh_true_when_change_detected():
|
|
assert analysis_service.needs_refresh({'change_detected': True, 'last_scraped': None}) is True
|
|
|
|
|
|
def test_needs_refresh_true_when_never_scraped():
|
|
assert analysis_service.needs_refresh({'change_detected': False, 'last_scraped': None}) is True
|
|
|
|
|
|
def test_needs_refresh_true_when_stale():
|
|
stale = (datetime.now(timezone.utc) - timedelta(hours=48)).isoformat()
|
|
assert analysis_service.needs_refresh({'change_detected': False, 'last_scraped': stale}) is True
|
|
|
|
|
|
def test_needs_refresh_false_when_fresh_and_unchanged():
|
|
fresh = (datetime.now(timezone.utc) - timedelta(hours=2)).isoformat()
|
|
assert analysis_service.needs_refresh({'change_detected': False, 'last_scraped': fresh}) is False
|
|
|
|
|
|
def test_needs_refresh_true_on_malformed_timestamp():
|
|
assert analysis_service.needs_refresh({'change_detected': False, 'last_scraped': 'not-a-date'}) is True
|
|
|
|
|
|
def test_analyse_from_cache_raises_without_cached_products():
|
|
_, competitor = _make_tenant_and_competitor()
|
|
with pytest.raises(ValueError):
|
|
analysis_service.analyse_from_cache('t1', competitor['id'], {})
|
|
|
|
|
|
def test_analyse_from_cache_uses_latest_product(monkeypatch):
|
|
tenant, competitor = _make_tenant_and_competitor()
|
|
product_repository.create({
|
|
'tenant_id': tenant['id'], 'competitor_id': competitor['id'],
|
|
'trip_name': 'Peru Classic', 'last_seen': '2026-01-01',
|
|
})
|
|
captured = {}
|
|
_install_fake_module(
|
|
monkeypatch, 'core.extractor',
|
|
extract_with_openrouter=lambda prompt, content: captured.update(prompt=prompt) or {'comments': 'ok', 'relevancy': 4}
|
|
)
|
|
|
|
result = analysis_service.analyse_from_cache(tenant['id'], competitor['id'], {'destination': 'Peru'})
|
|
|
|
assert result['tripName'] == 'Peru Classic'
|
|
assert result['comments'] == 'ok'
|
|
assert result['relevancy'] == 4
|
|
assert 'Peru Classic' in captured['prompt']
|
|
|
|
|
|
def test_analyse_from_cache_returns_structured_fields_resultstable_expects(monkeypatch):
|
|
"""
|
|
Regression test — a real "complete" job showed an empty-looking
|
|
results table because analyse_from_cache() returned only whatever
|
|
ad-hoc shape the LLM invented for a bare narrative prompt, never
|
|
the cached product's own tripName/price/duration fields that
|
|
ResultsTable.vue actually renders (item.result.tripName,
|
|
.majorityPrice/.gbpPrice, .duration, .comments).
|
|
"""
|
|
tenant, competitor = _make_tenant_and_competitor()
|
|
product_repository.create({
|
|
'tenant_id': tenant['id'], 'competitor_id': competitor['id'],
|
|
'trip_name': 'Inca Trail Classic', 'trip_code': 'INC15',
|
|
'duration_days': 15, 'majority_price_usd': 2780,
|
|
'price_low_usd': 2500, 'price_high_usd': 3100,
|
|
'gbp_price': 2190, 'last_seen': '2026-01-01',
|
|
})
|
|
_install_fake_module(
|
|
monkeypatch, 'core.extractor',
|
|
extract_with_openrouter=lambda prompt, content: {'comments': 'strong match', 'relevancy': 5}
|
|
)
|
|
|
|
result = analysis_service.analyse_from_cache(tenant['id'], competitor['id'], {'destination': 'Peru'})
|
|
|
|
assert result['tripName'] == 'Inca Trail Classic'
|
|
assert result['tripCode'] == 'INC15'
|
|
assert result['duration'] == 15
|
|
assert result['majorityPrice'] == 2780
|
|
assert result['priceLow'] == 2500
|
|
assert result['priceHigh'] == 3100
|
|
assert result['gbpPrice'] == 2190
|
|
assert result['comments'] == 'strong match'
|
|
assert result['relevancy'] == 5
|
|
|
|
|
|
def test_analyse_from_cache_falls_back_to_cached_relevancy_when_llm_omits_it(monkeypatch):
|
|
tenant, competitor = _make_tenant_and_competitor()
|
|
product_repository.create({
|
|
'tenant_id': tenant['id'], 'competitor_id': competitor['id'],
|
|
'trip_name': 'Peru Classic', 'relevancy': 3, 'last_seen': '2026-01-01',
|
|
})
|
|
_install_fake_module(
|
|
monkeypatch, 'core.extractor',
|
|
extract_with_openrouter=lambda prompt, content: {'comments': 'ok'} # no relevancy key
|
|
)
|
|
|
|
result = analysis_service.analyse_from_cache(tenant['id'], competitor['id'], {'destination': 'Peru'})
|
|
|
|
assert result['relevancy'] == 3
|
|
|
|
|
|
def test_run_competitor_analysis_takes_fast_path_when_fresh(monkeypatch):
|
|
fresh = (datetime.now(timezone.utc) - timedelta(hours=1)).isoformat()
|
|
tenant, competitor = _make_tenant_and_competitor(change_detected=False, last_scraped=fresh)
|
|
product_repository.create({
|
|
'tenant_id': tenant['id'], 'competitor_id': competitor['id'], 'trip_name': 'Peru Classic',
|
|
})
|
|
_install_fake_module(monkeypatch, 'core.extractor', extract_with_openrouter=lambda p, content: {'ok': True})
|
|
|
|
result = analysis_service.run_competitor_analysis(tenant['id'], competitor, {})
|
|
|
|
assert result['path'] == 'fast'
|
|
|
|
|
|
def test_run_competitor_analysis_takes_refresh_path_and_persists_product(monkeypatch):
|
|
tenant, competitor = _make_tenant_and_competitor(change_detected=True)
|
|
|
|
fake_page_data = {'success': True, 'url': competitor['website'], 'text': 'scraped content'}
|
|
_install_fake_module(monkeypatch, 'core.scraper', scrape=lambda url, country=None: fake_page_data)
|
|
_install_fake_module(
|
|
monkeypatch, 'core.extractor',
|
|
extract_with_openrouter=lambda prompt, content: {
|
|
'tripName': 'Peru Classic', 'duration': 15, 'majorityPrice': 2190,
|
|
'priceLow': 2050, 'priceHigh': 2300, 'comments': 'Solid value', 'relevancy': 4,
|
|
}
|
|
)
|
|
_install_fake_module(
|
|
monkeypatch, 'industries.adventure_travel.prompt',
|
|
build_prompt=lambda *a, **k: 'a built prompt'
|
|
)
|
|
|
|
result = analysis_service.run_competitor_analysis(tenant['id'], competitor, {
|
|
'destination': 'Peru', 'duration': 15, 'travel_style': 'Classic',
|
|
'product_name': 'Inca Trail', 'tab_type': 'Standard',
|
|
})
|
|
|
|
assert result['path'] == 'refresh'
|
|
assert result['product']['trip_name'] == 'Peru Classic'
|
|
|
|
updated_competitor = competitor_repository.get_by_id(competitor['id'])
|
|
# change_detected is intentionally NOT reset here — detection and
|
|
# extraction happen in the same scrape now, so it stays visible
|
|
# (status badges, next cron run) until a later scrape confirms no
|
|
# further change. See test_run_competitor_analysis_refresh_path_
|
|
# clears_change_detected_when_confirmed_stable below.
|
|
assert updated_competitor['change_detected'] is True
|
|
assert updated_competitor['last_scraped'] is not None
|
|
|
|
# No prior product existed for this competitor+trip_name — this is
|
|
# a brand-new-product discovery: is_new flag set, alert created.
|
|
assert result['product']['is_new'] is True
|
|
assert result['product']['first_seen'] is not None
|
|
alerts = alert_repository.list_recent(tenant['id'])
|
|
assert len(alerts) == 1
|
|
assert alerts[0]['alert_type'] == 'new_product'
|
|
assert alerts[0]['product_id'] == result['product']['id']
|
|
|
|
|
|
def test_run_competitor_analysis_uses_confirmed_url_over_competitor_website(monkeypatch):
|
|
# Regression test — run_competitor_analysis() used to always scrape
|
|
# competitor['website'] unconditionally, silently discarding whatever
|
|
# trip-specific URL was actually matched/confirmed at UrlConfirmation.vue
|
|
# for this run. context['confirmed_url'] must win when present.
|
|
tenant, competitor = _make_tenant_and_competitor(change_detected=True)
|
|
captured_urls = []
|
|
|
|
def fake_scrape(url, country=None):
|
|
captured_urls.append(url)
|
|
return {'success': True, 'url': url, 'text': 'scraped content'}
|
|
|
|
_install_fake_module(monkeypatch, 'core.scraper', scrape=fake_scrape)
|
|
_install_fake_module(
|
|
monkeypatch, 'core.extractor',
|
|
extract_with_openrouter=lambda prompt, content: {'tripName': 'Peru Classic', 'majorityPrice': 2190}
|
|
)
|
|
_install_fake_module(monkeypatch, 'industries.adventure_travel.prompt', build_prompt=lambda *a, **k: '')
|
|
|
|
analysis_service.run_competitor_analysis(tenant['id'], competitor, {
|
|
'confirmed_url': 'https://intrepidtravel.com/peru-classic-15-days',
|
|
})
|
|
|
|
assert captured_urls == ['https://intrepidtravel.com/peru-classic-15-days']
|
|
assert captured_urls[0] != competitor['website']
|
|
|
|
|
|
def test_run_competitor_analysis_falls_back_to_website_without_confirmed_url(monkeypatch):
|
|
tenant, competitor = _make_tenant_and_competitor(change_detected=True)
|
|
captured_urls = []
|
|
|
|
def fake_scrape(url, country=None):
|
|
captured_urls.append(url)
|
|
return {'success': True, 'url': url, 'text': 'scraped content'}
|
|
|
|
_install_fake_module(monkeypatch, 'core.scraper', scrape=fake_scrape)
|
|
_install_fake_module(
|
|
monkeypatch, 'core.extractor',
|
|
extract_with_openrouter=lambda prompt, content: {'tripName': 'Peru Classic', 'majorityPrice': 2190}
|
|
)
|
|
_install_fake_module(monkeypatch, 'industries.adventure_travel.prompt', build_prompt=lambda *a, **k: '')
|
|
|
|
analysis_service.run_competitor_analysis(tenant['id'], competitor, {})
|
|
|
|
assert captured_urls == [competitor['website']]
|
|
|
|
|
|
def test_run_competitor_analysis_flags_generic_page_for_root_domain(monkeypatch):
|
|
tenant, competitor = _make_tenant_and_competitor(change_detected=True)
|
|
fake_page_data = {'success': True, 'url': 'https://intrepidtravel.com', 'text': 'homepage content'}
|
|
_install_fake_module(monkeypatch, 'core.scraper', scrape=lambda url, country=None: fake_page_data)
|
|
_install_fake_module(
|
|
monkeypatch, 'core.extractor',
|
|
extract_with_openrouter=lambda prompt, content: {'tripName': ''}
|
|
)
|
|
_install_fake_module(monkeypatch, 'industries.adventure_travel.prompt', build_prompt=lambda *a, **k: '')
|
|
|
|
result = analysis_service.run_competitor_analysis(tenant['id'], competitor, {
|
|
'confirmed_url': 'https://intrepidtravel.com',
|
|
})
|
|
|
|
assert result['is_generic_page'] is True
|
|
|
|
|
|
def test_run_competitor_analysis_no_generic_page_flag_for_trip_specific_url(monkeypatch):
|
|
tenant, competitor = _make_tenant_and_competitor(change_detected=True)
|
|
fake_page_data = {
|
|
'success': True, 'url': 'https://intrepidtravel.com/peru-classic-15-days', 'text': 'trip page content',
|
|
}
|
|
_install_fake_module(monkeypatch, 'core.scraper', scrape=lambda url, country=None: fake_page_data)
|
|
_install_fake_module(
|
|
monkeypatch, 'core.extractor',
|
|
extract_with_openrouter=lambda prompt, content: {'tripName': 'Peru Classic'}
|
|
)
|
|
_install_fake_module(monkeypatch, 'industries.adventure_travel.prompt', build_prompt=lambda *a, **k: '')
|
|
|
|
result = analysis_service.run_competitor_analysis(tenant['id'], competitor, {
|
|
'confirmed_url': 'https://intrepidtravel.com/peru-classic-15-days',
|
|
})
|
|
|
|
assert result['is_generic_page'] is False
|
|
|
|
|
|
def test_run_competitor_analysis_fast_path_flags_generic_page_from_cached_url(monkeypatch):
|
|
fresh = (datetime.now(timezone.utc) - timedelta(hours=1)).isoformat()
|
|
tenant, competitor = _make_tenant_and_competitor(change_detected=False, last_scraped=fresh)
|
|
product_repository.create({
|
|
'tenant_id': tenant['id'], 'competitor_id': competitor['id'],
|
|
'trip_name': '', 'url': 'https://intrepidtravel.com',
|
|
})
|
|
_install_fake_module(monkeypatch, 'core.extractor', extract_with_openrouter=lambda p, content: {'ok': True})
|
|
|
|
result = analysis_service.run_competitor_analysis(tenant['id'], competitor, {})
|
|
|
|
assert result['path'] == 'fast'
|
|
assert result['is_generic_page'] is True
|
|
|
|
|
|
def test_save_scrape_result_updates_existing_product_without_duplicating(monkeypatch):
|
|
tenant, competitor = _make_tenant_and_competitor()
|
|
existing = product_repository.create({
|
|
'tenant_id': tenant['id'], 'competitor_id': competitor['id'],
|
|
'trip_name': 'Peru Classic', 'majority_price_usd': 2190, 'is_new': True,
|
|
})
|
|
|
|
extracted = {'tripName': 'Peru Classic', 'majorityPrice': 2190, 'duration': 15}
|
|
product = analysis_service._save_scrape_result(
|
|
tenant['id'], competitor, extracted, {'url': competitor['website']}
|
|
)
|
|
|
|
assert product['id'] == existing['id'] # updated in place, not duplicated
|
|
assert product['is_new'] is False
|
|
assert price_history_repository.list_for_product(product['id']) == []
|
|
assert alert_repository.list_recent(tenant['id']) == []
|
|
|
|
|
|
def test_save_scrape_result_creates_price_history_and_alert_above_threshold(monkeypatch):
|
|
tenant, competitor = _make_tenant_and_competitor()
|
|
existing = product_repository.create({
|
|
'tenant_id': tenant['id'], 'competitor_id': competitor['id'],
|
|
'trip_name': 'Peru Classic', 'majority_price_usd': 2490, 'is_new': False,
|
|
})
|
|
|
|
extracted = {'tripName': 'Peru Classic', 'majorityPrice': 2190, 'duration': 15} # -12%
|
|
product = analysis_service._save_scrape_result(
|
|
tenant['id'], competitor, extracted, {'url': competitor['website']}
|
|
)
|
|
|
|
assert product['id'] == existing['id']
|
|
assert product['majority_price_usd'] == 2190
|
|
|
|
history = price_history_repository.list_for_product(product['id'])
|
|
assert len(history) == 1
|
|
assert history[0]['change_amount'] == -300
|
|
assert history[0]['change_percent'] == -12.0
|
|
|
|
alerts = alert_repository.list_recent(tenant['id'])
|
|
assert len(alerts) == 1
|
|
assert alerts[0]['alert_type'] == 'price_change'
|
|
assert 'dropped' in alerts[0]['message']
|
|
assert alerts[0]['change_percent'] == -12.0
|
|
|
|
|
|
def test_save_scrape_result_price_change_below_threshold_no_alert(monkeypatch):
|
|
tenant, competitor = _make_tenant_and_competitor()
|
|
product_repository.create({
|
|
'tenant_id': tenant['id'], 'competitor_id': competitor['id'],
|
|
'trip_name': 'Peru Classic', 'majority_price_usd': 2200, 'is_new': False,
|
|
})
|
|
|
|
extracted = {'tripName': 'Peru Classic', 'majorityPrice': 2190, 'duration': 15} # ~-0.45%
|
|
product = analysis_service._save_scrape_result(
|
|
tenant['id'], competitor, extracted, {'url': competitor['website']}
|
|
)
|
|
|
|
# Price history is still recorded (it genuinely changed)...
|
|
assert len(price_history_repository.list_for_product(product['id'])) == 1
|
|
# ...but a sub-threshold move doesn't warrant an alert.
|
|
assert alert_repository.list_recent(tenant['id']) == []
|
|
|
|
|
|
def test_save_scrape_result_unchanged_price_no_history_no_alert(monkeypatch):
|
|
tenant, competitor = _make_tenant_and_competitor()
|
|
product_repository.create({
|
|
'tenant_id': tenant['id'], 'competitor_id': competitor['id'],
|
|
'trip_name': 'Peru Classic', 'majority_price_usd': 2190, 'is_new': False,
|
|
})
|
|
|
|
extracted = {'tripName': 'Peru Classic', 'majorityPrice': 2190, 'duration': 15, 'comments': 'refreshed'}
|
|
product = analysis_service._save_scrape_result(
|
|
tenant['id'], competitor, extracted, {'url': competitor['website']}
|
|
)
|
|
|
|
assert product['comments'] == 'refreshed' # still updated with fresh fields
|
|
assert price_history_repository.list_for_product(product['id']) == []
|
|
assert alert_repository.list_recent(tenant['id']) == []
|
|
|
|
|
|
def test_save_scrape_result_price_raised_alert_wording(monkeypatch):
|
|
tenant, competitor = _make_tenant_and_competitor()
|
|
product_repository.create({
|
|
'tenant_id': tenant['id'], 'competitor_id': competitor['id'],
|
|
'trip_name': 'Machu Picchu Trek', 'majority_price_usd': 1850, 'is_new': False,
|
|
})
|
|
|
|
extracted = {'tripName': 'Machu Picchu Trek', 'majorityPrice': 2050, 'duration': 8} # +10.8%
|
|
analysis_service._save_scrape_result(tenant['id'], competitor, extracted, {'url': competitor['website']})
|
|
|
|
alerts = alert_repository.list_recent(tenant['id'])
|
|
assert len(alerts) == 1
|
|
assert 'raised' in alerts[0]['message']
|
|
assert alerts[0]['change_percent'] > 0
|
|
|
|
|
|
def test_save_scrape_result_persists_full_field_set():
|
|
"""
|
|
Regression test — _save_scrape_result()'s common_fields dict used to
|
|
silently drop service_level/group_size/meals/start_location/
|
|
end_location/departures/seasonality/start_days/target_audience/
|
|
hotels/activities/trip_code on write, even though every one of these
|
|
already has a products column and the LLM already returns them
|
|
(industries/adventure_travel/prompt.py's build_prompt() schema).
|
|
ResultsTable.vue's transposed sheet-layout grid needs every one of
|
|
these to render (frontend/src/config/resultFields.js).
|
|
"""
|
|
tenant, competitor = _make_tenant_and_competitor()
|
|
extracted = {
|
|
'tripName': 'Inca Trail Classic', 'tripCode': 'INC15',
|
|
'serviceLevel': 'Premium', 'groupSize': 12, 'duration': 15, 'meals': 14,
|
|
'startLocation': 'Lima', 'endLocation': 'Cusco', 'departures': 52,
|
|
'seasonality': 'Year-round', 'startDays': 'Saturday, Monday',
|
|
'targetAudience': 'Premium travellers', 'hotels': 'Comfortable hotels',
|
|
'activities': 'City tours, Machu Picchu', 'majorityPrice': 2780, 'duration_days': 15,
|
|
}
|
|
product = analysis_service._save_scrape_result(
|
|
tenant['id'], competitor, extracted, {'url': competitor['website']}
|
|
)
|
|
|
|
assert product['trip_code'] == 'INC15'
|
|
assert product['service_level'] == 'Premium'
|
|
assert product['group_size'] == 12
|
|
assert product['meals'] == 14
|
|
assert product['start_location'] == 'Lima'
|
|
assert product['end_location'] == 'Cusco'
|
|
assert product['departures'] == 52
|
|
assert product['seasonality'] == 'Year-round'
|
|
assert product['start_days'] == 'Saturday, Monday'
|
|
assert product['target_audience'] == 'Premium travellers'
|
|
assert product['hotels'] == 'Comfortable hotels'
|
|
assert product['activities'] == 'City tours, Machu Picchu'
|
|
|
|
|
|
def test_save_scrape_result_upserts_ngs_meta_when_ngs_fields_present():
|
|
tenant, competitor = _make_tenant_and_competitor()
|
|
extracted = {
|
|
'tripName': 'Nat Geo Signature', 'majorityPrice': 4500,
|
|
'exclusiveAccess': 'After-hours museum access',
|
|
'groupLeader': 'Local archaeologist guide',
|
|
'sustainability': 'Carbon-offset flights included',
|
|
}
|
|
product = analysis_service._save_scrape_result(
|
|
tenant['id'], competitor, extracted, {'url': competitor['website']}
|
|
)
|
|
|
|
meta = ngs_meta_repository.get_by_product_id(product['id'])
|
|
assert meta is not None
|
|
assert meta['exclusive_access'] == 'After-hours museum access'
|
|
assert meta['group_leader'] == 'Local archaeologist guide'
|
|
assert meta['sustainability'] == 'Carbon-offset flights included'
|
|
|
|
|
|
def test_save_scrape_result_updates_existing_ngs_meta_without_duplicating():
|
|
tenant, competitor = _make_tenant_and_competitor()
|
|
extracted = {'tripName': 'Nat Geo Signature', 'exclusiveAccess': 'Version 1'}
|
|
product = analysis_service._save_scrape_result(
|
|
tenant['id'], competitor, extracted, {'url': competitor['website']}
|
|
)
|
|
analysis_service._save_scrape_result(
|
|
tenant['id'], competitor,
|
|
{'tripName': 'Nat Geo Signature', 'exclusiveAccess': 'Version 2'},
|
|
{'url': competitor['website']},
|
|
)
|
|
|
|
meta = ngs_meta_repository.get_by_product_id(product['id'])
|
|
assert meta['exclusive_access'] == 'Version 2'
|
|
|
|
|
|
def test_save_scrape_result_skips_ngs_meta_for_standard_run():
|
|
tenant, competitor = _make_tenant_and_competitor()
|
|
extracted = {'tripName': 'Peru Classic', 'majorityPrice': 2190}
|
|
product = analysis_service._save_scrape_result(
|
|
tenant['id'], competitor, extracted, {'url': competitor['website']}
|
|
)
|
|
|
|
assert ngs_meta_repository.get_by_product_id(product['id']) is None
|
|
|
|
|
|
def test_analyse_from_cache_returns_full_field_set(monkeypatch):
|
|
tenant, competitor = _make_tenant_and_competitor()
|
|
product_repository.create({
|
|
'tenant_id': tenant['id'], 'competitor_id': competitor['id'],
|
|
'trip_name': 'Inca Trail Classic', 'url': 'https://intrepidtravel.com/inca-trail',
|
|
'service_level': 'Premium', 'group_size': 12, 'duration_days': 15, 'meals': 14,
|
|
'start_location': 'Lima', 'end_location': 'Cusco', 'departures': 52,
|
|
'seasonality': 'Year-round', 'start_days': 'Saturday', 'target_audience': 'Premium travellers',
|
|
'hotels': 'Comfortable hotels', 'activities': 'City tours', 'date_used': '2026-08-15',
|
|
'last_seen': '2026-01-01',
|
|
})
|
|
_install_fake_module(
|
|
monkeypatch, 'core.extractor',
|
|
extract_with_openrouter=lambda prompt, content: {'comments': 'ok', 'relevancy': 4}
|
|
)
|
|
|
|
result = analysis_service.analyse_from_cache(tenant['id'], competitor['id'], {'destination': 'Peru'})
|
|
|
|
assert result['link'] == 'https://intrepidtravel.com/inca-trail'
|
|
assert result['serviceLevel'] == 'Premium'
|
|
assert result['groupSize'] == 12
|
|
assert result['meals'] == 14
|
|
assert result['startLocation'] == 'Lima'
|
|
assert result['endLocation'] == 'Cusco'
|
|
assert result['departures'] == 52
|
|
assert result['seasonality'] == 'Year-round'
|
|
assert result['startDays'] == 'Saturday'
|
|
assert result['targetAudience'] == 'Premium travellers'
|
|
assert result['hotels'] == 'Comfortable hotels'
|
|
assert result['activities'] == 'City tours'
|
|
assert result['dateUsed'] == '2026-08-15'
|
|
|
|
|
|
def test_analyse_from_cache_includes_ngs_fields_when_tab_type_ngs(monkeypatch):
|
|
tenant, competitor = _make_tenant_and_competitor()
|
|
product = product_repository.create({
|
|
'tenant_id': tenant['id'], 'competitor_id': competitor['id'],
|
|
'trip_name': 'Nat Geo Signature', 'last_seen': '2026-01-01',
|
|
})
|
|
ngs_meta_repository.create({
|
|
'product_id': product['id'], 'exclusive_access': 'After-hours access',
|
|
'group_leader': 'Local expert', 'sustainability': 'Offset flights',
|
|
})
|
|
_install_fake_module(
|
|
monkeypatch, 'core.extractor',
|
|
extract_with_openrouter=lambda prompt, content: {'comments': 'ok', 'relevancy': 4}
|
|
)
|
|
|
|
result = analysis_service.analyse_from_cache(
|
|
tenant['id'], competitor['id'], {'destination': 'Peru', 'tab_type': 'NGS'}
|
|
)
|
|
|
|
assert result['exclusiveAccess'] == 'After-hours access'
|
|
assert result['groupLeader'] == 'Local expert'
|
|
assert result['sustainability'] == 'Offset flights'
|
|
|
|
|
|
def test_analyse_from_cache_skips_ngs_lookup_for_standard_tab(monkeypatch):
|
|
tenant, competitor = _make_tenant_and_competitor()
|
|
product_repository.create({
|
|
'tenant_id': tenant['id'], 'competitor_id': competitor['id'],
|
|
'trip_name': 'Peru Classic', 'last_seen': '2026-01-01',
|
|
})
|
|
_install_fake_module(
|
|
monkeypatch, 'core.extractor',
|
|
extract_with_openrouter=lambda prompt, content: {'comments': 'ok', 'relevancy': 4}
|
|
)
|
|
|
|
result = analysis_service.analyse_from_cache(
|
|
tenant['id'], competitor['id'], {'destination': 'Peru', 'tab_type': 'Standard'}
|
|
)
|
|
|
|
assert 'exclusiveAccess' not in result
|
|
|
|
|
|
def test_detect_change_true_on_first_scrape():
|
|
_, competitor = _make_tenant_and_competitor()
|
|
assert analysis_service.detect_change(competitor['id'], 'somehash') is True
|
|
# First scrape never had a prior hash to compare against, so it
|
|
# shouldn't flag change_detected — there's nothing to diff against.
|
|
assert competitor_repository.get_by_id(competitor['id'])['change_detected'] is False
|
|
|
|
|
|
def test_detect_change_true_and_flags_competitor_on_diff():
|
|
_, competitor = _make_tenant_and_competitor()
|
|
scrape_repository.create({
|
|
'tenant_id': 't1', 'competitor_id': competitor['id'], 'triggered_by': 'cron',
|
|
'status': 'complete', 'content_hash': 'old-hash', 'competitors_total': 1, 'competitors_done': 1,
|
|
})
|
|
|
|
changed = analysis_service.detect_change(competitor['id'], 'new-hash')
|
|
|
|
assert changed is True
|
|
updated = competitor_repository.get_by_id(competitor['id'])
|
|
assert updated['change_detected'] is True
|
|
assert updated['change_detected_at'] is not None
|
|
|
|
|
|
def test_detect_change_false_when_hash_unchanged():
|
|
_, competitor = _make_tenant_and_competitor()
|
|
scrape_repository.create({
|
|
'tenant_id': 't1', 'competitor_id': competitor['id'], 'triggered_by': 'cron',
|
|
'status': 'complete', 'content_hash': 'same-hash', 'competitors_total': 1, 'competitors_done': 1,
|
|
})
|
|
|
|
assert analysis_service.detect_change(competitor['id'], 'same-hash') is False
|
|
|
|
|
|
def test_run_competitor_analysis_refresh_path_skips_extraction_when_hash_unchanged(monkeypatch):
|
|
tenant, competitor = _make_tenant_and_competitor(change_detected=True)
|
|
product_repository.create({
|
|
'tenant_id': tenant['id'], 'competitor_id': competitor['id'], 'trip_name': 'Peru Classic',
|
|
})
|
|
|
|
from core.scraper import compute_content_hash
|
|
|
|
fake_page_data = {'success': True, 'url': competitor['website'], 'text': 'unchanged content'}
|
|
_install_fake_module(monkeypatch, 'core.scraper', scrape=lambda url, country=None: fake_page_data)
|
|
prior_hash = compute_content_hash('unchanged content')
|
|
scrape_repository.create({
|
|
'tenant_id': tenant['id'], 'competitor_id': competitor['id'], 'triggered_by': 'cron',
|
|
'status': 'complete', 'content_hash': prior_hash, 'competitors_total': 1, 'competitors_done': 1,
|
|
})
|
|
|
|
# extract_with_openrouter backs both the full-page extraction call
|
|
# (refresh path, content=full scraped text) AND analyse_from_cache's
|
|
# much cheaper narrative-only call (content=''). Skipping "LLM
|
|
# extraction" means never invoking it with the full page content —
|
|
# so the assertion checks what content it was called with, not
|
|
# whether it was called at all.
|
|
calls_with_content = []
|
|
_install_fake_module(
|
|
monkeypatch, 'core.extractor',
|
|
extract_with_openrouter=lambda prompt, content: calls_with_content.append(content) or {'narrative': 'cached'}
|
|
)
|
|
_install_fake_module(monkeypatch, 'industries.adventure_travel.prompt', build_prompt=lambda *a, **k: '')
|
|
|
|
result = analysis_service.run_competitor_analysis(tenant['id'], competitor, {})
|
|
|
|
assert result['path'] == 'refresh'
|
|
assert result['unchanged'] is True
|
|
assert calls_with_content == [''] # only the cheap narrative-only call happened, never full extraction
|
|
updated = competitor_repository.get_by_id(competitor['id'])
|
|
assert updated['last_scraped'] is not None
|
|
# Confirmed no further change since the flag was set — this scrape
|
|
# is what clears it (see the bug this fixes: a prior version reset
|
|
# change_detected in the *changed* branch instead, making it
|
|
# unobservable to anything outside a single scrape call).
|
|
assert updated['change_detected'] is False
|
|
|
|
|
|
def test_force_refresh_bypasses_fast_path_gate_even_when_fresh(monkeypatch):
|
|
fresh = (datetime.now(timezone.utc) - timedelta(hours=1)).isoformat()
|
|
tenant, competitor = _make_tenant_and_competitor(change_detected=False, last_scraped=fresh)
|
|
product_repository.create({
|
|
'tenant_id': tenant['id'], 'competitor_id': competitor['id'], 'trip_name': 'Peru Classic',
|
|
})
|
|
|
|
fake_page_data = {'success': True, 'url': competitor['website'], 'text': 'fresh content from a real scrape'}
|
|
_install_fake_module(monkeypatch, 'core.scraper', scrape=lambda url, country=None: fake_page_data)
|
|
_install_fake_module(
|
|
monkeypatch, 'core.extractor',
|
|
extract_with_openrouter=lambda prompt, content: {'tripName': 'Peru Classic', 'majorityPrice': 2190}
|
|
)
|
|
_install_fake_module(monkeypatch, 'industries.adventure_travel.prompt', build_prompt=lambda *a, **k: '')
|
|
|
|
# This competitor looks fresh — needs_refresh() alone would take the
|
|
# fast path — but force_refresh must override that decision entirely.
|
|
result = analysis_service.run_competitor_analysis(tenant['id'], competitor, {'force_refresh': True})
|
|
|
|
assert result['path'] == 'refresh'
|
|
assert result.get('product') is not None # proves _save_scrape_result() ran — a real extraction, not cache reuse
|
|
|
|
|
|
def test_force_refresh_bypasses_unchanged_shortcut_even_when_hash_matches(monkeypatch):
|
|
tenant, competitor = _make_tenant_and_competitor(change_detected=False)
|
|
product_repository.create({
|
|
'tenant_id': tenant['id'], 'competitor_id': competitor['id'], 'trip_name': 'Peru Classic',
|
|
})
|
|
|
|
from core.scraper import compute_content_hash
|
|
|
|
fake_page_data = {'success': True, 'url': competitor['website'], 'text': 'unchanged content'}
|
|
_install_fake_module(monkeypatch, 'core.scraper', scrape=lambda url, country=None: fake_page_data)
|
|
prior_hash = compute_content_hash('unchanged content')
|
|
scrape_repository.create({
|
|
'tenant_id': tenant['id'], 'competitor_id': competitor['id'], 'triggered_by': 'cron',
|
|
'status': 'complete', 'content_hash': prior_hash, 'competitors_total': 1, 'competitors_done': 1,
|
|
})
|
|
|
|
calls_with_content = []
|
|
_install_fake_module(
|
|
monkeypatch, 'core.extractor',
|
|
extract_with_openrouter=lambda prompt, content: calls_with_content.append(content) or {
|
|
'tripName': 'Peru Classic', 'majorityPrice': 2190
|
|
}
|
|
)
|
|
_install_fake_module(monkeypatch, 'industries.adventure_travel.prompt', build_prompt=lambda *a, **k: '')
|
|
|
|
result = analysis_service.run_competitor_analysis(tenant['id'], competitor, {'force_refresh': True})
|
|
|
|
assert result['path'] == 'refresh'
|
|
assert not result.get('unchanged') # the cache-reuse shortcut was bypassed, not taken
|
|
assert calls_with_content == ['unchanged content'] # a REAL extraction ran with the full page text, not ''
|
|
updated = competitor_repository.get_by_id(competitor['id'])
|
|
# Bookkeeping (hash-confirmed-stable flag clear) still happens even
|
|
# though the cache-reuse decision itself was bypassed.
|
|
assert updated['change_detected'] is False
|
|
|
|
|
|
def test_change_detected_stays_visible_until_next_scrape_confirms_stable(monkeypatch):
|
|
"""
|
|
Regression test for the bug where change_detected was set True by
|
|
detect_change() and then immediately reset False before the same
|
|
function call returned — making it unobservable to the
|
|
CompetitorsPage status badge, the Apps Script sidebar, or the n8n
|
|
daily cron's `change_detected = true` filter clause. Exercises two
|
|
consecutive scrapes: the first finds a diff (flag should stay True
|
|
after it returns), the second finds no further diff (flag should
|
|
only clear then).
|
|
"""
|
|
tenant, competitor = _make_tenant_and_competitor()
|
|
product_repository.create({
|
|
'tenant_id': tenant['id'], 'competitor_id': competitor['id'], 'trip_name': 'Peru Classic',
|
|
})
|
|
_install_fake_module(monkeypatch, 'industries.adventure_travel.prompt', build_prompt=lambda *a, **k: '')
|
|
_install_fake_module(
|
|
monkeypatch, 'core.extractor',
|
|
extract_with_openrouter=lambda prompt, content: (
|
|
{'tripName': 'Peru Classic', 'majorityPrice': 2190} if content else {'narrative': 'cached'}
|
|
)
|
|
)
|
|
|
|
# First scrape — no prior hash exists, so detect_change() treats it
|
|
# as an initial baseline. changed=True takes the extraction branch.
|
|
_install_fake_module(
|
|
monkeypatch, 'core.scraper',
|
|
scrape=lambda url, country=None: {'success': True, 'url': url, 'text': 'version A'}
|
|
)
|
|
analysis_service.run_competitor_analysis(tenant['id'], competitor, {})
|
|
competitor = competitor_repository.get_by_id(competitor['id'])
|
|
assert competitor['change_detected'] is False # first-ever scrape has nothing to diff against
|
|
|
|
# Second scrape — content genuinely changed vs "version A".
|
|
competitor['last_scraped'] = None # force needs_refresh() to take the refresh path again
|
|
_install_fake_module(
|
|
monkeypatch, 'core.scraper',
|
|
scrape=lambda url, country=None: {'success': True, 'url': url, 'text': 'version B'}
|
|
)
|
|
analysis_service.run_competitor_analysis(tenant['id'], competitor, {})
|
|
competitor = competitor_repository.get_by_id(competitor['id'])
|
|
assert competitor['change_detected'] is True # visible — not reset by the same call that set it
|
|
|
|
# Third scrape — identical to "version B", nothing further changed.
|
|
competitor['last_scraped'] = None
|
|
analysis_service.run_competitor_analysis(tenant['id'], competitor, {})
|
|
competitor = competitor_repository.get_by_id(competitor['id'])
|
|
assert competitor['change_detected'] is False # now cleared, confirmed stable
|
|
|
|
|
|
def test_run_competitor_analysis_refresh_path_raises_on_scrape_failure(monkeypatch):
|
|
tenant, competitor = _make_tenant_and_competitor(change_detected=True)
|
|
_install_fake_module(
|
|
monkeypatch, 'core.scraper',
|
|
scrape=lambda url, country=None: {'success': False, 'text': 'scrape_blocked'}
|
|
)
|
|
_install_fake_module(monkeypatch, 'core.extractor', extract_with_openrouter=lambda p, content: {})
|
|
_install_fake_module(monkeypatch, 'industries.adventure_travel.prompt', build_prompt=lambda *a, **k: '')
|
|
|
|
with pytest.raises(ValueError):
|
|
analysis_service.run_competitor_analysis(tenant['id'], competitor, {})
|