assets/annual-estimates.example.jsonl
{"date":"2026-12-31","epsAvg":1.5,"epsHigh":1.6,"epsLow":1.4,"fiscalYear":"2026","numAnalystsEps":6,"numAnalystsRevenue":6,"revenueAvg":500000000,"symbol":"LISTA"}
{"date":"2027-12-31","epsAvg":1.9,"epsHigh":2.05,"epsLow":1.75,"fiscalYear":"2027","numAnalystsEps":6,"numAnalystsRevenue":6,"revenueAvg":575000000,"symbol":"LISTA"}
{"date":"2028-12-31","epsAvg":2.4,"epsHigh":2.6,"epsLow":2.2,"fiscalYear":"2028","numAnalystsEps":5,"numAnalystsRevenue":5,"revenueAvg":670000000,"symbol":"LISTA"}
{"date":"2026-12-31","epsAvg":2.0,"epsHigh":2.2,"epsLow":1.8,"fiscalYear":"2026","numAnalystsEps":4,"numAnalystsRevenue":4,"revenueAvg":800000000,"symbol":"LISTB"}
{"date":"2027-12-31","epsAvg":2.6,"epsHigh":2.85,"epsLow":2.35,"fiscalYear":"2027","numAnalystsEps":4,"numAnalystsRevenue":4,"revenueAvg":950000000,"symbol":"LISTB"}
{"date":"2028-12-31","epsAvg":3.4,"epsHigh":3.8,"epsLow":3.0,"fiscalYear":"2028","numAnalystsEps":4,"numAnalystsRevenue":4,"revenueAvg":1150000000,"symbol":"LISTB"}
{"date":"2026-12-31","epsAvg":0.3,"epsHigh":0.35,"epsLow":0.25,"fiscalYear":"2026","numAnalystsEps":3,"numAnalystsRevenue":3,"revenueAvg":180000000,"symbol":"LISTC"}
{"date":"2027-12-31","epsAvg":0.32,"epsHigh":0.37,"epsLow":0.27,"fiscalYear":"2027","numAnalystsEps":3,"numAnalystsRevenue":3,"revenueAvg":185000000,"symbol":"LISTC"}
{"date":"2028-12-31","epsAvg":0.34,"epsHigh":0.4,"epsLow":0.28,"fiscalYear":"2028","numAnalystsEps":3,"numAnalystsRevenue":3,"revenueAvg":190000000,"symbol":"LISTC"}
assets/broad-screen-audit.example.json
{
"audit_schema_version": 3,
"contract_revision": "3.5",
"runtime": {
"skill_name": "us-undervalued-growth-screener",
"skill_version": "3.6.1",
"schema_version": 3,
"contract_revision": "3.5",
"runtime_fingerprint": "ug-v3.6.1-claude-code-direct-fmp-20260830"
},
"conclusion_scope": "full_listing_universe",
"generated_at": "2026-08-28T14:28:34.377069+00:00",
"analysis_as_of": "2026-08-22T14:00:00-07:00",
"candidate_generation_mode": "full_universe_fundamentals",
"candidate_pool_status": "sufficient",
"selection_outcome": "selected",
"selected_symbols": [
"ACME",
"CYCLE",
"STALE"
],
"deep_dive_plan": {
"selected_symbols": [
"ACME",
"CYCLE",
"STALE"
],
"selected_count": 3,
"all_selected_must_be_resolved": true,
"budget_locked": true,
"budget_change_requires_rescreen": true,
"user_confirmation_required": false,
"user_continue_instruction_allowed": false,
"selected_set_is_committed": true,
"max_deep_dive_candidates": 3,
"selected_set_sha256": "3a45c382a4cdff829d63e0b378e7705866ca1d689de6ec7934381d30ba359065",
"commitment_payload": {
"analysis_as_of": "2026-08-22T14:00:00-07:00",
"max_deep_dive_candidates": 3,
"selected_symbols": [
"ACME",
"CYCLE",
"STALE"
]
}
},
"filters": {
"requested_min_market_cap": 500000000,
"requested_max_market_cap": 20000000000,
"min_market_cap": 500000000,
"max_market_cap": 20000000000,
"min_price": 5.0,
"min_average_daily_dollar_volume": 5000000,
"hard_min_average_daily_dollar_volume": 1000000,
"minimum_average_volume_period_days": 20,
"max_deep_dive_candidates": 3,
"preferred_forward_pe": 20.0,
"high_growth_exception_max_forward_pe": 30.0,
"near_miss_max_forward_pe": 22.0,
"preferred_ev_to_fcf": 20.0,
"preferred_fcf_yield_pct": 5.0,
"minimum_revenue_growth_pct": 8.0,
"minimum_per_share_growth_pct": 12.0,
"near_miss_min_per_share_growth_pct": 8.0,
"high_growth_exception_growth_pct": 20.0,
"minimum_roic_pct": 8.0,
"preferred_max_dilution_pct": 3.0,
"preferred_max_net_debt_to_ebitda": 2.5,
"hard_max_net_debt_to_ebitda": 4.0,
"maximum_forward_pe_for_economic_screen": 60.0,
"minimum_listing_data_coverage_pct": 95.0,
"minimum_discovery_analyst_count": 2,
"maximum_enrichment_attempts": 60,
"sector_review_selection_penalty": 5.0,
"near_miss_selection_penalty": 4.0,
"missing_deep_dive_field_penalty": 1.5,
"forward_pe_reconciliation_tolerance_pct": 5.0,
"maximum_forward_eps_dispersion_pct": 100.0,
"max_estimate_age_days": 45,
"maximum_fy1_horizon_days": 430
},
"scope": {
"requested_min_market_cap": 500000000.0,
"requested_max_market_cap": 20000000000.0,
"user_requested_scope": {
"min_market_cap": 500000000.0,
"max_market_cap": 20000000000.0,
"source": "skill_default"
},
"retrieval_min_market_cap": 500000000.0,
"retrieval_max_market_cap": 20000000000.0,
"retrieval_scope_explicit": true,
"scope_override_authorized": false,
"scope_reduced": false,
"scope_reduction_mode": "none",
"scope_reduction_disclosed": true,
"scope_reduction_reason": null,
"user_scope_evidence": null,
"user_requested_scope_complete": true,
"executed_scope_complete": true,
"bounded_sampling_ready": false,
"screening_scope_ready": true,
"executed_scope": {
"min_market_cap": 500000000.0,
"max_market_cap": 20000000000.0
},
"scope_complete": true,
"reasons": [],
"enumeration": {
"verified": true,
"provider_reported_total": 6,
"rows_fetched": 6,
"pages_fetched": 1,
"pagination_exhausted": true,
"band_audit": [],
"bands_well_formed": false,
"bands_cover_executed_range": false,
"bands_cover_requested_range": false,
"bands_verified": false
}
},
"universe": {
"row_count": 6,
"decision_counts": {
"in_scope": 4,
"excluded": 1,
"out_of_scope": 1
},
"listing_data_complete_count": 6,
"listing_data_complete_pct": 100.0,
"in_scope_count": 4,
"source_ids": [
"universe-fmp-20260822"
],
"artifact_path": "universe-audit-results.example.jsonl",
"artifact_sha256": "24e07ba0d17aa88df007e6e42df33dc2cf329326b5732f3521f57f529f8dee58"
},
"candidate_pool": {
"row_count": 6,
"decision_counts": {
"selected": 3,
"screened_out": 1,
"excluded": 2
},
"discovery_evaluable_count": 4,
"discovery_evaluable_pct": 66.6667,
"selection_eligible_count": 3,
"selected_count": 3,
"in_scope_covered_count": 4,
"in_scope_missing_count": 0,
"in_scope_missing_symbols": [],
"coverage_complete": true,
"coverage_scope": "full_listing_universe",
"listing_coverage_complete": true,
"generation_audit": {
"valid": true,
"selection_method": "full_universe_fundamentals",
"input_row_count": 6,
"selected_count": 6,
"selected_symbols": [
"ACME",
"CYCLE",
"MNA",
"OTCX",
"SCREEN",
"STALE"
],
"coverage_plan": {
"coverage_plan_valid": true,
"user_requested_range_spanned": true
}
},
"generation_review_reasons": [],
"fundamental_complete_count": 6,
"fundamental_complete_pct": 100.0,
"source_ids": [
"universe-fmp-20260822"
],
"symbols_not_in_universe": [],
"artifact_path": "broad-screen-results.example.jsonl",
"artifact_sha256": "55847bd84f650acd74f3381b2c2e125ed2e505f593d66f06085952f681e7ae3f"
},
"enrichment": {
"status": "complete",
"next_action": "proceed_to_deep_dive",
"discovery_pool_required": false,
"attempted_count": 6,
"resolved_count": 6,
"unresolved_count": 0,
"resolution_pct": 100.0,
"all_rows_resolved": true,
"maximum_attempts": 60,
"candidate_pool_exhaustion_declared": true,
"candidate_pool_exhausted": true,
"candidate_pool_covers_in_scope": true,
"candidate_pool_scope_verified": true,
"conclusion_scope": "full_listing_universe",
"queue_count": 0,
"queue_symbols": [],
"artifact_path": "enrichment-queue.example.json",
"artifact_sha256": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570"
},
"coverage_interpretation": {
"listing_universe_audited": true,
"universe_enumeration_verified": true,
"full_universe_fundamentals_claimed": true,
"candidate_pool_fundamentals_only": false,
"full_market_fundamental_coverage_is_not_a_completion_gate": true,
"no_candidates_requires_zero_unresolved_rows_and_exhausted_pool": true,
"bounded_pool_final_ranking_allowed_when_generation_is_audited": true,
"market_wide_no_candidates_requires_full_listing_universe_coverage": true
},
"row_count": 6,
"decision_counts": {
"selected": 3,
"screened_out": 1,
"excluded": 2
},
"source_ids": [
"universe-fmp-20260822"
]
}
assets/broad-screen-results.example.jsonl
{"broad_score":111.37,"candidate_generation_mode":"full_universe_fundamentals","company_name":"Acme Software, Inc.","decision":{"blocking_review_reasons":[],"deep_dive_requirements":[],"enrichment_exhausted":false,"enrichment_exhaustion_reason":null,"enrichment_source_ids":[],"exception_admitted":false,"guideline_misses":[],"hard_reasons":[],"preselection_status":"passed","resolution":"resolved","review_reasons":[],"screen_fail_reasons":[],"selection_eligible":true,"selection_reason":"highest deep-dive priority within budget","status":"selected"},"deep_dive_priority_score":117.37,"discovery_evaluable":true,"enrichment_attempted":true,"enrichment_resolved":true,"exchange":"NASDAQ","fundamental_complete":true,"fundamental_completeness_count":7,"industry":null,"metrics":{"analyst_count":4,"average_daily_dollar_volume":45000000.0,"average_daily_dollar_volume_method":"provider_average_dollar_volume","average_volume_period_days":20,"cyclicality_score":1,"dilution_pct":-1.9,"ev_to_fcf":27.33,"fcf_yield_pct":3.6,"forward_eps":3.0007501875468865,"forward_eps_dispersion_pct":null,"forward_estimate_as_of":"2026-08-22T14:00:00-07:00","forward_estimate_source_ids":["universe-fmp-20260822"],"forward_fiscal_year":"FY2027","forward_horizon_days":311.125,"forward_metric_origin":"provider_fy1_pe_with_eps_reconciliation","forward_pe":13.33,"forward_pe_period":"FY1","forward_period_end":"2027-06-30","forward_validation_reasons":[],"liquidity_valid_for_screen":true,"liquidity_validation_reasons":[],"market_cap":5000000000.0,"net_debt_to_ebitda":-0.42,"per_share_growth_pct":22.0,"price":40.0,"revenue_growth_pct":14.0,"roic_pct":18.5,"sector_profile_type":"general","standard_fcf":180000000.0},"sector":null,"selection_eligible":true,"symbol":"ACME"}
{"broad_score":98.0,"candidate_generation_mode":"full_universe_fundamentals","company_name":"Cycle Equipment Corp.","decision":{"blocking_review_reasons":[],"deep_dive_requirements":[],"enrichment_exhausted":false,"enrichment_exhaustion_reason":null,"enrichment_source_ids":[],"exception_admitted":false,"guideline_misses":[],"hard_reasons":[],"preselection_status":"passed","resolution":"resolved","review_reasons":[],"screen_fail_reasons":[],"selection_eligible":true,"selection_reason":"highest deep-dive priority within budget","status":"selected"},"deep_dive_priority_score":98.0,"discovery_evaluable":true,"enrichment_attempted":true,"enrichment_resolved":true,"exchange":"NYSE","fundamental_complete":true,"fundamental_completeness_count":7,"industry":null,"metrics":{"analyst_count":4,"average_daily_dollar_volume":30000000.0,"average_daily_dollar_volume_method":"provider_average_dollar_volume","average_volume_period_days":20,"cyclicality_score":4,"dilution_pct":0.5,"ev_to_fcf":null,"fcf_yield_pct":5.5,"forward_eps":3.5714285714285716,"forward_eps_dispersion_pct":null,"forward_estimate_as_of":"2026-08-22T14:00:00-07:00","forward_estimate_source_ids":["universe-fmp-20260822"],"forward_fiscal_year":"FY2027","forward_horizon_days":311.125,"forward_metric_origin":"provider_fy1_pe_with_eps_reconciliation","forward_pe":14.0,"forward_pe_period":"FY1","forward_period_end":"2027-06-30","forward_validation_reasons":[],"liquidity_valid_for_screen":true,"liquidity_validation_reasons":[],"market_cap":4000000000.0,"net_debt_to_ebitda":1.0,"per_share_growth_pct":18.0,"price":50.0,"revenue_growth_pct":12.0,"roic_pct":15.0,"sector_profile_type":"general","standard_fcf":220000000.0},"sector":null,"selection_eligible":true,"symbol":"CYCLE"}
{"broad_score":105.6,"candidate_generation_mode":"full_universe_fundamentals","company_name":"Stale Quote Corp.","decision":{"blocking_review_reasons":[],"deep_dive_requirements":[],"enrichment_exhausted":false,"enrichment_exhaustion_reason":null,"enrichment_source_ids":[],"exception_admitted":false,"guideline_misses":[],"hard_reasons":[],"preselection_status":"passed","resolution":"resolved","review_reasons":[],"screen_fail_reasons":[],"selection_eligible":true,"selection_reason":"highest deep-dive priority within budget","status":"selected"},"deep_dive_priority_score":111.6,"discovery_evaluable":true,"enrichment_attempted":true,"enrichment_resolved":true,"exchange":"NASDAQ","fundamental_complete":true,"fundamental_completeness_count":7,"industry":null,"metrics":{"analyst_count":4,"average_daily_dollar_volume":25000000.0,"average_daily_dollar_volume_method":"provider_average_dollar_volume","average_volume_period_days":20,"cyclicality_score":1,"dilution_pct":1.0,"ev_to_fcf":null,"fcf_yield_pct":5.3,"forward_eps":2.3333333333333335,"forward_eps_dispersion_pct":null,"forward_estimate_as_of":"2026-08-22T14:00:00-07:00","forward_estimate_source_ids":["universe-fmp-20260822"],"forward_fiscal_year":"FY2027","forward_horizon_days":311.125,"forward_metric_origin":"provider_fy1_pe_with_eps_reconciliation","forward_pe":15.0,"forward_pe_period":"FY1","forward_period_end":"2027-06-30","forward_validation_reasons":[],"liquidity_valid_for_screen":true,"liquidity_validation_reasons":[],"market_cap":3000000000.0,"net_debt_to_ebitda":0.5,"per_share_growth_pct":20.0,"price":35.0,"revenue_growth_pct":13.0,"roic_pct":17.0,"sector_profile_type":"general","standard_fcf":160000000.0},"sector":null,"selection_eligible":true,"symbol":"STALE"}
{"broad_score":50.0,"candidate_generation_mode":"full_universe_fundamentals","company_name":"Screened Out Inc.","decision":{"blocking_review_reasons":[],"deep_dive_requirements":[],"enrichment_exhausted":false,"enrichment_exhaustion_reason":null,"enrichment_source_ids":[],"exception_admitted":false,"guideline_misses":["per_share_growth_below_guideline","revenue_growth_below_guideline","roic_below_guideline"],"hard_reasons":[],"preselection_status":"screened_out","resolution":"resolved","review_reasons":[],"screen_fail_reasons":["growth_and_valuation_combination_below_review_threshold"],"selection_eligible":false,"status":"screened_out"},"deep_dive_priority_score":null,"discovery_evaluable":true,"enrichment_attempted":true,"enrichment_resolved":true,"exchange":"NASDAQ","fundamental_complete":true,"fundamental_completeness_count":7,"industry":null,"metrics":{"analyst_count":4,"average_daily_dollar_volume":15000000.0,"average_daily_dollar_volume_method":"provider_average_dollar_volume","average_volume_period_days":20,"cyclicality_score":1,"dilution_pct":1.0,"ev_to_fcf":null,"fcf_yield_pct":5.0,"forward_eps":0.6451612903225806,"forward_eps_dispersion_pct":null,"forward_estimate_as_of":"2026-08-22T14:00:00-07:00","forward_estimate_source_ids":["universe-fmp-20260822"],"forward_fiscal_year":"FY2027","forward_horizon_days":311.125,"forward_metric_origin":"provider_fy1_pe_with_eps_reconciliation","forward_pe":31.0,"forward_pe_period":"FY1","forward_period_end":"2027-06-30","forward_validation_reasons":[],"liquidity_valid_for_screen":true,"liquidity_validation_reasons":[],"market_cap":2000000000.0,"net_debt_to_ebitda":1.0,"per_share_growth_pct":5.0,"price":20.0,"revenue_growth_pct":-4.0,"roic_pct":7.0,"sector_profile_type":"general","standard_fcf":100000000.0},"sector":null,"selection_eligible":false,"symbol":"SCREEN"}
{"broad_score":null,"candidate_generation_mode":"full_universe_fundamentals","company_name":"Acquired Company, Inc.","decision":{"blocking_review_reasons":["average_daily_dollar_volume_unavailable_or_non_positive"],"deep_dive_requirements":[],"enrichment_exhausted":false,"enrichment_exhaustion_reason":null,"enrichment_source_ids":[],"exception_admitted":false,"guideline_misses":[],"hard_reasons":["inactive_symbol"],"preselection_status":"excluded","resolution":"resolved","review_reasons":["average_daily_dollar_volume_unavailable_or_non_positive"],"screen_fail_reasons":[],"selection_eligible":false,"status":"excluded"},"deep_dive_priority_score":null,"discovery_evaluable":false,"enrichment_attempted":true,"enrichment_resolved":true,"exchange":"NASDAQ","fundamental_complete":true,"fundamental_completeness_count":7,"industry":null,"metrics":{"analyst_count":4,"average_daily_dollar_volume":null,"average_daily_dollar_volume_method":"provider_average_dollar_volume","average_volume_period_days":20,"cyclicality_score":1,"dilution_pct":0.0,"ev_to_fcf":null,"fcf_yield_pct":8.0,"forward_eps":1.7494444444444444,"forward_eps_dispersion_pct":null,"forward_estimate_as_of":"2026-08-22T14:00:00-07:00","forward_estimate_source_ids":["universe-fmp-20260822"],"forward_fiscal_year":"FY2027","forward_horizon_days":311.125,"forward_metric_origin":"provider_fy1_pe_with_eps_reconciliation","forward_pe":18.0,"forward_pe_period":"FY1","forward_period_end":"2027-06-30","forward_validation_reasons":[],"liquidity_valid_for_screen":false,"liquidity_validation_reasons":["average_daily_dollar_volume_unavailable_or_non_positive"],"market_cap":1500000000.0,"net_debt_to_ebitda":0.0,"per_share_growth_pct":20.0,"price":31.49,"revenue_growth_pct":20.0,"roic_pct":20.0,"sector_profile_type":"general","standard_fcf":120000000.0},"sector":null,"selection_eligible":false,"symbol":"MNA"}
{"broad_score":null,"candidate_generation_mode":"full_universe_fundamentals","company_name":"OTC Example Corp.","decision":{"blocking_review_reasons":[],"deep_dive_requirements":["liquidity_below_preferred"],"enrichment_exhausted":false,"enrichment_exhaustion_reason":null,"enrichment_source_ids":[],"exception_admitted":false,"guideline_misses":[],"hard_reasons":["exchange_out_of_scope"],"preselection_status":"excluded","resolution":"resolved","review_reasons":[],"screen_fail_reasons":[],"selection_eligible":false,"status":"excluded"},"deep_dive_priority_score":null,"discovery_evaluable":false,"enrichment_attempted":true,"enrichment_resolved":true,"exchange":"OTC","fundamental_complete":true,"fundamental_completeness_count":7,"industry":null,"metrics":{"analyst_count":4,"average_daily_dollar_volume":2000000.0,"average_daily_dollar_volume_method":"provider_average_dollar_volume","average_volume_period_days":20,"cyclicality_score":1,"dilution_pct":1.0,"ev_to_fcf":null,"fcf_yield_pct":6.2,"forward_eps":0.6666666666666666,"forward_eps_dispersion_pct":null,"forward_estimate_as_of":"2026-08-22T14:00:00-07:00","forward_estimate_source_ids":["universe-fmp-20260822"],"forward_fiscal_year":"FY2027","forward_horizon_days":311.125,"forward_metric_origin":"provider_fy1_pe_with_eps_reconciliation","forward_pe":12.0,"forward_pe_period":"FY1","forward_period_end":"2027-06-30","forward_validation_reasons":[],"liquidity_valid_for_screen":true,"liquidity_validation_reasons":[],"market_cap":800000000.0,"net_debt_to_ebitda":1.0,"per_share_growth_pct":20.0,"price":8.0,"revenue_growth_pct":15.0,"roic_pct":15.0,"sector_profile_type":"general","standard_fcf":50000000.0},"sector":null,"selection_eligible":false,"symbol":"OTCX"}
assets/broad-screen-shortlist.example.json
{
"runtime": {
"skill_name": "us-undervalued-growth-screener",
"skill_version": "3.6.1",
"schema_version": 3,
"contract_revision": "3.5",
"runtime_fingerprint": "ug-v3.6.1-claude-code-direct-fmp-20260830"
},
"candidate_generation_mode": "full_universe_fundamentals",
"candidate_pool_status": "sufficient",
"selection_outcome": "selected",
"selected_symbols": [
"ACME",
"CYCLE",
"STALE"
],
"deep_dive_plan": {
"selected_symbols": [
"ACME",
"CYCLE",
"STALE"
],
"selected_count": 3,
"all_selected_must_be_resolved": true,
"budget_locked": true,
"budget_change_requires_rescreen": true,
"user_confirmation_required": false,
"user_continue_instruction_allowed": false,
"selected_set_is_committed": true,
"max_deep_dive_candidates": 3,
"selected_set_sha256": "3a45c382a4cdff829d63e0b378e7705866ca1d689de6ec7934381d30ba359065",
"commitment_payload": {
"analysis_as_of": "2026-08-22T14:00:00-07:00",
"max_deep_dive_candidates": 3,
"selected_symbols": [
"ACME",
"CYCLE",
"STALE"
]
}
}
}
assets/candidate-input.example.json
{
"schema_version": 3,
"analysis_as_of": "2026-08-22T14:00:00-07:00",
"run_metadata": {
"run_id": "example-20260822",
"status": "complete",
"base_repository_commit": "759151ee92e0eed9a7f2ddf4d6b0e4d6707df409",
"unprocessed_candidates": [],
"selected_symbols": [
"ACME",
"CYCLE",
"STALE"
]
},
"price_basis": {
"as_of": "2026-08-21T16:00:00-04:00",
"session": "regular_close",
"timezone": "America/New_York",
"source_id": "quote-20260821"
},
"config": {
"min_market_cap": 500000000,
"max_market_cap": 20000000000,
"min_price": 5.0,
"min_average_daily_dollar_volume": 5000000,
"hard_min_average_daily_dollar_volume": 1000000,
"max_candidates": 10,
"minimum_data_quality_score": 70,
"minimum_constant_multiple_upside_pct": 30,
"minimum_stressed_upside_pct": 0,
"multiple_contraction_pct": 20,
"maximum_cyclicality_without_normalization": 2,
"allow_special_cases": true,
"require_minimum_upside": true,
"require_positive_stress_case": false,
"max_quote_age_days": 7,
"max_estimate_age_days": 45,
"max_corporate_action_check_age_days": 14,
"forecast_bridge_tolerance_pct": 2,
"gaap_reconciliation_tolerance_pct": 2,
"fcf_reconciliation_tolerance_pct": 5,
"max_deep_dive_candidates": 3,
"cash_consistency_tolerance_pct": 3,
"max_market_context_age_days": 14,
"minimum_analyst_count_for_ranked_horizon": 3,
"require_forward_current_metric": true,
"require_peer_set": true,
"require_market_context": true,
"require_screening_audit": true,
"biopharma_loe_stress_multiples": [
6,
8
],
"require_full_scope": true,
"minimum_listing_data_coverage_pct": 95.0,
"discovery_pool_max_size": 120,
"discovery_pool_per_sector_market_cap_cell": 3,
"minimum_discovery_analyst_count": 2,
"maximum_enrichment_attempts": 60,
"max_discovery_pool_candidates": 120,
"discovery_per_sector_bucket_limit": 3,
"near_miss_max_forward_pe": 22.0,
"near_miss_min_per_share_growth_pct": 8.0,
"hard_max_net_debt_to_ebitda": 4.0,
"maximum_forward_pe_for_economic_screen": 60.0,
"sector_review_selection_penalty": 5.0,
"near_miss_selection_penalty": 4.0,
"missing_deep_dive_field_penalty": 1.5,
"max_policy_rate_source_age_days": 60,
"max_market_rate_source_age_days": 7,
"max_inflation_source_age_days": 45,
"max_gdp_source_age_days": 120,
"max_market_valuation_source_age_days": 14,
"max_small_mid_context_source_age_days": 14,
"minimum_final_score_for_eligible": 70.0,
"minimum_sbc_adjusted_fcf_yield_pct_for_eligible": 3.0,
"maximum_ev_to_fcf_for_eligible": 30.0,
"minimum_roic_pct_for_eligible": 8.0,
"maximum_net_debt_to_ebitda_for_eligible": 3.0,
"maximum_dilution_pct_for_eligible": 5.0,
"minimum_low_case_upside_pct_for_eligible": 15.0,
"maximum_quality_gate_failures_for_conditional": 2,
"severe_loe_stress_downside_pct": -25.0,
"final_three_minimum_score": 75.0,
"final_three_minimum_data_quality": 80,
"final_three_minimum_stress_upside_pct": 0.0,
"final_three_minimum_sbc_adjusted_fcf_yield_pct": 5.0,
"final_three_maximum_ev_to_fcf": 20.0,
"final_three_minimum_low_case_upside_pct": 20.0
},
"market_context": {
"summary": "US small- and mid-cap valuations are below the large-cap market, while elevated Treasury yields make multiple expansion an unsuitable base-case assumption.",
"as_of": "2026-08-22T12:00:00-07:00",
"policy_rate_pct": 3.625,
"treasury_10y_yield_pct": 4.74,
"real_gdp_growth_pct": 1.5,
"inflation_yoy_pct": 3.4,
"market_forward_pe": 21.2,
"small_mid_cap_valuation_context": "Small and mid caps trade at a discount to mega caps, but financing-sensitive businesses require a higher margin of safety.",
"sector_cycle_notes": [
"Software demand is stable but selective.",
"Commercial biopharma requires product-concentration and LOE stress checks."
],
"source_ids": [
"macro-fed-20260822",
"macro-treasury-20260822",
"macro-bea-bls-20260822",
"market-valuation-20260822"
]
},
"screening_funnel": {
"universe_count": 6,
"listing_in_scope_count": 4,
"candidate_pool_count": 6,
"discovery_evaluable_count": 4,
"deep_dive_selected_count": 3,
"preflight_passed_count": 3,
"deep_dive_completed_count": 3,
"notes": [
"Synthetic schema-3 / contract-3.5 strict-screen smoke universe."
]
},
"candidates": [
{
"identity": {
"symbol": "ACME",
"company_name": "Acme Software, Inc.",
"exchange": "NASDAQ",
"sector": "Technology",
"industry": "Software - Application",
"currency": "USD",
"price": 40.0,
"price_source_id": "quote-20260821",
"market_cap": 5000000000,
"average_daily_dollar_volume": 45000000,
"special_case": "none",
"average_daily_dollar_volume_method": "provider_average_dollar_volume",
"average_volume_period_days": 20,
"liquidity_source_ids": [
"universe-fmp-20260822"
],
"liquidity_valid_for_screen": true
},
"screening_decision": {
"status": "passed",
"reasons": [],
"source_ids": [
"consensus-20260822"
]
},
"corporate_action_check": {
"checked_at": "2026-08-22T08:05:00-07:00",
"listing_status": "active",
"mna_status": "none",
"symbol_active": true,
"latest_material_event_at": "2026-08-10T16:05:00-04:00",
"source_ids": [
"corporate-action-check",
"acme-ir"
]
},
"latest_earnings": {
"latest_report_type": "quarterly",
"quarter": {
"period": "FY2026 Q2",
"period_end": "2026-06-30",
"published_at": "2026-08-10T16:05:00-04:00",
"source_ids": [
"acme-10q",
"acme-ir"
],
"growth_state": "accelerating_or_strong",
"metrics": {
"revenue": 270000000,
"revenue_yoy_pct": 14.0,
"gaap_operating_margin_pct": 19.0,
"gaap_eps": 0.78,
"adjusted_eps": 0.82,
"operating_cash_flow": 60000000,
"standard_fcf": 52000000
},
"guidance": [
"FY2026 revenue growth 12%-14%",
"GAAP operating margin 18%-20%"
],
"key_kpis": [
"ARR +15%",
"NRR 110%"
],
"one_time_items": [
"No material one-time item"
],
"period_type": "quarter"
},
"full_year": null
},
"financials": {
"revenue_ttm": 1000000000,
"cash_flow_ttm": {
"method": "sum_4_discrete",
"operating_cash_flow": 210000000,
"capex_cash_outflow": 30000000,
"standard_fcf": 180000000,
"company_adjusted_fcf": 190000000,
"company_adjusted_fcf_definition": "Standard FCF plus restructuring cash payments",
"source_ids": [
"acme-10q"
]
},
"cash_flow_periods": [
{
"period": "FY2025 Q3",
"period_end": "2025-09-30",
"period_type": "discrete_quarter",
"operating_cash_flow": 45000000,
"capex_cash_outflow": 7000000,
"source_ids": [
"acme-10q"
]
},
{
"period": "FY2025 Q4",
"period_end": "2025-12-31",
"period_type": "discrete_quarter",
"operating_cash_flow": 50000000,
"capex_cash_outflow": 8000000,
"source_ids": [
"acme-10q"
]
},
{
"period": "FY2026 Q1",
"period_end": "2026-03-31",
"period_type": "discrete_quarter",
"operating_cash_flow": 55000000,
"capex_cash_outflow": 7000000,
"source_ids": [
"acme-10q"
]
},
{
"period": "FY2026 Q2",
"period_end": "2026-06-30",
"period_type": "discrete_quarter",
"operating_cash_flow": 60000000,
"capex_cash_outflow": 8000000,
"source_ids": [
"acme-10q"
]
}
],
"sbc_ttm": 25000000,
"total_debt": 100000000,
"cash_classification": {
"corporate_cash": 160000000,
"marketable_securities": 20000000,
"customer_or_settlement_funds": 0,
"restricted_cash": 0
},
"ebitda_ttm": 190000000,
"roic_pct": 18.5,
"roe_pct": 24.0,
"histories": {
"revenue": [
{
"date": "2023-12-31",
"period": "FY2023",
"value": 650000000
},
{
"date": "2024-12-31",
"period": "FY2024",
"value": 760000000
},
{
"date": "2025-12-31",
"period": "FY2025",
"value": 880000000
}
],
"gaap_eps": [
{
"date": "2023-12-31",
"period": "FY2023",
"value": 1.55
},
{
"date": "2024-12-31",
"period": "FY2024",
"value": 2.05
},
{
"date": "2025-12-31",
"period": "FY2025",
"value": 2.55
}
],
"fcf_per_share": [
{
"date": "2023-12-31",
"period": "FY2023",
"value": 1.3
},
{
"date": "2024-12-31",
"period": "FY2024",
"value": 1.6
},
{
"date": "2025-12-31",
"period": "FY2025",
"value": 1.95
}
],
"diluted_shares": [
{
"date": "2023-12-31",
"period": "FY2023",
"value": 130000000
},
{
"date": "2024-12-31",
"period": "FY2024",
"value": 128000000
},
{
"date": "2025-12-31",
"period": "FY2025",
"value": 125000000
}
]
}
},
"valuation_case": {
"basis": "eps",
"periods": {
"current": {
"label": "FY2026E",
"period_kind": "fy1",
"metric": 3.0,
"metric_basis": "gaap",
"source_type": "market_consensus",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"source_ids": [
"consensus-20260822"
],
"analyst_count": 9,
"estimate_low": 2.9,
"estimate_high": 3.1,
"period_end": "2026-12-31",
"years": 0.35
},
"year_2": {
"label": "FY2028E",
"period_kind": "fy2",
"metric": 4.2,
"years": 2.0,
"metric_basis": "gaap",
"source_type": "market_consensus",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"source_ids": [
"consensus-20260822"
],
"analyst_count": 8,
"estimate_low": 4.0,
"estimate_high": 4.4,
"period_end": "2028-12-31"
},
"year_3": {
"label": "FY2029E",
"period_kind": "fy3",
"metric": 5.0,
"years": 3.0,
"metric_basis": "gaap",
"source_type": "market_consensus",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"source_ids": [
"consensus-20260822"
],
"analyst_count": 6,
"estimate_low": 4.7,
"estimate_high": 5.3,
"period_end": "2029-12-31"
}
},
"supplied_current_multiple": 13.333333,
"peer_median_multiple": {
"value": 17.0,
"metric_basis": "gaap",
"multiple_period_kind": "fy1",
"source_ids": [
"peer-data"
]
}
},
"forecast_bridge": {
"periods": {
"year_2": {
"metric_basis": "gaap",
"metric_numerator": 525000000.0,
"metric_denominator": 125000000.0,
"drivers": {
"revenue": 1300000000,
"gross_margin_pct": 80.0,
"operating_margin_pct": 50.350535540408956,
"net_interest_income": 10000000,
"tax_rate_pct": 21.0,
"diluted_shares": 125000000,
"other_pre_tax_income": 0.0
},
"source_ids": [
"consensus-20260822",
"analyst-model"
],
"construction_method": "independent_driver_model",
"driver_provenance": {
"revenue": {
"origin": "market_consensus",
"source_ids": [
"consensus-20260822"
],
"target_solved": false
},
"gross_margin_pct": {
"origin": "historical_run_rate",
"source_ids": [
"analyst-model",
"acme-10q"
],
"target_solved": false
},
"operating_margin_pct": {
"origin": "historical_run_rate",
"source_ids": [
"analyst-model",
"acme-10q"
],
"target_solved": false
},
"net_interest_income": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
},
"tax_rate_pct": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
},
"diluted_shares": {
"origin": "market_consensus",
"source_ids": [
"consensus-20260822"
],
"target_solved": false
},
"other_pre_tax_income": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
}
}
},
"year_3": {
"metric_basis": "gaap",
"metric_numerator": 615000000.0,
"metric_denominator": 123000000.0,
"drivers": {
"revenue": 1480000000,
"gross_margin_pct": 81.0,
"operating_margin_pct": 51.78925761204242,
"net_interest_income": 12000000,
"tax_rate_pct": 21.0,
"diluted_shares": 123000000,
"other_pre_tax_income": 0.0
},
"source_ids": [
"consensus-20260822",
"analyst-model"
],
"construction_method": "independent_driver_model",
"driver_provenance": {
"revenue": {
"origin": "market_consensus",
"source_ids": [
"consensus-20260822"
],
"target_solved": false
},
"gross_margin_pct": {
"origin": "historical_run_rate",
"source_ids": [
"analyst-model",
"acme-10q"
],
"target_solved": false
},
"operating_margin_pct": {
"origin": "historical_run_rate",
"source_ids": [
"analyst-model",
"acme-10q"
],
"target_solved": false
},
"net_interest_income": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
},
"tax_rate_pct": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
},
"diluted_shares": {
"origin": "market_consensus",
"source_ids": [
"consensus-20260822"
],
"target_solved": false
},
"other_pre_tax_income": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
}
}
}
}
},
"gaap_reconciliation": {
"periods": {}
},
"score_components": {
"growth_sustainability": 18,
"valuation_attractiveness": 18,
"financial_quality": 14,
"fcf_earnings_quality": 14,
"competitive_advantage": 9,
"capital_allocation": 9,
"catalyst_risk_balance": 8
},
"cyclicality": {
"score": 1,
"position": "non_cyclical",
"peak_profit_risk": false,
"normalization_required": false,
"normalization": null
},
"sector_profile": {
"type": "general",
"kpis": {}
},
"flags": {
"otc": false,
"penny_stock": false,
"spac_or_shell": false,
"pre_revenue_or_development_stage": false,
"repeated_equity_financing": false,
"pending_mna": false,
"unresolved_material_weakness": false,
"sec_investigation": false,
"meme_only": false,
"restatement_unresolved": false,
"extreme_illiquidity": false
},
"peers": [
{
"symbol": "PEER1",
"company_name": "Peer One",
"selection_reason": "Comparable recurring-revenue model",
"multiple": 16.0,
"metric_basis": "gaap",
"multiple_period_kind": "fy1",
"source_id": "peer-data"
},
{
"symbol": "PEER2",
"company_name": "Peer Two",
"selection_reason": "Similar customer base and margins",
"multiple": 18.0,
"metric_basis": "gaap",
"multiple_period_kind": "fy1",
"source_id": "peer-data"
},
{
"symbol": "PEER3",
"company_name": "Peer Three",
"selection_reason": "Similar growth stage and capital intensity",
"multiple": 17.0,
"metric_basis": "gaap",
"multiple_period_kind": "fy1",
"source_id": "peer-data"
}
],
"qualitative": {
"business_overview": "Synthetic recurring-revenue software company.",
"investment_thesis": "Revenue and GAAP EPS can compound at attractive rates without requiring multiple expansion.",
"growth_drivers": [
"ARR growth",
"Operating leverage",
"Net share reduction"
],
"discount_reasons": [
"Recent growth normalization",
"Small-cap liquidity discount"
],
"discount_classification": "Mostly temporary",
"market_may_be_right": "NRR could weaken and reduce the durability of the model.",
"rerating_conditions": [
"NRR remains above 105%",
"GAAP margin reaches 22%"
],
"market_may_be_missing": "Per-share growth is stronger than headline revenue growth because diluted shares are declining.",
"catalysts": [
"New enterprise product launch in 6-12 months",
"Continued buybacks"
],
"best_catalyst": "GAAP operating-margin expansion.",
"maximum_risk": "A sustained NRR decline below 100%.",
"do_not_buy_reason": "The next quarter could confirm a sharper growth slowdown.",
"bear_case": "EPS reaches only $3.50 and the multiple falls to 10x, implying $35.",
"invalidation_conditions": [
"Revenue growth below 8% for two quarters",
"FCF turns negative",
"Diluted shares rise more than 3% YoY"
],
"next_earnings_kpis": [
"ARR growth",
"NRR",
"GAAP operating margin",
"Diluted shares"
],
"ev_to_fcf": 26.8
},
"evidence": {
"identity.price": [
"quote-20260821"
],
"latest_earnings": [
"acme-10q",
"acme-ir"
],
"financials.revenue_ttm": [
"acme-10q"
],
"financials.cash_flow_ttm": [
"acme-10q"
],
"financials.sbc_ttm": [
"acme-10q"
],
"financials.total_debt": [
"acme-10q"
],
"financials.cash_classification": [
"acme-10q"
],
"financials.histories.diluted_shares": [
"acme-10q"
],
"corporate_action_check": [
"corporate-action-check",
"acme-ir"
],
"financials.roic_pct": [
"acme-10q",
"analyst-model"
],
"financials.ebitda_ttm": [
"acme-10q",
"analyst-model"
]
},
"sources": [
{
"id": "acme-10q",
"tier": 1,
"kind": "sec_filing",
"title": "Synthetic Form 10-Q",
"published_at": "2026-08-10T16:05:00-04:00",
"retrieved_at": "2026-08-22T10:05:00-07:00",
"url": "https://www.sec.gov/Archives/edgar/data/1000000/000100000026000010/acme-20260630.htm",
"supports": [
"financials.cash_classification",
"financials.cash_flow_ttm",
"financials.ebitda_ttm",
"financials.histories.diluted_shares",
"financials.revenue_ttm",
"financials.roic_pct",
"financials.sbc_ttm",
"financials.total_debt",
"latest_earnings",
"financials.cash_flow_periods.2025-09-30",
"financials.cash_flow_periods.2025-12-31",
"financials.cash_flow_periods.2026-03-31",
"financials.cash_flow_periods.2026-06-30"
]
},
{
"id": "acme-ir",
"tier": 2,
"kind": "company_ir",
"title": "Synthetic earnings release",
"published_at": "2026-08-10T16:05:00-04:00",
"retrieved_at": "2026-08-22T10:06:00-07:00",
"url": "https://investors.example.com/news/acme-q2-2026-results",
"supports": [
"latest_earnings",
"corporate_action_check"
]
},
{
"id": "corporate-action-check",
"tier": 3,
"kind": "exchange_status",
"title": "Synthetic active-listing and M&A status check",
"published_at": "2026-08-22T08:00:00-07:00",
"retrieved_at": "2026-08-22T08:05:00-07:00",
"url": "https://www.nasdaq.com/market-activity/stocks/acme",
"supports": [
"corporate_action_check"
]
},
{
"id": "consensus-20260822",
"tier": 3,
"kind": "consensus",
"title": "Synthetic GAAP EPS consensus",
"published_at": "2026-08-22T09:00:00-07:00",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"url": "https://example.com/consensus",
"supports": [
"valuation_case.periods.current",
"valuation_case.periods.year_2",
"valuation_case.periods.year_3"
]
},
{
"id": "analyst-model",
"tier": 4,
"kind": "analyst_calculation",
"title": "Synthetic forecast bridge",
"published_at": "2026-08-22T11:00:00-07:00",
"retrieved_at": "2026-08-22T11:00:00-07:00",
"url": "local://analyst-model",
"supports": [
"forecast_bridge"
]
},
{
"id": "peer-data",
"tier": 3,
"kind": "market_data",
"title": "Synthetic peer forward GAAP P/E set",
"published_at": "2026-08-22T09:00:00-07:00",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"url": "https://example.com/peers",
"supports": [
"peers"
]
}
]
},
{
"identity": {
"symbol": "CYCLE",
"company_name": "Cycle Equipment Corp.",
"exchange": "NASDAQ",
"sector": "Technology",
"industry": "Software - Application",
"currency": "USD",
"price": 40.0,
"price_source_id": "quote-20260821",
"market_cap": 5000000000,
"average_daily_dollar_volume": 45000000,
"special_case": "none",
"average_daily_dollar_volume_method": "provider_average_dollar_volume",
"average_volume_period_days": 20,
"liquidity_source_ids": [
"universe-fmp-20260822"
],
"liquidity_valid_for_screen": true
},
"screening_decision": {
"status": "passed",
"reasons": [],
"source_ids": [
"consensus-20260822"
]
},
"corporate_action_check": {
"checked_at": "2026-08-22T08:05:00-07:00",
"listing_status": "active",
"mna_status": "none",
"symbol_active": true,
"latest_material_event_at": "2026-08-10T16:05:00-04:00",
"source_ids": [
"corporate-action-check",
"acme-ir"
]
},
"latest_earnings": {
"latest_report_type": "quarterly",
"quarter": {
"period": "FY2026 Q2",
"period_end": "2026-06-30",
"published_at": "2026-08-10T16:05:00-04:00",
"source_ids": [
"acme-10q",
"acme-ir"
],
"growth_state": "accelerating_or_strong",
"metrics": {
"revenue": 270000000,
"revenue_yoy_pct": 14.0,
"gaap_operating_margin_pct": 19.0,
"gaap_eps": 0.78,
"adjusted_eps": 0.82,
"operating_cash_flow": 60000000,
"standard_fcf": 52000000
},
"guidance": [
"FY2026 revenue growth 12%-14%",
"GAAP operating margin 18%-20%"
],
"key_kpis": [
"ARR +15%",
"NRR 110%"
],
"one_time_items": [
"No material one-time item"
],
"period_type": "quarter"
},
"full_year": null
},
"financials": {
"revenue_ttm": 1000000000,
"cash_flow_ttm": {
"method": "sum_4_discrete",
"operating_cash_flow": 210000000,
"capex_cash_outflow": 30000000,
"standard_fcf": 180000000,
"company_adjusted_fcf": 190000000,
"company_adjusted_fcf_definition": "Standard FCF plus restructuring cash payments",
"source_ids": [
"acme-10q"
]
},
"cash_flow_periods": [
{
"period": "FY2025 Q3",
"period_end": "2025-09-30",
"period_type": "discrete_quarter",
"operating_cash_flow": 45000000,
"capex_cash_outflow": 7000000,
"source_ids": [
"acme-10q"
]
},
{
"period": "FY2025 Q4",
"period_end": "2025-12-31",
"period_type": "discrete_quarter",
"operating_cash_flow": 50000000,
"capex_cash_outflow": 8000000,
"source_ids": [
"acme-10q"
]
},
{
"period": "FY2026 Q1",
"period_end": "2026-03-31",
"period_type": "discrete_quarter",
"operating_cash_flow": 55000000,
"capex_cash_outflow": 7000000,
"source_ids": [
"acme-10q"
]
},
{
"period": "FY2026 Q2",
"period_end": "2026-06-30",
"period_type": "discrete_quarter",
"operating_cash_flow": 60000000,
"capex_cash_outflow": 8000000,
"source_ids": [
"acme-10q"
]
}
],
"sbc_ttm": 25000000,
"total_debt": 100000000,
"cash_classification": {
"corporate_cash": 160000000,
"marketable_securities": 20000000,
"customer_or_settlement_funds": 0,
"restricted_cash": 0
},
"ebitda_ttm": 190000000,
"roic_pct": 18.5,
"roe_pct": 24.0,
"histories": {
"revenue": [
{
"date": "2023-12-31",
"period": "FY2023",
"value": 650000000
},
{
"date": "2024-12-31",
"period": "FY2024",
"value": 760000000
},
{
"date": "2025-12-31",
"period": "FY2025",
"value": 880000000
}
],
"gaap_eps": [
{
"date": "2023-12-31",
"period": "FY2023",
"value": 1.55
},
{
"date": "2024-12-31",
"period": "FY2024",
"value": 2.05
},
{
"date": "2025-12-31",
"period": "FY2025",
"value": 2.55
}
],
"fcf_per_share": [
{
"date": "2023-12-31",
"period": "FY2023",
"value": 1.3
},
{
"date": "2024-12-31",
"period": "FY2024",
"value": 1.6
},
{
"date": "2025-12-31",
"period": "FY2025",
"value": 1.95
}
],
"diluted_shares": [
{
"date": "2023-12-31",
"period": "FY2023",
"value": 130000000
},
{
"date": "2024-12-31",
"period": "FY2024",
"value": 128000000
},
{
"date": "2025-12-31",
"period": "FY2025",
"value": 125000000
}
]
}
},
"valuation_case": {
"basis": "eps",
"periods": {
"current": {
"label": "FY2026E",
"period_kind": "fy1",
"metric": 3.0,
"metric_basis": "gaap",
"source_type": "market_consensus",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"source_ids": [
"consensus-20260822"
],
"analyst_count": 9,
"estimate_low": 2.9,
"estimate_high": 3.1,
"period_end": "2026-12-31",
"years": 0.35
},
"year_2": {
"label": "FY2028E",
"period_kind": "fy2",
"metric": 4.2,
"years": 2.0,
"metric_basis": "gaap",
"source_type": "market_consensus",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"source_ids": [
"consensus-20260822"
],
"analyst_count": 8,
"estimate_low": 4.0,
"estimate_high": 4.4,
"period_end": "2028-12-31"
},
"year_3": {
"label": "FY2029E",
"period_kind": "fy3",
"metric": 5.0,
"years": 3.0,
"metric_basis": "gaap",
"source_type": "market_consensus",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"source_ids": [
"consensus-20260822"
],
"analyst_count": 6,
"estimate_low": 4.7,
"estimate_high": 5.3,
"period_end": "2029-12-31"
}
},
"supplied_current_multiple": 13.333333,
"peer_median_multiple": {
"value": 17.0,
"metric_basis": "gaap",
"multiple_period_kind": "fy1",
"source_ids": [
"peer-data"
]
}
},
"forecast_bridge": {
"periods": {
"year_2": {
"metric_basis": "gaap",
"metric_numerator": 525000000.0,
"metric_denominator": 125000000.0,
"drivers": {
"revenue": 1300000000,
"gross_margin_pct": 80.0,
"operating_margin_pct": 50.350535540408956,
"net_interest_income": 10000000,
"tax_rate_pct": 21.0,
"diluted_shares": 125000000,
"other_pre_tax_income": 0.0
},
"source_ids": [
"consensus-20260822",
"analyst-model"
],
"construction_method": "independent_driver_model",
"driver_provenance": {
"revenue": {
"origin": "market_consensus",
"source_ids": [
"consensus-20260822"
],
"target_solved": false
},
"gross_margin_pct": {
"origin": "historical_run_rate",
"source_ids": [
"analyst-model",
"acme-10q"
],
"target_solved": false
},
"operating_margin_pct": {
"origin": "historical_run_rate",
"source_ids": [
"analyst-model",
"acme-10q"
],
"target_solved": false
},
"net_interest_income": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
},
"tax_rate_pct": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
},
"diluted_shares": {
"origin": "market_consensus",
"source_ids": [
"consensus-20260822"
],
"target_solved": false
},
"other_pre_tax_income": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
}
}
},
"year_3": {
"metric_basis": "gaap",
"metric_numerator": 615000000.0,
"metric_denominator": 123000000.0,
"drivers": {
"revenue": 1480000000,
"gross_margin_pct": 81.0,
"operating_margin_pct": 51.78925761204242,
"net_interest_income": 12000000,
"tax_rate_pct": 21.0,
"diluted_shares": 123000000,
"other_pre_tax_income": 0.0
},
"source_ids": [
"consensus-20260822",
"analyst-model"
],
"construction_method": "independent_driver_model",
"driver_provenance": {
"revenue": {
"origin": "market_consensus",
"source_ids": [
"consensus-20260822"
],
"target_solved": false
},
"gross_margin_pct": {
"origin": "historical_run_rate",
"source_ids": [
"analyst-model",
"acme-10q"
],
"target_solved": false
},
"operating_margin_pct": {
"origin": "historical_run_rate",
"source_ids": [
"analyst-model",
"acme-10q"
],
"target_solved": false
},
"net_interest_income": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
},
"tax_rate_pct": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
},
"diluted_shares": {
"origin": "market_consensus",
"source_ids": [
"consensus-20260822"
],
"target_solved": false
},
"other_pre_tax_income": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
}
}
}
}
},
"gaap_reconciliation": {
"periods": {}
},
"score_components": {
"growth_sustainability": 18,
"valuation_attractiveness": 18,
"financial_quality": 14,
"fcf_earnings_quality": 14,
"competitive_advantage": 9,
"capital_allocation": 9,
"catalyst_risk_balance": 8
},
"cyclicality": {
"score": 4,
"position": "late_cycle",
"peak_profit_risk": true,
"normalization_required": true,
"normalization": null
},
"sector_profile": {
"type": "general",
"kpis": {}
},
"flags": {
"otc": false,
"penny_stock": false,
"spac_or_shell": false,
"pre_revenue_or_development_stage": false,
"repeated_equity_financing": false,
"pending_mna": false,
"unresolved_material_weakness": false,
"sec_investigation": false,
"meme_only": false,
"restatement_unresolved": false,
"extreme_illiquidity": false
},
"peers": [
{
"symbol": "PEER1",
"company_name": "Peer One",
"selection_reason": "Comparable recurring-revenue model",
"multiple": 16.0,
"metric_basis": "gaap",
"multiple_period_kind": "fy1",
"source_id": "peer-data"
},
{
"symbol": "PEER2",
"company_name": "Peer Two",
"selection_reason": "Similar customer base and margins",
"multiple": 18.0,
"metric_basis": "gaap",
"multiple_period_kind": "fy1",
"source_id": "peer-data"
},
{
"symbol": "PEER3",
"company_name": "Peer Three",
"selection_reason": "Similar growth stage and capital intensity",
"multiple": 17.0,
"metric_basis": "gaap",
"multiple_period_kind": "fy1",
"source_id": "peer-data"
}
],
"qualitative": {
"business_overview": "Synthetic recurring-revenue software company.",
"investment_thesis": "Revenue and GAAP EPS can compound at attractive rates without requiring multiple expansion.",
"growth_drivers": [
"ARR growth",
"Operating leverage",
"Net share reduction"
],
"discount_reasons": [
"Recent growth normalization",
"Small-cap liquidity discount"
],
"discount_classification": "Mostly temporary",
"market_may_be_right": "NRR could weaken and reduce the durability of the model.",
"rerating_conditions": [
"NRR remains above 105%",
"GAAP margin reaches 22%"
],
"market_may_be_missing": "Per-share growth is stronger than headline revenue growth because diluted shares are declining.",
"catalysts": [
"New enterprise product launch in 6-12 months",
"Continued buybacks"
],
"best_catalyst": "GAAP operating-margin expansion.",
"maximum_risk": "A sustained NRR decline below 100%.",
"do_not_buy_reason": "The next quarter could confirm a sharper growth slowdown.",
"bear_case": "EPS reaches only $3.50 and the multiple falls to 10x, implying $35.",
"invalidation_conditions": [
"Revenue growth below 8% for two quarters",
"FCF turns negative",
"Diluted shares rise more than 3% YoY"
],
"next_earnings_kpis": [
"ARR growth",
"NRR",
"GAAP operating margin",
"Diluted shares"
],
"ev_to_fcf": 26.8
},
"evidence": {
"identity.price": [
"quote-20260821"
],
"latest_earnings": [
"acme-10q",
"acme-ir"
],
"financials.revenue_ttm": [
"acme-10q"
],
"financials.cash_flow_ttm": [
"acme-10q"
],
"financials.sbc_ttm": [
"acme-10q"
],
"financials.total_debt": [
"acme-10q"
],
"financials.cash_classification": [
"acme-10q"
],
"financials.histories.diluted_shares": [
"acme-10q"
],
"corporate_action_check": [
"corporate-action-check",
"acme-ir"
],
"financials.roic_pct": [
"acme-10q",
"analyst-model"
],
"financials.ebitda_ttm": [
"acme-10q",
"analyst-model"
]
},
"sources": [
{
"id": "acme-10q",
"tier": 1,
"kind": "sec_filing",
"title": "Synthetic Form 10-Q",
"published_at": "2026-08-10T16:05:00-04:00",
"retrieved_at": "2026-08-22T10:05:00-07:00",
"url": "https://www.sec.gov/Archives/edgar/data/1000000/000100000026000010/acme-20260630.htm",
"supports": [
"financials.cash_classification",
"financials.cash_flow_ttm",
"financials.ebitda_ttm",
"financials.histories.diluted_shares",
"financials.revenue_ttm",
"financials.roic_pct",
"financials.sbc_ttm",
"financials.total_debt",
"latest_earnings",
"financials.cash_flow_periods.2025-09-30",
"financials.cash_flow_periods.2025-12-31",
"financials.cash_flow_periods.2026-03-31",
"financials.cash_flow_periods.2026-06-30"
]
},
{
"id": "acme-ir",
"tier": 2,
"kind": "company_ir",
"title": "Synthetic earnings release",
"published_at": "2026-08-10T16:05:00-04:00",
"retrieved_at": "2026-08-22T10:06:00-07:00",
"url": "https://investors.example.com/news/acme-q2-2026-results",
"supports": [
"latest_earnings",
"corporate_action_check"
]
},
{
"id": "corporate-action-check",
"tier": 3,
"kind": "exchange_status",
"title": "Synthetic active-listing and M&A status check",
"published_at": "2026-08-22T08:00:00-07:00",
"retrieved_at": "2026-08-22T08:05:00-07:00",
"url": "https://www.nasdaq.com/market-activity/stocks/acme",
"supports": [
"corporate_action_check"
]
},
{
"id": "consensus-20260822",
"tier": 3,
"kind": "consensus",
"title": "Synthetic GAAP EPS consensus",
"published_at": "2026-08-22T09:00:00-07:00",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"url": "https://example.com/consensus",
"supports": [
"valuation_case.periods.current",
"valuation_case.periods.year_2",
"valuation_case.periods.year_3"
]
},
{
"id": "analyst-model",
"tier": 4,
"kind": "analyst_calculation",
"title": "Synthetic forecast bridge",
"published_at": "2026-08-22T11:00:00-07:00",
"retrieved_at": "2026-08-22T11:00:00-07:00",
"url": "local://analyst-model",
"supports": [
"forecast_bridge"
]
},
{
"id": "peer-data",
"tier": 3,
"kind": "market_data",
"title": "Synthetic peer forward GAAP P/E set",
"published_at": "2026-08-22T09:00:00-07:00",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"url": "https://example.com/peers",
"supports": [
"peers"
]
}
]
},
{
"identity": {
"symbol": "STALE",
"company_name": "Stale Quote Corp.",
"exchange": "NASDAQ",
"sector": "Technology",
"industry": "Software - Application",
"currency": "USD",
"price": 40.0,
"price_source_id": "quote-20260821",
"market_cap": 5000000000,
"average_daily_dollar_volume": 45000000,
"special_case": "none",
"average_daily_dollar_volume_method": "provider_average_dollar_volume",
"average_volume_period_days": 20,
"liquidity_source_ids": [
"universe-fmp-20260822"
],
"liquidity_valid_for_screen": true
},
"screening_decision": {
"status": "passed",
"reasons": [],
"source_ids": [
"consensus-20260822"
]
},
"corporate_action_check": {
"checked_at": "2026-08-22T08:05:00-07:00",
"listing_status": "active",
"mna_status": "none",
"symbol_active": true,
"latest_material_event_at": "2026-08-22T17:00:00-04:00",
"source_ids": [
"corporate-action-check",
"acme-ir"
]
},
"latest_earnings": {
"latest_report_type": "quarterly",
"quarter": {
"period": "FY2026 Q2",
"period_end": "2026-06-30",
"published_at": "2026-08-22T17:00:00-04:00",
"source_ids": [
"acme-10q",
"acme-ir"
],
"growth_state": "accelerating_or_strong",
"metrics": {
"revenue": 270000000,
"revenue_yoy_pct": 14.0,
"gaap_operating_margin_pct": 19.0,
"gaap_eps": 0.78,
"adjusted_eps": 0.82,
"operating_cash_flow": 60000000,
"standard_fcf": 52000000
},
"guidance": [
"FY2026 revenue growth 12%-14%",
"GAAP operating margin 18%-20%"
],
"key_kpis": [
"ARR +15%",
"NRR 110%"
],
"one_time_items": [
"No material one-time item"
],
"period_type": "quarter"
},
"full_year": null
},
"financials": {
"revenue_ttm": 1000000000,
"cash_flow_ttm": {
"method": "sum_4_discrete",
"operating_cash_flow": 210000000,
"capex_cash_outflow": 30000000,
"standard_fcf": 180000000,
"company_adjusted_fcf": 190000000,
"company_adjusted_fcf_definition": "Standard FCF plus restructuring cash payments",
"source_ids": [
"acme-10q"
]
},
"cash_flow_periods": [
{
"period": "FY2025 Q3",
"period_end": "2025-09-30",
"period_type": "discrete_quarter",
"operating_cash_flow": 45000000,
"capex_cash_outflow": 7000000,
"source_ids": [
"acme-10q"
]
},
{
"period": "FY2025 Q4",
"period_end": "2025-12-31",
"period_type": "discrete_quarter",
"operating_cash_flow": 50000000,
"capex_cash_outflow": 8000000,
"source_ids": [
"acme-10q"
]
},
{
"period": "FY2026 Q1",
"period_end": "2026-03-31",
"period_type": "discrete_quarter",
"operating_cash_flow": 55000000,
"capex_cash_outflow": 7000000,
"source_ids": [
"acme-10q"
]
},
{
"period": "FY2026 Q2",
"period_end": "2026-06-30",
"period_type": "discrete_quarter",
"operating_cash_flow": 60000000,
"capex_cash_outflow": 8000000,
"source_ids": [
"acme-10q"
]
}
],
"sbc_ttm": 25000000,
"total_debt": 100000000,
"cash_classification": {
"corporate_cash": 160000000,
"marketable_securities": 20000000,
"customer_or_settlement_funds": 0,
"restricted_cash": 0
},
"ebitda_ttm": 190000000,
"roic_pct": 18.5,
"roe_pct": 24.0,
"histories": {
"revenue": [
{
"date": "2023-12-31",
"period": "FY2023",
"value": 650000000
},
{
"date": "2024-12-31",
"period": "FY2024",
"value": 760000000
},
{
"date": "2025-12-31",
"period": "FY2025",
"value": 880000000
}
],
"gaap_eps": [
{
"date": "2023-12-31",
"period": "FY2023",
"value": 1.55
},
{
"date": "2024-12-31",
"period": "FY2024",
"value": 2.05
},
{
"date": "2025-12-31",
"period": "FY2025",
"value": 2.55
}
],
"fcf_per_share": [
{
"date": "2023-12-31",
"period": "FY2023",
"value": 1.3
},
{
"date": "2024-12-31",
"period": "FY2024",
"value": 1.6
},
{
"date": "2025-12-31",
"period": "FY2025",
"value": 1.95
}
],
"diluted_shares": [
{
"date": "2023-12-31",
"period": "FY2023",
"value": 130000000
},
{
"date": "2024-12-31",
"period": "FY2024",
"value": 128000000
},
{
"date": "2025-12-31",
"period": "FY2025",
"value": 125000000
}
]
}
},
"valuation_case": {
"basis": "eps",
"periods": {
"current": {
"label": "FY2026E",
"period_kind": "fy1",
"metric": 3.0,
"metric_basis": "gaap",
"source_type": "market_consensus",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"source_ids": [
"consensus-20260822"
],
"analyst_count": 9,
"estimate_low": 2.9,
"estimate_high": 3.1,
"period_end": "2026-12-31",
"years": 0.35
},
"year_2": {
"label": "FY2028E",
"period_kind": "fy2",
"metric": 4.2,
"years": 2.0,
"metric_basis": "gaap",
"source_type": "market_consensus",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"source_ids": [
"consensus-20260822"
],
"analyst_count": 8,
"estimate_low": 4.0,
"estimate_high": 4.4,
"period_end": "2028-12-31"
},
"year_3": {
"label": "FY2029E",
"period_kind": "fy3",
"metric": 5.0,
"years": 3.0,
"metric_basis": "gaap",
"source_type": "market_consensus",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"source_ids": [
"consensus-20260822"
],
"analyst_count": 6,
"estimate_low": 4.7,
"estimate_high": 5.3,
"period_end": "2029-12-31"
}
},
"supplied_current_multiple": 13.333333,
"peer_median_multiple": {
"value": 17.0,
"metric_basis": "gaap",
"multiple_period_kind": "fy1",
"source_ids": [
"peer-data"
]
}
},
"forecast_bridge": {
"periods": {
"year_2": {
"metric_basis": "gaap",
"metric_numerator": 525000000.0,
"metric_denominator": 125000000.0,
"drivers": {
"revenue": 1300000000,
"gross_margin_pct": 80.0,
"operating_margin_pct": 50.350535540408956,
"net_interest_income": 10000000,
"tax_rate_pct": 21.0,
"diluted_shares": 125000000,
"other_pre_tax_income": 0.0
},
"source_ids": [
"consensus-20260822",
"analyst-model"
],
"construction_method": "independent_driver_model",
"driver_provenance": {
"revenue": {
"origin": "market_consensus",
"source_ids": [
"consensus-20260822"
],
"target_solved": false
},
"gross_margin_pct": {
"origin": "historical_run_rate",
"source_ids": [
"analyst-model",
"acme-10q"
],
"target_solved": false
},
"operating_margin_pct": {
"origin": "historical_run_rate",
"source_ids": [
"analyst-model",
"acme-10q"
],
"target_solved": false
},
"net_interest_income": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
},
"tax_rate_pct": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
},
"diluted_shares": {
"origin": "market_consensus",
"source_ids": [
"consensus-20260822"
],
"target_solved": false
},
"other_pre_tax_income": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
}
}
},
"year_3": {
"metric_basis": "gaap",
"metric_numerator": 615000000.0,
"metric_denominator": 123000000.0,
"drivers": {
"revenue": 1480000000,
"gross_margin_pct": 81.0,
"operating_margin_pct": 51.78925761204242,
"net_interest_income": 12000000,
"tax_rate_pct": 21.0,
"diluted_shares": 123000000,
"other_pre_tax_income": 0.0
},
"source_ids": [
"consensus-20260822",
"analyst-model"
],
"construction_method": "independent_driver_model",
"driver_provenance": {
"revenue": {
"origin": "market_consensus",
"source_ids": [
"consensus-20260822"
],
"target_solved": false
},
"gross_margin_pct": {
"origin": "historical_run_rate",
"source_ids": [
"analyst-model",
"acme-10q"
],
"target_solved": false
},
"operating_margin_pct": {
"origin": "historical_run_rate",
"source_ids": [
"analyst-model",
"acme-10q"
],
"target_solved": false
},
"net_interest_income": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
},
"tax_rate_pct": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
},
"diluted_shares": {
"origin": "market_consensus",
"source_ids": [
"consensus-20260822"
],
"target_solved": false
},
"other_pre_tax_income": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
}
}
}
}
},
"gaap_reconciliation": {
"periods": {}
},
"score_components": {
"growth_sustainability": 18,
"valuation_attractiveness": 18,
"financial_quality": 14,
"fcf_earnings_quality": 14,
"competitive_advantage": 9,
"capital_allocation": 9,
"catalyst_risk_balance": 8
},
"cyclicality": {
"score": 1,
"position": "non_cyclical",
"peak_profit_risk": false,
"normalization_required": false,
"normalization": null
},
"sector_profile": {
"type": "general",
"kpis": {}
},
"flags": {
"otc": false,
"penny_stock": false,
"spac_or_shell": false,
"pre_revenue_or_development_stage": false,
"repeated_equity_financing": false,
"pending_mna": false,
"unresolved_material_weakness": false,
"sec_investigation": false,
"meme_only": false,
"restatement_unresolved": false,
"extreme_illiquidity": false
},
"peers": [
{
"symbol": "PEER1",
"company_name": "Peer One",
"selection_reason": "Comparable recurring-revenue model",
"multiple": 16.0,
"metric_basis": "gaap",
"multiple_period_kind": "fy1",
"source_id": "peer-data"
},
{
"symbol": "PEER2",
"company_name": "Peer Two",
"selection_reason": "Similar customer base and margins",
"multiple": 18.0,
"metric_basis": "gaap",
"multiple_period_kind": "fy1",
"source_id": "peer-data"
},
{
"symbol": "PEER3",
"company_name": "Peer Three",
"selection_reason": "Similar growth stage and capital intensity",
"multiple": 17.0,
"metric_basis": "gaap",
"multiple_period_kind": "fy1",
"source_id": "peer-data"
}
],
"qualitative": {
"business_overview": "Synthetic recurring-revenue software company.",
"investment_thesis": "Revenue and GAAP EPS can compound at attractive rates without requiring multiple expansion.",
"growth_drivers": [
"ARR growth",
"Operating leverage",
"Net share reduction"
],
"discount_reasons": [
"Recent growth normalization",
"Small-cap liquidity discount"
],
"discount_classification": "Mostly temporary",
"market_may_be_right": "NRR could weaken and reduce the durability of the model.",
"rerating_conditions": [
"NRR remains above 105%",
"GAAP margin reaches 22%"
],
"market_may_be_missing": "Per-share growth is stronger than headline revenue growth because diluted shares are declining.",
"catalysts": [
"New enterprise product launch in 6-12 months",
"Continued buybacks"
],
"best_catalyst": "GAAP operating-margin expansion.",
"maximum_risk": "A sustained NRR decline below 100%.",
"do_not_buy_reason": "The next quarter could confirm a sharper growth slowdown.",
"bear_case": "EPS reaches only $3.50 and the multiple falls to 10x, implying $35.",
"invalidation_conditions": [
"Revenue growth below 8% for two quarters",
"FCF turns negative",
"Diluted shares rise more than 3% YoY"
],
"next_earnings_kpis": [
"ARR growth",
"NRR",
"GAAP operating margin",
"Diluted shares"
],
"ev_to_fcf": 26.8
},
"evidence": {
"identity.price": [
"quote-20260821"
],
"latest_earnings": [
"acme-10q",
"acme-ir"
],
"financials.revenue_ttm": [
"acme-10q"
],
"financials.cash_flow_ttm": [
"acme-10q"
],
"financials.sbc_ttm": [
"acme-10q"
],
"financials.total_debt": [
"acme-10q"
],
"financials.cash_classification": [
"acme-10q"
],
"financials.histories.diluted_shares": [
"acme-10q"
],
"corporate_action_check": [
"corporate-action-check",
"acme-ir"
],
"financials.roic_pct": [
"acme-10q",
"analyst-model"
],
"financials.ebitda_ttm": [
"acme-10q",
"analyst-model"
]
},
"sources": [
{
"id": "acme-10q",
"tier": 1,
"kind": "sec_filing",
"title": "Synthetic Form 10-Q",
"published_at": "2026-08-10T16:05:00-04:00",
"retrieved_at": "2026-08-22T10:05:00-07:00",
"url": "https://www.sec.gov/Archives/edgar/data/1000000/000100000026000010/acme-20260630.htm",
"supports": [
"financials.cash_classification",
"financials.cash_flow_ttm",
"financials.ebitda_ttm",
"financials.histories.diluted_shares",
"financials.revenue_ttm",
"financials.roic_pct",
"financials.sbc_ttm",
"financials.total_debt",
"latest_earnings",
"financials.cash_flow_periods.2025-09-30",
"financials.cash_flow_periods.2025-12-31",
"financials.cash_flow_periods.2026-03-31",
"financials.cash_flow_periods.2026-06-30"
]
},
{
"id": "acme-ir",
"tier": 2,
"kind": "company_ir",
"title": "Synthetic earnings release",
"published_at": "2026-08-10T16:05:00-04:00",
"retrieved_at": "2026-08-22T10:06:00-07:00",
"url": "https://investors.example.com/news/acme-q2-2026-results",
"supports": [
"latest_earnings",
"corporate_action_check"
]
},
{
"id": "corporate-action-check",
"tier": 3,
"kind": "exchange_status",
"title": "Synthetic active-listing and M&A status check",
"published_at": "2026-08-22T08:00:00-07:00",
"retrieved_at": "2026-08-22T08:05:00-07:00",
"url": "https://www.nasdaq.com/market-activity/stocks/acme",
"supports": [
"corporate_action_check"
]
},
{
"id": "consensus-20260822",
"tier": 3,
"kind": "consensus",
"title": "Synthetic GAAP EPS consensus",
"published_at": "2026-08-22T09:00:00-07:00",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"url": "https://example.com/consensus",
"supports": [
"valuation_case.periods.current",
"valuation_case.periods.year_2",
"valuation_case.periods.year_3"
]
},
{
"id": "analyst-model",
"tier": 4,
"kind": "analyst_calculation",
"title": "Synthetic forecast bridge",
"published_at": "2026-08-22T11:00:00-07:00",
"retrieved_at": "2026-08-22T11:00:00-07:00",
"url": "local://analyst-model",
"supports": [
"forecast_bridge"
]
},
{
"id": "peer-data",
"tier": 3,
"kind": "market_data",
"title": "Synthetic peer forward GAAP P/E set",
"published_at": "2026-08-22T09:00:00-07:00",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"url": "https://example.com/peers",
"supports": [
"peers"
]
}
]
},
{
"identity": {
"symbol": "OTCX",
"company_name": "OTC Example Corp.",
"exchange": "OTC",
"sector": "Technology",
"industry": "Software - Application",
"currency": "USD",
"price": 40.0,
"price_source_id": "quote-20260821",
"market_cap": 5000000000,
"average_daily_dollar_volume": 45000000,
"special_case": "none",
"average_daily_dollar_volume_method": "provider_average_dollar_volume",
"average_volume_period_days": 20,
"liquidity_source_ids": [
"universe-fmp-20260822"
],
"liquidity_valid_for_screen": true
},
"screening_decision": {
"status": "passed",
"reasons": [],
"source_ids": [
"consensus-20260822"
]
},
"corporate_action_check": {
"checked_at": "2026-08-22T08:05:00-07:00",
"listing_status": "active",
"mna_status": "none",
"symbol_active": true,
"latest_material_event_at": "2026-08-10T16:05:00-04:00",
"source_ids": [
"corporate-action-check",
"acme-ir"
]
},
"latest_earnings": {
"latest_report_type": "quarterly",
"quarter": {
"period": "FY2026 Q2",
"period_end": "2026-06-30",
"published_at": "2026-08-10T16:05:00-04:00",
"source_ids": [
"acme-10q",
"acme-ir"
],
"growth_state": "accelerating_or_strong",
"metrics": {
"revenue": 270000000,
"revenue_yoy_pct": 14.0,
"gaap_operating_margin_pct": 19.0,
"gaap_eps": 0.78,
"adjusted_eps": 0.82,
"operating_cash_flow": 60000000,
"standard_fcf": 52000000
},
"guidance": [
"FY2026 revenue growth 12%-14%",
"GAAP operating margin 18%-20%"
],
"key_kpis": [
"ARR +15%",
"NRR 110%"
],
"one_time_items": [
"No material one-time item"
],
"period_type": "quarter"
},
"full_year": null
},
"financials": {
"revenue_ttm": 1000000000,
"cash_flow_ttm": {
"method": "sum_4_discrete",
"operating_cash_flow": 210000000,
"capex_cash_outflow": 30000000,
"standard_fcf": 180000000,
"company_adjusted_fcf": 190000000,
"company_adjusted_fcf_definition": "Standard FCF plus restructuring cash payments",
"source_ids": [
"acme-10q"
]
},
"cash_flow_periods": [
{
"period": "FY2025 Q3",
"period_end": "2025-09-30",
"period_type": "discrete_quarter",
"operating_cash_flow": 45000000,
"capex_cash_outflow": 7000000,
"source_ids": [
"acme-10q"
]
},
{
"period": "FY2025 Q4",
"period_end": "2025-12-31",
"period_type": "discrete_quarter",
"operating_cash_flow": 50000000,
"capex_cash_outflow": 8000000,
"source_ids": [
"acme-10q"
]
},
{
"period": "FY2026 Q1",
"period_end": "2026-03-31",
"period_type": "discrete_quarter",
"operating_cash_flow": 55000000,
"capex_cash_outflow": 7000000,
"source_ids": [
"acme-10q"
]
},
{
"period": "FY2026 Q2",
"period_end": "2026-06-30",
"period_type": "discrete_quarter",
"operating_cash_flow": 60000000,
"capex_cash_outflow": 8000000,
"source_ids": [
"acme-10q"
]
}
],
"sbc_ttm": 25000000,
"total_debt": 100000000,
"cash_classification": {
"corporate_cash": 160000000,
"marketable_securities": 20000000,
"customer_or_settlement_funds": 0,
"restricted_cash": 0
},
"ebitda_ttm": 190000000,
"roic_pct": 18.5,
"roe_pct": 24.0,
"histories": {
"revenue": [
{
"date": "2023-12-31",
"period": "FY2023",
"value": 650000000
},
{
"date": "2024-12-31",
"period": "FY2024",
"value": 760000000
},
{
"date": "2025-12-31",
"period": "FY2025",
"value": 880000000
}
],
"gaap_eps": [
{
"date": "2023-12-31",
"period": "FY2023",
"value": 1.55
},
{
"date": "2024-12-31",
"period": "FY2024",
"value": 2.05
},
{
"date": "2025-12-31",
"period": "FY2025",
"value": 2.55
}
],
"fcf_per_share": [
{
"date": "2023-12-31",
"period": "FY2023",
"value": 1.3
},
{
"date": "2024-12-31",
"period": "FY2024",
"value": 1.6
},
{
"date": "2025-12-31",
"period": "FY2025",
"value": 1.95
}
],
"diluted_shares": [
{
"date": "2023-12-31",
"period": "FY2023",
"value": 130000000
},
{
"date": "2024-12-31",
"period": "FY2024",
"value": 128000000
},
{
"date": "2025-12-31",
"period": "FY2025",
"value": 125000000
}
]
}
},
"valuation_case": {
"basis": "eps",
"periods": {
"current": {
"label": "FY2026E",
"period_kind": "fy1",
"metric": 3.0,
"metric_basis": "gaap",
"source_type": "market_consensus",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"source_ids": [
"consensus-20260822"
],
"analyst_count": 9,
"estimate_low": 2.9,
"estimate_high": 3.1,
"period_end": "2026-12-31",
"years": 0.35
},
"year_2": {
"label": "FY2028E",
"period_kind": "fy2",
"metric": 4.2,
"years": 2.0,
"metric_basis": "gaap",
"source_type": "market_consensus",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"source_ids": [
"consensus-20260822"
],
"analyst_count": 8,
"estimate_low": 4.0,
"estimate_high": 4.4,
"period_end": "2028-12-31"
},
"year_3": {
"label": "FY2029E",
"period_kind": "fy3",
"metric": 5.0,
"years": 3.0,
"metric_basis": "gaap",
"source_type": "market_consensus",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"source_ids": [
"consensus-20260822"
],
"analyst_count": 6,
"estimate_low": 4.7,
"estimate_high": 5.3,
"period_end": "2029-12-31"
}
},
"supplied_current_multiple": 13.333333,
"peer_median_multiple": {
"value": 17.0,
"metric_basis": "gaap",
"multiple_period_kind": "fy1",
"source_ids": [
"peer-data"
]
}
},
"forecast_bridge": {
"periods": {
"year_2": {
"metric_basis": "gaap",
"metric_numerator": 525000000.0,
"metric_denominator": 125000000.0,
"drivers": {
"revenue": 1300000000,
"gross_margin_pct": 80.0,
"operating_margin_pct": 50.350535540408956,
"net_interest_income": 10000000,
"tax_rate_pct": 21.0,
"diluted_shares": 125000000,
"other_pre_tax_income": 0.0
},
"source_ids": [
"consensus-20260822",
"analyst-model"
],
"construction_method": "independent_driver_model",
"driver_provenance": {
"revenue": {
"origin": "market_consensus",
"source_ids": [
"consensus-20260822"
],
"target_solved": false
},
"gross_margin_pct": {
"origin": "historical_run_rate",
"source_ids": [
"analyst-model",
"acme-10q"
],
"target_solved": false
},
"operating_margin_pct": {
"origin": "historical_run_rate",
"source_ids": [
"analyst-model",
"acme-10q"
],
"target_solved": false
},
"net_interest_income": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
},
"tax_rate_pct": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
},
"diluted_shares": {
"origin": "market_consensus",
"source_ids": [
"consensus-20260822"
],
"target_solved": false
},
"other_pre_tax_income": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
}
}
},
"year_3": {
"metric_basis": "gaap",
"metric_numerator": 615000000.0,
"metric_denominator": 123000000.0,
"drivers": {
"revenue": 1480000000,
"gross_margin_pct": 81.0,
"operating_margin_pct": 51.78925761204242,
"net_interest_income": 12000000,
"tax_rate_pct": 21.0,
"diluted_shares": 123000000,
"other_pre_tax_income": 0.0
},
"source_ids": [
"consensus-20260822",
"analyst-model"
],
"construction_method": "independent_driver_model",
"driver_provenance": {
"revenue": {
"origin": "market_consensus",
"source_ids": [
"consensus-20260822"
],
"target_solved": false
},
"gross_margin_pct": {
"origin": "historical_run_rate",
"source_ids": [
"analyst-model",
"acme-10q"
],
"target_solved": false
},
"operating_margin_pct": {
"origin": "historical_run_rate",
"source_ids": [
"analyst-model",
"acme-10q"
],
"target_solved": false
},
"net_interest_income": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
},
"tax_rate_pct": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
},
"diluted_shares": {
"origin": "market_consensus",
"source_ids": [
"consensus-20260822"
],
"target_solved": false
},
"other_pre_tax_income": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
}
}
}
}
},
"gaap_reconciliation": {
"periods": {}
},
"score_components": {
"growth_sustainability": 18,
"valuation_attractiveness": 18,
"financial_quality": 14,
"fcf_earnings_quality": 14,
"competitive_advantage": 9,
"capital_allocation": 9,
"catalyst_risk_balance": 8
},
"cyclicality": {
"score": 1,
"position": "non_cyclical",
"peak_profit_risk": false,
"normalization_required": false,
"normalization": null
},
"sector_profile": {
"type": "general",
"kpis": {}
},
"flags": {
"otc": true,
"penny_stock": false,
"spac_or_shell": false,
"pre_revenue_or_development_stage": false,
"repeated_equity_financing": false,
"pending_mna": false,
"unresolved_material_weakness": false,
"sec_investigation": false,
"meme_only": false,
"restatement_unresolved": false,
"extreme_illiquidity": false
},
"peers": [
{
"symbol": "PEER1",
"company_name": "Peer One",
"selection_reason": "Comparable recurring-revenue model",
"multiple": 16.0,
"metric_basis": "gaap",
"multiple_period_kind": "fy1",
"source_id": "peer-data"
},
{
"symbol": "PEER2",
"company_name": "Peer Two",
"selection_reason": "Similar customer base and margins",
"multiple": 18.0,
"metric_basis": "gaap",
"multiple_period_kind": "fy1",
"source_id": "peer-data"
},
{
"symbol": "PEER3",
"company_name": "Peer Three",
"selection_reason": "Similar growth stage and capital intensity",
"multiple": 17.0,
"metric_basis": "gaap",
"multiple_period_kind": "fy1",
"source_id": "peer-data"
}
],
"qualitative": {
"business_overview": "Synthetic recurring-revenue software company.",
"investment_thesis": "Revenue and GAAP EPS can compound at attractive rates without requiring multiple expansion.",
"growth_drivers": [
"ARR growth",
"Operating leverage",
"Net share reduction"
],
"discount_reasons": [
"Recent growth normalization",
"Small-cap liquidity discount"
],
"discount_classification": "Mostly temporary",
"market_may_be_right": "NRR could weaken and reduce the durability of the model.",
"rerating_conditions": [
"NRR remains above 105%",
"GAAP margin reaches 22%"
],
"market_may_be_missing": "Per-share growth is stronger than headline revenue growth because diluted shares are declining.",
"catalysts": [
"New enterprise product launch in 6-12 months",
"Continued buybacks"
],
"best_catalyst": "GAAP operating-margin expansion.",
"maximum_risk": "A sustained NRR decline below 100%.",
"do_not_buy_reason": "The next quarter could confirm a sharper growth slowdown.",
"bear_case": "EPS reaches only $3.50 and the multiple falls to 10x, implying $35.",
"invalidation_conditions": [
"Revenue growth below 8% for two quarters",
"FCF turns negative",
"Diluted shares rise more than 3% YoY"
],
"next_earnings_kpis": [
"ARR growth",
"NRR",
"GAAP operating margin",
"Diluted shares"
],
"ev_to_fcf": 26.8
},
"evidence": {
"identity.price": [
"quote-20260821"
],
"latest_earnings": [
"acme-10q",
"acme-ir"
],
"financials.revenue_ttm": [
"acme-10q"
],
"financials.cash_flow_ttm": [
"acme-10q"
],
"financials.sbc_ttm": [
"acme-10q"
],
"financials.total_debt": [
"acme-10q"
],
"financials.cash_classification": [
"acme-10q"
],
"financials.histories.diluted_shares": [
"acme-10q"
],
"corporate_action_check": [
"corporate-action-check",
"acme-ir"
],
"financials.roic_pct": [
"acme-10q",
"analyst-model"
],
"financials.ebitda_ttm": [
"acme-10q",
"analyst-model"
]
},
"sources": [
{
"id": "acme-10q",
"tier": 1,
"kind": "sec_filing",
"title": "Synthetic Form 10-Q",
"published_at": "2026-08-10T16:05:00-04:00",
"retrieved_at": "2026-08-22T10:05:00-07:00",
"url": "https://www.sec.gov/Archives/edgar/data/1000000/000100000026000010/acme-20260630.htm",
"supports": [
"financials.cash_classification",
"financials.cash_flow_ttm",
"financials.ebitda_ttm",
"financials.histories.diluted_shares",
"financials.revenue_ttm",
"financials.roic_pct",
"financials.sbc_ttm",
"financials.total_debt",
"latest_earnings",
"financials.cash_flow_periods.2025-09-30",
"financials.cash_flow_periods.2025-12-31",
"financials.cash_flow_periods.2026-03-31",
"financials.cash_flow_periods.2026-06-30"
]
},
{
"id": "acme-ir",
"tier": 2,
"kind": "company_ir",
"title": "Synthetic earnings release",
"published_at": "2026-08-10T16:05:00-04:00",
"retrieved_at": "2026-08-22T10:06:00-07:00",
"url": "https://investors.example.com/news/acme-q2-2026-results",
"supports": [
"latest_earnings",
"corporate_action_check"
]
},
{
"id": "corporate-action-check",
"tier": 3,
"kind": "exchange_status",
"title": "Synthetic active-listing and M&A status check",
"published_at": "2026-08-22T08:00:00-07:00",
"retrieved_at": "2026-08-22T08:05:00-07:00",
"url": "https://www.nasdaq.com/market-activity/stocks/acme",
"supports": [
"corporate_action_check"
]
},
{
"id": "consensus-20260822",
"tier": 3,
"kind": "consensus",
"title": "Synthetic GAAP EPS consensus",
"published_at": "2026-08-22T09:00:00-07:00",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"url": "https://example.com/consensus",
"supports": [
"valuation_case.periods.current",
"valuation_case.periods.year_2",
"valuation_case.periods.year_3"
]
},
{
"id": "analyst-model",
"tier": 4,
"kind": "analyst_calculation",
"title": "Synthetic forecast bridge",
"published_at": "2026-08-22T11:00:00-07:00",
"retrieved_at": "2026-08-22T11:00:00-07:00",
"url": "local://analyst-model",
"supports": [
"forecast_bridge"
]
},
{
"id": "peer-data",
"tier": 3,
"kind": "market_data",
"title": "Synthetic peer forward GAAP P/E set",
"published_at": "2026-08-22T09:00:00-07:00",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"url": "https://example.com/peers",
"supports": [
"peers"
]
}
]
},
{
"identity": {
"symbol": "SCREEN",
"company_name": "Screened Out Inc."
},
"screening_decision": {
"status": "screened_out",
"reasons": [
"Forward P/E 31x and FY2025 revenue declined 4%"
],
"source_ids": [
"consensus-20260822"
]
},
"sources": [
{
"id": "consensus-20260822",
"tier": 3,
"kind": "consensus",
"title": "Synthetic GAAP EPS consensus",
"published_at": "2026-08-22T09:00:00-07:00",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"url": "https://example.com/consensus",
"supports": [
"valuation_case.periods.current",
"valuation_case.periods.year_2",
"valuation_case.periods.year_3"
]
}
]
},
{
"identity": {
"symbol": "MNA",
"company_name": "Acquired Company, Inc.",
"exchange": "NASDAQ",
"sector": "Technology",
"industry": "Software - Application",
"currency": "USD",
"price": 40.0,
"price_source_id": "quote-20260821",
"market_cap": 5000000000,
"average_daily_dollar_volume": 45000000,
"special_case": "none",
"average_daily_dollar_volume_method": "provider_average_dollar_volume",
"average_volume_period_days": 20,
"liquidity_source_ids": [
"universe-fmp-20260822"
],
"liquidity_valid_for_screen": true
},
"screening_decision": {
"status": "passed",
"reasons": [],
"source_ids": [
"consensus-20260822"
]
},
"corporate_action_check": {
"checked_at": "2026-08-22T08:05:00-07:00",
"listing_status": "delisted",
"mna_status": "completed",
"symbol_active": false,
"latest_material_event_at": "2026-08-10T16:05:00-04:00",
"source_ids": [
"corporate-action-check",
"acme-ir"
]
},
"latest_earnings": {
"latest_report_type": "quarterly",
"quarter": {
"period": "FY2026 Q2",
"period_end": "2026-06-30",
"published_at": "2026-08-10T16:05:00-04:00",
"source_ids": [
"acme-10q",
"acme-ir"
],
"growth_state": "accelerating_or_strong",
"metrics": {
"revenue": 270000000,
"revenue_yoy_pct": 14.0,
"gaap_operating_margin_pct": 19.0,
"gaap_eps": 0.78,
"adjusted_eps": 0.82,
"operating_cash_flow": 60000000,
"standard_fcf": 52000000
},
"guidance": [
"FY2026 revenue growth 12%-14%",
"GAAP operating margin 18%-20%"
],
"key_kpis": [
"ARR +15%",
"NRR 110%"
],
"one_time_items": [
"No material one-time item"
],
"period_type": "quarter"
},
"full_year": null
},
"financials": {
"revenue_ttm": 1000000000,
"cash_flow_ttm": {
"method": "sum_4_discrete",
"operating_cash_flow": 210000000,
"capex_cash_outflow": 30000000,
"standard_fcf": 180000000,
"company_adjusted_fcf": 190000000,
"company_adjusted_fcf_definition": "Standard FCF plus restructuring cash payments",
"source_ids": [
"acme-10q"
]
},
"cash_flow_periods": [
{
"period": "FY2025 Q3",
"period_end": "2025-09-30",
"period_type": "discrete_quarter",
"operating_cash_flow": 45000000,
"capex_cash_outflow": 7000000,
"source_ids": [
"acme-10q"
]
},
{
"period": "FY2025 Q4",
"period_end": "2025-12-31",
"period_type": "discrete_quarter",
"operating_cash_flow": 50000000,
"capex_cash_outflow": 8000000,
"source_ids": [
"acme-10q"
]
},
{
"period": "FY2026 Q1",
"period_end": "2026-03-31",
"period_type": "discrete_quarter",
"operating_cash_flow": 55000000,
"capex_cash_outflow": 7000000,
"source_ids": [
"acme-10q"
]
},
{
"period": "FY2026 Q2",
"period_end": "2026-06-30",
"period_type": "discrete_quarter",
"operating_cash_flow": 60000000,
"capex_cash_outflow": 8000000,
"source_ids": [
"acme-10q"
]
}
],
"sbc_ttm": 25000000,
"total_debt": 100000000,
"cash_classification": {
"corporate_cash": 160000000,
"marketable_securities": 20000000,
"customer_or_settlement_funds": 0,
"restricted_cash": 0
},
"ebitda_ttm": 190000000,
"roic_pct": 18.5,
"roe_pct": 24.0,
"histories": {
"revenue": [
{
"date": "2023-12-31",
"period": "FY2023",
"value": 650000000
},
{
"date": "2024-12-31",
"period": "FY2024",
"value": 760000000
},
{
"date": "2025-12-31",
"period": "FY2025",
"value": 880000000
}
],
"gaap_eps": [
{
"date": "2023-12-31",
"period": "FY2023",
"value": 1.55
},
{
"date": "2024-12-31",
"period": "FY2024",
"value": 2.05
},
{
"date": "2025-12-31",
"period": "FY2025",
"value": 2.55
}
],
"fcf_per_share": [
{
"date": "2023-12-31",
"period": "FY2023",
"value": 1.3
},
{
"date": "2024-12-31",
"period": "FY2024",
"value": 1.6
},
{
"date": "2025-12-31",
"period": "FY2025",
"value": 1.95
}
],
"diluted_shares": [
{
"date": "2023-12-31",
"period": "FY2023",
"value": 130000000
},
{
"date": "2024-12-31",
"period": "FY2024",
"value": 128000000
},
{
"date": "2025-12-31",
"period": "FY2025",
"value": 125000000
}
]
}
},
"valuation_case": {
"basis": "eps",
"periods": {
"current": {
"label": "FY2026E",
"period_kind": "fy1",
"metric": 3.0,
"metric_basis": "gaap",
"source_type": "market_consensus",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"source_ids": [
"consensus-20260822"
],
"analyst_count": 9,
"estimate_low": 2.9,
"estimate_high": 3.1,
"period_end": "2026-12-31",
"years": 0.35
},
"year_2": {
"label": "FY2028E",
"period_kind": "fy2",
"metric": 4.2,
"years": 2.0,
"metric_basis": "gaap",
"source_type": "market_consensus",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"source_ids": [
"consensus-20260822"
],
"analyst_count": 8,
"estimate_low": 4.0,
"estimate_high": 4.4,
"period_end": "2028-12-31"
},
"year_3": {
"label": "FY2029E",
"period_kind": "fy3",
"metric": 5.0,
"years": 3.0,
"metric_basis": "gaap",
"source_type": "market_consensus",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"source_ids": [
"consensus-20260822"
],
"analyst_count": 6,
"estimate_low": 4.7,
"estimate_high": 5.3,
"period_end": "2029-12-31"
}
},
"supplied_current_multiple": 13.333333,
"peer_median_multiple": {
"value": 17.0,
"metric_basis": "gaap",
"multiple_period_kind": "fy1",
"source_ids": [
"peer-data"
]
}
},
"forecast_bridge": {
"periods": {
"year_2": {
"metric_basis": "gaap",
"metric_numerator": 525000000.0,
"metric_denominator": 125000000.0,
"drivers": {
"revenue": 1300000000,
"gross_margin_pct": 80.0,
"operating_margin_pct": 50.350535540408956,
"net_interest_income": 10000000,
"tax_rate_pct": 21.0,
"diluted_shares": 125000000,
"other_pre_tax_income": 0.0
},
"source_ids": [
"consensus-20260822",
"analyst-model"
],
"construction_method": "independent_driver_model",
"driver_provenance": {
"revenue": {
"origin": "market_consensus",
"source_ids": [
"consensus-20260822"
],
"target_solved": false
},
"gross_margin_pct": {
"origin": "historical_run_rate",
"source_ids": [
"analyst-model",
"acme-10q"
],
"target_solved": false
},
"operating_margin_pct": {
"origin": "historical_run_rate",
"source_ids": [
"analyst-model",
"acme-10q"
],
"target_solved": false
},
"net_interest_income": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
},
"tax_rate_pct": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
},
"diluted_shares": {
"origin": "market_consensus",
"source_ids": [
"consensus-20260822"
],
"target_solved": false
},
"other_pre_tax_income": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
}
}
},
"year_3": {
"metric_basis": "gaap",
"metric_numerator": 615000000.0,
"metric_denominator": 123000000.0,
"drivers": {
"revenue": 1480000000,
"gross_margin_pct": 81.0,
"operating_margin_pct": 51.78925761204242,
"net_interest_income": 12000000,
"tax_rate_pct": 21.0,
"diluted_shares": 123000000,
"other_pre_tax_income": 0.0
},
"source_ids": [
"consensus-20260822",
"analyst-model"
],
"construction_method": "independent_driver_model",
"driver_provenance": {
"revenue": {
"origin": "market_consensus",
"source_ids": [
"consensus-20260822"
],
"target_solved": false
},
"gross_margin_pct": {
"origin": "historical_run_rate",
"source_ids": [
"analyst-model",
"acme-10q"
],
"target_solved": false
},
"operating_margin_pct": {
"origin": "historical_run_rate",
"source_ids": [
"analyst-model",
"acme-10q"
],
"target_solved": false
},
"net_interest_income": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
},
"tax_rate_pct": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
},
"diluted_shares": {
"origin": "market_consensus",
"source_ids": [
"consensus-20260822"
],
"target_solved": false
},
"other_pre_tax_income": {
"origin": "analyst_assumption",
"source_ids": [
"analyst-model"
],
"target_solved": false
}
}
}
}
},
"gaap_reconciliation": {
"periods": {}
},
"score_components": {
"growth_sustainability": 18,
"valuation_attractiveness": 18,
"financial_quality": 14,
"fcf_earnings_quality": 14,
"competitive_advantage": 9,
"capital_allocation": 9,
"catalyst_risk_balance": 8
},
"cyclicality": {
"score": 1,
"position": "non_cyclical",
"peak_profit_risk": false,
"normalization_required": false,
"normalization": null
},
"sector_profile": {
"type": "general",
"kpis": {}
},
"flags": {
"otc": false,
"penny_stock": false,
"spac_or_shell": false,
"pre_revenue_or_development_stage": false,
"repeated_equity_financing": false,
"pending_mna": false,
"unresolved_material_weakness": false,
"sec_investigation": false,
"meme_only": false,
"restatement_unresolved": false,
"extreme_illiquidity": false
},
"peers": [
{
"symbol": "PEER1",
"company_name": "Peer One",
"selection_reason": "Comparable recurring-revenue model",
"multiple": 16.0,
"metric_basis": "gaap",
"multiple_period_kind": "fy1",
"source_id": "peer-data"
},
{
"symbol": "PEER2",
"company_name": "Peer Two",
"selection_reason": "Similar customer base and margins",
"multiple": 18.0,
"metric_basis": "gaap",
"multiple_period_kind": "fy1",
"source_id": "peer-data"
},
{
"symbol": "PEER3",
"company_name": "Peer Three",
"selection_reason": "Similar growth stage and capital intensity",
"multiple": 17.0,
"metric_basis": "gaap",
"multiple_period_kind": "fy1",
"source_id": "peer-data"
}
],
"qualitative": {
"business_overview": "Synthetic recurring-revenue software company.",
"investment_thesis": "Revenue and GAAP EPS can compound at attractive rates without requiring multiple expansion.",
"growth_drivers": [
"ARR growth",
"Operating leverage",
"Net share reduction"
],
"discount_reasons": [
"Recent growth normalization",
"Small-cap liquidity discount"
],
"discount_classification": "Mostly temporary",
"market_may_be_right": "NRR could weaken and reduce the durability of the model.",
"rerating_conditions": [
"NRR remains above 105%",
"GAAP margin reaches 22%"
],
"market_may_be_missing": "Per-share growth is stronger than headline revenue growth because diluted shares are declining.",
"catalysts": [
"New enterprise product launch in 6-12 months",
"Continued buybacks"
],
"best_catalyst": "GAAP operating-margin expansion.",
"maximum_risk": "A sustained NRR decline below 100%.",
"do_not_buy_reason": "The next quarter could confirm a sharper growth slowdown.",
"bear_case": "EPS reaches only $3.50 and the multiple falls to 10x, implying $35.",
"invalidation_conditions": [
"Revenue growth below 8% for two quarters",
"FCF turns negative",
"Diluted shares rise more than 3% YoY"
],
"next_earnings_kpis": [
"ARR growth",
"NRR",
"GAAP operating margin",
"Diluted shares"
],
"ev_to_fcf": 26.8
},
"evidence": {
"identity.price": [
"quote-20260821"
],
"latest_earnings": [
"acme-10q",
"acme-ir"
],
"financials.revenue_ttm": [
"acme-10q"
],
"financials.cash_flow_ttm": [
"acme-10q"
],
"financials.sbc_ttm": [
"acme-10q"
],
"financials.total_debt": [
"acme-10q"
],
"financials.cash_classification": [
"acme-10q"
],
"financials.histories.diluted_shares": [
"acme-10q"
],
"corporate_action_check": [
"corporate-action-check",
"acme-ir"
],
"financials.roic_pct": [
"acme-10q",
"analyst-model"
],
"financials.ebitda_ttm": [
"acme-10q",
"analyst-model"
]
},
"sources": [
{
"id": "acme-10q",
"tier": 1,
"kind": "sec_filing",
"title": "Synthetic Form 10-Q",
"published_at": "2026-08-10T16:05:00-04:00",
"retrieved_at": "2026-08-22T10:05:00-07:00",
"url": "https://www.sec.gov/Archives/edgar/data/1000000/000100000026000010/acme-20260630.htm",
"supports": [
"financials.cash_classification",
"financials.cash_flow_ttm",
"financials.ebitda_ttm",
"financials.histories.diluted_shares",
"financials.revenue_ttm",
"financials.roic_pct",
"financials.sbc_ttm",
"financials.total_debt",
"latest_earnings",
"financials.cash_flow_periods.2025-09-30",
"financials.cash_flow_periods.2025-12-31",
"financials.cash_flow_periods.2026-03-31",
"financials.cash_flow_periods.2026-06-30"
]
},
{
"id": "acme-ir",
"tier": 2,
"kind": "company_ir",
"title": "Synthetic earnings release",
"published_at": "2026-08-10T16:05:00-04:00",
"retrieved_at": "2026-08-22T10:06:00-07:00",
"url": "https://investors.example.com/news/acme-q2-2026-results",
"supports": [
"latest_earnings",
"corporate_action_check"
]
},
{
"id": "corporate-action-check",
"tier": 3,
"kind": "exchange_status",
"title": "Synthetic active-listing and M&A status check",
"published_at": "2026-08-22T08:00:00-07:00",
"retrieved_at": "2026-08-22T08:05:00-07:00",
"url": "https://www.nasdaq.com/market-activity/stocks/acme",
"supports": [
"corporate_action_check"
]
},
{
"id": "consensus-20260822",
"tier": 3,
"kind": "consensus",
"title": "Synthetic GAAP EPS consensus",
"published_at": "2026-08-22T09:00:00-07:00",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"url": "https://example.com/consensus",
"supports": [
"valuation_case.periods.current",
"valuation_case.periods.year_2",
"valuation_case.periods.year_3"
]
},
{
"id": "analyst-model",
"tier": 4,
"kind": "analyst_calculation",
"title": "Synthetic forecast bridge",
"published_at": "2026-08-22T11:00:00-07:00",
"retrieved_at": "2026-08-22T11:00:00-07:00",
"url": "local://analyst-model",
"supports": [
"forecast_bridge"
]
},
{
"id": "peer-data",
"tier": 3,
"kind": "market_data",
"title": "Synthetic peer forward GAAP P/E set",
"published_at": "2026-08-22T09:00:00-07:00",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"url": "https://example.com/peers",
"supports": [
"peers"
]
}
]
}
],
"global_sources": [
{
"id": "quote-20260821",
"tier": 3,
"kind": "market_data",
"title": "Synthetic regular-session closing quote snapshot",
"published_at": "2026-08-21T16:00:00-04:00",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"url": "https://example.com/quote",
"supports": [
"identity.price",
"price_basis"
]
},
{
"id": "macro-fed-20260822",
"tier": 1,
"kind": "official_macro",
"title": "Synthetic Federal Reserve policy-rate record",
"published_at": "2026-08-21T12:00:00-04:00",
"retrieved_at": "2026-08-22T09:05:00-07:00",
"url": "https://www.federalreserve.gov/monetarypolicy/fomccalendars.htm",
"supports": [
"market_context.policy_rate_pct",
"market_context.summary"
]
},
{
"id": "macro-treasury-20260822",
"tier": 1,
"kind": "official_macro",
"title": "Synthetic Treasury yield record",
"published_at": "2026-08-21T18:00:00-04:00",
"retrieved_at": "2026-08-22T09:06:00-07:00",
"url": "https://home.treasury.gov/resource-center/data-chart-center/interest-rates",
"supports": [
"market_context.treasury_10y_yield_pct",
"market_context.summary"
]
},
{
"id": "macro-bea-bls-20260822",
"tier": 1,
"kind": "official_statistics",
"title": "Synthetic GDP and inflation release set",
"published_at": "2026-08-20T08:30:00-04:00",
"retrieved_at": "2026-08-22T09:07:00-07:00",
"url": "https://www.bea.gov/news",
"supports": [
"market_context.real_gdp_growth_pct",
"market_context.inflation_yoy_pct"
]
},
{
"id": "market-valuation-20260822",
"tier": 3,
"kind": "market_data",
"title": "Synthetic US market forward valuation snapshot",
"published_at": "2026-08-21T16:00:00-04:00",
"retrieved_at": "2026-08-22T09:08:00-07:00",
"url": "https://example.com/valuation",
"supports": [
"market_context.market_forward_pe",
"market_context.small_mid_cap_valuation_context"
]
},
{
"id": "universe-fmp-20260822",
"tier": 3,
"kind": "market_data",
"title": "Synthetic screened US common-stock universe",
"published_at": "2026-08-21T16:00:00-04:00",
"retrieved_at": "2026-08-22T09:10:00-07:00",
"url": "https://example.com/universe",
"supports": [
"screening_audit",
"screening_funnel"
]
}
],
"screening_audit": {
"audit_schema_version": 3,
"contract_revision": "3.5",
"runtime": {
"skill_name": "us-undervalued-growth-screener",
"skill_version": "3.6.1",
"schema_version": 3,
"contract_revision": "3.5",
"runtime_fingerprint": "ug-v3.6.1-claude-code-direct-fmp-20260830"
},
"conclusion_scope": "full_listing_universe",
"generated_at": "2026-08-28T14:28:34.377069+00:00",
"analysis_as_of": "2026-08-22T14:00:00-07:00",
"candidate_generation_mode": "full_universe_fundamentals",
"candidate_pool_status": "sufficient",
"selection_outcome": "selected",
"selected_symbols": [
"ACME",
"CYCLE",
"STALE"
],
"deep_dive_plan": {
"selected_symbols": [
"ACME",
"CYCLE",
"STALE"
],
"selected_count": 3,
"all_selected_must_be_resolved": true,
"budget_locked": true,
"budget_change_requires_rescreen": true,
"user_confirmation_required": false,
"user_continue_instruction_allowed": false,
"selected_set_is_committed": true,
"max_deep_dive_candidates": 3,
"selected_set_sha256": "3a45c382a4cdff829d63e0b378e7705866ca1d689de6ec7934381d30ba359065",
"commitment_payload": {
"analysis_as_of": "2026-08-22T14:00:00-07:00",
"max_deep_dive_candidates": 3,
"selected_symbols": [
"ACME",
"CYCLE",
"STALE"
]
}
},
"filters": {
"requested_min_market_cap": 500000000,
"requested_max_market_cap": 20000000000,
"min_market_cap": 500000000,
"max_market_cap": 20000000000,
"min_price": 5.0,
"min_average_daily_dollar_volume": 5000000,
"hard_min_average_daily_dollar_volume": 1000000,
"minimum_average_volume_period_days": 20,
"max_deep_dive_candidates": 3,
"preferred_forward_pe": 20.0,
"high_growth_exception_max_forward_pe": 30.0,
"near_miss_max_forward_pe": 22.0,
"preferred_ev_to_fcf": 20.0,
"preferred_fcf_yield_pct": 5.0,
"minimum_revenue_growth_pct": 8.0,
"minimum_per_share_growth_pct": 12.0,
"near_miss_min_per_share_growth_pct": 8.0,
"high_growth_exception_growth_pct": 20.0,
"minimum_roic_pct": 8.0,
"preferred_max_dilution_pct": 3.0,
"preferred_max_net_debt_to_ebitda": 2.5,
"hard_max_net_debt_to_ebitda": 4.0,
"maximum_forward_pe_for_economic_screen": 60.0,
"minimum_listing_data_coverage_pct": 95.0,
"minimum_discovery_analyst_count": 2,
"maximum_enrichment_attempts": 60,
"sector_review_selection_penalty": 5.0,
"near_miss_selection_penalty": 4.0,
"missing_deep_dive_field_penalty": 1.5,
"forward_pe_reconciliation_tolerance_pct": 5.0,
"maximum_forward_eps_dispersion_pct": 100.0,
"max_estimate_age_days": 45,
"maximum_fy1_horizon_days": 430
},
"scope": {
"requested_min_market_cap": 500000000.0,
"requested_max_market_cap": 20000000000.0,
"user_requested_scope": {
"min_market_cap": 500000000.0,
"max_market_cap": 20000000000.0,
"source": "skill_default"
},
"retrieval_min_market_cap": 500000000.0,
"retrieval_max_market_cap": 20000000000.0,
"retrieval_scope_explicit": true,
"scope_override_authorized": false,
"scope_reduced": false,
"scope_reduction_mode": "none",
"scope_reduction_disclosed": true,
"scope_reduction_reason": null,
"user_scope_evidence": null,
"user_requested_scope_complete": true,
"executed_scope_complete": true,
"bounded_sampling_ready": false,
"screening_scope_ready": true,
"executed_scope": {
"min_market_cap": 500000000.0,
"max_market_cap": 20000000000.0
},
"scope_complete": true,
"reasons": [],
"enumeration": {
"verified": true,
"provider_reported_total": 6,
"rows_fetched": 6,
"pages_fetched": 1,
"pagination_exhausted": true,
"band_audit": [],
"bands_well_formed": false,
"bands_cover_executed_range": false,
"bands_cover_requested_range": false,
"bands_verified": false
}
},
"universe": {
"row_count": 6,
"decision_counts": {
"in_scope": 4,
"excluded": 1,
"out_of_scope": 1
},
"listing_data_complete_count": 6,
"listing_data_complete_pct": 100.0,
"in_scope_count": 4,
"source_ids": [
"universe-fmp-20260822"
],
"artifact_path": "universe-audit-results.example.jsonl",
"artifact_sha256": "24e07ba0d17aa88df007e6e42df33dc2cf329326b5732f3521f57f529f8dee58"
},
"candidate_pool": {
"row_count": 6,
"decision_counts": {
"selected": 3,
"screened_out": 1,
"excluded": 2
},
"discovery_evaluable_count": 4,
"discovery_evaluable_pct": 66.6667,
"selection_eligible_count": 3,
"selected_count": 3,
"in_scope_covered_count": 4,
"in_scope_missing_count": 0,
"in_scope_missing_symbols": [],
"coverage_complete": true,
"coverage_scope": "full_listing_universe",
"listing_coverage_complete": true,
"generation_audit": {
"valid": true,
"selection_method": "full_universe_fundamentals",
"input_row_count": 6,
"selected_count": 6,
"selected_symbols": [
"ACME",
"CYCLE",
"MNA",
"OTCX",
"SCREEN",
"STALE"
],
"coverage_plan": {
"coverage_plan_valid": true,
"user_requested_range_spanned": true
}
},
"generation_review_reasons": [],
"fundamental_complete_count": 6,
"fundamental_complete_pct": 100.0,
"source_ids": [
"universe-fmp-20260822"
],
"symbols_not_in_universe": [],
"artifact_path": "broad-screen-results.example.jsonl",
"artifact_sha256": "55847bd84f650acd74f3381b2c2e125ed2e505f593d66f06085952f681e7ae3f"
},
"enrichment": {
"status": "complete",
"next_action": "proceed_to_deep_dive",
"discovery_pool_required": false,
"attempted_count": 6,
"resolved_count": 6,
"unresolved_count": 0,
"resolution_pct": 100.0,
"all_rows_resolved": true,
"maximum_attempts": 60,
"candidate_pool_exhaustion_declared": true,
"candidate_pool_exhausted": true,
"candidate_pool_covers_in_scope": true,
"candidate_pool_scope_verified": true,
"conclusion_scope": "full_listing_universe",
"queue_count": 0,
"queue_symbols": [],
"artifact_path": "enrichment-queue.example.json",
"artifact_sha256": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570"
},
"coverage_interpretation": {
"listing_universe_audited": true,
"universe_enumeration_verified": true,
"full_universe_fundamentals_claimed": true,
"candidate_pool_fundamentals_only": false,
"full_market_fundamental_coverage_is_not_a_completion_gate": true,
"no_candidates_requires_zero_unresolved_rows_and_exhausted_pool": true,
"bounded_pool_final_ranking_allowed_when_generation_is_audited": true,
"market_wide_no_candidates_requires_full_listing_universe_coverage": true
},
"row_count": 6,
"decision_counts": {
"selected": 3,
"screened_out": 1,
"excluded": 2
},
"source_ids": [
"universe-fmp-20260822"
]
},
"runtime": {
"skill_name": "us-undervalued-growth-screener",
"skill_version": "3.6.1",
"schema_version": 3,
"contract_revision": "3.5",
"runtime_fingerprint": "ug-v3.6.1-claude-code-direct-fmp-20260830"
}
}
assets/claude-code-config.example.json
{
"min_market_cap": 500000000,
"max_market_cap": 20000000000,
"min_price": 5.0,
"hard_min_average_daily_dollar_volume": 1000000,
"min_average_daily_dollar_volume": 5000000,
"max_api_calls": 350,
"company_screener_limit": 1000,
"minimum_market_cap_band_width": 25000000,
"maximum_market_cap_band_depth": 12,
"bulk_estimate_years": 5,
"bulk_estimate_minimum_coverage_pct": 20.0,
"pre_enrichment_limit": 180,
"seed_limit_cap": 200,
"quality_probe_limit": 35,
"candidate_packet_reserve_calls": 30,
"retry_reserve_calls": 25,
"exact_liquidity_limit": 40,
"provider_prefilter_pool_size": 30,
"provider_prefilter_minimum_pool": 12,
"provider_prefilter_per_lane": 8,
"max_deep_dive_candidates": 3,
"full_snapshot_pool_size": 50,
"full_snapshot_deep_dive_candidates": 5,
"full_snapshot_max_staleness_days": 7,
"full_snapshot_screening_clock_skew_seconds": 300,
"full_snapshot_collection_clock_skew_seconds": 300,
"minimum_discovery_analyst_count": 2,
"minimum_average_volume_period_days": 20,
"maximum_forward_eps_dispersion_pct": 100.0,
"maximum_fy1_horizon_days": 430,
"forward_pe_reconciliation_tolerance_pct": 5.0,
"compact_stdout_max_bytes": 20000,
"bulk_endpoints": {
"ratios_ttm": "ratios-ttm-bulk",
"key_metrics_ttm": "key-metrics-ttm-bulk",
"income_growth": "income-statement-growth-bulk",
"analyst_estimates": "analyst-estimates-bulk",
"eod": "eod-bulk"
},
"cache": {
"path": ".cache/us-garp/fmp-cache.sqlite3",
"raw_store_dir": ".cache/us-garp/raw"
}
}
assets/discovery-audit.example.json
{
"schema_version": 3,
"runtime": {
"skill_name": "us-undervalued-growth-screener",
"skill_version": "3.6.1",
"schema_version": 3,
"contract_revision": "3.5",
"runtime_fingerprint": "ug-v3.6.1-claude-code-direct-fmp-20260830"
},
"generated_at": "2026-08-28T14:42:38.687447+00:00",
"valid": true,
"input_row_count": 3,
"in_scope_row_count": 3,
"selected_count": 3,
"selection_method": "sector_market_cap_stratified_validated_liquidity",
"max_pool": 5,
"per_sector_bucket_limit": 2,
"market_cap_buckets": [
500000000,
2000000000,
5000000000,
10000000000,
20000000000
],
"selected_symbols": [
"LISTA",
"LISTB",
"LISTC"
],
"selected_by_sector": {
"Healthcare": 1,
"Industrials": 1,
"Technology": 1
},
"selected_by_market_cap_bucket": {
"2000000000-5000000000": 1,
"500000000-2000000000": 1,
"5000000000-10000000000": 1
},
"exclusion_counts": {},
"liquidity_validation": {
"basis_validated": true,
"minimum_window_days": 20,
"required_minimum_window_days": 20,
"selected_with_validated_liquidity": 3,
"unverified_reason_counts": {}
},
"coverage_plan": {
"method": "sector_market_cap_stratified_validated_liquidity",
"user_requested_range_spanned": true,
"market_cap_buckets_cover_user_requested_range": true,
"input_exchange_coverage": [
"NASDAQ",
"NYSE",
"NYSE AMERICAN"
],
"required_exchange_families": [
"NYSE",
"NASDAQ",
"NYSE AMERICAN"
],
"single_band_only": false,
"coverage_plan_valid": true,
"bounded_pool_disclosed": true
},
"scope": {
"user_requested_min_market_cap": 500000000.0,
"user_requested_max_market_cap": 20000000000.0,
"executed_min_market_cap": 500000000.0,
"executed_max_market_cap": 20000000000.0,
"scope_reduced": false,
"scope_reduction_authorized": false,
"scope_reduction_mode": "none",
"scope_reduction_disclosed": true,
"scope_reduction_reason": null,
"user_scope_evidence": null,
"user_requested_scope_complete": true,
"scope_valid": true
},
"source_ids": [
"universe-fmp-20260822"
],
"artifact_path": "discovery-pool.example.jsonl",
"artifact_sha256": "e91fcff25394fafdd125c609885d0d3362d6c035c79c868953cabcc5ebd8bca4"
}
assets/discovery-pool.example.jsonl
{"average_daily_dollar_volume":20000000.0,"average_volume":null,"company_name":"Listing Only A","discovery_rank":1,"discovery_reason":"sector_market_cap_stratified_validated_liquidity","exchange":"NASDAQ","industry":"Software","liquidity_basis_valid":true,"liquidity_method":"provider_average_dollar_volume","liquidity_source_ids":["universe-fmp-20260822"],"liquidity_window_days":20,"market_cap":2500000000.0,"market_cap_bucket":"2000000000-5000000000","price":25.0,"sector":"Technology","symbol":"LISTA"}
{"average_daily_dollar_volume":15000000.0,"average_volume":null,"company_name":"Listing Only B","discovery_rank":2,"discovery_reason":"sector_market_cap_stratified_validated_liquidity","exchange":"NYSE","industry":"Machinery","liquidity_basis_valid":true,"liquidity_method":"provider_average_dollar_volume","liquidity_source_ids":["universe-fmp-20260822"],"liquidity_window_days":20,"market_cap":6000000000.0,"market_cap_bucket":"5000000000-10000000000","price":40.0,"sector":"Industrials","symbol":"LISTB"}
{"average_daily_dollar_volume":3000000.0,"average_volume":null,"company_name":"Listing Only C","discovery_rank":3,"discovery_reason":"sector_market_cap_stratified_validated_liquidity","exchange":"NYSE AMERICAN","industry":"Medical Devices","liquidity_basis_valid":true,"liquidity_method":"provider_average_dollar_volume","liquidity_source_ids":["universe-fmp-20260822"],"liquidity_window_days":20,"market_cap":900000000.0,"market_cap_bucket":"500000000-2000000000","price":12.0,"sector":"Healthcare","symbol":"LISTC"}
assets/enriched-candidate-pool.example.jsonl
{"analyst_count": 6, "average_daily_dollar_volume": 20000000.0, "average_volume": null, "company_name": "Listing Only A", "discovery_rank": 1, "discovery_reason": "sector_market_cap_stratified_validated_liquidity", "enrichment_attempted": true, "enrichment_exhausted": false, "enrichment_exhaustion_reason": null, "enrichment_resolved": true, "enrichment_source_ids": ["consensus-20260822"], "eps_growth_pct": 26.470776, "estimate_normalization_reasons": [], "estimate_normalization_status": "valid", "estimate_periods": [{"eps_analyst_count": 6, "eps_avg": 1.5, "eps_high": 1.6, "eps_low": 1.4, "fiscal_year": "2026", "period": "FY2026", "period_end": "2026-12-31T00:00:00+00:00", "revenue_analyst_count": 6, "revenue_avg": 500000000.0}, {"eps_analyst_count": 6, "eps_avg": 1.9, "eps_high": 2.05, "eps_low": 1.75, "fiscal_year": "2027", "period": "FY2027", "period_end": "2027-12-31T00:00:00+00:00", "revenue_analyst_count": 6, "revenue_avg": 575000000.0}, {"eps_analyst_count": 5, "eps_avg": 2.4, "eps_high": 2.6, "eps_low": 2.2, "fiscal_year": "2028", "period": "FY2028", "period_end": "2028-12-31T00:00:00+00:00", "revenue_analyst_count": 5, "revenue_avg": 670000000.0}], "estimate_series_contiguous": true, "exchange": "NASDAQ", "forward_eps": 1.5, "forward_eps_analyst_count": 6, "forward_eps_high": 1.6, "forward_eps_low": 1.4, "forward_estimate_as_of": "2026-08-22T09:00:00-07:00", "forward_estimate_dispersion_pct": 13.333333333333345, "forward_estimate_source_ids": ["consensus-20260822"], "forward_fiscal_year": "2026", "forward_metric_basis": "fy1", "forward_metric_origin": "computed_from_price_and_fy1_eps", "forward_metric_period": "FY1", "forward_metric_period_end": "2026-12-31T00:00:00+00:00", "forward_metric_source_ids": ["consensus-20260822"], "forward_pe": 16.666667, "forward_pe_period": "FY1", "forward_period_end": "2026-12-31T00:00:00+00:00", "fy1_analyst_count": 6, "fy1_eps": 1.5, "fy1_horizon_days": 130, "fy1_pe": 16.666667, "growth_estimate_source_ids": ["consensus-20260822"], "growth_horizon_end_period": "FY2028", "growth_horizon_end_period_end": "2028-12-31T00:00:00+00:00", "growth_horizon_start_period": "FY1", "growth_horizon_start_period_end": "2026-12-31T00:00:00+00:00", "growth_horizon_years": 2.001369, "industry": "Software", "liquidity_basis_valid": true, "liquidity_method": "provider_average_dollar_volume", "liquidity_source_ids": ["universe-fmp-20260822"], "liquidity_window_days": 20, "market_cap": 2500000000.0, "market_cap_bucket": "2000000000-5000000000", "normalized_estimates_version": "3.6.0", "price": 25.0, "raw_forward_candidate": {"analyst_count": 6, "computed_pe": 16.666667, "eps": 1.5, "eps_high": 1.6, "eps_low": 1.4, "fiscal_year": "2026", "period_end": "2026-12-31T00:00:00+00:00"}, "revenue_growth_pct": 15.746783, "sector": "Technology", "symbol": "LISTA", "country": "US", "currency": "USD"}
{"analyst_count": 4, "average_daily_dollar_volume": 15000000.0, "average_volume": null, "company_name": "Listing Only B", "discovery_rank": 2, "discovery_reason": "sector_market_cap_stratified_validated_liquidity", "enrichment_attempted": true, "enrichment_exhausted": false, "enrichment_exhaustion_reason": null, "enrichment_resolved": true, "enrichment_source_ids": ["consensus-20260822"], "eps_growth_pct": 30.360389, "estimate_normalization_reasons": [], "estimate_normalization_status": "valid", "estimate_periods": [{"eps_analyst_count": 4, "eps_avg": 2.0, "eps_high": 2.2, "eps_low": 1.8, "fiscal_year": "2026", "period": "FY2026", "period_end": "2026-12-31T00:00:00+00:00", "revenue_analyst_count": 4, "revenue_avg": 800000000.0}, {"eps_analyst_count": 4, "eps_avg": 2.6, "eps_high": 2.85, "eps_low": 2.35, "fiscal_year": "2027", "period": "FY2027", "period_end": "2027-12-31T00:00:00+00:00", "revenue_analyst_count": 4, "revenue_avg": 950000000.0}, {"eps_analyst_count": 4, "eps_avg": 3.4, "eps_high": 3.8, "eps_low": 3.0, "fiscal_year": "2028", "period": "FY2028", "period_end": "2028-12-31T00:00:00+00:00", "revenue_analyst_count": 4, "revenue_avg": 1150000000.0}], "estimate_series_contiguous": true, "exchange": "NYSE", "forward_eps": 2.0, "forward_eps_analyst_count": 4, "forward_eps_high": 2.2, "forward_eps_low": 1.8, "forward_estimate_as_of": "2026-08-22T09:00:00-07:00", "forward_estimate_dispersion_pct": 20.000000000000007, "forward_estimate_source_ids": ["consensus-20260822"], "forward_fiscal_year": "2026", "forward_metric_basis": "fy1", "forward_metric_origin": "computed_from_price_and_fy1_eps", "forward_metric_period": "FY1", "forward_metric_period_end": "2026-12-31T00:00:00+00:00", "forward_metric_source_ids": ["consensus-20260822"], "forward_pe": 20.0, "forward_pe_period": "FY1", "forward_period_end": "2026-12-31T00:00:00+00:00", "fy1_analyst_count": 4, "fy1_eps": 2.0, "fy1_horizon_days": 130, "fy1_pe": 20.0, "growth_estimate_source_ids": ["consensus-20260822"], "growth_horizon_end_period": "FY2028", "growth_horizon_end_period_end": "2028-12-31T00:00:00+00:00", "growth_horizon_start_period": "FY1", "growth_horizon_start_period_end": "2026-12-31T00:00:00+00:00", "growth_horizon_years": 2.001369, "industry": "Machinery", "liquidity_basis_valid": true, "liquidity_method": "provider_average_dollar_volume", "liquidity_source_ids": ["universe-fmp-20260822"], "liquidity_window_days": 20, "market_cap": 6000000000.0, "market_cap_bucket": "5000000000-10000000000", "normalized_estimates_version": "3.6.0", "price": 40.0, "raw_forward_candidate": {"analyst_count": 4, "computed_pe": 20.0, "eps": 2.0, "eps_high": 2.2, "eps_low": 1.8, "fiscal_year": "2026", "period_end": "2026-12-31T00:00:00+00:00"}, "revenue_growth_pct": 19.880908, "sector": "Industrials", "symbol": "LISTB", "country": "US", "currency": "USD"}
{"analyst_count": 3, "average_daily_dollar_volume": 3000000.0, "average_volume": null, "company_name": "Listing Only C", "discovery_rank": 3, "discovery_reason": "sector_market_cap_stratified_validated_liquidity", "enrichment_attempted": true, "enrichment_exhausted": false, "enrichment_exhaustion_reason": null, "enrichment_resolved": true, "enrichment_source_ids": ["consensus-20260822"], "eps_growth_pct": 6.453573, "estimate_normalization_reasons": [], "estimate_normalization_status": "valid", "estimate_periods": [{"eps_analyst_count": 3, "eps_avg": 0.3, "eps_high": 0.35, "eps_low": 0.25, "fiscal_year": "2026", "period": "FY2026", "period_end": "2026-12-31T00:00:00+00:00", "revenue_analyst_count": 3, "revenue_avg": 180000000.0}, {"eps_analyst_count": 3, "eps_avg": 0.32, "eps_high": 0.37, "eps_low": 0.27, "fiscal_year": "2027", "period": "FY2027", "period_end": "2027-12-31T00:00:00+00:00", "revenue_analyst_count": 3, "revenue_avg": 185000000.0}, {"eps_analyst_count": 3, "eps_avg": 0.34, "eps_high": 0.4, "eps_low": 0.28, "fiscal_year": "2028", "period": "FY2028", "period_end": "2028-12-31T00:00:00+00:00", "revenue_analyst_count": 3, "revenue_avg": 190000000.0}], "estimate_series_contiguous": true, "exchange": "NYSE AMERICAN", "forward_eps": 0.3, "forward_eps_analyst_count": 3, "forward_eps_high": 0.35, "forward_eps_low": 0.25, "forward_estimate_as_of": "2026-08-22T09:00:00-07:00", "forward_estimate_dispersion_pct": 33.33333333333333, "forward_estimate_source_ids": ["consensus-20260822"], "forward_fiscal_year": "2026", "forward_metric_basis": "fy1", "forward_metric_origin": "computed_from_price_and_fy1_eps", "forward_metric_period": "FY1", "forward_metric_period_end": "2026-12-31T00:00:00+00:00", "forward_metric_source_ids": ["consensus-20260822"], "forward_pe": 40.0, "forward_pe_period": "FY1", "forward_period_end": "2026-12-31T00:00:00+00:00", "fy1_analyst_count": 3, "fy1_eps": 0.3, "fy1_horizon_days": 130, "fy1_pe": 40.0, "growth_estimate_source_ids": ["consensus-20260822"], "growth_horizon_end_period": "FY2028", "growth_horizon_end_period_end": "2028-12-31T00:00:00+00:00", "growth_horizon_start_period": "FY1", "growth_horizon_start_period_end": "2026-12-31T00:00:00+00:00", "growth_horizon_years": 2.001369, "industry": "Medical Devices", "liquidity_basis_valid": true, "liquidity_method": "provider_average_dollar_volume", "liquidity_source_ids": ["universe-fmp-20260822"], "liquidity_window_days": 20, "market_cap": 900000000.0, "market_cap_bucket": "500000000-2000000000", "normalized_estimates_version": "3.6.0", "price": 12.0, "raw_forward_candidate": {"analyst_count": 3, "computed_pe": 40.0, "eps": 0.3, "eps_high": 0.35, "eps_low": 0.25, "fiscal_year": "2026", "period_end": "2026-12-31T00:00:00+00:00"}, "revenue_growth_pct": 2.738334, "sector": "Healthcare", "symbol": "LISTC", "country": "US", "currency": "USD"}
assets/enrichment-queue.example.json
[]
assets/global-sources.example.json
[
{
"id": "quote-20260821",
"tier": 3,
"kind": "market_data",
"title": "Synthetic regular-session closing quote snapshot",
"published_at": "2026-08-21T16:00:00-04:00",
"retrieved_at": "2026-08-22T09:00:00-07:00",
"url": "https://example.com/quote",
"supports": [
"identity.price",
"price_basis"
]
},
{
"id": "macro-fed-20260822",
"tier": 1,
"kind": "official_macro",
"title": "Synthetic Federal Reserve policy-rate record",
"published_at": "2026-08-21T12:00:00-04:00",
"retrieved_at": "2026-08-22T09:05:00-07:00",
"url": "https://www.federalreserve.gov/monetarypolicy/fomccalendars.htm",
"supports": [
"market_context.policy_rate_pct",
"market_context.summary"
]
},
{
"id": "macro-treasury-20260822",
"tier": 1,
"kind": "official_macro",
"title": "Synthetic Treasury yield record",
"published_at": "2026-08-21T18:00:00-04:00",
"retrieved_at": "2026-08-22T09:06:00-07:00",
"url": "https://home.treasury.gov/resource-center/data-chart-center/interest-rates",
"supports": [
"market_context.treasury_10y_yield_pct",
"market_context.summary"
]
},
{
"id": "macro-bea-bls-20260822",
"tier": 1,
"kind": "official_statistics",
"title": "Synthetic GDP and inflation release set",
"published_at": "2026-08-20T08:30:00-04:00",
"retrieved_at": "2026-08-22T09:07:00-07:00",
"url": "https://www.bea.gov/news",
"supports": [
"market_context.real_gdp_growth_pct",
"market_context.inflation_yoy_pct"
]
},
{
"id": "market-valuation-20260822",
"tier": 3,
"kind": "market_data",
"title": "Synthetic US market forward valuation snapshot",
"published_at": "2026-08-21T16:00:00-04:00",
"retrieved_at": "2026-08-22T09:08:00-07:00",
"url": "https://example.com/valuation",
"supports": [
"market_context.market_forward_pe",
"market_context.small_mid_cap_valuation_context"
]
},
{
"id": "universe-fmp-20260822",
"tier": 3,
"kind": "market_data",
"title": "Synthetic screened US common-stock universe",
"published_at": "2026-08-21T16:00:00-04:00",
"retrieved_at": "2026-08-22T09:10:00-07:00",
"url": "https://example.com/universe",
"supports": [
"screening_audit",
"screening_funnel"
]
}
]
assets/listing-only-input.example.jsonl
{"average_daily_dollar_volume":20000000,"average_daily_dollar_volume_method":"provider_average_dollar_volume","average_volume_period_days":20,"company_name":"Listing Only A","exchange":"NASDAQ","industry":"Software","is_actively_trading":true,"is_common_stock":true,"liquidity_source_ids":["universe-fmp-20260822"],"liquidity_valid_for_screen":true,"market_cap":2500000000,"price":25.0,"sector":"Technology","symbol":"LISTA"}
{"average_daily_dollar_volume":15000000,"average_daily_dollar_volume_method":"provider_average_dollar_volume","average_volume_period_days":20,"company_name":"Listing Only B","exchange":"NYSE","industry":"Machinery","is_actively_trading":true,"is_common_stock":true,"liquidity_source_ids":["universe-fmp-20260822"],"liquidity_valid_for_screen":true,"market_cap":6000000000,"price":40.0,"sector":"Industrials","symbol":"LISTB"}
{"average_daily_dollar_volume":3000000,"average_daily_dollar_volume_method":"provider_average_dollar_volume","average_volume_period_days":20,"company_name":"Listing Only C","exchange":"NYSE AMERICAN","industry":"Medical Devices","is_actively_trading":true,"is_common_stock":true,"liquidity_source_ids":["universe-fmp-20260822"],"liquidity_valid_for_screen":true,"market_cap":900000000,"price":12.0,"sector":"Healthcare","symbol":"LISTC"}
assets/market-context.example.json
{
"summary": "US small- and mid-cap valuations are below the large-cap market, while elevated Treasury yields make multiple expansion an unsuitable base-case assumption.",
"as_of": "2026-08-22T12:00:00-07:00",
"policy_rate_pct": 3.625,
"treasury_10y_yield_pct": 4.74,
"real_gdp_growth_pct": 1.5,
"inflation_yoy_pct": 3.4,
"market_forward_pe": 21.2,
"small_mid_cap_valuation_context": "Small and mid caps trade at a discount to mega caps, but financing-sensitive businesses require a higher margin of safety.",
"sector_cycle_notes": [
"Software demand is stable but selective.",
"Commercial biopharma requires product-concentration and LOE stress checks."
],
"source_ids": [
"macro-fed-20260822",
"macro-treasury-20260822",
"macro-bea-bls-20260822",
"market-valuation-20260822"
]
}
assets/screening-config.example.json
{
"min_market_cap": 500000000,
"max_market_cap": 20000000000,
"min_price": 5.0,
"min_average_daily_dollar_volume": 5000000,
"hard_min_average_daily_dollar_volume": 1000000,
"max_candidates": 10,
"minimum_data_quality_score": 70,
"minimum_constant_multiple_upside_pct": 30,
"minimum_stressed_upside_pct": 0,
"multiple_contraction_pct": 20,
"maximum_cyclicality_without_normalization": 2,
"allow_special_cases": true,
"require_minimum_upside": true,
"require_positive_stress_case": false,
"max_quote_age_days": 7,
"max_estimate_age_days": 45,
"max_corporate_action_check_age_days": 14,
"forecast_bridge_tolerance_pct": 2,
"gaap_reconciliation_tolerance_pct": 2,
"fcf_reconciliation_tolerance_pct": 5,
"max_deep_dive_candidates": 5,
"cash_consistency_tolerance_pct": 3,
"max_market_context_age_days": 14,
"minimum_analyst_count_for_ranked_horizon": 3,
"require_forward_current_metric": true,
"require_peer_set": true,
"require_market_context": true,
"require_screening_audit": true,
"biopharma_loe_stress_multiples": [
6,
8
],
"require_full_scope": true,
"minimum_listing_data_coverage_pct": 95.0,
"discovery_pool_max_size": 120,
"discovery_pool_per_sector_market_cap_cell": 3,
"minimum_discovery_analyst_count": 2,
"maximum_enrichment_attempts": 60,
"max_discovery_pool_candidates": 120,
"discovery_per_sector_bucket_limit": 3,
"near_miss_max_forward_pe": 22.0,
"near_miss_min_per_share_growth_pct": 8.0,
"hard_max_net_debt_to_ebitda": 4.0,
"maximum_forward_pe_for_economic_screen": 60.0,
"sector_review_selection_penalty": 5.0,
"near_miss_selection_penalty": 4.0,
"missing_deep_dive_field_penalty": 1.5,
"max_policy_rate_source_age_days": 60,
"max_market_rate_source_age_days": 7,
"max_inflation_source_age_days": 45,
"max_gdp_source_age_days": 120,
"max_market_valuation_source_age_days": 14,
"max_small_mid_context_source_age_days": 14,
"requested_min_market_cap": 500000000,
"requested_max_market_cap": 20000000000,
"minimum_average_volume_period_days": 20,
"forward_pe_reconciliation_tolerance_pct": 5.0,
"maximum_forward_eps_dispersion_pct": 100.0,
"maximum_fy1_horizon_days": 430,
"selection_lane_quota_core_garp": 2,
"selection_lane_quota_high_growth": 1,
"selection_lane_quota_near_miss": 1,
"selection_lane_quota_cyclical": 1,
"maximum_selected_per_sector": 2,
"preferred_forward_pe": 20.0,
"high_growth_exception_max_forward_pe": 30.0,
"preferred_ev_to_fcf": 20.0,
"preferred_fcf_yield_pct": 5.0,
"minimum_revenue_growth_pct": 8.0,
"minimum_per_share_growth_pct": 12.0,
"high_growth_exception_growth_pct": 20.0,
"minimum_roic_pct": 8.0,
"preferred_max_dilution_pct": 3.0,
"preferred_max_net_debt_to_ebitda": 2.5,
"minimum_final_score_for_eligible": 70.0,
"minimum_sbc_adjusted_fcf_yield_pct_for_eligible": 3.0,
"maximum_ev_to_fcf_for_eligible": 30.0,
"minimum_roic_pct_for_eligible": 8.0,
"maximum_net_debt_to_ebitda_for_eligible": 3.0,
"maximum_dilution_pct_for_eligible": 5.0,
"minimum_low_case_upside_pct_for_eligible": 15.0,
"maximum_quality_gate_failures_for_conditional": 2,
"severe_loe_stress_downside_pct": -25.0,
"final_three_minimum_score": 75.0,
"final_three_minimum_data_quality": 80,
"final_three_minimum_stress_upside_pct": 0.0,
"final_three_minimum_sbc_adjusted_fcf_yield_pct": 5.0,
"final_three_maximum_ev_to_fcf": 20.0,
"final_three_minimum_low_case_upside_pct": 20.0
}
assets/universe-audit-results.example.jsonl
{"company_name":"Acme Software, Inc.","eligible_for_candidate_generation":true,"exchange":"NASDAQ","industry":null,"listing_data_complete":true,"listing_decision":{"reasons":[],"status":"in_scope"},"metrics":{"average_daily_dollar_volume":45000000.0,"average_daily_dollar_volume_method":"provider_average_dollar_volume","average_volume_period_days":20,"is_actively_trading":true,"is_common_stock":true,"liquidity_source_ids":["universe-fmp-20260822"],"liquidity_valid_for_screen":true,"market_cap":5000000000.0,"price":40.0},"sector":null,"symbol":"ACME"}
{"company_name":"Cycle Equipment Corp.","eligible_for_candidate_generation":true,"exchange":"NYSE","industry":null,"listing_data_complete":true,"listing_decision":{"reasons":[],"status":"in_scope"},"metrics":{"average_daily_dollar_volume":30000000.0,"average_daily_dollar_volume_method":"provider_average_dollar_volume","average_volume_period_days":20,"is_actively_trading":true,"is_common_stock":true,"liquidity_source_ids":["universe-fmp-20260822"],"liquidity_valid_for_screen":true,"market_cap":4000000000.0,"price":50.0},"sector":null,"symbol":"CYCLE"}
{"company_name":"Stale Quote Corp.","eligible_for_candidate_generation":true,"exchange":"NASDAQ","industry":null,"listing_data_complete":true,"listing_decision":{"reasons":[],"status":"in_scope"},"metrics":{"average_daily_dollar_volume":25000000.0,"average_daily_dollar_volume_method":"provider_average_dollar_volume","average_volume_period_days":20,"is_actively_trading":true,"is_common_stock":true,"liquidity_source_ids":["universe-fmp-20260822"],"liquidity_valid_for_screen":true,"market_cap":3000000000.0,"price":35.0},"sector":null,"symbol":"STALE"}
{"company_name":"Screened Out Inc.","eligible_for_candidate_generation":true,"exchange":"NASDAQ","industry":null,"listing_data_complete":true,"listing_decision":{"reasons":[],"status":"in_scope"},"metrics":{"average_daily_dollar_volume":15000000.0,"average_daily_dollar_volume_method":"provider_average_dollar_volume","average_volume_period_days":20,"is_actively_trading":true,"is_common_stock":true,"liquidity_source_ids":["universe-fmp-20260822"],"liquidity_valid_for_screen":true,"market_cap":2000000000.0,"price":20.0},"sector":null,"symbol":"SCREEN"}
{"company_name":"Acquired Company, Inc.","eligible_for_candidate_generation":false,"exchange":"NASDAQ","industry":null,"listing_data_complete":true,"listing_decision":{"reasons":["inactive_symbol"],"status":"excluded"},"metrics":{"average_daily_dollar_volume":null,"average_daily_dollar_volume_method":"provider_average_dollar_volume","average_volume_period_days":20,"is_actively_trading":false,"is_common_stock":true,"liquidity_source_ids":["universe-fmp-20260822"],"liquidity_valid_for_screen":false,"market_cap":1500000000.0,"price":31.49},"sector":null,"symbol":"MNA"}
{"company_name":"OTC Example Corp.","eligible_for_candidate_generation":false,"exchange":"OTC","industry":null,"listing_data_complete":true,"listing_decision":{"reasons":["exchange_out_of_scope"],"status":"out_of_scope"},"metrics":{"average_daily_dollar_volume":2000000.0,"average_daily_dollar_volume_method":"provider_average_dollar_volume","average_volume_period_days":20,"is_actively_trading":true,"is_common_stock":true,"liquidity_source_ids":["universe-fmp-20260822"],"liquidity_valid_for_screen":true,"market_cap":800000000.0,"price":8.0},"sector":null,"symbol":"OTCX"}
assets/universe-band-audit.example.json
[
{
"min_market_cap": 500000000,
"max_market_cap": 20000000000,
"rows_fetched": 6,
"provider_exhausted": true
}
]
assets/universe-input.example.jsonl
{"analyst_count":4,"average_daily_dollar_volume":45000000,"average_daily_dollar_volume_method":"provider_average_dollar_volume","average_volume_period_days":20,"common_stock":true,"company_name":"Acme Software, Inc.","cyclicality_score":1,"dilution_pct":-1.9,"eps_growth_pct":22.0,"estimate_series_contiguous":true,"ev_to_fcf":27.33,"exchange":"NASDAQ","fcf_yield_pct":3.6,"forward_eps":3.0007501875468865,"forward_estimate_as_of":"2026-08-22T14:00:00-07:00","forward_estimate_source_ids":["universe-fmp-20260822"],"forward_fiscal_year":"FY2027","forward_metric_origin":"provider_fy1_pe_with_eps_reconciliation","forward_pe":13.33,"forward_pe_period":"FY1","forward_period_end":"2027-06-30","forward_validation_reasons":[],"is_actively_trading":true,"liquidity_source_ids":["universe-fmp-20260822"],"liquidity_valid_for_screen":true,"market_cap":5000000000,"net_debt_to_ebitda":-0.42,"price":40.0,"revenue_growth_pct":14.0,"roic_pct":18.5,"standard_fcf":180000000,"symbol":"ACME"}
{"analyst_count":4,"average_daily_dollar_volume":30000000,"average_daily_dollar_volume_method":"provider_average_dollar_volume","average_volume_period_days":20,"common_stock":true,"company_name":"Cycle Equipment Corp.","cyclicality_score":4,"dilution_pct":0.5,"eps_growth_pct":18.0,"estimate_series_contiguous":true,"exchange":"NYSE","fcf_yield_pct":5.5,"forward_eps":3.5714285714285716,"forward_estimate_as_of":"2026-08-22T14:00:00-07:00","forward_estimate_source_ids":["universe-fmp-20260822"],"forward_fiscal_year":"FY2027","forward_metric_origin":"provider_fy1_pe_with_eps_reconciliation","forward_pe":14.0,"forward_pe_period":"FY1","forward_period_end":"2027-06-30","forward_validation_reasons":[],"is_actively_trading":true,"liquidity_source_ids":["universe-fmp-20260822"],"liquidity_valid_for_screen":true,"market_cap":4000000000,"net_debt_to_ebitda":1.0,"normalized_eps":3.2,"price":50.0,"revenue_growth_pct":12.0,"roic_pct":15.0,"standard_fcf":220000000,"symbol":"CYCLE"}
{"analyst_count":4,"average_daily_dollar_volume":25000000,"average_daily_dollar_volume_method":"provider_average_dollar_volume","average_volume_period_days":20,"common_stock":true,"company_name":"Stale Quote Corp.","cyclicality_score":1,"dilution_pct":1.0,"eps_growth_pct":20.0,"estimate_series_contiguous":true,"exchange":"NASDAQ","fcf_yield_pct":5.3,"forward_eps":2.3333333333333335,"forward_estimate_as_of":"2026-08-22T14:00:00-07:00","forward_estimate_source_ids":["universe-fmp-20260822"],"forward_fiscal_year":"FY2027","forward_metric_origin":"provider_fy1_pe_with_eps_reconciliation","forward_pe":15.0,"forward_pe_period":"FY1","forward_period_end":"2027-06-30","forward_validation_reasons":[],"is_actively_trading":true,"liquidity_source_ids":["universe-fmp-20260822"],"liquidity_valid_for_screen":true,"market_cap":3000000000,"net_debt_to_ebitda":0.5,"price":35.0,"revenue_growth_pct":13.0,"roic_pct":17.0,"standard_fcf":160000000,"symbol":"STALE"}
{"analyst_count":4,"average_daily_dollar_volume":15000000,"average_daily_dollar_volume_method":"provider_average_dollar_volume","average_volume_period_days":20,"common_stock":true,"company_name":"Screened Out Inc.","cyclicality_score":1,"dilution_pct":1.0,"eps_growth_pct":5.0,"estimate_series_contiguous":true,"exchange":"NASDAQ","fcf_yield_pct":5.0,"forward_eps":0.6451612903225806,"forward_estimate_as_of":"2026-08-22T14:00:00-07:00","forward_estimate_source_ids":["universe-fmp-20260822"],"forward_fiscal_year":"FY2027","forward_metric_origin":"provider_fy1_pe_with_eps_reconciliation","forward_pe":31.0,"forward_pe_period":"FY1","forward_period_end":"2027-06-30","forward_validation_reasons":[],"is_actively_trading":true,"liquidity_source_ids":["universe-fmp-20260822"],"liquidity_valid_for_screen":true,"market_cap":2000000000,"net_debt_to_ebitda":1.0,"price":20.0,"revenue_growth_pct":-4.0,"roic_pct":7.0,"standard_fcf":100000000,"symbol":"SCREEN"}
{"analyst_count":4,"average_daily_dollar_volume":0,"average_daily_dollar_volume_method":"provider_average_dollar_volume","average_volume_period_days":20,"common_stock":true,"company_name":"Acquired Company, Inc.","cyclicality_score":1,"dilution_pct":0.0,"eps_growth_pct":20.0,"estimate_series_contiguous":true,"exchange":"NASDAQ","fcf_yield_pct":8.0,"forward_eps":1.7494444444444444,"forward_estimate_as_of":"2026-08-22T14:00:00-07:00","forward_estimate_source_ids":["universe-fmp-20260822"],"forward_fiscal_year":"FY2027","forward_metric_origin":"provider_fy1_pe_with_eps_reconciliation","forward_pe":18.0,"forward_pe_period":"FY1","forward_period_end":"2027-06-30","forward_validation_reasons":[],"is_actively_trading":false,"liquidity_source_ids":["universe-fmp-20260822"],"liquidity_valid_for_screen":true,"market_cap":1500000000,"net_debt_to_ebitda":0.0,"price":31.49,"revenue_growth_pct":20.0,"roic_pct":20.0,"standard_fcf":120000000,"symbol":"MNA"}
{"analyst_count":4,"average_daily_dollar_volume":2000000,"average_daily_dollar_volume_method":"provider_average_dollar_volume","average_volume_period_days":20,"common_stock":true,"company_name":"OTC Example Corp.","cyclicality_score":1,"dilution_pct":1.0,"eps_growth_pct":20.0,"estimate_series_contiguous":true,"exchange":"OTC","fcf_yield_pct":6.2,"forward_eps":0.6666666666666666,"forward_estimate_as_of":"2026-08-22T14:00:00-07:00","forward_estimate_source_ids":["universe-fmp-20260822"],"forward_fiscal_year":"FY2027","forward_metric_origin":"provider_fy1_pe_with_eps_reconciliation","forward_pe":12.0,"forward_pe_period":"FY1","forward_period_end":"2027-06-30","forward_validation_reasons":[],"is_actively_trading":true,"liquidity_source_ids":["universe-fmp-20260822"],"liquidity_valid_for_screen":true,"market_cap":800000000,"net_debt_to_ebitda":1.0,"price":8.0,"revenue_growth_pct":15.0,"roic_pct":15.0,"standard_fcf":50000000,"symbol":"OTCX"}
references/autonomous-execution.md
# Autonomous Execution Contract — v3.6 Runtime / Contract 3.5
## Purpose
Make a minimal instruction sufficient for a complete screen. The operator supplies the research request; the skill supplies defaults, current data acquisition, fallback routing, deterministic screening, primary-source underwriting, checkpointing, repair, and final reporting.
Never pause after an intermediate stage to ask for “continue.” Exit code `2` from a helper script is an internal continuation signal, not a user handoff.
In Claude Code, the mandatory default is the local direct-FMP runner. Do not consume the listing universe through repeated MCP calls. Run `run_pipeline.py`, keep raw payloads in its cache/artifact directories, and read only the compact summary, audits, and selected candidate packets.
## Runtime Identity
All run artifacts must contain the installed runtime metadata:
```json
{
"skill_name": "us-undervalued-growth-screener",
"skill_version": "3.6.1",
"schema_version": 3,
"contract_revision": "3.5",
"runtime_fingerprint": "ug-v3.6.1-claude-code-direct-fmp-20260830"
}
```
Run every helper with `--version` before a live screen. Reject stale or mixed artifacts. This prevents a cached v3.1-v3.5 package from producing a seemingly valid contract-3.5 result.
## Request-Scope and Evidence Guardrails
- Preserve the default USD 500M–20B user request unless the user explicitly changes it.
- Never turn one convenient market-cap band into the requested scope. Stream exhausted pages/bands to disk or use a reproducible provider-prefilter grid across the entire requested range.
- Use only 20+ session average-dollar-liquidity evidence. Raw single-session volume is not ADDV.
- Run raw annual consensus rows through `normalize_estimates.py`; a current multiple must be NTM/FY1, dated, sourced, positive, and reconciled to price/EPS.
- Run `manage_run_state.py next-action` after each checkpoint and execute every returned symbol. The selected set is committed; changing its size requires rerunning the Broad Screen.
## Staged Coverage Model
### Listing coverage
Audit the complete requested listing universe for identity, active/common-stock status, quote, market cap, liquidity, sector, and exchange. This stage does not require full financial statements.
### Bounded economic pool
Generate a reproducible bounded pool by provider prefilter, available estimates, or stratified discovery. The generation audit must prove exactly how the pool was selected from the listing universe.
### Selected-name underwriting
Use SEC and company IR for the small selected set. Complete corporate-action, financial, forecast, peer, cycle, and sector evidence before ranking.
## Completion Invariants
A final ranking requires:
```text
valid runtime metadata
valid market context
valid listing-universe enumeration
valid candidate-pool generation audit
candidate pool exhausted
all candidate-pool rows resolved
queue empty
all selected symbols saved as verified candidate records
unprocessed_candidates empty
strict evaluator exit 0
```
A bounded pool can support a final **scoped ranking**. It cannot support a market-wide no-candidates claim unless every in-scope listing was economically covered.
## Attempted Is Not Resolved
`enrichment_attempted=true` records a query attempt or partial data receipt. A row becomes resolved only when it receives one of:
- `selected`
- `deferred_by_budget`
- `screened_out`
- `excluded`
- `unavailable_after_enrichment`
`unavailable_after_enrichment` requires a specific exhaustion reason and resolving source IDs.
## Deterministic Broad-Screen Authority
Always execute `screen_universe.py`. Preserve its decisions and requirements. The LLM may explain a decision but must not replace it with an informal gate.
Examples:
- Revenue growth below a guideline does not by itself remove a low-multiple, credible per-share grower.
- A cyclical candidate is routed to mid-cycle normalization, not automatically rejected.
- A high-growth 21–30x candidate may advance under the high-growth exception.
- Sector-specific rows remain review-blocked until the appropriate measure exists.
## Immutable Requested Scope
For a minimal request, retain the default USD 500M–20B request even when execution uses streaming, pagination, or a bounded discovery pool. Never rewrite the requested scope to a convenient band such as USD 3B–4B. A narrower executed scope requires explicit user authorization and must remain visibly separate from the original request.
## Liquidity and Forward-Horizon Gates
- ADDV requires provider-average evidence or price multiplied by an average-volume series covering at least 20 trading days. One-session volume is invalid.
- Current formal P/E requires NTM/FY1 metadata and a positive matching EPS. Distant outer-year estimates, missing intervening years, pre-operating names, zero-crossing ranges, or extreme dispersion cannot become the current multiple.
- Run raw annual estimates through `normalize_estimates.py` with an explicit estimate-as-of timestamp before deterministic screening.
## Selected-Set Commitment
The Broad Screen owns the selected set. Every committed selected symbol must be underwritten to a terminal verified record. The model cannot later ask whether to research two instead of three, or silently drop difficult names. Reduce the budget only by rerunning the deterministic Broad Screen and recording omitted names as `deferred_by_budget`.
A packet-stage provider-budget stop invalidates the entire broad-screen selection,
including rejected rows: every active decision becomes `deferred_by_budget`,
selection eligibility is disabled, and active lane/reason fields are removed.
The first original decision remains in `prior_decision`, with top-level selection
fields in `prior_selection`. These fields preserve the selected/rejected audit
trail only; they never authorize deep dives. Rerun in a new empty output directory
after restoring the budget.
## Provider Fallback Hierarchy
1. Use bulk listing and estimate/fundamental data when available.
2. Use a reproducible provider prefilter.
3. Build a stratified discovery pool and enrich it.
4. Use SEC/IR only for the selected set.
Do not stop because a premium endpoint is unavailable. Do not make an API-plan upgrade or CSV upload the primary user action.
## Candidate-Pool Scope
### Full-universe fundamentals
May support a market-wide final ranking or no-candidates conclusion when the full in-scope universe is economically covered and resolved.
### Provider prefilter or stratified discovery
May support a scoped final ranking after the bounded pool is audited, exhausted, and resolved. The report must state the bounded conclusion scope.
### User-supplied pool
May support only a user-supplied-pool conclusion unless the user explicitly claims broader coverage and supplies evidence.
## Tool-Budget Discipline
- Build a pool that can be completed.
- Limit deep dives to three by default for a minimal one-shot request; an explicit user override may raise the cap to five.
- Checkpoint after every selected candidate.
- Start selected deep dives while lower-priority enrichment continues when useful, but do not publish final until the pool is resolved.
- If tool budget becomes constrained, reduce the pool prospectively or resolve lower-priority rows as `deferred_by_budget`; never silently abandon them.
## Mandatory Repair Loop
After strict evaluation:
1. Read contract review reasons and candidate warnings.
2. Acquire missing obtainable evidence.
3. Correct source types, periods, cash classification, bridges, or funnel counts.
4. Rerun the deterministic evaluator.
5. Repeat until final or until a genuine unavailable condition is fully documented.
A genuine unavailable condition can lead to `review_required` or `unavailable_after_enrichment`; it does not justify invented numbers.
## Failure Behavior
When finality cannot be achieved, return a clearly labeled partial diagnostic, not a formal ranking. State the exact unresolved rows, selected candidate state, next internal action, and preserved artifact path. Do not invite the user to say “continue”; continue automatically when execution capacity remains.
## v3.5 Result-Quality and Publication Loop
1. Select through the deterministic multi-lane plan rather than a single global score.
2. Resolve every selected name and run the formal eligibility quality gate.
3. Permit `conditional` and `review_required`; do not lower quality floors to manufacture a ranked list.
4. Generate JSON and Markdown only after strict evaluation.
5. Run `prepublish_audit.py`. Exit code 2 is an internal repair instruction.
6. Run `bundle_run_artifacts.py` and present the self-contained ZIP.
The narrative layer must never announce completion before these steps finish. It must not contain “continue,” “next turn,” or similar handoff language in a final report.
### Provider-prefilter recall protection
Never use one narrow provider query as the whole economic pool. Retrieve and save four separate lanes—core GARP, high-growth exception, quality near miss, and cyclical normalization—then combine them with `build_provider_prefilter_pool.py`. This avoids systematically dropping GDDY-like low-headline-growth compounders, KGS-like high-growth exceptions, and AMKR-like cyclicals before primary-source review.
references/checkpointing.md
# Checkpoint and Completion Guide — Contract v3.5
## Purpose
Preserve every stage of a long screen, make resumption deterministic, and prevent selected candidates or unresolved pool rows from disappearing.
## Layout
```text
run/
├── run_state.json
├── screening_audit.json
├── screening_funnel.json
├── market_context.json
├── global_sources.json
├── universe_artifact.jsonl
├── candidate_pool_artifact.jsonl
├── candidates/
│ ├── SYMBOL.draft.json
│ └── SYMBOL.verified.json
└── final-snapshot.json
```
Write state atomically. Every state and audit carries v3.5 runtime metadata.
## Initialize
Use `manage_run_state.py init` with live market context, source ledger, price basis, and optional base repository commit.
## Attach the Audit
Use `set-screening-audit` only with a contract-3.5 audit whose runtime fingerprint matches the installed skill. The manager independently hashes the listing and candidate artifacts and checks their row counts and decisions.
Once a run contains any candidate at stage `verified`, `set-screening-audit` rejects every attachment, including the identical audit, with exit 1 before changing any files. Verification belongs to the existing audit and must not silently carry over to new evidence. To rescreen, initialize a new run directory, attach its audit, and verify its candidates again. Re-saving a verified candidate as `draft` is refused, so the guard cannot be bypassed by downgrading. While `candidate_pool_status` is `sufficient_pending_enrichment`, `save-candidate --stage verified` is refused and deep dives are saved as `draft`; resolve the pool, reattach the resolved audit, then save as `verified`. A run containing only draft candidates can still replace its audit; drafts remain unfinished and selected symbols must still be verified.
A run saved by an earlier version can hold verified candidates that this version can no longer complete: for example, candidates verified under a `sufficient_pending_enrichment` audit, or verified with `--allow-unselected` before any audit was attached. Reattaching, re-verifying and downgrading are all refused for such a run. Start over in a new run directory and verify the candidates against its audit. Do not use `init --force` on the same directory for this: it resets `run_state.json` but leaves the old `candidates/*.json` and `audit/*` files and appends to the existing `events.ndjson`.
Attaching or replacing an audit is all-or-nothing. The manager validates the metadata and hashes staged copies of every artifact before touching the run, then commits the artifacts, audit, event log, and `run_state.json` together, restoring all of them if any step fails. A rejected command leaves the prior checkpoint byte-identical and exits 1.
Details of the commit:
- Artifact names are compared case-insensitively, so `UNIVERSE-AUDIT-RESULTS.JSONL` collides with `universe-audit-results.jsonl` on every filesystem. An enrichment entry and a generation entry may point at the same file only when the SHA-256 is identical.
- A replaced file keeps its existing permission bits. New files get the default mode.
- Backups are plain copies with the permission bits only, so a macOS immutable flag on a live file is not copied to the backup.
- Backups are deleted only after the commit succeeded or the restore finished with no failures. A restore that fails or is interrupted keeps the `.set-screening-audit-*` directory, and the command continues restoring the remaining files before it reports. If the directory cannot be removed after a clean outcome, the command prints a warning naming it; delete it by hand.
- Non-finite numbers (`Infinity`), deeply nested JSON, and other malformed values are reported as `ERROR:` with exit 1 and no file changes.
If the error says the prior checkpoint could not be fully restored, each failure line names the target and the `backup-N.tmp` file in the kept `.set-screening-audit-*` directory that holds its prior bytes. Copy that backup over the target by hand. Targets with no backup were new files and can be deleted. A process killed during the commit cannot roll back: `prepublish_audit.py` rejects the resulting hash mismatch. Rerunning `set-screening-audit` repairs a run without verified candidates. For a legacy interrupted replacement in a run with verified candidates, restore the prior checkpoint from backups or initialize a new run directory and verify again; reattachment is rejected.
Limits that are documented rather than handled:
- The command assumes one writer. An event appended to `events.ndjson` by another process while the command runs is lost when the staged copy replaces the log.
- A live file that is a symbolic link is replaced by a regular file, and a rollback restores a regular file, not the link.
## Save Every Selected Candidate
For each selected symbol, save a draft early and replace it with a verified record after primary-source review. A selected company that becomes an M&A exclusion still receives a verified `excluded` candidate record.
## Funnel Counts
Maintain:
```text
universe_count
listing_in_scope_count
candidate_pool_count
discovery_evaluable_count
deep_dive_selected_count
preflight_passed_count
deep_dive_completed_count
```
`preflight_passed_count` and `deep_dive_completed_count` refer only to selected symbols. The evaluator independently recomputes both counts.
## Completion Gate
The manager enforces:
```text
unprocessed_candidates = selected_symbols − verified_candidate_symbols
```
It refuses `complete` when:
- the pool is unresolved or not exhausted,
- the queue is nonempty,
- generation audit/runtime evidence is invalid,
- any selected symbol lacks a verified record,
- funnel counts are inconsistent.
## Assemble and Evaluate
Assemble only from the attached audit, live source ledger, current context, and verified candidates. Then run:
```bash
python3 skills/us-undervalued-growth-screener/scripts/evaluate_candidates.py \
--input <final-snapshot.json> \
--artifact-root <run-root> \
--output-dir <final-dir> \
--strict --require-final --language ja
```
## Resume
On resume:
1. verify installed runtime with `--version`;
2. inspect `run_state.json` runtime metadata;
3. reject stale contract revisions/fingerprints;
4. inspect selected, verified, unprocessed, and enrichment queue states;
5. continue from the earliest incomplete stage.
## v3.5 Publication Checkpoint
After strict evaluation, store the generated report JSON/Markdown under the run directory, run the prepublication audit, then build the complete ZIP. A run is operationally complete only when:
```text
strict evaluator exit = 0
prepublish audit exit = 0
bundle builder exit = 0
bundle manifest contains every referenced artifact
```
Do not delete intermediate audit or candidate files before bundling.
references/claude-code-execution.md
# Claude Code Direct-FMP Execution — v3.6
## Purpose
The v3.6 path prevents bulk FMP responses from consuming the language-model context. Claude Code launches one local Python process. Python performs provider retrieval, persistent caching, listing enumeration, FY1 normalization, liquidity calculation, four-lane discovery, and deterministic broad screening. Claude reads only the compact summary and selected-company evidence packets.
The underwriting and final-ranking contract remains schema 3 / contract 3.5.
## Required environment
```bash
export FMP_API_KEY="..."
```
Never commit or print the key. Cache identities and raw-artifact metadata remove `apikey` and `api_key` fields.
The direct client requires `requests`; SQLite is provided by Python's standard library.
## Standard command
```bash
python3 skills/us-undervalued-growth-screener/scripts/run_pipeline.py \
--config skills/us-undervalued-growth-screener/assets/claude-code-config.example.json \
--output-dir reports/us-undervalued-growth-screener
```
The program prints one compact JSON object, normally below 20 KB. It does not print provider payloads.
## Reused repository components
- Repository-level generated FMP-client registry and vendoring pattern.
- Stable-first / legacy-v3 fallback, request pacing, retry, circuit breaking, and call-budget conventions.
- `normalize_estimates.py` for dated FY1/FY2/FY3 normalization.
- `build_provider_prefilter_pool.py` for four-lane candidate-pool construction.
- `screen_universe.py` for deterministic broad-screen decisions and deep-dive commitment.
- Existing checkpoint, evaluation, prepublication-audit, and bundle scripts.
## Retrieval strategy
1. Enumerate NASDAQ, NYSE, and AMEX with the company screener.
2. If a response reaches the configured limit, recursively split the market-cap band. A listing enumeration is complete only when all leaf bands are unsaturated.
3. Attempt bulk ratios, key metrics, income-statement growth, annual analyst estimates, and daily EOD datasets.
4. If bulk annual estimates cover enough symbols, normalize all covered listings. Otherwise choose a deterministic sector × market-cap seed and use per-symbol estimate calls.
5. Prefer bulk 20-day EOD volume. If unavailable, rank exact-liquidity work by the four economic lanes and fetch per-symbol history only for the bounded target set.
6. Build core GARP, high-growth exception, quality near-miss, and cyclical-normalization lanes.
7. Produce a 12–30 name audited provider-prefilter pool and select up to three deep-dive names.
8. Fetch candidate-level FMP data once. Store full payloads under `provider/candidate-data/`; write compact projected packets under `candidate-packets/`.
Bulk endpoint names remain configuration-driven because availability varies by FMP plan. A missing bulk endpoint is a fallback condition, not permission to invent data.
## Context discipline
Claude may read:
```text
run-summary.json
NEXT_ACTION.json
audit/listing-enumeration-audit.json
audit/provider-prefilter-audit.json
audit/broad-screen-audit.json
audit/partial-run-diagnostic.json
audit/enriched-estimates.partial.jsonl
candidate-packets/*.fmp-packet.json
```
The CLI stores provider raw responses in a sibling, attempt-specific
`.provider-raw/<run-id>/` directory so creating the FMP client cannot make a
new run directory look stale. An explicitly supplied raw-store path must also
remain outside the screen run directory.
`audit/partial-run-diagnostic.json` is a commit marker for a budget-exhausted
`screen-full-snapshot` attempt. Read the partial JSONL, summary, or preserved
artifacts only after verifying the marker's recorded SHA-256 values and row
counts. A missing or mismatched marker means the preceding files are not
authoritative. Screening has no `--resume`; rerun the same verified snapshot
into a new empty run directory after the provider budget is restored. The
snapshot is read-only. Existing
cache/raw records are not deleted or rewritten, while successful calls may add
new records and a later rerun may reuse them.
Claude must not load the raw provider-response tree into context. Open a raw file only to resolve a specific named mismatch. Provider packets remain secondary evidence; SEC/IR verification is mandatory for formal underwriting.
## Persistent cache
The generated client uses SQLite. Default TTLs are:
| Dataset | TTL |
|---|---:|
| Quotes | 1 hour |
| Listing universe | 7 days |
| Analyst estimates | 3 days |
| Statements / ratios / metrics | 7 days |
| Historical prices | 1 day |
The first run may issue many HTTP requests inside one Python invocation. Later runs reuse the cache and primarily refresh prices, estimates, and selected-company data.
## Key artifacts
```text
<run>/run-summary.json
<run>/NEXT_ACTION.json
<run>/audit/listing-enumeration-audit.json
<run>/audit/universe.jsonl
<run>/audit/enriched-estimates.jsonl
<run>/audit/provider-prefilter-pool.jsonl
<run>/audit/provider-prefilter-audit.json
<run>/audit/broad-screen-results.jsonl
<run>/audit/broad-screen-audit.json
<run>/audit/partial-run-diagnostic.json # only for budget-exhausted screen attempts
<run>/audit/enriched-estimates.partial.jsonl
<run>/candidate-packets/<SYMBOL>.fmp-packet.json
<run>/provider/candidate-data/<SYMBOL>/*.json
```
## Handoff to underwriting
`run_pipeline.py` stops at `ready_for_underwriting` when selected symbols exist. Claude must then:
1. perform corporate-action preflight;
2. verify current quarter and full year separately using SEC/IR;
3. reconstruct standard FCF with primary-source period evidence;
4. build same-basis valuation periods and an independent forecast bridge;
5. verify SBC, dilution, ROIC, leverage, peers, and cycle/sector evidence;
6. save all selected symbols, regardless of final status;
7. run strict evaluation, prepublication audit, and bundling.
A helper exit code 2 is an internal continuation signal. It is not a reason to ask the user for another turn.
## Deep-dive budget versus lane coverage
The bounded direct-FMP example config keeps `max_deep_dive_candidates: 3` to limit downstream underwriting work. This is a total candidate limit, not a promise of coverage across all four research lanes. The default lane targets are core GARP 2, high-growth exception 1, quality near miss 1, and cyclical normalization 1: five slots in total.
When the limit is below that total, the first selection pass walks candidates in priority order and applies each lane target as a cap. With a limit of three, at least one lane must be absent. At a limit of five or more, the first pass fills lane targets in lane order. In either case, later passes fill unused slots by priority without lane caps; the final pass can relax the sector preference if it would otherwise leave slots unused. Targets and the two-name sector preference therefore do not guarantee the final lane counts or sector counts. Only eligible candidates can be selected.
To allow the five-slot plan on the bounded path, copy `assets/claude-code-config.example.json` to your run configuration, set `max_deep_dive_candidates` to `5`, and pass that copy to `run_pipeline.py --config`. This allows more underwriting work but does not guarantee that every lane has an eligible selected name. Rerun the broad screen through the pipeline to establish the new selected set; when reducing the limit, omitted eligible names must become `deferred_by_budget`. Do not simply stop underwriting part of an already committed set: every selected symbol must be resolved.
The full-snapshot path uses the separate `full_snapshot_deep_dive_candidates` setting and requires it to equal `5`. Changing the bounded-path limit does not change that requirement. Neither a three-name nor a five-name deep-dive set alone establishes market-wide economic coverage; that depends on the full-snapshot verification contract.
references/data-contract.md
# Schema-v3 / Screening-Audit-v3.5 Data Contract
## Contract Versions
```text
Top-level snapshot schema_version: 3
Screening audit audit_schema_version: 3
Screening audit contract_revision: 3.5
```
Snapshot schema 3 retains the accounting, source, forecast, and candidate structure introduced in v3.0. Screening contract 3.5 hardens immutable requested scope, average-liquidity evidence, current NTM/FY1 normalization, selected-set commitments, universe enumeration, enrichment resolution, completion, source freshness, and report-disposition rules.
## v3.5 Discovery Evidence Contract
### User-request scope versus executed scope
The default user request remains USD 500 million–20 billion unless the user explicitly changes it. Store both:
```json
{
"user_requested_scope": {
"min_market_cap": 500000000,
"max_market_cap": 20000000000,
"source": "skill_default | explicit_user_request"
},
"executed_scope": {
"min_market_cap": 500000000,
"max_market_cap": 20000000000
},
"scope_reduced": false,
"scope_override_authorized": false,
"user_scope_evidence": null
}
```
Internal context or tool-budget pressure never authorizes a narrower market-cap request. Use a cross-range provider prefilter or stream exhausted bands to disk. A single convenient band is incomplete unless the user explicitly requested that band.
### Average daily dollar volume
A row is valid for discovery only when it provides an allowed averaging method, a window of at least 20 sessions, and resolving source IDs. Raw one-session `volume` is never interpreted as average volume.
```json
{
"average_daily_dollar_volume": 12000000,
"average_daily_dollar_volume_method": "provider_average_dollar_volume",
"average_volume_period_days": 20,
"liquidity_source_ids": ["quote-source"]
}
```
### Current forward metric
`forward_pe` alone is invalid. Current valuation must identify NTM or FY1 and reconcile to price and a positive current forward EPS:
```json
{
"forward_pe_period": "FY1",
"forward_fiscal_year": "FY2027",
"forward_period_end": "2027-06-30",
"forward_eps": 3.25,
"forward_pe": 15.4,
"forward_estimate_as_of": "2026-08-28T12:00:00Z",
"forward_estimate_source_ids": ["consensus-source"],
"forward_metric_origin": "computed_from_price_and_fy1_eps",
"analyst_count": 6
}
```
Outer-year substitutions, pre-operating companies, a forecast range crossing zero, excessive dispersion, stale estimates, or non-contiguous annual rows cannot become current Forward P/E inputs. Use `normalize_estimates.py` to produce this structure from raw dated annual estimates.
### Committed selected set
The broad-screen audit commits the selected set with a SHA-256 payload. Every selected symbol must receive a verified candidate record. Reducing the deep-dive count requires a fresh deterministic broad-screen run; narrative-stage discretion cannot drop an already selected name.
## Top-Level Snapshot
```json
{
"schema_version": 3,
"runtime": {
"skill_name": "us-undervalued-growth-screener",
"skill_version": "3.6.1",
"schema_version": 3,
"contract_revision": "3.5",
"runtime_fingerprint": "ug-v3.6.1-claude-code-direct-fmp-20260830"
},
"analysis_as_of": "ISO-8601 with timezone",
"run_metadata": {
"run_id": "string",
"status": "partial | complete",
"base_repository_commit": "optional SHA",
"selected_symbols": [],
"unprocessed_candidates": []
},
"price_basis": {
"as_of": "ISO-8601 with timezone",
"session": "regular_close | pre_market | after_hours | intraday",
"timezone": "America/New_York",
"source_id": "source-id"
},
"config": {},
"market_context": {},
"screening_funnel": {},
"global_sources": [],
"screening_audit": {},
"candidates": []
}
```
All timestamps must be at or before `analysis_as_of` unless a field explicitly represents a future fiscal period.
## Source Ledger
```json
{
"id": "unique-id",
"tier": 1,
"kind": "sec_filing",
"title": "Form 10-Q",
"published_at": "2026-08-10T16:05:00-04:00",
"data_as_of": "2026-06-30T00:00:00-04:00",
"retrieved_at": "2026-08-22T10:05:00-07:00",
"url": "https://...",
"supports": ["latest_earnings.quarter", "financials.cash_flow_ttm"]
}
```
`supports` is always a non-empty string array. `data_as_of` is optional but preferred for dynamic fields whose data date differs from publication time. Retrieval time never makes old data current.
| Kind | Usual tier |
|---|---:|
| `sec_filing` | 1 |
| `official_macro`, `official_statistics` | 1 |
| `company_ir` | 2 |
| `exchange_status` | 2 or 3 |
| `market_data`, `consensus`, `news`, `third_party_transcript` | 3 |
| `analyst_calculation`, `analyst_assumption`, `local_artifact` | 4 |
A third-party transcript is never `company_ir`.
## Market Context
Required fields:
```json
{
"summary": "non-placeholder narrative",
"as_of": "ISO-8601",
"policy_rate_pct": 3.625,
"treasury_10y_yield_pct": 4.74,
"real_gdp_growth_pct": 1.5,
"inflation_yoy_pct": 3.4,
"market_forward_pe": 21.2,
"small_mid_cap_valuation_context": "current context",
"sector_cycle_notes": [],
"source_ids": [],
"inferences": []
}
```
Field-level evidence and default maximum ages:
| Field | Default max age | Source rule |
|---|---:|---|
| Policy rate | 60 days | official macro/statistics |
| 10-year Treasury | 7 days | official macro/statistics |
| Inflation | 45 days | official statistics |
| GDP | 120 days | official statistics |
| Market forward P/E | 14 days | current market-data/research source |
| Small/mid-cap context | 14 days | current market-data/research source |
Market-rate expectation language such as “markets price hikes/cuts” requires a source supporting `market_context.market_rate_expectations` or an `analyst_inference` object with resolving source IDs.
## Layered Screening Audit
```json
{
"audit_schema_version": 3,
"contract_revision": "3.5",
"runtime": {
"skill_name": "us-undervalued-growth-screener",
"skill_version": "3.6.1",
"schema_version": 3,
"contract_revision": "3.5",
"runtime_fingerprint": "ug-v3.6.1-claude-code-direct-fmp-20260830"
},
"conclusion_scope": "full_listing_universe | provider_prefilter | stratified_discovery_pool | bounded_available_fundamentals | user_supplied",
"generated_at": "ISO-8601",
"analysis_as_of": "ISO-8601",
"candidate_generation_mode": "liquidity_stratified_estimates",
"candidate_pool_status": "sufficient | sufficient_pending_enrichment | no_qualifying_candidates | no_qualifying_candidates_in_bounded_pool | insufficient_data",
"selection_outcome": "selected | selected_pending_enrichment | no_candidates | no_candidates_in_bounded_pool | insufficient_data",
"selected_symbols": ["AAA"],
"filters": {},
"source_ids": ["listing-source", "estimate-source"],
"scope": {
"requested_min_market_cap": 500000000,
"requested_max_market_cap": 20000000000,
"retrieval_min_market_cap": 500000000,
"retrieval_max_market_cap": 20000000000,
"retrieval_scope_explicit": true,
"scope_override_authorized": false,
"scope_complete": true,
"reasons": [],
"enumeration": {
"verified": true,
"provider_reported_total": 1786,
"rows_fetched": 1786,
"pages_fetched": 18,
"pagination_exhausted": true,
"band_audit": [],
"bands_verified": false
}
},
"universe": {
"row_count": 1786,
"in_scope_count": 1600,
"listing_data_complete_count": 1770,
"listing_data_complete_pct": 99.1,
"decision_counts": {},
"artifact_path": "audit/universe-audit-results.jsonl",
"artifact_sha256": "64-character SHA-256"
},
"candidate_pool": {
"row_count": 120,
"discovery_evaluable_count": 40,
"selection_eligible_count": 8,
"selected_count": 5,
"in_scope_covered_count": 120,
"in_scope_missing_count": 1480,
"in_scope_missing_symbols": ["..."],
"coverage_complete": true,
"coverage_scope": "stratified_discovery_pool",
"listing_coverage_complete": false,
"generation_audit": {
"audit_schema_version": 2,
"runtime": {},
"selection_method": "sector_market_cap_stratified",
"input_row_count": 1786,
"selected_count": 120,
"selected_symbols": [],
"source_ids": [],
"artifact_path": "discovery/discovery-pool.jsonl",
"artifact_sha256": "64-character SHA-256",
"valid": true
},
"fundamental_complete_count": 12,
"decision_counts": {},
"symbols_not_in_universe": [],
"artifact_path": "audit/broad-screen-results.jsonl",
"artifact_sha256": "64-character SHA-256"
},
"enrichment": {
"status": "pending | complete",
"next_action": "build_discovery_pool | enrich_queue | expand_candidate_pool | verify_candidate_pool_exhaustion | proceed_to_deep_dive | publish_no_candidates",
"discovery_pool_required": false,
"attempted_count": 120,
"resolved_count": 118,
"unresolved_count": 2,
"resolution_pct": 98.3333,
"all_rows_resolved": false,
"maximum_attempts": 120,
"candidate_pool_exhaustion_declared": false,
"candidate_pool_exhausted": false,
"candidate_pool_covers_in_scope": false,
"queue_count": 2,
"queue_symbols": ["BBB", "CCC"],
"artifact_path": "enrichment-queue.json",
"artifact_sha256": "64-character SHA-256"
}
}
```
## Scope and Enumeration
`scope_complete=true` requires all of:
```text
retrieval_scope_explicit = true
retrieval bounds cover requested bounds
enumeration.verified = true
```
Enumeration is verified when:
- pagination is exhausted and any provider-reported total is consistent with fetched rows, or
- every row in `band_audit` has valid lower/upper bounds, a nonnegative row count, and `provider_exhausted=true`, and the ordered bands cover the complete requested market-cap range without gaps.
A matching lower/upper market-cap bound alone is insufficient. `bands_verified=true` is derived from the band evidence; it is not accepted as self-attestation.
## Requested Scope, Executed Scope, and Authorization
Keep the user's request separate from the data slice actually executed:
```json
{
"user_requested_scope": {
"min_market_cap": 500000000,
"max_market_cap": 20000000000,
"source": "default_contract | explicit_user_request"
},
"executed_scope": {
"min_market_cap": 500000000,
"max_market_cap": 20000000000,
"reduction_authorized": false,
"authorization_source": null
}
}
```
A tool/context budget is not authorization to replace the default USD 500M–20B request with a USD 3B–4B request. Stream pages or market-cap bands to disk. A narrower execution may be published only as an explicitly user-authorized scoped run; otherwise the contract remains partial.
## Listing Liquidity Evidence
A listing or discovery row may satisfy ADDV only with a provider-average dollar-volume value or `price × average_volume`, where the averaging window is at least 20 trading days:
```json
{
"average_daily_dollar_volume": 12500000,
"average_daily_dollar_volume_method": "provider_average_dollar_volume | price_times_average_volume",
"average_volume_period_days": 20,
"liquidity_source_ids": ["quote-history-source"]
}
```
Single-session volume, an unlabeled `volume` field, or a share-volume floor cannot populate ADDV.
## Current Forward Estimate Normalization
The formal current multiple must use a resolving NTM or FY1 period:
```json
{
"forward_pe": 14.2,
"forward_eps": 3.10,
"forward_pe_period": "NTM | FY1",
"forward_period_end": "2027-06-30",
"forward_fiscal_year": "FY2027",
"forward_estimate_as_of": "2026-08-28T00:00:00-04:00",
"forward_estimate_source_ids": ["consensus-source"],
"forward_estimate_analyst_count": 6,
"estimate_normalization_status": "valid"
}
```
For FY1, the period must be the nearest supported fiscal year after `analysis_as_of` and within the configured horizon. A missing FY1/NTM row, pre-operating status, a forecast range crossing zero, extreme dispersion, or noncontiguous outer-year-only estimates produces `estimate_normalization_status="unavailable"`. Canonical `forward_pe` and `forward_eps` are then null; raw outer-year values may be retained only under `raw_forward_candidate` for diagnostics.
## Selected-Set Commitment
The deterministic Broad Screen commits its selected symbols and budget:
```json
{
"selected_symbols": ["AAA", "BBB"],
"selected_set_sha256": "64-character SHA-256",
"max_deep_dive_candidates": 3
}
```
Every committed selected symbol must reach a verified terminal candidate record. The model cannot later process only a preferred subset. To change the budget or selected set, rerun the deterministic Broad Screen so omitted names become `deferred_by_budget`.
## Universe Artifact
`universe-audit-results.jsonl` has one row per listing-universe symbol. Allowed decisions include:
- `in_scope`
- `liquidity_review`
- `out_of_scope`
- `excluded`
`listing_data_complete` refers only to listing/quote/scope fields, not financial statements.
## Candidate-Pool Artifact
`broad-screen-results.jsonl` has one row per bounded economic candidate.
### Resolution fields
```json
{
"enrichment_attempted": true,
"enrichment_resolved": false,
"selection_eligible": false,
"decision": {
"status": "needs_enrichment",
"resolution": "unresolved",
"selection_eligible": false
}
}
```
`enrichment_attempted` means a source was queried or at least one economic value was supplied. It does not imply resolution.
### Allowed final broad statuses
| Status | Resolution | Can be selected? | Score |
|---|---|---:|---:|
| `selected` | resolved | yes | numeric |
| `deferred_by_budget` | resolved | no | numeric |
| `screened_out` | resolved | no | numeric when assessable |
| `excluded` | resolved | no | optional |
| `unavailable_after_enrichment` | resolved | no | null |
| `needs_enrichment` | unresolved | no | null |
Before budget assignment, economically plausible rows may use `passed`, `passed_exception`, `near_miss_review`, or `sector_review_required`. The serialized final artifact converts selected/deferred rows to `selected` or `deferred_by_budget` while preserving `preselection_status` and deep-dive requirements.
### Unavailable after enrichment
A row may be resolved as `unavailable_after_enrichment` only with:
```json
{
"enrichment_exhausted": true,
"enrichment_exhaustion_reason": "specific reason",
"enrichment_source_ids": ["attempt-source"]
}
```
The source IDs must resolve in the ledger.
## Candidate-Pool Outcomes
### `sufficient / selected`
- one or more selected rows;
- candidate pool exhausted;
- all rows resolved;
- queue empty.
### `sufficient_pending_enrichment / selected_pending_enrichment`
- one or more selected rows;
- unresolved rows or unverified exhaustion remain.
Deep dives may start, but candidates are saved only as `draft`: `save-candidate --stage verified` is refused while the pool is pending. Formal final output is prohibited. Resolve the pool, reattach the resolved audit with `set-screening-audit`, and then save the candidates as `verified`.
### `no_qualifying_candidates / no_candidates`
A **market-wide** conclusion is allowed only when:
```text
conclusion_scope = full_listing_universe
selected_symbols = []
candidate_pool.coverage_complete = true
candidate_pool.in_scope_missing_count = 0
candidate_pool_exhaustion_declared = true
candidate_pool_exhausted = true
all_rows_resolved = true
unresolved_count = 0
queue_count = 0
discovery_evaluable_count > 0
```
### `no_qualifying_candidates_in_bounded_pool / no_candidates_in_bounded_pool`
A reproducibly generated bounded pool may reach this status when its generation audit is valid, every pool row is resolved, exhaustion is verified, the queue is empty, and at least one row was economically assessable. The report must name the bounded `conclusion_scope` and must not make a market-wide claim.
### `insufficient_data`
- no selected rows;
- unresolved evidence remains, enumeration is incomplete, or no row was economically assessable.
Do not describe this as “no qualifying candidates.”
## Soft Guidelines and Hard Broad Failures
Growth, preferred valuation, ROIC, and preferred leverage values are guidelines. Store misses in `guideline_misses`; do not automatically reject a row for one miss.
Hard failures belong in `screen_fail_reasons`, for example:
- `non_positive_standard_fcf`
- `negative_roic`
- `excessive_leverage`
- `extreme_forward_valuation`
- `negative_forward_growth`
- `valuation_not_supported_by_growth`
- `growth_and_valuation_combination_below_review_threshold`
Cyclical normalization is normally a `deep_dive_requirement`, not a hard failure.
## Candidate Deep-Dive Record
Each selected candidate retains schema-v3 structures for:
- identity and price basis,
- sources and evidence,
- corporate-action check,
- latest quarter and full year as separate records,
- financials and TTM cash-flow reconstruction,
- forward same-basis valuation periods,
- forecast bridge and GAAP reconciliation,
- peer set,
- sector profile,
- cyclicality normalization,
- thesis, catalysts, risks, and invalidation conditions,
- score components and penalties.
## Runtime Compatibility
Every snapshot, screening audit, discovery audit, and checkpoint must match the installed v3.6 runtime metadata. A mismatch in skill version, schema version, contract revision, or runtime fingerprint is a hard contract error. File names and human-readable version labels are not sufficient proof.
## Driver-Derived Forecast Contract
For each rankable future period, `forecast_bridge` must contain operating drivers sufficient to calculate the metric independently.
### EPS
```json
{
"revenue": 1000000000,
"operating_margin_pct": 20.0,
"net_interest_and_other": -5000000,
"tax_rate_pct": 22.0,
"diluted_shares": 100000000,
"after_tax_adjustments": 10000000,
"source_ids": ["forecast-source"]
}
```
The evaluator derives GAAP net income from revenue, margin, interest/other, and tax, then adds sourced after-tax adjustments only for an adjusted basis. A supplied `metric_numerator` is only a cross-check. A bridge that merely sets numerator equal to target EPS times shares fails.
For adjusted periods, driver-derived GAAP EPS must tie to the GAAP reconciliation and driver adjustments must tie to the reconciliation adjustment amount.
### FCF per share
Derive standard FCF from OCF minus positive capex cash outflow, or from a sourced revenue/FCF-margin model, then divide by diluted shares.
## TTM Cash-Flow Evidence
A TTM method is valid only when the construction periods have resolving source IDs:
- `reported_ttm`: source IDs for the reported TTM value;
- `sum_4_discrete`: source IDs for all four discrete quarters;
- `fy_plus_current_ytd_minus_prior_ytd`: source IDs for FY, current YTD, and prior-year YTD.
## Cash and Financial-Quality Evidence
Ordinary companies may normalize reported cash/equivalents into `corporate_cash`. Payments and custodial businesses must explicitly separate corporate cash, customer/settlement funds, and restricted cash. ROIC and EBITDA used for scoring or leverage require resolving primary-source or transparent analyst-calculation evidence.
## Sector and Peak-Profit Evidence
Sector aliases `biopharma`, `pharma`, `biotechnology`, `royalty_biopharma`, and `drug_delivery_platform` normalize to `commercial_biopharma`. Material product/royalty concentration, nearest LOE, source IDs, and 6x/8x stress scenarios are required.
`peak_profit_risk=true` requires sourced mid-cycle normalization even when the numeric cyclicality score is 1 or 2.
## Completion and Checkpoint Invariants
```text
unprocessed_candidates = selected_symbols - verified_candidate_symbols
```
A run cannot be complete when:
- screening audit is invalid;
- candidate pool status is `insufficient_data` or `sufficient_pending_enrichment`;
- candidate pool does not cover every in-scope listing symbol;
- pool exhaustion was not explicitly declared and independently verified;
- any broad row is unresolved;
- queue is nonempty;
- a selected symbol is not verified;
- `no_qualifying_candidates` is claimed without an assessable exhausted pool.
## Markdown Output Contract
The Markdown report must separately show:
### Broad-screen stage
- selected,
- deferred by budget,
- unresolved/review rows,
- unavailable after enrichment,
- screened out,
- excluded.
### Deep-dive stage
- ranked,
- review required,
- screened out,
- excluded.
Never display `0 / 0 / 0 / 0 / 0` for deep-dive counts in a way that hides nonzero broad-screen dispositions.
## v3.5 Multi-Lane Selection and Publication Contract
### Selection lanes
Every selected Broad Screen row records one of:
```text
core_garp
high_growth_exception
quality_near_miss
cyclical_normalization
```
The default five-name plan targets 2/1/1/1 names across these lanes and applies a two-name sector cap when alternatives exist. Unused lane capacity is deterministically backfilled. The selected set remains committed and every selected symbol must receive a verified terminal candidate record.
### Independent driver provenance
Each year-2/year-3 forecast bridge uses:
```json
{
"construction_method": "independent_driver_model",
"driver_provenance": {
"revenue": {
"origin": "market_consensus",
"source_ids": ["consensus-source"],
"target_solved": false
},
"operating_margin_pct": {
"origin": "historical_run_rate",
"source_ids": ["10-k-source", "analyst-model"],
"target_solved": false
}
}
}
```
Allowed origins are `company_guidance`, `market_consensus`, `historical_run_rate`, `company_target`, `analyst_assumption`, and `primary_source_calculation`. Any driver solved backwards from target EPS or FCF fails the bridge.
### Explicit cash-flow period support
A generic `financials.cash_flow_ttm` support label does not validate four discrete periods. Each source must explicitly support its period path, for example:
```text
financials.cash_flow_periods.2026-06-30
financials.ttm_reconstruction.latest_fy
financials.cash_flow_ttm.reported_ttm
```
### Corporate transitions
Recent spin-offs, transformative acquisitions, and large divestitures are corporate-transition evidence, not sector valuation special cases. Store `identity.special_case = none` and a sourced `corporate_transition` object. Missing pro-forma normalization routes the candidate to `review_required`, not hard exclusion.
### Publication bundle
A formal publication contains the complete run directory plus `BUNDLE_MANIFEST.json`. The manifest records runtime identity, file paths, sizes, SHA-256 values, and the passing prepublication audit. Summary-only ZIP files are invalid.
### Multi-lane provider-prefilter generation audit
A provider-prefilter pool records:
```json
{
"selection_method": "provider_prefilter",
"lane_input_counts": {
"core_garp": 30,
"high_growth_exception": 20,
"quality_near_miss": 25,
"cyclical_normalization": 25
},
"lane_selected_counts": {
"core_garp": 15,
"high_growth_exception": 15,
"quality_near_miss": 15,
"cyclical_normalization": 15
},
"lane_coverage_count": 4,
"minimum_pool": 30,
"pool_adequate": true,
"provider_exhausted": false
}
```
A valid high-recall bounded pool normally has at least 30 rows and at least three represented lanes. `provider_exhausted=true` may waive the row floor, but not liquidity evidence or lane disclosure.
### v3.6.1 discovery additions
The discovery audit additionally carries:
```json
{
"provider_exhausted_scope": "estimate_seed",
"listing_provider_exhausted": true,
"estimate_seed_exhausted": true,
"economic_candidate_universe_exhausted": false,
"seed_audit": {
"seed_selection_basis": "stratified_liquidity_proxy",
"economic_metrics_available_for_seed": false,
"cell_count": 55,
"quota_method": "sqrt_hamilton",
"alphabetic_tie_break_used_count": 0,
"hash_tie_break_used_count": 3,
"seed_limit_configured": 180,
"seed_limit_effective": 180,
"reserved_calls": 130
},
"quality_probe": {"attempted": [], "resolved": [], "source_id": "fmp-key-metrics-ttm-<date>", "calls_used": 0},
"fcf_prefilter_excluded_symbols": [],
"fcf_prefilter_exclusions": [{"symbol": "AAA", "lane": "core_garp", "reason": "fcf_yield_below_prefilter_floor"}]
}
```
`provider_exhausted` may waive the pool row and lane floors only together with `provider_exhausted_scope` in {`economic_candidate_universe`, `full_input`}; a missing scope fails closed (no waiver), `estimate_seed` exhaustion never waives either floor, and `pool_floor_waived` / `lane_floor_waived` record the outcomes. The `sharded_snapshot` candidate-generation mode is distinct from a bounded `provider_prefilter`: it requires a screen-ready same-read snapshot bundle, a 64-character digest binding the explicit listing-enumeration audit, manifest, universe, and all shard hashes, economic attempts and classification totals exactly equal to the frozen universe, current per-row normalization provenance, and a final pool containing only `evaluable` rows. Enumeration verification recomputes continuous full-range coverage independently for every requested exchange, and the frozen market-cap/price scope must contain the later screen request. Exact-liquidity attempts continue past empty/failed histories until the 30–50 name target is met or every eligible name has an explicit outcome; only the latter can justify a short-pool exhaustion claim. That proof supports `conclusion_scope: full_listing_universe` even though deep-dive work is budgeted to the deterministic top pool. The unit gate fails closed on unknown context: a row is exempt only when PROVEN domestic (country `US`, no non-US ISIN, not an ADR/ADS, currency USD or unstated) — a missing country, non-USD currency, non-US ISIN, or ADR/ADS flag requires `unit_reconciliation_verified: true` (listing/statement currency and ordinary-shares-per-ADS evidence); without it the row carries the blocking `unit_reconciliation_required` review and, on the direct-FMP path, resolves as `unavailable_after_enrichment`. `normalize_listing` preserves `isin` and `is_adr` for this purpose. Implausible ratios (forward P/E below 2, FCF yield above 50%, reported EPS above twice the listing price) fail the broad screen as `unit_mismatch_suspected` — suspected currency/ADS-unit mismatch, never "deep value". `sector_profile_type` is inferred from an explicit FMP-taxonomy map (`INDUSTRY_PROFILE_MAP` + family-prefix rules + BDC name needles + the `sector_profile_overrides` config pin); `capital_markets` (advisory/investment banking) is labelled but deliberately not a blocked profile. The run summary and report JSON carry a tri-state `ranking_scope` (`final_marketwide` / `final_scoped` / `diagnostic`) with per-stage coverage counts and percentages (`economic_attempt_*`, `economically_evaluable_*`, `quality_probe_*`, `deep_dive_*`); `final_marketwide` requires estimate acquisition attempted for every listing-universe symbol (exact counts) — a seed-based run is at most `final_scoped` and its conclusions never generalize to the market. Sector-profile rows (`sector_profile_type` in reit / insurance / bank / asset_manager / bdc / mlp) skip the general `excessive_leverage` hard gate and instead require sector-specific valuation evidence (`sector_specific_valuation_required`). `economic_candidate_universe_exhausted` and `economic_screen_scope_complete` are true only when bulk estimates covered every listing-universe symbol (exact `covered_symbol_count == universe_symbol_count`; not a configurable ratio), or when the same exact invariant is proven by a verified `sharded_snapshot`; merely using the bulk route (from 20% coverage) does not qualify. `latest_actual_eps` must be a verified reported figure (`latest_actual_verified: true`, `latest_actual_basis`, `latest_actual_source_ids` resolving to an annual statement accepted at or before `analysis_as_of`, or a provider row marked actual whose period has ended); otherwise the actual-derived fields are null. `growth_pattern` and `current_year_growth_pct` are derived on the consensus basis from `fy0_consensus_eps` (the provider's prior-year row, never labelled actual; `growth_pattern_basis`), and `estimate_basis_likely_adjusted` flags a >15% gap between that row and the GAAP actual. Pool rows may carry `quality_probe_attempted`, `quality_probe_resolved`, `quality_probe_source_ids`, `sbc_adjusted_fcf_yield_pct`, `provider_prefilter_flags` (`weak_fcf_support`, `earnings_recovery`, `foreign_private_issuer_review`), and the growth-basis fields `latest_actual_eps`, `fy1_eps_below_latest_actual`, `current_year_growth_pct`, `eps_growth_actual_to_fy3_pct`, `growth_pattern`.
`run-summary.json` / `NEXT_ACTION.json` add `listing_enumeration_complete`, `economic_screen_scope_complete`, `listing_universe_count`, `estimate_seed_count`, `estimate_seed_coverage_pct`, `valid_estimate_count`, `valid_estimate_coverage_pct`; `scope_complete` remains as a deprecated alias of listing-enumeration completeness. The contract-validated `screening_audit.scope` block is unchanged.
### Full-snapshot partial budget diagnostic
`screen-full-snapshot` may return exit 2 with a provider-budget diagnostic
before it can publish a final screening result. The non-final diagnostic has
this shape:
```json
{
"schema_version": 1,
"status": "provider_budget_exhausted",
"stage": "liquidity | quality_probe | candidate_packets",
"snapshot_id": "string",
"snapshot_verification_digest": "64-character SHA-256",
"progress": {
"target_symbols": ["AAA", "BBB"],
"attempted_symbols": ["AAA"],
"completed_symbols": [],
"pending_symbols": ["AAA", "BBB"],
"interrupted_symbols": ["AAA"],
"interrupted_at": {"symbol": "AAA", "operation": "provider_method"}
},
"no_final_marketwide_conclusion": true,
"artifacts": {
"partial_enriched_estimates": {"path": "...", "sha256": "...", "row_count": 2},
"run_summary": {"path": "...", "sha256": "..."},
"next_action": {"path": "...", "sha256": "..."}
},
"commit_marker": "audit/partial-run-diagnostic.json"
}
```
The target and progress arrays are uppercase, duplicate-free, and retain target
order. `completed` means all provider calls required by the stage finished;
the interrupted current symbol can therefore be in `attempted` and `pending`.
Consumers must verify the marker exists and every listed artifact hash and row
count matches before reading any partial output. Screening has no resume cursor:
rerun the same verified snapshot after restoring the provider budget. The
snapshot is read-only; existing cache/raw records are retained, successful
calls may append new records, and later runs may reuse them. If packet
generation is interrupted, the broad-screen audit is explicitly non-authoritative:
its status and ranking scope are `incomplete`/`diagnostic`, its top-level,
deep-dive, candidate-pool, and generation-audit selection commitment fields are
cleared, and every broad-screen row is downgraded from `selected` to
`deferred_by_budget`. The candidate artifact hash is rewritten after this
invalidation. Each screen attempt also requires a new empty run directory;
the CLI adds a unique attempt suffix so retries cannot mix stale final
artifacts with a partial run.
references/full-universe-snapshot.md
# Full-universe estimate snapshot (v3.7 — sharded collection)
The v3.6.1 pilot economically evaluates ~4% of the listed universe and is
therefore permanently `ranking_scope: final_scoped`. The v3.7 path earns
`final_marketwide` the only honest way: by attempting estimate acquisition for
EVERY listed symbol at least once. On plans where bulk endpoints are 402, that
means collecting per-symbol across deterministic shards within each run's API
budget, persisted into a frozen snapshot.
## Stage: `collect-estimates`
```bash
python3 scripts/run_pipeline.py \
--stage collect-estimates \
--shard-index 0 --shard-count 8 \
--snapshot-dir .cache/us-garp/snapshot-2026-08 \
--config assets/claude-code-config.example.json \
[--resume]
```
- The FIRST invocation enumerates listings and **freezes the universe** into
the snapshot directory (`snapshot-manifest.json` + `universe.jsonl` +
`listing-enumeration-audit.json`,
identified by `snapshot_id` = timestamp + universe SHA-256). Every later
shard run writes into that frozen snapshot; new listings and delistings go
to the NEXT snapshot, never mixed in.
- Collection is current-only: a historical or future `--analysis-as-of` is
rejected before client/cache/raw/snapshot creation. Each row records
`snapshot_normalization_as_of`; shard manifests aggregate its bounds so a
later screen cannot treat an old FY-period normalization as current.
- Shard membership is `sha256(symbol) % shard_count` — the same symbol always
lands in the same shard, so multi-day collection is deterministic.
- Each attempted symbol is normalized (FY1-FY3 EPS/revenue, analyst counts)
and classified into exactly one bucket:
`evaluable / no_estimates / negative_eps / unit_mismatch / excluded`
(precedence: excluded > unit_mismatch > no_estimates > negative_eps >
evaluable; the unit gate is the round-8 fail-closed
`requires_unit_reconciliation`).
- **Provider failures are never classified.** The client returns an empty
list for HTTP failures, offline cache misses and invalid JSON as well as
for genuinely empty consensus; the stage distinguishes them via the
client's per-call diagnostics (calls / cache hits / failure count) and
records failures as `fetch_failed` — the symbol stays uncollected, the
shard stays `partial` (`shard_partial_fetch_failures`, exit 3), and the
marketwide invariant cannot be satisfied by outages. HTTP-200 error
bodies (`{"Error Message": ...}` plan-limit style) are validated in the
client itself: recorded as `provider_error_payload`, never cached; a
malformed object that predates validation in an existing cache is
flagged `unexpected_payload_shape` on read and likewise becomes a fetch
failure.
- Budget exhaustion mid-shard exits with code **3**, records the shard as
`partial` in the manifest, and is resumable with `--resume`. A shard file
that already exists without `--resume` is refused.
- **The frozen universe is re-verified on every load** (manifest schema,
row count, symbol uniqueness, canonical SHA-256) before any API access or
shard append; a swapped `universe.jsonl` is refused.
- Every collected row carries `snapshot_retrieved_at` — the ACTUAL fetch
time. Cache-served rows (any cache hit, including a failed stable HTTP
attempt falling back to the v3 cache) are stamped from the cache entry's
creation time; when that provenance cannot be resolved the stamp stays
**null** (counted as `retrieval_time_unknown`, never back-filled with
"now"). Shards aggregate `oldest_retrieved_at` / `newest_retrieved_at`,
and a run that collects nothing does NOT refresh the shard's `as_of`
freshness stamp.
- Shard summaries (`shard-<i>-summary.json`) and the manifest carry attempted
/ expected / fetch-failed counts, per-bucket classification, `as_of`,
retrieval and normalization bounds, and calls used.
## Stage: `screen-full-snapshot`
After every shard is complete, run the current-only market-wide screen:
```bash
python3 scripts/run_pipeline.py \
--stage screen-full-snapshot \
--snapshot-dir .cache/us-garp/snapshot-2026-08 \
--config assets/claude-code-config.example.json \
--output-dir reports/us-undervalued-growth-screener
```
This stage performs its read-only snapshot preflight before constructing the
FMP client or creating cache, provider-raw, or run directories. It refuses an
incomplete, stale, future-dated, tampered, or semantically misclassified
snapshot with exit code 1 and creates no run artifact tree.
The screen is deliberately current-only. Exact-liquidity and TTM quality
probes use current provider evidence, so an operator-supplied historical
`--analysis-as-of` is rejected before those calls rather than mixing current
fundamentals into a historical ranking. The allowed wall-clock skew defaults
to five minutes.
For a ready snapshot, the stage:
1. uses the verified in-memory shard rows from the same reads that produced
the readiness verdict;
2. rechecks gap-free exhausted bands for every requested exchange, requires
the frozen market-cap/price scope to contain the screen request, then binds
that audit, the manifest, frozen-universe SHA-256, and every shard SHA-256
into a `snapshot_verification_digest` carried by all downstream audits;
3. records the estimate-attempt classification and retrieval provenance for
every frozen symbol in `universe-audit-results.jsonl`;
4. builds a deterministic 30–50 name multi-lane pool (default 50), backfills
failed/empty exact-liquidity calls until the target succeeds or every
eligible symbol has an explicit outcome, probes every preliminary-pool
name for current quality evidence, and never selects an unprobed deep-dive
name; and
5. commits five deep-dive slots for the market-wide path.
A verified snapshot with no economically evaluable names can produce a formal
market-wide no-candidate result. Missing coverage, an unresolved enrichment
queue, or a short pool that cannot prove exhaustion remains diagnostic and
exits 2. A successful underwriting handoff or verified no-candidate result
exits 0.
### Budget exhaustion and partial screen diagnostics
Liquidity, quality-probe, and candidate-packet enrichment are fail-closed. If
the provider budget is exhausted, `screen-full-snapshot` exits 2 and persists
`audit/enriched-estimates.partial.jsonl` and
`audit/partial-run-diagnostic.json`. The diagnostic is written last as the
commit marker and records the snapshot ID/digest, stage, target-order
progress, interrupted symbol and provider operation, provider counters, and
SHA-256/row-count records for the partial summary and rows. Consumers must
verify the marker and every recorded hash before reading partial artifacts;
without a valid marker, the preceding files are not authoritative.
The current symbol is `attempted` and `interrupted`, and remains `pending`; a
symbol is `completed` only after all provider calls required by its current
stage finish. Quality-probe interruption preserves a current row even when its
key-metrics call succeeded but its actual-EPS call did not. A packet interrupted
inside an endpoint is not written as a packet; completed packets and provider
raw/cache evidence remain available. The already-written broad-screen audit is
rewritten as `status: incomplete`, `conclusion_scope: diagnostic`, and
`ranking_scope: diagnostic`, with top-level, deep-dive, candidate-pool, and
generation-audit selection commitment fields cleared. Broad-screen rows are
rewritten from `selected` to `deferred_by_budget`, and the audit's row hash is
updated before the diagnostic marker is written.
Each screen attempt uses a new empty run directory. The CLI adds a unique
attempt suffix to the run ID and stores provider raw responses in a sibling,
attempt-specific `.provider-raw/<run-id>/` directory; direct callers are
rejected if they pass a non-empty output directory. An explicitly supplied raw
store must also remain outside the run directory. This prevents a retry from
mixing partial output with stale final artifacts from an earlier attempt.
There is no `--resume` for screening. After the budget is restored, rerun the
same verified snapshot; successful new calls may append cache/raw records and
the rerun may reuse existing records, but the snapshot itself remains
read-only. Partial output never authorizes a market-wide conclusion or a real
trade.
## Readiness and content binding
Two layers, deliberately separate:
- **`snapshot_status(manifest)` → `collection_ready`** — manifest-level
aggregation only (all shards complete with zero `fetch_failed`,
classification counts summing exactly to the frozen universe,
`retrieval_time_unknown == 0`, `normalization_time_unknown == 0`). It trusts
the manifest's own counters and
therefore proves nothing about the shard files.
- **`load_verified_snapshot(snapshot_dir, screening_as_of=..., max_staleness_days=...)`
→ `verdict + verified rows`** — the ONLY source of the screening verdict and
consumable rows. It
reads the hash-bound listing-enumeration audit and every `shard-*.jsonl`
exactly once and verifies against the frozen universe:
file presence and SHA-256 vs the manifest, no duplicate symbols, every
symbol in the frozen universe and in its `stable_shard` shard, the union
covering the universe EXACTLY, classifications limited to allowed values,
per-shard counts matching the manifest — plus staleness measured from the
ACTUAL aggregated `oldest_retrieved_at` and
`oldest_normalization_as_of` against
`screening_as_of - max_staleness_days` (never the operator-supplied
collection `as_of`).
Only a run screened from this same-read bundle with
`ready_for_screening: true`, an exact classification total, and a bound
`snapshot_verification_digest` may emit `ranking_scope: final_marketwide`.
## Operating on the FMP Starter plan
~2,371 symbols ≈ 1 estimate call each. With a 350-call per-run budget and the
750-call daily plan limit, 8 shards (~300 symbols each) collect in 2 shards/day
over 4 days. After the initial build, only incremental refresh is needed
(new snapshot per refresh cycle; post-earnings and revised names first).
## Shared coverage semantics
`scripts/coverage_semantics.py` is the single source of truth for
`classify_ranking_scope` / `build_coverage_block` /
`derive_ranking_scope_from_audit` / `validate_coverage_block`; both
`run_pipeline` (discovery) and `evaluate_candidates` (final evaluation) import
it, so the tri-state semantics cannot drift between the two sides.
references/methodology-ja.md
# 米国割安成長株スクリーニング運用ガイド
## 1. 目的
米国上場企業の中から、単なる低PER株ではなく、**現在のバリュエーションが上昇しなくても、EPSまたはFCF per shareの健全な成長だけで、2～3年間に30～50％程度以上の株価上昇を期待できる企業**を発見する。
重視するのは次の組み合わせである。
- 中長期の売上成長
- 1株当たり利益・FCFの成長
- 高い資本効率
- 良好な利益の質
- 過大でない負債
- 小さいSBC・希薄化
- 妥当な同業比較
- 景気循環を平準化しても残る割安度
## v3.5の必須統制
v3.5では、schema v3の会計・証拠管理を維持しつつ、v3.1で発生した「未解決候補が残っているのに該当なしで完了する」問題を修正する。
1. 全母集団では上場状態、株価、時価総額、流動性を監査する。
2. Provider総件数とページ枯渇、または全時価総額帯の枯渇証拠を記録する。取得上下限が一致するだけでは母集団完全性としない。
3. 市場全体の大半についてROIC、FCF、負債、Forward PERを取得することを完了条件にしない。
4. ベンダープリフィルター、利用可能な予想値、またはセクター×時価総額×流動性で層化した限定候補プールを作る。
5. `enrichment_attempted`と`enrichment_resolved`を分離する。試行数や評価可能数の閾値だけでは未解決キューを完了扱いにしない。
6. 売上成長率、1株成長率、ROIC、優先PERは単独の機械的足切りではなくガイドラインとして扱う。
7. PER21～30倍の高成長例外や、売上成長率が低めでもForward PERと1株成長が魅力的な近似候補を詳細調査へ進める。
8. シクリカル度3～5は`mid_cycle_normalization_required`を付けて詳細調査へ進める。金融・REIT・自動車販売など、一般企業指標が不適切な業種は専用指標が揃うまでブロックする。
9. エンリッチメント順はGARP適合可能性、予想人数、不足項目、ベース効果で決め、流動性やティッカー順だけで決めない。
10. `no_qualifying_candidates`は、プール枯渇、未解決0、キュー0、全行最終処分、評価可能行1件以上のときだけ許可する。
11. `unprocessed = selected - verified`を不変条件とし、選定した全銘柄に検証済みレコードを作る。
12. Markdownには一次選定、予算繰越、要補完、取得不能、一次落ち、除外をすべて表示する。
13. 市場指標の鮮度は取得日時ではなく、`data_as_of`または公表日時で検証する。
従来のschema v3統制も継続する。四半期と通期の分離、NTM/FY1倍率、予想人数、情報源型、EV/FCFの現金定義、医薬品LOE、自動車販売会社のFloorplan Debt、企業イベント事前確認、設備投資符号、TTM再構築、標準FCFとAdjusted FCFの分離、GAAP調整表、チェックポイント、`--require-final`である。
### v3.5追加統制
1. 監査、チェックポイント、最終スナップショットに同一の実行時フィンガープリントを必須とし、v3.1／v3.2の古い成果物を拒否する。
2. 監査済み限定プールは、その範囲を明示した正式ランキングを作れるが、市場全体の「該当なし」と表現しない。
3. EPS・FCF予想は、売上高、利益率、金利・その他、税率、税引後調整、希薄化後株式数から独立再計算する。目標EPS×株数で逆算した分子は検証にならない。
4. Adjusted予想は、ドライバーから算出したGAAP指標と税引後調整額をGAAP調整表へ別々に一致させる。
5. TTMキャッシュフローを構成する各期間に情報源IDを要求する。
6. 現金分類、ROIC、EBITDAに根拠がなければ、データ品質点を制限する。
7. `biopharma`、`royalty_biopharma`、`drug_delivery_platform`等もcommercial biopharmaへ正規化し、製品・ロイヤリティ集中、LOE、6倍・8倍ストレスを要求する。
8. `peak_profit_risk=true`なら、シクリカル度が1～2でも平準化を必須とする。
9. Broad Screenの決定論的ステータスを説明生成段階で上書きしない。
10. ユーザー依頼の時価総額範囲と実際の取得範囲を分離し、内部予算を理由に依頼範囲を書き換えない。
11. 流動性は20営業日以上の平均売買代金またはProvider平均出来高から計算し、単日出来高を使用しない。
12. Forward PERは日付付きのNTMまたはFY1 EPSからのみ計算し、FY1欠損時にFY2・FY3を代用しない。
13. Broad Screenで選定した集合をハッシュで確定し、全選定銘柄を完了させる。予算変更はBroad Screenの再実行で行う。
14. Helperのexit code 2は同一実行内の継続指示であり、ユーザーへ「続けて」と求めない。
## 段階的スクリーニング
### レイヤーA：全母集団の上場・流動性監査
取引所、普通株・アクティブ状態、株価、時価総額、売買代金を確認する。この層は対象範囲の監査であり、全社の財務精査を意味しない。
### レイヤーB：限定された経済候補プール
Forward評価と1株当たり成長予想を取得できる透明な候補プールを作る。ベンダーのプリフィルターを優先し、利用できない場合はセクターと時価総額で層化した流動性の高い銘柄を予想値で補完する。
### レイヤーC：一次情報による精査
選定した銘柄だけをSEC・IRで確認し、キャッシュフロー、会計基準、同業、景気循環、業種KPI、シナリオを完成させる。
## 2. 調査範囲
標準条件：
- NYSE、Nasdaq、NYSE American
- 普通株
- 時価総額5億～200億ドルを中心
- 株価5ドル以上
- 平均売買代金500万ドル以上
- 最終候補は最大10銘柄。条件を満たさなければ無理に10銘柄へ増やさない
原則除外：
- OTC、ペニー株、極端な低流動性銘柄
- 実質的な事業を持たないSPAC・シェル
- 売上がほとんどない開発段階バイオ
- 継続的な増資を前提とする企業
- M&A価格に株価が固定された合併裁定銘柄
- 未解決の重大な会計・内部統制・ガバナンス問題
- 事業実態よりミーム需給だけで評価される銘柄
除外した企業と理由を記録し、黙って候補から消さない。
## 3. 段階的スクリーニング
### 第0段階：全上場母集団の監査
依頼範囲の各銘柄について、上場状態、普通株判定、取引所、株価、時価総額、流動性、情報源、実際の取得上下限を保存する。この層は対象範囲と売買可能性の監査であり、全社の完全な財務データを意味しない。
### 第1段階：限定候補プールの作成と補完
再現可能なベンダー絞り込み、利用可能な一括予想値、またはセクター×時価総額×流動性の層化抽出を使う。限定プールだけにForward評価と1株当たり成長予想を補完する。Forward評価、1株成長、予想人数などのブロック項目が足りない行は`needs_enrichment`とし、点数を付けない。金融・REIT・自動車販売は専用指標が揃うまで同様にブロックする。高シクリカル企業の平準化不足は非ブロックとし、魅力的な候補は`sector_review_required`として実スコア付きで詳細調査へ進める。
候補発見の目安：
- 売上成長率10％前後以上
- EPSまたはFCF per share成長率15％前後以上、20～30％を高評価
- Forward PER 20倍以下を中心とし、高成長・高品質なら30倍まで例外
- 標準FCF黒字または業種別適切指標
- ROIC 10％前後以上
- 業種調整後Net Debt / EBITDA 2.5倍以下
- 希薄化後株式数の年率増加3％以下
- 十分な売買代金
限定プールは、全行が解決済みで、プール枯渇を確認した場合だけ完了扱いにする。試行しただけでは解決済みにしない。選定0であっても、未解決行やキューが残る場合は`insufficient_data`であり、「該当なし」ではない。正式な「該当なし」には、評価可能な行が1件以上必要である。
### 第2段階：企業イベント事前確認
詳細調査前に、上場状態、M&A、上場停止・廃止、ティッカー変更、株式分割、スピンオフ、最新重要イベント日時を確認する。進行中・完了済みM&Aや非アクティブ銘柄は通常ランキングから除外し、選定済みであれば除外レコードとして完了保存する。
### 第3段階：一次情報による精査
選定した全銘柄をSEC・IRで確認し、`eligible`、`review_required`、`screened_out`、`excluded`のいずれかへ完了させる。
最終的に必ず答える問い：
1. 成長は持続可能か。
2. 総額ではなく1株当たりでも成長しているか。
3. 利益は営業キャッシュフローに裏付けられているか。
4. 割安理由は一時的か、構造的か。
5. PERが上昇しなくても十分なリターンがあるか。
6. PERが20％低下してもプラスのリターンが残るか。
7. どの数値・事象で投資仮説を無効化するか。
## 4. 分析時点の固定
候補間で比較する前に、分析スナップショットを固定する。
記録項目：
- 分析日時とタイムゾーン
- 株価取得日時、通常終値・プレマーケット・時間外の区分
- 最新決算期と発表日時
- 最新の重要な企業イベント
- コンセンサス取得日時
- FY1、FY2、FY3の会計年度対応
- 通貨と株式分割調整の有無
決算発表後の情報と決算発表前の終値を混在させない。混在する場合は株価を更新するか、明確に「イベント前評価」と表示する。厳格モードでは除外する。
## 5. 情報源と分類
重要な数字を次の4種類へ分類する。
- `reported_fact`：報告済み実績
- `company_guidance`：会社予想
- `market_consensus`：市場コンセンサス
- `analyst_estimate`：分析上の独自予想
優先順位：
1. SEC提出資料
2. 企業の決算リリース、株主書簡、説明資料、Prepared Remarks、Investor Day
3. 信頼できるデータベンダーの会社予想・コンセンサス
4. ニュース・外部分析は補足
情報が矛盾する場合はSEC提出資料を優先し、差異を説明する。
コンセンサスには、取得日、対象年度、FY1/FY2/NTM、GAAP/Adjusted、アナリスト数、可能なら予想レンジを付ける。
## 6. 主要計算
### 標準FCF
設備投資は**正のキャッシュ支出額**へ正規化する。ベンダーが負号付きで返す値をそのまま入力しない。
```text
標準FCF = 営業キャッシュフロー - 設備投資キャッシュ支出額
```
TTMは、報告済みTTM、独立した4四半期の合計、または「直近通期＋当期YTD－前年同期YTD」のいずれかで作る。3カ月・6カ月・9カ月の累計値を重複合計しない。会社独自Adjusted FCFは定義を付けて別表示し、標準FCFの代替にしない。
### SBC控除後の経済的FCF
```text
SBC調整後FCF = 標準FCF - Stock-Based Compensation
```
これはGAAPのキャッシュフロー指標ではなく、株主経済価値を見る補助指標である。標準FCFと併記する。
### FCF per share
```text
FCF per share = 標準FCF / 希薄化後加重平均株式数
```
### ネットデットと利用可能現金
```text
利用可能現金 = 企業現金 + 株主に利用可能な有価証券
Net Debt = 有利子負債 - 利用可能現金
```
顧客資金、加盟店・決済フロート、制限現金を企業現金へ含めない。特に決済会社では総現金とCorporate cashを分ける。
### ROIC
原則としてNOPATを平均投下資本で割る。ベンダー値を使う場合は定義を確認する。銀行・保険・REIT等には業種別指標を使う。
## 7. 成長の評価
過去3～5年：
- 売上CAGR
- GAAP EPS CAGR
- 標準FCF per share CAGR
- 基本・希薄化後株式数の増減
- 粗利益率、営業利益率、FCFマージン
- ROIC推移
将来2～3年：
- 今期・来期の売上成長
- 今期・来期のEPS成長
- 2年後・3年後のEPSまたはFCF per share
- 将来の希薄化後株式数
市場拡大、シェア上昇、ストック収益、スイッチングコスト、ネットワーク効果、製品ミックス、海外展開、営業レバレッジは加点する。
顧客・製品集中、M&Aだけの成長、株式発行を伴う成長、特需、補助金依存、コモディティ価格だけの増益、数量減を伴う値上げ、利益を犠牲にした売上成長は減点する。
## 8. EPS・FCF予想の整合性
コンセンサスをそのまま転載せず、少なくとも以下へ分解する。
```text
売上高
× 粗利益率
- 営業費用
= 営業利益
+/- 金利・その他
- 税金
= 純利益
÷ 希薄化後株式数
= EPS
```
FCFは、利益、非現金費用、運転資本、設備投資に分解する。
利益率の急上昇、税率の異常低下、過大な自社株買い、希薄化無視など、説明のない前提を警告する。
## 9. GAAPとNon-GAAP
現在、2年後、3年後の評価指標は、`gaap`、`company_adjusted`、`analyst_normalized`、`sector_defined`のいずれか一つへ統一する。現在GAAP、将来Adjustedのような混在では、PER据え置きシナリオを計算しない。Adjustedまたは分析者正規化指標は、GAAP指標、調整項目、反復性、算術一致、情報源IDを示す。
GAAP営業利益、Adjusted営業利益、GAAP EPS、Adjusted EPS、GAAP純利益、Adjusted EBITDA、標準FCF、Adjusted FCFを混同しない。
毎期繰り返すSBC、リストラクチャリング、買収費用、無形資産償却等を「一時的」として除外している企業は、Adjusted利益を割り引いて評価する。
## 10. 利益の質
確認項目：
- 純利益と営業キャッシュフローの乖離
- 売掛金・DSO
- 在庫・在庫日数
- 未請求売上・契約資産
- ソフトウェア開発費やR&Dの資本化
- リストラクチャリング・買収費用の常態化
- のれん・無形資産・減損
- 税効果・資産売却益・年金会計
- 関連当事者取引
- 監査法人変更
- Internal Control Weakness、修正再表示、SEC調査
- KPI定義の頻繁な変更
運転資本の一時的改善でFCFが増えている場合、通常状態のFCFを別途評価する。
## 11. SBC・希薄化・自社株買い
必ず確認：
- SBC金額
- SBC / 売上高
- SBC / 営業キャッシュフロー
- 基本・希薄化後株式数の3年推移
- RSU、オプション、ワラント、転換社債、ATM、Shelf、公募
- 実際の買い戻し額・株数
自社株買いの発表額ではなく、SBC発行分を差し引いた純減少株式数を評価する。負債で高値買い戻しをしていないかも確認する。
## 12. バリュエーションとシナリオ
利用指標：
- NTM/FY1/FY2予想PER
- EV/EBITDA、EV/EBIT、EV/FCF
- FCF利回り、SBC調整後FCF利回り
- PEG
- 自社5年中央値
- 真に比較可能な同業中央値
### PER据え置き
```text
将来株価 = 将来EPS × 現在Forward PER
上昇率 = 将来株価 / 現在株価 - 1
年率リターン = (将来株価 / 現在株価)^(1/年数) - 1
```
### PER20％低下
```text
ストレスPER = 現在PER × 0.8
ストレス将来株価 = 将来EPS × ストレスPER
```
### 同業中央値
比較基準が同一の場合のみ補助的に計算する。ランキングはリレーティングを前提にしない。
## 13. 同業比較
3～5社を、事業構造、顧客、成長段階、収益モデル、利益率、資本集約度で選ぶ。選定理由を説明する。
比較する指標：
- 成長率
- 利益率
- ROICまたは業種別収益性
- レバレッジ
- FCF利回り
- SBC
- 希薄化
- Forward valuation
対象企業だけが安い理由を、一時的・構造的・合理的のいずれかに分類する。
## 14. シクリカル評価
1～5で評価する。
- 1：景気影響が小さい
- 2：やや影響
- 3：中程度
- 4：大きい
- 5：極めて大きい
シクリカル企業は5～10年の売上、利益率、EPS、FCF、受注、受注残、Book-to-Bill、在庫、稼働率、顧客設備投資、商品価格、景気後退時の利益減少を確認する。
ピーク利益ではなく、ミッドサイクルEPS・FCFで実質PER・EV/FCFを計算する。
## 15. 直近決算
記録：
- 決算期・発表日
- 売上高、前年比、前四半期比、Organic Growth
- 粗利益率、GAAP営業利益・利益率
- GAAP EPS、Adjusted EPS
- 営業キャッシュフロー、FCF、設備投資、SBC
- 希薄化後株式数
- 会社予想と修正方向
- セグメント、受注、受注残、業種別KPI
- 一時項目、運転資本、税率
成長状態を、加速、安定、やや減速、明確に減速、底打ち、ピークアウトに分類する。
## 16. カタリスト・リスク・無効化条件
6～24カ月のカタリストを、時期、実現確率、影響経路とともに記載する。
最大リスクを抽象語だけで書かない。無効化条件は数値・事象で示す。
例：
- 売上成長率が2四半期連続で8％未満
- 受注残が前年比減少
- 粗利益率が55％未満
- NRRが100％未満
- Net Debt / EBITDAが3倍超
- 年率希薄化率が5％超
- FCF赤字化
- 主要顧客喪失
- 通期ガイダンス撤回
## 17. Fail-Closed方針
次の場合はランキングしない。
- 株価・決算・予想の基準時点が不整合
- 業種に不適切な評価指標
- 重要数値の出典不足
- データ品質が最低基準未満
- Hard exclusionが未解決
- 将来1株指標の整合性を確認できない
- 必須の同業比較・サイクル平準化ができない
状態：
- `eligible`：ランキング可能
- `review_required`：有望だが未解決事項あり
- `screened_out`：上場・取引は有効だが経済条件で一次選考落ち
- `excluded`：M&A完了、非アクティブ、上場廃止、ハード除外条件または無効入力
未処理銘柄がある実行は`provisional`とし、完了済み銘柄の計算を表示しても最終3銘柄は選定しない。
条件を満たす企業がなければ「現時点では該当なし」と結論づける。
## v3.5 候補品質の改善
- コアGARP、高成長例外、割安ニアミス、ミッドサイクル精査銘柄を複数レーンで選び、単一スタイルへの偏りを抑える。
- 30%上昇余地だけでなく、SBC控除後FCF、EV/FCF、ROIC、負債、希薄化、予想下限ケース、LOE尾部リスクを正式ランク条件にする。
- 軽微な不足は`conditional`、重大な不足は`review_required`とし、質の弱い候補を無理にランキングしない。
- 売上、利益率、金利、税率、株式数など各予想ドライバーに独立した根拠を要求し、目標EPSからの逆算を禁止する。
- スピンオフや大型買収はハード除外ではなく、Pro Forma平準化を必要とする企業移行イベントとして扱う。
- 最終レポートは事前監査を通し、参照した全原票を含む自己完結ZIPとして提供する。
### Providerプリフィルターの再現率保護
単一の「PER20倍以下・売上成長8%以上・低シクリカル」という条件だけで候補プールを作らない。コアGARP、高成長例外、割安ニアミス、シクリカル平準化の4レーンを別々に取得して統合する。限定プールは原則30銘柄以上、少なくとも3レーンを含め、選定前に全行を解決する。
references/methodology.md
# US Undervalued Growth Screening Methodology
## 1. Objective
Identify US-listed companies whose **per-share economic value** can compound fast enough to produce attractive two- to three-year returns without relying on multiple expansion.
The central test is:
> If the current valuation multiple remains unchanged, can healthy EPS or free-cash-flow-per-share growth alone support a credible 30% to 50% total price increase over two to three years?
A low P/E ratio is not sufficient. The candidate must also demonstrate sustainable growth, acceptable financial risk, high-quality earnings, sensible capital allocation, limited dilution, and a defensible business position.
## Version 3.5 Mandatory Controls
Contract 3.5 adds the following non-negotiable controls:
1. Every artifact carries the installed runtime fingerprint; stale v3.1-v3.4 artifacts are rejected.
2. A bounded provider/discovery pool can support a scoped final ranking only when its generation is audited and every pool row is resolved; it cannot support a market-wide no-candidates claim.
3. Forecasts are reconstructed from revenue, margin, interest/other, tax, adjustments, and diluted shares. A numerator reverse-engineered from target EPS is not a forecast bridge.
4. Adjusted forecasts separately reconcile the driver-derived GAAP metric and after-tax adjustments.
5. TTM cash-flow periods require resolving source evidence.
6. Cash classification, ROIC, and EBITDA require evidence before quality points can be awarded.
7. Commercial-biopharma aliases trigger concentration, LOE, and 6x/8x stress controls.
8. `peak_profit_risk=true` requires normalization regardless of the numeric cycle score.
9. The deterministic broad-screen decision is authoritative; the narrative layer cannot reintroduce a mechanical revenue-growth or cyclical exclusion.
10. Preserve the user-requested market-cap range independently from the executed retrieval range; internal budget pressure cannot silently redefine the request.
11. Accept only 20+ session average-dollar-liquidity evidence; one-session volume cannot determine discovery eligibility or order.
12. Normalize dated annual estimates to the current NTM/FY1 horizon before computing Forward P/E; never substitute FY2/FY3 because FY1 is missing.
13. Commit the selected set and complete every selected symbol; any budget reduction requires a new deterministic broad-screen run.
14. Treat helper exit code 2 as an internal continuation signal and never require a separate user “continue” turn.
## Inherited v3.2 Controls Retained in v3.5
Contract revision 3.2 preserves schema-v3 accounting and evidence controls while correcting the v3.1 completion and candidate-routing defects:
1. Audit listing, quote, market-cap, and liquidity fields across the full requested universe.
2. Prove universe enumeration through provider totals plus exhausted pagination or exhausted market-cap-band audits. Requested/retrieved endpoints alone do not prove completeness.
3. Do **not** require complete statements, ROIC, FCF, leverage, and forward estimates for most of the market.
4. Generate a bounded candidate pool through a provider prefilter, available estimates, or a deterministic sector/market-cap/liquidity fallback.
5. Distinguish `enrichment_attempted` from `enrichment_resolved`. Attempt-count and evaluable-count thresholds never complete a pool with unresolved rows.
6. Permit high-quality near misses and P/E 21–30 high-growth exceptions. Growth thresholds are guidelines, not isolated hard gates.
7. Route strong cyclicals to deep dive with `mid_cycle_normalization_required`; do not automatically reject them. Financial-sector and auto-dealer valuation/leverage gaps remain blocking.
8. Prioritize enrichment by GARP information value, not ticker order or liquidity alone.
9. A valid no-candidates result requires an exhausted pool, zero unresolved rows, an empty queue, all final dispositions, and at least one economically assessable row.
10. Enforce `unprocessed = selected - verified`; every selected symbol ends with a verified candidate record.
11. Render broad-screen selected, deferred, review, unavailable, screened-out, and excluded rows in the final Markdown instead of hiding them behind empty deep-dive counts.
12. Validate dynamic market fields against their `data_as_of`/publication date and require official sources for policy, Treasury, inflation, and GDP.
The existing schema-v3 controls remain mandatory: separate quarter/full-year records; NTM/FY1 formal valuation; analyst-count rankability; typed source ledgers; consistent EV/FCF cash definition; commercial-biopharma concentration/LOE stress; auto-dealer floorplan treatment; corporate-action preflight; evidence-derived quality; positive capex-outflow convention; explicit TTM reconstruction; standard versus adjusted FCF separation; same-basis periods; GAAP reconciliation; checkpointing; and `--require-final`.
## Tiered Screening Process
### Layer A: Listing-universe audit
Audit all retrieved symbols for exchange, active/common-stock status, price, market cap, and liquidity. This layer establishes scope and trading eligibility; it does not claim full financial coverage.
### Layer B: Bounded economic discovery
Create a transparent candidate pool with forward valuation and per-share growth evidence. Use a provider prefilter when available. If bulk fundamentals are unavailable, stratify up to 120 liquid in-scope symbols across sectors and market-cap bands, enrich them with available estimates, and record every attempt.
### Layer C: Primary-source underwriting
Verify the selected set with SEC and IR materials, normalize cash flow and accounting basis, compare genuine peers, apply cycle/sector controls, and calculate formal scenarios.
## 2. Universe
### Default scope
- Exchanges: NYSE, Nasdaq, NYSE American.
- Security type: operating-company common stock.
- Market capitalization: normally USD 500 million to USD 20 billion.
- Minimum price: configurable; default USD 5.
- Minimum average daily dollar volume: configurable; default USD 5 million.
- Final ranking: no more than 10 candidates, and fewer when evidence is weak.
### Allowed exceptions
Include a company outside the default market-cap range only when it strongly satisfies the methodology. Record the reason for the exception.
### Hard or presumptive exclusions
- OTC securities.
- Penny stocks or extreme liquidity risk.
- Shell companies and pre-combination SPACs.
- Development-stage biotechnology companies with little operating revenue.
- Businesses dependent on repeated equity issuance for survival.
- Pending merger-arbitrage situations where price is anchored to transaction terms.
- Companies with unresolved material restatements, internal-control weaknesses, severe governance concerns, or active SEC investigations.
- Meme-driven securities without operating support.
Preserve excluded names and reasons in the screening audit trail.
## 3. Staged Screening Process
### Stage 0: Full listing-universe audit
Retain one row per requested symbol with active/common-stock status, exchange, price, market capitalization, liquidity, source ID, and actual retrieval bounds. This layer proves scope and tradability; it does not claim full financial coverage.
### Stage 1: Bounded candidate generation and enrichment
Use a reproducible provider prefilter, already-available estimates, or a deterministic sector × market-cap × liquidity discovery pool. Enrich only the bounded pool with forward valuation and per-share growth. Missing blocking discovery metrics become `needs_enrichment` and receive no score. Financial-sector or auto-dealer valuation/leverage gaps remain blocking. A cyclical normalization requirement is nonblocking: a strong candidate may receive `sector_review_required`, a real score, and selection for mid-cycle deep-dive verification.
Suggested guidelines for assessable rows:
- Revenue growth near 10% or better where appropriate.
- EPS or FCF-per-share growth near 15% or better; 20%–30% preferred.
- Forward P/E normally 20x or lower, with justified high-growth exceptions through 30x.
- Positive standard FCF or a sector-appropriate substitute.
- ROIC near 10% or better where relevant.
- Net debt/EBITDA normally 2.5x or lower using sector adjustments.
- Diluted-share growth normally 3% or lower.
- Adequate dollar-volume liquidity.
The pool is complete only after every bounded row is resolved and the operator has verified exhaustion. A query attempt does not equal resolution. `no_qualifying_candidates` additionally requires zero selected rows and at least one economically assessable row. If no assessable row exists, the result is `insufficient_data`, not “no candidates.”
### Stage 2: Corporate-action preflight
Before deep financial work, verify active listing/symbol, pending or completed M&A, delisting/suspension, ticker changes, splits, spin-offs, and the latest material-event time. Pending/completed M&A and inactive/delisted securities are hard exclusions from normal GARP ranking. Preserve source IDs and checkpoint the result.
### Stage 3: Primary-source underwriting
For every selected name, verify load-bearing facts with SEC filings or company IR and complete one auditable record even when the final classification is review-required, screened-out, or excluded.
The deep dive must answer:
1. Is growth real and durable?
2. Is growth visible on a per-share basis?
3. Is reported profit supported by operating cash flow?
4. Is the valuation low for a temporary reason or a structural one?
5. Does the return case survive without re-rating?
6. Does the return case survive a 20% multiple contraction?
7. What measurable facts would invalidate the thesis?
## 4. Immutable Analysis Snapshot
Create one analysis snapshot before comparing candidates.
Record:
- Analysis timestamp and timezone.
- Price timestamp, session, and timezone.
- Whether price is regular close, pre-market, or after-hours.
- Latest earnings period and publication timestamp.
- Latest material corporate action timestamp.
- Consensus retrieval timestamp.
- Fiscal-year mapping used for FY1, FY2, and FY3.
- Currency and split-adjustment status.
### Stale-data gate
A price is stale when it predates a later earnings release or material corporate action while the analysis uses the post-event information. Either update the price or clearly label the report as a pre-event valuation. In strict mode, reject the candidate.
### Fiscal-year consistency
Do not mix calendar-year estimates with fiscal-year estimates. Do not compare a trailing metric for one company with a forward metric for another without labeling the mismatch.
## 5. Source Classification and Hierarchy
Classify every important figure as one of:
- `reported_fact`
- `company_guidance`
- `market_consensus`
- `analyst_estimate`
Primary-source hierarchy:
1. SEC 10-K, 10-Q, 8-K, 20-F, or 6-K.
2. Company earnings release, shareholder letter, investor presentation, prepared remarks, or investor-day material.
3. Company guidance transcribed by a reputable data source.
4. Market-data vendor or consensus database.
5. Reputable news or analysis for context only.
When sources conflict, prefer the SEC filing, then explain the discrepancy.
For consensus estimates record:
- retrieval date,
- fiscal period,
- FY1/FY2/NTM classification,
- GAAP or adjusted basis,
- analyst count,
- estimate range or dispersion when available.
Never silently fill missing values.
## 6. Core Financial Definitions
### Standard free cash flow
Normalize capex as a **non-negative cash-outflow magnitude**. A vendor value reported with a negative cash-flow sign must be converted before evaluation.
```text
Standard FCF = operating cash flow - capex cash outflow
```
Declare one TTM method: explicitly reported TTM; four unique discrete quarters; or latest FY plus current YTD minus prior-year comparable YTD. Never sum overlapping cumulative 3-, 6-, and 9-month cash-flow statements. If the company presents adjusted FCF, show it separately with its definition; do not substitute it for standard FCF.
### SBC-adjusted economic FCF
```text
SBC-adjusted FCF = standard FCF - stock-based compensation
```
This is an economic lens, not a GAAP cash-flow measure. Show both values. Do not double-count SBC if the selected FCF definition already deducts it.
### FCF per share
```text
FCF per share = standard FCF / diluted weighted-average shares
```
Use diluted shares and a consistent period.
### Enterprise value
```text
Enterprise value = market capitalization + total debt + preferred stock
+ non-controlling interest - cash and equivalents
```
Use a consistent definition and document material adjustments.
### Net debt / EBITDA
```text
Usable cash = corporate cash + eligible marketable securities
Net debt / EBITDA = (total debt - usable cash) / EBITDA
```
Exclude customer, merchant, settlement, and restricted funds from usable cash. Use normalized EBITDA for cyclicals. Negative net debt is acceptable and should not be forced to zero.
### ROIC
Preferred definition:
```text
NOPAT = adjusted operating income * (1 - normalized cash tax rate)
Invested capital = average operating assets - average operating liabilities
ROIC = NOPAT / average invested capital
```
Use a vendor ROIC only after confirming its definition. For financial companies, use sector-appropriate returns instead.
## 7. Growth Analysis
Measure both aggregate and per-share growth.
Required historical views when available:
- Three- to five-year revenue CAGR.
- Three-year GAAP EPS CAGR.
- Three-year standard FCF-per-share CAGR.
- Basic and diluted-share change.
- Gross, operating, and FCF margin trends.
- ROIC trend.
Required forward views:
- Current fiscal-year revenue and EPS growth.
- Next fiscal-year revenue and EPS growth.
- Two- and three-year EPS or FCF-per-share forecast.
- Forecast diluted-share count.
### Growth-quality checklist
Positive drivers include:
- market expansion,
- market-share gains,
- recurring revenue,
- switching costs,
- network effects,
- pricing backed by stable volume,
- mix improvement,
- new products or capacity,
- international expansion,
- cross-selling,
- aftermarket revenue,
- operating leverage.
Negative drivers include:
- customer or product concentration,
- acquisition-only growth,
- equity-funded acquisitions,
- temporary demand surges,
- subsidy dependence,
- commodity-price-only growth,
- price increases with declining volume,
- deteriorating customer-acquisition economics,
- growth achieved by sacrificing unit economics.
## 8. Forecast Bridge
Do not treat consensus as self-validating. Reconstruct the estimate.
At minimum bridge:
```text
Revenue
x gross margin
- operating expenses
= operating income
+/- interest and other income
- taxes
= net income
/ diluted shares
= EPS
```
For FCF, bridge:
```text
Operating income / net income
+ non-cash charges
+/- working-capital change
- capex
= standard FCF
/ diluted shares
= FCF per share
```
Test whether margins, tax rate, interest, and share count are plausible. Flag forecasts that rely on unexplained linear extrapolation, abrupt margin expansion, or aggressive buyback assumptions.
## 9. GAAP and Non-GAAP Discipline
The current, year-2, and year-3 valuation metrics must use one accounting basis: `gaap`, `company_adjusted`, `analyst_normalized`, or `sector_defined`. A current GAAP metric cannot be compared with future adjusted consensus under a constant multiple. Mixed bases block the scenario. Company-adjusted and analyst-normalized metrics require an arithmetic reconciliation from GAAP with labeled adjustment rows, recurring/nonrecurring classification, and source IDs.
Always distinguish:
- GAAP operating income,
- adjusted operating income,
- GAAP net income,
- GAAP EPS,
- adjusted EPS,
- adjusted EBITDA,
- standard FCF,
- company-defined adjusted FCF.
Investigate recurring exclusions:
- stock-based compensation,
- restructuring costs,
- acquisition costs,
- intangible amortization,
- litigation costs,
- impairment charges,
- one-time tax benefits.
A cost that recurs every quarter or year is not economically one-time. Penalize large or widening GAAP/non-GAAP gaps.
## 10. Earnings and Cash-Flow Quality
Review:
- Net income versus operating cash flow.
- Accounts-receivable growth and DSO.
- Inventory growth and days inventory.
- Unbilled revenue and contract assets.
- Capitalized software or R&D.
- Recurring restructuring and acquisition charges.
- Goodwill and intangible growth.
- Asset sales, tax benefits, or pension income.
- Related-party transactions.
- Auditor changes.
- Material weaknesses, restatements, or SEC inquiries.
- Frequent changes to KPI definitions.
Working-capital releases can temporarily inflate FCF. Identify the structural and temporary portions.
## 11. Stock-Based Compensation, Dilution, and Buybacks
Collect:
- SBC amount.
- SBC / revenue.
- SBC / operating cash flow.
- Basic and diluted shares for at least three years.
- RSUs, options, warrants, convertibles, ATM programs, shelves, and recent offerings.
- Actual repurchase amount and shares retired.
Evaluate buybacks by **net share reduction**, not authorization announcements.
```text
Net share change = ending diluted shares / beginning diluted shares - 1
```
Ask:
- Did repurchases offset employee issuance?
- Were shares bought above reasonable value?
- Was debt increased to fund repurchases?
- Did diluted shares actually decline?
## 12. Valuation
Use multiple measures:
- NTM, FY1, and FY2 P/E.
- EV/EBITDA.
- EV/EBIT.
- EV/FCF.
- FCF yield.
- SBC-adjusted FCF yield.
- PEG.
- Five-year own-history median.
- Genuine peer median.
### Value-trap checks
A low multiple is not attractive when:
- earnings are at a cycle peak,
- orders or revenue are decelerating,
- EPS includes one-time tax or asset-sale gains,
- leverage is excessive,
- market share is structurally eroding,
- customer concentration is severe,
- EPS rises while FCF does not,
- dilution transfers value away from owners.
## 13. Return Scenarios
### Constant-multiple scenario
For an EPS-based candidate:
```text
Future price = future diluted EPS * current forward P/E
Upside = future price / current price - 1
CAGR = (future price / current price)^(1 / years) - 1
```
For other sectors, substitute the appropriate per-share metric and multiple.
### 20% multiple-contraction stress
```text
Stressed multiple = current multiple * 0.80
Stressed future price = future metric * stressed multiple
```
### Peer-median scenario
Use only when peers are economically comparable and the same metric basis is available.
```text
Peer case price = future metric * peer median multiple
```
The peer case is secondary. Never rank primarily on assumed re-rating.
## 14. Peer Selection
Choose three to five peers based on:
- business economics,
- customers and end markets,
- growth stage,
- recurring versus transactional revenue,
- margins,
- capital intensity,
- geographic mix.
Explain each peer choice. Do not use a faster-growing or unrelated company merely to make the target look cheap.
Compare:
- growth,
- margins,
- ROIC or sector return measure,
- leverage,
- FCF yield,
- SBC,
- dilution,
- forward valuation.
Classify the discount as:
- temporary and potentially resolvable,
- structural and difficult to resolve,
- justified by superior risk,
- unsupported because the peer set is weak.
## 15. Cyclicality
Rate cyclicality from 1 to 5:
1. Low economic sensitivity.
2. Mild sensitivity.
3. Moderate sensitivity.
4. High sensitivity.
5. Extreme cyclicality.
For cyclical companies analyze five to ten years when available:
- revenue,
- operating margin,
- EPS,
- FCF,
- orders and backlog,
- book-to-bill,
- inventory and utilization,
- customer and industry capex,
- commodity prices,
- price versus volume,
- recession drawdowns.
Calculate mid-cycle revenue, margin, EPS, and FCF when possible. Use normalized earnings for valuation, not peak earnings.
## 16. Latest-Earnings Assessment
Record:
- fiscal period and publication date,
- revenue and year-over-year / sequential growth,
- organic growth,
- gross and operating margins,
- GAAP and adjusted EPS,
- operating cash flow and FCF,
- capex and SBC,
- diluted shares,
- guidance changes,
- consensus differences,
- segment results,
- orders/backlog and sector KPIs,
- one-time items,
- working-capital and tax changes.
Classify the growth state:
- accelerating,
- stable,
- modestly decelerating,
- clearly decelerating,
- cycle bottoming,
- cycle peaking.
A headline beat does not override deteriorating guidance, cash flow, backlog, or core KPIs.
## 17. Catalysts, Risks, and Invalidation
Evaluate catalysts over six to 24 months and specify timing, probability, and likely mechanism.
Examples:
- growth reacceleration,
- end of inventory correction,
- order recovery,
- new product or capacity,
- margin improvement,
- debt reduction,
- actual buybacks,
- divestiture or spin-off,
- regulatory or litigation resolution.
Define the largest risk concretely. Write measurable invalidation conditions, such as:
- revenue growth below a threshold for two quarters,
- backlog declining year over year,
- gross margin below a threshold,
- NRR below a threshold,
- net leverage above a threshold,
- annualized dilution above a threshold,
- FCF turning negative,
- loss of a major customer,
- guidance withdrawal.
## 18. Fail-Closed Decision Policy
Do not rank a candidate when:
- critical snapshot data is stale,
- the valuation basis is inappropriate,
- source provenance is missing for load-bearing numbers,
- data-quality score is below the configured floor,
- a hard-exclusion flag is unresolved,
- future per-share estimates cannot be reconciled,
- peer or cyclical normalization is essential but unavailable.
Status values:
- `eligible`: enough evidence to rank.
- `review_required`: promising but unresolved evidence prevents final ranking.
- `screened_out`: active/researchable security that failed the broad economic screen.
- `excluded`: hard exclusion, inactive security, special-situation pricing, or invalid input.
A partial run or a run with unprocessed candidates is `provisional`. It may show completed candidate calculations but must not publish final-three selections.
## 19. Final Selection
Select up to three categories:
- Highest conviction.
- Most undervalued.
- Largest upside.
For each answer:
1. Why it qualifies.
2. Return from per-share growth alone.
3. What the market may be missing.
4. Most important catalyst.
5. Largest risk.
6. Best reason not to buy now.
7. Bear case.
8. Invalidation condition.
9. KPI to monitor next quarter.
The same company may occupy multiple categories. No candidate is preferable to a forced recommendation.
## Version 3.5 Candidate-Quality Improvements
- Use deterministic multi-lane sampling to improve discovery diversity and avoid selecting only one factor style.
- Apply formal quality gates after the upside screen so low-P/E/high-EV-FCF transition stories do not become automatic winners.
- Test low-consensus-case upside in addition to average consensus.
- Require independent, sourced driver provenance and reject target-solved bridges.
- Treat spin-offs and transformative M&A as pro-forma corporate transitions.
- Publish only self-contained, pre-audited run bundles.
references/migration-v1-to-v2.md
# Migration from Schema Version 1 to Version 2
## Breaking Change
Schema version 2 is intentionally fail-closed. A version 1 input cannot be converted by relabeling `schema_version`; it lacks evidence and period semantics needed to prevent false rankings.
## Required Changes
### 1. Add run metadata and funnel
Add `run_metadata`, `screening_funnel`, and explicit `partial`/`complete` status.
### 2. Replace self-attestation
Remove reliance on `research_completeness`. Add:
- candidate-level `evidence` map,
- resolving `sources` rows,
- field-specific source IDs.
A Boolean claim such as `forecast_bridge_verified: true` no longer earns quality credit.
### 3. Add corporate-action preflight
Create `corporate_action_check` with listing status, M&A status, active-symbol flag, timestamp, latest material event, and source IDs.
### 4. Normalize capex sign and TTM period semantics
Replace ambiguous `capex_ttm` with non-negative `capex_cash_outflow`. Select one TTM method:
- `reported_ttm`
- `sum_4_discrete`
- `fy_plus_current_ytd_minus_prior_ytd`
Do not sum overlapping YTD periods.
### 5. Separate standard and company-adjusted FCF
Use `cash_flow_ttm.standard_fcf` for OCF minus capex. Keep `company_adjusted_fcf` and its definition separate.
### 6. Classify cash
Replace a single cash value with:
- `corporate_cash`
- `marketable_securities`
- `customer_or_settlement_funds`
- `restricted_cash`
### 7. Replace flat valuation fields with period objects
Version 1:
```json
{
"metric_basis": "gaap",
"current_metric": 3.0,
"year_2_metric": 4.2,
"year_3_metric": 5.0
}
```
Version 2:
```json
{
"periods": {
"current": {"metric": 3.0, "metric_basis": "gaap", "period_kind": "fy1"},
"year_2": {"metric": 4.2, "metric_basis": "gaap", "period_kind": "fy2", "years": 2},
"year_3": {"metric": 5.0, "metric_basis": "gaap", "period_kind": "fy3", "years": 3}
}
}
```
Add source type, retrieval timestamp, source IDs, analyst count, and estimate range to each period.
### 8. Add arithmetic forecast bridge
For each future metric, provide numerator, denominator, operating drivers, metric basis, and source IDs.
### 9. Reconcile adjusted metrics to GAAP
For every `company_adjusted` or `analyst_normalized` period, provide the GAAP metric plus labeled adjustment rows and sources.
### 10. Use the four-status model
- `eligible`
- `review_required`
- `screened_out`
- `excluded`
Ordinary economic screen failures belong in `screened_out`; completed M&A or inactive listing belongs in `excluded`.
## Migration Procedure
1. Preserve the original v1 JSON for audit.
2. Re-run corporate-action checks and quote freshness.
3. Re-read cash-flow periods and normalize capex signs.
4. Re-query consensus with explicit GAAP/adjusted basis.
5. Build forecast and reconciliation evidence.
6. Create v2 candidate checkpoints.
7. Run strict evaluation.
There is no automatic migration script because missing evidence cannot be reconstructed safely from a v1 file alone.
references/migration-v2-to-v3.md
# Migration from Schema v2 to v3
## Breaking Changes
Schema v3 makes the research run—not just each candidate—auditable and finalizable.
### New top-level requirements
- `market_context` must contain current numeric context and source IDs.
- `global_sources` must use the strict source ledger schema.
- `screening_audit` must reference or embed a hash-verifiable row-level universe artifact.
- `run_metadata.status = complete` is valid only when every selected broad-screen symbol has a candidate record.
### Source ledger
`supports` changed from an informal string to a required array:
```json
{"supports": ["latest_earnings", "financials.cash_flow_ttm"]}
```
Source `kind` and `tier` must be compatible. Third-party transcripts are not company IR.
### Latest earnings
Replace a single mixed record with separate records:
```json
{
"latest_report_type": "annual",
"quarter": {"period_type": "quarter", "period": "Q4 FY2026"},
"full_year": {"period_type": "full_year", "period": "FY2026"}
}
```
### Formal valuation basis
The current formal period must be `ntm` or `fy1`. A TTM metric may be supplemental but cannot drive the constant-multiple ranking scenario.
### Estimate breadth
A consensus horizon with fewer than the configured minimum analysts is supplemental unless accompanied by a fully sourced independent model.
### Cash and EV/FCF
EV now uses:
```text
market cap + total debt - corporate cash - eligible marketable securities
```
Customer/settlement funds and restricted cash are excluded from shareholder-available cash.
### New sector controls
- Commercial biopharma concentration may be derived from product and total revenue.
- Near-LOE concentrated franchises receive 6x and 8x stress scenarios.
- Auto dealerships require floorplan-adjusted leverage.
- Payments require corporate/settlement cash separation.
### New scripts and flags
- `screen_universe.py`: complete row-level broad-screen audit.
- `research_contract.py`: schema-v3 validation.
- `evaluate_candidates.py --require-final`: exit 2 for a provisional or globally invalid run after writing diagnostics.
## Migration Procedure
1. Create current market context and global source files.
2. Run the broad-screen audit script.
3. Initialize a new v3 run; do not mutate a v2 state directory in place.
4. Convert earnings records and forward periods.
5. Correct source `supports` types and source classifications.
6. Reconcile cash/debt and forward forecast bases.
7. Save every selected candidate.
8. Run strict evaluation with `--require-final`.
references/migration-v3-to-v3.1.md
# Migration Guide — v3.0 to v3.1
## Why v3.1 Exists
A v3.0 execution correctly refused to publish a ranking, but it failed operationally because it required near-complete financial coverage for the full market, silently used a reduced USD 10B–20B universe, selected unresolved sector-review rows using equal placeholder scores, and produced inconsistent selected/unprocessed state. The distributed standalone package also differed from the package embedded in the repository overlay.
```text
Top-level snapshot schema: 3
Screening audit schema: 3 / contract revision 3.1
Run-state schema: 2
Skill version: 3.1.0
```
v3.1 keeps the strict primary-source, accounting, and forward-valuation controls while replacing the impossible market-wide financial-completeness requirement with a two-layer coverage model.
## Required Changes
### Screening artifacts
Old single artifact:
```text
broad-screen-results.jsonl
```
v3.1 artifacts:
```text
universe-audit-results.jsonl
broad-screen-results.jsonl
enrichment-queue.json
broad-screen-audit.json
broad-screen-shortlist.json
```
### Audit metadata
```text
audit_schema_version: 3
contract_revision: 3.1
candidate_pool_status: sufficient | no_qualifying_candidates | insufficient_data
selection_outcome: selected | no_candidates | insufficient_data
```
### Checkpoint attachment
Replace the legacy single `--artifact` invocation with:
```bash
--universe-artifact .../universe-audit-results.jsonl \
--candidate-artifact .../broad-screen-results.jsonl
```
### Plan-gated fundamentals
Do not abandon the run or demand a paid endpoint. Build a transparent stratified discovery pool with `build_discovery_pool.py`, enrich that bounded pool with available estimates/fundamentals, and rerun `screen_universe.py`.
### Status migration
| v3.0 behavior | v3.1 behavior |
|---|---|
| Missing full-universe financial fields counted as global audit failure | Missing candidate-pool economics become `needs_enrichment` |
| Review-only rows could share a placeholder score | Review rows use `broad_score=null` and cannot be selected |
| Listing-only pool could be mistaken for no candidates | `insufficient_data`; final no-candidates conclusion prohibited |
| Reduced provider scope could continue | Scope marked incomplete; final contract blocked |
| Selected count could disagree with unprocessed state | `unprocessed = selected - verified` invariant enforced |
## Package Migration
1. Replace the entire skill directory; do not mix v3.0 and v3.1 scripts or assets.
2. Start a new run directory and regenerate both audit layers.
3. Remove the old standalone package.
4. Install only the v3.1 package whose SHA-256 matches the package embedded in the v3.1 overlay.
references/migration-v3.1-to-v3.2.md
# Migration v3.1 → v3.2
## Why v3.2 Exists
A v3.1 live run produced `ranking_status=final` and `no_qualifying_candidates` while 15 candidate-pool rows remained in an enrichment queue and `candidate_pool_exhausted=false`. The direct cause was a completion shortcut based on minimum attempted/evaluable counts. The same run also failed to advance strong cyclical/high-growth rows, treated growth guidelines as mechanical cutoffs, hid broad-screen review/failure rows in Markdown, and accepted universe/macro freshness evidence that was too weak.
v3.2 removes those paths.
## Version Changes
```text
Skill version: 3.2.0
Top-level snapshot schema: 3
Screening audit schema: 3
Screening contract revision: 3.2
```
## Breaking Audit Changes
v3.1 broad-screen audit artifacts are not valid v3.2 evidence. Regenerate:
```text
universe-audit-results.jsonl
broad-screen-results.jsonl
enrichment-queue.json
broad-screen-audit.json
broad-screen-shortlist.json
```
### New scope proof
`scope.enumeration` is required:
```json
{
"verified": true,
"provider_reported_total": 1786,
"rows_fetched": 1786,
"pages_fetched": 18,
"pagination_exhausted": true,
"band_audit": [],
"bands_verified": false
}
```
### New enrichment proof
```json
{
"attempted_count": 120,
"resolved_count": 118,
"unresolved_count": 2,
"resolution_pct": 98.3333,
"all_rows_resolved": false,
"candidate_pool_exhaustion_declared": false,
"candidate_pool_exhausted": false,
"candidate_pool_covers_in_scope": false,
"queue_count": 2,
"queue_symbols": ["BBB", "CCC"]
}
```
Attempted is not resolved.
### New candidate-pool coverage proof
A market-wide no-candidates conclusion requires the economic candidate pool to represent every in-scope listing symbol:
```json
{
"in_scope_covered_count": 1786,
"in_scope_missing_count": 0,
"in_scope_missing_symbols": [],
"coverage_complete": true
}
```
`candidate_pool_exhaustion_declared=true` is an operator/agent declaration. `candidate_pool_exhausted=true` is emitted only after the script independently confirms full in-scope coverage and zero unresolved rows.
### New candidate-pool statuses
```text
sufficient
sufficient_pending_enrichment
no_qualifying_candidates
insufficient_data
```
A final no-candidates outcome requires coverage of every in-scope listing symbol, explicit pool exhaustion, zero unresolved rows, an empty queue, all final dispositions, and at least one economically assessable row.
## Candidate Decision Changes
- Growth and preferred valuation thresholds are soft guidelines.
- `near_miss_review` may advance when valuation and per-share growth justify deeper work.
- P/E 21–30 high-growth exceptions remain eligible.
- Cyclical normalization is nonblocking for broad selection; the selected deep dive must calculate mid-cycle economics before ranking.
- Bank/REIT/BDC/MLP valuation and auto-dealer floorplan-adjusted leverage remain blocking.
- `unavailable_after_enrichment` requires a specific reason and resolving source IDs.
- Enrichment queue ordering is GARP-priority based, with liquidity only a tiebreaker.
## Source and Market-Context Changes
- Dynamic market fields use `data_as_of` or publication date for freshness.
- Policy, Treasury, inflation, and GDP require official support.
- Market-implied rate-path statements require explicit source support or a sourced analyst inference.
## Report Changes
Markdown now separates:
- broad selected,
- deferred by budget,
- unresolved/review,
- unavailable after enrichment,
- broad screened out,
- broad excluded,
- deep-dive ranked/review/screened-out/excluded.
An empty deep-dive set no longer hides nonzero broad-stage dispositions.
## Upgrade Procedure
1. Replace the complete `skills/us-undervalued-growth-screener/` directory. Do not mix v3.1 and v3.2 scripts or assets.
2. Replace the committed `.skill` package with the package generated from the same v3.2 source tree.
3. Add/update the `skills-index.yaml` entry from `snippets/skills-index-entry.yaml`.
4. Discard or migrate old run-state/audit artifacts; do not resume a v3.1 broad-screen audit.
5. Re-enumerate the listing universe with pagination/band exhaustion proof.
6. Rebuild and enrich the bounded candidate pool.
7. Run the v3.2 tests and strict final smoke test.
8. Verify standalone and overlay-embedded `.skill` SHA-256 values are identical.
references/migration-v3.2-to-v3.3.md
# Migration v3.2 → v3.3
## Why v3.3 Exists
A live run intended for v3.2 was generated by a stale v3.1 runtime. Subsequent review also found that forecast bridges could be circular, commercial biopharma aliases could bypass LOE stress, peak-profit risk could avoid normalization, and data quality could remain too high without cash, TTM-period, ROIC, or EBITDA evidence.
v3.3 makes those failures mechanically detectable.
## Breaking Changes
```text
skill version: 3.3.0
snapshot schema: 3
screening audit schema: 3
contract revision: 3.3
runtime fingerprint: ug-v3.3-driver-bridge-bounded-pool-20260828
```
Old audits and snapshots are not resumable. Regenerate them.
## Required Changes
1. Replace the complete skill directory and `.skill` package from the same release.
2. Verify all helper scripts with `--version`.
3. Regenerate listing, discovery, broad-screen, checkpoint, and final artifacts.
4. Add `runtime` metadata to every audit/snapshot.
5. Add a valid generation audit for bounded pools.
6. Rebuild forecasts from operating drivers rather than a preselected numerator.
7. Add TTM-period source IDs, cash classification, ROIC/EBITDA evidence, and sector/cycle evidence.
8. Use normalized `commercial_biopharma` sector semantics or a supported alias.
9. Rerun all tests and the strict final smoke test.
## Candidate Data Changes
- Forecast drivers now require sufficient information to independently calculate the metric.
- Adjusted forecasts must tie both GAAP and adjustment portions to the reconciliation.
- Cash classification records `corporate_cash`, eligible marketable securities, restricted/customer funds, and normalization method.
- TTM cash-flow methods require source IDs for reported TTM or each construction period.
- `peak_profit_risk=true` requires normalization.
- Commercial biopharma requires product concentration and LOE evidence.
## Package Migration
Delete any installed same-name older skill before installing v3.3. Confirm the package SHA-256 and `--version` output after installation. Do not trust a filename alone.
references/migration-v3.3-to-v3.4.md
# Migration v3.3 → v3.4
## Why this release is breaking
A live v3.3 run exposed five discovery/orchestration defects:
1. the default USD 500M–20B request was silently rewritten to USD 3B–4B;
2. one session's volume was labeled as average daily dollar volume;
3. a pre-operating company without FY1 estimates appeared at roughly 4x using distant outer-year EPS;
4. five deterministic selections were later proposed to be reduced to two without rerunning the screen;
5. the workflow again asked the user for a separate “continue” turn.
v3.4 makes these conditions fail closed.
## Required input changes
### Liquidity
Discovery rows need one of the supported 20+ session average-liquidity forms, plus source IDs:
```json
{
"average_daily_dollar_volume": 12000000,
"average_daily_dollar_volume_method": "provider_average_dollar_volume",
"average_volume_period_days": 20,
"liquidity_source_ids": ["quote-source"]
}
```
Raw single-session `volume` is ignored for ADDV.
### Current forward valuation
`forward_pe` alone is no longer accepted. Use `normalize_estimates.py` or supply dated NTM/FY1 metadata, positive EPS, analyst count, source IDs, and price/EPS reconciliation.
### Scope
The default user request remains USD 500M–20B. Internal budget pressure cannot turn a single band into the request. A genuinely narrower user request needs explicit authorization evidence and matching requested/executed bounds.
### Selected-set commitment
Every Broad Screen-selected symbol must receive a verified terminal candidate record. Change the deep-dive budget only by rerunning `screen_universe.py`; omitted eligible names then become `deferred_by_budget` deterministically.
## Upgrade steps
1. Replace the full `skills/us-undervalued-growth-screener/` directory.
2. Replace `skill-packages/us-undervalued-growth-screener.skill`.
3. Remove stale installed/cached copies of the same skill name.
4. Verify the five runtime entry points with `--version`.
5. Rebuild discovery liquidity from provider average volume or a 20+ session series.
6. Normalize dated annual estimates through `normalize_estimates.py`.
7. Discard v3.3 audits, checkpoints, and snapshots.
8. Run the skill tests, index validation, package-drift checks, and repository CI.
references/migration-v3.4-to-v3.5.md
# Migration v3.4 → v3.5
## Breaking changes
1. Runtime identity is now `3.5.0 / contract 3.5 / ug-v3.5-multilane-quality-bundle-20260828`.
2. Default deep-dive budget is five and uses multi-lane selection.
3. Candidate metrics must retain `average_volume` and `liquidity_source_ids`.
4. Provider-side share-volume filters prevent a full-listing enumeration claim.
5. Forecast bridges require `construction_method=independent_driver_model` and per-driver provenance with `target_solved=false`.
6. Discrete/YTD/reported TTM cash-flow evidence requires exact support paths.
7. Recent spin-offs and transformative M&A use `corporate_transition`, not `identity.special_case`.
8. Formal eligibility has score, FCF, ROIC, leverage, dilution, low-case, and LOE quality gates. A new `conditional` status is available.
9. Final-three categories are nullable and use category-specific thresholds.
10. Final publication requires `prepublish_audit.py` and `bundle_run_artifacts.py`.
## Candidate migration
Add driver provenance under every year-2/year-3 bridge:
```json
{
"construction_method": "independent_driver_model",
"driver_provenance": {
"revenue": {
"origin": "market_consensus",
"source_ids": ["consensus-source"],
"target_solved": false
}
}
}
```
Add exact cash-flow supports to source rows:
```text
financials.cash_flow_periods.<period-end>
financials.ttm_reconstruction.<component>
financials.cash_flow_ttm.reported_ttm
```
## Run migration
Discard v3.4 audits/checkpoints; their fingerprint is intentionally incompatible. Re-run runtime preflight, Broad Screen, candidate verification, strict evaluation, prepublication audit, and bundle generation.
references/migration-v3.5-to-v3.6.md
# Migration v3.5 → v3.6
v3.6 changes the preferred execution architecture while retaining schema 3 and the contract-3.5 underwriting/evaluation semantics.
## Runtime identity
```text
skill_version = 3.6.0
schema_version = 3
contract_revision = 3.5
runtime_fingerprint = ug-v3.6-claude-code-direct-fmp-20260829
```
Do not reuse v3.5 audits or checkpoints. The evidence schema is compatible, but runtime identity is intentionally different so a cached package cannot be mistaken for the direct-FMP release.
## New preferred path
```bash
export FMP_API_KEY="..."
python3 skills/us-undervalued-growth-screener/scripts/run_pipeline.py \
--config skills/us-undervalued-growth-screener/assets/claude-code-config.example.json \
--output-dir reports/us-undervalued-growth-screener
```
The runner keeps bulk FMP responses on disk, uses a persistent SQLite cache, normalizes FY1 estimates, validates 20-day ADDV, builds the four discovery lanes, and prints only a compact run summary. Claude Code then reads selected candidate packets and performs the existing SEC/IR underwriting and strict final sequence.
## Reused central infrastructure
The skill's `fmp_client.py` is generated from the repository-level `scripts/fmp_client/` source of truth. Regenerate and verify it with:
```bash
python3 scripts/generate_fmp_client.py
python3 scripts/generate_fmp_client.py --check
```
## New files
- `scripts/fmp_client/specials/garp.py.tmpl`
- generated `skills/us-undervalued-growth-screener/scripts/fmp_client.py`
- `skills/us-undervalued-growth-screener/scripts/run_pipeline.py`
- `skills/us-undervalued-growth-screener/assets/claude-code-config.example.json`
- `skills/us-undervalued-growth-screener/references/claude-code-execution.md`
- `commands/us-undervalued-growth-screen.md`
## Operational changes
- Direct FMP HTTP calls replace bulk FMP MCP calls in Claude Code.
- Adaptive market-cap-band splitting prevents silent company-screener truncation.
- Bulk analyst-estimate and EOD endpoints are attempted first; bounded per-symbol fallbacks remain available.
- Exact liquidity work is allocated by GARP opportunity lane.
- Raw provider payloads are never printed to stdout and should not be loaded into model context.
- The Claude Code default deep-dive commitment is three names.
- `evaluate_candidates.py`, `prepublish_audit.py`, and `bundle_run_artifacts.py` remain the publication gates.
references/migration-v3.6-to-v3.6.1.md
# Migration v3.6 → v3.6.1
v3.6.1 hardens the bounded per-symbol fallback that runs when FMP bulk endpoints are plan-gated (HTTP 402). Schema 3 and contract 3.5 are unchanged; the discovery stage gains an honest scope vocabulary, a stratified seed with a documented selection basis, a pre-selection quality probe, and a persistent endpoint-capability cache.
## Runtime identity
```text
skill_version = 3.6.1
schema_version = 3
contract_revision = 3.5
runtime_fingerprint = ug-v3.6.1-claude-code-direct-fmp-20260830
```
Runtime identity is intentionally different from v3.6.0. Do not mix v3.6.0 audits, packets, or checkpoints with v3.6.1 artifacts; rerun discovery.
## Why
A live run on 2026-08-30 (2,371 listings, all five bulk endpoints 402) showed that the fallback seeded 80 of 2,371 names with no economic data at seed time, that the within-cell ranking saturated at USD 100M/day and fell back to ticker order for large caps, that `provider_exhausted=true` was passed unconditionally, that `scope_complete=true` was read as economic coverage, and that a name with EV/FCF 126x reached the deep-dive slots because no FCF evidence existed before selection.
## Behavioural changes
### Seed selection (`diversified_seed`)
- Cells are sector × market-cap bucket. Quota per cell ∝ √(cell size), reconciled to the seed limit with Hamilton apportionment; every non-empty cell gets at least one seat. The result is independent of cell iteration order.
- Within a cell: `pre_enrichment_score` desc → raw single-day dollar volume desc → market cap desc → fewer missing price/volume fields → `sha256(analysis_date:symbol)`. The raw ticker string is never a tie-break. The log10 dollar-volume term is no longer capped.
- `audit/seed-audit.json` (also embedded in `provider-prefilter-audit.json` and `run-summary.json`) records `seed_selection_basis` (`stratified_liquidity_proxy` when economic fields are absent for the majority of rows), `economic_metrics_available_for_seed`, `cell_count`, `quota_method`, tie-break counters, and the configured/effective seed limits.
### Dynamic seed limit
New config keys with defaults: `pre_enrichment_limit: 180` (was 80), `seed_limit_cap: 200`, `quality_probe_limit: 35`, `candidate_packet_reserve_calls: 30`, `retry_reserve_calls: 25`.
```text
reserved = quality_probe_limit + exact_liquidity_limit
+ candidate_packet_reserve_calls + retry_reserve_calls
effective = min(pre_enrichment_limit, seed_limit_cap,
max_api_calls - api_calls_made - reserved)
```
An effective limit below 20 fails the run with `estimate seed budget insufficient` instead of silently producing a thin pool. The probe reserve is counted twice per target (key metrics + annual income statement).
### Quality probe before pool selection
After estimate normalization, the union of lane rows is ranked by best lane score and the top `quality_probe_limit` symbols receive one `key-metrics-ttm` call each. Rows gain `roic_pct`, `fcf_yield_pct`, `ev_to_fcf`, `net_debt_to_ebitda`, `sbc_revenue_pct`, and `sbc_adjusted_fcf_yield_pct` (computed on the market-cap basis: FCF yield − SBC/revenue × revenue/market cap). `audit/quality-probe-audit.json` records attempts, resolutions, and calls used.
Lane scores now include an FCF-yield term (weight 1.0 core_garp / quality_near_miss, 0.5 high_growth_exception) and a leverage penalty above 2.5x net debt / EBITDA. A probe-resolved row with SBC-adjusted (or standard) FCF yield below 1% is excluded from every lane except `high_growth_exception`, where it stays with `provider_prefilter_flags: ["weak_fcf_support"]` and a −10 score. Exclusions are listed under `fcf_prefilter_excluded_symbols` in the discovery audit.
### Honest scope fields
Route selection and completeness are separate thresholds: the bulk route is used from `bulk_estimate_minimum_coverage_pct` (20%), but `economic_screen_scope_complete` / `economic_candidate_universe_exhausted` are true only when the bulk estimates covered every listing-universe symbol (exact count equality, `covered_symbol_count >= universe_symbol_count`; deliberately not a configurable ratio, so no setting can declare a partially covered run complete). A 25%- or even 99%-covered bulk run is a bounded economic screen, exactly like the per-symbol fallback.
`run-summary.json` and `NEXT_ACTION.json` add `listing_enumeration_complete`, `economic_screen_scope_complete`, `listing_universe_count`, `estimate_seed_count`, `estimate_seed_coverage_pct`, `valid_estimate_count`, `valid_estimate_coverage_pct`. `scope_complete` is retained for readers of v3.6.0 output and now carries `scope_complete_deprecated_note`. The discovery audit adds `listing_provider_exhausted`, `estimate_seed_exhausted`, `economic_candidate_universe_exhausted`, and `provider_exhausted_scope` (`estimate_seed` on the fallback path). The contract-validated `screening_audit.scope` block is unchanged.
### Growth basis
**Actuals are verified or absent, and growth is compared on one basis.** `latest_actual_eps` is populated only from (a) a provider row explicitly marked as actual whose period has ended AND whose publication timestamp is at or before `analysis_as_of` (rows without a provable publication time are rejected — a current snapshot replayed against a historical as-of must not leak later-published actuals), or (b) the annual income statement fetched during the quality probe whose `acceptedDate` (with time of day; the date-only `filingDate` is never accepted) is at or before `analysis_as_of` (`latest_actual_verified: true`, `latest_actual_basis: gaap_diluted`, `latest_actual_source_ids`). An unmarked prior-year estimate row is never reported as an actual; it is exposed as `fy0_consensus_eps` and, because it shares the provider's (usually adjusted) basis with FY1/FY3, it is the comparator for `current_year_growth_pct`, `fy1_eps_below_fy0_consensus`, and `growth_pattern` (`growth_pattern_basis: consensus_same_basis`; `unknown` without a prior-year row). The GAAP actual is reported alongside (`current_year_growth_pct_vs_gaap_actual`, `fy1_eps_below_latest_actual`) and, when it differs from the FY0 consensus by more than 15%, `estimate_basis_likely_adjusted: true` warns that GAAP-vs-consensus comparisons are mixed-basis. Found on DOCS: FY1 non-GAAP consensus $1.35 vs GAAP actual $0.98 looked like a decline; same-basis FY26 non-GAAP $1.52 → $1.35 is the real (and still negative) comparison.
`normalize_estimates.py` adds `latest_actual_eps`, `latest_actual_period_end`, `fy1_eps_below_latest_actual`, `current_year_growth_pct`, `eps_growth_fy1_to_fy3_pct` (alias of `eps_growth_pct`), `eps_growth_actual_to_fy3_pct`, `growth_pattern` (`steady | accelerating | trough_recovery | declining | unknown`), and `growth_basis_source_ids`. A `trough_recovery` row is removed from `core_garp` and admitted to `quality_near_miss` with the `earnings_recovery` flag.
### Deep-dive selection with a small budget
`screen_universe._selection_lane` now also routes `growth_pattern == trough_recovery` rows to `quality_near_miss` (the pool-stage rule alone did not reach the final lane). When `max_deep_dive_candidates` is smaller than the lane plan total (default 2/1/1/1 = 5), selection walks the priority order and treats each lane quota as a cap instead of filling lanes in plan order; with a 3-name budget the best cyclical can now win a slot rather than every slot going to the first two lanes. Budgets at or above the plan total keep the lane-first fill.
### Cyclicality and foreign private issuers
Gold, silver, precious/base metals, copper, uranium, coal, metals & mining, and mineral names classify as cyclicality 4; aluminum and semiconductor equipment as 3 (the bare `semiconductor` needle was removed so equipment names do not inherit 4). Rows whose ISIN prefix (or, failing that, listing country) is not `US` carry `foreign_private_issuer_review`, and their packets add `form_20f_6k_verification` to `required_next_checks`. Neither flag excludes a name.
### Pool floor and sector profiles (round-4 review)
- `build_provider_prefilter_pool` waives the minimum-pool row floor only when `provider_exhausted_scope` is a full exhaustion (`economic_candidate_universe` or `full_input`). The per-symbol fallback passes `estimate_seed`, which never waives the floor; `pool_floor_waived` is recorded in the audit. Round-5 hardening removed the legacy missing-scope waiver: an unstated scope now fails closed.
- `normalize_listing` infers `sector_profile_type` (reit / insurance / bank / asset_manager / bdc / mlp / auto_dealership) from sector+industry text, so `screen_universe`'s existing sector gates actually fire: such names without sector-specific valuation evidence go to `sector_specific_valuation_required`, and the general-company `excessive_leverage` hard gate is skipped for them (a mortgage REIT at 13x net debt/EBITDA is normal, not a failure). On the direct-FMP path such rows are then declared enrichment-exhausted (the provider cannot supply P/FFO-AFFO, P/TBV, or adjusted leverage) and resolve as `unavailable_after_enrichment` -- audited and excluded from selection, available for manual sector underwriting in a scoped follow-up.
- The Markdown report renames "Universe scope" to "Listing enumeration" and adds an "Economic estimate coverage" line (mode, covered/universe counts, conclusion scope), so the bounded economic scope is visible in prose, not only in JSON.
### Unit reconciliation, taxonomy map, and the ET filing clock (round-5 review)
- Foreign issuers (non-US ISIN prefix or listing country) are now blocked from selection unless `unit_reconciliation_verified: true` with currency/ADS-unit evidence: a USD ADS price against local-currency statements produced QFIN at forward P/E 0.45x and 94% FCF yield — a unit mismatch, not deep value. On the direct-FMP path such rows are declared enrichment-exhausted and resolve as `unavailable_after_enrichment`. Independent circuit breakers stop implausible ratios on any issuer (`minimum_plausible_forward_pe` 2.0, `maximum_plausible_fcf_yield_pct` 50, reported EPS > 2x price) as `unit_mismatch_suspected`.
- Sector-profile inference now uses an explicit map over real FMP taxonomy labels instead of substring needles: `Auto - Dealerships` maps to `auto_dealership` (the old needle never matched it), `Investment - Banking & Investment Services` maps to `capital_markets` (labelled, valued on ordinary multiples — never the deposit-bank gate), `Banks - *` prefixes map to `bank`, and BDCs are caught by industry (`Business Development ...`) or company-name needles, with `sector_profile_overrides` in config to pin names the listing frame cannot classify (e.g. `{"ARCC": "bdc"}`).
- FMP `acceptedDate` / naive publish stamps are US/Eastern (the SEC acceptance clock), not UTC: a 17:23 ET acceptance read as UTC leaked the filing 4-5 hours early. Both `_verified_annual_actual` and the consensus publish check convert from `America/New_York`; date-only publish stamps count only after the whole publication day (ET) has passed.
- The scope-less pool-floor waiver was removed (fail closed; see the round-4 section above).
### Fail-closed unit context, actual attempt counts, capital_markets exemption (round-8 review)
- The unit gate is inverted from "block what is proven foreign" to "exempt only what is proven domestic": the shared `requires_unit_reconciliation()` (in `screen_universe`, used by both discovery's exhaustion marker and the screen's blocking gate) demands reconciliation evidence whenever the country is missing or non-US, the currency is non-USD, the ISIN is non-US, or the row is an ADR/ADS. `normalize_listing` now preserves `isin` and `is_adr` so those signals survive into the pipeline — a provider dropping the country field can no longer route a CNY-denominated row through as domestic.
- Discovery records the ACTUAL attempted count (`economic_attempt_count = len(estimate frame)`) in the generation audit; the evaluator copies it and fails closed to `diagnostic` when the field is missing, zero, or exceeds the universe (a 180 seed LIMIT over a 50-name universe must never display 360% coverage).
- `capital_markets` (advisory/investment banking) is excluded from the sector-enrichment exhaustion marker: only profiles the screen actually blocks (`SECTOR_PROFILES` + `auto_dealership`) are declared provider-unservable.
### Honest ranking scope (round-6 review)
- Listing 2,371 names and economically comparing 2,371 names are different claims: a run that attempted estimates for 180 seeds (98 evaluable, 4.1% of the universe) is a **scoped pilot**, not a market ranking, and the remaining ~96% are *unexamined*, not rejected. Every run now carries a tri-state `ranking_scope` — `final_marketwide` (estimate acquisition attempted for every listed symbol, exact counts), `final_scoped` (a bounded, fully processed subset; conclusions bind only to it), or `diagnostic` (unresolved queue) — plus per-stage coverage counts and percentages (`economic_attempt_*`, `economically_evaluable_*`, `quality_probe_*`, `deep_dive_*`) in the run summary and report JSON.
- The report title, scope banner, and the no-qualifying-candidates conclusion state the subset explicitly ("Scoped Pilot (180 of 2,371 listed names economically attempted)"; "the remaining names were never economically compared").
- `final_marketwide` is currently unreachable on the direct-FMP Starter path by design: reaching it requires the v3.7 sharded full-universe estimate collection (persistent snapshot store, ~8 deterministic shards within the per-run call budget) tracked as a follow-up issue, together with a discovery-recall gold-set harness (Recall@K against known past candidates).
### Endpoint capability cache
The generated client remembers a 402/403 bulk response in the SQLite cache (`capability:<url>`, 30-day TTL) and pre-disables that endpoint on later runs without spending a call. `respect_capability_cache=False` re-probes unconditionally. Diagnostics add `capability_cache_hits` and `remaining_calls`.
## Operator checklist
1. Build `market-context.json` and `global-sources.json` **before** running `run_pipeline.py`; every source `retrieved_at` must precede `analysis_as_of`.
2. Run discovery, then `manage_run_state.py init` / `set-screening-audit` / `set-funnel --preflight-passed-count N`.
3. `set-screening-audit` now copies the enrichment queue and provider-prefilter pool into `run/audit/` and rewrites their paths to the same run-relative base as the universe/candidate artifacts, and `prepublish_audit.py` also accepts audit-relative bare names, so `--artifact-root <run>` resolves every artifact from one root.
4. Keep shared `fmp-*` source entries byte-identical across candidate ledgers and the global ledger.
references/original-prompt-mapping.md
# Source Prompt Coverage Map
The original Japanese prompt remains the methodological authority. The skill reorganizes it into progressive research instructions, a source-linked schema, deterministic calculations, audit statuses, and tests.
| Original prompt area | Skill implementation |
|---|---|
| Undervalued growth rather than low P/E | `SKILL.md` overview; methodology objective; minimum constant-multiple upside gate |
| NYSE/Nasdaq/NYSE American and USD 500M–20B focus | scope/configuration and screening funnel |
| Exclusions and liquidity | corporate-action preflight, flags, liquidity gates, hard-exclusion log |
| SEC/IR source priority | evidence map, source ledger, research checklist, data-quality weights |
| Consensus date/period/type/analyst count | separate current/year-2/year-3 valuation records |
| Price timestamp/session and post-event consistency | common price basis, latest-event timestamp, stale-data gate |
| Revenue, EPS, FCF-per-share growth | historical CAGRs and forward per-share periods |
| ROIC, margins, leverage | normalized financials and score components |
| Standard FCF = OCF − capex | non-negative capex convention and three TTM reconstruction methods |
| Company-adjusted FCF disclosure | separate supplemental field and definition; never substitutes for standard FCF |
| GAAP vs non-GAAP | per-period metric basis and arithmetic GAAP reconciliation |
| SBC, dilution, and actual buybacks | SBC/FCF metrics, diluted-share history, explicit penalties |
| Three-to-five genuine peers | sourced peer rows, basis/period validation, peer-median scenario |
| Strict cyclical review | sourced normalization object and fail-closed gate |
| Latest earnings and growth-state classification | latest-earnings object and full report section |
| Sector-specific KPI rules | sector profile; commercial-biopharma LOE and payments cash/take-rate controls |
| Earnings/accounting quality | TTM reconstruction, working-capital review, source checklist |
| Constant multiple, 20% contraction, peer median | deterministic evaluator |
| EPS/FCF forecast construction | numerator/denominator bridge, drivers, arithmetic tolerance |
| Why discounted and market may be right | qualitative contract and report section |
| Catalysts, risk, and invalidation | detailed candidate and final-three sections |
| 100-point ranking | seven preserved score components plus deterministic penalties |
| Complete output contract | full Markdown renderer and structured JSON |
| Do not guess or force recommendations | null contract, evidence gates, review status, no-qualifier result |
| Preserve incomplete work honestly | checkpoint manager and provisional partial-run status |
## Version 2 Enhancements
1. Corporate-action preflight runs before detailed financial research, catching completed acquisitions and inactive symbols early.
2. Data-quality points require resolving evidence and arithmetic; self-attested completeness flags are ignored.
3. Capex sign convention is explicit and negative normalized outflows fail closed.
4. TTM OCF/capex supports four discrete quarters or FY+YTD−prior YTD without double-counting cumulative statements.
5. Standard FCF and company-adjusted FCF are separate.
6. Corporate cash is separated from customer/settlement and restricted funds.
7. Current, year-2, and year-3 accounting bases are stored separately; mixed bases block scenarios.
8. Adjusted/normalized periods require GAAP reconciliation.
9. `screened_out` is distinct from hard `excluded`.
10. Long runs checkpoint every candidate and can resume.
11. Partial coverage produces a provisional report with no final-three claims.
12. Commercial-biopharma LOE and payments take-rate/cash controls are explicit.
13. Full Markdown output now follows all major sections of the original prompt.
14. Synthetic tests cover evidence, FCF, period basis, corporate actions, sector controls, partial runs, and packaging behavior.
## Version 3 Review-Driven Controls
| Original requirement | v3 implementation |
|---|---|
| Current market valuation, rates, economy, sector cycle | Mandatory sourced `market_context`; placeholders/future dates rejected |
| Latest price and latest earnings | Quote/event timestamp gate; latest quarter and full year separated |
| Forward P/E constant-multiple analysis | NTM/FY1 current metric required; TTM supplemental only |
| Do not uncritically use consensus | Analyst-count/range and rankable-horizon controls; forecast bridge required |
| Compare 3–5 peers | Same-basis sourced peer gate |
| Do not mechanically reject 21–30x high growers | Broad-screen high-growth exception |
| Industry-specific leverage | Auto-dealer floorplan-adjusted leverage gate |
| Product concentration and structural risk | Derived/sourced commercial-biopharma concentration and LOE stress |
| Important numbers need source/period/retrieval date | Strict source ledger with typed `supports` arrays |
| Scope and candidate counts | Full listing audit plus bounded candidate-pool artifact, explicit scope, enrichment counts, and SHA-256 |
| Do not recommend when conditions are unmet | `--require-final`, no final-three on provisional runs, explicit no-candidate result |
## Version 3.1 Execution Controls
| Original requirement | v3.5 execution implementation |
|---|---|
| Screen the major US exchanges, with emphasis on USD 500M–20B | Full listing-universe audit records requested and retrieved bounds; reduced scope is disclosed and cannot silently complete |
| Do not guess missing numbers | Missing discovery data is `needs_enrichment`; no placeholder zeros or scores |
| Use multiple metrics but respect industry characteristics | General-company candidate screen plus explicit sector-review routing for banks, REITs, insurers, BDCs, MLPs, cyclicals, and auto dealers |
| Do not force ten recommendations | Deep-dive set is bounded; evidence-based `no_candidates` is a valid final result |
| Important data must be current and sourced | Full listing source, candidate-estimate source, SEC/IR sources, artifact hashes, and timestamps are separate |
| Conditions not met means no recommendation | `insufficient_data` triggers more discovery; it is never relabeled as “no candidates” |
| Complete the task from the screening request | Plan-gated bulk statements automatically route to provider-prefilter or stratified estimate enrichment before selected-name primary-source underwriting |
## Version 3.3 Evidence and Runtime Controls
| Original requirement | v3.5 implementation |
|---|---|
| Do not mix old and current data/processes | Runtime version, contract, schema, and fingerprint are embedded and hard-validated. |
| Do not uncritically use consensus | EPS/FCF is independently rebuilt from operating drivers; circular numerator checks fail. |
| Distinguish GAAP and adjusted forecasts | Driver-derived GAAP and after-tax adjustments separately tie to the reconciliation. |
| Standard FCF and source period integrity | TTM construction method and every component-period source are verified. |
| Do not overstate cash or EV/FCF | Corporate cash normalization and customer/restricted-fund separation feed evaluator-calculated EV. |
| Treat product concentration/LOE structurally | Biopharma aliases normalize to commercial-biopharma and require concentration/LOE/6x/8x stress. |
| Do not use peak cyclical profits | `peak_profit_risk=true` forces sourced normalization even with a low numeric cycle score. |
| Important numbers need sources | ROIC, EBITDA, cash, TTM periods, sector evidence, and forecasts affect quality points and caps. |
| Do not overclaim scope | Audited bounded pools produce explicitly scoped conclusions; market-wide no-candidates requires full economic coverage. |
## Version 3.5 Discovery and Autonomy Controls
| Original requirement | v3.5 implementation |
|---|---|
| Screen the requested US small/mid-cap range | Immutable `user_requested_scope` plus separately audited `executed_scope`; internal single-band narrowing is rejected |
| Exclude extreme illiquidity using trading liquidity | 20+ session ADDV provenance contract; raw one-session volume is ignored |
| Use current Forward P/E and do not mix periods | Dated NTM/FY1 normalization, price/EPS reconciliation, analyst breadth, range-dispersion and continuity checks |
| Do not force the user to supervise internal steps | Exit code 2 and `next-action` are same-execution continuation; selected-set commitment requires every selected name to finish |
references/output-template.md
# Required Output Template — v3.5
## 1. Files
Write both:
```text
us_undervalued_growth_<timestamp>.json
us_undervalued_growth_<timestamp>.md
```
Use the evaluator-generated files as the canonical calculation artifacts. Preserve timestamps and source IDs.
## 2. Run Status
At the beginning, state:
- analysis timestamp,
- quote timestamp and session,
- strict-mode flag,
- `ranking_status` (`final` or `provisional`),
- deep-dive input/ranked/review/screened-out/excluded counts,
- broad-screen selected/deferred/unresolved/unavailable/screened-out/excluded counts.
When `ranking_status = provisional`, `selection_outcome = insufficient_data`, or `selection_outcome = selected_pending_enrichment`:
- place a prominent incomplete-coverage warning,
- list unprocessed candidates,
- do not present final-three selections,
- do not describe the displayed order as the final market ranking.
## 3. Screening Funnel and Market Context
Include:
- requested and actual retrieval scope,
- full listing-universe count and listing-data coverage,
- bounded candidate-pool count and generation mode,
- discovery-evaluable count, enrichment attempted/resolved/unresolved counts, resolution percentage, queue, and exhaustion state,
- `candidate_pool_status` and `selection_outcome`,
- selected/deferred/enrichment/sector-review/screened-out/excluded counts,
- corporate-action preflight pass count,
- detailed-underwriting count,
- market valuation, rates, growth, inflation, and sector-cycle assumptions,
- main data sources and as-of dates.
## 4. A. Ranking Table
Recommended columns:
| Rank | Company | Ticker | Price | Market cap | Valuation basis | Current multiple | Standard FCF yield | ROIC/sector return | Diluted-share CAGR | Cyclicality | 3Y constant-multiple upside | Final score | Data quality |
|---:|---|---|---:|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|
Use `確認できず` / `not verified` for missing values.
## 5. B. Scenario Table
Include current metric/multiple and year-2/year-3 results for:
- constant multiple,
- configured multiple contraction,
- peer median only when basis/period/source validation passes.
Do not render a scenario when accounting bases are mixed.
## 6. C. Candidate Detail
For each eligible candidate, use the following order.
### 1. Basic Information
- company, ticker, exchange,
- sector/industry/special case,
- price, market cap, liquidity,
- price timestamp and source,
- business overview.
### 2. Investment Thesis
Three to five sentences covering per-share compounding, valuation, quality, and why the opportunity exists.
### 3. Valuation
- basis, accounting basis, period kind,
- current metric and recomputed multiple,
- peer median and validity,
- EV/FCF where meaningful,
- standard and SBC-adjusted FCF yield,
- supplied-multiple reconciliation warning if any.
### 4. Growth History and Forecast
- revenue CAGR,
- GAAP EPS CAGR,
- standard FCF-per-share CAGR,
- diluted-share CAGR,
- current/year-2/year-3 metrics,
- forecast-bridge verdict and source classification,
- analyst count and dispersion.
### 5. Latest Earnings
- fiscal period and publication time,
- revenue and growth,
- GAAP operating margin,
- GAAP/adjusted EPS,
- OCF and standard FCF,
- guidance, key KPIs, segments, one-time items,
- growth-state classification.
### 6. Growth Drivers
List the two- to three-year drivers in priority order.
### 7. Peer Comparison
Show three to five genuine peers with multiple, basis, period, and selection reason. Explain why the comparison is valid.
### 8. Why It Is Discounted
- market concern,
- temporary versus structural classification,
- why the market may be correct,
- conditions for discount narrowing,
- whether returns remain attractive without narrowing.
### 9. Constant-Multiple Scenario
| Item | Current | Year 2 | Year 3 |
|---|---:|---:|---:|
| Per-share metric | | | |
| Multiple | | | |
| Implied price | | | |
| Upside | — | | |
| Annualized return | — | | |
### 10. Multiple-Contraction Scenario
Show year-2/year-3 price and return using the configured contraction, normally 20%.
### 11. Catalysts
List 6–24 month catalysts with timing and probability where available.
### 12. Maximum Risk
State the single most important concrete risk.
### 13. Invalidation Conditions
Use measurable thresholds or events.
### 14. Cyclical Assessment
- score 1–5,
- cycle position,
- peak-profit risk,
- normalized metric/multiple and method when required.
### 15. Score
Show all seven raw components, raw total, explicit penalties, and final score.
### 16. Cash-Flow and Evidence Audit
- TTM method,
- OCF, capex cash outflow, standard FCF,
- company-adjusted FCF and definition,
- corporate versus settlement cash where relevant,
- data-quality score and item breakdown,
- warnings, blockers, and unresolved data.
## 7. D. Review-Required and Deferred Work
### Deep-dive review blockers
| Ticker | Company | Final score | Blockers | Evidence needed |
|---|---|---:|---|---|
Keep these out of the ranked list until blockers are resolved.
### Broad-screen dispositions
Always surface nonzero broad-screen rows that still require or defer work:
| Ticker | Company | Status | Fwd P/E | Revenue growth | Per-share growth | Priority | Reasons / requirements |
|---|---|---|---:|---:|---:|---:|---|
Include `selected`, `deferred_by_budget`, `needs_enrichment`, `sector_review_required`/preselection requirements, `near_miss_review`, and `unavailable_after_enrichment` as applicable. A selected cyclical must show its mid-cycle requirement.
## 8. E. Screened-Out Log
Show deep-dive and broad-screen failures separately.
| Stage | Ticker | Company | Reason |
|---|---|---|---|
Do not hide broad-screen failures merely because no deep-dive candidate record exists. Do not mix these with corporate-action or listing exclusions.
## 9. F. Hard Exclusion Log
Include inactive/delisted symbols, completed/pending M&A, OTC, shell/SPAC, extreme illiquidity, invalid sector basis, and other hard flags.
| Ticker | Company | Reason |
|---|---|---|
## 10. G. Final Three
Render only for a final run with at least one eligible candidate.
Categories:
- highest conviction,
- most undervalued,
- largest upside.
For each selection include all nine required points:
1. reason it is most attractive,
2. upside from EPS/FCF-per-share growth alone,
3. what the market may be missing,
4. best catalyst,
5. largest risk,
6. best reason not to buy now,
7. bear case,
8. invalidation conditions,
9. next-earnings KPIs.
The same ticker may occupy more than one category.
## 11. H. Source Ledger
Show source ID, tier, kind, publication/retrieval timestamps, and supported fields. Do not cite a source for a field it does not support.
## 12. I. Unresolved Data and Global Warnings
List:
- unprocessed candidates,
- missing funnel data,
- stale estimates or quotes,
- source conflicts,
- weak analyst coverage,
- basis mismatches,
- unresolved sector/cycle evidence,
- limitations of the run.
If none, state none explicitly.
## 13. JSON Output Contract
The evaluator output contains:
```json
{
"schema_version": 3,
"runtime": {"skill_version": "3.6.1", "contract_revision": "3.5", "runtime_fingerprint": "..."},
"analysis_as_of": "...",
"run_metadata": {},
"ranking_status": "final",
"strict_mode": true,
"config": {},
"contract": {"valid": true},
"market_context": {},
"global_sources": [],
"price_basis": {},
"screening_funnel": {},
"screening_audit": {"valid": true},
"broad_screen": {
"counts": {
"selected": 0,
"deferred_by_budget": 0,
"review_required": 0,
"screened_out": 0,
"excluded": 0,
"unavailable_after_enrichment": 0
},
"selected": [],
"deferred_by_budget": [],
"review_required": [],
"screened_out": [],
"excluded": [],
"unavailable_after_enrichment": []
},
"counts": {
"input_candidates": 0,
"ranked": 0,
"review_required": 0,
"screened_out": 0,
"excluded": 0
},
"ranked_candidates": [],
"review_required": [],
"screened_out": [],
"excluded": [],
"final_three": {},
"global_warnings": []
}
```
Each candidate result preserves:
- status and reasons,
- identity and source-linked corporate-action result,
- normalized financial metrics and TTM method,
- valuation periods and scenarios,
- forecast/reconciliation details,
- sector profile,
- score components and penalties,
- data-quality item details,
- warnings and unresolved fields,
- qualitative thesis fields,
- source ledger.
## 14. Schema-v3 Mandatory Presentation Controls
The final Markdown must additionally show:
- forward metric period kind and accounting basis,
- analyst count/range for each forecast horizon,
- whether each horizon is rankable,
- market-context values and source-backed timestamp,
- listing-enumeration evidence, broad-screen artifact row count, and SHA-256,
- broad-screen attempted/resolved/unresolved and pool-exhaustion state,
- all nonzero broad-screen disposition groups,
- runtime/version and bounded/full conclusion scope,
- separate latest-quarter and latest-full-year blocks,
- evaluator-calculated EV/FCF and cash definition,
- cash/debt reconciliation status,
- sector stress scenarios where applicable,
- source `supports` values,
- `final` versus `provisional` status.
Never render example/placeholder text. Always show 2Y base, 2Y 20%-contraction, 3Y base, and 3Y 20%-contraction as four separately labeled values. A provisional report must not populate the final-three section. `No qualifying candidates` may appear only when the exhausted pool has zero unresolved rows, an empty queue, all final dispositions, and at least one economically assessable row.
## v3.5 Conditional Candidates and Publication Bundle
Insert a distinct **Conditional Candidates** section between ranked and review-required names. State each failed quality gate and the KPI that would promote the name to `eligible`.
`final_three` labels are optional and category-gated. Use `確認できず` when no candidate meets the relevant conviction, cash-flow valuation, or low-case upside threshold.
Before publication, provide:
```text
prepublish-audit.json
us-undervalued-growth-screen-<date>.zip
BUNDLE_MANIFEST.json inside the ZIP
```
The ZIP must include every file referenced by the final snapshot: universe audit, candidate-pool audit/results, generation audit, enrichment queue, candidate records, source ledger, market context, final JSON, and final Markdown.
references/research-checklist.md
# Primary-Source Research Checklist — Schema 3 / Contract 3.5 / Audit v3.5
Use this checklist for every broad-screen survivor. Mark each item `verified`, `not applicable`, or `not verified`. A blank field or a Boolean assertion without a resolving source ID is not a pass.
## v3.5 Runtime and Independent-Bridge Checks
- [ ] Every helper `--version` reports v3.6.1 / contract 3.5 / the expected fingerprint.
- [ ] Audit, checkpoint, and snapshot runtime metadata match the installed skill.
- [ ] The user-requested scope remains separate from the executed retrieval scope; no internal single-band rewrite occurred.
- [ ] Discovery liquidity uses a documented 20+ session average and source IDs, never one-session volume.
- [ ] Current Forward P/E is dated NTM/FY1 and reconciles to positive EPS and price; no distant outer-year substitution.
- [ ] Every Broad Screen-selected symbol is present in the committed set and reaches a verified terminal candidate record.
- [ ] A bounded pool has a valid generation audit and a clearly labeled conclusion scope.
- [ ] Forecast EPS/FCF is independently derived from operating drivers; numerator is not reverse-engineered from target per-share metric.
- [ ] Adjusted forecast GAAP and adjustment portions separately tie to the reconciliation.
- [ ] Each TTM cash-flow component period has resolving source IDs.
- [ ] Corporate cash, eligible securities, customer/settlement funds, and restricted cash are correctly classified.
- [ ] ROIC and EBITDA have source-linked inputs or transparent calculations.
- [ ] Commercial-biopharma aliases trigger concentration, LOE, and 6x/8x stress checks.
- [ ] `peak_profit_risk=true` has sourced normalization.
- [ ] Broad-screen statuses were not overridden by an informal revenue-growth or cycle gate.
## 0. Checkpoint and Run Integrity
- [ ] Initialize a run with `manage_run_state.py`.
- [ ] Record `analysis_as_of`, common quote basis, repository commit, and configuration.
- [ ] Record requested and actual retrieval scope; do not silently narrow the market-cap range.
- [ ] Preserve `user_requested_scope` separately from `executed_scope`; tool/context budget is not scope authorization.
- [ ] If execution is narrower, confirm explicit user authorization and label the limited conclusion scope.
- [ ] Reject single-session volume as ADDV; require provider-average or 20+ trading-day average-volume evidence with source IDs.
- [ ] Prove universe enumeration with provider total plus exhausted pagination or exhausted market-cap-band audits.
- [ ] Attach both listing-universe and candidate-pool audit artifacts with verified SHA-256.
- [ ] Record listing-data coverage separately from candidate-pool economic coverage.
- [ ] Record candidate-pool status (`sufficient`, `sufficient_pending_enrichment`, `no_qualifying_candidates`, or `insufficient_data`).
- [ ] Preserve the enrichment queue and documented attempts when bulk fundamentals are unavailable.
- [ ] Record attempted, resolved, and unresolved counts separately.
- [ ] Confirm every in-scope listing symbol is represented in the candidate pool before a market-wide no-candidates claim.
- [ ] Confirm `candidate_pool_exhaustion_declared=true` only after every candidate row is resolved or evidenced as unavailable.
- [ ] Confirm independently verified `candidate_pool_exhausted=true`; do not rely on the declaration alone.
- [ ] Confirm only genuinely scored, selection-eligible rows enter the deep-dive set.
- [ ] Permit strong cyclicals to enter deep dive with a mid-cycle-normalization requirement.
- [ ] Treat growth thresholds as guidelines; document isolated misses rather than using them as automatic rejects.
- [ ] Record every screening-funnel stage.
- [ ] Save each candidate after preflight and after full verification.
- [ ] Enforce `unprocessed_candidates = selected_symbols - verified_symbols`.
- [ ] Verify the selected-set SHA-256 commitment and configured deep-dive budget.
- [ ] Resolve every committed selected symbol; never let an LLM silently choose a smaller subset.
- [ ] Keep an interrupted run `partial`; do not publish final-three selections.
- [ ] Before publishing “no candidates,” confirm full in-scope candidate coverage, explicit and verified exhaustion, zero queue/unresolved rows, all final dispositions, and at least one assessable row.
- [ ] Confirm Markdown exposes nonzero broad-screen review, unavailable, screened-out, and exclusion groups.
## 1. Corporate-Action Preflight — Perform First
- [ ] Confirm exchange listing is active.
- [ ] Confirm symbol is active and quote volume is plausible.
- [ ] Search for pending or completed acquisitions/mergers.
- [ ] Check delisting, suspension, ticker change, split, reverse split, and spin-off.
- [ ] Record `checked_at` and `latest_material_event_at`.
- [ ] Attach resolving source IDs.
- [ ] Hard-exclude completed/pending M&A and inactive/delisted securities from normal GARP ranking.
- [ ] Route rumors, unknown status, or stale preflight to review.
## 2. Snapshot Integrity
- [ ] Record current-price timestamp, timezone, and session.
- [ ] Confirm regular-close versus after-hours/pre-market/intraday.
- [ ] Record latest earnings period and publication timestamp.
- [ ] Confirm no earnings release or material event occurred after the price timestamp.
- [ ] Confirm split adjustment for price, EPS, and shares.
- [ ] Confirm reporting currency.
- [ ] Map FY1, FY2, and FY3 to the company's fiscal calendar.
- [ ] Normalize the formal current valuation period as NTM or FY1 with period end, estimate-as-of date, analyst count, and source IDs.
- [ ] Reject outer-year-only, pre-operating, zero-crossing, or excessively dispersed consensus as a current Forward P/E.
- [ ] Confirm fiscal versus calendar-year estimates.
## 3. Source Ledger and Evidence Map
For every load-bearing field, create a source-ledger row with:
- source ID,
- tier,
- document kind,
- title,
- publication and retrieval timestamps,
- URL or filing identifier,
- supported field names.
Create candidate-level `evidence` entries for price, latest earnings, revenue, cash flow, SBC, debt, cash classification, diluted shares, and corporate-action preflight.
### Tier 1 — SEC
- [ ] 10-K
- [ ] 10-Q
- [ ] 8-K
- [ ] DEF 14A
- [ ] S-3 / 424B5 when dilution risk exists
- [ ] Form 4 when insider activity matters
- [ ] 20-F / 6-K for foreign issuers
### Tier 2 — Company IR
- [ ] Earnings release
- [ ] Shareholder letter
- [ ] Investor presentation
- [ ] Prepared remarks/transcript supplied by company
- [ ] Guidance
- [ ] Investor-day materials
### Tier 3/4 — Supplemental
- [ ] Quote/market-data vendor
- [ ] Consensus database with retrieval time and analyst count
- [ ] Exchange/company status source
- [ ] Analyst model or calculation with explicit assumptions
- [ ] News/context source
When sources conflict, prefer the latest correcting SEC filing and document the discrepancy.
## 4. Business and Growth
- [ ] Summarize business model and revenue drivers.
- [ ] Identify recurring, transactional, cyclical, project, and acquired revenue.
- [ ] Calculate three- to five-year revenue CAGR.
- [ ] Calculate three-year GAAP EPS CAGR when meaningful.
- [ ] Calculate three-year standard FCF-per-share CAGR.
- [ ] Separate organic and acquisition growth.
- [ ] Separate price, volume, mix, and FX where material.
- [ ] Identify customer, product, platform, channel, and geographic concentration.
- [ ] Assess market share and competitive advantages.
- [ ] Classify growth as accelerating, stable, modestly decelerating, clearly decelerating, bottoming, or peaking.
## 5. Latest Earnings
- [ ] Fiscal period/end date/release timestamp
- [ ] Revenue, YoY, sequential, and organic growth
- [ ] Gross profit and margin
- [ ] GAAP operating income and margin
- [ ] Adjusted operating income and margin
- [ ] GAAP and adjusted EPS
- [ ] Operating cash flow
- [ ] Standard FCF
- [ ] Capex cash-outflow magnitude
- [ ] SBC
- [ ] Basic and diluted shares
- [ ] Guidance and revision direction
- [ ] Same-basis consensus comparison
- [ ] Segment performance
- [ ] Orders, backlog, or sector KPIs
- [ ] One-time costs/gains
- [ ] Working-capital and tax-rate changes
Do not let a headline beat override deteriorating guidance, cash flow, backlog, take rate, prescription growth, or another core KPI.
## 6. TTM Cash Flow and FCF
- [ ] Identify source statement as standalone quarter, YTD cumulative, FY, or explicit TTM.
- [ ] Normalize every `capex_cash_outflow` as a non-negative magnitude.
- [ ] Select one method: `reported_ttm`, `sum_4_discrete`, or `fy_plus_current_ytd_minus_prior_ytd`.
- [ ] For `sum_4_discrete`, confirm four unique standalone quarters.
- [ ] For YTD reconstruction, confirm comparable current/prior periods.
- [ ] Recalculate standard FCF = OCF − capex cash outflow.
- [ ] Reconcile supplied and reconstructed values within configured tolerance.
- [ ] Keep company-adjusted FCF separate and document its definition.
- [ ] Calculate SBC-adjusted economic FCF separately.
- [ ] Explain working-capital contributions.
## 7. GAAP / Adjusted Discipline
- [ ] Record a separate `metric_basis` for current, year-2, and year-3 periods.
- [ ] Confirm all periods use one basis.
- [ ] Never combine current GAAP EPS with future adjusted EPS under one multiple.
- [ ] For adjusted/normalized metrics, record the GAAP starting metric.
- [ ] List every adjustment, amount, label, and recurring status.
- [ ] Confirm arithmetic tie-out to the valuation metric.
- [ ] Attach source IDs.
- [ ] Distinguish company non-GAAP from analyst-created normalization.
- [ ] Flag recurring exclusions and changed definitions.
Review exclusions:
- SBC
- restructuring
- acquisition costs
- intangible amortization
- acquired IPR&D
- litigation
- impairment
- tax items
- asset-sale gains
- other recurring “one-time” items
## 8. Accounting and Earnings Quality
- [ ] Reconcile net income and operating cash flow.
- [ ] Review receivables, DSO, inventory, and inventory days.
- [ ] Review unbilled revenue, contract assets, deferred revenue, and customer advances.
- [ ] Review capitalized software/R&D.
- [ ] Review goodwill, intangibles, and impairments.
- [ ] Review asset-sale gains, tax effects, and pension/non-operating income.
- [ ] Review related parties, auditor changes, material weaknesses, restatements, and SEC investigations.
## 9. Balance Sheet and Cash Classification
- [ ] Corporate cash
- [ ] Marketable securities available to shareholders
- [ ] Customer/settlement funds
- [ ] Restricted cash
- [ ] Total debt
- [ ] Net debt using only eligible cash
- [ ] Net debt/EBITDA
- [ ] Interest expense and coverage
- [ ] Debt maturities, rates, covenants, leases, preferred stock, and minority interests
- [ ] Goodwill/intangibles and contingent acquisition obligations
For payments/marketplace businesses, never treat settlement float as corporate cash.
## 10. SBC, Dilution, and Capital Allocation
- [ ] SBC amount, SBC/revenue, and SBC/OCF
- [ ] Basic and diluted share history
- [ ] Annualized diluted-share change
- [ ] RSUs, options, warrants, convertibles
- [ ] ATM, shelf, and public-offering history
- [ ] Actual buyback dollars and shares
- [ ] Average repurchase price
- [ ] Net share reduction after employee issuance
- [ ] Buyback funding source
- [ ] Acquisition returns and impairment history
## 11. Forecast Quality and Arithmetic Bridge
For every current/year-2/year-3 metric used:
- [ ] Period label and period kind
- [ ] Metric basis
- [ ] Source classification
- [ ] Retrieval time and source IDs
- [ ] Analyst count
- [ ] Estimate low/high or dispersion
For year-2/year-3:
- [ ] Revenue assumption
- [ ] Gross/operating or FCF margin assumption
- [ ] Interest assumption
- [ ] Tax rate
- [ ] Diluted shares
- [ ] Buyback/dilution assumptions
- [ ] Acquisition/divestiture assumptions
- [ ] Metric numerator and denominator
- [ ] Arithmetic tie to valuation metric within tolerance
Flag single-analyst outer-year estimates and unexplained linear extrapolation.
## 12. Valuation and Scenarios
- [ ] Sector-appropriate valuation basis
- [ ] Current multiple recomputed as price/current metric
- [ ] Supplied multiple reconciled
- [ ] NTM/FY1/FY2 multiple labels
- [ ] EV/EBITDA, EV/EBIT, EV/FCF where appropriate
- [ ] Standard and SBC-adjusted FCF yield
- [ ] Own-history median
- [ ] Three to five genuine peers
- [ ] Same-basis/same-period peer median
- [ ] Constant-multiple 2Y and 3Y cases
- [ ] 20% contraction 2Y and 3Y cases
- [ ] Peer-median case only when valid
## 13. Sector-Specific Review
### Commercial biopharma
- [ ] Product-level revenue and top-product concentration
- [ ] Nearest material LOE date
- [ ] Patent settlement/litigation status
- [ ] Next-generation formulation/indication strategy
- [ ] Replacement pipeline and launch costs
- [ ] Source IDs for concentration and LOE
### Payments
- [ ] TPV growth
- [ ] Revenue/TPV and gross-profit/TPV
- [ ] Current and prior gross-profit take rate
- [ ] Corporate versus settlement cash
- [ ] FX and country exposure
- [ ] Standard versus adjusted FCF
- [ ] Source IDs
Use `sector-kpis.md` for all other sectors.
## 14. Cyclical Review
For scores 3–5:
- [ ] Five- to ten-year revenue, margin, EPS, and FCF history
- [ ] Orders/backlog/book-to-bill
- [ ] Inventory/utilization/capacity
- [ ] Customer/industry capex
- [ ] Commodity/freight exposure
- [ ] Price versus volume
- [ ] Recession drawdown
- [ ] Mid-cycle revenue and margin
- [ ] Normalized per-share metric and multiple
- [ ] Normalization method and source IDs
## 15. Peer Set
- [ ] Three to five genuine peers
- [ ] Selection reason per peer
- [ ] Same metric basis and period kind
- [ ] Growth, margins, ROIC/sector return, leverage, SBC, dilution, and valuation compared
- [ ] Discount classified as temporary, structural, justified, or unverified
## 16. Governance and Thesis
- [ ] Management capital allocation
- [ ] Insider ownership and transactions with 10b5-1 context
- [ ] Dual-class/founder control
- [ ] Related parties and compensation alignment
- [ ] Three- to five-sentence thesis
- [ ] Why discounted and why market may be right
- [ ] Six- to 24-month catalysts
- [ ] Largest concrete risk
- [ ] Bear case
- [ ] Measurable invalidation conditions
- [ ] Next-quarter KPIs
- [ ] Best reason not to buy now
## 17. Final Gate
A candidate may be `eligible` only when:
- [ ] Corporate action is fresh, active, and sourced.
- [ ] No hard flag is unresolved.
- [ ] Price and event timestamps are consistent.
- [ ] Core financial fields have Tier 1/2 evidence.
- [ ] TTM cash flow and capex signs are valid.
- [ ] Current/future metric bases match.
- [ ] Forecast bridge and required GAAP reconciliation pass.
- [ ] Valuation basis is sector-appropriate.
- [ ] Peer set is valid or explicitly unavailable without inventing a median.
- [ ] Data-quality score meets threshold.
- [ ] Required cyclical/sector evidence is complete.
- [ ] Minimum upside gate passes.
- [ ] Invalidation conditions are measurable.
- [ ] Run is complete before final-three publication.
## 18. Schema-v3 Autonomous Completion Checks
Before presenting a live run, confirm:
- [ ] Market context contains no placeholder text, is not future-dated, and has policy rate, 10Y yield, inflation, market forward P/E, small/mid-cap context, and resolving source IDs.
- [ ] `screening_audit` hashes to the full row-level universe artifact.
- [ ] Every audit-selected symbol has a verified candidate checkpoint.
- [ ] No selected symbol remains in `unprocessed_candidates`.
- [ ] `source.supports` is an array for every source.
- [ ] Source kind/tier is correct; no third-party transcript is labeled company IR.
- [ ] Q4 and full-year figures are stored separately.
- [ ] Current formal valuation metric is NTM or FY1.
- [ ] At least one future horizon is rankable after analyst-count checks.
- [ ] Current/future metric bases match.
- [ ] EV/FCF uses corporate cash plus eligible marketable securities consistently.
- [ ] Latest-release and normalized cash/debt values reconcile or have a source-backed note.
- [ ] High-growth 21–30x P/E exceptions were not mechanically discarded.
- [ ] Auto dealers use floorplan-adjusted leverage.
- [ ] Commercial biopharma product concentration and LOE are sourced or derived.
- [ ] `evaluate_candidates.py --strict --require-final` exits 0.
- [ ] The first diagnostic report was inspected and repairable blockers were corrected.
## v3.5 Final Quality Checklist
- [ ] Broad Screen selected lanes are diversified across core/high-growth/near-miss/cyclical opportunities where available.
- [ ] Candidate rows retain `average_volume` and `liquidity_source_ids`.
- [ ] Provider-side share-volume filters are disclosed and do not count as full listing enumeration.
- [ ] Every forecast driver has origin, source IDs, and `target_solved=false`.
- [ ] No margin, share count, or residual adjustment was reverse-solved merely to match target EPS.
- [ ] Every discrete cash-flow period has explicit source support.
- [ ] SEC filing evidence uses an accession-specific Archives URL.
- [ ] Recent spin-offs/acquisitions have pro-forma transition normalization.
- [ ] Formal eligible names pass score, FCF, ROIC, leverage, dilution, low-case, and LOE quality gates.
- [ ] `final_three` categories remain null when category thresholds are not met.
- [ ] `prepublish_audit.py` exits 0.
- [ ] `bundle_run_artifacts.py` produces a self-contained, manifest-hashed ZIP.
references/review-regression-matrix.md
# Review Regression Matrix — v3.6
The first section records inherited contract-3.5 controls; the final section covers the v3.6 Claude Code acquisition architecture.
| Observed failure | Prevention | Regression/release check |
|---|---|---|
| Default USD 500M–20B request was silently changed to USD 3B–4B | Immutable `user_requested_scope`, separate `executed_scope`, and explicit authorization gate | scope-preservation and unauthorized-reduction tests |
| One-session volume was labeled as ADDV and used to build the pool | Shared average-liquidity validator requires provider average or 20+ trading days and source IDs | single-session rejection and 20-day/provider-average acceptance tests |
| FRMI-like outer-year EPS produced a misleading 4x current Forward P/E | Estimate normalizer requires a resolving NTM/FY1 horizon and clears canonical values for outer-year/pre-operating/zero-crossing data | normalizer outer-year, FY1-horizon, and FRMI regression tests |
| Five selected names were later reduced to two by model preference | Selected-set SHA-256 commitment; every selected symbol must reach a terminal verified record; budget changes require re-screening | selected-commitment and all-selected completion tests |
| Intermediate output asked the user whether to continue or choose a subset | Exit code 2 is an internal continuation signal; run-state next action is deterministic and never requires user confirmation | autonomous-continuation and no-user-choice tests |
| A cached v3.1 package produced artifacts after v3.5 was installed | Single-source runtime metadata and fingerprint in every audit/snapshot; hard mismatch rejection; `--version` on all entry points | stale-runtime and cross-script version tests |
| LLM replaced deterministic broad-screen rules with an 8% revenue gate | Script output is authoritative; soft misses remain `guideline_misses`; near-miss/high-growth/cyclical routing preserved | broad-screen near-miss and high-growth tests |
| Cyclicals were rejected instead of normalized | Cycle 3–5 or `peak_profit_risk=true` forces sourced normalization before eligibility | peak-profit and cyclical normalization tests |
| A bounded 20-name pool was treated as a full-market conclusion | Explicit conclusion scope and validated generation audit; bounded ranking/no-candidate wording | bounded-pool completion and scoped no-candidate tests |
| Candidate pool stayed unresolved while run claimed final | Exhausted + all resolved + queue zero required; checkpoint and evaluator recompute | unresolved-pool finality tests |
| Forecast bridge passed because numerator was set to EPS × shares | Driver-derived revenue/margin/interest/tax/share reconstruction; numerator is cross-check only | circular-numerator rejection and valid driver bridge tests |
| Adjusted EPS bridge did not tie back to GAAP | Driver GAAP EPS and after-tax adjustments must tie to GAAP reconciliation | adjusted bridge reconciliation tests |
| `biopharma` alias bypassed LOE controls | Sector aliases normalize to `commercial_biopharma`; concentration/LOE/6x/8x evidence required | biopharma alias LOE test |
| Peak-profit risk was flagged but normalization was marked unnecessary | `peak_profit_risk=true` overrides a low numeric cyclicality score | peak-profit normalization test |
| EV/FCF text used manual cash while evaluator could not classify cash | Ordinary cash normalization; payments/custodial cash separation; evaluator-calculated EV only | cash normalization and payments cash tests |
| Data-quality score was 100 despite missing ROIC/EBITDA evidence | Source-linked ROIC/EBITDA evidence required; fail-closed quality cap | financial-quality evidence-cap test |
| TTM FCF periods cited only the latest 10-Q | Every TTM method/component requires resolving period source IDs | TTM period-source test |
| Funnel said preflight passed 0 although selected candidates passed | Evaluator recomputes selected-candidate preflight and completion counts | funnel-count consistency test |
| Report mixed 2Y base with 3Y stress | Explicit 2Y/3Y base and 2Y/3Y contraction columns | scenario-rendering regression |
| Official-statistics label pointed to a nonofficial domain | Source kind/domain validation | official-domain validation test |
| Standalone and overlay packages differed | Build package once from canonical source, embed identical bytes, compare SHA-256 and logical contents | release packaging checks |
| Global priority selected only one factor style | Four deterministic selection lanes with quotas, sector cap, and backfill | multi-lane opportunity-type test |
| Candidate serializer dropped average-volume provenance | Broad-screen metrics preserve `average_volume` and `liquidity_source_ids` | serializer provenance test |
| Provider share-volume filters were called full-universe enumeration | Retrieval-filter inspection downgrades conclusion scope and enumeration claim | provider-prefilter scope test |
| DKS-like low-P/E name ranked despite EV/FCF 79x and sub-1% adjusted FCF yield | Post-score quality eligibility gate routes severe weak-FCF names to review | low-FCF quality-gate test |
| Driver margins were solved backwards to consensus EPS | Per-driver provenance and `target_solved=false` contract | target-solved bridge rejection test |
| SEC browse page stood in for an immutable filing | Accession-specific SEC Archives URL required | SEC browse-page rejection test |
| Generic TTM support label stood in for four discrete quarters | Exact period support paths required | discrete-period support test |
| Recent spin-off was hard-excluded as an invalid special case | Corporate-transition normalization routes to review | recent-spin-off migration test |
| One eligible name was forced into every final-three category | Category-specific thresholds and nullable labels | nullable final-three test |
| Final ZIP contained only summaries, not audit evidence | Prepublication audit plus deterministic complete-run bundle | bundle completeness test |
| Final prose asked the user to continue | Prepublication prose audit rejects unfinished-run language | continue-language rejection test |
## v3.6 Claude Code direct-FMP regressions
| Observed failure | v3.6 prevention | Regression/release check |
|---|---|---|
| Bulk FMP JSON consumed the model context before underwriting | Direct Python pipeline writes provider payloads to disk and prints a compact summary only | direct-pipeline compact-summary test |
| Existing skills duplicated FMP transport, fallback, and retry logic | GARP client is generated from the repository-level `scripts/fmp_client/` registry/special-template source of truth | generator-source/package equivalence check |
| Repeated runs refetched unchanged provider data | SQLite response cache with dataset TTLs | persistent-cache cross-client test |
| Company-screener responses silently truncated at the provider limit | Adaptive overlapping market-cap-band splitting; scope is complete only when every leaf is unsaturated | saturated-band enumeration test |
| Per-symbol liquidity work was allocated before economic relevance was known | Bulk EOD first; fallback exact-liquidity targets are chosen round-robin from the four GARP lanes | direct-pipeline lane-target test |
| Overlapping core/high-growth symbols made the provider-pool audit report too few represented lanes | Lane coverage is calculated from final candidate memberships, not only the insertion loop | overlapping-lane provider-pool test |
| Full FMP packets still burdened the model after discovery | Full payloads remain under `provider/candidate-data`; projected packets contain only underwriting-relevant fields and artifact paths | candidate-packet size/content review |
| Claude stopped and requested another user turn | `NEXT_ACTION.json` sets `user_confirmation_required=false`; Claude Code command forbids “Continue” handoff | command contract and prepublish phrase gate |
references/scoring-rubric.md
# Scoring Rubric, Evidence Quality, Penalties, and Gates — v3.5
## 1. Philosophy
Use two independent dimensions:
1. **Investment score** — attractiveness of the company and return setup.
2. **Data-quality score** — whether the evidence is sufficient to trust the investment score.
A high investment score never overrides missing evidence. A candidate below the configured data-quality floor is `review_required`, not rankable.
## 2. Raw Investment Score — 100 Points
### A. Growth Sustainability — 20
Assess:
- two- to three-year revenue and per-share growth visibility,
- durability of drivers,
- organic versus acquisition growth,
- customer/product concentration,
- forecast breadth and plausibility,
- sector-specific lifecycle or cycle risks.
Guide:
- 17–20: durable 20%+ per-share growth or exceptional 15%+ visibility.
- 13–16: credible mid-teens growth with manageable risks.
- 8–12: growth exists but is volatile, concentrated, or fading.
- 0–7: weak or unsupported growth.
### B. Valuation Attractiveness — 20
Assess:
- same-basis current/forward multiple,
- standard and SBC-adjusted FCF yield,
- own-history and peer comparison,
- constant-multiple upside,
- 20% contraction resilience,
- value-trap risk.
Do not award a high score merely for low trailing P/E.
### C. Financial Quality and Capital Efficiency — 15
Assess:
- ROIC/sector return metric,
- margins and operating leverage,
- usable corporate cash,
- net leverage and refinancing risk,
- goodwill/intangible burden,
- resilience under stress.
### D. FCF and Earnings Quality — 15
Assess:
- correctly reconstructed standard FCF,
- OCF versus net income,
- working-capital quality,
- standard versus company-adjusted FCF gap,
- SBC-adjusted economics,
- accounting red flags.
### E. Competitive Advantage — 10
Assess:
- switching costs,
- network effects,
- brand/data/regulatory moat,
- recurring revenue,
- customer retention,
- market-share durability.
### F. Capital Allocation, SBC, and Dilution — 10
Assess:
- actual net share change,
- buyback price discipline and funding,
- SBC burden,
- acquisition returns and impairments,
- debt repayment and reinvestment quality.
### G. Catalyst / Risk Balance — 10
Assess:
- catalyst timing and probability,
- identifiable route to estimate delivery,
- downside asymmetry,
- measurable invalidation conditions,
- reason not to buy now.
## 2A. Broad-Screen Guidance Versus Hard Gates
Broad-screen growth and preferred valuation thresholds are guidelines. Store isolated misses in `guideline_misses`; they reduce priority or trigger a near-miss review but do not automatically reject a company.
Selection-eligible patterns include:
- preferred forward valuation plus credible per-share growth,
- P/E 21–30 with 20%+ per-share growth,
- attractive P/E and mid-teens per-share growth even when headline revenue growth is below the preferred range,
- strong cyclical growth/valuation combinations that require mid-cycle normalization.
Hard broad failures are reserved for severe conditions such as non-positive standard FCF when applicable, negative ROIC, excessive leverage beyond the hard limit, extreme forward valuation unsupported by growth, or negative revenue and per-share growth.
`sector_review_required` is selection-eligible when the remaining work is nonblocking cyclical normalization. Missing bank/REIT/BDC/MLP valuation or auto-dealer floorplan-adjusted leverage remains blocking and produces `needs_enrichment`.
## 3. Evidence-Derived Data-Quality Score — 100
The evaluator awards points only when fields and resolving source IDs pass. Self-attested completeness flags are ignored.
| Item | Weight | Required evidence |
|---|---:|---|
| Quote verified | 5 | Price source resolves. |
| Latest earnings verified | 7 | Quarter/full-year separation, timestamps, Tier 1/2 source. |
| Core financials and TTM periods verified | 10 | Revenue, TTM OCF/capex periods, SBC, debt, cash, and shares have primary evidence. |
| Guidance/consensus labeled | 8 | Each valuation period has type, date, analysts, and sources. |
| Driver-derived forecast bridge | 15 | Independent operating-driver arithmetic and source evidence pass. |
| Diluted shares verified | 8 | History and Tier 1/2 evidence. |
| SBC verified | 8 | TTM SBC and Tier 1/2 evidence. |
| Peer set verified | 8 | At least three genuine same-basis forward peers. |
| GAAP/non-GAAP reconciled | 8 | Adjusted periods tie GAAP and adjustment components. |
| Corporate actions verified | 4 | Fresh active listing/symbol/M&A preflight. |
| Cyclical/peak-profit normalization | 5 | Sourced normalization whenever cycle or peak-profit risk requires it. |
| Cash classification verified | 5 | Source-backed corporate cash; customer/restricted funds separated where applicable. |
| ROIC/EBITDA evidence verified | 4 | Reported or transparent analyst-calculated values with sources. |
| Sector risk verified | 5 | LOE/concentration, payments cash, or auto-dealer leverage evidence as applicable. |
| **Total** | **100** | |
Default minimum: 70. A score of 100 means deterministic evidence checks passed; it does not guarantee the thesis is correct.
## 4. Deterministic Penalties
Penalties subtract from the raw 100-point investment score.
### Data-quality penalty
| Data quality | Penalty |
|---:|---:|
| 90–100 | 0 |
| 80–89 | 2 |
| 70–79 | 5 |
| 60–69 | 10 |
| below 60 | 20 |
The minimum-quality gate may still prevent ranking regardless of penalty.
### Cyclicality penalty
| Score | Penalty |
|---:|---:|
| 1 | 0 |
| 2 | 1 |
| 3 | 3 |
| 4 | 6 |
| 5 | 10 |
Missing required normalization is also a review blocker.
### Estimate breadth
Use the minimum analyst count among available year-2/year-3 metrics.
| Minimum count | Penalty |
|---:|---:|
| 5+ | 0 |
| 3–4 | 2 |
| 1–2 | 4 |
| unavailable | 5 |
### Estimate dispersion
Use the maximum normalized low/high spread among future periods.
| Dispersion | Penalty |
|---:|---:|
| ≤20% | 0 |
| >20–30% | 1 |
| >30–50% | 2 |
| >50% | 4 |
### SBC / revenue
| SBC/revenue | Penalty |
|---:|---:|
| missing | 2 |
| ≤5% | 0 |
| >5–10% | 2 |
| >10–15% | 3 |
| >15–25% | 5 |
| >25% | 8 |
### Annualized diluted-share growth
| Growth | Penalty |
|---:|---:|
| missing | 2 |
| ≤3% | 0 |
| >3–5% | 2 |
| >5–10% | 4 |
| >10% | 8 |
Net share reduction is not inferred from a buyback authorization.
### Liquidity
- Below preferred dollar-volume threshold but above hard floor: 2 points.
- Below hard floor: hard exclusion as extreme illiquidity.
### Sector-specific risk
Current deterministic examples:
- Commercial biopharma with top-product concentration ≥50% and nearest material LOE within five years: 2 points.
- The same condition with LOE within three years: 4 points.
Missing required sector evidence causes review rather than a silent neutral score.
## 5. Gates and Statuses
### `excluded`
Hard exclusion includes:
- inactive, delisted, or suspended symbol,
- pending or completed M&A/special-situation pricing,
- OTC, penny stock, shell/SPAC, or development-stage economics,
- repeated financing dependence,
- unresolved restatement/material weakness/SEC investigation,
- meme-only thesis,
- extreme illiquidity,
- invalid special-case valuation basis.
### `screened_out`
Use when the security is valid and active but does not satisfy the broad economic screen. Preserve reasons and sources.
### `review_required`
Examples:
- stale quote/event mismatch,
- stale or unresolved corporate-action preflight,
- TTM/capex reconstruction failure,
- mixed GAAP/adjusted periods,
- forecast bridge or required reconciliation failure,
- data quality below threshold,
- missing cyclicality normalization,
- missing sector-critical evidence,
- insufficient constant-multiple upside when configured,
- materially conflicting supplied multiple.
### `eligible`
All hard and review gates pass. Eligible does not mean “buy”; it means the candidate may be compared and ranked.
## 6. Run-Level Ranking Status
- `final`: run metadata says complete and no symbols remain unprocessed.
- `provisional`: partial run or non-empty unprocessed list.
A provisional report may show completed candidate calculations, but `final_three` must remain empty.
## 7. Confidence Labels
- `high`: eligible, data quality ≥85, analyst breadth ≥5, and no major unresolved caveat.
- `medium`: eligible with data quality 70–84 or analyst breadth 3–4.
- `low`: eligible only under an exception or with fragile estimates.
- `not_ranked`: review-required, screened-out, or excluded.
## 8. Sorting and Tie-Breaking
Sort eligible candidates by:
1. final score,
2. data-quality score,
3. preferred three-year constant-multiple upside,
4. lower cyclicality,
5. symbol for deterministic stability.
Final selections use separate lenses:
- highest conviction: final score, data quality, confidence,
- most undervalued: SBC-adjusted FCF yield and lower current multiple,
- largest upside: preferred three-year constant-multiple upside.
## 9. Schema-v3 Quality Caps
The lowest applicable cap wins.
| Failure | Maximum data-quality score |
|---|---:|
| Latest quarter/full-year mixed or incomplete | 55 |
| Source ledger kind/domain/supports invalid | 60 |
| Forward basis or driver-derived forecast bridge invalid | 60 |
| Core financial/TTM-period evidence incomplete | 65 |
| Required cyclical or peak-profit normalization missing | 65 |
| Required sector/LOE stress evidence missing | 65 |
| ROIC/EBITDA evidence incomplete | 65 |
| Broad-screen audit/runtime invalid | 65 |
| Cash classification incomplete or conflicting | 70 |
| Market context absent, stale, future-dated, or placeholder | 70 |
These caps prevent unsupported 90–100 quality scores.
## 10. Forecast-Horizon Rankability
- Current formal valuation: NTM or FY1.
- A consensus future horizon normally requires at least three analysts.
- A one- or two-analyst horizon is supplemental unless a sourced independent model passes the forecast bridge.
- A candidate must have at least one rankable two- or three-year horizon.
- TTM multiples, unsupported outer-year estimates, and mixed metric bases cannot satisfy the 30% upside gate.
## 11. Run-Level Finality
A final ranking requires:
```text
run_metadata.status = complete
unprocessed_candidates = []
screening_audit.valid = true
screening_audit.runtime matches installed v3.6 fingerprint
screening_audit.candidate_pool.generation_audit.valid = true for bounded pools
screening_audit.enrichment.candidate_pool_exhausted = true
screening_audit.enrichment.unresolved_count = 0
screening_audit.enrichment.queue_count = 0
screening_audit.enrichment.all_rows_resolved = true
contract.valid = true
```
A full-market no-candidates conclusion additionally requires full in-scope economic coverage. A bounded-pool no-candidates conclusion must be explicitly scoped to the audited bounded pool and requires at least one economically assessable row. Attempt counts and evaluable-count thresholds cannot replace pool resolution.
The CLI `--require-final` returns exit code 2 otherwise. Diagnostic files are still written for the repair loop.
## 12. v3.5 Formal Eligibility Quality Gate
Passing the 30% base-case upside test is necessary but not sufficient. A formal `eligible` name must satisfy all configured floors:
| Gate | Default |
|---|---:|
| Final investment score | ≥70 |
| SBC-adjusted FCF yield | ≥3%, **or** EV/FCF ≤30x |
| ROIC / sector-equivalent return | ≥8% |
| Net Debt / EBITDA | ≤3.0x when applicable |
| Diluted-share CAGR | ≤5% |
| Supported low-consensus-case upside | ≥15% |
| Severe LOE stress | no loss worse than -25% |
A candidate with one or two ordinary failures may be labeled `conditional`. A candidate with more failures, severely weak FCF support (for example SBC-adjusted yield below 1% or EV/FCF above 50x), or severe LOE downside is `review_required`.
### Category-specific final selections
`final_three` categories may be null. Never force the sole eligible name into every label.
- **Highest conviction:** score ≥75, DQ ≥80, stress upside ≥0.
- **Most undervalued:** SBC-adjusted FCF yield ≥5% or EV/FCF ≤20x.
- **Largest upside:** supported low-consensus-case upside ≥20%.
If no ranked candidate meets a category threshold, render `確認できず` / `not verified`.
references/sector-kpis.md
# Sector-Specific Valuation and KPI Rules
Apply these rules in addition to the common framework. A sector-specific metric replaces, rather than decorates, an inappropriate generic metric.
## 1. SaaS and Software
### Required KPIs
- ARR and ARR growth
- Net revenue retention (NRR)
- Gross retention
- RPO and current RPO
- Subscription versus services mix
- Gross margin
- FCF margin
- Rule of 40
- SBC / revenue
- Diluted-share growth
- Capitalized software development
- Customer concentration
### Valuation
Use EV/revenue only as a secondary measure. Prefer EV/FCF, FCF yield, normalized operating margin, and per-share FCF growth once FCF is positive.
### Red flags
- ARR growth falling faster than revenue growth
- NRR below 100%
- SBC / revenue above 15% without rapid decline
- FCF driven mainly by SBC or deferred revenue
- GAAP losses masked by recurring adjusted exclusions
## 2. Semiconductors and Semiconductor Equipment
### Required KPIs
- Inventory and inventory days
- Book-to-bill
- Backlog
- Utilization
- Lead times
- Channel inventory
- Customer capex
- End-market mix
- Product mix
- Gross-margin cycle
- Customer concentration
### Valuation
Use normalized/mid-cycle EPS, EBIT, and FCF. A low trailing P/E at peak margins is not cheap.
### Red flags
- Inventory growth exceeding revenue growth
- Backlog cancellation or book-to-bill below 1
- Margin well above historical median without structural evidence
- Customer capex rolling over
## 3. Industrials and Capital Goods
### Required KPIs
- Orders
- Backlog
- Book-to-bill
- Organic growth
- Price and volume
- Utilization
- Customer capex
- Aftermarket/service mix
- Working-capital requirements
### Valuation
Use EV/EBIT, EV/FCF, and normalized EPS. For highly cyclical businesses calculate mid-cycle margins.
## 4. Retail, Restaurants, and Consumer Companies
### Required KPIs
- Comparable sales
- Traffic
- Average ticket
- Unit growth
- Store-level margin
- New-unit payback
- Inventory turnover
- Markdown rate
- Customer-acquisition cost
- Direct-to-consumer mix
- International growth
### Valuation
Use P/E and FCF yield with lease-aware enterprise value where material. Distinguish growth from unit expansion, price, traffic, and mix.
### Red flags
- Price-led comps with declining traffic
- Inventory rising faster than sales
- New-unit returns deteriorating
- Buybacks funded by leverage while store economics weaken
## 5. Healthcare Services and Medical Devices
### Required KPIs
- Procedure or patient volume
- Reimbursement trends
- Utilization
- Installed base
- Consumable/recurring revenue
- Pipeline or product-launch milestones
- Gross margin
- Regulatory actions
- Customer or payer concentration
### Valuation
Use P/E, EV/EBIT, and EV/FCF. For product-concentrated businesses, scenario-test loss of exclusivity or reimbursement changes.
## 6. Commercial Biotechnology and Specialty Pharmaceuticals
### Eligibility
Exclude pre-revenue or development-stage companies. Admit only businesses with meaningful commercial revenue and a valid earnings/standard-FCF framework.
### Required KPIs
- Product-level revenue and growth
- Top-product revenue concentration (`top_product_revenue_pct`)
- Prescription, patient, or demand growth
- Gross-to-net deductions
- Market share
- Nearest material loss-of-exclusivity date (`nearest_material_loe_date`)
- ANDA/patent settlement or litigation status
- Next-generation formulation, indication, or lifecycle-extension strategy
- Replacement pipeline probability and milestones
- R&D and launch intensity
- Acquisition integration and contingent obligations
- Source IDs supporting concentration and LOE
### Valuation and Risk
Use P/E and standard FCF only when operating economics are established. Do not mix a current GAAP metric with future adjusted consensus. Treat product concentration and LOE as structural risk even when near-term growth is strong. Do not count pipeline NPV as certain value.
The evaluator applies a sector penalty when the top product is at least 50% of revenue and nearest material LOE is within five years; missing sourced concentration/LOE evidence is review-required in strict mode.
## 7. Payments, Marketplaces, and Money Movement
### Required KPIs
- TPV or payment volume and growth
- Revenue/TPV and gross-profit/TPV
- Current and prior gross-profit take rate (`gross_profit_to_tpv_pct`, `gross_profit_to_tpv_prior_pct`)
- Transaction count and active merchants/users
- Cross-border, country, and FX exposure
- Credit or chargeback losses where applicable
- Corporate cash and marketable securities
- Customer/merchant/settlement funds and restricted cash
- Standard FCF versus company-adjusted FCF
- Working-capital normalization
- Dilution and SBC
- Source IDs for cash classification and take rates
### Valuation and Risk
Use corporate cash only for net-debt analysis; settlement float is not shareholder cash. A revenue surge caused by lower-margin pass-through volume is not equivalent to gross-profit growth. Track gross-profit/TPV and operating leverage. If the take rate declines, explain whether mix, geography, competition, or accounting presentation caused it.
Strict mode requires sourced separation of corporate and settlement cash.
## 8. Banks
Do not use EV/EBITDA.
### Required KPIs
- P/TBV and P/book
- ROTCE and ROE
- Net interest margin
- Deposit beta
- Deposit mix and uninsured deposits
- Loan growth
- Nonperforming loans
- Net charge-offs
- Provision expense
- CET1
- Commercial real-estate exposure
- Securities marks and accumulated other comprehensive income
### Scenario basis
Use TBV per share or book value per share and an appropriate P/TBV or P/book multiple. Model credit and deposit stress.
## 9. Insurance
### Required KPIs
- P/book or P/TBV
- ROE
- Combined ratio
- Underwriting income
- Reserve development
- Investment income
- Catastrophe exposure
- Premium growth
- Retention
### Scenario basis
Use book/TBV per share or normalized EPS. Adjust for reserve quality and catastrophe-cycle conditions.
## 10. REITs
Do not use ordinary EPS or P/E as the primary basis.
### Required KPIs
- P/FFO and P/AFFO
- AFFO growth
- NAV premium/discount
- Same-store NOI
- Occupancy
- Leasing spreads
- Lease maturities
- Net debt / EBITDA
- Fixed-charge coverage
- Dividend payout
### Scenario basis
Use AFFO per share or NAV per share. Include refinancing and cap-rate sensitivity.
## 11. Business Development Companies (BDCs)
### Required KPIs
- Price/NAV
- Net investment income per share
- Non-accruals
- Portfolio yield
- Debt/equity
- Interest coverage
- First-lien exposure
- Payment-in-kind income
- Dividend coverage
- NAV trend
### Scenario basis
Use NAV per share or normalized NII per share. Do not use industrial-company EV/EBITDA.
## 12. MLPs and Midstream Partnerships
### Required KPIs
- Distributable cash flow per unit
- Distribution coverage
- Leverage
- Contracted versus commodity-sensitive cash flow
- Counterparty concentration
- Maintenance versus growth capex
- Unit issuance/buybacks
- IDRs or sponsor conflicts
### Scenario basis
Prefer DCF per unit, FCF per unit, and EV/EBITDA on a consistently defined basis. Treat tax structure separately.
## 13. Energy, Mining, Materials, Chemicals, Steel, and Shipping
### Required KPIs
- Commodity or freight prices
- Production/volume
- Realized price
- Unit costs
- Capacity/utilization
- Capital intensity
- Maintenance capex
- Reserve life where relevant
- Contract coverage
- Balance-sheet sensitivity
### Valuation
Use normalized commodity assumptions and mid-cycle margins. Do not annualize a spot-price windfall.
## 14. Homebuilders and Housing-Related Companies
### Required KPIs
- Orders and cancellations
- Backlog
- Community count
- ASP
- Gross margin excluding unusual items
- Incentives
- Land inventory
- Net debt/capital
- Mortgage-rate sensitivity
### Valuation
Use normalized EPS, book value, and land-cycle analysis. A low P/E at peak closings may be misleading.
## 15. Advertising, Staffing, Transportation, Airlines, and Autos
These are normally cyclicality score 3 to 5.
### Required KPIs
- Volume and utilization
- Price/yield
- Customer budget or hiring trends
- Capacity
- Load factor or miles/hours where relevant
- Used-asset values
- Backlog/order intake
- Labor and fuel/input costs
Use normalized operating margins and recession drawdown history.
## 16. Auto Dealerships and Floorplan Debt
Do not apply general-company net-debt/EBITDA mechanically to auto dealers. Obtain a sourced sector-adjusted leverage measure that explicitly excludes inventory floorplan financing when appropriate.
Required schema-v3 fields:
- `adjusted_net_debt_to_ebitda`
- `floorplan_debt_excluded = true`
- source IDs
Without these fields, route the candidate to `review_required`, not `screened_out` solely because unadjusted leverage appears high.
## 17. Commercial Biopharma LOE Stress
When a top product contributes at least 50% of revenue and a material LOE is within five years:
- calculate or source product concentration,
- record the nearest material LOE date,
- distinguish base patent, pediatric exclusivity, settlements, formulation/use patents, and replacement products,
- calculate 6x and 8x year-3 metric stress cases in addition to the generic 20% multiple-contraction case.
Product concentration may be derived as:
```text
top product revenue / total revenue
```
only when both values share the same period and source evidence.
## v3.5 Normalization Rules
- Normalize `biopharma`, `pharma`, `biotechnology`, `royalty_biopharma`, and `drug_delivery_platform` to `commercial_biopharma` when commercial product or royalty economics exist.
- Commercial biopharma requires sourced product/revenue-stream concentration, nearest material LOE, and configured 6x/8x stress scenarios.
- `peak_profit_risk=true` forces mid-cycle normalization regardless of the numeric cyclicality score.
- Sector evidence is part of the data-quality score and can cap it at 65 when missing.
requirements.txt
# Runtime dependencies for standalone .skill installs (issue #330).
# Verified by scripts/check_skill_deps.py.
requests>=2.31.0
scripts/build_discovery_pool.py
#!/usr/bin/env python3
"""Build an audited, bounded discovery pool for the US GARP screener.
Contract v3.5 retains and extends fail-closed controls that operate *before* any symbol is
prioritized:
1. Liquidity must be based on a genuine average (normally 20+ sessions), not
``price × today's volume``.
2. The user's requested market-cap scope is preserved separately from the
executed/retrieved scope. A narrower execution range is never silently
rewritten as the original request.
The script does not fetch market data. It validates and stratifies rows supplied
by the calling agent or provider connector, then writes a reproducible pool and
an audit record.
"""
from __future__ import annotations
import argparse
import csv
import hashlib
import json
import math
import sys
from collections import defaultdict
from collections.abc import Mapping, Sequence
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
try:
from skill_version import runtime_metadata
except ModuleNotFoundError: # Supports importlib-based unit loading.
import importlib.util as _importlib_util
_version_path = Path(__file__).with_name("skill_version.py")
_version_spec = _importlib_util.spec_from_file_location("skill_version", _version_path)
if _version_spec is None or _version_spec.loader is None:
raise
_version_module = _importlib_util.module_from_spec(_version_spec)
_version_spec.loader.exec_module(_version_module)
runtime_metadata = _version_module.runtime_metadata
try:
from screening_semantics import normalize_liquidity
except ModuleNotFoundError:
import importlib.util as _semantics_importlib_util
_semantics_path = Path(__file__).with_name("screening_semantics.py")
_semantics_spec = _semantics_importlib_util.spec_from_file_location(
"screening_semantics", _semantics_path
)
if _semantics_spec is None or _semantics_spec.loader is None:
raise
_semantics_module = _semantics_importlib_util.module_from_spec(_semantics_spec)
_semantics_spec.loader.exec_module(_semantics_module)
normalize_liquidity = _semantics_module.normalize_liquidity
ALLOWED_EXCHANGES = {"NYSE", "NASDAQ", "NYSE AMERICAN", "NYSEAMERICAN", "AMEX"}
DEFAULT_BUCKETS = [500_000_000, 2_000_000_000, 5_000_000_000, 10_000_000_000, 20_000_000_000]
MIN_AVERAGE_WINDOW_DAYS = 20
VALID_LIQUIDITY_METHODS = {
"mean_dollar_volume",
"mean_20d_dollar_volume",
"provider_average_dollar_volume",
"average_volume_x_price",
"provider_avg_volume_x_price",
"mean_20d_volume_x_price",
}
class DiscoveryError(ValueError):
pass
def _mapping(value: Any) -> dict[str, Any]:
return dict(value) if isinstance(value, Mapping) else {}
def _text(value: Any) -> str | None:
if isinstance(value, str):
value = value.strip()
return value or None
return None
def _number(value: Any) -> float | None:
if isinstance(value, bool):
return None
if isinstance(value, (int, float)):
result = float(value)
return result if math.isfinite(result) else None
if isinstance(value, str):
try:
result = float(value.replace(",", "").strip())
except ValueError:
return None
return result if math.isfinite(result) else None
return None
def _integer(value: Any) -> int | None:
number = _number(value)
return int(number) if number is not None and number.is_integer() else None
def _bool(value: Any) -> bool | None:
if isinstance(value, bool):
return value
if isinstance(value, str):
lowered = value.strip().lower()
if lowered in {"true", "1", "yes", "y"}:
return True
if lowered in {"false", "0", "no", "n"}:
return False
return None
def _load_rows(path: Path) -> list[dict[str, Any]]:
suffix = path.suffix.lower()
if suffix == ".csv":
with path.open(newline="", encoding="utf-8-sig") as handle:
return [dict(row) for row in csv.DictReader(handle)]
if suffix in {".jsonl", ".ndjson"}:
rows: list[dict[str, Any]] = []
for line_no, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
if not line.strip():
continue
value = json.loads(line)
if not isinstance(value, Mapping):
raise DiscoveryError(f"line {line_no} is not an object")
rows.append(dict(value))
return rows
value = json.loads(path.read_text(encoding="utf-8"))
if isinstance(value, list):
return [_mapping(row) for row in value]
if isinstance(value, Mapping) and isinstance(value.get("rows"), list):
return [_mapping(row) for row in value["rows"]]
raise DiscoveryError("input must be a JSON array, {rows:[...]}, JSONL, or CSV")
def _metric(row: Mapping[str, Any], *keys: str) -> float | None:
for key in keys:
value = _number(row.get(key))
if value is not None:
return value
return None
def _liquidity_evidence(row: Mapping[str, Any], price: float | None) -> dict[str, Any]:
"""Delegate to the shared fail-closed liquidity normalizer."""
normalized = normalize_liquidity(row, minimum_period_days=MIN_AVERAGE_WINDOW_DAYS)
return {
"value": normalized.get("value"),
"valid": normalized.get("valid_for_screen") is True,
"method": normalized.get("method"),
"window_days": normalized.get("period_days"),
"source_ids": normalized.get("source_ids", []),
"average_volume": normalized.get("average_volume"),
"reasons": sorted(set(normalized.get("reasons") or [])),
}
def _normalized_listing(row: Mapping[str, Any]) -> dict[str, Any]:
price = _metric(row, "price", "last")
liquidity = _liquidity_evidence(row, price)
return {
"symbol": (_text(row.get("symbol")) or _text(row.get("ticker")) or "UNKNOWN").upper(),
"company_name": _text(row.get("company_name"))
or _text(row.get("companyName"))
or _text(row.get("name")),
"exchange": (
_text(row.get("exchange")) or _text(row.get("exchangeShortName")) or ""
).upper(),
"sector": (_text(row.get("sector")) or "Unknown").strip(),
"industry": _text(row.get("industry")),
"price": price,
"market_cap": _metric(row, "market_cap", "marketCap"),
"average_daily_dollar_volume": liquidity["value"],
"liquidity_method": liquidity["method"],
"liquidity_window_days": liquidity["window_days"],
"liquidity_source_ids": liquidity["source_ids"],
"average_volume": liquidity["average_volume"],
"liquidity_basis_valid": liquidity["valid"],
"liquidity_review_reasons": liquidity["reasons"],
"is_actively_trading": _bool(row.get("is_actively_trading"))
if "is_actively_trading" in row
else _bool(row.get("isActivelyTrading")),
"is_common_stock": _bool(row.get("is_common_stock"))
if "is_common_stock" in row
else _bool(row.get("common_stock")),
"source_row": dict(row),
}
def _bucket_label(market_cap: float, buckets: Sequence[float]) -> str:
for low, high in zip(buckets, buckets[1:]):
if low <= market_cap < high:
return f"{int(low)}-{int(high)}"
if market_cap == buckets[-1]:
return f"{int(buckets[-2])}-{int(buckets[-1])}"
return "out-of-range"
def build_pool(
rows: Sequence[Mapping[str, Any]],
*,
min_market_cap: float,
max_market_cap: float,
min_price: float,
hard_min_adv: float,
max_pool: int,
per_cell: int,
buckets: Sequence[float] | None = None,
user_requested_min_market_cap: float | None = None,
user_requested_max_market_cap: float | None = None,
scope_reduction_authorized: bool = False,
scope_reduction_reason: str | None = None,
user_scope_evidence: str | None = None,
default_liquidity_source_ids: Sequence[str] | None = None,
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
if max_pool <= 0 or per_cell <= 0:
raise DiscoveryError("max_pool and per_cell must be positive")
if min_market_cap >= max_market_cap:
raise DiscoveryError("executed market-cap range must be increasing")
requested_min = float(
user_requested_min_market_cap
if user_requested_min_market_cap is not None
else min_market_cap
)
requested_max = float(
user_requested_max_market_cap
if user_requested_max_market_cap is not None
else max_market_cap
)
if requested_min >= requested_max:
raise DiscoveryError("user-requested market-cap range must be increasing")
scope_reduced = min_market_cap > requested_min or max_market_cap < requested_max
scope_reason = _text(scope_reduction_reason)
# Automatic resource-bounded execution is valid only as a disclosed bounded result.
# Explicit user authorization is recorded separately; it is not required to
# produce a scoped diagnostic/ranking.
# Resource pressure may bound the discovery *row count*, but it may not
# rewrite the user's requested market-cap range. A narrower executed range
# is invalid; an explicitly narrower user request must be passed as the
# user-requested bounds themselves.
scope_valid = not scope_reduced
bucket_edges = list(buckets or DEFAULT_BUCKETS)
if bucket_edges[0] != min_market_cap or bucket_edges[-1] != max_market_cap:
bucket_edges = (
[min_market_cap]
+ [edge for edge in bucket_edges if min_market_cap < edge < max_market_cap]
+ [max_market_cap]
)
if any(a >= b for a, b in zip(bucket_edges, bucket_edges[1:])):
raise DiscoveryError("market-cap buckets must be strictly increasing")
normalized: list[dict[str, Any]] = []
exclusions: dict[str, int] = defaultdict(int)
liquidity_reason_counts: dict[str, int] = defaultdict(int)
for raw in rows:
source_row = dict(raw)
if not source_row.get("liquidity_source_ids") and default_liquidity_source_ids:
source_row["liquidity_source_ids"] = list(default_liquidity_source_ids)
row = _normalized_listing(source_row)
exchange = row["exchange"]
price = row["price"]
market_cap = row["market_cap"]
adv = row["average_daily_dollar_volume"]
active = row["is_actively_trading"]
common = row["is_common_stock"]
reason: str | None = None
if exchange not in ALLOWED_EXCHANGES:
reason = "exchange_out_of_scope"
elif active is False:
reason = "inactive_symbol"
elif common is False:
reason = "not_common_stock"
elif price is None or price < min_price:
reason = "price_below_minimum"
elif market_cap is None or not (min_market_cap <= market_cap <= max_market_cap):
reason = "market_cap_out_of_range"
elif row["liquidity_basis_valid"] is not True:
reason = "validated_average_liquidity_unavailable"
for item in row["liquidity_review_reasons"]:
liquidity_reason_counts[str(item)] += 1
elif adv is None or adv < hard_min_adv:
reason = "liquidity_below_hard_floor"
if reason:
exclusions[reason] += 1
continue
row["market_cap_bucket"] = _bucket_label(float(market_cap), bucket_edges)
normalized.append(row)
cells: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
for row in normalized:
sector = str(row["sector"]).strip().upper() or "UNKNOWN"
cells[(sector, str(row["market_cap_bucket"]))].append(row)
for values in cells.values():
values.sort(
key=lambda row: (
-(row["average_daily_dollar_volume"] or 0),
-(row["market_cap"] or 0),
row["symbol"],
)
)
selected: list[dict[str, Any]] = []
selected_symbols: set[str] = set()
cell_keys = sorted(cells)
for depth in range(per_cell):
layer = [cells[cell][depth] for cell in cell_keys if len(cells[cell]) > depth]
layer.sort(
key=lambda row: (
-(row["average_daily_dollar_volume"] or 0),
-(row["market_cap"] or 0),
row["symbol"],
)
)
for row in layer:
if len(selected) >= max_pool:
break
if row["symbol"] in selected_symbols:
continue
selected.append(row)
selected_symbols.add(row["symbol"])
if len(selected) >= max_pool:
break
if len(selected) < max_pool:
remaining = sorted(
(row for row in normalized if row["symbol"] not in selected_symbols),
key=lambda row: (
-(row["average_daily_dollar_volume"] or 0),
-(row["market_cap"] or 0),
row["symbol"],
),
)
for row in remaining:
if len(selected) >= max_pool:
break
selected.append(row)
selected_symbols.add(row["symbol"])
output_rows: list[dict[str, Any]] = []
for rank, row in enumerate(selected, start=1):
output_rows.append(
{
"symbol": row["symbol"],
"company_name": row["company_name"],
"exchange": row["exchange"],
"sector": row["sector"],
"industry": row["industry"],
"price": row["price"],
"market_cap": row["market_cap"],
"average_daily_dollar_volume": row["average_daily_dollar_volume"],
"average_volume": row["average_volume"],
"liquidity_method": row["liquidity_method"],
"liquidity_window_days": row["liquidity_window_days"],
"liquidity_source_ids": row["liquidity_source_ids"],
"liquidity_basis_valid": True,
"market_cap_bucket": row["market_cap_bucket"],
"discovery_rank": rank,
"discovery_reason": "sector_market_cap_stratified_validated_liquidity",
}
)
sector_counts: dict[str, int] = defaultdict(int)
bucket_counts: dict[str, int] = defaultdict(int)
for row in output_rows:
sector_counts[str(row["sector"])] += 1
bucket_counts[str(row["market_cap_bucket"])] += 1
windows = [
int(row["liquidity_window_days"])
for row in selected
if row.get("liquidity_window_days") is not None
]
input_exchanges = sorted({str(row["exchange"]) for row in normalized if row.get("exchange")})
requested_range_spanned = bool(
min_market_cap <= requested_min
and max_market_cap >= requested_max
and bucket_edges[0] <= requested_min
and bucket_edges[-1] >= requested_max
)
coverage_plan = {
"method": "sector_market_cap_stratified_validated_liquidity",
"user_requested_range_spanned": requested_range_spanned,
"market_cap_buckets_cover_user_requested_range": requested_range_spanned,
"input_exchange_coverage": input_exchanges,
"required_exchange_families": ["NYSE", "NASDAQ", "NYSE AMERICAN"],
"single_band_only": bool(scope_reduced),
"coverage_plan_valid": bool(scope_valid and requested_range_spanned),
"bounded_pool_disclosed": True,
}
audit = {
"schema_version": 3,
"runtime": runtime_metadata(),
"generated_at": datetime.now(timezone.utc).isoformat(),
"valid": bool(
scope_valid and all(row.get("liquidity_basis_valid") is True for row in selected)
),
"input_row_count": len(rows),
"in_scope_row_count": len(normalized),
"selected_count": len(output_rows),
"selection_method": "sector_market_cap_stratified_validated_liquidity",
"max_pool": max_pool,
"per_sector_bucket_limit": per_cell,
"market_cap_buckets": bucket_edges,
"selected_symbols": [row["symbol"] for row in output_rows],
"selected_by_sector": dict(sorted(sector_counts.items())),
"selected_by_market_cap_bucket": dict(sorted(bucket_counts.items())),
"exclusion_counts": dict(sorted(exclusions.items())),
"liquidity_validation": {
"basis_validated": all(row.get("liquidity_basis_valid") is True for row in selected),
"minimum_window_days": min(windows) if windows else None,
"required_minimum_window_days": MIN_AVERAGE_WINDOW_DAYS,
"selected_with_validated_liquidity": len(selected),
"unverified_reason_counts": dict(sorted(liquidity_reason_counts.items())),
},
"coverage_plan": coverage_plan,
"scope": {
"user_requested_min_market_cap": requested_min,
"user_requested_max_market_cap": requested_max,
"executed_min_market_cap": float(min_market_cap),
"executed_max_market_cap": float(max_market_cap),
"scope_reduced": scope_reduced,
"scope_reduction_authorized": scope_reduction_authorized,
"scope_reduction_mode": "invalid_internal_narrowing" if scope_reduced else "none",
"scope_reduction_disclosed": (not scope_reduced) or bool(scope_reason),
"scope_reduction_reason": scope_reason,
"user_scope_evidence": _text(user_scope_evidence),
"user_requested_scope_complete": requested_range_spanned,
"scope_valid": scope_valid,
},
}
return output_rows, audit
def _canonical_line(row: Mapping[str, Any]) -> str:
return json.dumps(dict(row), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n"
def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create a bounded stratified discovery pool using validated average liquidity."
)
parser.add_argument("--input", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--source-id", action="append", required=True)
parser.add_argument(
"--min-market-cap", type=float, default=500_000_000, help="Executed pool lower bound."
)
parser.add_argument(
"--max-market-cap", type=float, default=20_000_000_000, help="Executed pool upper bound."
)
parser.add_argument("--user-requested-min-market-cap", type=float, default=500_000_000)
parser.add_argument("--user-requested-max-market-cap", type=float, default=20_000_000_000)
parser.add_argument(
"--user-scope-override-authorized",
action="store_true",
help="Required only when the user explicitly requested non-default bounds.",
)
parser.add_argument("--user-scope-evidence")
parser.add_argument("--scope-reduction-reason")
parser.add_argument(
"--allow-reduced-scope",
action="store_true",
help="Deprecated and non-authoritative; internal narrowing remains invalid.",
)
parser.add_argument("--min-price", type=float, default=5.0)
parser.add_argument("--hard-min-adv", type=float, default=1_000_000)
parser.add_argument("--max-pool", type=int, default=120)
parser.add_argument("--per-cell", type=int, default=3)
return parser.parse_args(argv)
def main(argv: Sequence[str] | None = None) -> int:
raw_argv = list(argv) if argv is not None else sys.argv[1:]
if "--version" in raw_argv:
print(json.dumps(runtime_metadata(), sort_keys=True))
return 0
args = parse_args(raw_argv)
try:
rows = _load_rows(args.input)
default_requested = (500_000_000.0, 20_000_000_000.0)
requested = (
float(args.user_requested_min_market_cap),
float(args.user_requested_max_market_cap),
)
if requested != default_requested and (
not args.user_scope_override_authorized or not _text(args.user_scope_evidence)
):
raise DiscoveryError(
"non-default user-requested bounds require --user-scope-override-authorized "
"and --user-scope-evidence"
)
pool, audit = build_pool(
rows,
min_market_cap=args.min_market_cap,
max_market_cap=args.max_market_cap,
min_price=args.min_price,
hard_min_adv=args.hard_min_adv,
max_pool=args.max_pool,
per_cell=args.per_cell,
user_requested_min_market_cap=args.user_requested_min_market_cap,
user_requested_max_market_cap=args.user_requested_max_market_cap,
scope_reduction_authorized=args.user_scope_override_authorized,
scope_reduction_reason=args.scope_reduction_reason,
user_scope_evidence=args.user_scope_evidence,
default_liquidity_source_ids=args.source_id,
)
args.output_dir.mkdir(parents=True, exist_ok=True)
pool_path = args.output_dir / "discovery-pool.jsonl"
pool_path.write_text("".join(_canonical_line(row) for row in pool), encoding="utf-8")
audit["source_ids"] = list(args.source_id)
audit["artifact_path"] = pool_path.name
audit["artifact_sha256"] = hashlib.sha256(pool_path.read_bytes()).hexdigest()
audit_path = args.output_dir / "discovery-audit.json"
audit_path.write_text(
json.dumps(audit, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
except (OSError, ValueError, json.JSONDecodeError, DiscoveryError) as exc:
print(f"ERROR: {exc}", file=sys.stderr)
return 1
print(f"wrote: {pool_path}")
print(f"wrote: {audit_path}")
if audit.get("valid") is not True or not pool:
print(
"INCOMPLETE: discovery pool has unverified liquidity, unauthorized reduced scope, or no eligible rows",
file=sys.stderr,
)
return 2
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/build_provider_prefilter_pool.py
#!/usr/bin/env python3
"""Build an audited multi-lane provider-prefilter candidate pool.
The host retrieves broad provider rows for four opportunity lanes and stores
one JSONL file per lane. This script deduplicates symbols, applies deterministic
lane-specific ranking, preserves validated liquidity/forward evidence, and emits
one candidate pool plus a generation audit consumable by screen_universe.py.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import os
import sys
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Any
try:
from screening_semantics import normalize_forward_valuation, normalize_liquidity
from skill_version import runtime_metadata
except ModuleNotFoundError:
import importlib.util
def _load(name: str):
path = Path(__file__).with_name(name + ".py")
spec = importlib.util.spec_from_file_location(name, path)
if spec is None or spec.loader is None:
raise RuntimeError(f"cannot load {name}")
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
_semantics = _load("screening_semantics")
_version = _load("skill_version")
normalize_forward_valuation = _semantics.normalize_forward_valuation
normalize_liquidity = _semantics.normalize_liquidity
runtime_metadata = _version.runtime_metadata
ALLOWED_LANES = {
"core_garp",
"high_growth_exception",
"quality_near_miss",
"cyclical_normalization",
}
# provider_exhausted_scope values that justify waiving the minimum-pool floor:
# only a fully exhausted economic candidate universe (or a caller-supplied
# full-input pool) qualifies. "estimate_seed" -- the per-symbol fallback path --
# never does: it exhausts a bounded sample, not the market.
FULL_EXHAUSTION_SCOPES = frozenset({"economic_candidate_universe", "full_input"})
def _number(value: Any) -> float | None:
if isinstance(value, bool):
return None
if isinstance(value, (int, float)) and math.isfinite(float(value)):
return float(value)
return None
def _text(value: Any) -> str | None:
return value.strip() if isinstance(value, str) and value.strip() else None
def _symbol(row: Mapping[str, Any]) -> str:
return (_text(row.get("symbol")) or "UNKNOWN").upper()
def _fcf_effective_pct(row: Mapping[str, Any]) -> float | None:
"""SBC-adjusted FCF yield when the probe derived it, else the standard yield.
``sbc_adjusted_fcf_yield_pct`` is computed by the quality probe on the
market-cap basis ((FCF - SBC) / market cap). A revenue-based SBC ratio is
never subtracted from a market-cap-based yield here: the denominators
differ and the result would be meaningless.
"""
adjusted = _number(row.get("sbc_adjusted_fcf_yield_pct"))
if adjusted is not None:
return adjusted
return _number(row.get("fcf_yield_pct"))
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
for line_no, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
if not line.strip():
continue
value = json.loads(line)
if not isinstance(value, Mapping):
raise ValueError(f"{path}:{line_no} is not a JSON object")
rows.append(dict(value))
return rows
def _canonical_jsonl(rows: Sequence[Mapping[str, Any]]) -> bytes:
return b"".join(
(
json.dumps(dict(row), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n"
).encode("utf-8")
for row in rows
)
def _lane_score(row: Mapping[str, Any], lane: str, *, analysis_as_of: str) -> float:
price = _number(row.get("price")) or _number(row.get("last"))
forward = normalize_forward_valuation(
row,
price=price,
analysis_as_of=analysis_as_of,
max_age_days=45,
reconciliation_tolerance_pct=5.0,
maximum_dispersion_pct=100.0,
maximum_fy1_horizon_days=430,
)
pe = _number(forward.get("forward_pe"))
eps_growth = _number(row.get("eps_growth_pct")) or _number(row.get("per_share_growth_pct"))
revenue_growth = _number(row.get("revenue_growth_pct"))
analysts = _number(row.get("analyst_count")) or 0.0
roic = _number(row.get("roic_pct"))
cyclicality = _number(row.get("cyclicality_score")) or 1.0
pe_component = max(-30.0, 35.0 - min(pe if pe is not None else 80.0, 80.0))
growth_component = max(-15.0, min(eps_growth if eps_growth is not None else -10.0, 50.0))
revenue_component = max(
-10.0, min(revenue_growth if revenue_growth is not None else -5.0, 20.0)
)
breadth = min(10.0, analysts * 1.5)
quality = min(10.0, max(0.0, roic or 0.0) / 2.0)
if lane == "core_garp":
score = pe_component * 1.4 + growth_component + revenue_component + breadth + quality
elif lane == "high_growth_exception":
score = (
growth_component * 1.6
+ revenue_component * 1.2
+ pe_component * 0.7
+ breadth
+ quality
)
elif lane == "quality_near_miss":
score = pe_component * 1.8 + growth_component * 0.9 + breadth + quality
else:
score = (
pe_component
+ growth_component * 1.2
+ revenue_component
+ breadth
- max(0.0, cyclicality - 3.0) * 2.0
)
fcf_yield = _number(row.get("fcf_yield_pct"))
fcf_weight = {"core_garp": 1.0, "quality_near_miss": 1.0, "high_growth_exception": 0.5}.get(
lane, 0.0
)
if fcf_yield is not None and fcf_weight:
score += max(-5.0, min(fcf_yield, 15.0)) * fcf_weight
nd_ebitda = _number(row.get("net_debt_to_ebitda"))
if nd_ebitda is not None:
score -= min(10.0, max(0.0, nd_ebitda - 2.5) * 4.0)
if eps_growth is not None and eps_growth > 100:
score -= 15.0
return round(score, 6)
def build_pool(
*,
universe_rows: Sequence[Mapping[str, Any]],
lane_rows: Mapping[str, Sequence[Mapping[str, Any]]],
analysis_as_of: str,
source_ids: Sequence[str],
per_lane: int,
max_pool: int,
minimum_pool: int,
requested_min_market_cap: float,
requested_max_market_cap: float,
provider_exhausted: bool,
provider_exhausted_scope: str | None = None,
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
if not universe_rows:
raise ValueError("universe is empty")
unknown = sorted(set(lane_rows) - ALLOWED_LANES)
if unknown:
raise ValueError(f"unknown lanes: {unknown}")
candidates: dict[str, dict[str, Any]] = {}
lane_input_counts: dict[str, int] = {}
invalid_liquidity: list[str] = []
fcf_prefilter_exclusions: list[dict[str, str]] = []
for lane in sorted(ALLOWED_LANES):
rows = [dict(row) for row in lane_rows.get(lane, [])]
lane_input_counts[lane] = len(rows)
for raw in rows:
symbol = _symbol(raw)
if symbol == "UNKNOWN":
continue
liquidity = normalize_liquidity(raw, minimum_period_days=20)
if liquidity.get("valid_for_screen") is not True:
invalid_liquidity.append(symbol)
continue
# Hard FCF-support floor: a quality-probe-resolved row with
# sub-1% (SBC-adjusted) FCF yield does not clear the prefilter
# for any lane except high_growth_exception, which is kept but
# tagged and penalized instead of dropped outright (an
# early-stage grower may legitimately run FCF-negative).
fcf_below_floor = bool(raw.get("quality_probe_resolved")) and (
(effective := _fcf_effective_pct(raw)) is not None and effective < 1.0
)
weak_fcf_flag = False
if fcf_below_floor and lane != "high_growth_exception":
fcf_prefilter_exclusions.append(
{
"symbol": symbol,
"lane": lane,
"reason": "fcf_yield_below_prefilter_floor",
}
)
continue
score = _lane_score(raw, lane, analysis_as_of=analysis_as_of)
if fcf_below_floor and lane == "high_growth_exception":
score = round(score - 10.0, 6)
weak_fcf_flag = True
item = candidates.setdefault(symbol, dict(raw))
item["symbol"] = symbol
lanes = set(item.get("provider_prefilter_lanes") or [])
lanes.add(lane)
item["provider_prefilter_lanes"] = sorted(lanes)
if weak_fcf_flag:
flags = set(item.get("provider_prefilter_flags") or [])
flags.add("weak_fcf_support")
item["provider_prefilter_flags"] = sorted(flags)
lane_scores = dict(item.get("provider_prefilter_lane_scores") or {})
lane_scores[lane] = score
item["provider_prefilter_lane_scores"] = lane_scores
item["provider_prefilter_best_score"] = max(lane_scores.values())
item.setdefault(
"average_daily_dollar_volume", liquidity.get("average_daily_dollar_volume")
)
item.setdefault("average_daily_dollar_volume_method", liquidity.get("method"))
item.setdefault("average_volume", liquidity.get("average_volume"))
item.setdefault("average_volume_period_days", liquidity.get("period_days"))
item.setdefault("liquidity_source_ids", liquidity.get("source_ids", []))
selected: list[dict[str, Any]] = []
selected_symbols: set[str] = set()
lane_added_counts = {lane: 0 for lane in ALLOWED_LANES}
def lane_rank(lane: str) -> list[dict[str, Any]]:
rows = [
row
for row in candidates.values()
if lane in (row.get("provider_prefilter_lanes") or [])
]
return sorted(
rows,
key=lambda row: (
-float((row.get("provider_prefilter_lane_scores") or {}).get(lane, -1e9)),
-float(row.get("average_daily_dollar_volume") or 0.0),
_symbol(row),
),
)
for lane in (
"core_garp",
"high_growth_exception",
"quality_near_miss",
"cyclical_normalization",
):
for row in lane_rank(lane):
if lane_added_counts[lane] >= per_lane or len(selected) >= max_pool:
break
symbol = _symbol(row)
if symbol in selected_symbols:
continue
selected.append(dict(row))
selected_symbols.add(symbol)
lane_added_counts[lane] += 1
remaining = sorted(
candidates.values(),
key=lambda row: (
-float(row.get("provider_prefilter_best_score") or -1e9),
-float(row.get("average_daily_dollar_volume") or 0.0),
_symbol(row),
),
)
for row in remaining:
if len(selected) >= max_pool:
break
symbol = _symbol(row)
if symbol not in selected_symbols:
selected.append(dict(row))
selected_symbols.add(symbol)
selected.sort(
key=lambda row: (-float(row.get("provider_prefilter_best_score") or -1e9), _symbol(row))
)
# The row-count floor may be waived only when the WHOLE economic candidate
# universe was exhausted. Exhausting a bounded slice of it (the estimate
# seed on the per-symbol fallback path) does not make a thin pool
# adequate. Fail closed: a caller that does not state its exhaustion scope
# gets no waiver either — an unstated scope proves nothing.
allow_small_pool = provider_exhausted and provider_exhausted_scope in FULL_EXHAUSTION_SCOPES
pool_adequate = len(selected) >= minimum_pool or allow_small_pool
# A symbol can legitimately qualify for multiple opportunity lanes. Audit
# represented lanes from final memberships rather than only the loop that
# first inserted the unique symbol; otherwise an overlapping core/high-growth
# name can make a valid four-lane pool appear to cover only one lane.
lane_selected_counts = {
lane: sum(1 for row in selected if lane in (row.get("provider_prefilter_lanes") or []))
for lane in ALLOWED_LANES
}
lane_coverage_count = sum(1 for lane, count in lane_selected_counts.items() if count > 0)
# Invalid provider rows are excluded before selection. They are disclosed but
# do not poison an otherwise fully validated candidate pool.
selected_liquidity_valid = all(
normalize_liquidity(row, minimum_period_days=20).get("valid_for_screen") is True
for row in selected
)
lane_floor_waived = allow_small_pool and lane_coverage_count < 3
valid = (
pool_adequate
and (lane_coverage_count >= 3 or lane_floor_waived)
and selected_liquidity_valid
)
fcf_prefilter_excluded_symbols = sorted({item["symbol"] for item in fcf_prefilter_exclusions})
audit = {
"runtime": runtime_metadata(),
"valid": valid,
"selection_method": "provider_prefilter",
"analysis_as_of": analysis_as_of,
"input_row_count": len(universe_rows),
"selected_count": len(selected),
"selected_symbols": sorted(selected_symbols),
"source_ids": list(dict.fromkeys(str(value) for value in source_ids if str(value).strip())),
"lane_input_counts": lane_input_counts,
"lane_selected_counts": lane_selected_counts,
"lane_coverage_count": lane_coverage_count,
"minimum_required_lanes": 3,
"lane_floor_waived": lane_floor_waived,
"per_lane": per_lane,
"max_pool": max_pool,
"minimum_pool": minimum_pool,
"provider_exhausted": provider_exhausted,
"provider_exhausted_scope": provider_exhausted_scope,
"pool_floor_waived": allow_small_pool and len(selected) < minimum_pool,
"pool_adequate": pool_adequate,
"fcf_prefilter_excluded_symbols": fcf_prefilter_excluded_symbols,
"fcf_prefilter_excluded_count": len(fcf_prefilter_excluded_symbols),
"fcf_prefilter_exclusions": fcf_prefilter_exclusions,
"invalid_liquidity_symbols": sorted(set(invalid_liquidity)),
"invalid_liquidity_excluded_count": len(set(invalid_liquidity)),
"liquidity_validation": {
"basis_validated": selected_liquidity_valid,
"minimum_window_days": 20,
"invalid_input_rows_excluded": len(set(invalid_liquidity)),
},
"coverage_plan": {
"coverage_plan_valid": True,
"user_requested_range_spanned": True,
"market_cap_buckets_cover_user_requested_range": True,
"single_band_only": False,
"multi_lane_provider_prefilter": True,
},
"scope": {
"user_requested_min_market_cap": requested_min_market_cap,
"user_requested_max_market_cap": requested_max_market_cap,
"user_requested_scope_complete": True,
"scope_valid": True,
},
}
return selected, audit
def parse_lane(value: str) -> tuple[str, Path]:
if "=" not in value:
raise argparse.ArgumentTypeError("--lane must be LANE=PATH")
lane, raw = value.split("=", 1)
if lane not in ALLOWED_LANES:
raise argparse.ArgumentTypeError(f"invalid lane {lane!r}")
return lane, Path(raw)
def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--universe", type=Path, required=True)
parser.add_argument("--lane", action="append", type=parse_lane, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--analysis-as-of", required=True)
parser.add_argument("--source-id", action="append", required=True)
parser.add_argument("--per-lane", type=int, default=15)
parser.add_argument("--max-pool", type=int, default=60)
parser.add_argument("--minimum-pool", type=int, default=30)
parser.add_argument("--requested-min-market-cap", type=float, default=500_000_000)
parser.add_argument("--requested-max-market-cap", type=float, default=20_000_000_000)
parser.add_argument("--provider-exhausted", action="store_true")
parser.add_argument("--provider-exhausted-scope")
parser.add_argument("--version", action="store_true")
return parser.parse_args(argv)
def main(argv: Sequence[str] | None = None) -> int:
raw_argv = list(sys.argv[1:] if argv is None else argv)
if "--version" in raw_argv:
print(json.dumps(runtime_metadata(), sort_keys=True))
return 0
args = parse_args(raw_argv)
if args.version:
print(json.dumps(runtime_metadata(), sort_keys=True))
return 0
try:
universe_rows = _read_jsonl(args.universe)
lanes: dict[str, list[dict[str, Any]]] = {lane: [] for lane in ALLOWED_LANES}
for lane, path in args.lane:
lanes[lane].extend(_read_jsonl(path))
pool, audit = build_pool(
universe_rows=universe_rows,
lane_rows=lanes,
analysis_as_of=args.analysis_as_of,
source_ids=args.source_id,
per_lane=args.per_lane,
max_pool=args.max_pool,
minimum_pool=args.minimum_pool,
requested_min_market_cap=args.requested_min_market_cap,
requested_max_market_cap=args.requested_max_market_cap,
provider_exhausted=args.provider_exhausted,
provider_exhausted_scope=args.provider_exhausted_scope,
)
except (OSError, ValueError, json.JSONDecodeError) as exc:
print(f"ERROR: {exc}")
return 2
args.output_dir.mkdir(parents=True, exist_ok=True)
pool_path = args.output_dir / "provider-prefilter-pool.jsonl"
audit_path = args.output_dir / "provider-prefilter-audit.json"
data = _canonical_jsonl(pool)
pool_path.write_bytes(data)
audit["artifact_path"] = pool_path.name
audit["artifact_sha256"] = hashlib.sha256(data).hexdigest()
temp = audit_path.with_suffix(".tmp")
temp.write_text(json.dumps(audit, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
os.replace(temp, audit_path)
print(f"wrote: {pool_path}")
print(f"wrote: {audit_path}")
return 0 if audit["valid"] else 2
if __name__ == "__main__":
raise SystemExit(main())
scripts/bundle_run_artifacts.py
#!/usr/bin/env python3
"""Create a deterministic, self-contained audit ZIP for a completed screen."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import sys
from collections.abc import Sequence
from pathlib import Path
from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo
try:
from prepublish_audit import AuditError, _read_json, audit_publication
from skill_version import runtime_metadata
except ModuleNotFoundError:
import importlib.util
def _load(name: str):
path = Path(__file__).with_name(name + ".py")
spec = importlib.util.spec_from_file_location(name, path)
if spec is None or spec.loader is None:
raise RuntimeError(f"cannot load {name}")
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
_audit = _load("prepublish_audit")
_version = _load("skill_version")
AuditError = _audit.AuditError
_read_json = _audit._read_json
audit_publication = _audit.audit_publication
runtime_metadata = _version.runtime_metadata
FIXED_DATE = (2026, 1, 1, 0, 0, 0)
EXCLUDED_PARTS = {"__pycache__", ".pytest_cache", ".git"}
SECRET_TERMS = {".env", "credentials", "api_key", "apikey", "access_token", "secret"}
def _include(path: Path, run_dir: Path) -> bool:
rel = path.relative_to(run_dir)
if any(part in EXCLUDED_PARTS for part in rel.parts):
return False
lower = rel.as_posix().lower()
if any(term in lower for term in SECRET_TERMS):
raise ValueError(f"refusing to bundle possible secret file: {rel}")
return path.is_file() and not path.name.endswith((".tmp", ".pyc"))
def _zip_info(name: str, mode: int = 0o644) -> ZipInfo:
info = ZipInfo(name, FIXED_DATE)
info.compress_type = ZIP_DEFLATED
info.external_attr = mode << 16
return info
def build_bundle(run_dir: Path, report_json: Path, report_md: Path, output: Path) -> dict:
run_dir = run_dir.resolve()
report = _read_json(report_json)
markdown = report_md.read_text(encoding="utf-8")
audit = audit_publication(report, report_markdown=markdown, artifact_root=run_dir)
if not audit["valid"]:
raise AuditError("prepublication audit failed: " + "; ".join(audit["errors"]))
files = sorted(
[path for path in run_dir.rglob("*") if _include(path, run_dir)],
key=lambda path: path.relative_to(run_dir).as_posix(),
)
manifest = {
"runtime": runtime_metadata(),
"run_dir_name": run_dir.name,
"files": [
{
"path": path.relative_to(run_dir).as_posix(),
"sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
"size": path.stat().st_size,
}
for path in files
],
"prepublication_audit": audit,
}
manifest_bytes = (json.dumps(manifest, ensure_ascii=False, indent=2) + "\n").encode("utf-8")
output.parent.mkdir(parents=True, exist_ok=True)
temp = output.with_suffix(output.suffix + ".tmp")
try:
with ZipFile(temp, "w") as archive:
for path in files:
rel = path.relative_to(run_dir).as_posix()
mode = path.stat().st_mode & 0o777
archive.writestr(_zip_info(rel, mode), path.read_bytes())
archive.writestr(_zip_info("BUNDLE_MANIFEST.json"), manifest_bytes)
os.replace(temp, output)
finally:
if temp.exists():
temp.unlink()
return {
"runtime": runtime_metadata(),
"output": str(output),
"file_count": len(files),
"sha256": hashlib.sha256(output.read_bytes()).hexdigest(),
}
def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--run-dir", type=Path, required=True)
parser.add_argument("--report-json", type=Path, required=True)
parser.add_argument("--report-md", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--version", action="store_true")
return parser.parse_args(argv)
def main(argv: Sequence[str] | None = None) -> int:
raw_argv = list(sys.argv[1:] if argv is None else argv)
if "--version" in raw_argv:
print(json.dumps(runtime_metadata(), sort_keys=True))
return 0
args = parse_args(raw_argv)
if args.version:
print(json.dumps(runtime_metadata(), sort_keys=True))
return 0
try:
result = build_bundle(args.run_dir, args.report_json, args.report_md, args.output)
except (AuditError, OSError, ValueError) as exc:
print(f"ERROR: {exc}")
return 2
print(json.dumps(result, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/coverage_semantics.py
"""Single source of truth for ranking-scope and coverage semantics.
Discovery (``run_pipeline``) and final evaluation (``evaluate_candidates``)
previously implemented the tri-state ranking scope twice, with subtly
different field access. Both now call into this module so the semantics
cannot drift when new acquisition modes (e.g. the v3.7 ``sharded_snapshot``)
are added.
The contract (rounds 6-8 of external review):
- ``final_marketwide``: estimate acquisition was attempted for EVERY
listing-universe symbol (exact counts), the economic-scope completeness
verdict agrees, and no enrichment queue is unresolved. Only then may an
output be read as a market-wide ranking.
- ``final_scoped``: a bounded, explicitly audited subset was fully
processed; conclusions bind only to that subset, never to the market.
- ``diagnostic``: fail-closed bucket — an unresolved queue, an empty
universe, a missing/zero attempted count, or an attempted count that
exceeds the universe (a configured LIMIT is not an attempt count).
"""
from __future__ import annotations
from collections.abc import Mapping
from typing import Any
RANKING_SCOPES = ("final_marketwide", "final_scoped", "diagnostic")
_COVERAGE_COUNT_FIELDS = (
"listing_universe_count",
"economic_attempt_count",
"economically_evaluable_count",
"quality_probe_count",
"deep_dive_count",
)
def _as_int(value: Any) -> int | None:
if isinstance(value, bool):
return None
if isinstance(value, int):
return value
if isinstance(value, float) and value.is_integer():
return int(value)
if isinstance(value, str):
try:
return int(value.strip())
except ValueError:
return None
return None
def _as_mapping(value: Any) -> Mapping[str, Any]:
return value if isinstance(value, Mapping) else {}
def classify_ranking_scope(
*,
economic_attempt_count: int,
listing_universe_count: int,
economic_scope_complete: bool,
unresolved_queue_count: int,
) -> str:
"""Classify how far a run's conclusion may be generalized.
Listing enumeration completeness never makes a market-wide ranking:
only attempting economic (estimate) acquisition for EVERY listed symbol
does. Empty or impossible work fails closed to ``diagnostic``.
"""
if unresolved_queue_count > 0:
return "diagnostic"
if (
listing_universe_count <= 0
or economic_attempt_count <= 0
or economic_attempt_count > listing_universe_count
):
return "diagnostic"
if economic_scope_complete and economic_attempt_count >= listing_universe_count:
return "final_marketwide"
return "final_scoped"
def build_coverage_block(
*,
ranking_scope: str,
listing_universe_count: int,
economic_attempt_count: int,
economically_evaluable_count: int,
quality_probe_count: int,
deep_dive_count: int,
) -> dict[str, Any]:
"""Assemble the canonical per-stage coverage block (counts + percentages)."""
def _pct(count: int) -> float:
if listing_universe_count <= 0:
return 0.0
return round(count / listing_universe_count * 100.0, 6)
return {
"ranking_scope": ranking_scope,
"listing_universe_count": listing_universe_count,
"economic_attempt_count": economic_attempt_count,
"economic_attempt_coverage_pct": _pct(economic_attempt_count),
"economically_evaluable_count": economically_evaluable_count,
"economically_evaluable_coverage_pct": _pct(economically_evaluable_count),
"quality_probe_count": quality_probe_count,
"quality_probe_coverage_pct": _pct(quality_probe_count),
"deep_dive_count": deep_dive_count,
"deep_dive_coverage_pct": _pct(deep_dive_count),
}
def validate_coverage_block(block: Mapping[str, Any]) -> list[str]:
"""Return human-readable problems with a coverage block (empty = valid)."""
problems: list[str] = []
scope = block.get("ranking_scope")
if scope not in RANKING_SCOPES:
problems.append(f"ranking_scope {scope!r} is not one of {RANKING_SCOPES}")
counts: dict[str, int] = {}
for field in _COVERAGE_COUNT_FIELDS:
value = _as_int(block.get(field))
if value is None or value < 0:
problems.append(f"{field} is missing or not a non-negative integer")
else:
counts[field] = value
universe = counts.get("listing_universe_count")
if universe is not None:
for field in _COVERAGE_COUNT_FIELDS[1:]:
value = counts.get(field)
if value is not None and value > universe:
problems.append(f"{field} ({value}) exceeds listing_universe_count ({universe})")
attempted = counts.get("economic_attempt_count")
evaluable = counts.get("economically_evaluable_count")
if attempted is not None and evaluable is not None and evaluable > attempted:
problems.append(
f"economically_evaluable_count ({evaluable}) exceeds "
f"economic_attempt_count ({attempted})"
)
return problems
def derive_ranking_scope_from_audit(
audit: Mapping[str, Any], *, deep_dive_count: int
) -> dict[str, Any]:
"""Derive the coverage block from a run's screening audit.
Copies discovery's ACTUAL ``economic_attempt_count`` — never the
configured seed LIMIT, which can exceed a narrow universe and fabricate
>100% coverage. Missing or impossible counts fail closed to
``diagnostic``.
"""
audit = _as_mapping(audit)
universe_total = _as_int(_as_mapping(audit.get("universe")).get("row_count")) or 0
generation = _as_mapping(_as_mapping(audit.get("candidate_pool")).get("generation_audit"))
bulk = _as_mapping(generation.get("bulk_estimate_audit"))
covered = _as_int(bulk.get("covered_symbol_count")) or 0
mode = str(generation.get("estimate_acquisition_mode") or "")
if mode == "analyst_estimates_bulk" and covered:
attempted: int | None = covered
else:
attempted = _as_int(generation.get("economic_attempt_count"))
evaluable = _as_int(generation.get("economically_evaluable_count")) or 0
probe_raw = _as_mapping(generation.get("quality_probe")).get("attempted")
if isinstance(probe_raw, list):
probe_count = len(probe_raw)
else:
probe_count = _as_int(probe_raw) or 0
queue_count = _as_int(_as_mapping(audit.get("enrichment")).get("unresolved_count")) or 0
snapshot_verification = _as_mapping(generation.get("snapshot_verification"))
snapshot_digest = snapshot_verification.get("snapshot_verification_digest")
verified_sharded_snapshot = bool(
mode == "sharded_snapshot"
and snapshot_verification.get("ready_for_screening") is True
and isinstance(snapshot_digest, str)
and len(snapshot_digest) == 64
and snapshot_verification.get("classification_matches_universe") is True
and _as_int(snapshot_verification.get("classified_total")) == universe_total
and attempted == universe_total
)
if (
queue_count > 0
or universe_total <= 0
or attempted is None
or attempted <= 0
or attempted > universe_total
or (mode == "sharded_snapshot" and not verified_sharded_snapshot)
):
scope = "diagnostic"
elif (mode == "analyst_estimates_bulk" and covered >= universe_total) or (
mode == "sharded_snapshot" and verified_sharded_snapshot
):
scope = "final_marketwide"
else:
scope = "final_scoped"
return build_coverage_block(
ranking_scope=scope,
listing_universe_count=universe_total,
economic_attempt_count=attempted or 0,
economically_evaluable_count=evaluable,
quality_probe_count=probe_count,
deep_dive_count=deep_dive_count,
)
scripts/estimate_snapshot.py
"""Persistent full-universe estimate snapshot (v3.7 sharded collection).
A snapshot freezes the listing universe at creation time and then accumulates
per-symbol estimate rows shard by shard across multiple runs/days, within each
run's API budget. ``screen-full-snapshot`` (a later stage) may only run once
every shard is complete and every frozen symbol is classified — that is what
finally justifies ``ranking_scope: final_marketwide``.
Layout inside ``--snapshot-dir``::
snapshot-manifest.json # id, frozen universe hash/count, shard states
universe.jsonl # the frozen listing universe (normalized rows)
shard-<i>.jsonl # per-symbol records: listing + estimates + class
Design invariants (round-7 review additions to issue #345):
- The universe is FROZEN by ``snapshot_id``; new listings/delistings go to
the next snapshot, never mixed in.
- Every shard records its own ``as_of``; the spread between the oldest and
newest shard is bounded by ``max_shard_age_spread_days`` before screening.
- Classification counts must sum exactly to the frozen universe count.
"""
from __future__ import annotations
import hashlib
import json
from collections.abc import Mapping, Sequence
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any
MANIFEST_NAME = "snapshot-manifest.json"
UNIVERSE_NAME = "universe.jsonl"
ENUMERATION_AUDIT_NAME = "listing-enumeration-audit.json"
SNAPSHOT_SCHEMA_VERSION = 2
# Per-symbol classification buckets. Precedence when several apply:
# excluded > unit_mismatch > no_estimates > negative_eps > evaluable.
# unit_mismatch dominates the estimate-derived buckets because a row whose
# listing/statement units cannot be reconciled is unusable even WITH
# estimates (fail closed, round-8 semantics).
CLASSIFICATIONS = ("evaluable", "no_estimates", "negative_eps", "unit_mismatch", "excluded")
def stable_shard(symbol: str, shard_count: int) -> int:
"""Deterministic shard assignment: the same symbol always lands together."""
if shard_count <= 0:
raise ValueError("shard_count must be positive")
digest = hashlib.sha256(symbol.strip().upper().encode("utf-8")).hexdigest()
return int(digest, 16) % shard_count
def _universe_sha256(rows: Sequence[Mapping[str, Any]]) -> str:
hasher = hashlib.sha256()
for row in rows:
hasher.update(
json.dumps(row, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode(
"utf-8"
)
)
hasher.update(b"\n")
return hasher.hexdigest()
def create_snapshot(
snapshot_dir: Path,
universe_rows: Sequence[Mapping[str, Any]],
*,
shard_count: int,
as_of: datetime,
enumeration_audit: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
"""Freeze the universe and initialize an empty manifest."""
if shard_count <= 0:
raise ValueError("shard_count must be positive")
if not universe_rows:
raise ValueError("cannot freeze an empty universe")
snapshot_dir.mkdir(parents=True, exist_ok=True)
if (snapshot_dir / MANIFEST_NAME).exists():
raise ValueError(f"snapshot already exists at {snapshot_dir}")
ordered = sorted(universe_rows, key=lambda row: str(row.get("symbol") or ""))
with (snapshot_dir / UNIVERSE_NAME).open("w", encoding="utf-8") as handle:
for row in ordered:
handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n")
sha = _universe_sha256(ordered)
enumeration_sha256: str | None = None
if enumeration_audit is not None:
enumeration_bytes = (
json.dumps(dict(enumeration_audit), indent=2, ensure_ascii=False) + "\n"
).encode("utf-8")
(snapshot_dir / ENUMERATION_AUDIT_NAME).write_bytes(enumeration_bytes)
enumeration_sha256 = hashlib.sha256(enumeration_bytes).hexdigest()
manifest = {
"schema_version": SNAPSHOT_SCHEMA_VERSION,
"snapshot_id": (
f"snap-{as_of.astimezone(timezone.utc).strftime('%Y%m%d-%H%M%S')}-{sha[:12]}"
),
"created_at": as_of.astimezone(timezone.utc).isoformat(),
"universe_sha256": sha,
"universe_count": len(ordered),
"listing_enumeration_audit_sha256": enumeration_sha256,
"normalization_policy": "current_only_per_retrieval",
"shard_count": shard_count,
"shards": {
str(index): {"status": "pending", "attempted": 0, "classified": {}}
for index in range(shard_count)
},
}
write_manifest(snapshot_dir, manifest)
return manifest
def load_manifest(snapshot_dir: Path) -> dict[str, Any]:
return json.loads((snapshot_dir / MANIFEST_NAME).read_text(encoding="utf-8"))
def write_manifest(snapshot_dir: Path, manifest: Mapping[str, Any]) -> None:
path = snapshot_dir / MANIFEST_NAME
tmp = path.with_suffix(".json.tmp")
tmp.write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
tmp.replace(path)
def load_universe(snapshot_dir: Path) -> list[dict[str, Any]]:
rows = [
json.loads(line)
for line in (snapshot_dir / UNIVERSE_NAME).read_text(encoding="utf-8").splitlines()
if line.strip()
]
return rows
def load_verified_universe(snapshot_dir: Path, manifest: Mapping[str, Any]) -> list[dict[str, Any]]:
"""Load the frozen universe and verify it against the manifest.
Refuses — before any API access or shard append — when the manifest
schema is unknown, the row count differs, symbols are missing/duplicated,
or the canonical SHA-256 no longer matches: a silently swapped
``universe.jsonl`` would otherwise break the freeze guarantee while
keeping ``classification_matches_universe`` true.
"""
if int(manifest.get("schema_version") or 0) != SNAPSHOT_SCHEMA_VERSION:
raise ValueError(
f"unsupported snapshot manifest schema_version {manifest.get('schema_version')!r}"
)
rows = load_universe(snapshot_dir)
expected_count = int(manifest.get("universe_count") or -1)
if len(rows) != expected_count:
raise ValueError(
f"universe.jsonl has {len(rows)} rows but the manifest froze {expected_count}"
)
symbols = [str(row.get("symbol") or "") for row in rows]
if "" in symbols or len(set(symbols)) != len(symbols):
raise ValueError("universe.jsonl symbols are missing or not unique")
sha = _universe_sha256(rows)
if sha != manifest.get("universe_sha256"):
raise ValueError(
"universe.jsonl does not match the frozen universe_sha256 — the snapshot "
"universe may have been modified; create a new snapshot instead"
)
return rows
def load_verified_enumeration(
snapshot_dir: Path, manifest: Mapping[str, Any]
) -> tuple[dict[str, Any], str]:
"""Load and semantically verify the listing-enumeration proof bound by the manifest."""
path = snapshot_dir / ENUMERATION_AUDIT_NAME
data = path.read_bytes()
actual_sha = hashlib.sha256(data).hexdigest()
if actual_sha != manifest.get("listing_enumeration_audit_sha256"):
raise ValueError("listing enumeration audit SHA-256 does not match the manifest")
audit = json.loads(data)
if not isinstance(audit, Mapping):
raise ValueError("listing enumeration audit is not a JSON object")
audit = dict(audit)
requested = [str(value).upper() for value in (audit.get("requested_exchanges") or [])]
retrieved = [str(value).upper() for value in (audit.get("retrieved_exchanges") or [])]
bands = [dict(row) for row in (audit.get("bands") or []) if isinstance(row, Mapping)]
band_exchanges = {str(row.get("exchange") or "").upper() for row in bands}
try:
minimum = float(audit["requested_min_market_cap"])
maximum = float(audit["requested_max_market_cap"])
min_price = float(audit["min_price"])
except (KeyError, TypeError, ValueError) as exc:
raise ValueError("listing enumeration audit lacks an explicit numeric scope") from exc
bands_well_formed = True
ranges_by_exchange: dict[str, list[tuple[float, float]]] = {
exchange: [] for exchange in requested
}
for row in bands:
exchange = str(row.get("exchange") or "").upper()
try:
band_minimum = float(row["min_market_cap"])
band_maximum = float(row["max_market_cap"])
except (KeyError, TypeError, ValueError):
bands_well_formed = False
continue
if (
exchange not in ranges_by_exchange
or band_minimum >= band_maximum
or row.get("provider_exhausted") is not True
):
bands_well_formed = False
continue
ranges_by_exchange[exchange].append((band_minimum, band_maximum))
def covers_requested_range(ranges: Sequence[tuple[float, float]]) -> bool:
cursor = minimum
for band_minimum, band_maximum in sorted(ranges):
if band_maximum < minimum or band_minimum > maximum:
continue
clipped_minimum = max(band_minimum, minimum)
clipped_maximum = min(band_maximum, maximum)
if clipped_minimum > cursor:
return False
cursor = max(cursor, clipped_maximum)
return cursor >= maximum
bands_cover_scope = bands_well_formed and all(
covers_requested_range(ranges_by_exchange[exchange]) for exchange in requested
)
valid = (
audit.get("method") == "adaptive_market_cap_bands"
and audit.get("retrieval_scope_explicit") is True
and audit.get("pagination_exhausted") is True
and audit.get("enumeration_verified") is True
and int(audit.get("saturated_leaf_count") or 0) == 0
and int(audit.get("row_count") or -1) == int(manifest.get("universe_count") or -2)
and bool(requested)
and sorted(set(requested)) == sorted(set(retrieved))
and set(requested).issubset(band_exchanges)
and bool(bands)
and bands_cover_scope
and minimum < maximum
and min_price >= 0
)
if not valid:
raise ValueError("listing enumeration audit does not prove the complete requested scope")
return audit, actual_sha
def shard_path(snapshot_dir: Path, shard_index: int) -> Path:
return snapshot_dir / f"shard-{shard_index}.jsonl"
def load_shard_rows(snapshot_dir: Path, shard_index: int) -> list[dict[str, Any]]:
path = shard_path(snapshot_dir, shard_index)
if not path.exists():
return []
return [
json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()
]
def append_shard_rows(
snapshot_dir: Path, shard_index: int, rows: Sequence[Mapping[str, Any]]
) -> None:
with shard_path(snapshot_dir, shard_index).open("a", encoding="utf-8") as handle:
for row in rows:
handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n")
def classify_symbol(
listing: Mapping[str, Any],
normalized: Mapping[str, Any],
*,
requires_unit_reconciliation: Any,
minimum_plausible_forward_pe: float = 2.0,
) -> str:
"""Classify one frozen-universe symbol after an estimate-acquisition attempt."""
if listing.get("is_actively_trading") is False or listing.get("is_common_stock") is False:
return "excluded"
if requires_unit_reconciliation(listing):
return "unit_mismatch"
forward_pe = normalized.get("forward_pe")
if isinstance(forward_pe, (int, float)) and 0 < forward_pe < minimum_plausible_forward_pe:
return "unit_mismatch"
# The normalizer NULLS a non-positive FY1 (reason non_positive_fy1_eps)
# before it ever reaches fy1_eps, so the raw candidate / reason list is
# the only way this bucket is reachable from real pipeline output.
reasons = normalized.get("estimate_normalization_reasons") or []
raw_candidate = normalized.get("raw_forward_candidate")
raw_eps = raw_candidate.get("eps") if isinstance(raw_candidate, Mapping) else None
if "non_positive_fy1_eps" in reasons or (isinstance(raw_eps, (int, float)) and raw_eps <= 0):
return "negative_eps"
periods = normalized.get("estimate_periods")
if not periods:
return "no_estimates"
fy1 = normalized.get("fy1_eps")
if isinstance(fy1, (int, float)) and fy1 <= 0:
return "negative_eps"
if fy1 is None:
return "no_estimates"
return "evaluable"
def update_shard(
snapshot_dir: Path,
manifest: dict[str, Any],
shard_index: int,
*,
status: str,
as_of: str,
calls_used: int,
classified: Mapping[str, int],
attempted: int,
expected: int,
fetch_failed: int = 0,
oldest_retrieved_at: str | None = None,
newest_retrieved_at: str | None = None,
retrieval_time_unknown: int = 0,
oldest_normalization_as_of: str | None = None,
newest_normalization_as_of: str | None = None,
normalization_time_unknown: int = 0,
) -> dict[str, Any]:
if status not in {"pending", "partial", "complete"}:
raise ValueError(f"unknown shard status {status!r}")
path = shard_path(snapshot_dir, shard_index)
shard_sha256 = hashlib.sha256(path.read_bytes()).hexdigest() if path.exists() else None
entry = {
"status": status,
"as_of": as_of,
"attempted": attempted,
"expected": expected,
"calls_used": calls_used,
"fetch_failed": fetch_failed,
"oldest_retrieved_at": oldest_retrieved_at,
"newest_retrieved_at": newest_retrieved_at,
"retrieval_time_unknown": retrieval_time_unknown,
"oldest_normalization_as_of": oldest_normalization_as_of,
"newest_normalization_as_of": newest_normalization_as_of,
"normalization_time_unknown": normalization_time_unknown,
"shard_sha256": shard_sha256,
"classified": dict(sorted(classified.items())),
}
manifest["shards"][str(shard_index)] = entry
write_manifest(snapshot_dir, manifest)
return manifest
def snapshot_status(manifest: Mapping[str, Any]) -> dict[str, Any]:
"""Aggregate readiness: complete shards, totals, and the exact-count invariant."""
shards = manifest.get("shards") or {}
totals: dict[str, int] = {name: 0 for name in CLASSIFICATIONS}
complete = 0
attempted_total = 0
fetch_failed_total = 0
retrieval_time_unknown_total = 0
normalization_time_unknown_total = 0
oldest_retrieved: str | None = None
newest_retrieved: str | None = None
for entry in shards.values():
if entry.get("status") == "complete":
complete += 1
attempted_total += int(entry.get("attempted") or 0)
fetch_failed_total += int(entry.get("fetch_failed") or 0)
retrieval_time_unknown_total += int(entry.get("retrieval_time_unknown") or 0)
normalization_time_unknown_total += int(entry.get("normalization_time_unknown") or 0)
oldest = entry.get("oldest_retrieved_at")
if (
isinstance(oldest, str)
and oldest
and (oldest_retrieved is None or oldest < oldest_retrieved)
):
oldest_retrieved = oldest
newest = entry.get("newest_retrieved_at")
if (
isinstance(newest, str)
and newest
and (newest_retrieved is None or newest > newest_retrieved)
):
newest_retrieved = newest
for name, count in (entry.get("classified") or {}).items():
totals[name] = totals.get(name, 0) + int(count or 0)
universe_count = int(manifest.get("universe_count") or 0)
classified_total = sum(totals.values())
all_shards_complete = complete == int(manifest.get("shard_count") or 0)
classification_matches_universe = classified_total == universe_count
# Round-4 review: collection completeness and freshness provenance are
# SEPARATE readiness axes — a shard full of unknown-provenance rows must
# never look screenable, and staleness is judged from the ACTUAL
# oldest/newest retrieval stamps, not the operator-supplied as_of.
freshness_provenance_complete = (
retrieval_time_unknown_total == 0
and normalization_time_unknown_total == 0
and (
attempted_total == 0 or (oldest_retrieved is not None and newest_retrieved is not None)
)
)
return {
"snapshot_id": manifest.get("snapshot_id"),
"shard_count": int(manifest.get("shard_count") or 0),
"complete_shards": complete,
"all_shards_complete": all_shards_complete,
"attempted_total": attempted_total,
"fetch_failed_total": fetch_failed_total,
"retrieval_time_unknown_total": retrieval_time_unknown_total,
"normalization_time_unknown_total": normalization_time_unknown_total,
"oldest_retrieved_at": oldest_retrieved,
"newest_retrieved_at": newest_retrieved,
"classified_totals": totals,
"classified_total": classified_total,
"universe_count": universe_count,
# The round-7 invariant: every frozen symbol is classified, exactly.
"classification_matches_universe": classification_matches_universe,
"freshness_provenance_complete": freshness_provenance_complete,
# Manifest-level aggregation ONLY: says nothing about the shard
# files' actual contents or staleness. ready_for_screening comes
# exclusively from verify_snapshot(), which re-reads and verifies
# every shard row against the frozen universe.
"collection_ready": (
all_shards_complete
and classification_matches_universe
and fetch_failed_total == 0
and freshness_provenance_complete
),
}
def _rows_from_bytes(data: bytes, *, path: Path) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
for line_no, raw in enumerate(data.decode("utf-8").splitlines(), 1):
if not raw.strip():
continue
value = json.loads(raw)
if not isinstance(value, Mapping):
raise ValueError(f"{path}:{line_no} is not a JSON object")
rows.append(dict(value))
return rows
def load_verified_snapshot(
snapshot_dir: Path,
*,
screening_as_of: datetime,
max_staleness_days: float,
clock_skew_seconds: float = 300.0,
) -> dict[str, Any]:
"""Return a verification verdict and the rows read by that verification.
``snapshot_status`` trusts the manifest's own counters; a tampered,
truncated or swapped ``shard-*.jsonl`` would sail through it. This
function reads each artifact exactly once, verifies it symbol by symbol,
and returns only those in-memory rows. The consumer therefore cannot
verify one set of bytes and later screen a different set of shard bytes.
The returned ``verification_digest`` binds the exact manifest bytes, the
canonical frozen-universe SHA-256, and every actual shard SHA-256.
Verification checks:
- the frozen universe itself (schema/count/uniqueness/SHA-256);
- each shard file's SHA-256 against the manifest;
- no duplicate symbols within or across shards;
- every symbol belongs to its shard per ``stable_shard``;
- every symbol is in the frozen universe, and the union of shard
symbols covers the frozen universe EXACTLY;
- every classification is an allowed value, and per-shard counts match
the manifest;
- retrieval stamps re-aggregated from the ROWS themselves and bounded on
BOTH sides: ``screening_as_of - max_staleness_days <= oldest`` and
``newest <= screening_as_of + clock_skew_seconds`` — an historical
``screening_as_of`` must never see data fetched after it (look-ahead).
``ready_for_screening`` is true ONLY when all of the above hold and the
manifest-level ``collection_ready`` aggregation agrees.
"""
if max_staleness_days <= 0:
raise ValueError("max_staleness_days must be positive")
manifest_path = snapshot_dir / MANIFEST_NAME
manifest_bytes = manifest_path.read_bytes()
manifest = json.loads(manifest_bytes)
if not isinstance(manifest, Mapping):
raise ValueError(f"{manifest_path} is not a JSON object")
manifest = dict(manifest)
if int(manifest.get("schema_version") or 0) != SNAPSHOT_SCHEMA_VERSION:
raise ValueError(
f"unsupported snapshot manifest schema_version {manifest.get('schema_version')!r}"
)
problems: list[str] = []
try:
universe_path = snapshot_dir / UNIVERSE_NAME
universe = _rows_from_bytes(universe_path.read_bytes(), path=universe_path)
expected_count = int(manifest.get("universe_count") or -1)
if len(universe) != expected_count:
raise ValueError(
f"universe.jsonl has {len(universe)} rows but the manifest froze {expected_count}"
)
universe_symbols = [str(row.get("symbol") or "") for row in universe]
if "" in universe_symbols or len(set(universe_symbols)) != len(universe_symbols):
raise ValueError("universe.jsonl symbols are missing or not unique")
universe_sha = _universe_sha256(universe)
if universe_sha != manifest.get("universe_sha256"):
raise ValueError(
"universe.jsonl does not match the frozen universe_sha256 — the snapshot "
"universe may have been modified; create a new snapshot instead"
)
except (OSError, ValueError, UnicodeDecodeError, json.JSONDecodeError) as exc:
universe = []
universe_sha = None
problems.append(str(exc))
expected_symbols = {str(row.get("symbol") or "") for row in universe}
shard_count = int(manifest.get("shard_count") or 0)
seen: dict[str, int] = {}
row_stamps: list[datetime] = []
unknown_stamp_rows = 0
normalization_stamps: list[datetime] = []
unknown_normalization_rows = 0
verified_rows: list[dict[str, Any]] = []
actual_shard_hashes: dict[str, str | None] = {}
try:
enumeration_audit, enumeration_sha = load_verified_enumeration(snapshot_dir, manifest)
except (OSError, ValueError, UnicodeDecodeError, json.JSONDecodeError) as exc:
enumeration_audit = {}
enumeration_sha = None
problems.append(str(exc))
for index in range(shard_count):
entry = (manifest.get("shards") or {}).get(str(index)) or {}
path = shard_path(snapshot_dir, index)
if not path.exists():
actual_shard_hashes[str(index)] = None
if int(entry.get("attempted") or 0) > 0:
problems.append(f"shard {index}: file missing but manifest records rows")
continue
try:
shard_bytes = path.read_bytes()
except OSError as exc:
actual_shard_hashes[str(index)] = None
problems.append(f"shard {index} ({path}): read failed: {exc}")
continue
recorded_sha = entry.get("shard_sha256")
actual_sha = hashlib.sha256(shard_bytes).hexdigest()
actual_shard_hashes[str(index)] = actual_sha
if not (isinstance(recorded_sha, str) and recorded_sha):
# A shard collected before SHA recording existed must be
# backfilled (a zero-collect --resume re-records it) or
# re-collected; readiness never accepts an unpinned shard.
problems.append(f"shard {index}: manifest lacks shard_sha256")
elif recorded_sha != actual_sha:
problems.append(f"shard {index}: file SHA-256 does not match the manifest")
recount: dict[str, int] = {}
shard_normalization_stamps: list[str] = []
shard_unknown_normalization = 0
try:
shard_rows = _rows_from_bytes(shard_bytes, path=path)
except UnicodeDecodeError as exc:
problems.append(f"shard {index} ({path}): invalid UTF-8: {exc}")
continue
except json.JSONDecodeError as exc:
problems.append(f"shard {index} ({path}): invalid JSON: {exc}")
continue
except ValueError as exc:
problems.append(f"shard {index} ({path}): row is not a JSON object: {exc}")
continue
expected_in_shard = sum(
1 for symbol in expected_symbols if stable_shard(symbol, shard_count) == index
)
if int(entry.get("attempted") or 0) != len(shard_rows):
problems.append(f"shard {index}: manifest attempted count does not match the file")
if int(entry.get("expected") or 0) != expected_in_shard:
problems.append(f"shard {index}: manifest expected count does not match the universe")
for row in shard_rows:
symbol = str(row.get("symbol") or "")
if not symbol:
problems.append(f"shard {index}: row without a symbol")
continue
if symbol in seen:
problems.append(f"shard {index}: duplicate symbol {symbol}")
continue
seen[symbol] = index
if expected_symbols and symbol not in expected_symbols:
problems.append(f"shard {index}: {symbol} is not in the frozen universe")
if shard_count and stable_shard(symbol, shard_count) != index:
problems.append(f"shard {index}: {symbol} belongs to another shard")
if row.get("snapshot_shard") != index:
problems.append(f"shard {index}: {symbol} carries the wrong snapshot_shard")
name = str(row.get("snapshot_classification") or "")
if name not in CLASSIFICATIONS:
problems.append(f"shard {index}: {symbol} has classification {name!r}")
recount[name] = recount.get(name, 0) + 1
stamp = row.get("snapshot_retrieved_at")
if isinstance(stamp, str) and stamp:
try:
stamp_dt = datetime.fromisoformat(stamp)
except ValueError:
problems.append(f"shard {index}: {symbol} has unparsable snapshot_retrieved_at")
else:
if stamp_dt.tzinfo is None:
stamp_dt = stamp_dt.replace(tzinfo=timezone.utc)
row_stamps.append(stamp_dt)
else:
unknown_stamp_rows += 1
normalization_stamp = row.get("snapshot_normalization_as_of")
if isinstance(normalization_stamp, str) and normalization_stamp:
try:
normalization_dt = datetime.fromisoformat(normalization_stamp)
except ValueError:
problems.append(
f"shard {index}: {symbol} has unparsable snapshot_normalization_as_of"
)
else:
if normalization_dt.tzinfo is None:
normalization_dt = normalization_dt.replace(tzinfo=timezone.utc)
normalization_stamps.append(normalization_dt)
shard_normalization_stamps.append(normalization_dt.isoformat())
else:
unknown_normalization_rows += 1
shard_unknown_normalization += 1
verified_rows.append(row)
recorded = {key: int(value or 0) for key, value in (entry.get("classified") or {}).items()}
if recorded != recount:
problems.append(f"shard {index}: manifest classification counts do not match the file")
recorded_oldest_normalization = entry.get("oldest_normalization_as_of")
recorded_newest_normalization = entry.get("newest_normalization_as_of")
actual_shard_oldest = (
min(shard_normalization_stamps) if shard_normalization_stamps else None
)
actual_shard_newest = (
max(shard_normalization_stamps) if shard_normalization_stamps else None
)
if (
recorded_oldest_normalization != actual_shard_oldest
or recorded_newest_normalization != actual_shard_newest
or int(entry.get("normalization_time_unknown") or 0) != shard_unknown_normalization
):
problems.append(
f"shard {index}: manifest normalization provenance does not match the file"
)
missing = expected_symbols - set(seen)
if missing:
problems.append(
f"{len(missing)} frozen-universe symbols never collected (e.g. {sorted(missing)[:5]})"
)
if unknown_stamp_rows:
problems.append(f"{unknown_stamp_rows} rows carry no retrieval stamp (unknown provenance)")
if unknown_normalization_rows:
problems.append(
f"{unknown_normalization_rows} rows carry no normalization as-of (unknown provenance)"
)
status = snapshot_status(manifest)
# Freshness is judged from the stamps RE-AGGREGATED out of the shard
# rows themselves (the manifest's own bounds are informational only),
# and bounded on BOTH sides: old data is stale, and data fetched after
# screening_as_of is look-ahead — fatal for historical/as-of runs.
actual_oldest = min(row_stamps) if row_stamps else None
actual_newest = max(row_stamps) if row_stamps else None
staleness_ok = actual_oldest is not None and actual_oldest >= screening_as_of - timedelta(
days=float(max_staleness_days)
)
no_future_retrievals = actual_newest is not None and actual_newest <= (
screening_as_of + timedelta(seconds=float(clock_skew_seconds))
)
actual_oldest_normalization = min(normalization_stamps) if normalization_stamps else None
actual_newest_normalization = max(normalization_stamps) if normalization_stamps else None
normalization_current = (
actual_oldest_normalization is not None
and actual_oldest_normalization
>= screening_as_of - timedelta(days=float(max_staleness_days))
)
no_future_normalization = (
actual_newest_normalization is not None
and actual_newest_normalization
<= screening_as_of + timedelta(seconds=float(clock_skew_seconds))
)
binding = {
"manifest_sha256": hashlib.sha256(manifest_bytes).hexdigest(),
"universe_sha256": universe_sha,
"listing_enumeration_audit_sha256": enumeration_sha,
"shard_sha256": actual_shard_hashes,
}
verification_digest = hashlib.sha256(
json.dumps(binding, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
contents_verified = not problems
verdict = {
**status,
"screening_as_of": screening_as_of.astimezone(timezone.utc).isoformat(),
"max_staleness_days": float(max_staleness_days),
"clock_skew_seconds": float(clock_skew_seconds),
"verified_oldest_retrieved_at": actual_oldest.isoformat() if actual_oldest else None,
"verified_newest_retrieved_at": actual_newest.isoformat() if actual_newest else None,
"verified_oldest_normalization_as_of": (
actual_oldest_normalization.isoformat() if actual_oldest_normalization else None
),
"verified_newest_normalization_as_of": (
actual_newest_normalization.isoformat() if actual_newest_normalization else None
),
"staleness_ok": staleness_ok,
"no_future_retrievals": no_future_retrievals,
"normalization_current": normalization_current,
"no_future_normalization": no_future_normalization,
"listing_enumeration": enumeration_audit,
"listing_enumeration_verified": bool(enumeration_audit),
"problem_count": len(problems),
"problems": problems[:50],
"contents_verified": contents_verified,
"snapshot_verification_digest": verification_digest,
"verification_binding": binding,
"ready_for_screening": (
contents_verified
and bool(status.get("collection_ready"))
and staleness_ok
and no_future_retrievals
and normalization_current
and no_future_normalization
),
}
return {
"verdict": verdict,
"verification_digest": verification_digest,
"rows": verified_rows if verdict["ready_for_screening"] else [],
}
def verify_snapshot(
snapshot_dir: Path,
*,
screening_as_of: datetime,
max_staleness_days: float,
clock_skew_seconds: float = 300.0,
) -> dict[str, Any]:
"""Deep readiness verdict; use ``load_verified_snapshot`` when screening."""
return load_verified_snapshot(
snapshot_dir,
screening_as_of=screening_as_of,
max_staleness_days=max_staleness_days,
clock_skew_seconds=clock_skew_seconds,
)["verdict"]
scripts/evaluate_candidates.py
#!/usr/bin/env python3
"""Validate and rank normalized US undervalued-growth candidates (schema v3, contract v3.5).
The evaluator performs no network access. Upstream research must prepare a
source-linked JSON snapshot matching references/data-contract.md. The script
normalizes cash flow, verifies forecast arithmetic, blocks accounting-basis
mixing, validates corporate-action preflight evidence, calculates valuation
scenarios, routes candidates fail-closed, and writes audit-ready JSON/Markdown.
"""
from __future__ import annotations
import argparse
import json
import math
import statistics
import sys
from collections.abc import Iterable, Mapping, Sequence
from copy import deepcopy
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from coverage_semantics import derive_ranking_scope_from_audit
try:
from skill_version import (
CONTRACT_REVISION,
RUNTIME_FINGERPRINT,
SCHEMA_VERSION,
SKILL_VERSION,
runtime_metadata,
)
except ModuleNotFoundError: # Supports importlib-based unit loading.
import importlib.util as _importlib_util
_version_path = Path(__file__).with_name("skill_version.py")
_version_spec = _importlib_util.spec_from_file_location("skill_version", _version_path)
if _version_spec is None or _version_spec.loader is None:
raise
_version_module = _importlib_util.module_from_spec(_version_spec)
_version_spec.loader.exec_module(_version_module)
CONTRACT_REVISION = _version_module.CONTRACT_REVISION
RUNTIME_FINGERPRINT = _version_module.RUNTIME_FINGERPRINT
SCHEMA_VERSION = _version_module.SCHEMA_VERSION
SKILL_VERSION = _version_module.SKILL_VERSION
runtime_metadata = _version_module.runtime_metadata
try:
from research_contract import ContractError, validate_and_normalize_snapshot
except ModuleNotFoundError: # Supports importlib-based unit loading.
import importlib.util
_contract_path = Path(__file__).with_name("research_contract.py")
_contract_spec = importlib.util.spec_from_file_location("research_contract", _contract_path)
if _contract_spec is None or _contract_spec.loader is None:
raise
_contract_module = importlib.util.module_from_spec(_contract_spec)
_contract_spec.loader.exec_module(_contract_module)
ContractError = _contract_module.ContractError
validate_and_normalize_snapshot = _contract_module.validate_and_normalize_snapshot
DEFAULT_CONFIG: dict[str, Any] = {
"min_market_cap": 500_000_000,
"max_market_cap": 20_000_000_000,
"min_price": 5.0,
"min_average_daily_dollar_volume": 5_000_000,
"hard_min_average_daily_dollar_volume": 1_000_000,
"minimum_listing_data_coverage_pct": 95.0,
"minimum_discovery_analyst_count": 2,
"maximum_enrichment_attempts": 60,
"max_candidates": 10,
"minimum_data_quality_score": 70,
"minimum_final_score_for_eligible": 70.0,
"minimum_sbc_adjusted_fcf_yield_pct_for_eligible": 3.0,
"maximum_ev_to_fcf_for_eligible": 30.0,
"minimum_roic_pct_for_eligible": 8.0,
"maximum_net_debt_to_ebitda_for_eligible": 3.0,
"maximum_dilution_pct_for_eligible": 5.0,
"minimum_low_case_upside_pct_for_eligible": 15.0,
"maximum_quality_gate_failures_for_conditional": 2,
"severe_loe_stress_downside_pct": -25.0,
"final_three_minimum_score": 75.0,
"final_three_minimum_data_quality": 80,
"final_three_minimum_stress_upside_pct": 0.0,
"final_three_minimum_sbc_adjusted_fcf_yield_pct": 5.0,
"final_three_maximum_ev_to_fcf": 20.0,
"final_three_minimum_low_case_upside_pct": 20.0,
"minimum_constant_multiple_upside_pct": 30.0,
"minimum_stressed_upside_pct": 0.0,
"multiple_contraction_pct": 20.0,
"maximum_cyclicality_without_normalization": 2,
"allow_special_cases": True,
"require_minimum_upside": True,
"require_positive_stress_case": False,
"max_quote_age_days": 7,
"max_estimate_age_days": 45,
"max_corporate_action_check_age_days": 14,
"forecast_bridge_tolerance_pct": 2.0,
"gaap_reconciliation_tolerance_pct": 2.0,
"fcf_reconciliation_tolerance_pct": 5.0,
"cash_consistency_tolerance_pct": 3.0,
"max_market_context_age_days": 14,
"max_policy_rate_source_age_days": 60,
"max_market_rate_source_age_days": 7,
"max_inflation_source_age_days": 45,
"max_gdp_source_age_days": 120,
"max_market_valuation_source_age_days": 14,
"max_small_mid_context_source_age_days": 14,
"minimum_analyst_count_for_ranked_horizon": 3,
"require_forward_current_metric": True,
"require_peer_set": True,
"require_market_context": True,
"require_screening_audit": True,
"biopharma_loe_stress_multiples": [6.0, 8.0],
}
ALLOWED_EXCHANGES = {
"NYSE",
"NASDAQ",
"NYSE AMERICAN",
"NYSEAMERICAN",
"AMEX",
}
ALLOWED_SPECIAL_CASES = {"none", "bank", "insurance", "reit", "bdc", "mlp"}
ALLOWED_VALUATION_BASES: dict[str, set[str]] = {
"none": {"eps", "fcf_per_share", "ebit_per_share"},
"bank": {"tbv_per_share", "book_value_per_share", "eps"},
"insurance": {"book_value_per_share", "tbv_per_share", "eps"},
"reit": {"affo_per_share", "nav_per_share"},
"bdc": {"nav_per_share", "book_value_per_share", "nii_per_share"},
"mlp": {"fcf_per_unit", "dcf_per_unit"},
}
ALLOWED_METRIC_BASES = {"gaap", "company_adjusted", "analyst_normalized", "sector_defined"}
ALLOWED_SOURCE_TYPES = {
"reported_fact",
"company_guidance",
"market_consensus",
"analyst_estimate",
}
ALLOWED_PERIOD_KINDS = {"ttm", "ntm", "fy1", "fy2", "fy3", "normalized", "other"}
ALLOWED_SESSIONS = {"regular_close", "pre_market", "after_hours", "intraday"}
ALLOWED_SCREENING_STATUSES = {
"passed",
"exception_admitted",
"selected",
"near_miss_review",
"sector_review_required",
"screened_out",
}
ALLOWED_CASH_FLOW_METHODS = {
"reported_ttm",
"sum_4_discrete",
"fy_plus_current_ytd_minus_prior_ytd",
}
ALLOWED_MNA_STATUSES = {"none", "rumored", "pending", "completed", "terminated", "unknown"}
ALLOWED_LISTING_STATUSES = {"active", "suspended", "delisted", "unknown"}
SCORE_LIMITS: dict[str, float] = {
"growth_sustainability": 20.0,
"valuation_attractiveness": 20.0,
"financial_quality": 15.0,
"fcf_earnings_quality": 15.0,
"competitive_advantage": 10.0,
"capital_allocation": 10.0,
"catalyst_risk_balance": 10.0,
}
QUALITY_WEIGHTS: dict[str, int] = {
"quote_verified": 5,
"latest_earnings_verified": 7,
"sec_financials_verified": 10,
"guidance_consensus_labeled": 8,
"forecast_bridge_verified": 15,
"diluted_shares_verified": 8,
"sbc_verified": 8,
"peer_set_verified": 8,
"gaap_non_gaap_reconciled": 8,
"corporate_actions_verified": 4,
"cyclical_normalization_verified": 5,
"cash_classification_verified": 5,
"roic_ebitda_evidence_verified": 4,
"sector_risk_verified": 5,
}
HARD_FLAG_LABELS: dict[str, str] = {
"otc": "OTC listing",
"penny_stock": "penny-stock characteristics",
"spac_or_shell": "SPAC or shell company",
"pre_revenue_or_development_stage": "pre-revenue or development-stage economics",
"repeated_equity_financing": "repeated equity financing dependence",
"pending_mna": "pending M&A / merger-arbitrage situation",
"unresolved_material_weakness": "unresolved material weakness",
"sec_investigation": "active or unresolved SEC investigation",
"meme_only": "meme-driven valuation without operating support",
"restatement_unresolved": "unresolved restatement",
"extreme_illiquidity": "extreme illiquidity",
}
class InputError(ValueError):
"""Raised when the top-level input cannot be evaluated."""
def _mapping(value: Any) -> dict[str, Any]:
return dict(value) if isinstance(value, Mapping) else {}
def _list(value: Any) -> list[Any]:
return list(value) if isinstance(value, list) else []
def _text(value: Any) -> str | None:
if isinstance(value, str):
stripped = value.strip()
return stripped or None
return None
def _number(value: Any) -> float | None:
if isinstance(value, bool):
return None
if isinstance(value, (int, float)):
numeric = float(value)
if math.isfinite(numeric):
return numeric
return None
def _integer(value: Any) -> int | None:
numeric = _number(value)
if numeric is None or not numeric.is_integer():
return None
return int(numeric)
def _round(value: float | None, digits: int = 4) -> float | None:
return round(value, digits) if value is not None and math.isfinite(value) else None
def parse_iso8601(value: Any, field_name: str, *, required: bool = False) -> datetime | None:
text = _text(value)
if text is None:
if required:
raise InputError(f"{field_name} is required")
return None
normalized = text[:-1] + "+00:00" if text.endswith("Z") else text
try:
parsed = datetime.fromisoformat(normalized)
except ValueError as exc:
raise InputError(f"{field_name} must be ISO-8601: {text!r}") from exc
if parsed.tzinfo is None:
raise InputError(f"{field_name} must include a timezone: {text!r}")
return parsed
def _parse_date(value: Any) -> datetime | None:
text = _text(value)
if not text:
return None
try:
return datetime.fromisoformat(text + "T00:00:00+00:00")
except ValueError:
return None
def _years_between(start: datetime, end: datetime) -> float:
return max((end - start).total_seconds() / (365.2425 * 24 * 60 * 60), 0.0)
def _pct(numerator: float | None, denominator: float | None) -> float | None:
if numerator is None or denominator is None or denominator == 0:
return None
return numerator / denominator * 100.0
def _cagr(start: float | None, end: float | None, years: float | None) -> float | None:
if start is None or end is None or years is None or years <= 0 or start <= 0 or end < 0:
return None
return ((end / start) ** (1.0 / years) - 1.0) * 100.0
def _history_cagr(records: Any) -> dict[str, Any]:
clean: list[dict[str, Any]] = []
for item in _list(records):
row = _mapping(item)
value = _number(row.get("value"))
if value is None:
continue
clean.append(
{
"value": value,
"date": _parse_date(row.get("date")),
"period": _text(row.get("period")),
}
)
if len(clean) < 2:
return {"cagr_pct": None, "years": None, "start": None, "end": None, "count": len(clean)}
if all(row["date"] is not None for row in clean):
clean.sort(key=lambda row: row["date"])
years = _years_between(clean[0]["date"], clean[-1]["date"])
else:
years = float(len(clean) - 1)
return {
"cagr_pct": _cagr(clean[0]["value"], clean[-1]["value"], years),
"years": years,
"start": clean[0]["value"],
"end": clean[-1]["value"],
"count": len(clean),
"start_period": clean[0]["period"],
"end_period": clean[-1]["period"],
}
def _format_number(value: Any, digits: int = 1, missing: str = "—") -> str:
numeric = _number(value)
if numeric is None:
return missing
abs_value = abs(numeric)
if abs_value >= 1_000_000_000:
return f"{numeric / 1_000_000_000:.{digits}f}B"
if abs_value >= 1_000_000:
return f"{numeric / 1_000_000:.{digits}f}M"
if abs_value >= 1_000:
return f"{numeric:,.{digits}f}"
return f"{numeric:.{digits}f}"
def _format_pct(value: Any, digits: int = 1, missing: str = "—") -> str:
numeric = _number(value)
if numeric is None:
return missing
return f"{numeric:+.{digits}f}%"
def _format_price(value: Any, currency: str = "USD", missing: str = "—") -> str:
numeric = _number(value)
if numeric is None:
return missing
prefix = "$" if currency.upper() == "USD" else f"{currency.upper()} "
return f"{prefix}{numeric:,.2f}"
def _deep_merge(base: Mapping[str, Any], override: Mapping[str, Any]) -> dict[str, Any]:
result = deepcopy(dict(base))
for key, value in override.items():
if isinstance(value, Mapping) and isinstance(result.get(key), Mapping):
result[key] = _deep_merge(result[key], value)
else:
result[key] = value
return result
def _scenario(
metric: float | None, multiple: float | None, price: float | None, years: float | None
) -> dict[str, Any] | None:
if metric is None or multiple is None or price is None or years is None:
return None
if metric <= 0 or multiple <= 0 or price <= 0 or years <= 0:
return None
target = metric * multiple
upside = (target / price - 1.0) * 100.0
annualized = ((target / price) ** (1.0 / years) - 1.0) * 100.0
return {
"metric": _round(metric),
"multiple": _round(multiple),
"years": _round(years),
"implied_price": _round(target),
"upside_pct": _round(upside),
"cagr_pct": _round(annualized),
}
def _min_valid(values: Iterable[int | None]) -> int | None:
valid = [value for value in values if value is not None and value >= 0]
return min(valid) if valid else None
def _estimate_breadth_penalty(count: int | None) -> int:
if count is None:
return 5
if count >= 5:
return 0
if count >= 3:
return 2
if count >= 1:
return 4
return 5
def _estimate_dispersion_penalty(dispersion_pct: float | None) -> int:
if dispersion_pct is None:
return 0
if dispersion_pct > 50:
return 4
if dispersion_pct > 30:
return 2
if dispersion_pct > 15:
return 1
return 0
def _data_quality_penalty(score: int) -> int:
if score >= 85:
return 0
if score >= 70:
return 2
if score >= 55:
return 5
return 10
def _cyclicality_penalty(score: int | None) -> int:
return {1: 0, 2: 0, 3: 2, 4: 5, 5: 8}.get(score or 0, 5)
def _sbc_penalty(sbc_revenue_pct: float | None) -> int:
if sbc_revenue_pct is None:
return 2
if sbc_revenue_pct > 25:
return 8
if sbc_revenue_pct > 15:
return 5
if sbc_revenue_pct > 10:
return 3
if sbc_revenue_pct > 5:
return 1
return 0
def _dilution_penalty(annualized_pct: float | None) -> int:
if annualized_pct is None:
return 2
if annualized_pct <= 3:
return 0
if annualized_pct <= 5:
return 2
if annualized_pct <= 10:
return 5
return 8
def _dispersion_from_bounds(
low: float | None, high: float | None, midpoint: float | None
) -> float | None:
if low is None or high is None or midpoint is None or midpoint == 0 or high < low:
return None
return (high - low) / abs(midpoint) * 100.0
def _source_index(candidate: Mapping[str, Any]) -> dict[str, dict[str, Any]]:
index: dict[str, dict[str, Any]] = {}
for source in _list(candidate.get("sources")):
item = _mapping(source)
source_id = _text(item.get("id"))
if source_id:
index[source_id] = item
return index
def _evidence_ids(candidate: Mapping[str, Any], field: str) -> list[str]:
ids: list[str] = []
evidence = _mapping(candidate.get("evidence"))
for value in _list(evidence.get(field)):
source_id = _text(value)
if source_id and source_id not in ids:
ids.append(source_id)
for source in _list(candidate.get("sources")):
item = _mapping(source)
source_id = _text(item.get("id"))
supports = {_text(value) for value in _list(item.get("supports"))}
if source_id and field in supports and source_id not in ids:
ids.append(source_id)
return ids
def _resolved_sources(
source_ids: Iterable[str | None], source_index: Mapping[str, Mapping[str, Any]]
) -> list[Mapping[str, Any]]:
return [
source_index[source_id]
for source_id in source_ids
if source_id and source_id in source_index
]
def _sources_resolve(
source_ids: Iterable[str | None], source_index: Mapping[str, Mapping[str, Any]]
) -> bool:
ids = [source_id for source_id in source_ids if source_id]
return bool(ids) and all(source_id in source_index for source_id in ids)
def _has_tier(
source_ids: Iterable[str | None], source_index: Mapping[str, Mapping[str, Any]], tiers: set[int]
) -> bool:
return any(
_integer(source.get("tier")) in tiers
for source in _resolved_sources(source_ids, source_index)
)
def _has_primary_source(source_index: Mapping[str, Mapping[str, Any]]) -> bool:
return any(_integer(source.get("tier")) in {1, 2} for source in source_index.values())
def _candidate_source_ids(
candidate: Mapping[str, Any], field: str, explicit: Any = None
) -> list[str]:
ids = _evidence_ids(candidate, field)
for item in _list(explicit):
source_id = _text(item)
if source_id and source_id not in ids:
ids.append(source_id)
return ids
def _validate_corporate_action(
candidate: Mapping[str, Any],
source_index: Mapping[str, Mapping[str, Any]],
analysis_as_of: datetime,
config: Mapping[str, Any],
) -> tuple[dict[str, Any], list[str], list[str], list[str]]:
check = _mapping(candidate.get("corporate_action_check"))
warnings: list[str] = []
review: list[str] = []
hard: list[str] = []
checked_at = None
try:
checked_at = parse_iso8601(check.get("checked_at"), "corporate_action_check.checked_at")
except InputError as exc:
review.append(str(exc))
source_ids = _candidate_source_ids(candidate, "corporate_action_check", check.get("source_ids"))
evidence_ok = _sources_resolve(source_ids, source_index)
if not evidence_ok:
review.append("corporate-action preflight lacks resolving source evidence")
listing_status = (_text(check.get("listing_status")) or "unknown").lower()
mna_status = (_text(check.get("mna_status")) or "unknown").lower()
symbol_active = check.get("symbol_active")
if listing_status not in ALLOWED_LISTING_STATUSES:
review.append(f"invalid corporate-action listing_status: {listing_status!r}")
if mna_status not in ALLOWED_MNA_STATUSES:
review.append(f"invalid corporate-action mna_status: {mna_status!r}")
if listing_status in {"delisted", "suspended"}:
hard.append(f"listing status is {listing_status}")
if symbol_active is False:
hard.append("symbol is not active")
if mna_status in {"pending", "completed"}:
hard.append(f"M&A status is {mna_status}; treat as special-situation pricing")
elif mna_status == "rumored":
review.append("M&A status is rumored and requires event-risk resolution")
elif mna_status == "unknown":
review.append("M&A status is unknown")
if listing_status == "unknown":
review.append("listing status is unknown")
if symbol_active is not True:
review.append("symbol_active is not positively verified")
age_days = None
if checked_at is not None:
age_days = (analysis_as_of - checked_at).total_seconds() / 86_400
if age_days < -1:
review.append("corporate-action check timestamp is after analysis_as_of")
max_age = _number(config.get("max_corporate_action_check_age_days")) or 14.0
if age_days > max_age:
review.append(
f"corporate-action check is stale ({age_days:.1f} days old; maximum {max_age:.0f})"
)
return (
{
"checked_at": _text(check.get("checked_at")),
"age_days": _round(age_days),
"listing_status": listing_status,
"mna_status": mna_status,
"symbol_active": symbol_active is True,
"latest_material_event_at": _text(check.get("latest_material_event_at")),
"source_ids": source_ids,
"evidence_ok": evidence_ok,
},
warnings,
review,
hard,
)
def _cash_flow_component(
row: Mapping[str, Any], label: str
) -> tuple[float | None, float | None, list[str]]:
warnings: list[str] = []
ocf = _number(row.get("operating_cash_flow"))
capex = _number(row.get("capex_cash_outflow"))
if capex is not None and capex < 0:
warnings.append(f"{label}.capex_cash_outflow must be non-negative; received {capex}")
capex = None
return ocf, capex, warnings
def _cash_flow_sources_support_period(
source_ids: Sequence[Any],
source_index: Mapping[str, Mapping[str, Any]],
*,
support_path: str,
) -> bool:
ids = [_text(value) for value in source_ids if _text(value)]
if not ids or not all(source_id in source_index for source_id in ids):
return False
for source_id in ids:
supports = {_text(value) for value in _list(source_index[source_id].get("supports"))}
if support_path in supports:
return True
return False
def _reconstruct_ttm_cash_flow(
financials: Mapping[str, Any],
tolerance_pct: float,
source_index: Mapping[str, Mapping[str, Any]],
) -> tuple[dict[str, Any], list[str], list[str]]:
ttm = _mapping(financials.get("cash_flow_ttm"))
method = (_text(ttm.get("method")) or "").lower()
warnings: list[str] = []
review: list[str] = []
source_checks: dict[str, bool] = {}
supplied_ocf, supplied_capex, component_warnings = _cash_flow_component(
ttm, "financials.cash_flow_ttm"
)
warnings.extend(component_warnings)
supplied_fcf = _number(ttm.get("standard_fcf"))
reconstructed_ocf: float | None = None
reconstructed_capex: float | None = None
used_periods: list[str] = []
if method not in ALLOWED_CASH_FLOW_METHODS:
review.append("cash_flow_ttm.method is missing or invalid")
elif method == "reported_ttm":
reconstructed_ocf = supplied_ocf
reconstructed_capex = supplied_capex
source_checks["reported_ttm"] = _cash_flow_sources_support_period(
_list(ttm.get("source_ids")),
source_index,
support_path="financials.cash_flow_ttm.reported_ttm",
)
if not source_checks["reported_ttm"]:
review.append(
"reported_ttm cash flow requires a source explicitly supporting financials.cash_flow_ttm.reported_ttm"
)
elif method == "sum_4_discrete":
periods: list[tuple[datetime, Mapping[str, Any]]] = []
for item in _list(financials.get("cash_flow_periods")):
row = _mapping(item)
if (_text(row.get("period_type")) or "").lower() != "discrete_quarter":
continue
period_end = _parse_date(row.get("period_end"))
if period_end is not None:
periods.append((period_end, row))
periods.sort(key=lambda pair: pair[0])
selected = periods[-4:]
if len(selected) != 4 or len({pair[0] for pair in selected}) != 4:
review.append("sum_4_discrete requires four unique discrete quarters")
else:
ocf_values: list[float] = []
capex_values: list[float] = []
for _, row in selected:
period_label = _text(row.get("period")) or _text(row.get("period_end")) or "unknown"
period_end_label = _text(row.get("period_end")) or period_label
support_path = f"financials.cash_flow_periods.{period_end_label}"
source_checks[period_label] = _cash_flow_sources_support_period(
_list(row.get("source_ids")), source_index, support_path=support_path
)
if not source_checks[period_label]:
review.append(
f"cash-flow period {period_label} requires a source explicitly supporting {support_path}"
)
ocf, capex, row_warnings = _cash_flow_component(
row, f"cash_flow_periods[{row.get('period')}]"
)
warnings.extend(row_warnings)
if ocf is None or capex is None:
review.append(
"sum_4_discrete contains a quarter with missing/invalid OCF or capex"
)
break
ocf_values.append(ocf)
capex_values.append(capex)
used_periods.append(
_text(row.get("period")) or _text(row.get("period_end")) or "unknown"
)
if len(ocf_values) == 4:
reconstructed_ocf = sum(ocf_values)
reconstructed_capex = sum(capex_values)
elif method == "fy_plus_current_ytd_minus_prior_ytd":
reconstruction = _mapping(financials.get("ttm_reconstruction"))
parts: dict[str, tuple[float | None, float | None]] = {}
for key in ("latest_fy", "current_ytd", "prior_ytd"):
row = _mapping(reconstruction.get(key))
support_path = f"financials.ttm_reconstruction.{key}"
source_checks[key] = _cash_flow_sources_support_period(
_list(row.get("source_ids")), source_index, support_path=support_path
)
if not source_checks[key]:
review.append(
f"ttm_reconstruction.{key} requires a source explicitly supporting {support_path}"
)
ocf, capex, row_warnings = _cash_flow_component(row, f"ttm_reconstruction.{key}")
warnings.extend(row_warnings)
parts[key] = (ocf, capex)
used_periods.append(_text(row.get("period")) or key)
if all(parts[key][0] is not None and parts[key][1] is not None for key in parts):
reconstructed_ocf = (
parts["latest_fy"][0] + parts["current_ytd"][0] - parts["prior_ytd"][0]
)
reconstructed_capex = (
parts["latest_fy"][1] + parts["current_ytd"][1] - parts["prior_ytd"][1]
)
if reconstructed_capex < 0:
review.append(
"YTD reconstruction produced negative capex cash outflow; period inputs are likely inconsistent"
)
reconstructed_capex = None
else:
review.append(
"fy_plus_current_ytd_minus_prior_ytd requires complete latest_fy/current_ytd/prior_ytd values"
)
canonical_ocf = reconstructed_ocf if reconstructed_ocf is not None else supplied_ocf
canonical_capex = reconstructed_capex if reconstructed_capex is not None else supplied_capex
calculated_fcf = (
canonical_ocf - canonical_capex
if canonical_ocf is not None and canonical_capex is not None
else None
)
tolerance = max(float(tolerance_pct), 0.0)
for supplied, calculated, label in (
(supplied_ocf, reconstructed_ocf, "operating cash flow"),
(supplied_capex, reconstructed_capex, "capex cash outflow"),
(supplied_fcf, calculated_fcf, "standard FCF"),
):
if supplied is None or calculated is None:
continue
denominator = max(abs(supplied), abs(calculated), 1.0)
mismatch = abs(supplied - calculated) / denominator * 100.0
if mismatch > tolerance:
warnings.append(f"supplied {label} differs from reconstructed value by {mismatch:.1f}%")
if mismatch > max(10.0, tolerance * 2):
review.append(f"{label} reconciliation exceeds fail-closed tolerance")
company_adjusted = _number(ttm.get("company_adjusted_fcf"))
adjusted_definition = _text(ttm.get("company_adjusted_fcf_definition"))
if company_adjusted is not None and not adjusted_definition:
review.append("company_adjusted_fcf is present without a definition")
return (
{
"method": method or None,
"operating_cash_flow": _round(canonical_ocf),
"capex_cash_outflow": _round(canonical_capex),
"standard_fcf": _round(calculated_fcf),
"supplied_standard_fcf": _round(supplied_fcf),
"company_adjusted_fcf": _round(company_adjusted),
"company_adjusted_fcf_definition": adjusted_definition,
"used_periods": used_periods,
"source_evidence": source_checks,
"source_evidence_ok": bool(source_checks) and all(source_checks.values()),
},
warnings,
review,
)
def _calculate_financial_metrics(
candidate: Mapping[str, Any],
config: Mapping[str, Any],
source_index: Mapping[str, Mapping[str, Any]],
) -> tuple[dict[str, Any], list[str], list[str], list[str]]:
identity = _mapping(candidate.get("identity"))
financials = _mapping(candidate.get("financials"))
histories = _mapping(financials.get("histories"))
warnings: list[str] = []
review: list[str] = []
unresolved: list[str] = []
market_cap = _number(identity.get("market_cap"))
revenue = _number(financials.get("revenue_ttm"))
cash_flow, cash_warnings, cash_review = _reconstruct_ttm_cash_flow(
financials,
_number(config.get("fcf_reconciliation_tolerance_pct")) or 5.0,
source_index,
)
warnings.extend(cash_warnings)
review.extend(cash_review)
ocf = _number(cash_flow.get("operating_cash_flow"))
standard_fcf = _number(cash_flow.get("standard_fcf"))
sbc = _number(financials.get("sbc_ttm"))
sbc_adjusted_fcf = standard_fcf - sbc if standard_fcf is not None and sbc is not None else None
total_debt = _number(financials.get("total_debt"))
cash_class = _mapping(financials.get("cash_classification"))
corporate_cash = _number(cash_class.get("corporate_cash"))
if corporate_cash is None:
corporate_cash = _number(cash_class.get("cash_and_equivalents"))
marketable_securities = _number(cash_class.get("marketable_securities")) or 0.0
customer_funds = _number(cash_class.get("customer_or_settlement_funds"))
restricted_cash = _number(cash_class.get("restricted_cash"))
usable_cash = corporate_cash + marketable_securities if corporate_cash is not None else None
ebitda = _number(financials.get("ebitda_ttm"))
net_debt = (
total_debt - usable_cash if total_debt is not None and usable_cash is not None else None
)
net_debt_ebitda = (
net_debt / ebitda if net_debt is not None and ebitda is not None and ebitda > 0 else None
)
enterprise_value = (
market_cap + total_debt - usable_cash
if market_cap is not None and total_debt is not None and usable_cash is not None
else None
)
ev_to_fcf = (
enterprise_value / standard_fcf
if enterprise_value is not None and standard_fcf is not None and standard_fcf > 0
else None
)
ev_to_ebitda = (
enterprise_value / ebitda
if enterprise_value is not None and ebitda is not None and ebitda > 0
else None
)
revenue_history = _history_cagr(histories.get("revenue"))
eps_history = _history_cagr(histories.get("gaap_eps"))
fcf_ps_history = _history_cagr(histories.get("fcf_per_share"))
share_history = _history_cagr(histories.get("diluted_shares"))
for name, value in (
("financials.revenue_ttm", revenue),
("financials.cash_flow_ttm.standard_fcf", standard_fcf),
("financials.sbc_ttm", sbc),
("identity.market_cap", market_cap),
("financials.cash_classification.corporate_cash", corporate_cash),
):
if value is None:
unresolved.append(name)
metrics = {
"cash_flow_ttm": cash_flow,
"standard_fcf": _round(standard_fcf),
"company_adjusted_fcf": cash_flow.get("company_adjusted_fcf"),
"sbc_adjusted_fcf": _round(sbc_adjusted_fcf),
"fcf_yield_pct": _round(_pct(standard_fcf, market_cap)),
"sbc_adjusted_fcf_yield_pct": _round(_pct(sbc_adjusted_fcf, market_cap)),
"fcf_margin_pct": _round(_pct(standard_fcf, revenue)),
"sbc_revenue_pct": _round(_pct(sbc, revenue)),
"sbc_ocf_pct": _round(_pct(sbc, ocf)),
"cash_definition": "corporate_cash_plus_marketable_securities",
"cash_classification_method": cash_class.get("classification_method"),
"cash_classification_derived": cash_class.get("corporate_cash_derived") is True,
"corporate_cash": _round(corporate_cash),
"marketable_securities": _round(marketable_securities),
"usable_corporate_cash": _round(usable_cash),
"customer_or_settlement_funds": _round(customer_funds),
"restricted_cash": _round(restricted_cash),
"total_debt": _round(total_debt),
"enterprise_value": _round(enterprise_value),
"ev_to_fcf": _round(ev_to_fcf),
"ev_to_ebitda": _round(ev_to_ebitda),
"net_debt": _round(net_debt),
"net_debt_to_ebitda": _round(net_debt_ebitda),
"roic_pct": _round(_number(financials.get("roic_pct"))),
"roe_pct": _round(_number(financials.get("roe_pct"))),
"revenue_cagr_pct": _round(revenue_history.get("cagr_pct")),
"gaap_eps_cagr_pct": _round(eps_history.get("cagr_pct")),
"fcf_per_share_cagr_pct": _round(fcf_ps_history.get("cagr_pct")),
"diluted_share_cagr_pct": _round(share_history.get("cagr_pct")),
"history_details": {
"revenue": {
key: _round(value) if isinstance(value, float) else value
for key, value in revenue_history.items()
},
"gaap_eps": {
key: _round(value) if isinstance(value, float) else value
for key, value in eps_history.items()
},
"fcf_per_share": {
key: _round(value) if isinstance(value, float) else value
for key, value in fcf_ps_history.items()
},
"diluted_shares": {
key: _round(value) if isinstance(value, float) else value
for key, value in share_history.items()
},
},
}
return metrics, warnings, review, unresolved
def _period_record(valuation: Mapping[str, Any], key: str) -> dict[str, Any]:
return _mapping(_mapping(valuation.get("periods")).get(key))
def _validate_forecast_bridge(
candidate: Mapping[str, Any],
valuation_basis: str,
valuation_periods: Mapping[str, Mapping[str, Any]],
source_index: Mapping[str, Mapping[str, Any]],
tolerance_pct: float,
) -> tuple[bool, dict[str, Any], list[str]]:
"""Rebuild future per-share metrics from operating drivers.
``metric_numerator / metric_denominator`` is retained only as an optional
cross-check. It is never the primary calculation because reverse-engineering
the numerator from consensus EPS would create a circular bridge.
"""
bridge = _mapping(candidate.get("forecast_bridge"))
bridge_periods = _mapping(bridge.get("periods"))
reconciliation_periods = _mapping(_mapping(candidate.get("gaap_reconciliation")).get("periods"))
details: dict[str, Any] = {}
reasons: list[str] = []
all_ok = True
def pct_value(value: float | None) -> float | None:
return value / 100.0 if value is not None else None
for key in ("year_2", "year_3"):
target = _mapping(valuation_periods.get(key))
target_metric = _number(target.get("metric"))
if target_metric is None:
details[key] = {"required": False, "valid": True, "reason": "target metric unavailable"}
continue
row = _mapping(bridge_periods.get(key))
drivers = _mapping(row.get("drivers"))
driver_provenance = _mapping(row.get("driver_provenance"))
construction_method = (_text(row.get("construction_method")) or "").lower()
source_ids = [_text(value) for value in _list(row.get("source_ids")) if _text(value)]
source_ok = _sources_resolve(source_ids, source_index)
target_basis = (_text(target.get("metric_basis")) or "").lower()
basis_ok = (_text(row.get("metric_basis")) or "").lower() == target_basis
calculated_numerator: float | None = None
calculated_denominator: float | None = None
driver_chain: dict[str, Any] = {}
drivers_ok = False
driver_provenance_ok = False
driver_provenance_details: dict[str, Any] = {}
adjustment_bridge_ok = True
adjustment_bridge_details: dict[str, Any] = {}
if valuation_basis == "eps":
revenue = _number(drivers.get("revenue"))
operating_margin = _number(drivers.get("operating_margin_pct"))
tax_rate = _number(drivers.get("tax_rate_pct"))
diluted_shares = _number(drivers.get("diluted_shares"))
net_interest_income = _number(drivers.get("net_interest_income"))
net_interest_expense = _number(drivers.get("net_interest_expense"))
other_pre_tax_income = _number(drivers.get("other_pre_tax_income"))
after_tax_adjustments = _number(drivers.get("after_tax_adjustments"))
interest_explicit = net_interest_income is not None or net_interest_expense is not None
if net_interest_income is None:
net_interest_income = 0.0
if net_interest_expense is None:
net_interest_expense = 0.0
if other_pre_tax_income is None:
other_pre_tax_income = 0.0
adjustment_required = target_basis not in {"gaap", "sector_defined"}
adjustment_ok = after_tax_adjustments is not None if adjustment_required else True
if after_tax_adjustments is None:
after_tax_adjustments = 0.0
drivers_ok = (
all(
value is not None
for value in (revenue, operating_margin, tax_rate, diluted_shares)
)
and interest_explicit
and adjustment_ok
and diluted_shares > 0
and 0 <= tax_rate <= 100
)
if drivers_ok:
operating_income = revenue * (operating_margin / 100.0)
pre_tax_income = (
operating_income
+ net_interest_income
- net_interest_expense
+ other_pre_tax_income
)
gaap_net_income = pre_tax_income * (1.0 - tax_rate / 100.0)
calculated_numerator = gaap_net_income + after_tax_adjustments
calculated_denominator = diluted_shares
driver_chain = {
"revenue": revenue,
"operating_margin_pct": operating_margin,
"operating_income": operating_income,
"net_interest_income": net_interest_income,
"net_interest_expense": net_interest_expense,
"other_pre_tax_income": other_pre_tax_income,
"pre_tax_income": pre_tax_income,
"tax_rate_pct": tax_rate,
"gaap_net_income": gaap_net_income,
"after_tax_adjustments": after_tax_adjustments,
"forecast_net_income": calculated_numerator,
"diluted_shares": diluted_shares,
}
if adjustment_required:
reconciliation_row = _mapping(reconciliation_periods.get(key))
reconciliation_gaap_metric = _number(reconciliation_row.get("gaap_metric"))
adjustment_rows = _list(reconciliation_row.get("adjustments"))
reconciliation_adjustment_per_share = 0.0
reconciliation_adjustments_ok = bool(adjustment_rows)
for item in adjustment_rows:
amount = _number(_mapping(item).get("amount"))
if amount is None:
reconciliation_adjustments_ok = False
break
reconciliation_adjustment_per_share += amount
expected_after_tax_adjustments = (
reconciliation_adjustment_per_share * diluted_shares
if reconciliation_adjustments_ok
else None
)
driver_gaap_metric = gaap_net_income / diluted_shares
gaap_diff_pct = (
abs(driver_gaap_metric - reconciliation_gaap_metric)
/ max(abs(reconciliation_gaap_metric), 1e-12)
* 100.0
if reconciliation_gaap_metric is not None
else None
)
adjustment_diff_pct = (
abs(after_tax_adjustments - expected_after_tax_adjustments)
/ max(abs(expected_after_tax_adjustments), 1.0)
* 100.0
if expected_after_tax_adjustments is not None
else None
)
adjustment_bridge_ok = bool(
reconciliation_adjustments_ok
and reconciliation_gaap_metric is not None
and gaap_diff_pct is not None
and gaap_diff_pct <= tolerance_pct
and adjustment_diff_pct is not None
and adjustment_diff_pct <= tolerance_pct
)
adjustment_bridge_details = {
"reconciliation_gaap_metric": _round(reconciliation_gaap_metric),
"driver_gaap_metric": _round(driver_gaap_metric),
"gaap_difference_pct": _round(gaap_diff_pct),
"reconciliation_adjustment_per_share": _round(
reconciliation_adjustment_per_share
),
"expected_after_tax_adjustments": _round(expected_after_tax_adjustments),
"driver_after_tax_adjustments": _round(after_tax_adjustments),
"adjustment_difference_pct": _round(adjustment_diff_pct),
}
elif valuation_basis == "fcf_per_share":
diluted_shares = _number(drivers.get("diluted_shares"))
operating_cash_flow = _number(drivers.get("operating_cash_flow"))
capex = _number(drivers.get("capex_cash_outflow"))
revenue = _number(drivers.get("revenue"))
fcf_margin = _number(drivers.get("fcf_margin_pct"))
if operating_cash_flow is not None and capex is not None and capex >= 0:
calculated_numerator = operating_cash_flow - capex
method = "operating_cash_flow_minus_capex"
elif revenue is not None and fcf_margin is not None:
calculated_numerator = revenue * (fcf_margin / 100.0)
method = "revenue_times_fcf_margin"
else:
method = None
calculated_denominator = diluted_shares
drivers_ok = (
calculated_numerator is not None
and diluted_shares is not None
and diluted_shares > 0
)
driver_chain = {
"method": method,
"revenue": revenue,
"fcf_margin_pct": fcf_margin,
"operating_cash_flow": operating_cash_flow,
"capex_cash_outflow": capex,
"standard_fcf": calculated_numerator,
"diluted_shares": diluted_shares,
}
elif valuation_basis == "ebit_per_share":
revenue = _number(drivers.get("revenue"))
operating_margin = _number(drivers.get("operating_margin_pct"))
diluted_shares = _number(drivers.get("diluted_shares"))
drivers_ok = (
all(value is not None for value in (revenue, operating_margin, diluted_shares))
and diluted_shares > 0
)
if drivers_ok:
calculated_numerator = revenue * (operating_margin / 100.0)
calculated_denominator = diluted_shares
driver_chain = {
"revenue": revenue,
"operating_margin_pct": operating_margin,
"forecast_ebit": calculated_numerator,
"diluted_shares": diluted_shares,
}
else:
# Sector-defined metrics may not have an income-statement bridge,
# but they must state a formula and provide independently sourced
# numerator/denominator inputs.
calculated_numerator = _number(row.get("metric_numerator"))
calculated_denominator = _number(row.get("metric_denominator"))
formula = _text(row.get("calculation_formula"))
drivers_ok = (
calculated_numerator is not None
and calculated_denominator is not None
and calculated_denominator > 0
and bool(formula)
and len(drivers) >= 2
)
driver_chain = {"calculation_formula": formula, **deepcopy(drivers)}
required_driver_names: list[str] = []
if valuation_basis == "eps":
required_driver_names = [
"revenue",
"operating_margin_pct",
"tax_rate_pct",
"diluted_shares",
]
if _number(drivers.get("net_interest_income")) is not None:
required_driver_names.append("net_interest_income")
elif _number(drivers.get("net_interest_expense")) is not None:
required_driver_names.append("net_interest_expense")
else:
required_driver_names.append("net_interest_income_or_expense")
if target_basis not in {"gaap", "sector_defined"}:
required_driver_names.append("after_tax_adjustments")
elif valuation_basis == "fcf_per_share":
required_driver_names = ["diluted_shares"]
if _number(drivers.get("operating_cash_flow")) is not None:
required_driver_names.extend(["operating_cash_flow", "capex_cash_outflow"])
else:
required_driver_names.extend(["revenue", "fcf_margin_pct"])
elif valuation_basis == "ebit_per_share":
required_driver_names = ["revenue", "operating_margin_pct", "diluted_shares"]
else:
required_driver_names = sorted(driver_provenance)
allowed_origins = {
"company_guidance",
"market_consensus",
"historical_run_rate",
"company_target",
"analyst_assumption",
"primary_source_calculation",
}
provenance_failures: list[str] = []
for driver_name in required_driver_names:
if driver_name == "net_interest_income_or_expense":
provenance_candidates = [
_mapping(driver_provenance.get("net_interest_income")),
_mapping(driver_provenance.get("net_interest_expense")),
]
item = next((candidate for candidate in provenance_candidates if candidate), {})
else:
item = _mapping(driver_provenance.get(driver_name))
origin = (_text(item.get("origin")) or "").lower()
ids = [_text(value) for value in _list(item.get("source_ids")) if _text(value)]
target_solved = (
item.get("target_solved") is True or item.get("derived_from_target_metric") is True
)
valid_item = (
bool(item)
and origin in allowed_origins
and _sources_resolve(ids, source_index)
and not target_solved
)
driver_provenance_details[driver_name] = {
"origin": origin or None,
"source_ids": ids,
"target_solved": target_solved,
"valid": valid_item,
}
if not valid_item:
provenance_failures.append(driver_name)
driver_provenance_ok = (
construction_method == "independent_driver_model"
and bool(required_driver_names)
and not provenance_failures
)
calculated_metric = (
calculated_numerator / calculated_denominator
if calculated_numerator is not None
and calculated_denominator is not None
and calculated_denominator > 0
else None
)
diff_pct = None
arithmetic_ok = False
if calculated_metric is not None:
denominator = max(abs(target_metric), 1e-12)
diff_pct = abs(calculated_metric - target_metric) / denominator * 100.0
arithmetic_ok = diff_pct <= tolerance_pct
supplied_numerator = _number(row.get("metric_numerator"))
supplied_denominator = _number(row.get("metric_denominator"))
numerator_crosscheck_ok = True
denominator_crosscheck_ok = True
if supplied_numerator is not None and calculated_numerator is not None:
numerator_diff = (
abs(supplied_numerator - calculated_numerator)
/ max(abs(calculated_numerator), 1.0)
* 100.0
)
numerator_crosscheck_ok = numerator_diff <= tolerance_pct
else:
numerator_diff = None
if supplied_denominator is not None and calculated_denominator is not None:
denominator_diff = (
abs(supplied_denominator - calculated_denominator)
/ max(abs(calculated_denominator), 1.0)
* 100.0
)
denominator_crosscheck_ok = denominator_diff <= tolerance_pct
else:
denominator_diff = None
valid = (
arithmetic_ok
and source_ok
and basis_ok
and drivers_ok
and driver_provenance_ok
and numerator_crosscheck_ok
and denominator_crosscheck_ok
and adjustment_bridge_ok
)
if not valid:
all_ok = False
if not drivers_ok:
reasons.append(f"{key} forecast bridge lacks a complete driver-derived model")
if not driver_provenance_ok:
reasons.append(
f"{key} forecast bridge drivers are not independently sourced or are target-solved"
)
if not arithmetic_ok:
reasons.append(
f"{key} driver-derived forecast does not tie to the valuation metric"
)
if not source_ok:
reasons.append(f"{key} forecast bridge source IDs do not resolve")
if not basis_ok:
reasons.append(f"{key} forecast bridge metric basis differs from valuation period")
if not numerator_crosscheck_ok:
reasons.append(
f"{key} supplied metric_numerator is not supported by the operating-driver model"
)
if not denominator_crosscheck_ok:
reasons.append(
f"{key} supplied metric_denominator differs from driver diluted shares"
)
if not adjustment_bridge_ok:
reasons.append(
f"{key} adjusted forecast bridge does not reconcile driver GAAP EPS and after-tax adjustments"
)
details[key] = {
"required": True,
"valid": valid,
"calculation_method": "driver_derived",
"calculated_numerator": _round(calculated_numerator),
"calculated_denominator": _round(calculated_denominator),
"calculated_metric": _round(calculated_metric),
"target_metric": _round(target_metric),
"difference_pct": _round(diff_pct),
"arithmetic_ok": arithmetic_ok,
"source_ok": source_ok,
"basis_ok": basis_ok,
"drivers_ok": drivers_ok,
"construction_method": construction_method or None,
"driver_provenance_ok": driver_provenance_ok,
"driver_provenance": driver_provenance_details,
"numerator_crosscheck_ok": numerator_crosscheck_ok,
"numerator_difference_pct": _round(numerator_diff),
"denominator_crosscheck_ok": denominator_crosscheck_ok,
"denominator_difference_pct": _round(denominator_diff),
"adjustment_bridge_ok": adjustment_bridge_ok,
"adjustment_bridge_details": adjustment_bridge_details,
"source_ids": source_ids,
"drivers": deepcopy(drivers),
"driver_chain": {
k: _round(v) if isinstance(v, float) else v for k, v in driver_chain.items()
},
}
return all_ok, details, list(dict.fromkeys(reasons))
def _validate_gaap_reconciliation(
candidate: Mapping[str, Any],
valuation_periods: Mapping[str, Mapping[str, Any]],
source_index: Mapping[str, Mapping[str, Any]],
tolerance_pct: float,
) -> tuple[bool, dict[str, Any], list[str]]:
reconciliation = _mapping(candidate.get("gaap_reconciliation"))
rows = _mapping(reconciliation.get("periods"))
details: dict[str, Any] = {}
reasons: list[str] = []
all_ok = True
for key in ("current", "year_2", "year_3"):
target = _mapping(valuation_periods.get(key))
target_metric = _number(target.get("metric"))
basis = (_text(target.get("metric_basis")) or "").lower()
if target_metric is None:
details[key] = {"required": False, "valid": True, "reason": "metric unavailable"}
continue
if basis in {"gaap", "sector_defined"}:
details[key] = {
"required": False,
"valid": True,
"reason": f"{basis} requires no adjustment bridge",
}
continue
row = _mapping(rows.get(key))
gaap_metric = _number(row.get("gaap_metric"))
adjustments = _list(row.get("adjustments"))
adjustment_total = 0.0
adjustments_ok = bool(adjustments)
normalized_adjustments: list[dict[str, Any]] = []
for item in adjustments:
adjustment = _mapping(item)
amount = _number(adjustment.get("amount"))
label = _text(adjustment.get("label"))
recurring = adjustment.get("recurring")
if amount is None or not label or recurring not in {True, False}:
adjustments_ok = False
continue
adjustment_total += amount
normalized_adjustments.append(
{"label": label, "amount": _round(amount), "recurring": recurring}
)
calculated = (
gaap_metric + adjustment_total if gaap_metric is not None and adjustments_ok else None
)
diff_pct = None
arithmetic_ok = False
if calculated is not None and target_metric != 0:
diff_pct = abs(calculated - target_metric) / abs(target_metric) * 100.0
arithmetic_ok = diff_pct <= tolerance_pct
source_ids = [_text(value) for value in _list(row.get("source_ids"))]
source_ok = _sources_resolve(source_ids, source_index)
valid = arithmetic_ok and source_ok and adjustments_ok
if not valid:
all_ok = False
if not adjustments_ok:
reasons.append(f"{key} GAAP reconciliation lacks complete adjustment rows")
if not arithmetic_ok:
reasons.append(f"{key} GAAP reconciliation does not tie to valuation metric")
if not source_ok:
reasons.append(f"{key} GAAP reconciliation source IDs do not resolve")
details[key] = {
"required": True,
"valid": valid,
"gaap_metric": _round(gaap_metric),
"adjustments": normalized_adjustments,
"calculated_metric": _round(calculated),
"target_metric": _round(target_metric),
"difference_pct": _round(diff_pct),
"source_ids": [value for value in source_ids if value],
}
return all_ok, details, list(dict.fromkeys(reasons))
def _calculate_valuation(
candidate: Mapping[str, Any],
config: Mapping[str, Any],
analysis_as_of: datetime,
source_index: Mapping[str, Mapping[str, Any]],
) -> tuple[dict[str, Any], list[str], list[str], list[str]]:
identity = _mapping(candidate.get("identity"))
valuation = _mapping(candidate.get("valuation_case"))
warnings: list[str] = []
review: list[str] = []
hard: list[str] = []
price = _number(identity.get("price"))
special_case = (_text(identity.get("special_case")) or "none").lower()
basis = (_text(valuation.get("basis")) or "").lower()
if special_case not in ALLOWED_SPECIAL_CASES:
hard.append(f"invalid special_case: {special_case!r}")
elif basis not in ALLOWED_VALUATION_BASES[special_case]:
hard.append(f"valuation basis {basis!r} is invalid for special_case {special_case!r}")
raw_periods = _mapping(valuation.get("periods"))
periods: dict[str, dict[str, Any]] = {}
bases: list[str] = []
min_count = _integer(config.get("minimum_analyst_count_for_ranked_horizon")) or 3
for key in ("current", "year_2", "year_3"):
row = _mapping(raw_periods.get(key))
metric = _number(row.get("metric"))
metric_basis = (_text(row.get("metric_basis")) or "").lower()
source_type = (_text(row.get("source_type")) or "").lower()
period_kind = (_text(row.get("period_kind")) or "other").lower()
if metric is not None and metric <= 0:
review.append(f"valuation_case.periods.{key}.metric must be positive")
if metric is not None and metric_basis not in ALLOWED_METRIC_BASES:
review.append(f"valuation_case.periods.{key}.metric_basis is missing or invalid")
if metric is not None and source_type not in ALLOWED_SOURCE_TYPES:
review.append(f"valuation_case.periods.{key}.source_type is missing or invalid")
if metric is not None and period_kind not in ALLOWED_PERIOD_KINDS:
review.append(f"valuation_case.periods.{key}.period_kind is invalid")
source_ids = [_text(value) for value in _list(row.get("source_ids"))]
if metric is not None and not _sources_resolve(source_ids, source_index):
review.append(f"valuation_case.periods.{key} source IDs do not resolve")
retrieved_at = None
age_days = None
try:
retrieved_at = parse_iso8601(
row.get("retrieved_at"), f"valuation_case.periods.{key}.retrieved_at"
)
except InputError as exc:
if source_type in {"market_consensus", "analyst_estimate", "company_guidance"}:
review.append(str(exc))
if retrieved_at is not None:
age_days = (analysis_as_of - retrieved_at).total_seconds() / 86_400
max_age = _number(config.get("max_estimate_age_days")) or 45.0
if age_days < -1:
review.append(f"{key} estimate retrieval timestamp is after analysis_as_of")
if source_type in {"market_consensus", "analyst_estimate"} and age_days > max_age:
review.append(
f"{key} estimate is stale ({age_days:.1f} days old; maximum {max_age:.0f})"
)
low = _number(row.get("estimate_low"))
high = _number(row.get("estimate_high"))
dispersion = _dispersion_from_bounds(low, high, metric)
if dispersion is not None and dispersion > 30:
warnings.append(f"{key} estimate dispersion is high at {dispersion:.1f}%")
years = _number(row.get("years"))
analyst_count = _integer(row.get("analyst_count"))
rankable = row.get("rankable") is True
if key == "current":
rankable = period_kind in {"ntm", "fy1"}
if not rankable and bool(config.get("require_forward_current_metric", True)):
review.append(
"formal constant-multiple analysis requires NTM or FY1 current metric; TTM is supplemental only"
)
if (
source_type in {"market_consensus", "analyst_estimate"}
and (analyst_count or 0) < min_count
and row.get("independent_model") is not True
):
review.append(f"current forward metric has fewer than {min_count} analysts")
periods[key] = {
"label": _text(row.get("label")),
"period_kind": period_kind,
"period_end": _text(row.get("period_end")),
"metric": _round(metric),
"metric_basis": metric_basis,
"source_type": source_type,
"retrieved_at": _text(row.get("retrieved_at")),
"age_days": _round(age_days),
"source_ids": [value for value in source_ids if value],
"analyst_count": analyst_count,
"estimate_low": _round(low),
"estimate_high": _round(high),
"dispersion_pct": _round(dispersion),
"years": years,
"independent_model": row.get("independent_model") is True,
"rankable": rankable,
}
if metric is not None and metric_basis:
bases.append(metric_basis)
current_metric = _number(periods["current"].get("metric"))
if price is None or price <= 0:
hard.append("current price must be positive")
if current_metric is None or current_metric <= 0:
hard.append("current per-share valuation metric must be positive")
basis_consistent = len(set(bases)) <= 1 and bool(bases)
if not basis_consistent:
review.append("current and future valuation periods do not use one consistent metric basis")
forward_current_valid = (
periods["current"].get("period_kind") in {"ntm", "fy1"}
and periods["current"].get("rankable") is True
)
current_multiple = (
price / current_metric if price and current_metric and current_metric > 0 else None
)
supplied_multiple = _number(valuation.get("supplied_current_multiple"))
multiple_diff_pct = None
if supplied_multiple is not None and current_multiple is not None:
multiple_diff_pct = abs(supplied_multiple - current_multiple) / current_multiple * 100.0
if multiple_diff_pct > 10:
warnings.append(
f"supplied current multiple differs from price/current metric by {multiple_diff_pct:.1f}%"
)
if multiple_diff_pct > 25:
review.append(
"supplied current multiple materially conflicts with price/current metric"
)
contraction_pct = _number(config.get("multiple_contraction_pct")) or 20.0
stress_multiple = current_multiple * (1 - contraction_pct / 100.0) if current_multiple else None
y2_metric = _number(periods["year_2"].get("metric"))
y3_metric = _number(periods["year_3"].get("metric"))
y2_years = _number(periods["year_2"].get("years")) or 2.0
y3_years = _number(periods["year_3"].get("years")) or 3.0
formal_valid = basis_consistent and forward_current_valid
if formal_valid:
constant_2 = _scenario(y2_metric, current_multiple, price, y2_years)
constant_3 = _scenario(y3_metric, current_multiple, price, y3_years)
stress_2 = _scenario(y2_metric, stress_multiple, price, y2_years)
stress_3 = _scenario(y3_metric, stress_multiple, price, y3_years)
else:
constant_2 = constant_3 = stress_2 = stress_3 = None
peers = _list(candidate.get("peers"))
peer_multiples: list[float] = []
peer_checks: list[bool] = []
current_basis = _text(periods["current"].get("metric_basis"))
current_period_kind = _text(periods["current"].get("period_kind"))
for peer in peers:
row = _mapping(peer)
multiple = _number(row.get("multiple"))
source_ok = _sources_resolve([_text(row.get("source_id"))], source_index)
valid = (
multiple is not None
and multiple > 0
and bool(_text(row.get("selection_reason")))
and (_text(row.get("metric_basis")) or "").lower() == (current_basis or "").lower()
and (_text(row.get("multiple_period_kind")) or "").lower()
== (current_period_kind or "").lower()
and source_ok
)
peer_checks.append(valid)
if valid:
peer_multiples.append(multiple)
calculated_peer_median = statistics.median(peer_multiples) if len(peer_multiples) >= 3 else None
supplied_peer = _mapping(valuation.get("peer_median_multiple"))
supplied_peer_value = _number(supplied_peer.get("value"))
peer_source_ids = [_text(value) for value in _list(supplied_peer.get("source_ids"))]
supplied_peer_ok = (
supplied_peer_value is not None
and supplied_peer_value > 0
and (_text(supplied_peer.get("metric_basis")) or "").lower()
== (current_basis or "").lower()
and (_text(supplied_peer.get("multiple_period_kind")) or "").lower()
== (current_period_kind or "").lower()
and _sources_resolve(peer_source_ids, source_index)
)
peer_median = supplied_peer_value if supplied_peer_ok else calculated_peer_median
peer_valid = formal_valid and len(peer_multiples) >= 3 and all(peer_checks)
if bool(config.get("require_peer_set", True)) and not peer_valid:
review.append("three to five sourced genuine peers on the same forward basis are required")
peer_2 = _scenario(y2_metric, peer_median, price, y2_years) if peer_valid else None
peer_3 = _scenario(y3_metric, peer_median, price, y3_years) if peer_valid else None
forecast_ok, forecast_details, forecast_reasons = _validate_forecast_bridge(
candidate,
basis,
periods,
source_index,
_number(config.get("forecast_bridge_tolerance_pct")) or 2.0,
)
reconciliation_ok, reconciliation_details, reconciliation_reasons = (
_validate_gaap_reconciliation(
candidate,
periods,
source_index,
_number(config.get("gaap_reconciliation_tolerance_pct")) or 2.0,
)
)
rankable_horizons: dict[str, bool] = {}
for key in ("year_2", "year_3"):
row = periods[key]
rankable = row.get("rankable") is True
if row.get("independent_model") is True:
rankable = rankable and forecast_ok
rankable_horizons[key] = bool(rankable and formal_valid)
if not any(rankable_horizons.values()):
review.append("no rankable two- or three-year forward scenario")
future_counts = [
_integer(periods[key].get("analyst_count"))
for key in ("year_2", "year_3")
if rankable_horizons.get(key) and _number(periods[key].get("metric")) is not None
]
minimum_analyst_count = _min_valid(future_counts)
valid_dispersions = [
_number(periods[key].get("dispersion_pct"))
for key in ("year_2", "year_3")
if rankable_horizons.get(key)
]
valid_dispersions = [value for value in valid_dispersions if value is not None]
maximum_dispersion = max(valid_dispersions) if valid_dispersions else None
return (
{
"basis": basis,
"periods": periods,
"metric_basis_consistent": basis_consistent,
"forward_current_valid": forward_current_valid,
"formal_forward_scenario_valid": formal_valid,
"rankable_horizons": rankable_horizons,
"current_metric": _round(current_metric),
"current_metric_basis": current_basis,
"current_period_kind": current_period_kind,
"current_multiple": _round(current_multiple),
"supplied_current_multiple": _round(supplied_multiple),
"current_multiple_difference_pct": _round(multiple_diff_pct),
"multiple_contraction_pct": _round(contraction_pct),
"stress_multiple": _round(stress_multiple),
"peer_median_multiple": _round(peer_median),
"calculated_peer_median_multiple": _round(calculated_peer_median),
"peer_set_basis_valid": peer_valid,
"minimum_analyst_count": minimum_analyst_count,
"maximum_estimate_dispersion_pct": _round(maximum_dispersion),
"forecast_bridge_valid": forecast_ok,
"forecast_bridge_details": forecast_details,
"gaap_reconciliation_valid": reconciliation_ok,
"gaap_reconciliation_details": reconciliation_details,
"constant_multiple": {"year_2": constant_2, "year_3": constant_3},
"multiple_contraction": {"year_2": stress_2, "year_3": stress_3},
"peer_median": {"year_2": peer_2, "year_3": peer_3},
},
warnings,
list(dict.fromkeys(review + forecast_reasons + reconciliation_reasons)),
hard,
)
def _validate_sector_profile(
candidate: Mapping[str, Any],
financial_metrics: Mapping[str, Any],
valuation: Mapping[str, Any],
analysis_as_of: datetime,
source_index: Mapping[str, Mapping[str, Any]],
config: Mapping[str, Any],
strict: bool,
) -> tuple[dict[str, Any], list[str], list[str], int]:
profile = _mapping(candidate.get("sector_profile"))
raw_profile_type = (_text(profile.get("type")) or "general").lower()
profile_aliases = {
"biopharma": "commercial_biopharma",
"pharma": "commercial_biopharma",
"biotechnology": "commercial_biopharma",
"royalty_biopharma": "commercial_biopharma",
"drug_delivery_platform": "commercial_biopharma",
}
profile_type = profile_aliases.get(raw_profile_type, raw_profile_type)
kpis = _mapping(profile.get("kpis"))
warnings: list[str] = []
review: list[str] = []
penalty = 0
details: dict[str, Any] = {
"type": profile_type,
"raw_type": raw_profile_type,
"type_normalized": profile_type != raw_profile_type,
"kpis": deepcopy(kpis),
}
source_ids = [_text(value) for value in _list(kpis.get("source_ids"))]
sector_sources_ok = _sources_resolve(source_ids, source_index) if source_ids else False
details["source_ids"] = [value for value in source_ids if value]
details["source_evidence_ok"] = sector_sources_ok
if profile_type == "commercial_biopharma":
concentration = _number(kpis.get("top_product_revenue_pct"))
if concentration is None:
concentration = _number(kpis.get("top_revenue_stream_pct"))
if concentration is None:
concentration = _number(kpis.get("top_program_revenue_pct"))
top_revenue = _number(kpis.get("top_product_revenue"))
if top_revenue is None:
top_revenue = _number(kpis.get("top_revenue_stream"))
total_revenue = _number(kpis.get("total_revenue"))
if (
concentration is None
and top_revenue is not None
and total_revenue
and total_revenue > 0
):
concentration = top_revenue / total_revenue * 100.0
details["concentration_derived"] = True
details["top_product_revenue_pct"] = _round(concentration)
loe_value = kpis.get("nearest_material_loe_date") or kpis.get("nearest_loe_date")
loe_date = _parse_date(loe_value)
loe_years = (
_years_between(analysis_as_of.astimezone(timezone.utc), loe_date) if loe_date else None
)
details["nearest_material_loe_date"] = _text(loe_value)
details["years_to_nearest_loe"] = _round(loe_years)
if concentration is None or loe_date is None or not sector_sources_ok:
if strict:
review.append(
"commercial biopharma requires sourced product concentration and nearest material LOE evidence"
)
elif loe_years is not None and loe_years <= 5 and concentration >= 50:
warnings.append(
f"high product concentration ({concentration:.1f}%) with nearest LOE in {loe_years:.1f} years"
)
penalty = 4 if loe_years <= 3 else 2
multiples = [
_number(value)
for value in _list(config.get("biopharma_loe_stress_multiples"))
if _number(value) is not None and _number(value) > 0
]
price = _number(_mapping(candidate.get("identity")).get("price"))
y3_metric = _number(
_mapping(_mapping(valuation.get("periods")).get("year_3")).get("metric")
)
y3_years = (
_number(_mapping(_mapping(valuation.get("periods")).get("year_3")).get("years"))
or 3.0
)
details["loe_stress_scenarios"] = [
{"multiple": multiple, **(_scenario(y3_metric, multiple, price, y3_years) or {})}
for multiple in sorted(set(multiples))
]
elif profile_type == "payments":
cash = _mapping(_mapping(candidate.get("financials")).get("cash_classification"))
if (
_number(cash.get("corporate_cash")) is None
or _number(cash.get("customer_or_settlement_funds")) is None
or not sector_sources_ok
):
if strict:
review.append(
"payments company requires sourced corporate cash separated from customer/settlement funds"
)
take_rate = _number(kpis.get("gross_profit_to_tpv_pct"))
if take_rate is None:
take_rate = _number(kpis.get("gross_profit_take_rate_pct"))
prior_take_rate = _number(kpis.get("gross_profit_to_tpv_prior_pct"))
if prior_take_rate is None:
prior_take_rate = _number(kpis.get("gross_profit_take_rate_prior_pct"))
details["gross_profit_to_tpv_pct"] = _round(take_rate)
details["gross_profit_to_tpv_prior_pct"] = _round(prior_take_rate)
if take_rate is not None and prior_take_rate is not None and take_rate < prior_take_rate:
warnings.append("gross-profit take rate declined versus the comparison period")
elif profile_type == "auto_dealership":
adjusted = _number(kpis.get("adjusted_net_debt_to_ebitda"))
details["adjusted_net_debt_to_ebitda"] = _round(adjusted)
details["floorplan_debt_excluded"] = kpis.get("floorplan_debt_excluded") is True
if (
adjusted is None
or kpis.get("floorplan_debt_excluded") is not True
or not sector_sources_ok
):
if strict:
review.append(
"auto dealership requires sourced leverage adjusted for floorplan debt"
)
elif adjusted > 2.5:
warnings.append(f"sector-adjusted net debt/EBITDA is elevated at {adjusted:.2f}x")
penalty = max(penalty, 2)
return details, warnings, review, penalty
def _calculate_score_components(
candidate: Mapping[str, Any],
) -> tuple[float, dict[str, float | None], list[str]]:
supplied = _mapping(candidate.get("score_components"))
normalized: dict[str, float | None] = {}
warnings: list[str] = []
total = 0.0
for key, maximum in SCORE_LIMITS.items():
value = _number(supplied.get(key))
if value is None:
normalized[key] = None
warnings.append(f"score component {key} is missing")
continue
if value < 0 or value > maximum:
normalized[key] = None
warnings.append(f"score component {key} must be between 0 and {maximum:g}")
continue
normalized[key] = _round(value, 2)
total += value
return total, normalized, warnings
def _calculate_data_quality(
candidate: Mapping[str, Any],
source_index: Mapping[str, Mapping[str, Any]],
price_source_id: str | None,
valuation: Mapping[str, Any],
financial_metrics: Mapping[str, Any],
sector_profile: Mapping[str, Any],
cyclicality_score: int | None,
corporate_action: Mapping[str, Any],
contract: Mapping[str, Any],
) -> tuple[int, dict[str, dict[str, Any]], list[str]]:
latest = _mapping(candidate.get("latest_earnings"))
financials = _mapping(candidate.get("financials"))
histories = _mapping(financials.get("histories"))
warnings: list[str] = []
details: dict[str, dict[str, Any]] = {}
def award(key: str, evidence_ok: bool, reason: str, evidence: Any = None) -> None:
details[key] = {
"evidence_ok": evidence_ok,
"awarded": evidence_ok,
"weight": QUALITY_WEIGHTS[key],
"reason": reason,
"evidence": evidence,
}
quote_ids = _candidate_source_ids(
candidate, "identity.price", [price_source_id] if price_source_id else []
)
quote_ok = _sources_resolve(quote_ids, source_index)
award("quote_verified", quote_ok, "price evidence resolves in source ledger", quote_ids)
latest_ids = _candidate_source_ids(candidate, "latest_earnings", latest.get("source_ids"))
latest_ok = (
contract.get("latest_earnings_valid") is True
and _sources_resolve(latest_ids, source_index)
and _has_tier(latest_ids, source_index, {1, 2})
)
award(
"latest_earnings_verified",
latest_ok,
"latest quarter/full-year records are separated and sourced",
latest_ids,
)
core_fields = [
"financials.revenue_ttm",
"financials.cash_flow_ttm",
"financials.sbc_ttm",
"financials.total_debt",
"financials.cash_classification",
"financials.histories.diluted_shares",
]
core_checks = {
field: _has_tier(_candidate_source_ids(candidate, field), source_index, {1, 2})
for field in core_fields
}
cash_flow_source_ok = (
_mapping(financial_metrics.get("cash_flow_ttm")).get("source_evidence_ok") is True
)
core_checks["cash_flow_period_sources"] = cash_flow_source_ok
award(
"sec_financials_verified",
all(core_checks.values()),
"core financial fields and TTM cash-flow periods have primary-source evidence",
core_checks,
)
period_rows = _mapping(valuation.get("periods"))
estimate_checks: dict[str, bool] = {}
for key in ("current", "year_2", "year_3"):
row = _mapping(period_rows.get(key))
if _number(row.get("metric")) is None:
continue
estimate_checks[key] = (
(_text(row.get("source_type")) or "") in ALLOWED_SOURCE_TYPES
and bool(_text(row.get("retrieved_at")))
and _sources_resolve(_list(row.get("source_ids")), source_index)
)
award(
"guidance_consensus_labeled",
bool(estimate_checks) and all(estimate_checks.values()),
"each valuation period is classified, dated, and sourced",
estimate_checks,
)
award(
"forecast_bridge_verified",
valuation.get("forecast_bridge_valid") is True,
"year-2/year-3 forecast arithmetic and source evidence pass",
valuation.get("forecast_bridge_details"),
)
diluted_ids = _candidate_source_ids(candidate, "financials.histories.diluted_shares")
award(
"diluted_shares_verified",
len(_list(histories.get("diluted_shares"))) >= 2
and _has_tier(diluted_ids, source_index, {1, 2}),
"diluted-share history has primary-source evidence",
diluted_ids,
)
sbc_ids = _candidate_source_ids(candidate, "financials.sbc_ttm")
award(
"sbc_verified",
_number(financials.get("sbc_ttm")) is not None and _has_tier(sbc_ids, source_index, {1, 2}),
"TTM SBC has primary-source evidence",
sbc_ids,
)
peers_ok = valuation.get("peer_set_basis_valid") is True
award(
"peer_set_verified",
peers_ok,
"three or more genuine peers have reasons, consistent forward basis, and sources",
valuation.get("calculated_peer_median_multiple"),
)
award(
"gaap_non_gaap_reconciled",
valuation.get("gaap_reconciliation_valid") is True,
"all non-GAAP/normalized periods reconcile to GAAP",
valuation.get("gaap_reconciliation_details"),
)
corporate_ok = (
corporate_action.get("evidence_ok") is True
and corporate_action.get("listing_status") == "active"
and corporate_action.get("mna_status") in {"none", "terminated"}
and corporate_action.get("symbol_active") is True
)
award(
"corporate_actions_verified",
corporate_ok,
"fresh listing/M&A preflight has resolving evidence",
corporate_action.get("source_ids"),
)
cyclicality = _mapping(candidate.get("cyclicality"))
peak_profit = cyclicality.get("peak_profit_risk") is True
cycle_required = peak_profit or not (cyclicality_score is not None and cyclicality_score <= 2)
if not cycle_required:
cycle_ok = True
cycle_evidence: Any = "not required for low cyclicality without peak-profit risk"
else:
normalization = _mapping(cyclicality.get("normalization"))
normalization_ids = [_text(value) for value in _list(normalization.get("source_ids"))]
cycle_ok = (
_number(normalization.get("normalized_metric")) is not None
and bool(_text(normalization.get("method")))
and _sources_resolve(normalization_ids, source_index)
)
cycle_evidence = normalization
award(
"cyclical_normalization_verified",
cycle_ok,
"mid-cycle normalization is evidenced whenever cyclicality or peak-profit risk requires it",
cycle_evidence,
)
cash = _mapping(financials.get("cash_classification"))
cash_ids = _candidate_source_ids(candidate, "financials.cash_classification")
profile_type = (_text(sector_profile.get("type")) or "general").lower()
cash_ok = _number(cash.get("corporate_cash")) is not None and _has_tier(
cash_ids, source_index, {1, 2}
)
if profile_type == "payments":
cash_ok = cash_ok and _number(cash.get("customer_or_settlement_funds")) is not None
award(
"cash_classification_verified",
cash_ok,
"corporate cash is source-backed and separated from restricted/customer funds where applicable",
cash_ids,
)
roic_ids = _candidate_source_ids(candidate, "financials.roic_pct")
ebitda_ids = _candidate_source_ids(candidate, "financials.ebitda_ttm")
roic_ok = _number(financials.get("roic_pct")) is not None and _has_tier(
roic_ids, source_index, {1, 2, 4}
)
ebitda_ok = _number(financials.get("ebitda_ttm")) is not None and _has_tier(
ebitda_ids, source_index, {1, 2, 4}
)
award(
"roic_ebitda_evidence_verified",
roic_ok and ebitda_ok,
"ROIC and EBITDA are calculated or reported with resolving evidence",
{"roic": roic_ids, "ebitda": ebitda_ids},
)
sector_required = profile_type in {"commercial_biopharma", "payments", "auto_dealership"}
if not sector_required:
sector_ok = True
sector_evidence: Any = "not required for general profile"
elif profile_type == "commercial_biopharma":
sector_ok = (
sector_profile.get("source_evidence_ok") is True
and _number(sector_profile.get("top_product_revenue_pct")) is not None
and bool(_text(sector_profile.get("nearest_material_loe_date")))
and bool(_list(sector_profile.get("loe_stress_scenarios")))
)
sector_evidence = sector_profile
elif profile_type == "payments":
sector_ok = sector_profile.get("source_evidence_ok") is True and cash_ok
sector_evidence = sector_profile
else:
sector_ok = (
sector_profile.get("source_evidence_ok") is True
and sector_profile.get("floorplan_debt_excluded") is True
)
sector_evidence = sector_profile
award(
"sector_risk_verified",
sector_ok,
"material sector-specific risks and stress cases are source-backed",
sector_evidence,
)
raw_score = sum(detail["weight"] for detail in details.values() if detail["awarded"])
caps = [
_integer(item.get("cap"))
for item in _list(contract.get("quality_caps"))
if _integer(item.get("cap")) is not None
]
intrinsic_caps: list[dict[str, Any]] = []
for key, cap, reason in (
("forecast_bridge_verified", 60, "forecast bridge is not independently driver-derived"),
("sec_financials_verified", 65, "core financial or TTM cash-flow evidence is incomplete"),
("cash_classification_verified", 70, "corporate cash classification is incomplete"),
("cyclical_normalization_verified", 65, "required mid-cycle normalization is missing"),
("sector_risk_verified", 65, "required sector-specific risk/LOE stress is missing"),
("roic_ebitda_evidence_verified", 65, "ROIC/EBITDA evidence is incomplete"),
):
if details.get(key, {}).get("evidence_ok") is not True:
caps.append(cap)
intrinsic_caps.append({"cap": cap, "reason": reason})
applied_cap = min(caps) if caps else 100
score = min(raw_score, applied_cap)
details["_quality_cap"] = {
"evidence_ok": applied_cap == 100,
"awarded": True,
"weight": 0,
"reason": "quality score is capped by global and candidate contract failures",
"evidence": deepcopy(_list(contract.get("quality_caps"))) + intrinsic_caps,
"raw_score": raw_score,
"applied_cap": applied_cap,
}
for key, detail in details.items():
if key != "_quality_cap" and not detail["evidence_ok"]:
warnings.append(f"data-quality item failed: {key}")
if applied_cap < raw_score:
warnings.append(f"data-quality score capped at {applied_cap} from raw {raw_score}")
return int(score), details, warnings
def _highest_scenario_upside(valuation: Mapping[str, Any], scenario_key: str) -> float | None:
scenario = _mapping(valuation.get(scenario_key))
rankable = _mapping(valuation.get("rankable_horizons"))
values = [
_number(_mapping(scenario.get(key)).get("upside_pct"))
for key in ("year_2", "year_3")
if rankable.get(key) is True
]
valid = [value for value in values if value is not None]
return max(valid) if valid else None
def _preferred_three_year_upside(valuation: Mapping[str, Any], scenario_key: str) -> float | None:
scenario = _mapping(valuation.get(scenario_key))
rankable = _mapping(valuation.get("rankable_horizons"))
if rankable.get("year_3") is True:
value = _number(_mapping(scenario.get("year_3")).get("upside_pct"))
if value is not None:
return value
if rankable.get("year_2") is True:
return _number(_mapping(scenario.get("year_2")).get("upside_pct"))
return None
def _estimate_range_scenarios(
valuation: Mapping[str, Any],
*,
price: float | None,
) -> dict[str, dict[str, Any]]:
"""Calculate low/high consensus scenarios using the same current multiple.
The base scenario can overstate robustness when analyst ranges are wide.
These scenarios are not alternative forecasts; they are deterministic
sensitivity checks using reported consensus bounds.
"""
current_multiple = _number(valuation.get("current_multiple"))
periods = _mapping(valuation.get("periods"))
output: dict[str, dict[str, Any]] = {}
for key in ("year_2", "year_3"):
row = _mapping(periods.get(key))
years = _number(row.get("years"))
low = _number(row.get("estimate_low"))
high = _number(row.get("estimate_high"))
output[key] = {
"low": _scenario(low, current_multiple, price, years)
if low is not None and low > 0
else None,
"high": _scenario(high, current_multiple, price, years)
if high is not None and high > 0
else None,
}
return output
def _best_low_case_upside(valuation: Mapping[str, Any]) -> float | None:
rankable = _mapping(valuation.get("rankable_horizons"))
ranges = _mapping(valuation.get("estimate_range_scenarios"))
values: list[float] = []
for key in ("year_2", "year_3"):
if rankable.get(key) is not True:
continue
value = _number(_mapping(_mapping(ranges.get(key)).get("low")).get("upside_pct"))
if value is not None:
values.append(value)
return max(values) if values else None
def _quality_eligibility_gate(
*,
financial_metrics: Mapping[str, Any],
valuation: Mapping[str, Any],
sector_profile: Mapping[str, Any],
final_score: float,
config: Mapping[str, Any],
) -> tuple[list[str], list[str]]:
"""Return (ordinary failures, severe failures) for formal eligibility.
The screen may surface low-P/E transition stories, but a final ranked name
must also show credible cash-flow support, acceptable capital efficiency,
and downside resilience. Failing this gate routes the name to conditional
or review-required rather than presenting a weak candidate as a formal win.
"""
failures: list[str] = []
severe: list[str] = []
if final_score < float(config.get("minimum_final_score_for_eligible", 70.0)):
failures.append("final_score_below_eligible_floor")
adj_yield = _number(financial_metrics.get("sbc_adjusted_fcf_yield_pct"))
ev_to_fcf = _number(financial_metrics.get("ev_to_fcf"))
min_adj_yield = float(config.get("minimum_sbc_adjusted_fcf_yield_pct_for_eligible", 3.0))
max_ev_fcf = float(config.get("maximum_ev_to_fcf_for_eligible", 30.0))
cashflow_supported = (adj_yield is not None and adj_yield >= min_adj_yield) or (
ev_to_fcf is not None and ev_to_fcf <= max_ev_fcf
)
if not cashflow_supported:
failures.append("valuation_not_supported_by_sbc_adjusted_fcf")
if (adj_yield is not None and adj_yield < 1.0) or (
ev_to_fcf is not None and ev_to_fcf > 50.0
):
severe.append("severely_weak_fcf_support")
roic = _number(financial_metrics.get("roic_pct"))
if roic is None or roic < float(config.get("minimum_roic_pct_for_eligible", 8.0)):
failures.append("roic_below_eligible_floor")
leverage = _number(financial_metrics.get("net_debt_to_ebitda"))
if leverage is not None and leverage > float(
config.get("maximum_net_debt_to_ebitda_for_eligible", 3.0)
):
failures.append("leverage_above_eligible_floor")
dilution = _number(financial_metrics.get("diluted_share_cagr_pct"))
if dilution is not None and dilution > float(
config.get("maximum_dilution_pct_for_eligible", 5.0)
):
failures.append("dilution_above_eligible_floor")
low_case = _best_low_case_upside(valuation)
min_low = float(config.get("minimum_low_case_upside_pct_for_eligible", 15.0))
if low_case is None or low_case < min_low:
failures.append("consensus_low_case_upside_below_eligible_floor")
loe_scenarios = _list(sector_profile.get("loe_stress_scenarios"))
loe_downsides = [
_number(_mapping(row).get("upside_pct"))
for row in loe_scenarios
if _number(_mapping(row).get("upside_pct")) is not None
]
if loe_downsides and min(loe_downsides) < float(
config.get("severe_loe_stress_downside_pct", -25.0)
):
failures.append("material_loe_tail_risk")
severe.append("material_loe_tail_risk")
return list(dict.fromkeys(failures)), list(dict.fromkeys(severe))
def _screened_out_result(candidate: Mapping[str, Any]) -> dict[str, Any]:
identity = _mapping(candidate.get("identity"))
decision = _mapping(candidate.get("screening_decision"))
reasons = [str(value) for value in _list(decision.get("reasons")) if str(value).strip()]
return {
"symbol": _text(identity.get("symbol")) or "UNKNOWN",
"company_name": _text(identity.get("company_name")) or "Unknown company",
"status": "screened_out",
"screening_reasons": reasons or ["broad-screen economics did not qualify"],
"identity": deepcopy(identity),
"sources": deepcopy(_list(candidate.get("sources"))),
}
def evaluate_candidate(
candidate: Mapping[str, Any],
*,
config: Mapping[str, Any],
analysis_as_of: datetime,
price_basis: Mapping[str, Any],
strict: bool,
) -> dict[str, Any]:
identity = _mapping(candidate.get("identity"))
symbol = _text(identity.get("symbol")) or "UNKNOWN"
company_name = _text(identity.get("company_name")) or "Unknown company"
screening = _mapping(candidate.get("screening_decision"))
screening_status = (_text(screening.get("status")) or "passed").lower()
if screening_status == "screened_out":
return _screened_out_result(candidate)
contract = _mapping(candidate.get("_contract"))
hard: list[str] = []
review: list[str] = list(_list(contract.get("review_reasons")))
warnings: list[str] = list(_list(contract.get("warnings")))
unresolved: list[str] = []
if screening_status not in ALLOWED_SCREENING_STATUSES:
review.append(f"invalid screening_decision.status: {screening_status!r}")
elif screening_status == "exception_admitted":
warnings.append("candidate was admitted as an explicit broad-screen exception")
elif screening_status == "near_miss_review":
warnings.append("candidate advanced from the broad-screen near-miss review lane")
elif screening_status == "sector_review_required":
warnings.append("candidate advanced with mandatory sector/cycle deep-dive requirements")
exchange = (_text(identity.get("exchange")) or "").upper()
if exchange not in ALLOWED_EXCHANGES:
hard.append(f"exchange {exchange!r} is outside NYSE/Nasdaq/NYSE American")
special_case = (_text(identity.get("special_case")) or "none").lower()
if special_case != "none" and not bool(config.get("allow_special_cases", True)):
hard.append(f"special-case sector {special_case!r} is disabled by configuration")
price = _number(identity.get("price"))
market_cap = _number(identity.get("market_cap"))
adv = _number(identity.get("average_daily_dollar_volume"))
if price is None or price <= 0:
hard.append("price must be positive")
if market_cap is None or market_cap <= 0:
hard.append("market cap must be positive")
min_price = _number(config.get("min_price")) or 5.0
if price is not None and price < min_price:
hard.append(f"price {price:.2f} is below configured minimum {min_price:.2f}")
hard_liquidity = _number(config.get("hard_min_average_daily_dollar_volume")) or 1_000_000
liquidity_penalty = 0
if adv is None:
review.append("average daily dollar volume is not verified")
liquidity_penalty = 2
elif adv < hard_liquidity:
hard.append(
f"average daily dollar volume {adv:.0f} is below hard minimum {hard_liquidity:.0f}"
)
elif adv < (_number(config.get("min_average_daily_dollar_volume")) or 5_000_000):
warnings.append("liquidity is below the preferred threshold")
liquidity_penalty = 2
min_cap = _number(config.get("min_market_cap")) or 500_000_000
max_cap = _number(config.get("max_market_cap")) or 20_000_000_000
if market_cap is not None and not (min_cap <= market_cap <= max_cap):
exception_reason = _text(screening.get("exception_reason"))
if not exception_reason:
review.append(
"market cap is outside the focus range without an explicit exception reason"
)
else:
warnings.append(f"market-cap exception: {exception_reason}")
flags = _mapping(candidate.get("flags"))
for key, label in HARD_FLAG_LABELS.items():
if flags.get(key) is True:
hard.append(label)
source_index = _source_index(candidate)
if not _has_primary_source(source_index):
review.append("no Tier 1 or Tier 2 primary source is present")
corporate_action, corp_warnings, corp_review, corp_hard = _validate_corporate_action(
candidate, source_index, analysis_as_of, config
)
warnings.extend(corp_warnings)
review.extend(corp_review)
hard.extend(corp_hard)
price_as_of = None
try:
price_as_of = parse_iso8601(price_basis.get("as_of"), "price_basis.as_of", required=True)
except InputError as exc:
review.append(str(exc))
session = (_text(price_basis.get("session")) or "").lower()
if session not in ALLOWED_SESSIONS:
review.append("price_basis.session is missing or invalid")
if price_as_of is not None:
quote_age_days = (analysis_as_of - price_as_of).total_seconds() / 86_400
max_quote_age = _number(config.get("max_quote_age_days")) or 7.0
if quote_age_days > max_quote_age:
message = f"price snapshot is stale ({quote_age_days:.1f} days old; maximum {max_quote_age:.0f})"
warnings.append(message)
if strict:
review.append(message)
if quote_age_days < -1:
review.append("price snapshot timestamp is after analysis_as_of")
latest = _mapping(candidate.get("latest_earnings"))
latest_records = _mapping(candidate.get("latest_earnings_records"))
latest_published = None
if _text(contract.get("latest_published_at")):
try:
latest_published = parse_iso8601(
contract.get("latest_published_at"), "latest_published_at"
)
except InputError as exc:
review.append(str(exc))
if latest_published and price_as_of and latest_published > price_as_of:
message = "price snapshot predates the latest earnings release"
warnings.append(message)
if strict:
review.append(message)
material_event = (
parse_iso8601(
corporate_action.get("latest_material_event_at"),
"corporate_action_check.latest_material_event_at",
)
if _text(corporate_action.get("latest_material_event_at"))
else None
)
if material_event and price_as_of and material_event > price_as_of:
message = "price snapshot predates a material corporate action"
warnings.append(message)
if strict:
review.append(message)
financial_metrics, fin_warnings, fin_review, fin_unresolved = _calculate_financial_metrics(
candidate, config, source_index
)
warnings.extend(fin_warnings)
if strict:
review.extend(fin_review)
unresolved.extend(fin_unresolved)
valuation, val_warnings, val_review, val_hard = _calculate_valuation(
candidate, config, analysis_as_of, source_index
)
valuation["estimate_range_scenarios"] = _estimate_range_scenarios(valuation, price=price)
warnings.extend(val_warnings)
if strict:
review.extend(val_review)
hard.extend(val_hard)
cyclicality = _mapping(candidate.get("cyclicality"))
cyclicality_score = _integer(cyclicality.get("score"))
if cyclicality_score not in {1, 2, 3, 4, 5}:
review.append("cyclicality score must be an integer from 1 to 5")
cyclicality_score = None
max_without_normalization = (
_integer(config.get("maximum_cyclicality_without_normalization")) or 2
)
normalization_required = (
cyclicality.get("normalization_required") is True
or cyclicality.get("peak_profit_risk") is True
or (cyclicality_score is not None and cyclicality_score > max_without_normalization)
)
normalization = _mapping(cyclicality.get("normalization"))
normalization_valid = not normalization_required or (
_number(normalization.get("normalized_metric")) is not None
and bool(_text(normalization.get("method")))
and _sources_resolve(_list(normalization.get("source_ids")), source_index)
)
if strict and normalization_required and not normalization_valid:
review.append("cyclical normalization is required but not evidenced")
sector_profile, sector_warnings, sector_review, sector_penalty = _validate_sector_profile(
candidate, financial_metrics, valuation, analysis_as_of, source_index, config, strict
)
warnings.extend(sector_warnings)
review.extend(sector_review)
price_source_id = _text(identity.get("price_source_id")) or _text(price_basis.get("source_id"))
quality_score, quality_details, quality_warnings = _calculate_data_quality(
candidate,
source_index,
price_source_id,
valuation,
financial_metrics,
sector_profile,
cyclicality_score,
corporate_action,
contract,
)
warnings.extend(quality_warnings)
minimum_quality = _integer(config.get("minimum_data_quality_score")) or 70
if quality_score < minimum_quality:
review.append(
f"data-quality score {quality_score} is below configured minimum {minimum_quality}"
)
score_total, score_components, score_warnings = _calculate_score_components(candidate)
warnings.extend(score_warnings)
if score_warnings and strict:
review.append("one or more raw score components are missing or invalid")
best_constant = _highest_scenario_upside(valuation, "constant_multiple")
best_stress = _highest_scenario_upside(valuation, "multiple_contraction")
min_upside = _number(config.get("minimum_constant_multiple_upside_pct")) or 30.0
if bool(config.get("require_minimum_upside", True)) and (
best_constant is None or best_constant < min_upside
):
review.append(
f"rankable forward constant-multiple upside does not reach configured minimum {min_upside:.1f}%"
)
min_stress = _number(config.get("minimum_stressed_upside_pct")) or 0.0
if bool(config.get("require_positive_stress_case", False)) and (
best_stress is None or best_stress < min_stress
):
review.append(
f"multiple-contraction stress upside does not reach configured minimum {min_stress:.1f}%"
)
minimum_analyst_count = _integer(valuation.get("minimum_analyst_count"))
max_dispersion = _number(valuation.get("maximum_estimate_dispersion_pct"))
penalties = {
"data_quality": _data_quality_penalty(quality_score),
"cyclicality": _cyclicality_penalty(cyclicality_score),
"estimate_breadth": _estimate_breadth_penalty(minimum_analyst_count),
"estimate_dispersion": _estimate_dispersion_penalty(max_dispersion),
"sbc": _sbc_penalty(_number(financial_metrics.get("sbc_revenue_pct"))),
"dilution": _dilution_penalty(_number(financial_metrics.get("diluted_share_cagr_pct"))),
"liquidity": liquidity_penalty,
"sector_specific_risk": sector_penalty,
}
total_penalty = sum(penalties.values())
final_score = max(0.0, min(100.0, score_total - total_penalty))
quality_gate_failures, severe_quality_gate_failures = _quality_eligibility_gate(
financial_metrics=financial_metrics,
valuation=valuation,
sector_profile=sector_profile,
final_score=final_score,
config=config,
)
hard = list(dict.fromkeys(hard))
review = [reason for reason in dict.fromkeys(review) if reason not in hard]
warnings = list(dict.fromkeys(warnings))
unresolved = list(dict.fromkeys(unresolved))
if hard:
status = "excluded"
elif review:
status = "review_required"
elif severe_quality_gate_failures:
status = "review_required"
review.extend(severe_quality_gate_failures)
elif quality_gate_failures:
maximum_conditional = int(config.get("maximum_quality_gate_failures_for_conditional", 2))
if len(quality_gate_failures) <= maximum_conditional:
status = "conditional"
else:
status = "review_required"
review.extend(quality_gate_failures)
else:
status = "eligible"
if status == "eligible" and quality_score >= 85 and (minimum_analyst_count or 0) >= 5:
confidence = "high"
elif status == "eligible":
confidence = "medium"
elif status == "conditional":
confidence = "low"
elif status == "review_required":
confidence = "low"
else:
confidence = "not_ranked"
return {
"symbol": symbol,
"company_name": company_name,
"status": status,
"status_reasons": hard
if status == "excluded"
else review
if status == "review_required"
else quality_gate_failures
if status == "conditional"
else [],
"hard_exclusion_reasons": hard,
"review_reasons": review,
"conditional_reasons": quality_gate_failures,
"quality_gate_failures": quality_gate_failures,
"severe_quality_gate_failures": severe_quality_gate_failures,
"warnings": warnings,
"unresolved_fields": unresolved,
"confidence": confidence,
"identity": deepcopy(identity),
"screening_decision": deepcopy(screening),
"corporate_action_check": corporate_action,
"latest_earnings": deepcopy(latest),
"latest_earnings_records": deepcopy(latest_records),
"financial_metrics": financial_metrics,
"valuation": valuation,
"cash_consistency": deepcopy(_mapping(contract.get("cash_consistency"))),
"cyclicality": {
"score": cyclicality_score,
"position": _text(cyclicality.get("position")),
"peak_profit_risk": cyclicality.get("peak_profit_risk") is True,
"normalization_required": normalization_required,
"normalization_valid": normalization_valid,
"normalization": deepcopy(normalization),
},
"sector_profile": sector_profile,
"data_quality_score": quality_score,
"data_quality_details": quality_details,
"raw_score": _round(score_total, 2),
"score_components": score_components,
"penalties": penalties,
"total_penalty": total_penalty,
"final_score": _round(final_score, 2),
"qualitative": deepcopy(_mapping(candidate.get("qualitative"))),
"peers": deepcopy(_list(candidate.get("peers"))),
"sources": deepcopy(_list(candidate.get("sources"))),
"contract": deepcopy(contract),
}
def _sort_key(result: Mapping[str, Any]) -> tuple[float, float, float, float, float, str]:
valuation = _mapping(result.get("valuation"))
constant_upside = _preferred_three_year_upside(valuation, "constant_multiple")
stress_upside = _preferred_three_year_upside(valuation, "multiple_contraction")
dilution = _number(_mapping(result.get("financial_metrics")).get("diluted_share_cagr_pct"))
return (
-(_number(result.get("final_score")) or 0.0),
-(constant_upside if constant_upside is not None else -10_000.0),
-(stress_upside if stress_upside is not None else -10_000.0),
-(_number(result.get("data_quality_score")) or 0.0),
dilution if dilution is not None else 10_000.0,
_text(result.get("symbol")) or "",
)
def _review_sort_key(result: Mapping[str, Any]) -> tuple[float, float, str]:
return (
-(_number(result.get("final_score")) or 0.0),
-(
_preferred_three_year_upside(_mapping(result.get("valuation")), "constant_multiple")
or -10_000
),
_text(result.get("symbol")) or "",
)
def _select_final_three(
ranked: Sequence[Mapping[str, Any]],
*,
ranking_status: str,
config: Mapping[str, Any],
) -> dict[str, Any]:
if not ranked or ranking_status != "final":
return {"highest_conviction": None, "most_undervalued": None, "largest_upside": None}
def summary(row: Mapping[str, Any]) -> dict[str, Any]:
valuation = _mapping(row.get("valuation"))
qualitative = _mapping(row.get("qualitative"))
return {
"symbol": row.get("symbol"),
"company_name": row.get("company_name"),
"final_score": row.get("final_score"),
"constant_multiple_upside_pct": _preferred_three_year_upside(
valuation, "constant_multiple"
),
"stress_upside_pct": _preferred_three_year_upside(valuation, "multiple_contraction"),
"low_case_upside_pct": _best_low_case_upside(valuation),
"data_quality_score": row.get("data_quality_score"),
"investment_thesis": qualitative.get("investment_thesis"),
"market_may_be_missing": qualitative.get("market_may_be_missing"),
"best_catalyst": qualitative.get("best_catalyst"),
"maximum_risk": qualitative.get("maximum_risk"),
"do_not_buy_reason": qualitative.get("do_not_buy_reason"),
"bear_case": qualitative.get("bear_case"),
"invalidation_conditions": qualitative.get("invalidation_conditions"),
"next_earnings_kpis": qualitative.get("next_earnings_kpis"),
}
min_score = float(config.get("final_three_minimum_score", 75.0))
min_dq = int(config.get("final_three_minimum_data_quality", 80))
min_stress = float(config.get("final_three_minimum_stress_upside_pct", 0.0))
min_adj_yield = float(config.get("final_three_minimum_sbc_adjusted_fcf_yield_pct", 5.0))
max_ev_fcf = float(config.get("final_three_maximum_ev_to_fcf", 20.0))
min_low_case = float(config.get("final_three_minimum_low_case_upside_pct", 20.0))
conviction_pool = [
row
for row in ranked
if (_number(row.get("final_score")) or 0.0) >= min_score
and (_number(row.get("data_quality_score")) or 0.0) >= min_dq
and (
_preferred_three_year_upside(_mapping(row.get("valuation")), "multiple_contraction")
or -10_000
)
>= min_stress
]
undervalued_pool = [
row
for row in ranked
if (
(
_number(_mapping(row.get("financial_metrics")).get("sbc_adjusted_fcf_yield_pct"))
or -10_000
)
>= min_adj_yield
or (_number(_mapping(row.get("financial_metrics")).get("ev_to_fcf")) or 10_000)
<= max_ev_fcf
)
]
upside_pool = [
row
for row in ranked
if (_best_low_case_upside(_mapping(row.get("valuation"))) or -10_000) >= min_low_case
]
highest_conviction = max(
conviction_pool,
key=lambda row: (
_number(row.get("final_score")) or 0,
_number(row.get("data_quality_score")) or 0,
1 if row.get("confidence") == "high" else 0,
),
default=None,
)
most_undervalued = max(
undervalued_pool,
key=lambda row: (
_number(_mapping(row.get("financial_metrics")).get("sbc_adjusted_fcf_yield_pct"))
or -10_000,
-(_number(_mapping(row.get("financial_metrics")).get("ev_to_fcf")) or 10_000),
),
default=None,
)
largest_upside = max(
upside_pool,
key=lambda row: (
_preferred_three_year_upside(_mapping(row.get("valuation")), "constant_multiple")
or -10_000
),
default=None,
)
return {
"highest_conviction": summary(highest_conviction) if highest_conviction else None,
"most_undervalued": summary(most_undervalued) if most_undervalued else None,
"largest_upside": summary(largest_upside) if largest_upside else None,
}
def _validate_funnel(funnel: Mapping[str, Any], strict: bool) -> list[str]:
warnings: list[str] = []
required = (
"universe_count",
"listing_in_scope_count",
"candidate_pool_count",
"discovery_evaluable_count",
"deep_dive_selected_count",
"deep_dive_completed_count",
)
values: dict[str, int | None] = {}
for key in required:
values[key] = _integer(funnel.get(key))
if values[key] is None:
warnings.append(f"screening_funnel.{key} is missing")
if all(values[key] is not None for key in required):
universe = int(values["universe_count"] or 0)
in_scope = int(values["listing_in_scope_count"] or 0)
pool = int(values["candidate_pool_count"] or 0)
evaluable = int(values["discovery_evaluable_count"] or 0)
selected = int(values["deep_dive_selected_count"] or 0)
completed = int(values["deep_dive_completed_count"] or 0)
if in_scope > universe:
warnings.append("listing_in_scope_count exceeds universe_count")
if pool > universe:
warnings.append("candidate_pool_count exceeds universe_count")
if evaluable > pool:
warnings.append("discovery_evaluable_count exceeds candidate_pool_count")
if selected > evaluable:
warnings.append("deep_dive_selected_count exceeds discovery_evaluable_count")
if completed > selected:
warnings.append("deep_dive_completed_count exceeds deep_dive_selected_count")
preflight = _integer(funnel.get("preflight_passed_count"))
selected = values.get("deep_dive_selected_count")
if preflight is not None and selected is not None and preflight > selected:
warnings.append("preflight_passed_count exceeds deep_dive_selected_count")
if strict and not funnel:
warnings.append("screening_funnel is absent; tiered coverage cannot be audited")
return warnings
def evaluate_snapshot(
payload: Mapping[str, Any],
*,
strict: bool = False,
top: int | None = None,
artifact_root: Path | None = None,
) -> dict[str, Any]:
try:
normalized_payload, contract = validate_and_normalize_snapshot(
payload, artifact_root=artifact_root
)
except ContractError as exc:
raise InputError(str(exc)) from exc
analysis_as_of = parse_iso8601(
normalized_payload.get("analysis_as_of"), "analysis_as_of", required=True
)
assert analysis_as_of is not None
price_basis = _mapping(normalized_payload.get("price_basis"))
session = (_text(price_basis.get("session")) or "").lower()
if session not in ALLOWED_SESSIONS:
raise InputError("price_basis.session is missing or invalid")
parse_iso8601(price_basis.get("as_of"), "price_basis.as_of", required=True)
config = _deep_merge(DEFAULT_CONFIG, _mapping(normalized_payload.get("config")))
max_candidates = top if top is not None else _integer(config.get("max_candidates")) or 10
if max_candidates <= 0:
raise InputError("max candidates must be positive")
candidates = _list(normalized_payload.get("candidates"))
candidate_pool_status = (_text(contract.get("candidate_pool_status")) or "").lower()
no_qualifying_candidates = candidate_pool_status in {
"no_qualifying_candidates",
"no_qualifying_candidates_in_bounded_pool",
}
unresolved_pool_statuses = {"insufficient_data", "sufficient_pending_enrichment"}
if (
not candidates
and not no_qualifying_candidates
and candidate_pool_status not in unresolved_pool_statuses
):
raise InputError(
"candidates must contain selected candidate records unless the audited pool is unresolved or found no qualifying candidates"
)
results = [
evaluate_candidate(
_mapping(candidate),
config=config,
analysis_as_of=analysis_as_of,
price_basis=price_basis,
strict=strict,
)
for candidate in candidates
]
eligible = sorted((row for row in results if row.get("status") == "eligible"), key=_sort_key)
conditional = sorted(
(row for row in results if row.get("status") == "conditional"), key=_review_sort_key
)
review = sorted(
(row for row in results if row.get("status") == "review_required"), key=_review_sort_key
)
screened_out = sorted(
(row for row in results if row.get("status") == "screened_out"),
key=lambda row: row.get("symbol") or "",
)
excluded = sorted(
(row for row in results if row.get("status") == "excluded"),
key=lambda row: row.get("symbol") or "",
)
ranked = eligible[:max_candidates]
for index, row in enumerate(ranked, start=1):
row["rank"] = index
funnel = _mapping(normalized_payload.get("screening_funnel"))
selected_symbol_set = {
str(value).upper()
for value in _list(contract.get("selected_symbols"))
if str(value).strip()
}
selected_results = [
row for row in results if str(row.get("symbol") or "").upper() in selected_symbol_set
]
actual_preflight = sum(
1
for row in selected_results
if _mapping(row.get("corporate_action_check")).get("evidence_ok") is True
and _mapping(row.get("corporate_action_check")).get("listing_status") == "active"
and _mapping(row.get("corporate_action_check")).get("mna_status") in {"none", "terminated"}
and _mapping(row.get("corporate_action_check")).get("symbol_active") is True
)
funnel_contract_errors: list[str] = []
declared_preflight = _integer(funnel.get("preflight_passed_count"))
if declared_preflight != actual_preflight:
funnel_contract_errors.append(
f"screening_funnel.preflight_passed_count={declared_preflight!r} does not equal verified selected-candidate preflight count {actual_preflight}"
)
declared_completed = _integer(funnel.get("deep_dive_completed_count"))
actual_completed = len(selected_results)
if declared_completed != actual_completed:
funnel_contract_errors.append(
f"screening_funnel.deep_dive_completed_count={declared_completed!r} does not equal selected candidate record count {actual_completed}"
)
if funnel_contract_errors:
contract["valid"] = False
contract.setdefault("review_reasons", []).extend(funnel_contract_errors)
run_metadata = _mapping(normalized_payload.get("run_metadata"))
declared_status = (_text(run_metadata.get("status")) or "partial").lower()
unprocessed = [str(value) for value in _list(run_metadata.get("unprocessed_candidates"))]
ranking_status = (
"final"
if declared_status == "complete" and not unprocessed and contract.get("valid") is True
else "provisional"
)
global_warnings = _validate_funnel(funnel, strict)
global_warnings.extend(_list(contract.get("warnings")))
global_warnings.extend(_list(contract.get("review_reasons")))
if ranking_status == "provisional":
global_warnings.append(
"run contract is incomplete; ranking and final-three selection are provisional"
)
if not ranked:
if ranking_status == "final" and no_qualifying_candidates:
if candidate_pool_status == "no_qualifying_candidates_in_bounded_pool":
global_warnings.append(
"completed bounded candidate pool contained no qualifying company; this is not a market-wide no-candidates claim"
)
else:
global_warnings.append(
"completed full-universe candidate pool contained no company that qualified under the configured GARP gates"
)
elif candidate_pool_status in unresolved_pool_statuses:
global_warnings.append(
"candidate-pool research is unresolved; no-candidates conclusion is prohibited"
)
else:
global_warnings.append("no candidate cleared all strict ranking gates")
broad_rows = _list(
_mapping(normalized_payload.get("screening_audit")).get("actual_candidate_rows")
)
broad_groups: dict[str, list[dict[str, Any]]] = {
"selected": [],
"deferred_by_budget": [],
"review_required": [],
"screened_out": [],
"excluded": [],
"unavailable_after_enrichment": [],
}
for raw in broad_rows:
row = _mapping(raw)
decision = _mapping(row.get("decision"))
status = (_text(decision.get("status")) or "").lower()
entry = {
"symbol": row.get("symbol"),
"company_name": row.get("company_name"),
"status": status,
"preselection_status": decision.get("preselection_status"),
"broad_score": row.get("broad_score"),
"deep_dive_priority_score": row.get("deep_dive_priority_score"),
"review_reasons": decision.get("review_reasons") or [],
"guideline_misses": decision.get("guideline_misses") or [],
"screen_fail_reasons": decision.get("screen_fail_reasons") or [],
"deep_dive_requirements": decision.get("deep_dive_requirements") or [],
"metrics": deepcopy(_mapping(row.get("metrics"))),
}
if status == "selected":
broad_groups["selected"].append(entry)
elif status == "deferred_by_budget":
broad_groups["deferred_by_budget"].append(entry)
elif status in {
"needs_enrichment",
"sector_review_required",
"near_miss_review",
"review_required",
}:
broad_groups["review_required"].append(entry)
elif status == "screened_out":
broad_groups["screened_out"].append(entry)
elif status == "excluded":
broad_groups["excluded"].append(entry)
elif status == "unavailable_after_enrichment":
broad_groups["unavailable_after_enrichment"].append(entry)
for values in broad_groups.values():
values.sort(key=lambda row: str(row.get("symbol") or ""))
broad_counts = {key: len(value) for key, value in broad_groups.items()}
coverage_info = _derive_ranking_scope(
_mapping(normalized_payload.get("screening_audit")), deep_dive_count=len(candidates)
)
return {
"schema_version": SCHEMA_VERSION,
"runtime": runtime_metadata(),
"generated_at": datetime.now(timezone.utc).isoformat(),
"analysis_as_of": analysis_as_of.isoformat(),
"strict_mode": strict,
"run_metadata": deepcopy(run_metadata),
"ranking_status": ranking_status,
"ranking_scope": coverage_info.get("ranking_scope"),
"coverage": coverage_info,
"price_basis": deepcopy(price_basis),
"config": config,
"global_sources": deepcopy(_list(normalized_payload.get("global_sources"))),
"market_context": deepcopy(_mapping(normalized_payload.get("market_context"))),
"screening_funnel": deepcopy(_mapping(normalized_payload.get("screening_funnel"))),
"screening_audit": deepcopy(_mapping(normalized_payload.get("screening_audit"))),
"broad_screen": {"counts": broad_counts, **broad_groups},
"contract": deepcopy(contract),
"counts": {
"input_candidates": len(candidates),
"eligible": len(eligible),
"ranked": len(ranked),
"conditional": len(conditional),
"review_required": len(review),
"screened_out": len(screened_out),
"excluded": len(excluded),
},
"ranked_candidates": ranked,
"conditional": conditional,
"review_required": review,
"screened_out": screened_out,
"excluded": excluded,
"final_three": _select_final_three(ranked, ranking_status=ranking_status, config=config),
"global_warnings": list(dict.fromkeys(global_warnings)),
}
def _label(language: str, en: str, ja: str) -> str:
return ja if language == "ja" else en
def _missing(language: str) -> str:
return "確認できず" if language == "ja" else "not verified"
def _md_escape(value: Any) -> str:
return str(value).replace("|", "\\|").replace("\n", " ") if value is not None else ""
def _scenario_value(result: Mapping[str, Any], scenario: str, horizon: str, key: str) -> Any:
return _mapping(_mapping(_mapping(result.get("valuation")).get(scenario)).get(horizon)).get(key)
def _derive_ranking_scope(audit: Mapping[str, Any], *, deep_dive_count: int) -> dict[str, Any]:
"""Classify how far the run's conclusion may be generalized, with coverage.
``final_marketwide`` requires estimate acquisition attempted for EVERY
listing-universe symbol (exact counts); a bounded but fully processed
subset is ``final_scoped`` (conclusions bind only to that subset); an
unresolved enrichment queue makes the output ``diagnostic``. Listing
enumeration completeness alone never yields a market-wide ranking.
"""
# Delegated to the shared coverage-semantics module (used by discovery
# too) so the tri-state semantics cannot drift between the two sides.
return derive_ranking_scope_from_audit(audit, deep_dive_count=deep_dive_count)
def _render_list(values: Any, missing: str) -> str:
rows = [str(value).strip() for value in _list(values) if str(value).strip()]
return "; ".join(rows) if rows else missing
def _render_latest_earnings(language: str, records: Mapping[str, Any], missing: str) -> list[str]:
lines: list[str] = []
for key, en, ja in (
("quarter", "Latest quarter", "直近四半期"),
("full_year", "Latest full year", "直近通期"),
):
record = _mapping(records.get(key))
if not record:
continue
metrics = _mapping(record.get("metrics"))
lines.append(f"**{_label(language, en, ja)} — {record.get('period', missing)}**")
lines.extend(
[
f"- **{_label(language, 'Period end / published', '期末／発表日時')}:** {record.get('period_end', missing)} / {record.get('published_at', missing)}",
f"- **{_label(language, 'Revenue / YoY', '売上高／前年比')}:** {_format_number(metrics.get('revenue'), 1, missing)} / {_format_pct(metrics.get('revenue_yoy_pct'), 1, missing)}",
f"- **{_label(language, 'GAAP operating income / margin', 'GAAP営業利益／利益率')}:** {_format_number(metrics.get('gaap_operating_income'), 1, missing)} / {_format_pct(metrics.get('gaap_operating_margin_pct'), 1, missing)}",
f"- **GAAP / Adjusted EPS:** {_format_number(metrics.get('gaap_eps'), 2, missing)} / {_format_number(metrics.get('adjusted_eps'), 2, missing)}",
f"- **{_label(language, 'OCF / standard FCF', '営業CF／標準FCF')}:** {_format_number(metrics.get('operating_cash_flow'), 1, missing)} / {_format_number(metrics.get('standard_fcf'), 1, missing)}",
f"- **{_label(language, 'Growth state', '成長状態')}:** {record.get('growth_state') or record.get('derived_growth_state') or missing}",
f"- **{_label(language, 'Guidance', 'ガイダンス')}:** {_render_list(record.get('guidance'), missing)}",
f"- **KPI:** {_render_list(record.get('key_kpis'), missing)}",
f"- **{_label(language, 'One-time items', '一時項目')}:** {_render_list(record.get('one_time_items'), missing)}",
]
)
return lines or [missing]
def _render_peers(language: str, peers: Sequence[Any], missing: str) -> list[str]:
lines = [
"| Ticker | Company | Multiple | Basis | Period | Selection reason |",
"|---|---|---:|---|---|---|",
]
if not peers:
lines.append(f"| {missing} | {missing} | {missing} | {missing} | {missing} | {missing} |")
return lines
for item in peers:
peer = _mapping(item)
lines.append(
"| {symbol} | {company} | {multiple} | {basis} | {period} | {reason} |".format(
symbol=_md_escape(peer.get("symbol", missing)),
company=_md_escape(peer.get("company_name", missing)),
multiple=_format_number(peer.get("multiple"), 2, missing),
basis=_md_escape(peer.get("metric_basis", missing)),
period=_md_escape(peer.get("multiple_period_kind", missing)),
reason=_md_escape(peer.get("selection_reason", missing)),
)
)
return lines
def _render_score_table(row: Mapping[str, Any], missing: str) -> list[str]:
components = _mapping(row.get("score_components"))
penalties = _mapping(row.get("penalties"))
lines = ["| Component | Score | Maximum |", "|---|---:|---:|"]
for key, maximum in SCORE_LIMITS.items():
lines.append(f"| {key} | {_format_number(components.get(key), 1, missing)} | {maximum:g} |")
lines.append(f"| Raw score | {_format_number(row.get('raw_score'), 1, missing)} | 100 |")
lines.append(f"| Penalties | -{_format_number(row.get('total_penalty'), 1, missing)} | — |")
lines.append(f"| Final score | {_format_number(row.get('final_score'), 1, missing)} | 100 |")
lines.append("")
lines.append(
"**Penalty breakdown:** " + ", ".join(f"{key}={value}" for key, value in penalties.items())
)
return lines
def _render_candidate_detail(language: str, row: Mapping[str, Any]) -> list[str]:
missing = _missing(language)
identity = _mapping(row.get("identity"))
metrics = _mapping(row.get("financial_metrics"))
cash_flow = _mapping(metrics.get("cash_flow_ttm"))
valuation = _mapping(row.get("valuation"))
periods = _mapping(valuation.get("periods"))
qualitative = _mapping(row.get("qualitative"))
latest_records = _mapping(row.get("latest_earnings_records"))
cyclicality = _mapping(row.get("cyclicality"))
lines: list[str] = []
lines.append(f"### {row.get('rank')}. {row.get('company_name')} ({row.get('symbol')})")
lines.append("")
lines.append(f"#### 1. {_label(language, 'Basic Information', '基本情報')}")
lines.extend(
[
f"- **{_label(language, 'Exchange / sector / industry', '取引所／セクター／業種')}:** {identity.get('exchange', missing)} / {identity.get('sector', missing)} / {identity.get('industry', missing)}",
f"- **{_label(language, 'Price / market cap', '株価／時価総額')}:** {_format_price(identity.get('price'), identity.get('currency', 'USD'), missing)} / {_format_number(identity.get('market_cap'), 1, missing)}",
f"- **{_label(language, 'Liquidity', '流動性')}:** {_format_number(identity.get('average_daily_dollar_volume'), 1, missing)}",
f"- **{_label(language, 'Business overview', '企業概要')}:** {qualitative.get('business_overview', missing)}",
]
)
lines.append("")
lines.append(f"#### 2. {_label(language, 'Investment Thesis', '投資仮説')}")
lines.append(str(qualitative.get("investment_thesis") or missing))
lines.append("")
lines.append(f"#### 3. {_label(language, 'Valuation', 'バリュエーション')}")
lines.extend(
[
f"- **{_label(language, 'Valuation basis', '評価基準')}:** {valuation.get('basis', missing)} / {valuation.get('current_metric_basis', missing)} / {valuation.get('current_period_kind', missing)}",
f"- **{_label(language, 'Current metric / multiple', '現在指標／倍率')}:** {_format_number(valuation.get('current_metric'), 2, missing)} / {_format_number(valuation.get('current_multiple'), 2, missing)}x",
f"- **{_label(language, 'Peer median multiple', '同業中央値倍率')}:** {_format_number(valuation.get('peer_median_multiple'), 2, missing)}x",
f"- **{_label(language, 'EV/FCF', 'EV/FCF')}:** {_format_number(metrics.get('ev_to_fcf'), 2, missing)}x",
f"- **{_label(language, 'Enterprise value / cash definition', '企業価値／現金定義')}:** {_format_number(metrics.get('enterprise_value'), 1, missing)} / {metrics.get('cash_definition', missing)}",
f"- **{_label(language, 'Standard / SBC-adjusted FCF yield', '標準／SBC調整後FCF利回り')}:** {_format_pct(metrics.get('fcf_yield_pct'), 1, missing)} / {_format_pct(metrics.get('sbc_adjusted_fcf_yield_pct'), 1, missing)}",
]
)
lines.append("")
lines.append(f"#### 4. {_label(language, 'Growth History and Forecasts', '成長実績と予想')}")
lines.extend(
[
f"- **{_label(language, 'Revenue CAGR', '売上CAGR')}:** {_format_pct(metrics.get('revenue_cagr_pct'), 1, missing)}",
f"- **{_label(language, 'GAAP EPS CAGR', 'GAAP EPS CAGR')}:** {_format_pct(metrics.get('gaap_eps_cagr_pct'), 1, missing)}",
f"- **{_label(language, 'FCF/share CAGR', 'FCF per share CAGR')}:** {_format_pct(metrics.get('fcf_per_share_cagr_pct'), 1, missing)}",
f"- **{_label(language, 'Current / year-2 / year-3 metric', '現在／2年後／3年後指標')}:** {_format_number(_mapping(periods.get('current')).get('metric'), 2, missing)} / {_format_number(_mapping(periods.get('year_2')).get('metric'), 2, missing)} / {_format_number(_mapping(periods.get('year_3')).get('metric'), 2, missing)}",
f"- **{_label(language, 'Forecast bridge', '予想ブリッジ')}:** {'PASS' if valuation.get('forecast_bridge_valid') else 'FAIL'}",
]
)
lines.append("")
lines.append(f"#### 5. {_label(language, 'Latest Earnings', '直近決算')}")
lines.extend(_render_latest_earnings(language, latest_records, missing))
lines.append("")
lines.append(f"#### 6. {_label(language, 'Growth Drivers', '成長ドライバー')}")
lines.append(_render_list(qualitative.get("growth_drivers"), missing))
lines.append("")
lines.append(f"#### 7. {_label(language, 'Peer Comparison', '同業他社比較')}")
lines.extend(_render_peers(language, _list(row.get("peers")), missing))
lines.append("")
lines.append(f"#### 8. {_label(language, 'Why the Stock Is Discounted', 'なぜ割安なのか')}")
lines.extend(
[
f"- **{_label(language, 'Market concerns', '市場の懸念')}:** {_render_list(qualitative.get('discount_reasons'), missing)}",
f"- **{_label(language, 'Temporary or structural', '一時的か構造的か')}:** {qualitative.get('discount_classification', missing)}",
f"- **{_label(language, 'How the market may be right', '市場評価が妥当な可能性')}:** {qualitative.get('market_may_be_right', missing)}",
f"- **{_label(language, 'Conditions for discount closure', 'ディスカウント縮小条件')}:** {_render_list(qualitative.get('rerating_conditions'), missing)}",
]
)
lines.append("")
lines.append(
f"#### 9. {_label(language, 'Constant-Multiple Scenario', '倍率据え置きシナリオ')}"
)
lines.append("| Item | Current | Year 2 | Year 3 |")
lines.append("|---|---:|---:|---:|")
lines.append(
f"| Metric | {_format_number(valuation.get('current_metric'), 2, missing)} | {_format_number(_mapping(periods.get('year_2')).get('metric'), 2, missing)} | {_format_number(_mapping(periods.get('year_3')).get('metric'), 2, missing)} |"
)
lines.append(
f"| Multiple | {_format_number(valuation.get('current_multiple'), 2, missing)}x | {_format_number(valuation.get('current_multiple'), 2, missing)}x | {_format_number(valuation.get('current_multiple'), 2, missing)}x |"
)
lines.append(
f"| Implied price | {_format_price(identity.get('price'), identity.get('currency', 'USD'), missing)} | {_format_price(_scenario_value(row, 'constant_multiple', 'year_2', 'implied_price'), identity.get('currency', 'USD'), missing)} | {_format_price(_scenario_value(row, 'constant_multiple', 'year_3', 'implied_price'), identity.get('currency', 'USD'), missing)} |"
)
lines.append(
f"| Upside | — | {_format_pct(_scenario_value(row, 'constant_multiple', 'year_2', 'upside_pct'), 1, missing)} | {_format_pct(_scenario_value(row, 'constant_multiple', 'year_3', 'upside_pct'), 1, missing)} |"
)
lines.append(
f"| Annualized return | — | {_format_pct(_scenario_value(row, 'constant_multiple', 'year_2', 'cagr_pct'), 1, missing)} | {_format_pct(_scenario_value(row, 'constant_multiple', 'year_3', 'cagr_pct'), 1, missing)} |"
)
lines.append("")
lines.append(
f"#### 10. {_label(language, '20% Multiple-Contraction Scenario', '倍率20％低下シナリオ')}"
)
lines.append(
f"- **2-year:** {_format_price(_scenario_value(row, 'multiple_contraction', 'year_2', 'implied_price'), identity.get('currency', 'USD'), missing)} / {_format_pct(_scenario_value(row, 'multiple_contraction', 'year_2', 'upside_pct'), 1, missing)}"
)
lines.append(
f"- **3-year:** {_format_price(_scenario_value(row, 'multiple_contraction', 'year_3', 'implied_price'), identity.get('currency', 'USD'), missing)} / {_format_pct(_scenario_value(row, 'multiple_contraction', 'year_3', 'upside_pct'), 1, missing)}"
)
lines.append("")
lines.append(f"#### 11. {_label(language, 'Catalysts', 'カタリスト')}")
lines.append(_render_list(qualitative.get("catalysts"), missing))
lines.append("")
lines.append(f"#### 12. {_label(language, 'Largest Risk', '最大のリスク')}")
lines.append(str(qualitative.get("maximum_risk") or missing))
lines.append("")
lines.append(f"#### 13. {_label(language, 'Invalidation Conditions', '投資仮説の無効化条件')}")
lines.append(_render_list(qualitative.get("invalidation_conditions"), missing))
lines.append("")
lines.append(f"#### 14. {_label(language, 'Cyclicality', 'シクリカル評価')}")
lines.extend(
[
f"- **{_label(language, 'Cyclicality score', 'シクリカル度')}:** {cyclicality.get('score', missing)}/5",
f"- **{_label(language, 'Cycle position', '現在のサイクル位置')}:** {cyclicality.get('position', missing)}",
f"- **{_label(language, 'Peak-profit risk', 'ピーク利益リスク')}:** {cyclicality.get('peak_profit_risk', missing)}",
f"- **{_label(language, 'Normalized metric', '平準化指標')}:** {_format_number(_mapping(cyclicality.get('normalization')).get('normalized_metric'), 2, missing)}",
]
)
lines.append("")
lines.append(f"#### 15. {_label(language, 'Overall Score', '総合評価')}")
lines.extend(_render_score_table(row, missing))
lines.append("")
lines.append(
f"#### 16. {_label(language, 'Cash-Flow and Evidence Audit', 'キャッシュフロー・証拠監査')}"
)
lines.extend(
[
f"- **{_label(language, 'TTM method', 'TTM再構築方法')}:** {cash_flow.get('method', missing)}",
f"- **{_label(language, 'OCF / capex / standard FCF', '営業CF／設備投資／標準FCF')}:** {_format_number(cash_flow.get('operating_cash_flow'), 1, missing)} / {_format_number(cash_flow.get('capex_cash_outflow'), 1, missing)} / {_format_number(cash_flow.get('standard_fcf'), 1, missing)}",
f"- **{_label(language, 'Company adjusted FCF', '会社Adjusted FCF')}:** {_format_number(cash_flow.get('company_adjusted_fcf'), 1, missing)} ({cash_flow.get('company_adjusted_fcf_definition', missing)})",
f"- **{_label(language, 'Data quality', 'データ品質')}:** {row.get('data_quality_score')}/100",
f"- **{_label(language, 'Cash consistency', '現金・負債整合性')}:** {_md_escape(row.get('cash_consistency'))}",
f"- **{_label(language, 'Warnings', '警告')}:** {_render_list(row.get('warnings'), missing)}",
f"- **{_label(language, 'Unresolved', '未解決')}:** {_render_list(row.get('unresolved_fields'), missing)}",
]
)
lines.append("")
return lines
def render_markdown(report: Mapping[str, Any], *, language: str = "en") -> str:
missing = _missing(language)
ranked = _list(report.get("ranked_candidates"))
conditional = _list(report.get("conditional"))
review = _list(report.get("review_required"))
screened_out = _list(report.get("screened_out"))
excluded = _list(report.get("excluded"))
broad = _mapping(report.get("broad_screen"))
broad_counts = _mapping(broad.get("counts"))
counts = _mapping(report.get("counts"))
funnel = _mapping(report.get("screening_funnel"))
price_basis = _mapping(report.get("price_basis"))
lines: list[str] = []
title = _label(
language,
"US Undervalued Growth Screening Report",
"米国株・割安成長株スクリーニングレポート",
)
coverage = _mapping(report.get("coverage"))
ranking_scope = _text(report.get("ranking_scope"))
if ranking_scope == "final_scoped":
evaluated = coverage.get("economic_attempt_count", "?")
listed = coverage.get("listing_universe_count", "?")
title += _label(
language,
f" — Scoped Pilot ({evaluated} of {listed} listed names economically attempted)",
f" — 限定パイロット（上場{listed}銘柄中{evaluated}銘柄のみ経済評価を試行）",
)
elif ranking_scope == "diagnostic":
title += _label(language, " — DIAGNOSTIC (incomplete scope)", " — 診断用（範囲未完了）")
if report.get("ranking_status") != "final":
title += _label(language, " — PROVISIONAL", " — 暫定")
lines.append(f"# {title}")
lines.append("")
if ranking_scope and ranking_scope != "final_marketwide":
attempt_pct = coverage.get("economic_attempt_coverage_pct")
attempt_pct_text = f"{float(attempt_pct):.2f}%" if attempt_pct is not None else missing
evaluable_pct = coverage.get("economically_evaluable_coverage_pct")
evaluable_pct_text = (
f"{float(evaluable_pct):.2f}%" if evaluable_pct is not None else missing
)
lines.append(
f"- **{_label(language, 'Ranking scope', 'ランキング範囲')}:** {ranking_scope} — "
+ _label(
language,
(
f"conclusions bind ONLY to the {coverage.get('economic_attempt_count', '?')} names whose estimate "
f"acquisition was attempted ({attempt_pct_text} of "
f"{coverage.get('listing_universe_count', '?')} listed; economically evaluable {coverage.get('economically_evaluable_count', '?')} = {evaluable_pct_text}; quality probe {coverage.get('quality_probe_count', '?')}, "
f"deep dive {coverage.get('deep_dive_count', '?')}). This is NOT a market-wide ranking."
),
(
f"結論は予想取得を試行した{coverage.get('economic_attempt_count', '?')}銘柄"
f"（上場{coverage.get('listing_universe_count', '?')}銘柄の{attempt_pct_text}、"
f"評価可能{coverage.get('economically_evaluable_count', '?')}銘柄・{evaluable_pct_text}、品質プローブ{coverage.get('quality_probe_count', '?')}銘柄、詳細分析{coverage.get('deep_dive_count', '?')}銘柄）"
"に限定され、市場全体ランキングではありません。"
),
)
)
lines.append(
f"- **{_label(language, 'Analysis as of', '分析実施日時')}:** {report.get('analysis_as_of')}"
)
lines.append(
f"- **{_label(language, 'Price basis', '株価基準')}:** {price_basis.get('as_of', missing)} / {price_basis.get('session', missing)}"
)
lines.append(
f"- **{_label(language, 'Ranking status', 'ランキング状態')}:** {report.get('ranking_status')}"
)
lines.append(
f"- **{_label(language, 'Strict mode', '厳格モード')}:** {report.get('strict_mode')}"
)
lines.append(
f"- **{_label(language, 'Deep-dive input / ranked / conditional / review / screened out / excluded', '詳細調査入力／ランキング／条件付き／要確認／スクリーニング落ち／除外')}:** {counts.get('input_candidates')} / {counts.get('ranked')} / {counts.get('conditional', 0)} / {counts.get('review_required')} / {counts.get('screened_out')} / {counts.get('excluded')}"
)
lines.append(
f"- **{_label(language, 'Broad-screen selected / deferred / unresolved / screened out / excluded / unavailable', '一次選定／予算繰越／未解決／一次落ち／除外／取得不能')}:** {broad_counts.get('selected', 0)} / {broad_counts.get('deferred_by_budget', 0)} / {broad_counts.get('review_required', 0)} / {broad_counts.get('screened_out', 0)} / {broad_counts.get('excluded', 0)} / {broad_counts.get('unavailable_after_enrichment', 0)}"
)
lines.append("")
lines.append(
f"## {_label(language, 'Screening Funnel and Market Context', 'スクリーニング・ファネルと市場前提')}"
)
lines.append("")
lines.append(f"- **Listing universe:** {funnel.get('universe_count', missing)}")
lines.append(f"- **Listing rows in scope:** {funnel.get('listing_in_scope_count', missing)}")
lines.append(
f"- **Fundamental/estimate candidate pool:** {funnel.get('candidate_pool_count', missing)}"
)
lines.append(
f"- **Discovery-evaluable candidates:** {funnel.get('discovery_evaluable_count', missing)}"
)
lines.append(f"- **Deep dives selected:** {funnel.get('deep_dive_selected_count', missing)}")
lines.append(
f"- **Corporate-action preflight passed:** {funnel.get('preflight_passed_count', missing)}"
)
lines.append(f"- **Deep dives completed:** {funnel.get('deep_dive_completed_count', missing)}")
market = _mapping(report.get("market_context"))
audit = _mapping(report.get("screening_audit"))
scope = _mapping(audit.get("scope"))
lines.append(
f"- **{_label(language, 'Candidate-generation mode', '候補生成モード')}:** {audit.get('candidate_generation_mode', missing)}"
)
lines.append(
f"- **{_label(language, 'Candidate-pool status', '候補プール状態')}:** {audit.get('candidate_pool_status', missing)}"
)
lines.append(
f"- **{_label(language, 'Selection outcome', '候補選定結果')}:** {audit.get('selection_outcome', missing)}"
)
lines.append(
f"- **{_label(language, 'Listing-data coverage', '上場・価格データカバレッジ')}:** {_format_pct(audit.get('actual_listing_data_complete_pct'), 1, missing)}"
)
lines.append(
f"- **{_label(language, 'Candidate-pool discovery coverage', '候補プール経済評価可能カバレッジ')}:** {_format_pct(audit.get('actual_candidate_pool_discovery_evaluable_pct'), 1, missing)}"
)
lines.append(
f"- **{_label(language, 'Candidate-pool full-fundamental coverage (informational)', '候補プール完全財務カバレッジ（参考）')}:** {_format_pct(audit.get('actual_candidate_pool_fundamental_complete_pct'), 1, missing)}"
)
lines.append(
f"- **{_label(language, 'Listing enumeration', 'リスティング列挙')}:** complete={scope.get('scope_complete', False)} / requested ${scope.get('requested_min_market_cap', missing)}-${scope.get('requested_max_market_cap', missing)} / retrieved ${scope.get('retrieval_min_market_cap', missing)}-${scope.get('retrieval_max_market_cap', missing)}"
)
generation = _mapping(_mapping(audit.get("candidate_pool")).get("generation_audit"))
if generation:
universe_total = _integer(_mapping(audit.get("universe")).get("row_count"))
mode = _text(generation.get("estimate_acquisition_mode")) or missing
bulk = _mapping(generation.get("bulk_estimate_audit"))
covered = _integer(bulk.get("covered_symbol_count"))
seeds = _integer(generation.get("economic_attempt_count"))
if mode == "analyst_estimates_bulk" and covered is not None and universe_total:
coverage = f"{covered} / {universe_total} ({covered / universe_total * 100.0:.1f}%)"
status = "complete" if covered >= universe_total else "partial"
elif seeds is not None and universe_total:
coverage = f"{seeds} / {universe_total} ({seeds / universe_total * 100.0:.1f}%)"
status = "partial"
else:
coverage = missing
status = "partial"
lines.append(
f"- **{_label(language, 'Economic estimate coverage', '経済スクリーン範囲')}:** {status} / mode={mode} / {coverage} -- {_label(language, 'never a market-wide conclusion under', '以下の範囲での結論に限定')} conclusion_scope={audit.get('conclusion_scope', missing)}"
)
lines.append(
f"- **{_label(language, 'Market assumptions', '市場前提')}:** {market.get('summary', missing)}"
)
lines.append(
f"- **Policy / 10Y / Inflation / Market Fwd P/E:** {_format_pct(market.get('policy_rate_pct'), 2, missing)} / {_format_pct(market.get('treasury_10y_yield_pct'), 2, missing)} / {_format_pct(market.get('inflation_yoy_pct'), 2, missing)} / {_format_number(market.get('market_forward_pe'), 2, missing)}x"
)
lines.append(
f"- **{_label(language, 'Universe audit SHA-256', '上場母集団監査SHA-256')}:** {audit.get('actual_universe_sha256', missing)}"
)
enrichment = _mapping(audit.get("enrichment"))
lines.append(
f"- **{_label(language, 'Candidate-pool audit SHA-256', '候補プール監査SHA-256')}:** {audit.get('actual_candidate_pool_sha256', missing)} / valid={audit.get('valid', False)}"
)
lines.append(
f"- **{_label(language, 'Enrichment resolved / unresolved / exhausted', 'エンリッチメント解決済み／未解決／プール枯渇')}:** {audit.get('actual_enrichment_resolved_count', missing)} / {audit.get('actual_enrichment_unresolved_count', missing)} / {audit.get('actual_candidate_pool_exhausted', False)}"
)
lines.append(
f"- **{_label(language, 'Next action', '次の処理')}:** {enrichment.get('next_action', missing)}"
)
lines.append("")
lines.append(f"## A. {_label(language, 'Ranking', 'ランキング一覧')}")
lines.append("")
if ranked:
lines.append(
"| Rank | Company | Ticker | Price | Market Cap | Forward Basis | Forward Multiple | EV/FCF | FCF Yield | ROIC | Diluted Share CAGR | Cyclicality | Rankable 3Y/2Y Upside | Score | DQ |"
)
lines.append("|---:|---|---|---:|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|")
for row in ranked:
identity = _mapping(row.get("identity"))
metrics = _mapping(row.get("financial_metrics"))
valuation = _mapping(row.get("valuation"))
lines.append(
"| {rank} | {company} | {symbol} | {price} | {cap} | {basis} | {multiple} | {evfcf} | {fcf} | {roic} | {dilution} | {cycle} | {upside} | {score} | {dq} |".format(
rank=row.get("rank"),
company=_md_escape(row.get("company_name")),
symbol=row.get("symbol"),
price=_format_price(
identity.get("price"), identity.get("currency", "USD"), missing
),
cap=_format_number(identity.get("market_cap"), 1, missing),
basis=f"{valuation.get('basis', missing)}/{valuation.get('current_metric_basis', missing)}",
multiple=_format_number(valuation.get("current_multiple"), 2, missing),
evfcf=_format_number(metrics.get("ev_to_fcf"), 2, missing),
fcf=_format_pct(metrics.get("fcf_yield_pct"), 1, missing),
roic=_format_pct(metrics.get("roic_pct"), 1, missing),
dilution=_format_pct(metrics.get("diluted_share_cagr_pct"), 1, missing),
cycle=_mapping(row.get("cyclicality")).get("score", missing),
upside=_format_pct(
_preferred_three_year_upside(valuation, "constant_multiple"), 1, missing
),
score=_format_number(row.get("final_score"), 1, missing),
dq=row.get("data_quality_score", missing),
)
)
else:
pool_status = _text(_mapping(report.get("contract")).get("candidate_pool_status"))
if report.get("ranking_status") == "final" and pool_status == "no_qualifying_candidates":
lines.append(
_label(
language,
"No qualifying candidates in the audited full universe.",
"監査済みの全対象ユニバースでは現時点で該当なし。",
)
)
elif (
report.get("ranking_status") == "final"
and pool_status == "no_qualifying_candidates_in_bounded_pool"
):
cov = _mapping(report.get("coverage"))
lines.append(
_label(
language,
(
f"No qualifying candidates among the {cov.get('deep_dive_count', '?')} deep-dived names "
f"selected from the {cov.get('economic_attempt_count', '?')} economically attempted "
f"(of {cov.get('listing_universe_count', '?')} listed); the remaining names were never "
"economically compared — this is not a market-wide conclusion."
),
(
f"経済評価を試行した{cov.get('economic_attempt_count', '?')}銘柄から選定した"
f"{cov.get('deep_dive_count', '?')}銘柄の詳細分析では該当なし"
f"（上場{cov.get('listing_universe_count', '?')}銘柄の残りは未審査であり、"
"市場全体の該当なしを意味しません）。"
),
)
)
else:
lines.append(
_label(
language,
"Ranking withheld: candidate-pool research is incomplete.",
"ランキング保留：候補プールの調査が未完了です。",
)
)
lines.append("")
lines.append(f"## B. {_label(language, 'Scenario Table', 'シナリオ表')}")
lines.append("")
lines.append(
"| Ticker | Current Metric | Current Multiple | 2Y Price | 2Y Upside | 2Y Stress Price | 2Y Stress Upside | 3Y Price | 3Y Upside | 3Y Stress Price | 3Y Stress Upside | Peer 3Y Price |"
)
lines.append("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|")
for row in ranked:
valuation = _mapping(row.get("valuation"))
currency = _mapping(row.get("identity")).get("currency", "USD")
lines.append(
f"| {row.get('symbol')} | {_format_number(valuation.get('current_metric'), 2, missing)} | {_format_number(valuation.get('current_multiple'), 2, missing)} | {_format_price(_scenario_value(row, 'constant_multiple', 'year_2', 'implied_price'), currency, missing)} | {_format_pct(_scenario_value(row, 'constant_multiple', 'year_2', 'upside_pct'), 1, missing)} | {_format_price(_scenario_value(row, 'multiple_contraction', 'year_2', 'implied_price'), currency, missing)} | {_format_pct(_scenario_value(row, 'multiple_contraction', 'year_2', 'upside_pct'), 1, missing)} | {_format_price(_scenario_value(row, 'constant_multiple', 'year_3', 'implied_price'), currency, missing)} | {_format_pct(_scenario_value(row, 'constant_multiple', 'year_3', 'upside_pct'), 1, missing)} | {_format_price(_scenario_value(row, 'multiple_contraction', 'year_3', 'implied_price'), currency, missing)} | {_format_pct(_scenario_value(row, 'multiple_contraction', 'year_3', 'upside_pct'), 1, missing)} | {_format_price(_scenario_value(row, 'peer_median', 'year_3', 'implied_price'), currency, missing)} |"
)
lines.append("")
lines.append(f"## C. {_label(language, 'Candidate Details', '各銘柄の詳細')}")
lines.append("")
for row in ranked:
lines.extend(_render_candidate_detail(language, row))
lines.append(f"## D. {_label(language, 'Conditional Candidates', '条件付き候補')}")
lines.append("")
if conditional:
lines.append("| Ticker | Company | Final Score | Conditions | Low-case Upside |")
lines.append("|---|---|---:|---|---:|")
for row in conditional:
lines.append(
f"| {row.get('symbol')} | {_md_escape(row.get('company_name'))} | {_format_number(row.get('final_score'), 1, missing)} | {_md_escape('; '.join(row.get('conditional_reasons') or []))} | {_format_pct(_best_low_case_upside(_mapping(row.get('valuation'))), 1, missing)} |"
)
else:
lines.append(_label(language, "None.", "なし。"))
lines.append("")
lines.append(f"## E. {_label(language, 'Review Required', '追加確認が必要な候補')}")
lines.append("")
if review:
lines.append(f"### {_label(language, 'Deep-dive review blockers', '詳細調査の要確認')}")
lines.append("| Ticker | Company | Final Score | Blockers | Evidence Needed |")
lines.append("|---|---|---:|---|---|")
for row in review:
lines.append(
f"| {row.get('symbol')} | {_md_escape(row.get('company_name'))} | {_format_number(row.get('final_score'), 1, missing)} | {_md_escape('; '.join(row.get('review_reasons') or []))} | {_md_escape('; '.join(row.get('unresolved_fields') or []) or missing)} |"
)
lines.append("")
broad_review = _list(broad.get("review_required"))
broad_deferred = _list(broad.get("deferred_by_budget"))
broad_selected = _list(broad.get("selected"))
broad_unavailable = _list(broad.get("unavailable_after_enrichment"))
if broad_selected or broad_deferred or broad_review or broad_unavailable:
lines.append(
f"### {_label(language, 'Broad-screen dispositions requiring or deferring work', '一次スクリーニングの要確認・繰越')}"
)
lines.append(
"| Ticker | Company | Status | Fwd P/E | Revenue Growth | Per-share Growth | Priority | Reasons / Requirements |"
)
lines.append("|---|---|---|---:|---:|---:|---:|---|")
for row in broad_selected + broad_deferred + broad_review + broad_unavailable:
metrics = _mapping(row.get("metrics"))
reasons = list(row.get("review_reasons") or []) + list(
row.get("deep_dive_requirements") or []
)
lines.append(
f"| {row.get('symbol')} | {_md_escape(row.get('company_name'))} | {row.get('status')} | {_format_number(metrics.get('forward_pe'), 2, missing)} | {_format_pct(metrics.get('revenue_growth_pct'), 1, missing)} | {_format_pct(metrics.get('per_share_growth_pct'), 1, missing)} | {_format_number(row.get('deep_dive_priority_score'), 1, missing)} | {_md_escape('; '.join(dict.fromkeys(reasons)) or missing)} |"
)
elif not review:
lines.append(_label(language, "None.", "なし。"))
lines.append("")
lines.append(f"## F. {_label(language, 'Screened-Out Log', 'スクリーニング落ちログ')}")
lines.append("")
if screened_out:
lines.append(
f"### {_label(language, 'Deep-dive screened out', '詳細調査でのスクリーニング落ち')}"
)
lines.append("| Ticker | Company | Reason |")
lines.append("|---|---|---|")
for row in screened_out:
lines.append(
f"| {row.get('symbol')} | {_md_escape(row.get('company_name'))} | {_md_escape('; '.join(row.get('screening_reasons') or []))} |"
)
lines.append("")
broad_screened = _list(broad.get("screened_out"))
if broad_screened:
lines.append(
f"### {_label(language, 'Broad-screen screened out', '一次スクリーニング落ち')}"
)
lines.append(
"| Ticker | Company | Fwd P/E | Revenue Growth | Per-share Growth | Guideline Misses | Screen-Fail Reasons |"
)
lines.append("|---|---|---:|---:|---:|---|---|")
for row in broad_screened:
metrics = _mapping(row.get("metrics"))
lines.append(
f"| {row.get('symbol')} | {_md_escape(row.get('company_name'))} | {_format_number(metrics.get('forward_pe'), 2, missing)} | {_format_pct(metrics.get('revenue_growth_pct'), 1, missing)} | {_format_pct(metrics.get('per_share_growth_pct'), 1, missing)} | {_md_escape('; '.join(row.get('guideline_misses') or []) or missing)} | {_md_escape('; '.join(row.get('screen_fail_reasons') or []) or missing)} |"
)
elif not screened_out:
lines.append(_label(language, "None.", "なし。"))
lines.append("")
lines.append(f"## G. {_label(language, 'Hard Exclusion Log', 'ハード除外ログ')}")
lines.append("")
broad_excluded = _list(broad.get("excluded"))
if excluded or broad_excluded:
lines.append("| Stage | Ticker | Company | Reason |")
lines.append("|---|---|---|---|")
for row in excluded:
lines.append(
f"| deep_dive | {row.get('symbol')} | {_md_escape(row.get('company_name'))} | {_md_escape('; '.join(row.get('hard_exclusion_reasons') or []))} |"
)
for row in broad_excluded:
reasons = list(row.get("review_reasons") or []) + list(
row.get("screen_fail_reasons") or []
)
lines.append(
f"| broad_screen | {row.get('symbol')} | {_md_escape(row.get('company_name'))} | {_md_escape('; '.join(reasons) or missing)} |"
)
else:
lines.append(_label(language, "None.", "なし。"))
lines.append("")
lines.append(f"## H. {_label(language, 'Final Three', '最終選定3銘柄')}")
lines.append("")
categories = [
("highest_conviction", "Highest conviction", "最も確度が高い"),
("most_undervalued", "Most undervalued", "最も割安"),
("largest_upside", "Largest upside", "最も上昇余地が大きい"),
]
for key, en, ja in categories:
entry = _mapping(_mapping(report.get("final_three")).get(key))
lines.append(f"### {_label(language, en, ja)}")
if not entry:
lines.append(missing)
lines.append("")
continue
lines.extend(
[
f"- **Ticker:** {entry.get('symbol')} — {entry.get('company_name')}",
f"- **{_label(language, 'Why it qualifies', '最も有望と判断した理由')}:** {entry.get('investment_thesis') or missing}",
f"- **{_label(language, 'Per-share growth upside', 'EPS/FCF成長だけの上昇余地')}:** {_format_pct(entry.get('constant_multiple_upside_pct'), 1, missing)}",
f"- **{_label(language, 'What the market may miss', '市場が見落としている可能性')}:** {entry.get('market_may_be_missing') or missing}",
f"- **{_label(language, 'Best catalyst', '最有力カタリスト')}:** {entry.get('best_catalyst') or missing}",
f"- **{_label(language, 'Largest risk', '最大のリスク')}:** {entry.get('maximum_risk') or missing}",
f"- **{_label(language, 'Reason not to buy now', '今買わない理由')}:** {entry.get('do_not_buy_reason') or missing}",
f"- **{_label(language, 'Bear case', '弱気シナリオ')}:** {entry.get('bear_case') or missing}",
f"- **{_label(language, 'Invalidation', '無効化条件')}:** {_render_list(entry.get('invalidation_conditions'), missing)}",
f"- **{_label(language, 'Next earnings KPIs', '次回決算KPI')}:** {_render_list(entry.get('next_earnings_kpis'), missing)}",
]
)
lines.append("")
source_rows: dict[str, Mapping[str, Any]] = {}
for source in _list(report.get("global_sources")):
item = _mapping(source)
source_id = _text(item.get("id"))
if source_id:
source_rows[source_id] = item
for row in ranked + review + screened_out + excluded:
for source in _list(row.get("sources")):
item = _mapping(source)
source_id = _text(item.get("id"))
if source_id:
source_rows[source_id] = item
lines.append(f"## I. {_label(language, 'Source Ledger', '情報源台帳')}")
lines.append("")
if source_rows:
lines.append("| Source ID | Tier | Kind | Published | Retrieved | Supports |")
lines.append("|---|---:|---|---|---|---|")
for source_id in sorted(source_rows):
source = source_rows[source_id]
lines.append(
f"| {_md_escape(source_id)} | {source.get('tier', missing)} | {_md_escape(source.get('kind', missing))} | {_md_escape(source.get('published_at', missing))} | {_md_escape(source.get('retrieved_at', missing))} | {_md_escape(', '.join(str(value) for value in _list(source.get('supports'))))} |"
)
else:
lines.append(missing)
lines.append("")
lines.append(
f"## J. {_label(language, 'Unresolved Data and Global Warnings', '未解決データと全体警告')}"
)
lines.append("")
warnings = _list(report.get("global_warnings"))
if warnings:
for warning in warnings:
lines.append(f"- {warning}")
else:
lines.append(_label(language, "None.", "なし。"))
lines.append("")
lines.append(
_label(
language,
"This report is a research screen, not an automatic order or guarantee of return.",
"本レポートは調査用スクリーニングであり、自動発注や収益保証ではありません。",
)
)
lines.append("")
return "\n".join(lines)
def _timestamp_for_filename(report: Mapping[str, Any]) -> str:
generated = _text(report.get("generated_at"))
if generated:
try:
parsed = parse_iso8601(generated, "generated_at")
if parsed:
return parsed.astimezone(timezone.utc).strftime("%Y%m%d_%H%M%S")
except InputError:
pass
return datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
def write_outputs(
report: Mapping[str, Any], output_dir: Path, *, language: str
) -> tuple[Path, Path]:
output_dir.mkdir(parents=True, exist_ok=True)
stamp = _timestamp_for_filename(report)
json_path = output_dir / f"us_undervalued_growth_{stamp}.json"
md_path = output_dir / f"us_undervalued_growth_{stamp}.md"
json_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
md_path.write_text(render_markdown(report, language=language), encoding="utf-8")
return json_path, md_path
def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Validate, score, and rank normalized US undervalued-growth candidates (schema v3, contract v3.5)."
)
parser.add_argument(
"--input",
required=True,
type=Path,
help="Normalized schema-3 / contract-3.5 candidate JSON input",
)
parser.add_argument(
"--output-dir",
type=Path,
default=Path("reports/us-undervalued-growth-screener"),
help="Output directory for JSON and Markdown reports",
)
parser.add_argument(
"--artifact-root",
type=Path,
default=None,
help="Root used to resolve and hash broad-screen audit artifacts; defaults to input directory",
)
parser.add_argument("--top", type=int, default=None, help="Maximum ranked candidates")
parser.add_argument("--language", choices=("en", "ja"), default="en")
parser.add_argument("--strict", action="store_true", help="Enable fail-closed research gates")
parser.add_argument("--stdout", action="store_true", help="Print the Markdown report to stdout")
parser.add_argument(
"--require-final",
action="store_true",
help="Return exit code 2 after writing outputs when the run is provisional or the global contract is invalid",
)
return parser.parse_args(argv)
def main(argv: Sequence[str] | None = None) -> int:
raw_argv = list(argv) if argv is not None else sys.argv[1:]
if "--version" in raw_argv:
print(json.dumps(runtime_metadata(), sort_keys=True))
return 0
args = parse_args(raw_argv)
try:
payload = json.loads(args.input.read_text(encoding="utf-8"))
if not isinstance(payload, Mapping):
raise InputError("top-level JSON must be an object")
artifact_root = args.artifact_root or args.input.parent
report = evaluate_snapshot(
payload, strict=args.strict, top=args.top, artifact_root=artifact_root
)
json_path, md_path = write_outputs(report, args.output_dir, language=args.language)
except (OSError, json.JSONDecodeError, InputError, ContractError, ValueError) as exc:
print(f"ERROR: {exc}", file=sys.stderr)
return 1
print(f"wrote: {json_path}")
print(f"wrote: {md_path}")
if args.stdout:
print()
print(md_path.read_text(encoding="utf-8"))
if args.require_final and (
report.get("ranking_status") != "final" or not _mapping(report.get("contract")).get("valid")
):
print(
"ERROR: final-run gate failed; see generated report for contract blockers",
file=sys.stderr,
)
return 2
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/fmp_client.py
#!/usr/bin/env python3
# GENERATED by scripts/generate_fmp_client.py — do not edit.
# Source of truth: scripts/fmp_client/specials/ and scripts/fmp_client/registry.py.
# Regenerate: python3 scripts/generate_fmp_client.py
"""FMP client for the Claude Code-native US GARP screening pipeline.
This module deliberately keeps bulk provider payloads out of the language-model
context. Successful responses are cached in SQLite and may be mirrored to a raw
artifact directory; callers receive Python objects and should emit only compact
summaries to stdout.
The implementation follows the repository's existing FMP conventions:
- FMP_API_KEY from the environment or constructor
- stable endpoints first, legacy v3 fallback where available
- request pacing, 429 retry, endpoint circuit breaking, and call budgeting
- deterministic diagnostics without exposing the API key
"""
from __future__ import annotations
import hashlib
import json
import os
import re
import sqlite3
import sys
import time
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
try:
import requests
except ImportError:
print("ERROR: requests library not found. Install with: pip install requests", file=sys.stderr)
raise
# Mirror of scripts/provider_contracts.py::redact_url (clients are standalone).
# Requires a `?`/`&` prefix, so this masks apikey=/api_key= in URL query
# strings, not a rendered params dict (unreachable today: only
# RequestException text -- which can embed the full request URL -- is
# ever printed here, never a bare params repr()).
_APIKEY_RE = re.compile(r"([?&](?:apikey|api_key)=)[^&\s]+", re.IGNORECASE)
def _redact_key(text: str) -> str:
"""Mask apikey=/api_key= query values so stderr never carries the key."""
return _APIKEY_RE.sub(r"\1REDACTED", str(text))
_PROVIDER_ERROR_KEYS = frozenset({"error message", "errormessage", "error"})
def _is_provider_error_payload(payload: Any) -> bool:
"""True for HTTP-200 error objects (e.g. {"Error Message": "plan limit"})."""
if not isinstance(payload, dict):
return False
keys = {str(key).strip().lower() for key in payload}
return bool(keys & _PROVIDER_ERROR_KEYS)
def _finite_number(value: Any) -> bool:
if isinstance(value, bool) or value is None:
return False
try:
number = float(value)
except (TypeError, ValueError):
return False
return number == number and number not in (float("inf"), float("-inf"))
# Exactly the estimate-value fields the normalizer consumes (its _pick key
# lists, lowercased). Substring matching is NOT allowed here: numAnalystsEps
# contains "eps" but is a count, never an estimate value.
_ESTIMATE_VALUE_KEYS = frozenset(
{
"epsavg",
"eps_avg",
"eps",
"estimated_eps",
"epslow",
"eps_low",
"epshigh",
"eps_high",
"revenueavg",
"revenue_avg",
"revenue",
"estimated_revenue",
}
)
def _parseable_estimate_date(value: Any) -> bool:
"""Full-string ISO parse, the same rule the normalizer applies.
Truncating to ten characters let ``2026-12-31garbage`` through; the
WHOLE string must parse.
"""
text = str(value or "").strip()
if not text:
return False
try:
datetime.fromisoformat(text.replace("Z", "+00:00"))
except ValueError:
return False
return True
def _valid_estimates_payload(payload: Any) -> bool:
"""Schema gate for analyst-estimates payloads (pre-cache AND on cache read).
A genuinely empty list is valid (no consensus). Anything else must be a
list of mappings that are not provider error objects, whose date parses
IN FULL under the normalizer's rule, and that hold a FINITE number in at
least one field the normalizer actually consumes —
``[{"Error Message": ...}]`` is an outage,
``[{"date": "2026-12-31garbage", ...}]`` and rows whose only "value" is
an analyst count are unusable noise; none may complete a symbol as
"no estimates".
"""
if payload == []:
return True
if not isinstance(payload, list):
return False
for row in payload:
if not isinstance(row, dict) or _is_provider_error_payload(row):
return False
if not _parseable_estimate_date(row.get("date")):
return False
has_finite_estimate = any(
str(key).strip().lower() in _ESTIMATE_VALUE_KEYS and _finite_number(value)
for key, value in row.items()
)
if not has_finite_estimate:
return False
return True
class ApiCallBudgetExceeded(RuntimeError):
"""Raised when the configured provider-call ceiling is reached."""
class FMPResponseError(RuntimeError):
"""Raised for malformed or unusable provider responses in strict mode."""
@dataclass(frozen=True)
class CachePolicy:
default_ttl_seconds: int = 86_400
quote_ttl_seconds: int = 3_600
historical_ttl_seconds: int = 86_400
estimates_ttl_seconds: int = 259_200
statements_ttl_seconds: int = 604_800
universe_ttl_seconds: int = 604_800
class SQLiteJsonCache:
"""Small persistent JSON cache keyed by URL and non-secret parameters."""
def __init__(self, path: str | Path):
self.path = Path(path)
self.path.parent.mkdir(parents=True, exist_ok=True)
self._connection = sqlite3.connect(self.path)
self._connection.execute(
"""
CREATE TABLE IF NOT EXISTS responses (
cache_key TEXT PRIMARY KEY,
created_at REAL NOT NULL,
payload TEXT NOT NULL
)
"""
)
self._connection.commit()
@staticmethod
def make_key(url: str, params: Mapping[str, Any]) -> str:
clean = {
str(k): v for k, v in params.items() if str(k).lower() not in {"apikey", "api_key"}
}
encoded = json.dumps(
[url, clean], ensure_ascii=False, sort_keys=True, separators=(",", ":")
)
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
def get_entry(self, key: str, ttl_seconds: int) -> tuple[Any, float] | None:
"""Return (payload, created_at) for a fresh entry, else None."""
row = self._connection.execute(
"SELECT created_at, payload FROM responses WHERE cache_key = ?", (key,)
).fetchone()
if row is None:
return None
created_at, payload = row
if ttl_seconds >= 0 and time.time() - float(created_at) > ttl_seconds:
self.delete(key)
return None
try:
return json.loads(payload), float(created_at)
except json.JSONDecodeError:
self.delete(key)
return None
def get(self, key: str, ttl_seconds: int) -> Any | None:
entry = self.get_entry(key, ttl_seconds)
return entry[0] if entry is not None else None
def delete(self, key: str) -> None:
self._connection.execute("DELETE FROM responses WHERE cache_key = ?", (key,))
self._connection.commit()
def put(self, key: str, payload: Any) -> None:
text = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
self._connection.execute(
"INSERT OR REPLACE INTO responses(cache_key, created_at, payload) VALUES(?, ?, ?)",
(key, time.time(), text),
)
self._connection.commit()
def close(self) -> None:
self._connection.close()
class FMPClient:
"""Stable-first FMP client with persistent caching and compact diagnostics."""
STABLE_URL = "https://financialmodelingprep.com/stable"
V3_URL = "https://financialmodelingprep.com/api/v3"
RATE_LIMIT_DELAY = 0.30
ENDPOINT_FAILURE_THRESHOLD = 3
CAPABILITY_CACHE_TTL_DAYS = 30
CAPABILITY_DISABLING_STATUS_CODES = (402, 403)
def _redact_secret(self, text) -> str:
"""Mask raw and URL-encoded key echoes before diagnostics are truncated."""
from urllib.parse import quote, quote_plus
text = _redact_key(text)
if not self.api_key:
return text
patterns = {re.escape(self.api_key)}
for encoded in (quote(self.api_key, safe=""), quote_plus(self.api_key, safe="")):
# Percent triplets are case-insensitive; the key's letters are not.
parts = re.split(r"(%[0-9A-Fa-f]{2})", encoded)
patterns.add(
"".join(
"(?i:" + re.escape(part) + ")"
if re.fullmatch(r"%[0-9A-Fa-f]{2}", part)
else re.escape(part)
for part in parts
)
)
# One pass prevents a short key from matching the replacement marker.
return re.sub("|".join(sorted(patterns, key=len, reverse=True)), "***REDACTED***", text)
def __init__(
self,
api_key: str | None = None,
*,
max_api_calls: int = 500,
cache_path: str | Path = ".cache/fmp/fmp-cache.sqlite3",
raw_store_dir: str | Path | None = None,
cache_policy: CachePolicy | None = None,
offline: bool = False,
timeout_seconds: int = 30,
rate_limit_delay: float | None = None,
respect_capability_cache: bool = True,
):
self.api_key = (api_key or os.getenv("FMP_API_KEY") or "").strip()
self.offline = bool(offline)
# Provenance of the payload returned by the most recent
# _request_json call: {"source": "cache"|"http"|None,
# "retrieved_at": epoch float|None}. Single-threaded CLI client.
self._last_response_meta: dict[str, Any] = {"source": None, "retrieved_at": None}
if not self.api_key and not self.offline:
raise ValueError("FMP API key required. Set FMP_API_KEY or pass api_key.")
self.max_api_calls = int(max_api_calls)
if self.max_api_calls <= 0:
raise ValueError("max_api_calls must be positive")
self.timeout_seconds = int(timeout_seconds)
self.rate_limit_delay = (
self.RATE_LIMIT_DELAY if rate_limit_delay is None else max(0.0, float(rate_limit_delay))
)
self.cache_policy = cache_policy or CachePolicy()
self.cache = SQLiteJsonCache(cache_path)
self.raw_store_dir = Path(raw_store_dir) if raw_store_dir else None
if self.raw_store_dir:
self.raw_store_dir.mkdir(parents=True, exist_ok=True)
self.session = requests.Session()
self.last_call_time = 0.0
self.api_calls_made = 0
self.cache_hits = 0
self.failures: list[dict[str, Any]] = []
self._endpoint_failures: dict[str, int] = {}
self._disabled_endpoints: set[str] = set()
self.respect_capability_cache = bool(respect_capability_cache)
self.capability_cache_hits = 0
@property
def remaining_calls(self) -> int:
"""API calls still available under the configured budget (never negative)."""
return max(0, self.max_api_calls - self.api_calls_made)
def __enter__(self) -> FMPClient:
return self
def __exit__(self, exc_type, exc, tb) -> None:
self.close()
def close(self) -> None:
self.cache.close()
self.session.close()
@staticmethod
def _safe_params(params: Mapping[str, Any]) -> dict[str, Any]:
return {str(k): v for k, v in params.items() if str(k).lower() not in {"apikey", "api_key"}}
def _write_raw(self, *, url: str, params: Mapping[str, Any], payload: Any) -> None:
if self.raw_store_dir is None:
return
safe_params = self._safe_params(params)
identity = json.dumps([url, safe_params], sort_keys=True, separators=(",", ":"))
digest = hashlib.sha256(identity.encode("utf-8")).hexdigest()
target = self.raw_store_dir / f"{digest}.json"
wrapper = {
"retrieved_at": datetime.now(timezone.utc).isoformat(),
"url": url,
"params": safe_params,
"payload": payload,
}
temp = target.with_suffix(".tmp")
temp.write_text(json.dumps(wrapper, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
os.replace(temp, target)
def _record_failure(self, endpoint: str, reason: str, status_code: int | None = None) -> None:
# `endpoint` is an identity key: the circuit breaker below and the
# capability-cache gate in `_fetch_statements` compare it against the
# raw url, so it is stored unredacted here and masked only where it
# is exposed (`diagnostics()['failure_samples']`).
item: dict[str, Any] = {"endpoint": endpoint, "reason": reason}
if status_code is not None:
item["status_code"] = status_code
self.failures.append(item)
count = self._endpoint_failures.get(endpoint, 0) + 1
self._endpoint_failures[endpoint] = count
if count >= self.ENDPOINT_FAILURE_THRESHOLD:
self._disabled_endpoints.add(endpoint)
def _request_json(
self,
url: str,
params: Mapping[str, Any] | None = None,
*,
ttl_seconds: int | None = None,
allow_empty: bool = False,
quiet: bool = False,
validate: Any | None = None,
) -> Any | None:
request_params = dict(params or {})
ttl = self.cache_policy.default_ttl_seconds if ttl_seconds is None else int(ttl_seconds)
cache_key = SQLiteJsonCache.make_key(url, request_params)
self._last_response_meta = {"source": None, "retrieved_at": None}
entry = self.cache.get_entry(cache_key, ttl)
if entry is not None:
cached, created_at = entry
if validate is not None and not validate(cached):
# A payload cached before validation existed (or by another
# tool) must not keep short-circuiting retries: purge it and
# fall through to a fresh HTTP attempt.
self._record_failure(url, "cached_payload_invalid")
self.cache.delete(cache_key)
else:
self.cache_hits += 1
self._last_response_meta = {"source": "cache", "retrieved_at": created_at}
return cached
if self.offline:
return None
if url in self._disabled_endpoints:
return None
if self.api_calls_made >= self.max_api_calls:
raise ApiCallBudgetExceeded(
f"FMP API call budget exhausted: {self.api_calls_made}/{self.max_api_calls}"
)
elapsed = time.time() - self.last_call_time
if elapsed < self.rate_limit_delay:
time.sleep(self.rate_limit_delay - elapsed)
request_params["apikey"] = self.api_key
attempts = 0
while attempts < 2:
if self.api_calls_made >= self.max_api_calls:
raise ApiCallBudgetExceeded(
f"FMP API call budget exhausted: {self.api_calls_made}/{self.max_api_calls}"
)
attempts += 1
# An attempt is consumed when dispatched (#607): the upstream
# server may have received the request even if the client times
# out, so a ReadTimeout must consume the same budget and pace
# the next call as a successful response. Exception paths
# return None; the stable->v3 fallback and later symbols then
# meet the exhausted-budget guard instead of the transport.
self.api_calls_made += 1
self.last_call_time = time.time()
try:
response = self.session.get(
url, params=request_params, timeout=self.timeout_seconds
)
except requests.RequestException as exc:
self._record_failure(url, f"request_exception:{exc.__class__.__name__}")
if not quiet:
print(
self._redact_secret(f"WARN: FMP request failed for {url}: {exc}"),
file=sys.stderr,
)
return None
if response.status_code == 429 and attempts < 2:
time.sleep(60)
continue
if response.status_code != 200:
self._record_failure(url, f"http_{response.status_code}", response.status_code)
if not quiet:
print(
f"WARN: FMP endpoint failed: {self._redact_secret(url)} HTTP {response.status_code}",
file=sys.stderr,
)
return None
try:
payload = response.json()
except ValueError:
self._record_failure(url, "invalid_json", response.status_code)
if not quiet:
print(
f"WARN: FMP returned invalid JSON: {self._redact_secret(url)}",
file=sys.stderr,
)
return None
if _is_provider_error_payload(payload):
# FMP reports plan limits and endpoint errors as HTTP 200
# bodies like {"Error Message": "..."}; treating those as
# data would cache the outage as an "empty" success.
self._record_failure(url, "provider_error_payload", response.status_code)
if not quiet:
print(
f"WARN: FMP returned an error payload: {self._redact_secret(url)}",
file=sys.stderr,
)
return None
if validate is not None and not validate(payload):
# Same outage class hidden one level down (e.g. an error
# object INSIDE a list) or a shape the endpoint never
# produces: a failure, never cached.
self._record_failure(url, "schema_validation_failed", response.status_code)
if not quiet:
print(
f"WARN: FMP payload failed schema validation: {self._redact_secret(url)}",
file=sys.stderr,
)
return None
if not allow_empty and payload in (None, [], {}):
self._record_failure(url, "empty_response", response.status_code)
return None
self._endpoint_failures[url] = 0
self.cache.put(cache_key, payload)
self._write_raw(url=url, params=request_params, payload=payload)
self._last_response_meta = {"source": "http", "retrieved_at": time.time()}
return payload
self._record_failure(url, "rate_limit_exhausted", 429)
return None
def _stable_then_v3(
self,
stable_endpoint: str,
stable_params: Mapping[str, Any],
*,
v3_endpoint: str | None = None,
v3_params: Mapping[str, Any] | None = None,
ttl_seconds: int | None = None,
allow_empty: bool = False,
validate: Any | None = None,
) -> Any | None:
stable_url = f"{self.STABLE_URL}/{stable_endpoint.lstrip('/')}"
data = self._request_json(
stable_url,
stable_params,
ttl_seconds=ttl_seconds,
allow_empty=allow_empty,
quiet=v3_endpoint is not None,
validate=validate,
)
if data is not None:
return data
if not v3_endpoint:
return None
v3_url = f"{self.V3_URL}/{v3_endpoint.lstrip('/')}"
return self._request_json(
v3_url,
v3_params if v3_params is not None else stable_params,
ttl_seconds=ttl_seconds,
allow_empty=allow_empty,
validate=validate,
)
def get_company_screener(
self,
*,
exchange: str,
min_market_cap: float,
max_market_cap: float,
min_price: float = 5.0,
limit: int = 10_000,
) -> list[dict[str, Any]]:
params = {
"exchange": exchange,
"marketCapMoreThan": int(min_market_cap),
"marketCapLowerThan": int(max_market_cap),
"priceMoreThan": float(min_price),
"isEtf": "false",
"isFund": "false",
"isActivelyTrading": "true",
"limit": int(limit),
}
data = self._stable_then_v3(
"company-screener",
params,
v3_endpoint="stock-screener",
v3_params=params,
ttl_seconds=self.cache_policy.universe_ttl_seconds,
allow_empty=True,
)
return [dict(row) for row in data] if isinstance(data, list) else []
def get_quotes(self, symbols: Sequence[str], batch_size: int = 5) -> dict[str, dict[str, Any]]:
result: dict[str, dict[str, Any]] = {}
clean = [str(symbol).strip().upper() for symbol in symbols if str(symbol).strip()]
for offset in range(0, len(clean), max(1, int(batch_size))):
batch = clean[offset : offset + max(1, int(batch_size))]
joined = ",".join(batch)
data = self._stable_then_v3(
"quote",
{"symbol": joined},
v3_endpoint=f"quote/{joined}",
v3_params={},
ttl_seconds=self.cache_policy.quote_ttl_seconds,
)
if isinstance(data, Mapping):
data = [data]
if isinstance(data, list):
for row in data:
if isinstance(row, Mapping) and row.get("symbol"):
result[str(row["symbol"]).upper()] = dict(row)
return result
def get_historical_prices(
self,
symbol: str,
*,
from_date: str,
to_date: str,
) -> list[dict[str, Any]]:
data = self._stable_then_v3(
"historical-price-eod/full",
{"symbol": symbol, "from": from_date, "to": to_date},
v3_endpoint=f"historical-price-full/{symbol}",
v3_params={"from": from_date, "to": to_date},
ttl_seconds=self.cache_policy.historical_ttl_seconds,
allow_empty=True,
)
if isinstance(data, Mapping):
data = data.get("historical") or data.get("historicalStockList") or []
return [dict(row) for row in data] if isinstance(data, list) else []
def get_analyst_estimates_detailed(
self, symbol: str, *, period: str = "annual", limit: int = 6
) -> dict[str, Any]:
"""Structured estimates fetch tied to the ACTUALLY adopted response.
Returns ``{"rows", "status", "served_from_cache", "retrieved_at"}``
where status is ``ok`` (validated rows), ``empty`` (a genuine empty
consensus from a successful response), or ``failed`` (any provider
failure — HTTP error, offline miss, error payload, schema failure).
``retrieved_at`` is the epoch time the adopted payload was ACTUALLY
fetched over HTTP (a cache hit keeps the cache entry's creation
time); consumers must treat a missing value as unknown, never "now".
"""
data = self._stable_then_v3(
"analyst-estimates",
{"symbol": symbol, "period": period, "limit": int(limit)},
v3_endpoint=f"analyst-estimates/{symbol}",
v3_params={"period": period, "limit": int(limit)},
ttl_seconds=self.cache_policy.estimates_ttl_seconds,
allow_empty=True,
validate=_valid_estimates_payload,
)
if data is None:
return {
"rows": [],
"status": "failed",
"served_from_cache": False,
"retrieved_at": None,
}
meta = dict(self._last_response_meta)
rows = [dict(row) for row in data]
return {
"rows": rows,
"status": "ok" if rows else "empty",
"served_from_cache": meta.get("source") == "cache",
"retrieved_at": meta.get("retrieved_at"),
}
def get_analyst_estimates(
self, symbol: str, *, period: str = "annual", limit: int = 6
) -> list[dict[str, Any]]:
return self.get_analyst_estimates_detailed(symbol, period=period, limit=limit)["rows"]
def get_income_statement(
self, symbol: str, *, period: str = "annual", limit: int = 6
) -> list[dict[str, Any]]:
return self._statement("income-statement", symbol, period=period, limit=limit)
def get_balance_sheet(
self, symbol: str, *, period: str = "annual", limit: int = 6
) -> list[dict[str, Any]]:
return self._statement("balance-sheet-statement", symbol, period=period, limit=limit)
def get_cash_flow(
self, symbol: str, *, period: str = "annual", limit: int = 6
) -> list[dict[str, Any]]:
return self._statement("cash-flow-statement", symbol, period=period, limit=limit)
def _statement(
self, endpoint: str, symbol: str, *, period: str, limit: int
) -> list[dict[str, Any]]:
data = self._stable_then_v3(
endpoint,
{"symbol": symbol, "period": period, "limit": int(limit)},
v3_endpoint=f"{endpoint}/{symbol}",
v3_params={"period": period, "limit": int(limit)},
ttl_seconds=self.cache_policy.statements_ttl_seconds,
allow_empty=True,
)
return [dict(row) for row in data] if isinstance(data, list) else []
def get_key_metrics_ttm(self, symbol: str) -> list[dict[str, Any]]:
data = self._stable_then_v3(
"key-metrics-ttm",
{"symbol": symbol},
v3_endpoint=f"key-metrics-ttm/{symbol}",
v3_params={},
ttl_seconds=self.cache_policy.statements_ttl_seconds,
allow_empty=True,
)
if isinstance(data, Mapping):
data = [data]
return [dict(row) for row in data] if isinstance(data, list) else []
def get_ratios_ttm(self, symbol: str) -> list[dict[str, Any]]:
data = self._stable_then_v3(
"ratios-ttm",
{"symbol": symbol},
v3_endpoint=f"ratios-ttm/{symbol}",
v3_params={},
ttl_seconds=self.cache_policy.statements_ttl_seconds,
allow_empty=True,
)
if isinstance(data, Mapping):
data = [data]
return [dict(row) for row in data] if isinstance(data, list) else []
def get_profile(self, symbol: str) -> dict[str, Any] | None:
data = self._stable_then_v3(
"profile",
{"symbol": symbol},
v3_endpoint=f"profile/{symbol}",
v3_params={},
ttl_seconds=self.cache_policy.statements_ttl_seconds,
allow_empty=True,
)
if isinstance(data, Mapping):
return dict(data)
if isinstance(data, list) and data and isinstance(data[0], Mapping):
return dict(data[0])
return None
def get_stock_peers(self, symbol: str) -> list[str]:
data = self._stable_then_v3(
"stock-peers",
{"symbol": symbol},
v3_endpoint="stock_peers",
v3_params={"symbol": symbol},
ttl_seconds=self.cache_policy.statements_ttl_seconds,
allow_empty=True,
)
if isinstance(data, Mapping):
peers = data.get("peersList") or data.get("peers") or []
return [str(value).upper() for value in peers if str(value).strip()]
if isinstance(data, list):
if data and isinstance(data[0], Mapping):
peers = data[0].get("peersList") or data[0].get("peers") or []
return [str(value).upper() for value in peers if str(value).strip()]
return [str(value).upper() for value in data if isinstance(value, str)]
return []
@staticmethod
def _capability_cache_key(url: str) -> str:
return f"capability:{url}"
def _capability_ttl_seconds(self) -> int:
return self.CAPABILITY_CACHE_TTL_DAYS * 86_400
def _get_cached_capability(self, url: str) -> dict[str, Any] | None:
payload = self.cache.get(self._capability_cache_key(url), self._capability_ttl_seconds())
return dict(payload) if isinstance(payload, Mapping) else None
def _put_cached_capability(self, url: str, status_code: int) -> None:
payload = {
"endpoint": url,
"status": status_code,
"checked_at": datetime.now(timezone.utc).isoformat(),
"ttl_days": self.CAPABILITY_CACHE_TTL_DAYS,
}
self.cache.put(self._capability_cache_key(url), payload)
def get_bulk_dataset(self, endpoint: str, **params: Any) -> list[dict[str, Any]]:
"""Fetch a stable bulk endpoint; return an empty list when unavailable.
Endpoint names remain configuration-driven because FMP plan access and
stable endpoint naming can vary. Callers must record an unavailable
endpoint and use the bounded per-symbol fallback rather than invent data.
A 402/403 response is remembered in the persistent SQLite cache as an
endpoint "capability" (plan access, not data). Later client instances
pointed at the same cache path skip the HTTP call entirely for a
capability entry that is still fresh (``CAPABILITY_CACHE_TTL_DAYS``),
so a known-unavailable bulk endpoint never spends provider budget
again within the TTL window. Set ``respect_capability_cache=False`` on
construction to re-probe every endpoint unconditionally.
"""
url = f"{self.STABLE_URL}/{endpoint.lstrip('/')}"
if self.respect_capability_cache:
cached = self._get_cached_capability(url)
if (
cached is not None
and cached.get("status") in self.CAPABILITY_DISABLING_STATUS_CODES
):
self.capability_cache_hits += 1
self._disabled_endpoints.add(url)
return []
failure_count_before = len(self.failures)
data = self._request_json(
url,
params,
ttl_seconds=self.cache_policy.statements_ttl_seconds,
allow_empty=True,
quiet=True,
)
if self.respect_capability_cache and len(self.failures) > failure_count_before:
last_failure = self.failures[-1]
status_code = last_failure.get("status_code")
if (
last_failure.get("endpoint") == url
and status_code in self.CAPABILITY_DISABLING_STATUS_CODES
):
self._put_cached_capability(url, status_code)
return [dict(row) for row in data] if isinstance(data, list) else []
def diagnostics(self) -> dict[str, Any]:
return {
"api_calls_made": self.api_calls_made,
"max_api_calls": self.max_api_calls,
"cache_hits": self.cache_hits,
"failure_count": len(self.failures),
"failure_samples": [
{**item, "endpoint": self._redact_secret(item.get("endpoint", ""))}
for item in self.failures[:20]
],
"disabled_endpoint_count": len(self._disabled_endpoints),
"capability_cache_hits": self.capability_cache_hits,
"remaining_calls": self.remaining_calls,
"offline": self.offline,
}
scripts/manage_run_state.py
#!/usr/bin/env python3
"""Atomic checkpoint/resume manager for schema-3 / contract-3.5 GARP screening runs.
The manager keeps the tiered listing/candidate-pool audit, selected-symbol set,
verified candidate checkpoints, and unprocessed list mathematically
consistent. It performs no network access.
"""
from __future__ import annotations
import argparse
import filecmp
import hashlib
import json
import os
import re
import shutil
import stat
import sys
import tempfile
from collections.abc import Mapping, Sequence
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
try:
from skill_version import (
CONTRACT_REVISION,
RUNTIME_FINGERPRINT,
SCHEMA_VERSION,
SKILL_VERSION,
runtime_metadata,
)
except ModuleNotFoundError: # Supports importlib-based unit loading.
import importlib.util as _importlib_util
_version_path = Path(__file__).with_name("skill_version.py")
_version_spec = _importlib_util.spec_from_file_location("skill_version", _version_path)
if _version_spec is None or _version_spec.loader is None:
raise
_version_module = _importlib_util.module_from_spec(_version_spec)
_version_spec.loader.exec_module(_version_module)
CONTRACT_REVISION = _version_module.CONTRACT_REVISION
RUNTIME_FINGERPRINT = _version_module.RUNTIME_FINGERPRINT
SCHEMA_VERSION = _version_module.SCHEMA_VERSION
SKILL_VERSION = _version_module.SKILL_VERSION
runtime_metadata = _version_module.runtime_metadata
RUN_STATE_SCHEMA_VERSION = 2
SNAPSHOT_SCHEMA_VERSION = 3
AUDIT_SCHEMA_VERSION = 3
SYMBOL_RE = re.compile(r"^[A-Za-z0-9.\-^]{1,20}$")
AUDIT_DOCUMENT = "audit/broad-screen-audit.json"
class RunStateError(ValueError):
pass
class _RollbackIncomplete(RunStateError):
"""A failed commit could not restore every file; its backups must be kept."""
keep_staging = True
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
def _text(value: Any) -> str | None:
if isinstance(value, str):
value = value.strip()
return value or None
return None
def _mapping(value: Any) -> dict[str, Any]:
return dict(value) if isinstance(value, Mapping) else {}
def _read_json(path: Path) -> Any:
try:
return json.loads(path.read_text(encoding="utf-8"))
except OSError as exc:
raise RunStateError(f"cannot read {path}: {exc}") from exc
except json.JSONDecodeError as exc:
raise RunStateError(f"invalid JSON in {path}: {exc}") from exc
def _atomic_write_json(path: Path, value: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temp = path.with_suffix(path.suffix + ".tmp")
temp.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
os.replace(temp, path)
def _append_event_line(path: Path, event: Mapping[str, Any]) -> None:
with path.open("a", encoding="utf-8") as handle:
handle.write(json.dumps(dict(event), ensure_ascii=False) + "\n")
def _append_event(run_dir: Path, event: Mapping[str, Any]) -> None:
_append_event_line(run_dir / "events.ndjson", event)
def _state_path(run_dir: Path) -> Path:
return run_dir / "run_state.json"
def _load_state(run_dir: Path) -> dict[str, Any]:
path = _state_path(run_dir)
if not path.is_file():
raise RunStateError(f"run state not found: {path}")
state = _read_json(path)
if not isinstance(state, Mapping):
raise RunStateError("run_state.json must be an object")
state = dict(state)
if state.get("schema_version") != RUN_STATE_SCHEMA_VERSION:
raise RunStateError(f"run-state schema_version must be {RUN_STATE_SCHEMA_VERSION}")
return state
def _save_state(run_dir: Path, state: Mapping[str, Any]) -> None:
payload = dict(state)
payload["updated_at"] = _now()
_atomic_write_json(_state_path(run_dir), payload)
def _validate_symbol(symbol: str) -> str:
normalized = symbol.strip().upper()
if not SYMBOL_RE.fullmatch(normalized):
raise RunStateError(f"invalid symbol: {symbol!r}")
return normalized
def _selected_symbols(state: Mapping[str, Any]) -> set[str]:
audit = _mapping(state.get("screening_audit"))
return {_validate_symbol(str(value)) for value in audit.get("selected_symbols", [])}
def _verified_symbols(state: Mapping[str, Any]) -> set[str]:
result: set[str] = set()
for symbol, metadata_raw in _mapping(state.get("candidates")).items():
if _mapping(metadata_raw).get("stage") == "verified":
result.add(_validate_symbol(symbol))
return result
def _state_invariants(state: Mapping[str, Any]) -> list[str]:
errors: list[str] = []
audit = _mapping(state.get("screening_audit"))
selected = _selected_symbols(state) if audit else set()
verified = _verified_symbols(state)
declared = {_validate_symbol(str(value)) for value in state.get("unprocessed_candidates", [])}
expected = selected - verified
if declared != expected:
errors.append(
"unprocessed_candidates must equal selected minus verified: "
f"declared={sorted(declared)}, expected={sorted(expected)}"
)
unexpected_verified = verified - selected
if unexpected_verified:
errors.append(
"verified candidates were not selected: " + ", ".join(sorted(unexpected_verified))
)
funnel = _mapping(state.get("screening_funnel"))
if audit:
universe = _mapping(audit.get("universe"))
candidate_pool = _mapping(audit.get("candidate_pool"))
expected_funnel = {
"universe_count": int(universe.get("row_count") or 0),
"listing_in_scope_count": int(universe.get("in_scope_count") or 0),
"candidate_pool_count": int(candidate_pool.get("row_count") or 0),
"discovery_evaluable_count": int(candidate_pool.get("discovery_evaluable_count") or 0),
"deep_dive_selected_count": len(selected),
"deep_dive_completed_count": len(verified & selected),
}
for key, value in expected_funnel.items():
if int(funnel.get(key) or 0) != value:
errors.append(f"screening_funnel.{key} does not match attached audit/run state")
outcome = (_text(audit.get("selection_outcome")) or "").lower()
pool_status = (_text(audit.get("candidate_pool_status")) or "").lower()
if pool_status == "sufficient":
if not selected or outcome != "selected":
errors.append(
"candidate_pool_status=sufficient requires selected symbols and selection_outcome=selected"
)
elif pool_status == "sufficient_pending_enrichment":
if not selected or outcome != "selected_pending_enrichment":
errors.append(
"candidate_pool_status=sufficient_pending_enrichment requires selected symbols and selection_outcome=selected_pending_enrichment"
)
elif pool_status == "no_qualifying_candidates":
if selected or outcome != "no_candidates":
errors.append(
"no_qualifying_candidates requires no selected symbols and selection_outcome=no_candidates"
)
elif pool_status == "no_qualifying_candidates_in_bounded_pool":
if selected or outcome != "no_candidates_in_bounded_pool":
errors.append(
"bounded-pool no-candidates requires no selected symbols and the scoped selection outcome"
)
elif pool_status == "insufficient_data":
if selected or outcome != "insufficient_data":
errors.append(
"insufficient_data requires no selected symbols and selection_outcome=insufficient_data"
)
elif audit:
errors.append("screening_audit.candidate_pool_status is invalid")
return errors
def cmd_init(args: argparse.Namespace) -> int:
run_dir: Path = args.run_dir
if _state_path(run_dir).exists() and not args.force:
raise RunStateError(f"run already exists: {run_dir}; use --force to replace")
config = _read_json(args.config) if args.config else {}
market_context = _read_json(args.market_context)
global_sources = _read_json(args.global_sources)
if not isinstance(config, Mapping) or not isinstance(market_context, Mapping):
raise RunStateError("config and market-context files must contain JSON objects")
if not isinstance(global_sources, list):
raise RunStateError("global-sources file must contain a JSON array")
summary = _text(market_context.get("summary"))
if not summary or "replace this" in summary.lower() or "placeholder" in summary.lower():
raise RunStateError(
"market context must be populated with current data; example placeholders are rejected"
)
if (
not isinstance(market_context.get("source_ids"), list)
or len(market_context["source_ids"]) < 2
):
raise RunStateError("market context must reference at least two global source IDs")
state = {
"schema_version": RUN_STATE_SCHEMA_VERSION,
"runtime": runtime_metadata(),
"run_id": args.run_id or run_dir.name,
"created_at": _now(),
"updated_at": _now(),
"status": "partial",
"analysis_as_of": args.analysis_as_of,
"base_repository_commit": args.base_commit,
"price_basis": {
"as_of": args.price_as_of,
"session": args.session,
"timezone": args.timezone,
"source_id": args.price_source_id,
},
"config": dict(config),
"market_context": dict(market_context),
"global_sources": list(global_sources),
"screening_funnel": {},
"screening_audit": {},
"candidates": {},
"unprocessed_candidates": [],
}
run_dir.mkdir(parents=True, exist_ok=True)
(run_dir / "candidates").mkdir(exist_ok=True)
(run_dir / "audit").mkdir(exist_ok=True)
_save_state(run_dir, state)
_append_event(run_dir, {"at": _now(), "event": "run_initialized", "run_id": state["run_id"]})
print(f"initialized: {run_dir}")
return 0
def _stage_artifact(
staging_dir: Path,
staged: dict[str, Path],
section: Mapping[str, Any],
source_path: Path,
target_name: str,
label: str,
) -> dict[str, Any]:
"""Copy an artifact into staging and verify the bytes that will be committed."""
if not source_path.is_file():
raise RunStateError(f"{label} artifact not found: {source_path}")
target = f"audit/{target_name}"
# Compare case-insensitively: on a case-insensitive filesystem two names that
# differ only by case are one file.
folded = target.casefold()
existing = next((name for name in staged if name.casefold() == folded), None)
if folded == AUDIT_DOCUMENT.casefold() or (existing is not None and existing != target):
raise RunStateError(f"{label} artifact name collides with another file: {target_name}")
staged_path = staging_dir / f"artifact-{len(staged)}.tmp"
shutil.copyfile(source_path, staged_path)
actual_sha = hashlib.sha256(staged_path.read_bytes()).hexdigest()
expected_sha = _text(section.get("artifact_sha256"))
if expected_sha != actual_sha:
raise RunStateError(f"{label} artifact SHA-256 does not match audit JSON")
if existing is not None:
# The same name twice is one artifact, but only if the bytes agree.
if hashlib.sha256(staged[existing].read_bytes()).hexdigest() != actual_sha:
raise RunStateError(f"{label} artifact name collides with another file: {target_name}")
staged_path.unlink()
else:
staged[target] = staged_path
normalized = dict(section)
normalized["artifact_path"] = target
normalized["artifact_sha256"] = actual_sha
return normalized
def _resolve_artifact_argument(
explicit: Path | None,
section: Mapping[str, Any],
audit_path: Path,
*,
label: str,
) -> Path:
if explicit is not None:
return explicit
relative = _text(section.get("artifact_path"))
if not relative:
raise RunStateError(f"{label} artifact path is required")
candidate = (audit_path.parent / relative).resolve()
if not candidate.is_file():
raise RunStateError(f"{label} artifact not found beside audit JSON: {candidate}")
return candidate
def _restore_one(target: Path, backup: Path) -> None:
"""Put one target back; tolerate a target whose rename never happened."""
if backup.is_file():
if target.is_file() and filecmp.cmp(backup, target, shallow=False):
return # Never moved (or already restored): the prior bytes are live.
os.replace(backup, target)
elif target.is_dir() and not target.is_symlink():
return # A directory in the way is never ours to remove.
else:
try:
target.unlink()
except FileNotFoundError:
pass # A new file that was never moved into place.
def _restore_files(
applied: Sequence[tuple[Path, Path]],
) -> tuple[list[str], BaseException | None]:
"""Restore every applied target; return the failures and the first interrupt.
One failure or interrupt must not skip the remaining files, so every file is
attempted. The caller re-raises the interrupt once the loop has finished.
"""
failures: list[str] = []
interrupt: BaseException | None = None
for target, backup in reversed(applied):
try:
_restore_one(target, backup)
except BaseException as exc: # noqa: BLE001 - see docstring
failures.append(f"{target}: {exc} (prior bytes are in {backup.name})")
if not isinstance(exc, Exception) and interrupt is None:
interrupt = exc
return failures, interrupt
def _commit_files(run_dir: Path, staging_dir: Path, staged: Mapping[str, Path]) -> None:
"""Move staged files over their live paths, restoring all of them if any move fails.
Mapping order is commit order; run_state.json goes last because it is what
a resumed run trusts. A process killed outright cannot run this rollback:
prepublish_audit then rejects the hash mismatch. Rerunning the command
repairs the run only when it has no verified candidates; otherwise restore
from backups or start a new run directory. An exception that leaves a file unrestored carries
keep_staging = True so the caller does not delete the backups.
"""
pending: list[tuple[Path, Path, Path]] = []
for index, (relative, source) in enumerate(staged.items()):
target = run_dir / relative
target.parent.mkdir(parents=True, exist_ok=True)
backup = staging_dir / f"backup-{index}.tmp"
if target.is_file():
# copyfile + copymode: copy2 would also copy BSD flags such as
# immutable, which would make the backup impossible to delete.
shutil.copyfile(target, backup)
shutil.copymode(target, backup)
# Keep the live file's permissions across the replacement.
os.chmod(source, stat.S_IMODE(target.stat().st_mode))
pending.append((source, target, backup))
applied: list[tuple[Path, Path]] = []
try:
for source, target, backup in pending:
# Recorded before the rename so a rename followed by an interrupt
# is still restored.
applied.append((target, backup))
os.replace(source, target)
except BaseException as exc:
failures, interrupt = _restore_files(applied)
if interrupt is not None:
if failures:
interrupt.keep_staging = True # type: ignore[attr-defined]
print(
"WARNING: restore was interrupted; prior files are kept in "
f"{staging_dir}: " + "; ".join(failures),
file=sys.stderr,
)
if interrupt is exc:
raise
raise interrupt from exc
if failures:
raise _RollbackIncomplete(
f"commit failed ({exc}) and the prior checkpoint could not be fully restored: "
+ "; ".join(failures)
+ f"; prior files are kept in {staging_dir}"
) from exc
if isinstance(exc, OSError):
raise RunStateError(
f"cannot commit screening audit: {exc}; prior checkpoint restored"
) from exc
raise
def _validate_selection(normalized: dict[str, Any], audit: Mapping[str, Any]) -> tuple[str, str]:
"""Validate the committed selection and store its normalized symbols; touches no files."""
selected = audit.get("selected_symbols")
if not isinstance(selected, list) or not all(
isinstance(value, str) and value.strip() for value in selected
):
raise RunStateError("audit selected_symbols must be an array of symbols")
normalized["selected_symbols"] = sorted({_validate_symbol(value) for value in selected})
deep_plan = _mapping(normalized.get("deep_dive_plan"))
planned = sorted({_validate_symbol(value) for value in deep_plan.get("selected_symbols", [])})
if planned != normalized["selected_symbols"]:
raise RunStateError("deep_dive_plan.selected_symbols must match selected_symbols")
selected_count = deep_plan.get("selected_count")
if (
type(selected_count) is not int
or selected_count < 0
or selected_count != len(normalized["selected_symbols"])
):
raise RunStateError(
"deep_dive_plan.selected_count must be a non-negative integer matching selected_symbols"
)
for key in (
"all_selected_must_be_resolved",
"budget_locked",
"budget_change_requires_rescreen",
"selected_set_is_committed",
):
if deep_plan.get(key) is not True:
raise RunStateError(f"deep_dive_plan.{key} must be true")
for key in ("user_confirmation_required", "user_continue_instruction_allowed"):
if deep_plan.get(key) is not False:
raise RunStateError(f"deep_dive_plan.{key} must be false")
max_deep_dives = int(deep_plan.get("max_deep_dive_candidates") or -1)
if max_deep_dives != int(
_mapping(normalized.get("filters")).get("max_deep_dive_candidates") or -2
):
raise RunStateError("deep_dive_plan.max_deep_dive_candidates must match filters")
expected_payload = {
"analysis_as_of": _text(normalized.get("analysis_as_of")),
"max_deep_dive_candidates": max_deep_dives,
"selected_symbols": normalized["selected_symbols"],
}
if _mapping(deep_plan.get("commitment_payload")) != expected_payload:
raise RunStateError(
"deep_dive_plan.commitment_payload must match the committed selected set"
)
expected_sha = hashlib.sha256(
json.dumps(expected_payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
if (_text(deep_plan.get("selected_set_sha256")) or "").lower() != expected_sha:
raise RunStateError("deep_dive_plan.selected_set_sha256 does not match commitment payload")
pool_status = (_text(normalized.get("candidate_pool_status")) or "").lower()
outcome = (_text(normalized.get("selection_outcome")) or "").lower()
expected_outcome = {
"sufficient": "selected",
"sufficient_pending_enrichment": "selected_pending_enrichment",
"no_qualifying_candidates": "no_candidates",
"no_qualifying_candidates_in_bounded_pool": "no_candidates_in_bounded_pool",
"insufficient_data": "insufficient_data",
}.get(pool_status)
if expected_outcome is None:
raise RunStateError("candidate_pool_status is invalid")
if outcome != expected_outcome:
raise RunStateError(f"selection_outcome must be {expected_outcome!r}")
if (
pool_status in {"sufficient", "sufficient_pending_enrichment"}
and not normalized["selected_symbols"]
):
raise RunStateError(f"candidate_pool_status={pool_status} requires selected symbols")
if (
pool_status not in {"sufficient", "sufficient_pending_enrichment"}
and normalized["selected_symbols"]
):
raise RunStateError(f"candidate_pool_status={pool_status} cannot have selected symbols")
return pool_status, outcome
def cmd_set_screening_audit(args: argparse.Namespace) -> int:
"""Replace a run's audit bundle all-or-nothing; any failure leaves the run unchanged."""
try:
return _attach_screening_audit(args)
except RunStateError:
raise
except (AttributeError, OverflowError, RecursionError, TypeError, ValueError) as exc:
raise RunStateError(f"screening audit is malformed: {exc}") from exc
def _attach_screening_audit(args: argparse.Namespace) -> int:
run_dir: Path = args.run_dir
state = _load_state(run_dir)
if _verified_symbols(state):
raise RunStateError(
"cannot attach a screening audit to a run with verified candidates; "
"initialize a new run directory and verify candidates against the new audit"
)
audit = _read_json(args.audit)
if not isinstance(audit, Mapping):
raise RunStateError("audit file must contain a JSON object")
audit = dict(audit)
if audit.get("audit_schema_version") != AUDIT_SCHEMA_VERSION:
raise RunStateError(f"audit_schema_version must be {AUDIT_SCHEMA_VERSION}")
if (_text(audit.get("contract_revision")) or "") != CONTRACT_REVISION:
raise RunStateError(f"contract_revision must be {CONTRACT_REVISION!r}")
runtime = _mapping(audit.get("runtime"))
expected_runtime = runtime_metadata()
for key in (
"skill_name",
"skill_version",
"schema_version",
"contract_revision",
"runtime_fingerprint",
):
if runtime.get(key) != expected_runtime.get(key):
raise RunStateError(
f"audit runtime {key} does not match installed skill v{SKILL_VERSION}"
)
normalized = dict(audit)
pool_status, outcome = _validate_selection(normalized, audit)
universe_section = _mapping(audit.get("universe"))
candidate_section = _mapping(audit.get("candidate_pool"))
listing_path = _resolve_artifact_argument(
args.universe_artifact or args.listing_artifact,
universe_section,
args.audit,
label="universe",
)
candidate_path = _resolve_artifact_argument(
args.candidate_artifact or args.artifact,
candidate_section,
args.audit,
label="candidate-pool",
)
# Inside the run directory so the commit's renames stay on one filesystem.
# The .tmp names keep a leftover from a killed process out of audit bundles.
staging_dir = Path(tempfile.mkdtemp(prefix=".set-screening-audit-", dir=run_dir))
keep_staging = False
try:
staged: dict[str, Path] = {}
normalized["universe"] = _stage_artifact(
staging_dir,
staged,
universe_section,
listing_path,
"universe-audit-results.jsonl",
"universe",
)
normalized["candidate_pool"] = _stage_artifact(
staging_dir,
staged,
candidate_section,
candidate_path,
"broad-screen-results.jsonl",
"candidate-pool",
)
# The discovery-stage artifacts (enrichment queue, provider-prefilter pool)
# are referenced by bare file names relative to the audit JSON. Copy them
# into run/audit/ and rewrite their paths to the same run-relative base as
# universe/candidate_pool, so prepublish_audit --artifact-root <run> resolves
# every artifact from one root.
enrichment_section = _mapping(normalized.get("enrichment"))
if _text(enrichment_section.get("artifact_path")):
enrichment_path = _resolve_artifact_argument(
None, enrichment_section, args.audit, label="enrichment"
)
normalized["enrichment"] = _stage_artifact(
staging_dir,
staged,
enrichment_section,
enrichment_path,
enrichment_path.name,
"enrichment",
)
generation_section = _mapping(normalized["candidate_pool"].get("generation_audit"))
if generation_section and _text(generation_section.get("artifact_path")):
generation_path = _resolve_artifact_argument(
None, generation_section, args.audit, label="candidate-pool generation"
)
normalized["candidate_pool"]["generation_audit"] = _stage_artifact(
staging_dir,
staged,
generation_section,
generation_path,
generation_path.name,
"candidate-pool generation",
)
state["screening_audit"] = normalized
state["screening_funnel"] = {
"universe_count": int(_mapping(normalized.get("universe")).get("row_count") or 0),
"listing_in_scope_count": int(
_mapping(normalized.get("universe")).get("in_scope_count") or 0
),
"candidate_pool_count": int(
_mapping(normalized.get("candidate_pool")).get("row_count") or 0
),
"discovery_evaluable_count": int(
_mapping(normalized.get("candidate_pool")).get("discovery_evaluable_count") or 0
),
"deep_dive_selected_count": len(normalized["selected_symbols"]),
"preflight_passed_count": 0,
"deep_dive_completed_count": 0,
}
selected_set = set(normalized["selected_symbols"])
# No candidate is verified here: the guard above rejects verified runs.
state["unprocessed_candidates"] = sorted(selected_set)
state["status"] = "partial"
state["updated_at"] = _now()
staged_audit = staging_dir / "audit-document.tmp"
_atomic_write_json(staged_audit, normalized)
staged[AUDIT_DOCUMENT] = staged_audit
staged_events = staging_dir / "events.tmp"
live_events = run_dir / "events.ndjson"
if live_events.exists():
shutil.copyfile(live_events, staged_events)
_append_event_line(
staged_events,
{
"at": _now(),
"event": "screening_audit_attached",
"selected_symbols": normalized["selected_symbols"],
"selection_outcome": outcome,
"candidate_pool_status": pool_status,
},
)
staged["events.ndjson"] = staged_events
staged_state = staging_dir / "run-state.tmp"
_atomic_write_json(staged_state, state)
staged["run_state.json"] = staged_state
_commit_files(run_dir, staging_dir, staged)
except BaseException as exc:
# Backups are deleted only after a commit or a restore that finished cleanly.
keep_staging = bool(getattr(exc, "keep_staging", False))
raise
finally:
if not keep_staging:
shutil.rmtree(staging_dir, ignore_errors=True)
if staging_dir.exists():
print(
f"WARNING: could not remove the staging directory {staging_dir}; "
"delete it by hand once the run is verified",
file=sys.stderr,
)
print(f"attached: {run_dir / AUDIT_DOCUMENT}")
return 0
def cmd_save_candidate(args: argparse.Namespace) -> int:
run_dir: Path = args.run_dir
state = _load_state(run_dir)
candidate = _read_json(args.candidate)
if not isinstance(candidate, Mapping):
raise RunStateError("candidate file must contain a JSON object")
identity = _mapping(candidate.get("identity"))
symbol_text = _text(identity.get("symbol"))
if not symbol_text:
raise RunStateError("candidate.identity.symbol is required")
symbol = _validate_symbol(symbol_text)
selected = _selected_symbols(state)
if symbol not in selected and not args.allow_unselected:
raise RunStateError(f"candidate {symbol} was not selected by the verified screening audit")
audit_section = _mapping(state.get("screening_audit"))
pool_status = (_text(audit_section.get("candidate_pool_status")) or "").lower()
if args.stage == "verified" and pool_status == "sufficient_pending_enrichment":
raise RunStateError(
"cannot save a verified candidate while candidate_pool_status is "
"sufficient_pending_enrichment; save as draft until the pool is resolved "
"and the resolved screening audit is attached"
)
previous = _mapping(_mapping(state.get("candidates")).get(symbol))
if previous.get("stage") == "verified" and args.stage != "verified":
raise RunStateError(
f"cannot downgrade verified candidate {symbol} to {args.stage}; "
"verification is tied to the attached audit"
)
candidate = dict(candidate)
candidate.setdefault("identity", {})["symbol"] = symbol
target = run_dir / "candidates" / f"{symbol}.json"
_atomic_write_json(target, candidate)
candidates = _mapping(state.get("candidates"))
candidates[symbol] = {
"path": str(target.relative_to(run_dir)),
"stage": args.stage,
"updated_at": _now(),
"company_name": _text(identity.get("company_name")),
}
state["candidates"] = candidates
state["unprocessed_candidates"] = sorted(selected - _verified_symbols(state))
funnel = _mapping(state.get("screening_funnel"))
funnel["deep_dive_completed_count"] = len(_verified_symbols(state) & selected)
state["screening_funnel"] = funnel
_save_state(run_dir, state)
_append_event(
run_dir, {"at": _now(), "event": "candidate_saved", "symbol": symbol, "stage": args.stage}
)
print(f"saved: {target}")
return 0
def cmd_set_funnel(args: argparse.Namespace) -> int:
run_dir: Path = args.run_dir
state = _load_state(run_dir)
funnel = _mapping(state.get("screening_funnel"))
if args.preflight_passed_count is not None:
if args.preflight_passed_count < 0:
raise RunStateError("preflight_passed_count must be non-negative")
selected_count = int(funnel.get("deep_dive_selected_count") or 0)
if args.preflight_passed_count > selected_count:
raise RunStateError("preflight_passed_count cannot exceed deep_dive_selected_count")
funnel["preflight_passed_count"] = args.preflight_passed_count
if args.note:
notes = list(funnel.get("notes") or [])
notes.extend(args.note)
funnel["notes"] = notes
state["screening_funnel"] = funnel
_save_state(run_dir, state)
_append_event(run_dir, {"at": _now(), "event": "funnel_updated", "funnel": funnel})
print(json.dumps(funnel, ensure_ascii=False, indent=2))
return 0
def cmd_set_unprocessed(args: argparse.Namespace) -> int:
"""Validate, not override, the mathematically expected unprocessed set."""
run_dir: Path = args.run_dir
state = _load_state(run_dir)
supplied = {_validate_symbol(value) for value in args.symbols}
expected = _selected_symbols(state) - _verified_symbols(state)
if supplied != expected:
raise RunStateError(
"cannot set an inconsistent unprocessed list; expected " + json.dumps(sorted(expected))
)
state["unprocessed_candidates"] = sorted(expected)
if expected:
state["status"] = "partial"
_save_state(run_dir, state)
print(json.dumps(sorted(expected), ensure_ascii=False))
return 0
def cmd_set_status(args: argparse.Namespace) -> int:
run_dir: Path = args.run_dir
state = _load_state(run_dir)
if args.status == "complete":
audit = _mapping(state.get("screening_audit"))
if not audit:
raise RunStateError(
"cannot mark complete before a verified tiered screening audit is attached"
)
invariants = _state_invariants(state)
if invariants:
raise RunStateError("cannot mark complete: " + "; ".join(invariants))
if _mapping(audit.get("scope")).get("screening_scope_ready") is not True:
raise RunStateError(
"cannot mark complete: requested scope is neither fully enumerated nor covered by an audited full-range stratified pool"
)
pool_status = (_text(audit.get("candidate_pool_status")) or "").lower()
if pool_status in {"insufficient_data", "sufficient_pending_enrichment"}:
raise RunStateError("cannot mark complete: candidate pool is unresolved")
if pool_status not in {
"sufficient",
"no_qualifying_candidates",
"no_qualifying_candidates_in_bounded_pool",
}:
raise RunStateError("cannot mark complete: candidate_pool_status is invalid")
enrichment = _mapping(audit.get("enrichment"))
if enrichment.get("candidate_pool_exhausted") is not True:
raise RunStateError("cannot mark complete: candidate pool is not exhausted")
if (
enrichment.get("all_rows_resolved") is not True
or int(enrichment.get("unresolved_count") or 0) != 0
):
raise RunStateError("cannot mark complete: unresolved candidate rows remain")
if int(enrichment.get("queue_count") or 0) != 0:
raise RunStateError("cannot mark complete: enrichment queue is not empty")
selected = _selected_symbols(state)
verified = _verified_symbols(state)
if pool_status == "sufficient" and selected != verified:
missing = sorted(selected - verified)
raise RunStateError(
"cannot mark complete; selected symbols are not verified: " + ", ".join(missing)
)
if pool_status in {
"no_qualifying_candidates",
"no_qualifying_candidates_in_bounded_pool",
} and (selected or verified):
raise RunStateError(
"no-qualifying-candidates completion cannot contain selected/verified names"
)
state["status"] = args.status
_save_state(run_dir, state)
_append_event(run_dir, {"at": _now(), "event": "status_updated", "status": args.status})
print(f"status: {args.status}")
return 0
def _candidate_paths(run_dir: Path, state: Mapping[str, Any], include_drafts: bool) -> list[Path]:
paths: list[Path] = []
selected = _selected_symbols(state)
for symbol, metadata_value in sorted(_mapping(state.get("candidates")).items()):
if symbol not in selected:
continue
metadata = _mapping(metadata_value)
if not include_drafts and metadata.get("stage") != "verified":
continue
relative = _text(metadata.get("path"))
if not relative:
raise RunStateError(f"candidate {symbol} has no path")
path = run_dir / relative
if not path.is_file():
raise RunStateError(f"candidate checkpoint missing: {path}")
paths.append(path)
return paths
def cmd_assemble(args: argparse.Namespace) -> int:
run_dir: Path = args.run_dir
state = _load_state(run_dir)
invariants = _state_invariants(state)
if invariants:
raise RunStateError("run state is inconsistent: " + "; ".join(invariants))
candidates: list[Any] = []
for path in _candidate_paths(run_dir, state, args.include_drafts):
candidate = _read_json(path)
if not isinstance(candidate, Mapping):
raise RunStateError(f"candidate checkpoint must be an object: {path}")
candidates.append(candidate)
selected = _selected_symbols(state)
audit = _mapping(state.get("screening_audit"))
pool_status = (_text(audit.get("candidate_pool_status")) or "").lower()
if selected and not candidates:
raise RunStateError("no selected candidate checkpoints are available for assembly")
if not selected and not (
state.get("status") == "complete"
and pool_status in {"no_qualifying_candidates", "no_qualifying_candidates_in_bounded_pool"}
):
raise RunStateError(
"zero-candidate snapshot is valid only after a complete no-qualifying-candidates screen"
)
payload = {
"schema_version": SNAPSHOT_SCHEMA_VERSION,
"runtime": runtime_metadata(),
"analysis_as_of": state.get("analysis_as_of"),
"run_metadata": {
"run_id": state.get("run_id"),
"status": state.get("status", "partial"),
"base_repository_commit": state.get("base_repository_commit"),
"selected_symbols": sorted(selected),
"unprocessed_candidates": state.get("unprocessed_candidates", []),
"checkpoint_generated_at": _now(),
},
"price_basis": state.get("price_basis", {}),
"config": state.get("config", {}),
"global_sources": state.get("global_sources", []),
"market_context": state.get("market_context", {}),
"screening_funnel": state.get("screening_funnel", {}),
"screening_audit": state.get("screening_audit", {}),
"candidates": candidates,
}
_atomic_write_json(args.output, payload)
_append_event(
run_dir,
{
"at": _now(),
"event": "snapshot_assembled",
"output": str(args.output),
"candidate_count": len(candidates),
},
)
print(f"assembled: {args.output}")
return 0
def cmd_status(args: argparse.Namespace) -> int:
run_dir: Path = args.run_dir
state = _load_state(run_dir)
candidates = _mapping(state.get("candidates"))
stage_counts: dict[str, int] = {}
for metadata in candidates.values():
stage = _text(_mapping(metadata).get("stage")) or "unknown"
stage_counts[stage] = stage_counts.get(stage, 0) + 1
audit = _mapping(state.get("screening_audit"))
invariants = _state_invariants(state)
summary = {
"run_id": state.get("run_id"),
"status": state.get("status"),
"updated_at": state.get("updated_at"),
"candidate_count": len(candidates),
"stage_counts": stage_counts,
"unprocessed_candidates": state.get("unprocessed_candidates", []),
"screening_funnel": state.get("screening_funnel", {}),
"screening_audit_attached": bool(audit),
"selection_outcome": audit.get("selection_outcome"),
"candidate_pool_status": audit.get("candidate_pool_status"),
"selected_symbols": audit.get("selected_symbols", []),
"state_consistent": not invariants,
"invariant_errors": invariants,
"next_action": _next_action_payload(state),
}
print(json.dumps(summary, ensure_ascii=False, indent=2))
return 2 if args.strict and invariants else 0
def _next_action_payload(state: Mapping[str, Any]) -> dict[str, Any]:
audit = _mapping(state.get("screening_audit"))
selected = sorted(_selected_symbols(state)) if audit else []
verified = sorted(_verified_symbols(state))
remaining = sorted(set(selected) - set(verified))
pool_status = (_text(audit.get("candidate_pool_status")) or "").lower()
enrichment = _mapping(audit.get("enrichment"))
if not audit:
action = "attach_screening_audit"
symbols: list[str] = []
elif (
int(enrichment.get("queue_count") or 0) > 0
or enrichment.get("all_rows_resolved") is not True
):
action = "continue_candidate_pool_enrichment"
symbols = sorted(str(v).upper() for v in enrichment.get("queue_symbols", []))
elif remaining:
action = "complete_all_selected_deep_dives"
symbols = remaining
elif state.get("status") != "complete":
action = "mark_complete_and_assemble"
symbols = []
else:
action = "assemble_and_run_strict_evaluation"
symbols = []
return {
"action": action,
"symbols": symbols,
"selected_symbols": selected,
"verified_symbols": verified,
"candidate_pool_status": pool_status,
"user_confirmation_required": False,
"instruction": (
"Continue in the same execution. Never ask the user which selected symbols to process. "
"A budget change requires rerunning the deterministic broad screen before any selected symbol is dropped."
),
}
def cmd_next_action(args: argparse.Namespace) -> int:
state = _load_state(args.run_dir)
invariants = _state_invariants(state)
payload = _next_action_payload(state)
payload["state_consistent"] = not invariants
payload["invariant_errors"] = invariants
print(json.dumps(payload, ensure_ascii=False, indent=2))
return 2 if invariants else 0
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Checkpoint and resume schema-3 / contract-3.5 undervalued-growth runs."
)
parser.add_argument(
"--version", action="store_true", help="Print installed skill/runtime metadata and exit"
)
sub = parser.add_subparsers(dest="command")
init = sub.add_parser("init")
init.add_argument("--run-dir", type=Path, required=True)
init.add_argument("--run-id")
init.add_argument("--analysis-as-of", required=True)
init.add_argument("--price-as-of", required=True)
init.add_argument(
"--session",
choices=("regular_close", "pre_market", "after_hours", "intraday"),
required=True,
)
init.add_argument("--timezone", default="America/New_York")
init.add_argument("--price-source-id", required=True)
init.add_argument("--base-commit")
init.add_argument("--config", type=Path)
init.add_argument("--market-context", type=Path, required=True)
init.add_argument("--global-sources", type=Path, required=True)
init.add_argument("--force", action="store_true")
init.set_defaults(func=cmd_init)
audit = sub.add_parser("set-screening-audit")
audit.add_argument("--run-dir", type=Path, required=True)
audit.add_argument("--audit", type=Path, required=True)
audit.add_argument("--universe-artifact", type=Path)
audit.add_argument("--listing-artifact", type=Path, help="Alias for --universe-artifact")
audit.add_argument("--candidate-artifact", type=Path)
audit.add_argument("--artifact", type=Path, help="Legacy alias for --candidate-artifact")
audit.set_defaults(func=cmd_set_screening_audit)
save = sub.add_parser("save-candidate")
save.add_argument("--run-dir", type=Path, required=True)
save.add_argument("--candidate", type=Path, required=True)
save.add_argument("--stage", choices=("draft", "verified"), default="draft")
save.add_argument("--allow-unselected", action="store_true")
save.set_defaults(func=cmd_save_candidate)
funnel = sub.add_parser("set-funnel")
funnel.add_argument("--run-dir", type=Path, required=True)
funnel.add_argument("--preflight-passed-count", type=int)
funnel.add_argument("--note", action="append")
funnel.set_defaults(func=cmd_set_funnel)
unprocessed = sub.add_parser("set-unprocessed")
unprocessed.add_argument("--run-dir", type=Path, required=True)
unprocessed.add_argument("symbols", nargs="*")
unprocessed.set_defaults(func=cmd_set_unprocessed)
status = sub.add_parser("set-status")
status.add_argument("--run-dir", type=Path, required=True)
status.add_argument("status", choices=("partial", "complete"))
status.set_defaults(func=cmd_set_status)
assemble = sub.add_parser("assemble")
assemble.add_argument("--run-dir", type=Path, required=True)
assemble.add_argument("--output", type=Path, required=True)
assemble.add_argument("--include-drafts", action="store_true")
assemble.set_defaults(func=cmd_assemble)
next_action = sub.add_parser("next-action")
next_action.add_argument("--run-dir", type=Path, required=True)
next_action.set_defaults(func=cmd_next_action)
show = sub.add_parser("status")
show.add_argument("--run-dir", type=Path, required=True)
show.add_argument("--strict", action="store_true")
show.set_defaults(func=cmd_status)
return parser
def main(argv: Sequence[str] | None = None) -> int:
parser = build_parser()
args = parser.parse_args(argv)
if args.version:
print(json.dumps(runtime_metadata(), sort_keys=True))
return 0
if not args.command:
parser.error("the following arguments are required: command")
try:
return int(args.func(args))
except (OSError, RunStateError) as exc:
print(f"ERROR: {exc}", file=sys.stderr)
return 1
if __name__ == "__main__":
raise SystemExit(main())
scripts/normalize_estimates.py
#!/usr/bin/env python3
"""Normalize annual consensus rows into contract-3.5 discovery metrics.
A current forward P/E is computed only from a dated FY1 estimate. The helper
never substitutes FY2/FY3 merely because FY1 is missing. It records estimate
breadth, range dispersion, period continuity, growth horizons, and source IDs so
the deterministic broad screen can reject FRMI-like outer-year artefacts.
"""
from __future__ import annotations
import argparse
import csv
import json
import math
import sys
from collections import defaultdict
from collections.abc import Mapping, Sequence
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from zoneinfo import ZoneInfo
# Provider publish/acceptance stamps without an explicit offset are US/Eastern
# wall time (the SEC acceptance clock), not UTC.
PROVIDER_PUBLISH_TZ = ZoneInfo("America/New_York")
try:
from skill_version import SKILL_VERSION, runtime_metadata
except ModuleNotFoundError: # pragma: no cover
import importlib.util as _importlib_util
_path = Path(__file__).with_name("skill_version.py")
_spec = _importlib_util.spec_from_file_location("skill_version", _path)
if _spec is None or _spec.loader is None:
raise
_module = _importlib_util.module_from_spec(_spec)
_spec.loader.exec_module(_module)
runtime_metadata = _module.runtime_metadata
SKILL_VERSION = _module.SKILL_VERSION
class NormalizeError(ValueError):
"""Raised for malformed estimate inputs."""
def _number(value: Any) -> float | None:
if isinstance(value, bool):
return None
if isinstance(value, (int, float)):
result = float(value)
return result if math.isfinite(result) else None
if isinstance(value, str):
try:
result = float(value.replace(",", "").strip())
except ValueError:
return None
return result if math.isfinite(result) else None
return None
def _integer(value: Any) -> int | None:
number = _number(value)
return int(number) if number is not None and number.is_integer() else None
def _text(value: Any) -> str | None:
if isinstance(value, str):
text = value.strip()
return text or None
return None
def _bool(value: Any) -> bool | None:
if isinstance(value, bool):
return value
if isinstance(value, str):
lowered = value.strip().lower()
if lowered in {"true", "1", "yes", "y"}:
return True
if lowered in {"false", "0", "no", "n"}:
return False
return None
def _parse_date(value: Any, label: str) -> datetime:
text = _text(value)
if not text:
raise NormalizeError(f"{label} is required")
try:
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
except ValueError as exc:
raise NormalizeError(f"{label} is not ISO-8601: {text!r}") from exc
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed
def _load_rows(path: Path) -> list[dict[str, Any]]:
suffix = path.suffix.lower()
if suffix == ".csv":
with path.open("r", encoding="utf-8-sig", newline="") as handle:
return [dict(row) for row in csv.DictReader(handle)]
text = path.read_text(encoding="utf-8")
if suffix == ".jsonl":
return [json.loads(line) for line in text.splitlines() if line.strip()]
value = json.loads(text)
if isinstance(value, list):
return [dict(row) for row in value if isinstance(row, Mapping)]
if isinstance(value, Mapping) and isinstance(value.get("rows"), list):
return [dict(row) for row in value["rows"] if isinstance(row, Mapping)]
raise NormalizeError(f"unsupported row container in {path}")
def _symbol(row: Mapping[str, Any]) -> str:
return (_text(row.get("symbol")) or _text(row.get("ticker")) or "UNKNOWN").upper()
def _period_end(row: Mapping[str, Any]) -> datetime | None:
for key in ("date", "period_end", "fiscal_period_end", "fiscalDateEnding"):
text = _text(row.get(key))
if not text:
continue
try:
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
except ValueError:
continue
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed
return None
def _pick(row: Mapping[str, Any], *keys: str) -> float | None:
for key in keys:
value = _number(row.get(key))
if value is not None:
return value
return None
def _period_record(row: Mapping[str, Any]) -> dict[str, Any] | None:
end = _period_end(row)
if end is None:
return None
fiscal_year = _text(row.get("fiscalYear")) or _text(row.get("fiscal_year")) or str(end.year)
fiscal_year = fiscal_year.upper().removeprefix("FY")
is_actual = (
_bool(row.get("isActual")) or _bool(row.get("is_actual")) or _bool(row.get("actual"))
)
return {
"fiscal_year": fiscal_year,
"period": f"FY{fiscal_year}",
"period_end": end.isoformat(),
"eps_avg": _pick(row, "epsAvg", "eps_avg", "eps", "estimated_eps"),
"eps_low": _pick(row, "epsLow", "eps_low"),
"eps_high": _pick(row, "epsHigh", "eps_high"),
"revenue_avg": _pick(row, "revenueAvg", "revenue_avg", "revenue", "estimated_revenue"),
"eps_analyst_count": _integer(row.get("numAnalystsEps"))
or _integer(row.get("num_analysts_eps")),
"revenue_analyst_count": _integer(row.get("numAnalystsRevenue"))
or _integer(row.get("num_analysts_revenue")),
"is_actual": bool(is_actual),
"published_at": _text(row.get("publishedDate"))
or _text(row.get("published_at"))
or _text(row.get("updatedAt"))
or None,
}
def _latest_actual_period(
periods: Sequence[Mapping[str, Any]], analysis_as_of: datetime
) -> Mapping[str, Any] | None:
"""Return the most recent provider-marked actual annual EPS row, else None.
Verified reported figures normally arrive through
``apply_verified_actual_eps`` (annual income statement with an accepted
date at or before ``analysis_as_of``); this helper only honours explicit
provider actual markers and never treats a prior-year estimate as actual.
"""
# Fail closed: only rows the provider explicitly marks as actuals count,
# and only when (a) their period has already ended AND (b) the row carries
# a publication timestamp at or before analysis_as_of. Without a provable
# publication time, a CURRENT provider snapshot replayed against a
# HISTORICAL analysis_as_of would leak an actual that was not yet public
# on that date (e.g. FY ended 2025-12-31, as-of 2026-01-05, filed
# 2026-02-27). An unmarked prior-year row is consensus, never an actual.
def _published_before(record: Mapping[str, Any]) -> bool:
raw = _text(record.get("published_at"))
if not raw:
return False
try:
parsed = datetime.fromisoformat(raw.replace("Z", "+00:00").replace(" ", "T"))
except ValueError:
# NormalizeError is a ValueError subclass; any unparsable stamp
# fails closed (treated as not yet published).
return False
if parsed.tzinfo is None:
if len(raw.strip()) <= 10:
# A date-only publish stamp cannot prove intraday precedence:
# count it only once the whole publication day (ET) has passed.
parsed = parsed.replace(hour=23, minute=59, second=59)
parsed = parsed.replace(tzinfo=PROVIDER_PUBLISH_TZ)
published = parsed.astimezone(timezone.utc)
return published <= analysis_as_of
marked_actual = [
record
for record in periods
if record.get("is_actual")
and _parse_date(record["period_end"], "period_end") <= analysis_as_of
and _published_before(record)
]
if marked_actual:
return max(marked_actual, key=lambda record: str(record["period_end"]))
return None
def _growth_pattern(
latest_actual_eps: float | None,
fy1_eps: float | None,
fy3_eps: float | None,
early_growth_pct: float | None,
late_growth_pct: float | None,
) -> str:
if latest_actual_eps is None or latest_actual_eps <= 0:
return "unknown"
if fy1_eps is not None and fy3_eps is not None:
if fy1_eps < latest_actual_eps and fy3_eps > latest_actual_eps:
return "trough_recovery"
if fy3_eps < latest_actual_eps:
return "declining"
if early_growth_pct is not None and late_growth_pct is not None:
if late_growth_pct > early_growth_pct + 1e-9:
return "accelerating"
return "steady"
def _dispersion_pct(avg: float | None, low: float | None, high: float | None) -> float | None:
if avg is None or low is None or high is None or avg == 0:
return None
return abs(high - low) / abs(avg) * 100.0
def _cagr(start: float | None, end: float | None, years: float) -> float | None:
if start is None or end is None or start <= 0 or end <= 0 or years <= 0:
return None
return ((end / start) ** (1.0 / years) - 1.0) * 100.0
def normalize_symbol(
symbol: str,
estimate_rows: Sequence[Mapping[str, Any]],
listing: Mapping[str, Any],
*,
analysis_as_of: datetime,
estimate_as_of: datetime,
source_ids: Sequence[str],
minimum_analysts: int,
max_dispersion_pct: float,
max_fy1_horizon_days: int,
forward_pe_tolerance_pct: float,
) -> dict[str, Any]:
price = _pick(listing, "price", "last")
periods = [record for row in estimate_rows if (record := _period_record(row)) is not None]
periods.sort(key=lambda record: str(record["period_end"]))
future = [
record
for record in periods
if _parse_date(record["period_end"], "period_end") > analysis_as_of
]
reasons: list[str] = []
operating_stage = (_text(listing.get("operating_stage")) or "").lower().replace(" ", "_")
if _bool(listing.get("is_pre_operating")) is True or operating_stage in {
"pre_operating",
"pre-revenue",
"pre_revenue",
"development_stage",
}:
reasons.append("pre_operating_company")
fy1 = future[0] if future else None
fy1_horizon_days: int | None = None
if fy1 is None:
reasons.append("fy1_estimate_unavailable")
else:
fy1_horizon_days = (_parse_date(fy1["period_end"], "fy1.period_end") - analysis_as_of).days
if fy1_horizon_days <= 0 or fy1_horizon_days > max_fy1_horizon_days:
reasons.append("invalid_fy1_horizon")
eps = _number(fy1.get("eps_avg"))
low = _number(fy1.get("eps_low"))
high = _number(fy1.get("eps_high"))
if eps is None or eps <= 0:
reasons.append("non_positive_fy1_eps")
if low is not None and high is not None and low <= 0 <= high:
reasons.append("fy1_eps_range_crosses_zero")
dispersion = _dispersion_pct(eps, low, high)
if dispersion is not None and dispersion > max_dispersion_pct:
reasons.append("fy1_estimate_dispersion_excessive")
analysts = _integer(fy1.get("eps_analyst_count"))
if analysts is None or analysts < minimum_analysts:
reasons.append("estimate_breadth_below_discovery_minimum")
series_contiguous = True
for left, right in zip(future, future[1:]):
delta_days = (
_parse_date(right["period_end"], "period_end")
- _parse_date(left["period_end"], "period_end")
).days
if not 300 <= delta_days <= 430:
series_contiguous = False
break
if len(future) >= 2 and not series_contiguous:
reasons.append("annual_estimate_series_not_contiguous")
forward_eps = _number(fy1.get("eps_avg")) if fy1 else None
forward_pe = (
price / forward_eps
if price is not None and price > 0 and forward_eps is not None and forward_eps > 0
else None
)
if price is None or price <= 0:
reasons.append("price_unavailable_or_non_positive")
supplied_pe = _pick(listing, "forward_pe", "fy1_pe", "ntm_pe")
if supplied_pe is not None and forward_pe is not None:
mismatch = abs(supplied_pe - forward_pe) / max(abs(forward_pe), 1e-9) * 100.0
if mismatch > forward_pe_tolerance_pct:
reasons.append("forward_pe_reconciliation_failed")
growth_end = future[2] if len(future) >= 3 else (future[1] if len(future) >= 2 else None)
growth_years = 0.0
if fy1 and growth_end:
growth_years = (
_parse_date(growth_end["period_end"], "growth_end.period_end")
- _parse_date(fy1["period_end"], "fy1.period_end")
).days / 365.25
eps_growth = _cagr(
forward_eps, _number(growth_end.get("eps_avg")) if growth_end else None, growth_years
)
revenue_growth = _cagr(
_number(fy1.get("revenue_avg")) if fy1 else None,
_number(growth_end.get("revenue_avg")) if growth_end else None,
growth_years,
)
# Growth-basis diagnostics (v3.6.1): compare the forward estimate path to
# the latest actual annual EPS so a trough-recovery FY1->FY3 CAGR (a
# current-year decline masked by an eventual rebound, e.g. YELP FY26E
# below FY25 actual) is visible rather than silently blended away.
latest_actual_period = _latest_actual_period(periods, analysis_as_of)
latest_actual_eps = (
_number(latest_actual_period.get("eps_avg")) if latest_actual_period else None
)
latest_actual_period_end = (
latest_actual_period.get("period_end") if latest_actual_period else None
)
# Same-basis reference: the provider's most recent prior-year row. It is
# CONSENSUS on the provider's (usually adjusted) basis -- never reported as
# an actual -- but it is the only figure on the same basis as FY1/FY3, so
# it is the correct comparator for "is the current year a decline".
fy0_candidates = [
record
for record in periods
if _parse_date(record["period_end"], "period_end") <= analysis_as_of
]
fy0 = (
max(fy0_candidates, key=lambda record: str(record["period_end"]))
if fy0_candidates
else None
)
fy0_consensus_eps = _number(fy0.get("eps_avg")) if fy0 else None
fy0_period_end = fy0.get("period_end") if fy0 else None
fy1_raw_eps = _number(fy1.get("eps_avg")) if fy1 else None
growth_end_raw_eps = _number(growth_end.get("eps_avg")) if growth_end else None
has_positive_actual = latest_actual_eps is not None and latest_actual_eps > 0
has_positive_fy0 = fy0_consensus_eps is not None and fy0_consensus_eps > 0
fy1_eps_below_latest_actual = (
fy1_raw_eps < latest_actual_eps if has_positive_actual and fy1_raw_eps is not None else None
)
fy1_eps_below_fy0_consensus = (
fy1_raw_eps < fy0_consensus_eps if has_positive_fy0 and fy1_raw_eps is not None else None
)
current_year_growth_pct = (
(fy1_raw_eps / fy0_consensus_eps - 1.0) * 100.0
if has_positive_fy0 and fy1_raw_eps is not None
else None
)
actual_to_growth_end_years = 0.0
if latest_actual_period is not None and growth_end is not None:
actual_to_growth_end_years = (
_parse_date(growth_end["period_end"], "growth_end.period_end")
- _parse_date(latest_actual_period["period_end"], "latest_actual.period_end")
).days / 365.25
eps_growth_actual_to_fy3 = (
_cagr(latest_actual_eps, growth_end_raw_eps, actual_to_growth_end_years)
if has_positive_actual
else None
)
fy2 = future[1] if len(future) >= 2 else None
early_growth_pct = None
late_growth_pct = None
if fy1 and fy2:
early_years = (
_parse_date(fy2["period_end"], "fy2.period_end")
- _parse_date(fy1["period_end"], "fy1.period_end")
).days / 365.25
early_growth_pct = _cagr(fy1_raw_eps, _number(fy2.get("eps_avg")), early_years)
if len(future) >= 3:
fy3 = future[2]
late_years = (
_parse_date(fy3["period_end"], "fy3.period_end")
- _parse_date(fy2["period_end"], "fy2.period_end")
).days / 365.25
late_growth_pct = _cagr(
_number(fy2.get("eps_avg")), _number(fy3.get("eps_avg")), late_years
)
growth_pattern = _growth_pattern(
fy0_consensus_eps,
fy1_raw_eps,
growth_end_raw_eps,
early_growth_pct,
late_growth_pct,
)
growth_pattern_basis = "consensus_same_basis" if has_positive_fy0 else "unknown"
status = "valid" if not reasons else "unavailable"
canonical_forward_eps = forward_eps if status == "valid" else None
canonical_forward_pe = forward_pe if status == "valid" else None
canonical_fy1 = fy1 if status == "valid" else None
result = dict(listing)
result.update(
{
"symbol": symbol,
"normalized_estimates_version": SKILL_VERSION,
"enrichment_attempted": True,
# The enrichment step is resolved whether it produced a usable FY1
# metric or exhausted the available evidence with explicit reasons.
"enrichment_resolved": True,
"forward_metric_basis": "fy1" if canonical_fy1 else None,
"forward_pe_period": "FY1" if canonical_fy1 else None,
"forward_metric_period": "FY1" if canonical_fy1 else None,
"forward_fiscal_year": canonical_fy1.get("fiscal_year") if canonical_fy1 else None,
"forward_period_end": canonical_fy1.get("period_end") if canonical_fy1 else None,
"forward_metric_period_end": canonical_fy1.get("period_end") if canonical_fy1 else None,
"forward_estimate_as_of": estimate_as_of.isoformat(),
"forward_estimate_source_ids": list(source_ids),
"forward_metric_source_ids": list(source_ids),
"forward_metric_origin": "computed_from_price_and_fy1_eps" if canonical_fy1 else None,
"fy1_horizon_days": fy1_horizon_days,
"forward_eps": canonical_forward_eps,
"fy1_eps": canonical_forward_eps,
"forward_eps_low": _number(canonical_fy1.get("eps_low")) if canonical_fy1 else None,
"forward_eps_high": _number(canonical_fy1.get("eps_high")) if canonical_fy1 else None,
"forward_estimate_dispersion_pct": _dispersion_pct(
canonical_forward_eps,
_number(canonical_fy1.get("eps_low")) if canonical_fy1 else None,
_number(canonical_fy1.get("eps_high")) if canonical_fy1 else None,
),
"forward_pe": round(canonical_forward_pe, 6)
if canonical_forward_pe is not None
else None,
"fy1_pe": round(canonical_forward_pe, 6) if canonical_forward_pe is not None else None,
"analyst_count": _integer(canonical_fy1.get("eps_analyst_count"))
if canonical_fy1
else None,
"forward_eps_analyst_count": _integer(canonical_fy1.get("eps_analyst_count"))
if canonical_fy1
else None,
"fy1_analyst_count": _integer(canonical_fy1.get("eps_analyst_count"))
if canonical_fy1
else None,
"eps_growth_pct": round(eps_growth, 6)
if eps_growth is not None and status == "valid"
else None,
# v3.6.1 alias: identical semantics to eps_growth_pct (FY1->FY3
# CAGR), kept under an explicit name alongside eps_growth_actual_to_fy3_pct.
"eps_growth_fy1_to_fy3_pct": round(eps_growth, 6)
if eps_growth is not None and status == "valid"
else None,
"latest_actual_eps": round(latest_actual_eps, 6)
if latest_actual_eps is not None
else None,
"latest_actual_period_end": latest_actual_period_end,
"latest_actual_verified": latest_actual_period is not None,
"latest_actual_basis": "provider_marked_actual"
if latest_actual_period is not None
else None,
"latest_actual_source_ids": list(source_ids)
if latest_actual_period is not None
else [],
"fy1_eps_below_latest_actual": fy1_eps_below_latest_actual,
"fy0_consensus_eps": round(fy0_consensus_eps, 6)
if fy0_consensus_eps is not None
else None,
"fy0_period_end": fy0_period_end,
"fy1_eps_below_fy0_consensus": fy1_eps_below_fy0_consensus,
"current_year_growth_pct": round(current_year_growth_pct, 6)
if current_year_growth_pct is not None
else None,
"growth_pattern_basis": growth_pattern_basis,
"estimate_basis_likely_adjusted": None,
"eps_growth_actual_to_fy3_pct": round(eps_growth_actual_to_fy3, 6)
if eps_growth_actual_to_fy3 is not None
else None,
"growth_pattern": growth_pattern,
"growth_basis_source_ids": list(source_ids),
"revenue_growth_pct": round(revenue_growth, 6)
if revenue_growth is not None and status == "valid"
else None,
"growth_horizon_start_period": "FY1" if canonical_fy1 else None,
"growth_horizon_start_period_end": canonical_fy1.get("period_end")
if canonical_fy1
else None,
"growth_horizon_end_period": growth_end.get("period")
if growth_end and status == "valid"
else None,
"growth_horizon_end_period_end": growth_end.get("period_end")
if growth_end and status == "valid"
else None,
"growth_horizon_years": round(growth_years, 6)
if growth_years > 0 and status == "valid"
else None,
"growth_estimate_source_ids": list(source_ids),
"estimate_series_contiguous": series_contiguous,
"estimate_periods": future[:4],
"estimate_normalization_status": status,
"estimate_normalization_reasons": sorted(set(reasons)),
# Preserve the raw candidate for diagnostics without allowing it to
# masquerade as the current FY1/NTM valuation in downstream screens.
"raw_forward_candidate": {
"fiscal_year": fy1.get("fiscal_year") if fy1 else None,
"period_end": fy1.get("period_end") if fy1 else None,
"eps": forward_eps,
"eps_low": _number(fy1.get("eps_low")) if fy1 else None,
"eps_high": _number(fy1.get("eps_high")) if fy1 else None,
"computed_pe": round(forward_pe, 6) if forward_pe is not None else None,
"analyst_count": _integer(fy1.get("eps_analyst_count")) if fy1 else None,
},
}
)
if reasons:
result.update(
{
"enrichment_exhausted": True,
"enrichment_exhaustion_reason": "; ".join(sorted(set(reasons))),
"enrichment_source_ids": list(source_ids),
}
)
else:
result.update(
{
"enrichment_exhausted": False,
"enrichment_exhaustion_reason": None,
"enrichment_source_ids": list(source_ids),
}
)
return result
def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Normalize raw annual estimates into current FY1/NTM discovery rows."
)
parser.add_argument("--estimates", type=Path, required=True)
parser.add_argument("--listing-input", type=Path, required=True)
parser.add_argument("--analysis-as-of", required=True)
parser.add_argument("--estimate-as-of", required=True, help="Estimate retrieval/data timestamp")
parser.add_argument("--source-id", action="append", required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--minimum-analysts", type=int, default=2)
parser.add_argument("--max-dispersion-pct", type=float, default=100.0)
parser.add_argument("--max-fy1-horizon-days", type=int, default=430)
parser.add_argument("--forward-pe-tolerance-pct", type=float, default=3.0)
return parser.parse_args(argv)
def main(argv: Sequence[str] | None = None) -> int:
raw_argv = list(argv) if argv is not None else sys.argv[1:]
if "--version" in raw_argv:
print(json.dumps(runtime_metadata(), sort_keys=True))
return 0
args = parse_args(raw_argv)
try:
analysis_as_of = _parse_date(args.analysis_as_of, "analysis_as_of")
estimate_as_of = _parse_date(args.estimate_as_of, "estimate_as_of")
estimate_rows = _load_rows(args.estimates)
listing_rows = _load_rows(args.listing_input)
listing_index = {_symbol(row): row for row in listing_rows}
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
for row in estimate_rows:
grouped[_symbol(row)].append(dict(row))
symbols = sorted(set(listing_index) | set(grouped))
output_rows = [
normalize_symbol(
symbol,
grouped.get(symbol, []),
listing_index.get(symbol, {"symbol": symbol}),
analysis_as_of=analysis_as_of,
estimate_as_of=estimate_as_of,
source_ids=args.source_id,
minimum_analysts=args.minimum_analysts,
max_dispersion_pct=args.max_dispersion_pct,
max_fy1_horizon_days=args.max_fy1_horizon_days,
forward_pe_tolerance_pct=args.forward_pe_tolerance_pct,
)
for symbol in symbols
]
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(
"".join(
json.dumps(row, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n"
for row in output_rows
),
encoding="utf-8",
)
except (OSError, ValueError, json.JSONDecodeError, NormalizeError) as exc:
print(f"ERROR: {exc}", file=sys.stderr)
return 1
print(f"wrote: {args.output}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
def apply_verified_actual_eps(
row: Mapping[str, Any],
*,
actual_eps: float | None,
period_end: str | None,
analysis_as_of: datetime,
source_ids: Sequence[str],
) -> dict[str, Any]:
"""Re-derive the growth-basis fields from a VERIFIED reported annual EPS.
``actual_eps``/``period_end`` must come from a filing whose accepted date is
at or before ``analysis_as_of`` (the caller checks that). When the actual is
missing, non-positive, or dated after ``analysis_as_of``, every
actual-derived field is reset to None and ``growth_pattern`` to
``"unknown"`` so nothing downstream can mistake consensus for a reported
figure.
"""
out = dict(row)
# growth_pattern / current_year_growth_pct stay on the consensus same-basis
# comparison made at normalization; a GAAP actual must never be compared
# against an (usually adjusted) consensus to relabel them.
reset = {
"latest_actual_eps": None,
"latest_actual_period_end": None,
"latest_actual_source_ids": [],
"latest_actual_verified": False,
"latest_actual_basis": None,
"fy1_eps_below_latest_actual": None,
"current_year_growth_pct_vs_gaap_actual": None,
"eps_growth_actual_to_fy3_pct": None,
"estimate_basis_likely_adjusted": None,
}
if actual_eps is None or actual_eps <= 0 or not period_end:
out.update(reset)
return out
end = _parse_date(period_end, "actual.period_end")
if end > analysis_as_of:
out.update(reset)
return out
fy1_eps = _number(out.get("fy1_eps"))
periods = [dict(p) for p in (out.get("estimate_periods") or []) if isinstance(p, Mapping)]
growth_end_period = _text(out.get("growth_horizon_end_period"))
growth_end = next((p for p in periods if _text(p.get("period")) == growth_end_period), None)
growth_end_eps = _number(growth_end.get("eps_avg")) if growth_end else None
growth_end_end = _text(growth_end.get("period_end")) if growth_end else None
fy1_below = fy1_eps < actual_eps if fy1_eps is not None else None
current_year_growth = (fy1_eps / actual_eps - 1.0) * 100.0 if fy1_eps is not None else None
fy0_consensus = _number(out.get("fy0_consensus_eps"))
# Consensus is usually adjusted (ex-SBC, ex-amortization); when the
# provider's own prior-year row differs from the GAAP actual by more than
# 15%, the bases differ and GAAP-vs-consensus comparisons are unreliable.
basis_flag = (
abs(fy0_consensus - actual_eps) / actual_eps > 0.15 if fy0_consensus is not None else None
)
actual_to_fy3 = None
if growth_end_eps is not None and growth_end_end:
years = (_parse_date(growth_end_end, "growth_end.period_end") - end).days / 365.25
actual_to_fy3 = _cagr(actual_eps, growth_end_eps, years)
out.update(
{
"latest_actual_eps": round(actual_eps, 6),
"latest_actual_period_end": period_end,
"latest_actual_source_ids": list(source_ids),
"latest_actual_verified": True,
"latest_actual_basis": "gaap_diluted",
"fy1_eps_below_latest_actual": fy1_below,
"current_year_growth_pct_vs_gaap_actual": round(current_year_growth, 6)
if current_year_growth is not None
else None,
"eps_growth_actual_to_fy3_pct": round(actual_to_fy3, 6)
if actual_to_fy3 is not None
else None,
"estimate_basis_likely_adjusted": basis_flag,
}
)
return out
scripts/prepublish_audit.py
#!/usr/bin/env python3
"""Fail-closed publication audit for US undervalued-growth screening runs.
This script validates the *published* report, not only the evaluator input. It
catches missing audit artifacts, inconsistent counts, scenario arithmetic drift,
forced final-three labels, and prose that asks the user to continue an unfinished
run. Exit 0 means the run can be bundled and shown as final; exit 2 means the
same execution must repair the diagnostics and rerun.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import sys
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Any
try:
from skill_version import runtime_metadata
except ModuleNotFoundError:
import importlib.util
_path = Path(__file__).with_name("skill_version.py")
_spec = importlib.util.spec_from_file_location("skill_version", _path)
if _spec is None or _spec.loader is None:
raise
_mod = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(_mod)
runtime_metadata = _mod.runtime_metadata
class AuditError(ValueError):
pass
def _mapping(value: Any) -> dict[str, Any]:
return dict(value) if isinstance(value, Mapping) else {}
def _list(value: Any) -> list[Any]:
return list(value) if isinstance(value, list) else []
def _number(value: Any) -> float | None:
if isinstance(value, bool):
return None
if isinstance(value, (int, float)) and math.isfinite(float(value)):
return float(value)
return None
def _text(value: Any) -> str | None:
return value.strip() if isinstance(value, str) and value.strip() else None
def _read_json(path: Path) -> dict[str, Any]:
try:
value = json.loads(path.read_text(encoding="utf-8"))
except OSError as exc:
raise AuditError(f"cannot read {path}: {exc}") from exc
except json.JSONDecodeError as exc:
raise AuditError(f"invalid JSON in {path}: {exc}") from exc
if not isinstance(value, Mapping):
raise AuditError(f"{path} must contain a JSON object")
return dict(value)
def _safe_path(root: Path, raw: str) -> Path:
candidate = (root / raw).resolve()
root_resolved = root.resolve()
try:
candidate.relative_to(root_resolved)
except ValueError as exc:
raise AuditError(f"artifact path escapes root: {raw}") from exc
return candidate
def _verify_artifact(section: Mapping[str, Any], root: Path, label: str, errors: list[str]) -> None:
path_text = _text(section.get("artifact_path"))
expected = _text(section.get("artifact_sha256"))
if not path_text:
errors.append(f"{label}.artifact_path is missing")
return
path = _safe_path(root, path_text)
if not path.is_file():
# Discovery artifacts written by run_pipeline are referenced relative
# to the audit directory; accept that base as well so a run root and
# its audit/ subdirectory both resolve.
fallback = (
_safe_path(root, f"audit/{path_text}") if not path_text.startswith("audit/") else None
)
if fallback is not None and fallback.is_file():
path = fallback
else:
errors.append(f"{label} artifact is missing: {path_text}")
return
actual = hashlib.sha256(path.read_bytes()).hexdigest()
if expected and actual != expected:
errors.append(f"{label} SHA-256 mismatch")
def _scenario_arithmetic(report: Mapping[str, Any], errors: list[str]) -> None:
for candidate in _list(report.get("ranked_candidates")):
row = _mapping(candidate)
symbol = _text(row.get("symbol")) or "UNKNOWN"
valuation = _mapping(row.get("valuation"))
for scenario_name in ("constant_multiple", "multiple_contraction", "peer_median"):
scenario = _mapping(valuation.get(scenario_name))
for horizon in ("year_2", "year_3"):
item = _mapping(scenario.get(horizon))
metric = _number(item.get("metric"))
multiple = _number(item.get("multiple"))
price = _number(item.get("implied_price"))
if None in {metric, multiple, price}:
continue
expected = metric * multiple
if abs(price - expected) > max(0.02, abs(expected) * 0.0005):
errors.append(
f"{symbol} {scenario_name}.{horizon} implied price does not equal metric × multiple"
)
def audit_publication(
report: Mapping[str, Any],
*,
report_markdown: str,
artifact_root: Path,
) -> dict[str, Any]:
errors: list[str] = []
warnings: list[str] = []
expected_runtime = runtime_metadata()
runtime = _mapping(report.get("runtime"))
for key, value in expected_runtime.items():
if runtime.get(key) != value:
errors.append(f"runtime.{key} does not match installed skill")
contract = _mapping(report.get("contract"))
if contract.get("valid") is not True:
errors.append("contract.valid is not true")
if report.get("ranking_status") != "final":
errors.append("ranking_status is not final")
run_metadata = _mapping(report.get("run_metadata"))
if run_metadata.get("status") != "complete":
errors.append("run_metadata.status is not complete")
if _list(run_metadata.get("unprocessed_candidates")):
errors.append("unprocessed_candidates is not empty")
audit = _mapping(report.get("screening_audit"))
_verify_artifact(_mapping(audit.get("universe")), artifact_root, "universe", errors)
_verify_artifact(_mapping(audit.get("candidate_pool")), artifact_root, "candidate_pool", errors)
enrichment = _mapping(audit.get("enrichment"))
if _text(enrichment.get("artifact_path")):
_verify_artifact(enrichment, artifact_root, "enrichment", errors)
generation = _mapping(_mapping(audit.get("candidate_pool")).get("generation_audit"))
if generation and _text(generation.get("artifact_path")):
_verify_artifact(generation, artifact_root, "candidate_pool_generation", errors)
if enrichment.get("queue_count") not in {0, None}:
errors.append("enrichment queue is not empty")
if enrichment.get("unresolved_count") not in {0, None}:
errors.append("enrichment unresolved_count is not zero")
if enrichment.get("all_rows_resolved") is not True:
errors.append("enrichment.all_rows_resolved is not true")
counts = _mapping(report.get("counts"))
status_counts = {
"eligible": len(_list(report.get("ranked_candidates"))),
"conditional": len(_list(report.get("conditional"))),
"review_required": len(_list(report.get("review_required"))),
"screened_out": len(_list(report.get("screened_out"))),
"excluded": len(_list(report.get("excluded"))),
}
for key, actual in status_counts.items():
declared = counts.get("ranked" if key == "eligible" else key)
if declared is not None and int(declared) != actual:
errors.append(f"counts.{key} does not match published rows")
for row in _list(report.get("ranked_candidates")):
candidate = _mapping(row)
if candidate.get("status") != "eligible":
errors.append(f"ranked candidate {candidate.get('symbol')} is not eligible")
if _list(candidate.get("quality_gate_failures")):
errors.append(f"ranked candidate {candidate.get('symbol')} has quality-gate failures")
ranked_symbols = {
str(_mapping(row).get("symbol") or "").upper()
for row in _list(report.get("ranked_candidates"))
}
for label, raw in _mapping(report.get("final_three")).items():
item = _mapping(raw)
symbol = str(item.get("symbol") or "").upper()
if symbol and symbol not in ranked_symbols:
errors.append(f"final_three.{label} references a non-ranked symbol")
forbidden_phrases = (
"say continue",
"send continue",
"tell me continue",
"続きと送",
"続けてと送",
"次のターンで",
"正式ランキングではなく暫定",
"temporary partial result",
)
lower_md = report_markdown.lower()
for phrase in forbidden_phrases:
if phrase.lower() in lower_md:
errors.append(f"published markdown contains unfinished-run phrase: {phrase!r}")
_scenario_arithmetic(report, errors)
return {
"runtime": expected_runtime,
"valid": not errors,
"errors": list(dict.fromkeys(errors)),
"warnings": list(dict.fromkeys(warnings)),
}
def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--report-json", type=Path, required=True)
parser.add_argument("--report-md", type=Path, required=True)
parser.add_argument("--artifact-root", type=Path, required=True)
parser.add_argument("--output", type=Path)
parser.add_argument("--version", action="store_true")
return parser.parse_args(argv)
def main(argv: Sequence[str] | None = None) -> int:
raw_argv = list(sys.argv[1:] if argv is None else argv)
if "--version" in raw_argv:
print(json.dumps(runtime_metadata(), sort_keys=True))
return 0
args = parse_args(raw_argv)
if args.version:
print(json.dumps(runtime_metadata(), sort_keys=True))
return 0
try:
report = _read_json(args.report_json)
markdown = args.report_md.read_text(encoding="utf-8")
result = audit_publication(
report, report_markdown=markdown, artifact_root=args.artifact_root
)
except (AuditError, OSError) as exc:
print(f"ERROR: {exc}")
return 2
payload = json.dumps(result, ensure_ascii=False, indent=2) + "\n"
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(payload, encoding="utf-8")
print(payload, end="")
return 0 if result["valid"] else 2
if __name__ == "__main__":
raise SystemExit(main())
scripts/research_contract.py
#!/usr/bin/env python3
"""Schema-v3 / contract-v3.5 research validation for the US GARP screener.
The module performs no network access. It validates source-ledger types,
market context, broad-screen audit artifacts, latest-earnings period separation,
forward-multiple inputs, estimate breadth, cash-definition consistency, and
run completeness before deterministic scoring.
"""
from __future__ import annotations
import hashlib
import json
import math
import re
from collections.abc import Mapping, Sequence
from copy import deepcopy
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from urllib.parse import urlparse
try:
from skill_version import (
CONTRACT_REVISION,
RUNTIME_FINGERPRINT,
SCHEMA_VERSION,
SKILL_VERSION,
runtime_metadata,
)
except ModuleNotFoundError: # Supports importlib-based unit loading.
import importlib.util as _importlib_util
_version_path = Path(__file__).with_name("skill_version.py")
_version_spec = _importlib_util.spec_from_file_location("skill_version", _version_path)
if _version_spec is None or _version_spec.loader is None:
raise
_version_module = _importlib_util.module_from_spec(_version_spec)
_version_spec.loader.exec_module(_version_module)
CONTRACT_REVISION = _version_module.CONTRACT_REVISION
RUNTIME_FINGERPRINT = _version_module.RUNTIME_FINGERPRINT
SCHEMA_VERSION = _version_module.SCHEMA_VERSION
SKILL_VERSION = _version_module.SKILL_VERSION
runtime_metadata = _version_module.runtime_metadata
try:
from screening_semantics import normalize_liquidity
except ModuleNotFoundError:
import importlib.util as _semantics_importlib_util
_semantics_path = Path(__file__).with_name("screening_semantics.py")
_semantics_spec = _semantics_importlib_util.spec_from_file_location(
"screening_semantics_contract", _semantics_path
)
if _semantics_spec is None or _semantics_spec.loader is None:
raise
_semantics_module = _semantics_importlib_util.module_from_spec(_semantics_spec)
_semantics_spec.loader.exec_module(_semantics_module)
normalize_liquidity = _semantics_module.normalize_liquidity
ALLOWED_SOURCE_KINDS = {
"sec_filing",
"company_ir",
"exchange_status",
"official_macro",
"official_statistics",
"market_data",
"consensus",
"news",
"third_party_transcript",
"analyst_calculation",
"analyst_assumption",
"local_artifact",
}
SOURCE_KIND_TIERS: dict[str, set[int]] = {
"sec_filing": {1},
"official_macro": {1, 3},
"official_statistics": {1, 3},
"company_ir": {2},
"exchange_status": {2, 3},
"market_data": {3},
"consensus": {3},
"news": {3},
"third_party_transcript": {3},
"analyst_calculation": {4},
"analyst_assumption": {4},
"local_artifact": {4},
}
OFFICIAL_SOURCE_DOMAINS = {
"federalreserve.gov",
"treasury.gov",
"bls.gov",
"bea.gov",
"census.gov",
"sec.gov",
"cftc.gov",
"fdic.gov",
"occ.gov",
"irs.gov",
"dol.gov",
}
THIRD_PARTY_TRANSCRIPT_DOMAINS = {
"fool.com",
"seekingalpha.com",
"stocktitan.net",
"marketscreener.com",
"investing.com",
"finance.yahoo.com",
}
def _hostname(url: str | None) -> str:
if not url:
return ""
try:
host = (urlparse(url).hostname or "").lower()
except ValueError:
return ""
return host[4:] if host.startswith("www.") else host
def _domain_matches(host: str, domains: set[str]) -> bool:
return any(host == domain or host.endswith("." + domain) for domain in domains)
def _is_accession_specific_sec_url(url: str | None) -> bool:
if not url:
return False
try:
parsed = urlparse(url)
except ValueError:
return False
if not _domain_matches((parsed.hostname or "").lower().removeprefix("www."), {"sec.gov"}):
return False
path = parsed.path.lower()
query = parsed.query.lower()
return "/archives/edgar/data/" in path or "doc=/archives/edgar/data/" in query
ALLOWED_AUDIT_DECISIONS = {
"passed",
"passed_exception",
"selected",
"deferred_by_budget",
"needs_enrichment",
"sector_review_required",
"near_miss_review",
"review_required",
"unavailable_after_enrichment",
"screened_out",
"excluded",
}
ALLOWED_UNIVERSE_AUDIT_DECISIONS = {
"in_scope",
"liquidity_review",
"out_of_scope",
"excluded",
"listing_data_incomplete",
}
ALLOWED_CANDIDATE_GENERATION_MODES = {
"full_universe_fundamentals",
"provider_prefilter",
"liquidity_stratified_estimates",
"available_fundamentals",
"user_supplied",
}
ALLOWED_REPORT_TYPES = {"quarterly", "annual"}
ALLOWED_EARNINGS_PERIOD_TYPES = {"quarter", "full_year"}
ALLOWED_FORWARD_PERIOD_KINDS = {"ntm", "fy1"}
ALLOWED_FUTURE_PERIOD_KINDS = {"fy2", "fy3", "other"}
PLACEHOLDER_RE = re.compile(
r"replace this|placeholder|synthetic text|todo|tbd|fill in|example only|ここに|要入力",
re.IGNORECASE,
)
class ContractError(ValueError):
"""Raised when the snapshot cannot be normalized."""
def _mapping(value: Any) -> dict[str, Any]:
return dict(value) if isinstance(value, Mapping) else {}
def _list(value: Any) -> list[Any]:
return list(value) if isinstance(value, list) else []
def _text(value: Any) -> str | None:
if isinstance(value, str):
value = value.strip()
return value or None
return None
def _number(value: Any) -> float | None:
if isinstance(value, bool):
return None
if isinstance(value, (int, float)):
result = float(value)
if math.isfinite(result):
return result
return None
def _integer(value: Any) -> int | None:
value = _number(value)
if value is not None and value.is_integer():
return int(value)
return None
def parse_iso8601(value: Any, field: str, *, required: bool = False) -> datetime | None:
text = _text(value)
if text is None:
if required:
raise ContractError(f"{field} is required")
return None
normalized = text[:-1] + "+00:00" if text.endswith("Z") else text
try:
parsed = datetime.fromisoformat(normalized)
except ValueError as exc:
raise ContractError(f"{field} must be ISO-8601: {text!r}") from exc
if parsed.tzinfo is None:
raise ContractError(f"{field} must include a timezone: {text!r}")
return parsed
def _parse_date(value: Any, field: str) -> datetime | None:
text = _text(value)
if not text:
return None
try:
parsed = datetime.fromisoformat(text)
except ValueError as exc:
raise ContractError(f"{field} must be YYYY-MM-DD: {text!r}") from exc
return parsed.replace(tzinfo=timezone.utc)
def _pct_diff(a: float | None, b: float | None) -> float | None:
if a is None or b is None:
return None
base = max(abs(a), abs(b), 1e-12)
return abs(a - b) / base * 100.0
def _is_string_list(value: Any, *, nonempty: bool = False) -> bool:
return (
isinstance(value, list)
and (not nonempty or bool(value))
and all(isinstance(item, str) and item.strip() for item in value)
)
def _is_placeholder(value: Any) -> bool:
text = _text(value)
return text is None or bool(PLACEHOLDER_RE.search(text))
def _canonical_jsonl(rows: Sequence[Mapping[str, Any]]) -> bytes:
return b"".join(
(
json.dumps(dict(row), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n"
).encode("utf-8")
for row in rows
)
def _safe_artifact_path(root: Path, raw_path: str) -> Path:
candidate = (root / raw_path).resolve()
root_resolved = root.resolve()
try:
candidate.relative_to(root_resolved)
except ValueError as exc:
raise ContractError("screening_audit.artifact_path escapes artifact_root") from exc
return candidate
def validate_source_ledger(
global_sources: Sequence[Any],
candidate_sources: Sequence[Any],
analysis_as_of: datetime,
) -> tuple[dict[str, dict[str, Any]], list[dict[str, Any]], list[str], list[str]]:
warnings: list[str] = []
review: list[str] = []
index: dict[str, dict[str, Any]] = {}
normalized: list[dict[str, Any]] = []
for position, raw in enumerate(list(global_sources) + list(candidate_sources)):
source = _mapping(raw)
source_id = _text(source.get("id"))
loc = f"sources[{position}]"
if not source_id:
review.append(f"{loc}.id is required")
continue
tier = _integer(source.get("tier"))
kind = (_text(source.get("kind")) or "").lower()
supports = source.get("supports")
retrieved_at = None
published_at = None
data_as_of = None
if tier not in {1, 2, 3, 4}:
review.append(f"source {source_id!r} has invalid tier")
if kind not in ALLOWED_SOURCE_KINDS:
review.append(f"source {source_id!r} has invalid kind {kind!r}")
elif tier not in SOURCE_KIND_TIERS[kind]:
review.append(f"source {source_id!r} kind {kind!r} is incompatible with tier {tier!r}")
if not _text(source.get("title")):
review.append(f"source {source_id!r} has no title")
source_url = _text(source.get("url"))
if not source_url:
review.append(f"source {source_id!r} has no URL")
host = _hostname(source_url)
if kind in {"official_macro", "official_statistics"} and not _domain_matches(
host, OFFICIAL_SOURCE_DOMAINS
):
review.append(
f"source {source_id!r} is classified as official but URL domain {host!r} is not an approved official domain"
)
if kind == "sec_filing":
if not _domain_matches(host, {"sec.gov"}):
review.append(
f"source {source_id!r} is classified as sec_filing but is not on sec.gov"
)
elif not _is_accession_specific_sec_url(source_url):
review.append(
f"source {source_id!r} must use an accession-specific SEC Archives URL, not a changing company browse page"
)
if kind == "company_ir" and _domain_matches(host, THIRD_PARTY_TRANSCRIPT_DOMAINS):
review.append(
f"source {source_id!r} is a third-party page and cannot be classified as company_ir"
)
if kind == "third_party_transcript" and not host:
review.append(f"source {source_id!r} transcript URL is invalid")
if not _is_string_list(supports, nonempty=True):
review.append(f"source {source_id!r}.supports must be a non-empty array of strings")
try:
retrieved_at = parse_iso8601(
source.get("retrieved_at"), f"source {source_id}.retrieved_at", required=True
)
except ContractError as exc:
review.append(str(exc))
try:
published_at = parse_iso8601(
source.get("published_at"), f"source {source_id}.published_at"
)
except ContractError as exc:
review.append(str(exc))
try:
data_as_of = parse_iso8601(source.get("data_as_of"), f"source {source_id}.data_as_of")
except ContractError as exc:
review.append(str(exc))
if retrieved_at and retrieved_at > analysis_as_of:
review.append(f"source {source_id!r} was retrieved after analysis_as_of")
if published_at and published_at > analysis_as_of:
review.append(f"source {source_id!r} was published after analysis_as_of")
if data_as_of and data_as_of > analysis_as_of:
review.append(f"source {source_id!r}.data_as_of is after analysis_as_of")
normalized_source = deepcopy(source)
normalized_source["id"] = source_id
normalized_source["kind"] = kind
normalized_source["tier"] = tier
if source_id in index:
if index[source_id] != normalized_source:
review.append(f"duplicate source ID {source_id!r} has conflicting definitions")
else:
warnings.append(f"duplicate source ID {source_id!r} was de-duplicated")
continue
index[source_id] = normalized_source
normalized.append(normalized_source)
return index, normalized, list(dict.fromkeys(warnings)), list(dict.fromkeys(review))
def _sources_resolve(source_ids: Any, source_index: Mapping[str, Mapping[str, Any]]) -> bool:
if not _is_string_list(source_ids, nonempty=True):
return False
return all(source_id in source_index for source_id in source_ids)
def _has_primary(source_ids: Any, source_index: Mapping[str, Mapping[str, Any]]) -> bool:
if not _sources_resolve(source_ids, source_index):
return False
return any(_integer(source_index[source_id].get("tier")) in {1, 2} for source_id in source_ids)
def validate_market_context(
context_raw: Any,
source_index: Mapping[str, Mapping[str, Any]],
analysis_as_of: datetime,
config: Mapping[str, Any],
) -> tuple[dict[str, Any], bool, list[str], list[str]]:
"""Validate current macro/valuation context against field-level evidence.
Retrieval time does not make an old valuation datum current. For each
dynamic field, use ``data_as_of`` when supplied, otherwise ``published_at``.
Official macro fields must be supported by an official source. Statements
about market-implied rate paths require an explicit supported inference.
"""
context = _mapping(context_raw)
warnings: list[str] = []
review: list[str] = []
summary = _text(context.get("summary"))
if _is_placeholder(summary) or (summary and len(summary) < 40):
review.append(
"market_context.summary is missing, too short, or still contains placeholder text"
)
as_of = None
try:
as_of = parse_iso8601(context.get("as_of"), "market_context.as_of", required=True)
except ContractError as exc:
review.append(str(exc))
if as_of:
if as_of > analysis_as_of:
review.append("market_context.as_of is after analysis_as_of")
max_age = _number(config.get("max_market_context_age_days")) or 14.0
age_days = (analysis_as_of - as_of).total_seconds() / 86_400
if age_days > max_age:
review.append(
f"market context is stale ({age_days:.1f} days old; maximum {max_age:.0f})"
)
required_numeric = (
"policy_rate_pct",
"treasury_10y_yield_pct",
"inflation_yoy_pct",
"market_forward_pe",
)
for key in required_numeric:
if _number(context.get(key)) is None:
review.append(f"market_context.{key} is required")
if _number(context.get("real_gdp_growth_pct")) is None:
warnings.append("market_context.real_gdp_growth_pct is not verified")
if not _text(context.get("small_mid_cap_valuation_context")):
review.append("market_context.small_mid_cap_valuation_context is required")
if not isinstance(context.get("sector_cycle_notes"), list):
review.append("market_context.sector_cycle_notes must be an array")
source_ids = context.get("source_ids")
if not _sources_resolve(source_ids, source_index) or len(source_ids) < 2:
review.append("market_context.source_ids must resolve to at least two sources")
source_ids = []
def supporting_sources(path: str) -> list[Mapping[str, Any]]:
rows: list[Mapping[str, Any]] = []
for source_id in source_ids if isinstance(source_ids, list) else []:
source = source_index.get(source_id)
if not source:
continue
supports = source.get("supports")
if isinstance(supports, list) and any(
isinstance(value, str) and (value == path or value.startswith(path + "."))
for value in supports
):
rows.append(source)
return rows
def effective_date(source: Mapping[str, Any]) -> datetime | None:
for key in ("data_as_of", "published_at"):
try:
value = parse_iso8601(source.get(key), f"source {source.get('id')}.{key}")
except ContractError:
value = None
if value is not None:
return value
return None
field_rules = {
"policy_rate_pct": {
"path": "market_context.policy_rate_pct",
"max_age": _number(config.get("max_policy_rate_source_age_days")) or 60.0,
"official": True,
},
"treasury_10y_yield_pct": {
"path": "market_context.treasury_10y_yield_pct",
"max_age": _number(config.get("max_market_rate_source_age_days")) or 7.0,
"official": True,
},
"inflation_yoy_pct": {
"path": "market_context.inflation_yoy_pct",
"max_age": _number(config.get("max_inflation_source_age_days")) or 45.0,
"official": True,
},
"real_gdp_growth_pct": {
"path": "market_context.real_gdp_growth_pct",
"max_age": _number(config.get("max_gdp_source_age_days")) or 120.0,
"official": True,
},
"market_forward_pe": {
"path": "market_context.market_forward_pe",
"max_age": _number(config.get("max_market_valuation_source_age_days"))
or _number(config.get("max_market_context_age_days"))
or 14.0,
"official": False,
},
"small_mid_cap_valuation_context": {
"path": "market_context.small_mid_cap_valuation_context",
"max_age": _number(config.get("max_small_mid_context_source_age_days"))
or _number(config.get("max_market_context_age_days"))
or 14.0,
"official": False,
},
}
official_kinds = {"official_macro", "official_statistics"}
for field, rule in field_rules.items():
if field == "real_gdp_growth_pct" and _number(context.get(field)) is None:
continue
if field == "small_mid_cap_valuation_context" and not _text(context.get(field)):
continue
rows = supporting_sources(str(rule["path"]))
if not rows:
review.append(f"{rule['path']} has no field-level supporting source")
continue
if rule["official"] and not any(
(_text(row.get("kind")) or "").lower() in official_kinds for row in rows
):
review.append(
f"{rule['path']} must be supported by an official macro/statistics source"
)
fresh = False
ages: list[float] = []
for row in rows:
stamp = effective_date(row)
if stamp is None:
continue
age_days = (analysis_as_of - stamp).total_seconds() / 86_400
ages.append(age_days)
if age_days <= float(rule["max_age"]):
fresh = True
if not fresh:
oldest_note = (
f"; observed ages={','.join(f'{age:.1f}' for age in ages)}" if ages else ""
)
review.append(
f"{rule['path']} lacks a source within {float(rule['max_age']):.0f} days{oldest_note}"
)
# Market-implied policy-path language must be traceable, not silently inferred.
summary_lower = (summary or "").lower()
expectation_terms = (
"markets price",
"market prices",
"priced in",
"rate hike",
"rate cut",
"市場は",
"織り込",
"利上げ",
"利下げ",
)
if any(term in summary_lower for term in expectation_terms):
expectation_sources = supporting_sources("market_context.market_rate_expectations")
inference_rows = _list(context.get("inferences"))
inference_supported = any(
(_text(_mapping(row).get("classification")) or "").lower() == "analyst_inference"
and _sources_resolve(_mapping(row).get("source_ids"), source_index)
for row in inference_rows
)
if not expectation_sources and not inference_supported:
review.append(
"market_context.summary contains a market-rate expectation without a supporting source or analyst_inference record"
)
normalized = deepcopy(context)
normalized["valid"] = not review
return normalized, not review, list(dict.fromkeys(warnings)), list(dict.fromkeys(review))
def _load_jsonl_artifact(
section: Mapping[str, Any],
artifact_root: Path | None,
*,
label: str,
) -> tuple[list[dict[str, Any]], str | None, list[str]]:
review: list[str] = []
rows_raw = section.get("embedded_rows")
if isinstance(rows_raw, list):
rows = [_mapping(row) for row in rows_raw]
return rows, hashlib.sha256(_canonical_jsonl(rows)).hexdigest(), review
artifact_path = _text(section.get("artifact_path"))
if not artifact_path:
return [], None, [f"{label}.artifact_path or embedded_rows is required"]
if artifact_root is None:
return [], None, [f"artifact_root is required to verify {label}.artifact_path"]
try:
path = _safe_artifact_path(artifact_root, artifact_path)
except ContractError as exc:
return [], None, [str(exc)]
if not path.is_file():
return [], None, [f"{label} artifact does not exist: {path}"]
raw = path.read_bytes()
digest = hashlib.sha256(raw).hexdigest()
rows: list[dict[str, Any]] = []
for line_number, line in enumerate(raw.decode("utf-8").splitlines(), start=1):
if not line.strip():
continue
try:
value = json.loads(line)
except json.JSONDecodeError:
review.append(f"{label} line {line_number} is not valid JSON")
continue
if not isinstance(value, Mapping):
review.append(f"{label} line {line_number} is not an object")
continue
rows.append(dict(value))
return rows, digest, review
def validate_screening_audit(
audit_raw: Any,
funnel_raw: Any,
source_index: Mapping[str, Mapping[str, Any]],
analysis_as_of: datetime,
artifact_root: Path | None,
config: Mapping[str, Any],
) -> tuple[dict[str, Any], bool, list[str], list[str]]:
"""Validate the v3.5 staged listing/candidate-pool audit.
Completion is resolution-based, never attempt-count based. A no-candidates
conclusion is valid only when the bounded candidate pool is explicitly
exhausted, every row has a final broad-screen disposition, and no
enrichment queue remains.
"""
audit = _mapping(audit_raw)
funnel = _mapping(funnel_raw)
warnings: list[str] = []
review: list[str] = []
if _integer(audit.get("audit_schema_version")) != 3:
normalized = deepcopy(audit)
normalized["valid"] = False
return (
normalized,
False,
warnings,
[
f"screening_audit.audit_schema_version must equal 3 for contract revision {CONTRACT_REVISION}"
],
)
if (_text(audit.get("contract_revision")) or "") != CONTRACT_REVISION:
review.append(f"screening_audit.contract_revision must equal {CONTRACT_REVISION!r}")
runtime = _mapping(audit.get("runtime"))
expected_runtime = runtime_metadata()
for key in (
"skill_name",
"skill_version",
"schema_version",
"contract_revision",
"runtime_fingerprint",
):
if runtime.get(key) != expected_runtime.get(key):
review.append(
f"screening_audit.runtime.{key} does not match the installed v{SKILL_VERSION} runtime"
)
try:
parse_iso8601(audit.get("generated_at"), "screening_audit.generated_at", required=True)
except ContractError as exc:
review.append(str(exc))
try:
audit_as_of = parse_iso8601(
audit.get("analysis_as_of"), "screening_audit.analysis_as_of", required=True
)
if audit_as_of and abs((audit_as_of - analysis_as_of).total_seconds()) > 1:
review.append("screening_audit.analysis_as_of does not match top-level analysis_as_of")
except ContractError as exc:
review.append(str(exc))
mode = (_text(audit.get("candidate_generation_mode")) or "").lower()
if mode not in ALLOWED_CANDIDATE_GENERATION_MODES:
review.append(f"screening_audit.candidate_generation_mode is invalid: {mode!r}")
if not _mapping(audit.get("filters")):
review.append("screening_audit.filters is required")
if not _sources_resolve(audit.get("source_ids"), source_index):
review.append("screening_audit.source_ids do not resolve")
scope = _mapping(audit.get("scope"))
requested_min = _number(scope.get("requested_min_market_cap"))
requested_max = _number(scope.get("requested_max_market_cap"))
retrieval_min = _number(scope.get("retrieval_min_market_cap"))
retrieval_max = _number(scope.get("retrieval_max_market_cap"))
bounds_valid = None not in {requested_min, requested_max, retrieval_min, retrieval_max}
retrieval_covers_requested = False
reduced = False
if not bounds_valid:
review.append("screening_audit.scope requires requested and retrieval market-cap bounds")
else:
assert requested_min is not None and requested_max is not None
assert retrieval_min is not None and retrieval_max is not None
if requested_min >= requested_max or retrieval_min >= retrieval_max:
review.append("screening_audit.scope market-cap bounds are invalid")
retrieval_covers_requested = (
retrieval_min <= requested_min and retrieval_max >= requested_max
)
reduced = not retrieval_covers_requested
if reduced and scope.get("scope_override_authorized") is not True:
review.append("reduced execution scope was not explicitly authorized by the user")
if reduced and (
not _text(scope.get("scope_reduction_reason"))
or not _text(scope.get("user_scope_evidence"))
):
review.append(
"authorized reduced scope requires scope_reduction_reason and user_scope_evidence"
)
if scope.get("retrieval_scope_explicit") is not True:
review.append("screening_audit.scope.retrieval_scope_explicit must be true")
user_scope = _mapping(scope.get("user_requested_scope"))
executed_scope = _mapping(scope.get("executed_scope"))
if (
_number(user_scope.get("min_market_cap")) != requested_min
or _number(user_scope.get("max_market_cap")) != requested_max
):
review.append("screening_audit.scope.user_requested_scope does not match requested bounds")
if (
_number(executed_scope.get("min_market_cap")) != retrieval_min
or _number(executed_scope.get("max_market_cap")) != retrieval_max
):
review.append("screening_audit.scope.executed_scope does not match retrieval bounds")
enumeration = _mapping(scope.get("enumeration"))
pages_fetched = _integer(enumeration.get("pages_fetched"))
band_rows = _list(enumeration.get("band_audit"))
band_ranges: list[tuple[float, float]] = []
bands_well_formed = bool(band_rows)
for position, raw_band in enumerate(band_rows):
band = _mapping(raw_band)
band_min = _number(band.get("min_market_cap"))
band_max = _number(band.get("max_market_cap"))
band_count = _integer(band.get("rows_fetched"))
if (
band_min is None
or band_max is None
or band_min >= band_max
or band_count is None
or band_count < 0
or band.get("provider_exhausted") is not True
):
bands_well_formed = False
review.append(
f"screening_audit.scope.enumeration.band_audit[{position}] is malformed or not exhausted"
)
continue
band_ranges.append((band_min, band_max))
band_ranges.sort()
bands_cover_retrieval = False
if (
bands_well_formed
and band_ranges
and retrieval_min is not None
and retrieval_max is not None
):
cursor = retrieval_min
bands_cover_retrieval = True
for band_min, band_max in band_ranges:
if band_max < retrieval_min or band_min > retrieval_max:
continue
clipped_min = max(band_min, retrieval_min)
clipped_max = min(band_max, retrieval_max)
if clipped_min > cursor:
bands_cover_retrieval = False
break
cursor = max(cursor, clipped_max)
bands_cover_retrieval = bands_cover_retrieval and cursor >= retrieval_max
bands_verified = bands_well_formed and bands_cover_retrieval
pagination_verified = enumeration.get("pagination_exhausted") is True or bands_verified
provider_total = _integer(enumeration.get("provider_reported_total"))
rows_fetched = _integer(enumeration.get("rows_fetched"))
provider_total_consistent = not (
rows_fetched is not None and provider_total is not None and rows_fetched < provider_total
)
full_enumeration_verified = bool(pagination_verified and provider_total_consistent)
executed_scope_complete = bool(
scope.get("retrieval_scope_explicit") is True and full_enumeration_verified
)
user_requested_scope_complete = bool(retrieval_covers_requested and executed_scope_complete)
if enumeration.get("verified") is not full_enumeration_verified:
review.append(
"screening_audit.scope.enumeration.verified does not match pagination/band evidence"
)
if (
enumeration.get("bands_well_formed") is not None
and enumeration.get("bands_well_formed") is not bands_well_formed
):
review.append(
"screening_audit.scope.enumeration.bands_well_formed does not match band evidence"
)
declared_cover = enumeration.get("bands_cover_executed_range")
if declared_cover is None:
declared_cover = enumeration.get("bands_cover_requested_range")
if declared_cover is not None and declared_cover is not bands_cover_retrieval:
review.append(
"screening_audit.scope.enumeration band-coverage flag does not match evidence"
)
if (
enumeration.get("bands_verified") is not None
and enumeration.get("bands_verified") is not bands_verified
):
review.append(
"screening_audit.scope.enumeration.bands_verified does not match band evidence"
)
if pages_fetched is not None and pages_fetched <= 0:
review.append("screening_audit.scope.enumeration.pages_fetched must be positive")
if not provider_total_consistent:
review.append("universe rows_fetched is below provider_reported_total")
if scope.get("executed_scope_complete") is not executed_scope_complete:
review.append(
"screening_audit.scope.executed_scope_complete does not match enumeration evidence"
)
if scope.get("user_requested_scope_complete") is not user_requested_scope_complete:
review.append("screening_audit.scope.user_requested_scope_complete does not match evidence")
if scope.get("scope_complete") is not user_requested_scope_complete:
review.append(
"screening_audit.scope.scope_complete must represent full user-requested enumeration"
)
universe_section = _mapping(audit.get("universe"))
candidate_section = _mapping(audit.get("candidate_pool"))
enrichment = _mapping(audit.get("enrichment"))
if not _sources_resolve(universe_section.get("source_ids"), source_index):
review.append("screening_audit.universe.source_ids do not resolve")
if (_integer(candidate_section.get("row_count")) or 0) > 0 and not _sources_resolve(
candidate_section.get("source_ids"), source_index
):
review.append("screening_audit.candidate_pool.source_ids do not resolve")
universe_rows, universe_digest, universe_review = _load_jsonl_artifact(
universe_section, artifact_root, label="screening_audit.universe"
)
candidate_rows, candidate_digest, candidate_review = _load_jsonl_artifact(
candidate_section, artifact_root, label="screening_audit.candidate_pool"
)
review.extend(universe_review)
review.extend(candidate_review)
if (
universe_digest
and (_text(universe_section.get("artifact_sha256")) or "").lower() != universe_digest
):
review.append("screening_audit.universe.artifact_sha256 does not match")
if (
candidate_digest
and (_text(candidate_section.get("artifact_sha256")) or "").lower() != candidate_digest
):
review.append("screening_audit.candidate_pool.artifact_sha256 does not match")
universe_symbols: set[str] = set()
universe_counts: dict[str, int] = {}
listing_complete = 0
for position, row in enumerate(universe_rows):
symbol = (_text(row.get("symbol")) or "").upper()
status = (_text(_mapping(row.get("listing_decision")).get("status")) or "").lower()
if not symbol:
review.append(f"universe audit row {position} has no symbol")
continue
if symbol in universe_symbols:
review.append(f"universe audit contains duplicate symbol {symbol}")
universe_symbols.add(symbol)
if status not in ALLOWED_UNIVERSE_AUDIT_DECISIONS:
review.append(f"universe audit row {symbol} has invalid status {status!r}")
continue
universe_counts[status] = universe_counts.get(status, 0) + 1
listing_complete += int(row.get("listing_data_complete") is True)
in_scope_universe_symbols = {
(_text(row.get("symbol")) or "").upper()
for row in universe_rows
if (_text(_mapping(row.get("listing_decision")).get("status")) or "").lower()
in {"in_scope", "liquidity_review"}
}
in_scope_universe_symbols.discard("")
candidate_symbols: set[str] = set()
candidate_counts: dict[str, int] = {}
selected_symbols: list[str] = []
discovery_evaluable = 0
fundamental_complete = 0
enrichment_attempted = 0
enrichment_resolved = 0
actual_queue_symbols: list[str] = []
for position, row in enumerate(candidate_rows):
symbol = (_text(row.get("symbol")) or "").upper()
decision = _mapping(row.get("decision"))
status = (_text(decision.get("status")) or "").lower()
if not symbol:
review.append(f"candidate-pool row {position} has no symbol")
continue
if symbol in candidate_symbols:
review.append(f"candidate-pool audit contains duplicate symbol {symbol}")
candidate_symbols.add(symbol)
if symbol not in universe_symbols:
review.append(f"candidate-pool symbol {symbol} is not present in the listing universe")
if status not in ALLOWED_AUDIT_DECISIONS:
review.append(f"candidate-pool row {symbol} has invalid decision status {status!r}")
continue
candidate_counts[status] = candidate_counts.get(status, 0) + 1
discovery_evaluable += int(row.get("discovery_evaluable") is True)
fundamental_complete += int(row.get("fundamental_complete") is True)
enrichment_attempted += int(row.get("enrichment_attempted") is True)
resolved = (
row.get("enrichment_resolved") is True or decision.get("resolution") == "resolved"
)
enrichment_resolved += int(resolved)
if not resolved:
actual_queue_symbols.append(symbol)
if status == "unavailable_after_enrichment":
if decision.get("enrichment_exhausted") is not True:
review.append(f"unavailable candidate {symbol} lacks enrichment_exhausted=true")
if not _text(decision.get("enrichment_exhaustion_reason")):
review.append(
f"unavailable candidate {symbol} lacks an enrichment exhaustion reason"
)
source_ids = decision.get("enrichment_source_ids")
if not _sources_resolve(source_ids, source_index):
review.append(
f"unavailable candidate {symbol} lacks resolving enrichment source IDs"
)
if status == "selected":
selected_symbols.append(symbol)
if row.get("discovery_evaluable") is not True:
review.append(f"selected candidate {symbol} is not discovery-evaluable")
if _number(row.get("broad_score")) is None:
review.append(f"selected candidate {symbol} lacks a real broad score")
if _integer(universe_section.get("row_count")) != len(universe_rows):
review.append("screening_audit.universe.row_count does not match artifact")
if _integer(candidate_section.get("row_count")) != len(candidate_rows):
review.append("screening_audit.candidate_pool.row_count does not match artifact")
calculated_listing_pct = listing_complete / len(universe_rows) * 100.0 if universe_rows else 0.0
declared_listing_pct = _number(universe_section.get("listing_data_complete_pct"))
if declared_listing_pct is None or abs(declared_listing_pct - calculated_listing_pct) > 0.05:
review.append("screening_audit.universe.listing_data_complete_pct does not match artifact")
minimum_listing_pct = _number(config.get("minimum_listing_data_coverage_pct")) or 95.0
if calculated_listing_pct < minimum_listing_pct:
review.append(
f"listing-data coverage is {calculated_listing_pct:.1f}%, below the configured {minimum_listing_pct:.1f}%"
)
declared_selected = sorted(str(value).upper() for value in _list(audit.get("selected_symbols")))
if declared_selected != sorted(selected_symbols):
review.append(
"screening_audit.selected_symbols does not match selected candidate-pool rows"
)
deep_plan = _mapping(audit.get("deep_dive_plan"))
if sorted(str(value).upper() for value in _list(deep_plan.get("selected_symbols"))) != sorted(
selected_symbols
):
review.append(
"screening_audit.deep_dive_plan.selected_symbols does not match selected rows"
)
if _integer(deep_plan.get("selected_count")) != len(selected_symbols):
review.append("screening_audit.deep_dive_plan.selected_count does not match selected rows")
for key in (
"all_selected_must_be_resolved",
"budget_locked",
"budget_change_requires_rescreen",
):
if deep_plan.get(key) is not True:
review.append(f"screening_audit.deep_dive_plan.{key} must be true")
if deep_plan.get("user_confirmation_required") is not False:
review.append("screening_audit.deep_dive_plan.user_confirmation_required must be false")
if deep_plan.get("user_continue_instruction_allowed") is not False:
review.append(
"screening_audit.deep_dive_plan.user_continue_instruction_allowed must be false"
)
if deep_plan.get("selected_set_is_committed") is not True:
review.append("screening_audit.deep_dive_plan.selected_set_is_committed must be true")
max_deep_dives = _integer(deep_plan.get("max_deep_dive_candidates"))
if max_deep_dives != _integer(_mapping(audit.get("filters")).get("max_deep_dive_candidates")):
review.append("screening_audit.deep_dive_plan.max_deep_dive_candidates must match filters")
commitment_payload = _mapping(deep_plan.get("commitment_payload"))
expected_payload = {
"analysis_as_of": _text(audit.get("analysis_as_of")),
"max_deep_dive_candidates": max_deep_dives,
"selected_symbols": sorted(selected_symbols),
}
if commitment_payload != expected_payload:
review.append(
"screening_audit.deep_dive_plan.commitment_payload does not match selected set"
)
expected_commitment_sha = hashlib.sha256(
json.dumps(expected_payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
if (_text(deep_plan.get("selected_set_sha256")) or "").lower() != expected_commitment_sha:
review.append(
"screening_audit.deep_dive_plan.selected_set_sha256 does not match commitment payload"
)
declared_not_in_universe = sorted(
str(value).upper() for value in _list(candidate_section.get("symbols_not_in_universe"))
)
actual_not_in_universe = sorted(candidate_symbols - universe_symbols)
if declared_not_in_universe != actual_not_in_universe:
review.append(
"screening_audit.candidate_pool.symbols_not_in_universe does not match artifacts"
)
actual_missing_in_scope = sorted(in_scope_universe_symbols - candidate_symbols)
actual_listing_coverage_complete = not actual_missing_in_scope
if _integer(candidate_section.get("in_scope_covered_count")) != len(
in_scope_universe_symbols & candidate_symbols
):
review.append(
"screening_audit.candidate_pool.in_scope_covered_count does not match artifacts"
)
if _integer(candidate_section.get("in_scope_missing_count")) != len(actual_missing_in_scope):
review.append(
"screening_audit.candidate_pool.in_scope_missing_count does not match artifacts"
)
declared_missing_in_scope = sorted(
str(value).upper() for value in _list(candidate_section.get("in_scope_missing_symbols"))
)
if declared_missing_in_scope != actual_missing_in_scope:
review.append(
"screening_audit.candidate_pool.in_scope_missing_symbols does not match artifacts"
)
if candidate_section.get("listing_coverage_complete") is not actual_listing_coverage_complete:
review.append(
"screening_audit.candidate_pool.listing_coverage_complete does not match artifacts"
)
coverage_scope = (_text(candidate_section.get("coverage_scope")) or "").lower()
generation_audit = _mapping(candidate_section.get("generation_audit"))
generation_valid = generation_audit.get("valid") is True
bounded_sampling_ready = False
if mode == "liquidity_stratified_estimates":
expected_generation_runtime = runtime_metadata()
generation_runtime = _mapping(generation_audit.get("runtime"))
for key in (
"skill_name",
"skill_version",
"schema_version",
"contract_revision",
"runtime_fingerprint",
):
if generation_runtime.get(key) != expected_generation_runtime.get(key):
generation_valid = False
review.append(
f"candidate-pool generation runtime {key} does not match installed skill"
)
if (
_text(generation_audit.get("selection_method")) or ""
).lower() != "sector_market_cap_stratified_validated_liquidity":
generation_valid = False
review.append(
"liquidity-stratified candidate pool must use validated average-liquidity selection"
)
liquidity_validation = _mapping(generation_audit.get("liquidity_validation"))
if liquidity_validation.get("basis_validated") is not True:
generation_valid = False
review.append("candidate-pool generation lacks validated average-liquidity evidence")
minimum_window = _integer(liquidity_validation.get("minimum_window_days"))
if minimum_window is None or minimum_window < 20:
generation_valid = False
review.append(
"candidate-pool generation average-liquidity window must be at least 20 days"
)
generation_scope = _mapping(generation_audit.get("scope"))
if (
generation_scope.get("scope_valid") is not True
or generation_scope.get("scope_reduced") is True
):
generation_valid = False
review.append(
"candidate-pool generation may not narrow the user-requested market-cap scope"
)
if _number(generation_scope.get("user_requested_min_market_cap")) != requested_min:
generation_valid = False
review.append("candidate-pool generation requested lower bound does not match")
if _number(generation_scope.get("user_requested_max_market_cap")) != requested_max:
generation_valid = False
review.append("candidate-pool generation requested upper bound does not match")
coverage_plan = _mapping(generation_audit.get("coverage_plan"))
if coverage_plan.get("coverage_plan_valid") is not True:
generation_valid = False
review.append("candidate-pool generation coverage_plan_valid must be true")
if coverage_plan.get("user_requested_range_spanned") is not True:
generation_valid = False
review.append("candidate-pool generation does not span the user-requested range")
if coverage_plan.get("market_cap_buckets_cover_user_requested_range") is not True:
generation_valid = False
review.append("candidate-pool generation buckets do not cover the user-requested range")
if coverage_plan.get("single_band_only") is True:
generation_valid = False
review.append("single-band candidate pool cannot represent the requested range")
invalid_liquidity_symbols: list[str] = []
for candidate in candidate_rows:
# screen_universe serializes normalized discovery metrics under
# candidate["metrics"]. Raw discovery input rows may still carry
# the fields at top level, so accept either representation without
# accepting single-session volume.
liquidity_payload = dict(candidate)
liquidity_payload.update(_mapping(candidate.get("metrics")))
evidence = normalize_liquidity(liquidity_payload, minimum_period_days=20)
if evidence.get("valid_for_screen") is not True:
invalid_liquidity_symbols.append(
(_text(candidate.get("symbol")) or "UNKNOWN").upper()
)
if invalid_liquidity_symbols:
generation_valid = False
review.append(
"candidate-pool rows lack valid average-liquidity evidence: "
+ ", ".join(sorted(invalid_liquidity_symbols))
)
if mode == "full_universe_fundamentals":
actual_pool_scope_verified = (
actual_listing_coverage_complete and user_requested_scope_complete
)
expected_scope = "full_listing_universe"
else:
actual_pool_scope_verified = generation_valid
expected_scope = {
"liquidity_stratified_estimates": "stratified_discovery_pool",
"provider_prefilter": "provider_prefilter",
"available_fundamentals": "bounded_available_fundamentals",
"user_supplied": "user_supplied",
}.get(mode, "bounded_candidate_pool")
declared_generation_symbols = sorted(
str(value).upper() for value in _list(generation_audit.get("actual_selected_symbols"))
)
if declared_generation_symbols != sorted(candidate_symbols):
review.append("candidate-pool generation audit symbols do not match candidate artifact")
if _integer(generation_audit.get("actual_input_row_count")) != len(universe_rows):
review.append(
"candidate-pool generation audit input count does not match listing frame"
)
bounded_sampling_ready = bool(
actual_pool_scope_verified and retrieval_covers_requested and not reduced
)
actual_screening_scope_ready = bool(
user_requested_scope_complete
or bounded_sampling_ready
or (reduced and scope.get("scope_override_authorized") is True and executed_scope_complete)
)
if scope.get("bounded_sampling_ready") is not bounded_sampling_ready:
review.append(
"screening_audit.scope.bounded_sampling_ready does not match generation evidence"
)
if scope.get("screening_scope_ready") is not actual_screening_scope_ready:
review.append("screening_audit.scope.screening_scope_ready does not match evidence")
if not actual_screening_scope_ready:
review.append(
"screening scope is not ready: use full requested enumeration or an audited stratified pool spanning the requested range"
)
if coverage_scope != expected_scope:
review.append(f"screening_audit.candidate_pool.coverage_scope must be {expected_scope!r}")
if candidate_section.get("coverage_complete") is not actual_pool_scope_verified:
review.append(
"screening_audit.candidate_pool.coverage_complete does not match verified pool scope"
)
enrichment_status = (_text(enrichment.get("status")) or "").lower()
if enrichment_status not in {"pending", "complete", "not_required"}:
review.append("screening_audit.enrichment.status is invalid")
exhausted = enrichment.get("candidate_pool_exhausted") is True
declared_exhausted = enrichment.get("candidate_pool_exhaustion_declared") is True
all_rows_resolved = enrichment_resolved == len(candidate_rows) and not actual_queue_symbols
pool_ready = (
enrichment_status in {"complete", "not_required"}
and exhausted
and declared_exhausted
and all_rows_resolved
and actual_pool_scope_verified
and actual_screening_scope_ready
)
if enrichment.get("candidate_pool_covers_in_scope") is not actual_listing_coverage_complete:
review.append(
"screening_audit.enrichment.candidate_pool_covers_in_scope does not match listing artifacts"
)
if enrichment.get("candidate_pool_scope_verified") is not actual_pool_scope_verified:
review.append(
"screening_audit.enrichment.candidate_pool_scope_verified does not match generation evidence"
)
if exhausted and not actual_pool_scope_verified:
review.append(
"candidate_pool_exhausted cannot be true before the candidate-pool generation scope is verified"
)
if enrichment_status in {"complete", "not_required"} and not pool_ready:
review.append(
"candidate-pool enrichment is marked complete before the pool is exhausted and every row is resolved"
)
declared_queue = sorted(str(value).upper() for value in _list(enrichment.get("queue_symbols")))
if declared_queue != sorted(actual_queue_symbols):
review.append(
"screening_audit.enrichment.queue_symbols does not match unresolved candidate rows"
)
if _integer(enrichment.get("queue_count")) != len(actual_queue_symbols):
review.append(
"screening_audit.enrichment.queue_count does not match unresolved candidate rows"
)
if _integer(enrichment.get("attempted_count")) != enrichment_attempted:
review.append("screening_audit.enrichment.attempted_count does not match candidate rows")
if _integer(enrichment.get("resolved_count")) != enrichment_resolved:
review.append("screening_audit.enrichment.resolved_count does not match candidate rows")
if _integer(enrichment.get("unresolved_count")) != len(actual_queue_symbols):
review.append("screening_audit.enrichment.unresolved_count does not match candidate rows")
if enrichment.get("all_rows_resolved") is not all_rows_resolved:
review.append("screening_audit.enrichment.all_rows_resolved does not match candidate rows")
calculated_resolution_pct = (
enrichment_resolved / len(candidate_rows) * 100.0 if candidate_rows else 0.0
)
declared_resolution_pct = _number(enrichment.get("resolution_pct"))
if (
declared_resolution_pct is None
or abs(declared_resolution_pct - calculated_resolution_pct) > 0.05
):
review.append("screening_audit.enrichment.resolution_pct does not match candidate rows")
if selected_symbols and pool_ready:
expected_pool_status = "sufficient"
elif selected_symbols:
expected_pool_status = "sufficient_pending_enrichment"
elif pool_ready and discovery_evaluable > 0:
expected_pool_status = (
"no_qualifying_candidates"
if expected_scope == "full_listing_universe"
else "no_qualifying_candidates_in_bounded_pool"
)
else:
expected_pool_status = "insufficient_data"
pool_status = (_text(audit.get("candidate_pool_status")) or "").lower()
if pool_status != expected_pool_status:
review.append(
f"screening_audit.candidate_pool_status must be {expected_pool_status!r} for the audited pool"
)
expected_outcome = {
"sufficient": "selected",
"sufficient_pending_enrichment": "selected_pending_enrichment",
"no_qualifying_candidates": "no_candidates",
"no_qualifying_candidates_in_bounded_pool": "no_candidates_in_bounded_pool",
"insufficient_data": "insufficient_data",
}[expected_pool_status]
outcome = (_text(audit.get("selection_outcome")) or "").lower()
if outcome != expected_outcome:
review.append(
f"screening_audit.selection_outcome must be {expected_outcome!r} for the audited pool"
)
if expected_pool_status in {"insufficient_data", "sufficient_pending_enrichment"}:
warnings.append(
"candidate pool remains unresolved; continue enrichment/exhaustion verification before publishing a final ranking or no-candidates conclusion"
)
if pool_status in {"no_qualifying_candidates", "no_qualifying_candidates_in_bounded_pool"}:
if (
not exhausted
or not declared_exhausted
or actual_queue_symbols
or not all_rows_resolved
or not actual_pool_scope_verified
):
review.append(
"no-candidates conclusion requires an explicitly exhausted, fully resolved, scope-verified candidate pool"
)
if pool_status == "no_qualifying_candidates" and not actual_listing_coverage_complete:
review.append(
"market-wide no_qualifying_candidates requires coverage of every in-scope listing"
)
universe_count = _integer(funnel.get("universe_count"))
candidate_pool_count = _integer(funnel.get("candidate_pool_count"))
if candidate_pool_count is None:
candidate_pool_count = _integer(funnel.get("broad_screen_count"))
selected_count = _integer(funnel.get("deep_dive_selected_count"))
if selected_count is None:
selected_count = _integer(funnel.get("deep_dive_count"))
if universe_count != len(universe_rows):
review.append("screening_funnel.universe_count does not equal universe artifact rows")
if candidate_pool_count != len(candidate_rows):
review.append(
"screening_funnel.candidate_pool_count does not equal candidate artifact rows"
)
if selected_count != len(selected_symbols):
review.append(
"screening_funnel.deep_dive_selected_count does not equal selected-symbol count"
)
if len(universe_rows) < len(candidate_rows) or len(candidate_rows) < len(selected_symbols):
review.append("screening funnel layer counts are inconsistent")
fundamental_pct = fundamental_complete / len(candidate_rows) * 100.0 if candidate_rows else 0.0
discovery_pct = discovery_evaluable / len(candidate_rows) * 100.0 if candidate_rows else 0.0
if mode != "full_universe_fundamentals":
warnings.append(
"tiered candidate-generation mode used; financial coverage applies only to the bounded candidate pool"
)
if candidate_rows and fundamental_pct < 50.0:
warnings.append(
f"candidate-pool full-fundamental coverage is {fundamental_pct:.1f}%; selected names still require complete primary-source underwriting"
)
normalized = deepcopy(audit)
normalized.update(
{
"actual_universe_sha256": universe_digest,
"actual_candidate_pool_sha256": candidate_digest,
"actual_candidate_rows": deepcopy(candidate_rows),
"actual_universe_row_count": len(universe_rows),
"actual_candidate_pool_row_count": len(candidate_rows),
"actual_universe_decision_counts": universe_counts,
"actual_candidate_decision_counts": candidate_counts,
"actual_selected_symbols": sorted(selected_symbols),
"actual_listing_data_complete_pct": round(calculated_listing_pct, 4),
"actual_candidate_pool_discovery_evaluable_pct": round(discovery_pct, 4),
"actual_candidate_pool_fundamental_complete_pct": round(fundamental_pct, 4),
"actual_enrichment_attempted_count": enrichment_attempted,
"actual_enrichment_resolved_count": enrichment_resolved,
"actual_enrichment_unresolved_count": len(actual_queue_symbols),
"actual_enrichment_resolution_pct": round(calculated_resolution_pct, 4),
"actual_enrichment_queue_symbols": sorted(actual_queue_symbols),
"actual_candidate_pool_exhausted": exhausted,
"actual_candidate_pool_coverage_complete": actual_pool_scope_verified,
"actual_listing_coverage_complete": actual_listing_coverage_complete,
"actual_candidate_pool_coverage_scope": expected_scope,
"actual_candidate_pool_missing_in_scope_symbols": actual_missing_in_scope,
"actual_executed_scope_complete": executed_scope_complete,
"actual_user_requested_scope_complete": user_requested_scope_complete,
"actual_bounded_sampling_ready": bounded_sampling_ready,
"actual_screening_scope_ready": actual_screening_scope_ready,
"screening_completion_ready": pool_ready,
"valid": not review,
}
)
return normalized, not review, list(dict.fromkeys(warnings)), list(dict.fromkeys(review))
def _derive_growth_state(metrics: Mapping[str, Any]) -> str:
values = [
_number(metrics.get("revenue_yoy_pct")),
_number(metrics.get("gaap_operating_income_yoy_pct")),
_number(metrics.get("gaap_eps_yoy_pct")),
_number(metrics.get("adjusted_eps_yoy_pct")),
]
values = [value for value in values if value is not None]
if len(values) < 2:
return "insufficient_data"
if sum(value < 0 for value in values) >= 2:
return "decelerating_or_contracting"
if sum(value >= 10 for value in values) >= 2:
return "accelerating_or_strong"
return "stable_or_mixed"
def _validate_earnings_record(
record_raw: Any,
expected_type: str,
source_index: Mapping[str, Mapping[str, Any]],
analysis_as_of: datetime,
location: str,
) -> tuple[dict[str, Any], datetime | None, list[str], list[str]]:
record = _mapping(record_raw)
warnings: list[str] = []
review: list[str] = []
period_type = (_text(record.get("period_type")) or "").lower()
period = _text(record.get("period"))
if period_type != expected_type:
review.append(f"{location}.period_type must equal {expected_type!r}")
if not period:
review.append(f"{location}.period is required")
elif "/" in period:
review.append(f"{location}.period must not combine quarter and full-year labels")
elif expected_type == "quarter" and not re.search(r"\bQ[1-4]\b", period, re.IGNORECASE):
review.append(f"{location}.period must identify a fiscal quarter")
elif expected_type == "full_year" and re.search(r"\bQ[1-4]\b", period, re.IGNORECASE):
review.append(f"{location}.period must contain only the full-year period")
try:
_parse_date(record.get("period_end"), f"{location}.period_end")
except ContractError as exc:
review.append(str(exc))
published = None
try:
published = parse_iso8601(
record.get("published_at"), f"{location}.published_at", required=True
)
except ContractError as exc:
review.append(str(exc))
if published and published > analysis_as_of:
review.append(f"{location}.published_at is after analysis_as_of")
if not _has_primary(record.get("source_ids"), source_index):
review.append(f"{location}.source_ids must resolve to SEC or company IR evidence")
metrics = _mapping(record.get("metrics"))
if _number(metrics.get("revenue")) is None:
review.append(f"{location}.metrics.revenue is required")
if (
_number(metrics.get("gaap_eps")) is None
and _number(metrics.get("gaap_operating_income")) is None
):
review.append(f"{location} requires GAAP EPS or GAAP operating income")
if not isinstance(record.get("guidance"), list):
review.append(f"{location}.guidance must be an array")
if not isinstance(record.get("key_kpis"), list):
review.append(f"{location}.key_kpis must be an array")
if not isinstance(record.get("one_time_items"), list):
review.append(f"{location}.one_time_items must be an array")
normalized = deepcopy(record)
if expected_type == "quarter":
derived = _derive_growth_state(metrics)
supplied = (_text(record.get("growth_state")) or "").lower()
normalized["derived_growth_state"] = derived
if supplied and derived != "insufficient_data" and supplied != derived:
review.append(f"{location}.growth_state conflicts with quarter metrics ({derived})")
elif not supplied:
normalized["growth_state"] = derived
return normalized, published, list(dict.fromkeys(warnings)), list(dict.fromkeys(review))
def validate_latest_earnings(
latest_raw: Any,
source_index: Mapping[str, Mapping[str, Any]],
analysis_as_of: datetime,
) -> tuple[dict[str, Any], dict[str, Any], datetime | None, bool, list[str], list[str]]:
latest = _mapping(latest_raw)
report_type = (_text(latest.get("latest_report_type")) or "").lower()
warnings: list[str] = []
review: list[str] = []
if report_type not in ALLOWED_REPORT_TYPES:
review.append("latest_earnings.latest_report_type must be quarterly or annual")
quarter: dict[str, Any] = {}
full_year: dict[str, Any] = {}
published_dates: list[datetime] = []
if latest.get("quarter") is not None:
quarter, published, row_warnings, row_review = _validate_earnings_record(
latest.get("quarter"),
"quarter",
source_index,
analysis_as_of,
"latest_earnings.quarter",
)
warnings.extend(row_warnings)
review.extend(row_review)
if published:
published_dates.append(published)
if latest.get("full_year") is not None:
full_year, published, row_warnings, row_review = _validate_earnings_record(
latest.get("full_year"),
"full_year",
source_index,
analysis_as_of,
"latest_earnings.full_year",
)
warnings.extend(row_warnings)
review.extend(row_review)
if published:
published_dates.append(published)
if report_type == "quarterly" and not quarter:
review.append("quarterly latest report requires latest_earnings.quarter")
if report_type == "annual" and not full_year:
review.append("annual latest report requires latest_earnings.full_year")
if not quarter and not full_year:
review.append("latest_earnings contains no report record")
primary = quarter if quarter else full_year
latest_published = max(published_dates) if published_dates else None
normalized = {
"latest_report_type": report_type,
"quarter": quarter or None,
"full_year": full_year or None,
"valid": not review,
}
return (
normalized,
primary,
latest_published,
not review,
list(dict.fromkeys(warnings)),
list(dict.fromkeys(review)),
)
def validate_forward_periods(
candidate: Mapping[str, Any], config: Mapping[str, Any]
) -> tuple[dict[str, bool], list[str], list[str]]:
valuation = _mapping(candidate.get("valuation_case"))
periods = _mapping(valuation.get("periods"))
warnings: list[str] = []
review: list[str] = []
rankable: dict[str, bool] = {}
min_count = _integer(config.get("minimum_analyst_count_for_ranked_horizon")) or 3
current = _mapping(periods.get("current"))
current_kind = (_text(current.get("period_kind")) or "").lower()
if current_kind not in ALLOWED_FORWARD_PERIOD_KINDS:
review.append("valuation current period must be NTM or FY1; TTM is supplemental only")
if _number(current.get("metric")) is None or _number(current.get("metric")) <= 0:
review.append("valuation current forward metric must be positive")
if not _text(current.get("period_end")):
review.append("valuation current period_end is required")
current_count = _integer(current.get("analyst_count"))
source_type = (_text(current.get("source_type")) or "").lower()
if source_type in {"market_consensus", "analyst_estimate"} and (current_count or 0) < min_count:
review.append(
f"current forward metric requires at least {min_count} analysts or an independent model"
)
ranges = {"year_2": (1.5, 2.5), "year_3": (2.5, 3.5)}
for key, (low, high) in ranges.items():
row = _mapping(periods.get(key))
metric = _number(row.get("metric"))
if metric is None:
rankable[key] = False
continue
kind = (_text(row.get("period_kind")) or "").lower()
years = _number(row.get("years"))
if kind not in ALLOWED_FUTURE_PERIOD_KINDS:
review.append(f"valuation {key} period_kind is invalid")
if years is None or not (low <= years <= high):
review.append(f"valuation {key}.years must be between {low} and {high}")
if not _text(row.get("period_end")):
review.append(f"valuation {key}.period_end is required")
count = _integer(row.get("analyst_count"))
row_source_type = (_text(row.get("source_type")) or "").lower()
independent = row.get("independent_model") is True
if (
row_source_type in {"market_consensus", "analyst_estimate"}
and not independent
and (count or 0) < min_count
):
warnings.append(
f"{key} estimate has only {count or 0} analysts and cannot be the sole ranked horizon"
)
rankable[key] = False
else:
rankable[key] = True
if not any(rankable.values()):
review.append("no adequately supported two- or three-year forecast horizon")
return rankable, list(dict.fromkeys(warnings)), list(dict.fromkeys(review))
def normalize_cash_classification(
candidate: dict[str, Any],
latest: Mapping[str, Any],
) -> tuple[list[str], list[str]]:
"""Normalize ordinary balance-sheet cash into ``corporate_cash``.
Payment processors and other custodial-cash businesses must provide an
explicit separation and are never auto-normalized.
"""
warnings: list[str] = []
review: list[str] = []
financials = _mapping(candidate.get("financials"))
cash = _mapping(financials.get("cash_classification"))
profile_type = (
_text(_mapping(candidate.get("sector_profile")).get("type")) or "general"
).lower()
custodial_profiles = {"payments", "broker", "custodian", "exchange"}
if _number(cash.get("corporate_cash")) is None:
explicit_cash = _number(cash.get("cash_and_equivalents"))
primary = (
_mapping(latest.get("quarter")) or _mapping(latest.get("full_year")) or _mapping(latest)
)
metrics = _mapping(primary.get("metrics"))
reported_cash = _number(metrics.get("cash_and_equivalents"))
if profile_type in custodial_profiles:
review.append(
"custodial/payments business requires explicit corporate_cash separate from customer funds"
)
else:
derived = explicit_cash if explicit_cash is not None else reported_cash
if derived is not None:
cash["corporate_cash"] = derived
cash["classification_method"] = "derived_from_reported_cash_and_equivalents"
cash["corporate_cash_derived"] = True
warnings.append("corporate_cash was normalized from reported cash and equivalents")
else:
review.append("corporate_cash or reported cash_and_equivalents is required")
if _number(cash.get("marketable_securities")) is None:
primary = (
_mapping(latest.get("quarter")) or _mapping(latest.get("full_year")) or _mapping(latest)
)
reported = _number(_mapping(primary.get("metrics")).get("marketable_securities"))
if reported is not None:
cash["marketable_securities"] = reported
cash["marketable_securities_derived"] = True
financials["cash_classification"] = cash
candidate["financials"] = financials
return warnings, review
def validate_cash_consistency(
candidate: Mapping[str, Any], latest: Mapping[str, Any], config: Mapping[str, Any]
) -> tuple[dict[str, Any], bool, list[str], list[str]]:
financials = _mapping(candidate.get("financials"))
cash = _mapping(financials.get("cash_classification"))
primary = _mapping(latest.get("quarter")) or _mapping(latest.get("full_year"))
metrics = _mapping(primary.get("metrics"))
warnings: list[str] = []
review: list[str] = []
tolerance = _number(config.get("cash_consistency_tolerance_pct")) or 3.0
reconciliation_note = _text(cash.get("reconciliation_note"))
checks: dict[str, Any] = {}
pairs = {
"cash_and_equivalents": (
_number(metrics.get("cash_and_equivalents")),
_number(cash.get("corporate_cash")),
),
"marketable_securities": (
_number(metrics.get("marketable_securities")),
_number(cash.get("marketable_securities")),
),
"total_debt": (_number(metrics.get("total_debt")), _number(financials.get("total_debt"))),
}
for key, (reported, normalized) in pairs.items():
diff = _pct_diff(reported, normalized)
checks[key] = {
"earnings_value": reported,
"normalized_value": normalized,
"difference_pct": diff,
}
if diff is not None and diff > tolerance:
message = (
f"{key} differs by {diff:.1f}% between latest earnings and normalized financials"
)
if reconciliation_note:
warnings.append(message + f"; reconciliation: {reconciliation_note}")
else:
review.append(message + " without a reconciliation note")
return (
{"checks": checks, "reconciliation_note": reconciliation_note},
not review,
list(dict.fromkeys(warnings)),
list(dict.fromkeys(review)),
)
def validate_and_normalize_snapshot(
payload_raw: Mapping[str, Any],
*,
artifact_root: Path | None = None,
) -> tuple[dict[str, Any], dict[str, Any]]:
payload = deepcopy(dict(payload_raw))
if _integer(payload.get("schema_version")) != SCHEMA_VERSION:
raise ContractError(f"schema_version must equal {SCHEMA_VERSION}")
payload_runtime = _mapping(payload.get("runtime"))
expected_runtime = runtime_metadata()
runtime_mismatches = [
key
for key in (
"skill_name",
"skill_version",
"schema_version",
"contract_revision",
"runtime_fingerprint",
)
if payload_runtime.get(key) != expected_runtime.get(key)
]
if runtime_mismatches:
raise ContractError(
"snapshot runtime does not match the installed skill; stale or cached package detected: "
+ ", ".join(runtime_mismatches)
)
analysis_as_of = parse_iso8601(payload.get("analysis_as_of"), "analysis_as_of", required=True)
assert analysis_as_of is not None
config = _mapping(payload.get("config"))
global_index, normalized_global_sources, source_warnings, source_review = (
validate_source_ledger(_list(payload.get("global_sources")), [], analysis_as_of)
)
market_context, market_valid, market_warnings, market_review = validate_market_context(
payload.get("market_context"), global_index, analysis_as_of, config
)
screening_audit, audit_valid, audit_warnings, audit_review = validate_screening_audit(
payload.get("screening_audit"),
payload.get("screening_funnel"),
global_index,
analysis_as_of,
artifact_root,
config,
)
global_review = source_review + market_review + audit_review
global_warnings = source_warnings + market_warnings + audit_warnings
selected_symbols = set(screening_audit.get("actual_selected_symbols") or [])
normalized_candidates: list[dict[str, Any]] = []
candidate_symbols: set[str] = set()
for raw_candidate in _list(payload.get("candidates")):
candidate = deepcopy(_mapping(raw_candidate))
identity = _mapping(candidate.get("identity"))
symbol = (_text(identity.get("symbol")) or "UNKNOWN").upper()
transition_aliases = {"recent_spin_off", "transformative_acquisition", "large_divestiture"}
legacy_special_case = (_text(identity.get("special_case")) or "none").lower()
transition_review: list[str] = []
transition_warnings: list[str] = []
if legacy_special_case in transition_aliases:
identity["special_case"] = "none"
candidate["identity"] = identity
transition = _mapping(candidate.get("corporate_transition"))
transition.setdefault("type", legacy_special_case)
transition.setdefault("source_ids", [])
candidate["corporate_transition"] = transition
transition_warnings.append(
f"legacy identity.special_case={legacy_special_case!r} was normalized to corporate_transition.type"
)
transition = _mapping(candidate.get("corporate_transition"))
transition_type = (_text(transition.get("type")) or "none").lower()
if transition_type != "none":
source_ids = transition.get("source_ids")
pro_forma = transition.get("pro_forma_normalization")
if not _sources_resolve(
source_ids,
global_index
| {
str(_mapping(src).get("id")): _mapping(src)
for src in _list(candidate.get("sources"))
if _text(_mapping(src).get("id"))
},
):
transition_review.append("corporate transition lacks resolving source evidence")
if transition_type in {
"recent_spin_off",
"transformative_acquisition",
} and not isinstance(pro_forma, Mapping):
transition_review.append(
"recent spin-off/acquisition requires a sourced pro-forma normalization"
)
candidate_symbols.add(symbol)
source_index, normalized_sources, candidate_source_warnings, candidate_source_review = (
validate_source_ledger(
normalized_global_sources, _list(candidate.get("sources")), analysis_as_of
)
)
latest, primary, latest_published, latest_valid, latest_warnings, latest_review = (
validate_latest_earnings(candidate.get("latest_earnings"), source_index, analysis_as_of)
if (_text(_mapping(candidate.get("screening_decision")).get("status")) or "passed")
!= "screened_out"
else ({}, {}, None, True, [], [])
)
rankable_horizons, forward_warnings, forward_review = (
validate_forward_periods(candidate, config) if primary else ({}, [], [])
)
cash_normalization_warnings, cash_normalization_review = (
normalize_cash_classification(candidate, latest) if primary else ([], [])
)
cash_consistency, cash_valid, cash_warnings, cash_review = (
validate_cash_consistency(candidate, latest, config) if primary else ({}, True, [], [])
)
review_reasons = list(
dict.fromkeys(
global_review
+ candidate_source_review
+ latest_review
+ forward_review
+ cash_normalization_review
+ cash_review
+ transition_review
)
)
warnings = list(
dict.fromkeys(
global_warnings
+ candidate_source_warnings
+ latest_warnings
+ forward_warnings
+ cash_normalization_warnings
+ cash_warnings
+ transition_warnings
)
)
quality_caps: list[dict[str, Any]] = []
if not market_valid:
quality_caps.append({"cap": 70, "reason": "market context is incomplete or invalid"})
if not audit_valid:
quality_caps.append(
{"cap": 65, "reason": "broad-screen audit is incomplete or unverified"}
)
if candidate_source_review or source_review:
quality_caps.append(
{"cap": 60, "reason": "source ledger schema or classification is invalid"}
)
if not latest_valid and primary:
quality_caps.append(
{"cap": 55, "reason": "latest earnings periods are incomplete or mixed"}
)
if forward_review and primary:
quality_caps.append({"cap": 60, "reason": "forward valuation basis is incomplete"})
if not cash_valid and primary:
quality_caps.append({"cap": 70, "reason": "cash/debt definitions are inconsistent"})
if transition_review:
quality_caps.append(
{"cap": 65, "reason": "corporate transition/pro-forma evidence is incomplete"}
)
candidate["sources"] = normalized_sources
candidate["latest_earnings_records"] = latest
candidate["latest_earnings"] = primary
periods = _mapping(_mapping(candidate.get("valuation_case")).get("periods"))
for key, rankable in rankable_horizons.items():
row = _mapping(periods.get(key))
row["rankable"] = rankable
periods[key] = row
if periods:
candidate.setdefault("valuation_case", {})["periods"] = periods
candidate["_contract"] = {
"review_reasons": review_reasons,
"warnings": warnings,
"quality_caps": quality_caps,
"source_schema_valid": not candidate_source_review and not source_review,
"latest_earnings_valid": latest_valid,
"latest_published_at": latest_published.isoformat() if latest_published else None,
"forward_basis_valid": not forward_review,
"rankable_horizons": rankable_horizons,
"cash_consistency_valid": cash_valid,
"cash_consistency": cash_consistency,
"global_contract_valid": not global_review,
}
normalized_candidates.append(candidate)
run_metadata = _mapping(payload.get("run_metadata"))
declared_status = (_text(run_metadata.get("status")) or "partial").lower()
unprocessed = {
str(value).upper() for value in _list(run_metadata.get("unprocessed_candidates"))
}
missing_selected = sorted(selected_symbols - candidate_symbols)
unexpected_unprocessed = sorted(unprocessed - selected_symbols)
selection_outcome = (_text(screening_audit.get("selection_outcome")) or "").lower()
candidate_pool_status = (_text(screening_audit.get("candidate_pool_status")) or "").lower()
no_qualified = candidate_pool_status in {
"no_qualifying_candidates",
"no_qualifying_candidates_in_bounded_pool",
}
completion_ready = (
screening_audit.get("valid") is True
and screening_audit.get("screening_completion_ready") is True
and candidate_pool_status
in {"sufficient", "no_qualifying_candidates", "no_qualifying_candidates_in_bounded_pool"}
)
if missing_selected:
global_review.append(
"selected broad-screen symbols are missing candidate records: "
+ ", ".join(missing_selected)
)
if unexpected_unprocessed:
global_review.append(
"unprocessed_candidates contains symbols that were not selected: "
+ ", ".join(unexpected_unprocessed)
)
if declared_status == "complete" and unprocessed:
global_review.append("run is marked complete but unprocessed_candidates is not empty")
if declared_status == "complete" and missing_selected:
global_review.append(
"run is marked complete but not all selected symbols were underwritten"
)
if declared_status == "complete" and not completion_ready:
global_review.append(
"run is marked complete before the staged screening contract is completion-ready"
)
if declared_status == "complete" and candidate_pool_status in {
"insufficient_data",
"sufficient_pending_enrichment",
}:
global_review.append("run is marked complete while the candidate pool is unresolved")
if declared_status == "complete" and not selected_symbols and not no_qualified:
global_review.append(
"complete run with no selected symbols requires a scope-qualified no-candidates status"
)
payload["global_sources"] = normalized_global_sources
payload["market_context"] = market_context
payload["screening_audit"] = screening_audit
payload["candidates"] = normalized_candidates
contract = {
"runtime": runtime_metadata(),
"valid": not global_review,
"market_context_valid": market_valid,
"screening_audit_valid": audit_valid,
"source_schema_valid": not source_review,
"selected_symbols": sorted(selected_symbols),
"missing_selected_symbols": missing_selected,
"selection_outcome": selection_outcome,
"candidate_pool_status": candidate_pool_status,
"screening_completion_ready": completion_ready,
"no_qualified_candidates": no_qualified,
"review_reasons": list(dict.fromkeys(global_review)),
"warnings": list(dict.fromkeys(global_warnings)),
}
payload["_contract"] = contract
return payload, contract
scripts/run_pipeline.py
#!/usr/bin/env python3
"""Claude Code-native direct-FMP discovery pipeline for the US GARP skill.
Bulk provider payloads stay on disk. The program prints one compact JSON summary
and emits auditable JSON/JSONL artifacts consumed by the existing contract-3.5
underwriting and evaluation workflow.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import os
import re
import uuid
from collections import defaultdict
from collections.abc import Mapping, Sequence
from copy import deepcopy
from dataclasses import dataclass
from datetime import date, datetime, timedelta, timezone
from pathlib import Path
from typing import Any
from zoneinfo import ZoneInfo
import estimate_snapshot as snapshot_store
from build_provider_prefilter_pool import ALLOWED_LANES, _lane_score, build_pool
from coverage_semantics import build_coverage_block, classify_ranking_scope
from fmp_client import ApiCallBudgetExceeded, FMPClient
from normalize_estimates import apply_verified_actual_eps, normalize_symbol
from screen_universe import DEFAULTS as SCREEN_DEFAULTS
from screen_universe import (
SECTOR_PROFILES,
_canonical_line,
requires_unit_reconciliation,
run_layered,
)
from skill_version import runtime_metadata
# SEC filing acceptance timestamps (FMP acceptedDate) are US/Eastern wall time.
SEC_FILING_TZ = ZoneInfo("America/New_York")
DEFAULT_CONFIG: dict[str, Any] = {
"min_market_cap": 500_000_000,
"max_market_cap": 20_000_000_000,
"min_price": 5.0,
"hard_min_average_daily_dollar_volume": 1_000_000,
"min_average_daily_dollar_volume": 5_000_000,
"max_api_calls": 350,
"company_screener_limit": 1_000,
"minimum_market_cap_band_width": 25_000_000,
"maximum_market_cap_band_depth": 12,
"bulk_estimate_years": 5,
"bulk_estimate_minimum_coverage_pct": 20.0,
"pre_enrichment_limit": 180,
"seed_limit_cap": 200,
"quality_probe_limit": 35,
"candidate_packet_reserve_calls": 30,
"retry_reserve_calls": 25,
"exact_liquidity_limit": 40,
"provider_prefilter_pool_size": 30,
"provider_prefilter_minimum_pool": 12,
"provider_prefilter_per_lane": 8,
"max_deep_dive_candidates": 3,
"full_snapshot_pool_size": 50,
"full_snapshot_deep_dive_candidates": 5,
"full_snapshot_max_staleness_days": 7,
"full_snapshot_screening_clock_skew_seconds": 300,
"full_snapshot_collection_clock_skew_seconds": 300,
# Symbol -> profile pins for names the listing-frame taxonomy cannot
# classify (e.g. BDCs filed under plain "Asset Management" whose name
# carries no BDC marker: {"ARCC": "bdc"}).
"sector_profile_overrides": {},
"minimum_discovery_analyst_count": 2,
"maximum_forward_eps_dispersion_pct": 100.0,
"maximum_fy1_horizon_days": 430,
"forward_pe_reconciliation_tolerance_pct": 5.0,
"minimum_average_volume_period_days": 20,
"minimum_revenue_growth_pct": 8.0,
"minimum_per_share_growth_pct": 12.0,
"preferred_forward_pe": 20.0,
"high_growth_exception_max_forward_pe": 30.0,
"high_growth_exception_growth_pct": 20.0,
"near_miss_max_forward_pe": 22.0,
"near_miss_min_per_share_growth_pct": 8.0,
"preferred_ev_to_fcf": 20.0,
"preferred_fcf_yield_pct": 5.0,
"minimum_roic_pct": 8.0,
"preferred_max_dilution_pct": 3.0,
"preferred_max_net_debt_to_ebitda": 2.5,
"hard_max_net_debt_to_ebitda": 4.0,
"maximum_forward_pe_for_economic_screen": 60.0,
"sector_review_selection_penalty": 5.0,
"near_miss_selection_penalty": 4.0,
"missing_deep_dive_field_penalty": 1.5,
"selection_lane_quota_core_garp": 2,
"selection_lane_quota_high_growth": 1,
"selection_lane_quota_near_miss": 1,
"selection_lane_quota_cyclical": 1,
"maximum_selected_per_sector": 2,
"maximum_enrichment_attempts": 80,
"minimum_listing_data_coverage_pct": 95.0,
"max_estimate_age_days": 45,
"bulk_endpoints": {
"ratios_ttm": "ratios-ttm-bulk",
"key_metrics_ttm": "key-metrics-ttm-bulk",
"income_growth": "income-statement-growth-bulk",
"analyst_estimates": "analyst-estimates-bulk",
"eod": "eod-bulk",
},
"cache": {"path": ".cache/us-garp/fmp-cache.sqlite3", "raw_store_dir": ".cache/us-garp/raw"},
"compact_stdout_max_bytes": 20_000,
}
EXCHANGES = ("NASDAQ", "NYSE", "AMEX")
NON_COMMON_NAME = re.compile(
r"\b(preferred|depositary shares?|warrants?|rights?|units?|notes?|debentures?|bonds?|closed[- ]end|income fund)\b",
re.IGNORECASE,
)
NON_COMMON_SYMBOL = re.compile(r"(?:[-.](?:P[A-Z]?|WS|WT|W|U|R))$", re.IGNORECASE)
CYCLICAL_RULES: tuple[tuple[int, tuple[str, ...]], ...] = (
(5, ("airline", "marine shipping", "oil tanker", "dry bulk", "steel", "coal", "iron ore")),
(
4,
(
"chemical",
"mining",
"oil & gas",
"oilfield",
"auto manufacturer",
"auto parts",
"homebuilding",
"building materials",
"forest products",
"paper",
"agricultural",
"farm products",
"meat products",
"gold",
"silver",
"precious metal",
"base metal",
"copper",
"uranium",
"coal",
"metals & mining",
"metals and mining",
"metal mining",
"mineral",
),
),
(
3,
(
"machinery",
"industrial distribution",
"advertising",
"staffing",
"casino",
"resort",
"hotel",
"transportation",
"trucking",
"railroad",
"construction",
"consumer cyclical",
"aluminum",
"semiconductor equipment",
),
),
)
@dataclass
class PipelineResult:
summary: dict[str, Any]
exit_code: int
def _ordered_unique_symbols(values: Sequence[Any]) -> list[str]:
"""Return canonical, duplicate-free symbols while preserving input order."""
output: list[str] = []
seen: set[str] = set()
for value in values:
if value is None:
continue
symbol = str(value).strip().upper()
if symbol and symbol not in seen:
seen.add(symbol)
output.append(symbol)
return output
class _OperationTrackingClient:
"""Proxy provider calls so an interrupted packet identifies its operation."""
def __init__(self, client: Any) -> None:
self._client = client
self.last_operation: str | None = None
def __getattr__(self, name: str) -> Any:
value = getattr(self._client, name)
if not callable(value):
return value
def tracked(*args: Any, **kwargs: Any) -> Any:
self.last_operation = name
return value(*args, **kwargs)
return tracked
def _number(value: Any) -> float | None:
if isinstance(value, bool):
return None
try:
result = float(value)
except (TypeError, ValueError):
return None
return result if math.isfinite(result) else None
def _first_number(row: Mapping[str, Any], *keys: str) -> float | None:
for key in keys:
value = _number(row.get(key))
if value is not None:
return value
return None
def _text(value: Any) -> str | None:
return value.strip() if isinstance(value, str) and value.strip() else None
def _symbol(row: Mapping[str, Any]) -> str:
return (_text(row.get("symbol")) or _text(row.get("ticker")) or "UNKNOWN").upper()
def _canonical_jsonl(rows: Sequence[Mapping[str, Any]]) -> bytes:
return b"".join(
(
json.dumps(dict(row), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n"
).encode("utf-8")
for row in rows
)
def _write_json(path: Path, value: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temp = path.with_suffix(path.suffix + ".tmp")
temp.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
os.replace(temp, path)
def _write_jsonl(path: Path, rows: Sequence[Mapping[str, Any]]) -> str:
path.parent.mkdir(parents=True, exist_ok=True)
data = _canonical_jsonl(rows)
temp = path.with_suffix(path.suffix + ".tmp")
temp.write_bytes(data)
os.replace(temp, path)
return hashlib.sha256(data).hexdigest()
def _artifact_record(path: Path, *, root: Path, row_count: int | None = None) -> dict[str, Any]:
"""Describe an existing artifact for the partial-run commit marker."""
relative = str(path.relative_to(root))
record: dict[str, Any] = {
"path": relative,
"sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
"bytes": path.stat().st_size,
}
if row_count is not None:
record["row_count"] = row_count
return record
def _persist_partial_snapshot_diagnostic(
*,
output_dir: Path,
audit_dir: Path,
snapshot_id: str,
verification_digest: str,
stage: str,
progress: Mapping[str, Any],
error: str,
client: Any,
partial_rows: Sequence[Mapping[str, Any]],
preserved_artifact_paths: Sequence[Path] = (),
) -> PipelineResult:
"""Persist a fail-closed partial screen result before returning exit 2.
The diagnostic is deliberately written last. It is the commit marker for
the per-file atomic writes that precede it; consumers must verify the
marker's hashes before reading any partial artifact.
"""
partial_path = audit_dir / "enriched-estimates.partial.jsonl"
partial_sha = _write_jsonl(partial_path, partial_rows)
partial_record = _artifact_record(
partial_path,
root=output_dir,
row_count=len(partial_rows),
)
preserved_records = [
_artifact_record(path, root=output_dir)
for path in preserved_artifact_paths
if path.is_file()
]
# Keep the explicit local variable as a guard against accidentally writing
# a diagnostic whose hash does not describe the bytes just committed.
if partial_record["sha256"] != partial_sha:
raise OSError("partial enrichment artifact hash changed during write")
common: dict[str, Any] = {
"schema_version": 1,
"runtime": runtime_metadata(),
"run_id": output_dir.name,
"status": "provider_budget_exhausted",
"stage": stage,
"snapshot_id": snapshot_id,
"snapshot_verification_digest": verification_digest,
"progress": dict(progress),
"error": error,
"provider_diagnostics": client.diagnostics(),
"no_final_marketwide_conclusion": True,
"recovery": {
"resume_supported": False,
"action": "rerun_screen_full_snapshot_after_provider_budget_reset",
"same_verified_snapshot_required": True,
"partial_artifact_is_not_a_final_conclusion": True,
},
"partial_diagnostic_path": "audit/partial-run-diagnostic.json",
}
summary = {
**common,
"artifacts": {
"partial_enriched_estimates": partial_record,
"preserved": preserved_records,
"partial_diagnostic": {"path": "audit/partial-run-diagnostic.json"},
},
}
next_action = {
**common,
"action": "rerun_screen_full_snapshot_after_provider_budget_reset",
"partial_artifacts_are_authoritative_only_after_marker_verification": True,
"artifacts": {
"partial_diagnostic": "audit/partial-run-diagnostic.json",
"partial_enriched_estimates": "audit/enriched-estimates.partial.jsonl",
},
}
run_summary_path = output_dir / "run-summary.json"
next_action_path = output_dir / "NEXT_ACTION.json"
_write_json(run_summary_path, summary)
_write_json(next_action_path, next_action)
diagnostic = {
**common,
"commit_marker": "audit/partial-run-diagnostic.json",
"artifacts": {
"partial_enriched_estimates": partial_record,
"run_summary": _artifact_record(run_summary_path, root=output_dir),
"next_action": _artifact_record(next_action_path, root=output_dir),
"preserved": preserved_records,
},
}
# This must remain the final write in this helper. A missing or invalid
# marker makes the preceding partial summary non-authoritative.
_write_json(audit_dir / "partial-run-diagnostic.json", diagnostic)
return PipelineResult(summary=summary, exit_code=2)
def _invalidate_final_screen_audit(audit: Mapping[str, Any]) -> dict[str, Any]:
"""Make a pre-packet screen audit explicitly non-authoritative."""
invalidated = dict(audit)
invalidated.update(
{
"status": "incomplete",
"conclusion_scope": "diagnostic",
"ranking_scope": "diagnostic",
"candidate_pool_status": "incomplete_budget_exhausted",
"selection_outcome": "incomplete_budget_exhausted",
"selected_symbols": [],
}
)
plan = dict(invalidated.get("deep_dive_plan") or {})
plan.update(
{
"status": "incomplete",
"selected_symbols": [],
"selected_count": 0,
"selected_set_sha256": None,
"commitment_payload": None,
"selected_set_is_committed": False,
"budget_locked": False,
"lane_counts": {},
"selection_method": "not_committed_due_to_provider_budget",
}
)
invalidated["deep_dive_plan"] = plan
candidate_pool = dict(invalidated.get("candidate_pool") or {})
generation_audit = dict(candidate_pool.get("generation_audit") or {})
generation_audit.update(
{
"valid": False,
"selection_method": "not_committed_due_to_provider_budget",
"selected_count": 0,
"selected_symbols": [],
"actual_selected_count": 0,
"actual_selected_symbols": [],
"selected_set_sha256": None,
"commitment_payload": None,
"selection_commitment": None,
"lane_selected_counts": {},
"lane_coverage_count": 0,
}
)
candidate_pool.update(
{
"status": "incomplete",
"selected_count": 0,
"selection_eligible_count": 0,
"selection_commitment": None,
"generation_audit": generation_audit,
}
)
invalidated["candidate_pool"] = candidate_pool
return invalidated
def _invalidate_candidate_decisions(
decisions: Sequence[Mapping[str, Any]],
) -> list[dict[str, Any]]:
"""Remove all active selection authority while retaining the original audit."""
invalidated: list[dict[str, Any]] = []
for raw in decisions:
row = deepcopy(dict(raw))
decision = dict(row.get("decision") or {})
row.setdefault("prior_decision", deepcopy(decision))
row.setdefault(
"prior_selection",
{
key: deepcopy(row[key])
for key in ("selection_eligible", "selection_lane")
if key in row
},
)
decision["status"] = "deferred_by_budget"
decision["selection_eligible"] = False
for key in ("preselection_status", "selection_lane", "selection_reason"):
decision.pop(key, None)
row["decision"] = decision
row["selection_eligible"] = False
row.pop("selection_lane", None)
invalidated.append(row)
return invalidated
def _require_fresh_screen_output_dir(output_dir: Path) -> None:
"""Prevent a retry from mixing partial artifacts with a prior run."""
if output_dir.exists() and any(output_dir.iterdir()):
raise ValueError(
f"screen-full-snapshot output directory must be new and empty: {output_dir}"
)
def _path_is_within(path: Path, parent: Path) -> bool:
"""Return whether ``path`` resolves below ``parent``."""
try:
path.resolve().relative_to(parent.resolve())
except ValueError:
return False
return True
def load_config(path: Path | None) -> dict[str, Any]:
config = json.loads(json.dumps(DEFAULT_CONFIG))
if path is None:
return config
value = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(value, Mapping):
raise ValueError("pipeline config must be a JSON object")
for key, item in value.items():
if isinstance(item, Mapping) and isinstance(config.get(key), Mapping):
config[key].update(item)
else:
config[key] = item
return config
def is_common_stock(row: Mapping[str, Any]) -> bool:
if bool(row.get("isEtf")) or bool(row.get("isFund")):
return False
if row.get("isActivelyTrading") is False:
return False
name = _text(row.get("companyName")) or _text(row.get("name")) or ""
symbol = _symbol(row)
if NON_COMMON_NAME.search(name) or NON_COMMON_SYMBOL.search(symbol):
return False
return symbol != "UNKNOWN"
# Explicit map over normalized FMP taxonomy labels (lowercased, single-spaced).
# Substring needles alone misroute real provider data: FMP's actual label is
# "Auto - Dealerships" (which the old "auto & truck dealership" needle never
# matched), and "Investment - Banking & Investment Services" is an
# advisory/capital-markets business, not a deposit bank valued on P/TBV and
# deposit betas.
INDUSTRY_PROFILE_MAP: dict[str, str] = {
"auto - dealerships": "auto_dealership",
"auto & truck dealerships": "auto_dealership",
"investment - banking & investment services": "capital_markets",
"financial - capital markets": "capital_markets",
"capital markets": "capital_markets",
}
# Prefix rules over FMP's "Family - Subtype" industry labels.
_INDUSTRY_PREFIX_RULES: tuple[tuple[str, str], ...] = (
("reit", "reit"),
("banks", "bank"),
("insurance", "insurance"),
("business development", "bdc"),
("asset management", "asset_manager"),
)
# FMP files most BDCs under "Asset Management"; the company name is the only
# listing-frame signal. Best-effort — operators can pin the rest through the
# sector_profile_overrides config map.
_BDC_NAME_NEEDLES = ("business development", " bdc")
_MLP_NEEDLES = (" master limited partnership", " mlp")
def _normalize_taxonomy_text(value: str | None) -> str:
return re.sub(r"\s+", " ", (value or "").strip().lower())
def infer_sector_profile_type(
sector: str | None, industry: str | None, company_name: str | None = None
) -> str:
"""Map provider taxonomy onto screen_universe's sector-profile gates.
Names whose standard company multiples are not comparable (mortgage REITs,
insurers, banks, asset managers, BDCs, MLPs, auto dealers) must reach the
broad screen tagged so that, absent sector-specific valuation evidence,
they are routed to ``sector_specific_valuation_required`` instead of being
scored (or excluded) on general-company metrics such as net debt/EBITDA.
``capital_markets`` (advisory / investment banking) is deliberately NOT a
blocked sector profile: those firms are valued on ordinary earnings
multiples — the label exists so they are never misrouted as deposit banks.
"""
industry_norm = _normalize_taxonomy_text(industry)
name_norm = f" {_normalize_taxonomy_text(company_name)} "
if any(needle in name_norm for needle in _BDC_NAME_NEEDLES):
return "bdc"
if industry_norm in INDUSTRY_PROFILE_MAP:
return INDUSTRY_PROFILE_MAP[industry_norm]
for prefix, profile in _INDUSTRY_PREFIX_RULES:
if industry_norm.startswith(prefix):
return profile
text = f" {_normalize_taxonomy_text(sector)} {industry_norm}{name_norm}"
if any(needle in text for needle in _MLP_NEEDLES):
return "mlp"
if " reit " in f"{text} ":
return "reit"
return "general"
def normalize_listing(row: Mapping[str, Any], exchange: str) -> dict[str, Any] | None:
price = _first_number(row, "price", "last")
market_cap = _first_number(row, "marketCap", "mktCap", "market_cap")
if price is None or market_cap is None:
return None
symbol = _symbol(row)
return {
"symbol": symbol,
"company_name": _text(row.get("companyName")) or _text(row.get("name")) or symbol,
"exchange": (
_text(row.get("exchangeShortName")) or _text(row.get("exchange")) or exchange
).upper(),
"sector": _text(row.get("sector")),
"industry": _text(row.get("industry")),
"price": price,
"market_cap": market_cap,
"volume": _first_number(row, "volume"),
"is_actively_trading": row.get("isActivelyTrading", True) is not False,
"is_common_stock": is_common_stock(row),
"common_stock": is_common_stock(row),
"currency": _text(row.get("currency")),
"country": _text(row.get("country")),
# Unit-context identity signals must survive normalization: the
# fail-closed unit gate needs them, and a dropped ISIN silently
# disabled the non-US-ISIN check in earlier versions.
"isin": _text(row.get("isin")),
"is_adr": row.get("isAdr") is True,
"sector_profile_type": infer_sector_profile_type(
_text(row.get("sector")),
_text(row.get("industry")),
_text(row.get("companyName")) or _text(row.get("name")),
),
}
def _coverage_count(value: Any) -> int:
"""Length of a list-valued audit field, or the int it already is."""
if isinstance(value, list):
return len(value)
try:
return int(value or 0)
except (TypeError, ValueError):
return 0
# classify_ranking_scope now lives in coverage_semantics (shared with the
# evaluator so the tri-state semantics cannot drift); imported above.
def collect_listing_universe(
client: FMPClient,
*,
min_market_cap: float,
max_market_cap: float,
min_price: float,
page_limit: int,
minimum_band_width: float,
maximum_depth: int,
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
"""Enumerate the listing frame with adaptive market-cap band splitting.
FMP's company screener has no portable pagination contract across legacy and
stable plans. A saturated response is therefore split into overlapping
market-cap bands until every leaf returns fewer rows than ``page_limit``.
Provider payloads stay in the FMP client's raw store; only normalized rows
and a compact enumeration audit are returned.
"""
if page_limit <= 0:
raise ValueError("company_screener_limit must be positive")
if minimum_band_width <= 0 or max_market_cap <= min_market_cap:
raise ValueError("invalid market-cap enumeration range")
rows: list[dict[str, Any]] = []
leaves: list[dict[str, Any]] = []
query_count = 0
saturated_leaves: list[dict[str, Any]] = []
for exchange in EXCHANGES:
stack: list[tuple[float, float, int]] = [(min_market_cap, max_market_cap, 0)]
while stack:
band_min, band_max, depth = stack.pop()
payload = client.get_company_screener(
exchange=exchange,
min_market_cap=band_min,
max_market_cap=band_max,
min_price=min_price,
limit=page_limit,
)
query_count += 1
saturated = len(payload) >= page_limit
can_split = depth < maximum_depth and (band_max - band_min) > minimum_band_width
if saturated and can_split:
midpoint = (band_min + band_max) / 2.0
# The provider uses MoreThan/LowerThan semantics on some plans.
# A one-dollar overlap prevents an exact-boundary omission; rows
# are deterministically deduplicated after collection.
stack.append((max(band_min, midpoint - 1.0), band_max, depth + 1))
stack.append((band_min, min(band_max, midpoint + 1.0), depth + 1))
continue
leaf = {
"exchange": exchange,
"min_market_cap": band_min,
"max_market_cap": band_max,
"row_count": len(payload),
# Contract 3.5 validators read `rows_fetched` per band.
"rows_fetched": len(payload),
"provider_exhausted": not saturated,
"depth": depth,
}
leaves.append(leaf)
if saturated:
saturated_leaves.append(leaf)
for item in payload:
normalized = normalize_listing(item, exchange)
if normalized is not None:
rows.append(normalized)
# Prefer one canonical listing row per ticker. Dual-listed or provider
# duplicate rows are resolved by the requested exchange order, then by the
# row with the larger market-cap observation.
exchange_rank = {value: idx for idx, value in enumerate(EXCHANGES)}
deduped: dict[str, dict[str, Any]] = {}
for row in rows:
symbol = _symbol(row)
current = deduped.get(symbol)
if current is None:
deduped[symbol] = row
continue
new_key = (
-exchange_rank.get(str(row.get("exchange") or "").upper(), 99),
_first_number(row, "market_cap") or 0.0,
)
old_key = (
-exchange_rank.get(str(current.get("exchange") or "").upper(), 99),
_first_number(current, "market_cap") or 0.0,
)
if new_key > old_key:
deduped[symbol] = row
normalized_rows = sorted(deduped.values(), key=lambda row: _symbol(row))
audit = {
"method": "adaptive_market_cap_bands",
"retrieval_scope_explicit": True,
"pagination_exhausted": not saturated_leaves,
"requested_exchanges": list(EXCHANGES),
"retrieved_exchanges": sorted({str(row.get("exchange") or "").upper() for row in leaves}),
"requested_min_market_cap": min_market_cap,
"requested_max_market_cap": max_market_cap,
"min_price": min_price,
"page_limit": page_limit,
"query_count": query_count,
"row_count": len(normalized_rows),
"bands": sorted(
leaves,
key=lambda row: (
str(row["exchange"]),
float(row["min_market_cap"]),
float(row["max_market_cap"]),
),
),
"saturated_leaf_count": len(saturated_leaves),
"enumeration_verified": not saturated_leaves,
}
return normalized_rows, audit
def market_cap_bucket(value: float) -> str:
bounds = (
500_000_000,
1_000_000_000,
2_000_000_000,
5_000_000_000,
10_000_000_000,
20_000_000_000,
)
for low, high in zip(bounds, bounds[1:]):
if low <= value < high:
return f"{low}-{high}"
return "other"
def classify_cyclicality(sector: str | None, industry: str | None) -> int:
text = f"{sector or ''} {industry or ''}".lower()
for score, needles in CYCLICAL_RULES:
if any(needle in text for needle in needles):
return score
if "financial" in text or "bank" in text or "insurance" in text:
return 3
return 2
def merge_bulk_rows(
rows_by_dataset: Mapping[str, Sequence[Mapping[str, Any]]],
) -> dict[str, dict[str, Any]]:
merged: dict[str, dict[str, Any]] = defaultdict(dict)
for dataset, rows in rows_by_dataset.items():
for row in rows:
symbol = _symbol(row)
if symbol == "UNKNOWN":
continue
merged[symbol][dataset] = dict(row)
return dict(merged)
def collect_bulk_annual_estimates(
client: FMPClient,
*,
endpoint: str | None,
universe_symbols: set[str],
analysis_as_of: datetime,
year_count: int,
) -> tuple[dict[str, list[dict[str, Any]]], dict[str, Any]]:
"""Collect annual estimate rows in bulk when the provider plan supports it."""
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
calls: list[dict[str, Any]] = []
if not endpoint or year_count <= 0:
return {}, {"available": False, "reason": "endpoint_not_configured", "calls": []}
for year in range(analysis_as_of.year, analysis_as_of.year + year_count):
rows = client.get_bulk_dataset(endpoint, year=year, period="annual")
calls.append({"year": year, "row_count": len(rows)})
for raw in rows:
symbol = _symbol(raw)
if symbol in universe_symbols:
grouped[symbol].append(dict(raw))
for values in grouped.values():
values.sort(key=lambda row: str(row.get("date") or row.get("period_end") or ""))
coverage_pct = (len(grouped) / len(universe_symbols) * 100.0) if universe_symbols else 0.0
return dict(grouped), {
"available": bool(grouped),
"covered_symbol_count": len(grouped),
"universe_symbol_count": len(universe_symbols),
"coverage_pct": coverage_pct,
"calls": calls,
}
def normalize_estimate_frame(
listings: Sequence[Mapping[str, Any]],
estimates_by_symbol: Mapping[str, Sequence[Mapping[str, Any]]],
*,
analysis_as_of: datetime,
source_id: str,
config: Mapping[str, Any],
) -> list[dict[str, Any]]:
output: list[dict[str, Any]] = []
for listing in listings:
symbol = _symbol(listing)
normalized = normalize_symbol(
symbol,
estimates_by_symbol.get(symbol, []),
listing,
analysis_as_of=analysis_as_of,
estimate_as_of=analysis_as_of,
source_ids=[source_id],
minimum_analysts=int(config["minimum_discovery_analyst_count"]),
max_dispersion_pct=float(config["maximum_forward_eps_dispersion_pct"]),
max_fy1_horizon_days=int(config["maximum_fy1_horizon_days"]),
forward_pe_tolerance_pct=float(config["forward_pe_reconciliation_tolerance_pct"]),
)
normalized["cyclicality_score"] = classify_cyclicality(
_text(normalized.get("sector")), _text(normalized.get("industry"))
)
output.append(normalized)
return output
def _metric_from_bulk(bundle: Mapping[str, Any], *keys: str) -> float | None:
for dataset in ("ratios_ttm", "key_metrics_ttm", "income_growth"):
row = bundle.get(dataset)
if not isinstance(row, Mapping):
continue
value = _first_number(row, *keys)
if value is not None:
return value
return None
def enrich_listing_from_bulk(row: Mapping[str, Any], bundle: Mapping[str, Any]) -> dict[str, Any]:
result = dict(row)
result["roic_pct"] = _metric_from_bulk(
bundle, "returnOnInvestedCapital", "roic", "returnOnInvestedCapitalTTM"
)
roic = _number(result.get("roic_pct"))
if roic is not None and abs(roic) <= 2:
result["roic_pct"] = roic * 100.0
result["ev_to_fcf"] = _metric_from_bulk(
bundle, "enterpriseValueOverFCF", "enterpriseValueOverFreeCashFlow", "evToFreeCashFlow"
)
result["net_debt_to_ebitda"] = _metric_from_bulk(bundle, "netDebtToEBITDA", "netDebtToEbitda")
result["revenue_growth_pct"] = _metric_from_bulk(
bundle, "growthRevenue", "revenueGrowth", "revenueGrowthTTM"
)
result["eps_growth_pct"] = _metric_from_bulk(bundle, "growthEPS", "epsgrowth", "epsGrowth")
result["dilution_pct"] = _metric_from_bulk(
bundle, "weightedAverageShsOutDilGrowth", "weightedAverageSharesGrowth"
)
fcf_yield = _metric_from_bulk(bundle, "freeCashFlowYield", "fcfYield")
if fcf_yield is not None and abs(fcf_yield) <= 2:
fcf_yield *= 100.0
result["fcf_yield_pct"] = fcf_yield
result["cyclicality_score"] = classify_cyclicality(
_text(result.get("sector")), _text(result.get("industry"))
)
return result
def pre_enrichment_score(row: Mapping[str, Any]) -> float:
score = 0.0
pe = _first_number(row, "priceEarningsRatio", "peRatio", "pe")
if pe is not None and pe > 0:
score += max(-20.0, 30.0 - min(pe, 60.0))
eps_growth = _first_number(row, "eps_growth_pct")
revenue_growth = _first_number(row, "revenue_growth_pct")
roic = _first_number(row, "roic_pct")
fcf_yield = _first_number(row, "fcf_yield_pct")
if eps_growth is not None:
score += max(-10.0, min(eps_growth, 40.0))
if revenue_growth is not None:
score += max(-5.0, min(revenue_growth, 20.0)) * 0.5
if roic is not None:
score += max(-5.0, min(roic, 30.0)) * 0.3
if fcf_yield is not None:
score += max(-5.0, min(fcf_yield, 15.0))
volume = _first_number(row, "volume") or 0.0
price = _first_number(row, "price") or 0.0
if volume > 0 and price > 0:
# Uncapped: this is a ranking key for seed/liquidity selection, not a
# bounded score, so large-cap/high-volume names must not tie here.
score += math.log10(max(volume * price, 1.0))
return round(score, 6)
def _cell_key(row: Mapping[str, Any]) -> tuple[str, str]:
sector = (_text(row.get("sector")) or "Unknown").lower()
cap = _first_number(row, "market_cap") or 0.0
return sector, market_cap_bucket(cap)
def _stable_tie_break(symbol: str, run_salt: str) -> str:
return hashlib.sha256(f"{run_salt}:{symbol}".encode()).hexdigest()
def _apportion_quota(
sizes: Mapping[tuple[str, str], int], limit: int
) -> dict[tuple[str, str], int]:
"""Hamilton (largest-remainder) apportionment of ``limit`` seats across cells.
Every non-empty cell gets at least one seat (capped by its own size);
weight is ``sqrt(cell size)`` so large cells still get proportionally
more seats without a single mega-cell crowding out everything else.
Deterministic and independent of dict/cell iteration order.
"""
keys = sorted(sizes)
if not keys:
return {}
if limit < len(keys):
ordered = sorted(keys, key=lambda key: (-sizes[key], key))
chosen = set(ordered[:limit])
return {key: (1 if key in chosen else 0) for key in keys}
weights = {key: math.sqrt(max(sizes[key], 1)) for key in keys}
total_weight = sum(weights.values()) or 1.0
ideal = {key: weights[key] / total_weight * limit for key in keys}
counts = {key: min(sizes[key], max(1, int(math.floor(ideal[key])))) for key in keys}
allocated = sum(counts.values())
remainder = limit - allocated
if remainder > 0:
# Largest fractional remainder first; skip cells already at capacity.
by_fraction = sorted(keys, key=lambda key: (-(ideal[key] - math.floor(ideal[key])), key))
while remainder > 0:
progressed = False
for key in by_fraction:
if remainder <= 0:
break
if counts[key] < sizes[key]:
counts[key] += 1
remainder -= 1
progressed = True
if not progressed:
break
elif remainder < 0:
# Defensive: only reachable if the min-1-per-cell floor overshot the
# limit while sizes still permitted trimming (rare, tiny-cell case).
by_excess = sorted(keys, key=lambda key: (-(counts[key] - 1), key))
idx = 0
while remainder < 0 and idx < len(by_excess) * 4:
key = by_excess[idx % len(by_excess)]
if counts[key] > 1:
counts[key] -= 1
remainder += 1
idx += 1
return counts
_ECONOMIC_SEED_FIELDS = ("eps_growth_pct", "revenue_growth_pct", "roic_pct", "fcf_yield_pct")
def diversified_seed(
rows: Sequence[Mapping[str, Any]],
limit: int,
*,
run_salt: str,
seed_limit_configured: int | None = None,
reserved_calls: int | None = None,
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
"""Stratified sector x market-cap seed selection for the estimate fallback path.
Replaces the prior dict-order round-robin (which biased the tail of the
seed toward alphabetically-early cells) with a deterministic
sqrt-weighted Hamilton apportionment and a within-cell ranking that never
falls back to the raw ticker string.
"""
cells: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
for raw in rows:
row = dict(raw)
cells[_cell_key(row)].append(row)
sizes = {key: len(values) for key, values in cells.items()}
quotas = _apportion_quota(sizes, limit)
hash_tie_break_used_count = 0
selected: list[dict[str, Any]] = []
for key in sorted(cells):
values = cells[key]
def _sort_key(row: Mapping[str, Any]) -> tuple[float, float, float, int, str]:
price = _first_number(row, "price") or 0.0
volume = _first_number(row, "volume") or 0.0
dollar_volume = price * volume
market_cap = _first_number(row, "market_cap") or 0.0
missing = sum(1 for field in ("price", "volume") if _first_number(row, field) is None)
return (
-pre_enrichment_score(row),
-dollar_volume,
-market_cap,
missing,
_stable_tie_break(_symbol(row), run_salt),
)
values.sort(key=_sort_key)
quota = quotas.get(key, 0)
# Count ties across the whole ranked cell (not just the chosen slice):
# the hash decides ordering at the selection boundary too, so a tie
# straddling `quota` still used the hash even though only one side
# of it was kept.
for idx in range(1, len(values)):
base_prev = _sort_key(values[idx - 1])[:-1]
base_cur = _sort_key(values[idx])[:-1]
if base_prev == base_cur:
hash_tie_break_used_count += 1
selected.extend(values[:quota])
economic_rows_with_data = sum(
1
for raw in rows
if any(_first_number(raw, field) is not None for field in _ECONOMIC_SEED_FIELDS)
)
economic_metrics_available_for_seed = bool(rows) and (
economic_rows_with_data / len(rows) >= 0.5
)
seed_audit = {
"seed_selection_basis": (
"stratified_economic_score"
if economic_metrics_available_for_seed
else "stratified_liquidity_proxy"
),
"economic_metrics_available_for_seed": economic_metrics_available_for_seed,
"cell_count": len(cells),
"quota_method": "sqrt_hamilton",
"alphabetic_tie_break_used_count": 0,
"hash_tie_break_used_count": hash_tie_break_used_count,
"seed_limit_configured": seed_limit_configured
if seed_limit_configured is not None
else limit,
"seed_limit_effective": limit,
"reserved_calls": reserved_calls if reserved_calls is not None else 0,
}
return selected, seed_audit
def compute_effective_seed_limit(
*,
pre_enrichment_limit: int,
seed_limit_cap: int,
max_api_calls: int,
api_calls_made: int,
quality_probe_limit: int,
exact_liquidity_limit: int,
candidate_packet_reserve_calls: int,
retry_reserve_calls: int,
) -> int:
"""Bound the per-symbol estimate fallback seed by the remaining call budget.
Reserves calls for the quality probe, exact-liquidity lookups, candidate
packet generation, and retries so the seed step cannot starve the rest of
the pipeline. Raises ``ValueError`` (caught by the existing failure-JSON
path in ``main``) when the remaining budget cannot afford a floor of 20
seeds -- silently truncating below that floor would make the discovery
pool too thin to be meaningful.
"""
reserved = (
int(quality_probe_limit) * 2 # key-metrics-ttm + annual income statement
+ int(exact_liquidity_limit)
+ int(candidate_packet_reserve_calls)
+ int(retry_reserve_calls)
)
remaining = int(max_api_calls) - int(api_calls_made)
effective = min(int(pre_enrichment_limit), int(seed_limit_cap), remaining - reserved)
if effective < 20:
raise ValueError(
"estimate seed budget insufficient: effective_limit="
f"{effective} < 20 (pre_enrichment_limit={pre_enrichment_limit}, "
f"seed_limit_cap={seed_limit_cap}, remaining_calls={remaining}, "
f"reserved_calls={reserved})"
)
return effective
def economic_scope_complete(
*, estimate_acquisition_mode: str, covered_symbol_count: int, universe_symbol_count: int
) -> bool:
"""True only when bulk estimates covered EVERY listing-universe symbol.
Completeness is an exact symbol-count equality, deliberately not a
configurable percentage: any ratio threshold (99%, 99.99%, or a user
lowering it to 20%) still declares "complete / exhausted" while uncovered
symbols remain, which is the misstatement this field exists to prevent.
Using the bulk route (allowed from ``bulk_estimate_minimum_coverage_pct``,
20%) says nothing about completeness.
"""
return (
estimate_acquisition_mode == "analyst_estimates_bulk"
and int(universe_symbol_count) > 0
and int(covered_symbol_count) >= int(universe_symbol_count)
)
_SECTOR_METRIC_FIELDS = (
"sector_forward_multiple",
"p_to_tbv",
"p_to_affo",
"p_to_book",
"sector_per_share_growth_pct",
"affo_per_share_growth_pct",
"tbv_per_share_growth_pct",
"sector_adjusted_net_debt_to_ebitda",
)
# Profiles whose missing sector metrics BLOCK screening (capital_markets is
# labelled but valued on ordinary multiples, so it is deliberately absent).
_BLOCKED_SECTOR_PROFILES = frozenset(SECTOR_PROFILES) | {"auto_dealership"}
def mark_sector_profile_exhaustion(
rows: Sequence[Mapping[str, Any]], *, source_id: str
) -> list[dict[str, Any]]:
"""Declare enrichment exhaustion for sector-profile rows the provider cannot serve.
Balance-sheet businesses (REITs, banks, insurers, BDCs, MLPs, asset
managers, auto dealers) need sector-specific valuation evidence
(P/FFO-AFFO, P/TBV, adjusted leverage) that the direct-FMP discovery path
does not provide. Without this declaration such rows stay
``needs_enrichment`` forever and the run cannot complete; with it,
``screen_universe`` resolves them as ``unavailable_after_enrichment`` —
still visible in the audit, honestly excluded from selection, and eligible
for manual sector underwriting in a scoped follow-up.
"""
output: list[dict[str, Any]] = []
for raw in rows:
row = dict(raw)
profile = _text(row.get("sector_profile_type")) or "general"
has_sector_metrics = any(
_first_number(row, field) is not None for field in _SECTOR_METRIC_FIELDS
)
if profile in _BLOCKED_SECTOR_PROFILES and not has_sector_metrics:
row.setdefault("enrichment_attempted", True)
row["enrichment_exhausted"] = True
row["enrichment_exhaustion_reason"] = (
f"sector-specific valuation metrics for profile '{profile}' are not "
"available from the direct-FMP discovery path"
)
sources = [
value
for value in (row.get("enrichment_source_ids") or [])
if isinstance(value, str) and value.strip()
]
if source_id not in sources:
sources.append(source_id)
row["enrichment_source_ids"] = sources
output.append(row)
return output
def mark_unit_reconciliation_exhaustion(
rows: Sequence[Mapping[str, Any]], *, source_id: str
) -> list[dict[str, Any]]:
"""Declare exhaustion for foreign issuers the direct-FMP path cannot reconcile.
A non-US issuer's USD listing price and local-currency (or differently
ratioed ADS) statements make every valuation ratio meaningless until the
listing/statement currencies and the ordinary-shares-per-ADS ratio are
verified (QFIN: forward P/E 0.45x, FCF yield 94% — unit mismatch, not deep
value). The direct-FMP discovery path carries no such evidence, so the
``unit_reconciliation_required`` blocking review would otherwise leave the
row ``needs_enrichment`` forever; declaring exhaustion lets
``screen_universe`` resolve it as ``unavailable_after_enrichment`` —
audited, excluded from selection, open to manual reconciliation.
"""
output: list[dict[str, Any]] = []
for raw in rows:
row = dict(raw)
if requires_unit_reconciliation(row):
row.setdefault("enrichment_attempted", True)
row["enrichment_exhausted"] = True
row["enrichment_exhaustion_reason"] = (
"listing/statement currency and ADS-unit reconciliation is not "
"available from the direct-FMP discovery path"
)
sources = [
value
for value in (row.get("enrichment_source_ids") or [])
if isinstance(value, str) and value.strip()
]
if source_id not in sources:
sources.append(source_id)
row["enrichment_source_ids"] = sources
output.append(row)
return output
def _is_foreign_private_issuer(row: Mapping[str, Any]) -> bool:
"""Heuristic FPI flag: ISIN country prefix when available, else listing country."""
isin = _text(row.get("isin"))
if isin:
return not isin.upper().startswith("US")
country = _text(row.get("country"))
if country:
return country.upper() != "US"
return False
def _verified_annual_actual(
statements: Sequence[Mapping[str, Any]], *, analysis_as_of: datetime
) -> tuple[float | None, str | None]:
"""Latest reported annual diluted EPS whose period AND filing precede analysis_as_of."""
best: tuple[str, float] | None = None
for raw in statements:
if not isinstance(raw, Mapping):
continue
if (_text(raw.get("period")) or "FY").upper() != "FY":
continue
period_end = _text(raw.get("date"))
# Only acceptedDate qualifies, and only with a time-of-day component:
# a bare date (or the filingDate field, which is date-only) cannot
# prove the filing preceded an intraday analysis_as_of, so it is
# treated as unknown and rejected (fail closed).
accepted = _text(raw.get("acceptedDate"))
eps = _first_number(raw, "epsDiluted", "eps")
if not period_end or not accepted or eps is None:
continue
if len(accepted.strip()) < 16:
continue
try:
end_dt = datetime.fromisoformat(period_end[:10]).replace(tzinfo=timezone.utc)
acc_dt = datetime.fromisoformat(accepted[:19].replace(" ", "T"))
except ValueError:
continue
if acc_dt.tzinfo is None:
# FMP's acceptedDate is the SEC acceptance clock: US/Eastern.
# Treating "17:23:10" as UTC would read it as 13:23 ET and leak a
# filing 4-5 hours before it was actually public.
acc_dt = acc_dt.replace(tzinfo=SEC_FILING_TZ)
acc_dt = acc_dt.astimezone(timezone.utc)
if end_dt > analysis_as_of or acc_dt > analysis_as_of:
continue
if best is None or period_end > best[0]:
best = (period_end, float(eps))
return (best[1], best[0]) if best else (None, None)
def apply_quality_probe(
client: Any,
rows: Sequence[Mapping[str, Any]],
*,
target_symbols: Sequence[str],
source_id: str,
analysis_as_of: datetime | None = None,
actual_source_id: str | None = None,
return_partial_on_budget: bool = False,
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
"""Enrich the top-ranked lane candidates with TTM quality metrics.
A failed/empty probe (4xx or no data) is recorded as
``quality_probe_resolved: False`` on that row rather than failing the run.
Full-snapshot screening may opt into returning rows and progress when the
provider call budget is exhausted; the bounded pipeline keeps the historic
exception behavior by default. Counts use FMP provider-budget counter deltas
(including consumed retries, excluding cache hits and unsent refusals). Clients
without that counter fall back to completed logical calls.
"""
ordered_targets = _ordered_unique_symbols(target_symbols)
targets = set(ordered_targets)
attempted: list[str] = []
resolved: list[str] = []
actual_resolved: list[str] = []
actual_calls = 0
metrics_calls = 0
completed: set[str] = set()
interrupted: list[str] = []
budget_error: str | None = None
interrupted_at: dict[str, str] | None = None
output: list[dict[str, Any]] = []
actual_required = analysis_as_of is not None and hasattr(client, "get_income_statement")
for raw in rows:
row = dict(raw)
symbol = _symbol(row)
if symbol in targets:
if budget_error is not None:
row.setdefault("quality_probe_attempted", False)
row.setdefault("quality_probe_resolved", False)
row.setdefault("quality_probe_complete", False)
output.append(row)
continue
attempted.append(symbol)
row["quality_probe_attempted"] = True
row["quality_probe_complete"] = False
calls_before = getattr(client, "api_calls_made", None)
completed_call = False
try:
payload = client.get_key_metrics_ttm(symbol)
completed_call = True
except ApiCallBudgetExceeded as exc:
if not return_partial_on_budget:
raise
budget_error = str(exc)
interrupted.append(symbol)
interrupted_at = {"symbol": symbol, "operation": "get_key_metrics_ttm"}
row["quality_probe_budget_exhausted"] = True
row["quality_probe_resolved"] = False
output.append(row)
continue
finally:
metrics_calls += (
client.api_calls_made - calls_before
if calls_before is not None
else int(completed_call)
)
bundle = payload[0] if payload else None
if isinstance(bundle, Mapping):
roic = _first_number(bundle, "returnOnInvestedCapitalTTM")
fcf_yield = _first_number(bundle, "freeCashFlowYieldTTM")
ev_to_fcf = _first_number(bundle, "evToFreeCashFlowTTM")
nd_ebitda = _first_number(bundle, "netDebtToEBITDATTM")
sbc = _first_number(bundle, "stockBasedCompensationToRevenueTTM")
if roic is not None:
row["roic_pct"] = roic * 100.0
if fcf_yield is not None:
row["fcf_yield_pct"] = fcf_yield * 100.0
if ev_to_fcf is not None:
row["ev_to_fcf"] = ev_to_fcf
if nd_ebitda is not None:
row["net_debt_to_ebitda"] = nd_ebitda
if sbc is not None:
row["sbc_revenue_pct"] = sbc * 100.0
# SBC-adjusted FCF yield on the SAME denominator as fcf_yield
# (market cap): (FCF - SBC) / mktcap = fcf_yield - SBC/revenue * revenue/mktcap,
# with revenue recovered from EV / (EV/Sales). Never subtract a
# revenue-based ratio from a market-cap-based yield directly.
ev = _first_number(bundle, "enterpriseValueTTM")
ev_to_sales = _first_number(bundle, "evToSalesTTM")
market_cap = _first_number(row, "market_cap")
if (
fcf_yield is not None
and sbc is not None
and ev is not None
and ev_to_sales
and market_cap
):
revenue = ev / ev_to_sales
row["sbc_adjusted_fcf_yield_pct"] = (
fcf_yield - sbc * revenue / market_cap
) * 100.0
row["quality_probe_source_ids"] = [source_id]
row["quality_probe_resolved"] = True
resolved.append(symbol)
else:
row["quality_probe_resolved"] = False
# Verified reported EPS: one annual income-statement call, accepted
# at or before analysis_as_of. Without it the growth-basis fields
# stay fail-closed (unknown) rather than borrowing a consensus row.
if actual_required:
calls_before = getattr(client, "api_calls_made", None)
completed_call = False
try:
statements = client.get_income_statement(symbol, period="annual", limit=2)
completed_call = True
except ApiCallBudgetExceeded as exc:
if not return_partial_on_budget:
raise
budget_error = str(exc)
interrupted.append(symbol)
interrupted_at = {"symbol": symbol, "operation": "get_income_statement"}
row["quality_probe_budget_exhausted"] = True
output.append(row)
continue
finally:
actual_calls += (
client.api_calls_made - calls_before
if calls_before is not None
else int(completed_call)
)
actual_eps, actual_end = _verified_annual_actual(
statements or [], analysis_as_of=analysis_as_of
)
row = apply_verified_actual_eps(
row,
actual_eps=actual_eps,
period_end=actual_end,
analysis_as_of=analysis_as_of,
source_ids=[actual_source_id] if actual_source_id else [],
)
if row.get("latest_actual_verified"):
actual_resolved.append(symbol)
row["quality_probe_complete"] = True
completed.add(symbol)
else:
row.setdefault("quality_probe_attempted", False)
row.setdefault("quality_probe_resolved", False)
row.setdefault("quality_probe_complete", False)
output.append(row)
attempted_symbols = [symbol for symbol in ordered_targets if symbol in set(attempted)]
completed_symbols = [symbol for symbol in ordered_targets if symbol in completed]
pending_symbols = [symbol for symbol in ordered_targets if symbol not in completed]
interrupted_symbols = [symbol for symbol in ordered_targets if symbol in set(interrupted)]
audit = {
"attempted": attempted_symbols,
"resolved": resolved,
"symbols": attempted_symbols,
"source_id": source_id,
"calls_used": metrics_calls + actual_calls,
"actual_eps_source_id": actual_source_id,
"actual_eps_calls": actual_calls,
"actual_eps_resolved": actual_resolved,
"budget_exhausted": budget_error is not None,
"attempted_symbols": attempted_symbols,
"completed_symbols": completed_symbols,
"pending_symbols": pending_symbols,
"interrupted_symbols": interrupted_symbols,
"interrupted_at": interrupted_at,
"error": budget_error,
}
return output, audit
def _best_lane_score(row: Mapping[str, Any], lanes: Sequence[str], *, analysis_as_of: str) -> float:
if not lanes:
return float("-inf")
return max(_lane_score(row, lane, analysis_as_of=analysis_as_of) for lane in lanes)
def select_quality_probe_targets(
rows: Sequence[Mapping[str, Any]],
lane_membership_map: Mapping[str, Sequence[str]],
*,
limit: int,
analysis_as_of: str,
) -> list[str]:
"""Rank the union of lane rows by best lane score; return the top ``limit`` symbols."""
ranked = sorted(
(row for row in rows if lane_membership_map.get(_symbol(row))),
key=lambda row: (
-_best_lane_score(
row, lane_membership_map.get(_symbol(row), []), analysis_as_of=analysis_as_of
),
_symbol(row),
),
)
return [_symbol(row) for row in ranked[:limit]]
def prior_business_days(as_of: date, count: int) -> list[date]:
days: list[date] = []
current = as_of
while len(days) < count:
if current.weekday() < 5:
days.append(current)
current -= timedelta(days=1)
return days
def apply_bulk_liquidity(
client: FMPClient,
rows: Sequence[Mapping[str, Any]],
*,
as_of: date,
endpoint: str,
source_id: str,
required_days: int = 20,
) -> tuple[list[dict[str, Any]], bool]:
volumes: dict[str, list[float]] = defaultdict(list)
successful_dates = 0
for day in prior_business_days(as_of, required_days + 8):
data = client.get_bulk_dataset(endpoint, date=day.isoformat())
if not data:
if successful_dates == 0:
return [dict(row) for row in rows], False
continue
successful_dates += 1
for item in data:
symbol = _symbol(item)
volume = _first_number(item, "volume")
if volume is not None and volume > 0:
volumes[symbol].append(volume)
if successful_dates >= required_days:
break
if successful_dates < required_days:
return [dict(row) for row in rows], False
output: list[dict[str, Any]] = []
for raw in rows:
row = dict(raw)
symbol = _symbol(row)
values = volumes.get(symbol, [])[:required_days]
price = _first_number(row, "price")
if len(values) >= required_days and price is not None and price > 0:
avg = sum(values) / len(values)
row.update(
{
"average_volume": avg,
"average_volume_period_days": required_days,
"average_daily_dollar_volume": avg * price,
"average_daily_dollar_volume_method": "price_x_20d_average_volume",
"liquidity_source_ids": [source_id],
}
)
output.append(row)
return output, True
def apply_symbol_liquidity(
client: FMPClient,
rows: Sequence[Mapping[str, Any]],
*,
as_of: date,
source_id: str,
limit: int,
required_days: int = 20,
target_symbols: Sequence[str] | None = None,
) -> list[dict[str, Any]]:
ranked = sorted(
(dict(row) for row in rows), key=lambda row: (-pre_enrichment_score(row), _symbol(row))
)
chosen = (
{str(value).upper() for value in target_symbols if str(value).strip()}
if target_symbols is not None
else {_symbol(row) for row in ranked[:limit]}
)
if len(chosen) > limit:
ordered = [_symbol(row) for row in ranked if _symbol(row) in chosen]
chosen = set(ordered[:limit])
start = (as_of - timedelta(days=45)).isoformat()
end = as_of.isoformat()
output: list[dict[str, Any]] = []
for raw in rows:
row = dict(raw)
symbol = _symbol(row)
if symbol not in chosen:
output.append(row)
continue
history = client.get_historical_prices(symbol, from_date=start, to_date=end)
volumes = [
value
for item in history
if (value := _first_number(item, "volume")) is not None and value > 0
][:required_days]
price = _first_number(row, "price")
if len(volumes) >= required_days and price is not None and price > 0:
avg = sum(volumes) / len(volumes)
row.update(
{
"average_volume": avg,
"average_volume_period_days": required_days,
"average_daily_dollar_volume": avg * price,
"average_daily_dollar_volume_method": "price_x_20d_average_volume",
"liquidity_source_ids": [source_id],
}
)
output.append(row)
return output
def select_liquidity_targets(
rows: Sequence[Mapping[str, Any]],
config: Mapping[str, Any],
*,
limit: int,
) -> list[str]:
"""Choose exact-liquidity work by opportunity lane, not ticker order."""
lane_rows: dict[str, list[dict[str, Any]]] = {lane: [] for lane in ALLOWED_LANES}
for raw in rows:
row = dict(raw)
for lane in lane_memberships(row, config):
lane_rows[lane].append(row)
for lane in lane_rows:
lane_rows[lane].sort(key=lambda row: (-pre_enrichment_score(row), _symbol(row)))
selected: list[str] = []
seen: set[str] = set()
pointers = {lane: 0 for lane in ALLOWED_LANES}
ordered_lanes = (
"core_garp",
"high_growth_exception",
"quality_near_miss",
"cyclical_normalization",
)
while len(selected) < limit:
added = False
for lane in ordered_lanes:
values = lane_rows[lane]
pointer = pointers[lane]
while pointer < len(values) and _symbol(values[pointer]) in seen:
pointer += 1
pointers[lane] = pointer
if pointer >= len(values):
continue
symbol = _symbol(values[pointer])
pointers[lane] = pointer + 1
selected.append(symbol)
seen.add(symbol)
added = True
if len(selected) >= limit:
break
if not added:
break
if len(selected) < limit:
remaining = sorted(rows, key=lambda row: (-pre_enrichment_score(row), _symbol(row)))
for row in remaining:
symbol = _symbol(row)
if symbol not in seen:
selected.append(symbol)
seen.add(symbol)
if len(selected) >= limit:
break
return selected
def lane_memberships(row: Mapping[str, Any], config: Mapping[str, Any]) -> list[str]:
pe = _first_number(row, "forward_pe", "fy1_pe")
growth = _first_number(row, "eps_growth_pct", "per_share_growth_pct")
revenue = _first_number(row, "revenue_growth_pct")
cyclicality = int(_first_number(row, "cyclicality_score") or 2)
lanes: list[str] = []
if pe is not None and growth is not None:
if pe <= float(config["preferred_forward_pe"]) and growth >= float(
config["minimum_per_share_growth_pct"]
):
lanes.append("core_garp")
if pe <= float(config["high_growth_exception_max_forward_pe"]) and growth >= float(
config["high_growth_exception_growth_pct"]
):
lanes.append("high_growth_exception")
if pe <= float(config["near_miss_max_forward_pe"]) and growth >= float(
config["near_miss_min_per_share_growth_pct"]
):
if (
revenue is None
or revenue < float(config["minimum_revenue_growth_pct"])
or growth < float(config["minimum_per_share_growth_pct"])
):
lanes.append("quality_near_miss")
if (
cyclicality >= 3
and pe <= float(config["high_growth_exception_max_forward_pe"])
and growth >= float(config["near_miss_min_per_share_growth_pct"])
):
lanes.append("cyclical_normalization")
# Trough-recovery names (FY1 dips below the latest actual before FY3
# recovers) show a headline FY1->FY3 CAGR that masks a current-year
# decline; they do not belong in core_garp but remain visible for
# deep-dive review.
growth_pattern = _text(row.get("growth_pattern"))
lane_set = set(lanes)
if growth_pattern == "trough_recovery" and "core_garp" in lane_set:
lane_set.discard("core_garp")
lane_set.add("quality_near_miss")
return sorted(lane_set)
def compact_candidate(row: Mapping[str, Any]) -> dict[str, Any]:
return {
"symbol": _symbol(row),
"company_name": row.get("company_name"),
"sector": row.get("sector"),
"industry": row.get("industry"),
"price": row.get("price"),
"market_cap": row.get("market_cap"),
"forward_pe": row.get("forward_pe"),
"forward_eps": row.get("forward_eps"),
"forward_fiscal_year": row.get("forward_fiscal_year"),
"analyst_count": row.get("analyst_count"),
"eps_growth_pct": row.get("eps_growth_pct"),
"revenue_growth_pct": row.get("revenue_growth_pct"),
"roic_pct": row.get("roic_pct"),
"fcf_yield_pct": row.get("fcf_yield_pct"),
"ev_to_fcf": row.get("ev_to_fcf"),
"net_debt_to_ebitda": row.get("net_debt_to_ebitda"),
"cyclicality_score": row.get("cyclicality_score"),
"provider_prefilter_lanes": row.get("provider_prefilter_lanes", []),
}
def _project_rows(
rows: Sequence[Mapping[str, Any]],
fields: Sequence[str],
*,
limit: int,
) -> list[dict[str, Any]]:
output: list[dict[str, Any]] = []
for raw in list(rows)[:limit]:
item = {key: raw.get(key) for key in fields if key in raw}
output.append(item)
return output
def _cash_flow_ttm_preview(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
quarters = list(rows)[:4]
if len(quarters) < 4:
return {"available": False, "reason": "fewer_than_four_provider_quarters"}
ocf_values: list[float] = []
capex_values: list[float] = []
sbc_values: list[float] = []
for row in quarters:
ocf = _first_number(row, "operatingCashFlow", "netCashProvidedByOperatingActivities")
capex = _first_number(row, "capitalExpenditure", "capitalExpenditures")
if ocf is None or capex is None:
return {"available": False, "reason": "provider_quarter_missing_ocf_or_capex"}
ocf_values.append(ocf)
capex_values.append(abs(capex))
sbc_values.append(_first_number(row, "stockBasedCompensation") or 0.0)
ocf_ttm = sum(ocf_values)
capex_ttm = sum(capex_values)
return {
"available": True,
"method": "sum_4_provider_quarters_preview",
"primary_source_verified": False,
"operating_cash_flow": ocf_ttm,
"capex_cash_outflow": capex_ttm,
"standard_fcf": ocf_ttm - capex_ttm,
"stock_based_compensation": sum(sbc_values),
"period_ends": [row.get("date") for row in quarters],
}
def build_fmp_packet(client: FMPClient, row: Mapping[str, Any], output_dir: Path) -> Path:
"""Write a compact provider evidence packet and keep full payloads on disk."""
symbol = _symbol(row)
profile = client.get_profile(symbol)
quote = client.get_quotes([symbol]).get(symbol)
estimates = client.get_analyst_estimates(symbol, period="annual", limit=6)
income_annual = client.get_income_statement(symbol, period="annual", limit=6)
income_quarterly = client.get_income_statement(symbol, period="quarter", limit=8)
balance_annual = client.get_balance_sheet(symbol, period="annual", limit=6)
balance_quarterly = client.get_balance_sheet(symbol, period="quarter", limit=8)
cash_flow_annual = client.get_cash_flow(symbol, period="annual", limit=6)
cash_flow_quarterly = client.get_cash_flow(symbol, period="quarter", limit=8)
key_metrics_ttm = client.get_key_metrics_ttm(symbol)
ratios_ttm = client.get_ratios_ttm(symbol)
peers = client.get_stock_peers(symbol)
raw_dir = output_dir.parent / "provider" / "candidate-data" / symbol
raw_dir.mkdir(parents=True, exist_ok=True)
raw_payloads = {
"profile": profile,
"quote": quote,
"analyst-estimates": estimates,
"income-annual": income_annual,
"income-quarterly": income_quarterly,
"balance-annual": balance_annual,
"balance-quarterly": balance_quarterly,
"cash-flow-annual": cash_flow_annual,
"cash-flow-quarterly": cash_flow_quarterly,
"key-metrics-ttm": key_metrics_ttm,
"ratios-ttm": ratios_ttm,
"peers": peers,
}
evidence_paths: dict[str, str] = {}
for name, payload in raw_payloads.items():
target = raw_dir / f"{name}.json"
_write_json(target, payload)
evidence_paths[name] = str(target.relative_to(output_dir.parent))
required_next_checks = [
"corporate_action_preflight",
"accession_specific_sec_or_official_ir_verification",
"quarter_and_full_year_period_separation",
"standard_fcf_primary_source_reconstruction",
"independent_forecast_bridge",
"peer_basis_alignment",
"sector_and_cycle_normalization",
]
if "foreign_private_issuer_review" in (row.get("provider_prefilter_flags") or []):
required_next_checks.append("form_20f_6k_verification")
income_fields = (
"date",
"calendarYear",
"period",
"reportedCurrency",
"revenue",
"grossProfit",
"operatingIncome",
"ebitda",
"netIncome",
"eps",
"epsdiluted",
"weightedAverageShsOut",
"weightedAverageShsOutDil",
"incomeTaxExpense",
"interestExpense",
"interestIncome",
)
cash_flow_fields = (
"date",
"calendarYear",
"period",
"reportedCurrency",
"netIncome",
"operatingCashFlow",
"capitalExpenditure",
"freeCashFlow",
"stockBasedCompensation",
"commonStockIssued",
"commonStockRepurchased",
"dividendsPaid",
"acquisitionsNet",
)
balance_fields = (
"date",
"calendarYear",
"period",
"reportedCurrency",
"cashAndCashEquivalents",
"shortTermInvestments",
"cashAndShortTermInvestments",
"totalCurrentAssets",
"totalAssets",
"shortTermDebt",
"longTermDebt",
"totalDebt",
"netDebt",
"goodwill",
"intangibleAssets",
"totalStockholdersEquity",
)
estimate_fields = (
"symbol",
"date",
"fiscalYear",
"revenueAvg",
"revenueLow",
"revenueHigh",
"epsAvg",
"epsLow",
"epsHigh",
"numAnalystsRevenue",
"numAnalystsEps",
"ebitdaAvg",
"ebitAvg",
"netIncomeAvg",
"sgaExpenseAvg",
)
packet = {
"runtime": runtime_metadata(),
"symbol": symbol,
"purpose": "provider evidence preview for primary-source underwriting",
"provider_data_is_not_primary_source_verified": True,
"discovery": compact_candidate(row),
"profile": profile,
"quote": {
key: quote.get(key)
for key in (
"symbol",
"name",
"price",
"marketCap",
"timestamp",
"exchange",
"yearHigh",
"yearLow",
"volume",
"avgVolume",
)
if isinstance(quote, Mapping) and key in quote
},
"annual_estimates": _project_rows(estimates, estimate_fields, limit=6),
"income_annual": _project_rows(income_annual, income_fields, limit=4),
"income_quarterly": _project_rows(income_quarterly, income_fields, limit=8),
"balance_latest": _project_rows(
balance_quarterly or balance_annual, balance_fields, limit=2
),
"cash_flow_annual": _project_rows(cash_flow_annual, cash_flow_fields, limit=4),
"cash_flow_quarterly": _project_rows(cash_flow_quarterly, cash_flow_fields, limit=8),
"cash_flow_ttm_preview": _cash_flow_ttm_preview(cash_flow_quarterly),
"key_metrics_ttm": dict(key_metrics_ttm[0]) if key_metrics_ttm else None,
"ratios_ttm": dict(ratios_ttm[0]) if ratios_ttm else None,
"peer_symbols": peers[:10],
"raw_evidence_paths": evidence_paths,
"required_next_checks": required_next_checks,
}
path = output_dir / f"{symbol}.fmp-packet.json"
_write_json(path, packet)
return path
def execute_pipeline(
client: FMPClient,
config: Mapping[str, Any],
*,
analysis_as_of: datetime,
output_dir: Path,
include_packets: bool = True,
) -> PipelineResult:
output_dir.mkdir(parents=True, exist_ok=True)
audit_dir = output_dir / "audit"
raw_dir = output_dir / "provider"
packet_dir = output_dir / "candidate-packets"
audit_dir.mkdir(parents=True, exist_ok=True)
raw_dir.mkdir(parents=True, exist_ok=True)
min_cap = float(config["min_market_cap"])
max_cap = float(config["max_market_cap"])
min_price = float(config["min_price"])
universe_rows, enumeration_audit = collect_listing_universe(
client,
min_market_cap=min_cap,
max_market_cap=max_cap,
min_price=min_price,
page_limit=int(config["company_screener_limit"]),
minimum_band_width=float(config["minimum_market_cap_band_width"]),
maximum_depth=int(config["maximum_market_cap_band_depth"]),
)
if not universe_rows:
raise ValueError("FMP company screener returned no normalized listing rows")
profile_overrides = {
str(key).upper(): str(value)
for key, value in (config.get("sector_profile_overrides") or {}).items()
}
if profile_overrides:
for row in universe_rows:
pinned = profile_overrides.get(str(row.get("symbol", "")).upper())
if pinned:
row["sector_profile_type"] = pinned
_write_json(audit_dir / "listing-enumeration-audit.json", enumeration_audit)
bulk_endpoints = dict(config.get("bulk_endpoints") or {})
bulk_rows: dict[str, list[dict[str, Any]]] = {}
year = analysis_as_of.year - 1
for name, params in (
("ratios_ttm", {}),
("key_metrics_ttm", {}),
("income_growth", {"year": year, "period": "annual"}),
):
endpoint = bulk_endpoints.get(name)
bulk_rows[name] = client.get_bulk_dataset(endpoint, **params) if endpoint else []
_write_json(raw_dir / f"{name}.json", bulk_rows[name])
bulk_merged = merge_bulk_rows(bulk_rows)
enriched_universe = [
enrich_listing_from_bulk(row, bulk_merged.get(_symbol(row), {})) for row in universe_rows
]
eod_endpoint = bulk_endpoints.get("eod")
bulk_liquidity = False
if eod_endpoint:
enriched_universe, bulk_liquidity = apply_bulk_liquidity(
client,
enriched_universe,
as_of=analysis_as_of.date(),
endpoint=eod_endpoint,
source_id=f"fmp-{eod_endpoint}-{analysis_as_of.date().isoformat()}",
required_days=int(config["minimum_average_volume_period_days"]),
)
estimate_source = f"fmp-analyst-estimates-{analysis_as_of.date().isoformat()}"
universe_symbols = {_symbol(row) for row in enriched_universe}
bulk_estimates, bulk_estimate_audit = collect_bulk_annual_estimates(
client,
endpoint=_text(bulk_endpoints.get("analyst_estimates")),
universe_symbols=universe_symbols,
analysis_as_of=analysis_as_of,
year_count=int(config["bulk_estimate_years"]),
)
_write_json(audit_dir / "bulk-estimates-audit.json", bulk_estimate_audit)
bulk_coverage = float(bulk_estimate_audit.get("coverage_pct") or 0.0)
use_bulk_estimates = bool(
bulk_estimates and bulk_coverage >= float(config["bulk_estimate_minimum_coverage_pct"])
)
seed_audit: dict[str, Any] | None = None
if use_bulk_estimates:
estimate_frame = [row for row in enriched_universe if _symbol(row) in bulk_estimates]
estimates_by_symbol = bulk_estimates
estimate_acquisition_mode = "analyst_estimates_bulk"
else:
pre_enrichment_limit = int(config["pre_enrichment_limit"])
effective_seed_limit = compute_effective_seed_limit(
pre_enrichment_limit=pre_enrichment_limit,
seed_limit_cap=int(config["seed_limit_cap"]),
max_api_calls=int(client.max_api_calls),
api_calls_made=int(client.api_calls_made),
quality_probe_limit=int(config["quality_probe_limit"]),
exact_liquidity_limit=int(config["exact_liquidity_limit"]),
candidate_packet_reserve_calls=int(config["candidate_packet_reserve_calls"]),
retry_reserve_calls=int(config["retry_reserve_calls"]),
)
estimate_frame, seed_audit = diversified_seed(
enriched_universe,
effective_seed_limit,
run_salt=analysis_as_of.date().isoformat(),
seed_limit_configured=pre_enrichment_limit,
reserved_calls=(
int(config["quality_probe_limit"]) * 2
+ int(config["exact_liquidity_limit"])
+ int(config["candidate_packet_reserve_calls"])
+ int(config["retry_reserve_calls"])
),
)
_write_json(audit_dir / "seed-audit.json", seed_audit)
estimates_by_symbol: dict[str, list[dict[str, Any]]] = {}
for row in estimate_frame:
symbol = _symbol(row)
estimates_by_symbol[symbol] = client.get_analyst_estimates(
symbol, period="annual", limit=6
)
estimate_acquisition_mode = "bounded_per_symbol_fallback"
normalized_estimates = normalize_estimate_frame(
estimate_frame,
estimates_by_symbol,
analysis_as_of=analysis_as_of,
source_id=estimate_source,
config=config,
)
if not bulk_liquidity:
liquidity_targets = select_liquidity_targets(
normalized_estimates,
config,
limit=int(config["exact_liquidity_limit"]),
)
normalized_estimates = apply_symbol_liquidity(
client,
normalized_estimates,
as_of=analysis_as_of.date(),
source_id=f"fmp-historical-eod-{analysis_as_of.date().isoformat()}",
limit=int(config["exact_liquidity_limit"]),
required_days=int(config["minimum_average_volume_period_days"]),
target_symbols=liquidity_targets,
)
else:
liquidity_targets = [_symbol(row) for row in normalized_estimates]
seed = estimate_frame
universe_sha = _write_jsonl(audit_dir / "universe.jsonl", enriched_universe)
# Quality probe (v3.6.1): rank the union of lane candidates by best lane
# score and pull TTM quality metrics (ROIC, FCF yield, EV/FCF, leverage,
# SBC/revenue) for the top slice before pool selection, so FCF-negative
# candidates like a 126x-EV/FCF miss can be screened out ahead of the
# deep-dive step instead of consuming one of its slots.
lane_membership_map = {
_symbol(row): lane_memberships(row, config) for row in normalized_estimates
}
quality_probe_limit = int(config["quality_probe_limit"])
quality_probe_targets = select_quality_probe_targets(
normalized_estimates,
lane_membership_map,
limit=quality_probe_limit,
analysis_as_of=analysis_as_of.isoformat(),
)
quality_probe_source_id = f"fmp-key-metrics-ttm-{analysis_as_of.date().isoformat()}"
normalized_estimates, quality_probe_audit = apply_quality_probe(
client,
normalized_estimates,
target_symbols=quality_probe_targets,
source_id=quality_probe_source_id,
analysis_as_of=analysis_as_of,
actual_source_id=f"fmp-income-statement-annual-{analysis_as_of.date().isoformat()}",
)
_write_json(audit_dir / "quality-probe-audit.json", quality_probe_audit)
normalized_estimates = mark_sector_profile_exhaustion(
normalized_estimates, source_id=quality_probe_source_id
)
normalized_estimates = mark_unit_reconciliation_exhaustion(
normalized_estimates, source_id=quality_probe_source_id
)
# growth_pattern is fixed at normalization (consensus basis) and the probe
# never relabels it; the recompute below is defensive so pool lane tags
# always match what lane_memberships would produce for the final rows.
lane_membership_map = {
_symbol(row): lane_memberships(row, config) for row in normalized_estimates
}
enriched_sha = _write_jsonl(audit_dir / "enriched-estimates.jsonl", normalized_estimates)
lanes: dict[str, list[dict[str, Any]]] = {lane: [] for lane in ALLOWED_LANES}
for row in normalized_estimates:
symbol = _symbol(row)
growth_pattern = _text(row.get("growth_pattern"))
is_fpi = _is_foreign_private_issuer(row)
for lane in lane_membership_map.get(symbol, []):
copy = dict(row)
flags = set(copy.get("provider_prefilter_flags") or [])
if growth_pattern == "trough_recovery":
flags.add("earnings_recovery")
if is_fpi:
flags.add("foreign_private_issuer_review")
if flags:
copy["provider_prefilter_flags"] = sorted(flags)
lanes[lane].append(copy)
for lane, rows in lanes.items():
_write_jsonl(audit_dir / f"lane-{lane}.jsonl", rows)
pool, discovery_audit = build_pool(
universe_rows=enriched_universe,
lane_rows=lanes,
analysis_as_of=analysis_as_of.isoformat(),
source_ids=[estimate_source],
per_lane=int(config["provider_prefilter_per_lane"]),
max_pool=int(config["provider_prefilter_pool_size"]),
minimum_pool=int(config["provider_prefilter_minimum_pool"]),
requested_min_market_cap=min_cap,
requested_max_market_cap=max_cap,
provider_exhausted=True,
provider_exhausted_scope="estimate_seed",
)
pool_sha = _write_jsonl(audit_dir / "provider-prefilter-pool.jsonl", pool)
discovery_audit.update(
{
"artifact_path": "provider-prefilter-pool.jsonl",
"artifact_sha256": pool_sha,
"universe_artifact_sha256": universe_sha,
"enriched_estimates_sha256": enriched_sha,
"estimate_acquisition_mode": estimate_acquisition_mode,
"bulk_estimate_audit": bulk_estimate_audit,
"exact_liquidity_target_count": len(liquidity_targets),
"quality_probe": quality_probe_audit,
# ACTUAL attempted count (len of the estimate frame), never the
# configured seed LIMIT: a 180 limit over a 50-name universe must
# not fabricate 360% coverage.
"economic_attempt_count": len(seed),
"economic_attempt_coverage_pct": round(
len(seed) / len(enriched_universe) * 100.0 if enriched_universe else 0.0, 6
),
"economically_evaluable_count": sum(
1
for row in normalized_estimates
if str(row.get("estimate_normalization_status") or "") == "valid"
),
"economically_evaluable_coverage_pct": round(
(
sum(
1
for row in normalized_estimates
if str(row.get("estimate_normalization_status") or "") == "valid"
)
/ len(enriched_universe)
* 100.0
if enriched_universe
else 0.0
),
6,
),
"listing_provider_exhausted": True,
"estimate_seed_exhausted": True,
"economic_candidate_universe_exhausted": economic_scope_complete(
estimate_acquisition_mode=estimate_acquisition_mode,
covered_symbol_count=int(bulk_estimate_audit.get("covered_symbol_count") or 0),
universe_symbol_count=int(bulk_estimate_audit.get("universe_symbol_count") or 0),
),
}
)
if seed_audit is not None:
discovery_audit["seed_audit"] = seed_audit
_write_json(audit_dir / "provider-prefilter-audit.json", discovery_audit)
screen_config = dict(SCREEN_DEFAULTS)
screen_config.update(config)
screen_config["max_deep_dive_candidates"] = int(config["max_deep_dive_candidates"])
universe_decisions, candidate_decisions, audit, selected, queue = run_layered(
enriched_universe,
pool,
screen_config,
analysis_as_of=analysis_as_of.isoformat(),
universe_source_ids=[f"fmp-company-screener-{analysis_as_of.date().isoformat()}"],
candidate_source_ids=[estimate_source],
candidate_generation_mode="provider_prefilter",
retrieval_min_market_cap=min_cap,
retrieval_max_market_cap=max_cap,
requested_min_market_cap=min_cap,
requested_max_market_cap=max_cap,
retrieval_scope_explicit=True,
candidate_pool_exhausted=True,
provider_reported_total=None,
pages_fetched=int(enumeration_audit["query_count"]),
pagination_exhausted=bool(enumeration_audit["enumeration_verified"]),
band_audit=enumeration_audit["bands"],
discovery_audit=discovery_audit,
)
universe_decisions_path = audit_dir / "universe-audit-results.jsonl"
candidate_decisions_path = audit_dir / "broad-screen-results.jsonl"
queue_path = audit_dir / "enrichment-queue.json"
universe_decisions_path.write_text(
"".join(_canonical_line(row) for row in universe_decisions), encoding="utf-8"
)
candidate_decisions_path.write_text(
"".join(_canonical_line(row) for row in candidate_decisions), encoding="utf-8"
)
_write_json(queue_path, queue)
audit["universe"].update(
{
"artifact_path": universe_decisions_path.name,
"artifact_sha256": hashlib.sha256(universe_decisions_path.read_bytes()).hexdigest(),
}
)
audit["candidate_pool"].update(
{
"artifact_path": candidate_decisions_path.name,
"artifact_sha256": hashlib.sha256(candidate_decisions_path.read_bytes()).hexdigest(),
}
)
audit["enrichment"].update(
{
"artifact_path": queue_path.name,
"artifact_sha256": hashlib.sha256(queue_path.read_bytes()).hexdigest(),
}
)
_write_json(audit_dir / "broad-screen-audit.json", audit)
selected_rows = {_symbol(row): row for row in pool if _symbol(row) in set(selected)}
packet_paths: list[str] = []
if include_packets:
for symbol in selected:
packet_paths.append(str(build_fmp_packet(client, selected_rows[symbol], packet_dir)))
listing_universe_count = len(enriched_universe)
estimate_seed_count = len(seed)
estimate_seed_coverage_pct = (
estimate_seed_count / listing_universe_count * 100.0 if listing_universe_count else 0.0
)
valid_estimate_count = sum(
1
for row in normalized_estimates
if str(row.get("estimate_normalization_status") or "") == "valid"
)
valid_estimate_coverage_pct = (
valid_estimate_count / listing_universe_count * 100.0 if listing_universe_count else 0.0
)
economic_screen_scope_complete = economic_scope_complete(
estimate_acquisition_mode=estimate_acquisition_mode,
covered_symbol_count=int(bulk_estimate_audit.get("covered_symbol_count") or 0),
universe_symbol_count=int(bulk_estimate_audit.get("universe_symbol_count") or 0),
)
old_scope_complete = bool(audit.get("scope", {}).get("scope_complete"))
quality_probe_count = _coverage_count(quality_probe_audit.get("attempted"))
deep_dive_count = len(selected)
ranking_scope = classify_ranking_scope(
economic_attempt_count=estimate_seed_count,
listing_universe_count=listing_universe_count,
economic_scope_complete=economic_screen_scope_complete,
unresolved_queue_count=len(queue),
)
coverage_block = build_coverage_block(
ranking_scope=ranking_scope,
listing_universe_count=listing_universe_count,
economic_attempt_count=estimate_seed_count,
economically_evaluable_count=valid_estimate_count,
quality_probe_count=quality_probe_count,
deep_dive_count=deep_dive_count,
)
# build_coverage_block also emits listing_universe_count; the summary
# already carries it at top level, which is harmless duplication.
coverage_block = {k: v for k, v in coverage_block.items() if k != "listing_universe_count"}
pool_status = str(audit.get("candidate_pool_status") or "")
selection_outcome = str(audit.get("selection_outcome") or "")
if selected and pool_status == "sufficient":
status = "ready_for_underwriting"
action = "complete_primary_source_underwriting"
elif not selected and pool_status == "no_qualifying_candidates":
status = "no_candidates_in_scoped_pool"
action = "publish_scoped_no_candidates"
else:
status = "needs_enrichment"
action = "repair_discovery_or_enrichment_contract"
next_action = {
"runtime": runtime_metadata(),
"action": action,
"symbols": selected,
"user_confirmation_required": False,
"read_only": [
"run-summary.json",
"audit/broad-screen-audit.json",
"candidate-packets/*.fmp-packet.json",
],
"do_not_read_bulk_provider_payloads_into_model_context": True,
"listing_enumeration_complete": old_scope_complete,
"economic_screen_scope_complete": economic_screen_scope_complete,
"listing_universe_count": listing_universe_count,
"estimate_seed_count": estimate_seed_count,
"estimate_seed_coverage_pct": round(estimate_seed_coverage_pct, 6),
"valid_estimate_count": valid_estimate_count,
"valid_estimate_coverage_pct": round(valid_estimate_coverage_pct, 6),
**coverage_block,
"required_after_underwriting": [
"manage_run_state.py assemble",
"evaluate_candidates.py --strict --require-final",
"prepublish_audit.py",
"bundle_run_artifacts.py",
],
}
_write_json(output_dir / "NEXT_ACTION.json", next_action)
summary = {
"runtime": runtime_metadata(),
"run_id": output_dir.name,
"status": status,
"analysis_as_of": analysis_as_of.isoformat(),
"conclusion_scope": audit.get("conclusion_scope"),
# `scope_complete` communicates *listing enumeration* completeness
# only -- it says nothing about economic (estimate/fundamental)
# coverage. See `economic_screen_scope_complete` for that.
"scope_complete": old_scope_complete,
"scope_complete_deprecated_note": (
"listing enumeration only; see economic_screen_scope_complete"
),
"listing_enumeration_complete": old_scope_complete,
"economic_screen_scope_complete": economic_screen_scope_complete,
"listing_universe_count": listing_universe_count,
"estimate_seed_count": estimate_seed_count,
"estimate_seed_coverage_pct": round(estimate_seed_coverage_pct, 6),
"valid_estimate_count": valid_estimate_count,
"valid_estimate_coverage_pct": round(valid_estimate_coverage_pct, 6),
**coverage_block,
"listing_enumeration_verified": bool(enumeration_audit["enumeration_verified"]),
"listing_query_count": int(enumeration_audit["query_count"]),
"universe_count": len(enriched_universe),
"seed_count": len(seed),
"seed_audit": seed_audit,
"normalized_estimate_count": len(normalized_estimates),
"estimate_acquisition_mode": estimate_acquisition_mode,
"bulk_estimate_coverage_pct": bulk_coverage,
"exact_liquidity_target_count": len(liquidity_targets),
"provider_prefilter_pool_count": len(pool),
"selected_symbols": selected,
"selected_candidates": [compact_candidate(selected_rows[symbol]) for symbol in selected],
"enrichment_queue_count": len(queue),
"candidate_pool_status": pool_status,
"selection_outcome": selection_outcome,
"bulk_liquidity_used": bulk_liquidity,
"lane_counts": {lane: len(rows) for lane, rows in lanes.items()},
"provider_diagnostics": client.diagnostics(),
"artifacts": {
"run_summary": "run-summary.json",
"next_action": "NEXT_ACTION.json",
"listing_enumeration_audit": "audit/listing-enumeration-audit.json",
"provider_prefilter_audit": "audit/provider-prefilter-audit.json",
"broad_screen_audit": "audit/broad-screen-audit.json",
"candidate_packets": [str(Path(path).relative_to(output_dir)) for path in packet_paths],
"raw_provider_dir": "provider",
},
}
_write_json(output_dir / "run-summary.json", summary)
exit_code = 0 if status in {"ready_for_underwriting", "no_candidates_in_scoped_pool"} else 2
return PipelineResult(summary=summary, exit_code=exit_code)
def prepare_screen_full_snapshot(
snapshot_dir: Path,
*,
analysis_as_of: datetime,
config: Mapping[str, Any],
screening_started_at: datetime | None = None,
) -> dict[str, Any]:
"""Read-only preflight for the current-only full-snapshot screen.
Screen-time liquidity and TTM quality probes are current provider data.
Historical replays must not mix that evidence into an old snapshot, so
this stage accepts only an ``analysis_as_of`` close to its wall-clock
start. The preflight runs before FMPClient construction in ``main``;
invalid snapshots therefore create no cache, raw-store, or run tree.
"""
started = screening_started_at or datetime.now(timezone.utc)
if started.tzinfo is None:
started = started.replace(tzinfo=timezone.utc)
if analysis_as_of.tzinfo is None:
analysis_as_of = analysis_as_of.replace(tzinfo=timezone.utc)
skew = float(config.get("full_snapshot_screening_clock_skew_seconds", 300))
if skew < 0:
raise ValueError("full_snapshot_screening_clock_skew_seconds must be non-negative")
if abs((analysis_as_of - started).total_seconds()) > skew:
raise ValueError(
"screen-full-snapshot is current-only because its liquidity and quality "
"enrichment use current provider evidence"
)
target_pool_size = int(config.get("full_snapshot_pool_size", 50))
if not 30 <= target_pool_size <= 50:
raise ValueError("full_snapshot_pool_size must be between 30 and 50")
deep_dive_limit = int(config.get("full_snapshot_deep_dive_candidates", 5))
if deep_dive_limit != 5:
raise ValueError("full_snapshot_deep_dive_candidates must be 5")
max_staleness = float(config.get("full_snapshot_max_staleness_days", 7))
bundle = snapshot_store.load_verified_snapshot(
snapshot_dir,
screening_as_of=analysis_as_of,
max_staleness_days=max_staleness,
clock_skew_seconds=skew,
)
verdict = dict(bundle["verdict"])
verdict.update(
{
"screening_started_at": started.astimezone(timezone.utc).isoformat(),
"screening_clock_skew_seconds": skew,
}
)
bundle["verdict"] = verdict
if verdict.get("ready_for_screening") is not True:
problems = verdict.get("problems") or []
detail = "; ".join(str(value) for value in problems[:5]) or "readiness checks failed"
raise ValueError(f"snapshot is not ready for screening: {detail}")
enumeration = dict(verdict.get("listing_enumeration") or {})
frozen_minimum = float(enumeration["requested_min_market_cap"])
frozen_maximum = float(enumeration["requested_max_market_cap"])
frozen_min_price = float(enumeration["min_price"])
requested_minimum = float(config["min_market_cap"])
requested_maximum = float(config["max_market_cap"])
requested_min_price = float(config["min_price"])
if (
frozen_minimum > requested_minimum
or frozen_maximum < requested_maximum
or frozen_min_price > requested_min_price
):
raise ValueError("snapshot listing enumeration does not cover the requested screen scope")
for row in bundle.get("rows") or []:
expected = snapshot_store.classify_symbol(
row,
row,
requires_unit_reconciliation=requires_unit_reconciliation,
minimum_plausible_forward_pe=float(config.get("minimum_plausible_forward_pe", 2.0)),
)
if row.get("snapshot_classification") != expected:
raise ValueError(
f"snapshot classification mismatch for {_symbol(row)}: "
f"recorded {row.get('snapshot_classification')!r}, expected {expected!r}"
)
return bundle
def validate_current_snapshot_collection(
*,
analysis_as_of: datetime,
config: Mapping[str, Any],
collection_started_at: datetime | None = None,
) -> datetime:
"""Reject historical/future normalization bases before collection side effects."""
started = collection_started_at or datetime.now(timezone.utc)
if started.tzinfo is None:
started = started.replace(tzinfo=timezone.utc)
if analysis_as_of.tzinfo is None:
analysis_as_of = analysis_as_of.replace(tzinfo=timezone.utc)
skew = float(config.get("full_snapshot_collection_clock_skew_seconds", 300))
if skew < 0:
raise ValueError("full_snapshot_collection_clock_skew_seconds must be non-negative")
if abs((analysis_as_of - started).total_seconds()) > skew:
raise ValueError(
"collect-estimates is current-only because analysis_as_of fixes the estimate "
"normalization basis"
)
return started.astimezone(timezone.utc)
def _snapshot_lane_rows(
rows: Sequence[Mapping[str, Any]],
config: Mapping[str, Any],
) -> dict[str, list[dict[str, Any]]]:
lane_rows: dict[str, list[dict[str, Any]]] = {lane: [] for lane in ALLOWED_LANES}
for raw in rows:
row = dict(raw)
symbol = _symbol(row)
growth_pattern = _text(row.get("growth_pattern"))
is_fpi = _is_foreign_private_issuer(row)
for lane in lane_memberships(row, config):
copy = dict(row)
flags = set(copy.get("provider_prefilter_flags") or [])
if growth_pattern == "trough_recovery":
flags.add("earnings_recovery")
if is_fpi:
flags.add("foreign_private_issuer_review")
if flags:
copy["provider_prefilter_flags"] = sorted(flags)
copy["symbol"] = symbol
lane_rows[lane].append(copy)
return lane_rows
def _attach_snapshot_attempts(
decisions: Sequence[Mapping[str, Any]],
snapshot_rows: Sequence[Mapping[str, Any]],
*,
snapshot_id: str,
verification_digest: str,
) -> list[dict[str, Any]]:
by_symbol = {_symbol(row): row for row in snapshot_rows}
output: list[dict[str, Any]] = []
for raw in decisions:
row = dict(raw)
source = by_symbol.get(_symbol(row))
if source is None:
raise ValueError(f"universe audit symbol {_symbol(row)} is absent from snapshot")
row["estimate_attempt"] = {
"snapshot_id": snapshot_id,
"snapshot_verification_digest": verification_digest,
"shard": source.get("snapshot_shard"),
"retrieved_at": source.get("snapshot_retrieved_at"),
"classification": source.get("snapshot_classification"),
"served_from_cache": source.get("snapshot_served_from_cache"),
}
output.append(row)
return output
def execute_screen_full_snapshot(
client: FMPClient,
config: Mapping[str, Any],
*,
analysis_as_of: datetime,
output_dir: Path,
prepared_snapshot: Mapping[str, Any],
include_packets: bool = True,
) -> PipelineResult:
"""Screen a deeply verified, current full-universe estimate snapshot."""
verdict = dict(prepared_snapshot.get("verdict") or {})
snapshot_rows = [dict(row) for row in (prepared_snapshot.get("rows") or [])]
verification_digest = str(prepared_snapshot.get("verification_digest") or "")
if (
verdict.get("ready_for_screening") is not True
or len(verification_digest) != 64
or not snapshot_rows
):
raise ValueError("execute_screen_full_snapshot requires a prepared verified snapshot")
_require_fresh_screen_output_dir(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
audit_dir = output_dir / "audit"
packet_dir = output_dir / "candidate-packets"
audit_dir.mkdir(parents=True, exist_ok=True)
snapshot_id = str(verdict.get("snapshot_id") or "")
universe_rows = sorted(snapshot_rows, key=_symbol)
evaluable_rows = [
dict(row) for row in universe_rows if row.get("snapshot_classification") == "evaluable"
]
target_pool_size = int(config.get("full_snapshot_pool_size", 50))
# The market-wide pool target and exact-liquidity work must move together;
# retaining the bounded path's default 40 would silently cap a 50-name pool.
eligible_rows = [row for row in evaluable_rows if lane_memberships(row, config)]
ordered_liquidity_targets = select_liquidity_targets(
eligible_rows,
config,
limit=len(eligible_rows),
)
ordered_liquidity_targets = _ordered_unique_symbols(ordered_liquidity_targets)
required_liquidity_days = int(config["minimum_average_volume_period_days"])
enriched_rows = [dict(row) for row in evaluable_rows]
liquidity_targets: list[str] = []
liquidity_attempted: list[str] = []
liquidity_completed: set[str] = set()
valid_liquidity_symbols: set[str] = set()
# Backfill past failed/empty histories. A short pool can claim exhaustion
# only after every economically eligible symbol has an explicit outcome.
for symbol in ordered_liquidity_targets:
liquidity_attempted.append(symbol)
try:
enriched_rows = apply_symbol_liquidity(
client,
enriched_rows,
as_of=analysis_as_of.date(),
source_id=f"fmp-historical-eod-{analysis_as_of.date().isoformat()}",
limit=1,
required_days=required_liquidity_days,
target_symbols=[symbol],
)
except ApiCallBudgetExceeded as exc:
return _persist_partial_snapshot_diagnostic(
output_dir=output_dir,
audit_dir=audit_dir,
snapshot_id=snapshot_id,
verification_digest=verification_digest,
stage="liquidity",
progress={
"target_symbols": ordered_liquidity_targets,
"attempted_symbols": liquidity_attempted,
"completed_symbols": [
value for value in ordered_liquidity_targets if value in liquidity_completed
],
"pending_symbols": [
value
for value in ordered_liquidity_targets
if value not in liquidity_completed
],
"interrupted_symbols": [symbol],
"interrupted_at": {
"symbol": symbol,
"operation": "get_historical_prices",
},
},
error=str(exc),
client=client,
partial_rows=enriched_rows,
)
liquidity_completed.add(symbol)
liquidity_targets.append(symbol)
row = next(item for item in enriched_rows if _symbol(item) == symbol)
if (
_first_number(row, "average_daily_dollar_volume") is not None
and int(_first_number(row, "average_volume_period_days") or 0)
>= required_liquidity_days
):
valid_liquidity_symbols.add(symbol)
if len(valid_liquidity_symbols) >= target_pool_size:
break
eligible_liquidity_exhausted = len(liquidity_targets) == len(ordered_liquidity_targets)
preliminary_pool, _ = build_pool(
universe_rows=universe_rows,
lane_rows=_snapshot_lane_rows(enriched_rows, config),
analysis_as_of=analysis_as_of.isoformat(),
source_ids=[f"snapshot:{snapshot_id}"],
per_lane=int(config["provider_prefilter_per_lane"]),
max_pool=target_pool_size,
minimum_pool=min(30, target_pool_size),
requested_min_market_cap=float(config["min_market_cap"]),
requested_max_market_cap=float(config["max_market_cap"]),
provider_exhausted=eligible_liquidity_exhausted,
provider_exhausted_scope=(
"economic_candidate_universe" if eligible_liquidity_exhausted else None
),
)
probe_symbols = [_symbol(row) for row in preliminary_pool]
probed_rows, quality_probe_audit = apply_quality_probe(
client,
enriched_rows,
target_symbols=probe_symbols,
source_id=f"fmp-key-metrics-ttm-{analysis_as_of.date().isoformat()}",
analysis_as_of=analysis_as_of,
actual_source_id=f"fmp-income-statement-annual-{analysis_as_of.date().isoformat()}",
return_partial_on_budget=True,
)
if quality_probe_audit.get("budget_exhausted"):
_write_json(audit_dir / "quality-probe-audit.json", quality_probe_audit)
return _persist_partial_snapshot_diagnostic(
output_dir=output_dir,
audit_dir=audit_dir,
snapshot_id=snapshot_id,
verification_digest=verification_digest,
stage="quality_probe",
progress={
"target_symbols": _ordered_unique_symbols(probe_symbols),
"attempted_symbols": quality_probe_audit.get("attempted_symbols", []),
"completed_symbols": quality_probe_audit.get("completed_symbols", []),
"pending_symbols": quality_probe_audit.get("pending_symbols", []),
"interrupted_symbols": quality_probe_audit.get("interrupted_symbols", []),
"interrupted_at": quality_probe_audit.get("interrupted_at"),
},
error=str(quality_probe_audit.get("error") or "provider API call budget exhausted"),
client=client,
partial_rows=probed_rows,
preserved_artifact_paths=[audit_dir / "quality-probe-audit.json"],
)
probed_rows = mark_sector_profile_exhaustion(
probed_rows,
source_id=f"fmp-key-metrics-ttm-{analysis_as_of.date().isoformat()}",
)
probed_rows = mark_unit_reconciliation_exhaustion(
probed_rows,
source_id=f"fmp-key-metrics-ttm-{analysis_as_of.date().isoformat()}",
)
# Never backfill the final pool with an unprobed name. If probes remove
# names from a full preliminary pool, the short-pool proof stays false and
# the stage fails closed instead of claiming exhaustion.
probed_symbol_set = set(probe_symbols)
final_input_rows = [row for row in probed_rows if _symbol(row) in probed_symbol_set]
pool, discovery_audit = build_pool(
universe_rows=universe_rows,
lane_rows=_snapshot_lane_rows(final_input_rows, config),
analysis_as_of=analysis_as_of.isoformat(),
source_ids=[f"snapshot:{snapshot_id}"],
per_lane=int(config["provider_prefilter_per_lane"]),
max_pool=target_pool_size,
minimum_pool=min(30, target_pool_size),
requested_min_market_cap=float(config["min_market_cap"]),
requested_max_market_cap=float(config["max_market_cap"]),
provider_exhausted=eligible_liquidity_exhausted,
provider_exhausted_scope=(
"economic_candidate_universe" if eligible_liquidity_exhausted else None
),
)
pool_sha = _write_jsonl(audit_dir / "provider-prefilter-pool.jsonl", pool)
classification_totals = dict(verdict.get("classified_totals") or {})
eligible_pool_exhausted = len(pool) >= target_pool_size or eligible_liquidity_exhausted
discovery_audit.update(
{
"valid": bool(discovery_audit.get("valid")) and eligible_pool_exhausted,
"selection_method": "sharded_snapshot_multilane",
"artifact_path": "provider-prefilter-pool.jsonl",
"artifact_sha256": pool_sha,
"input_row_count": len(universe_rows),
"selected_count": len(pool),
"selected_symbols": sorted(_symbol(row) for row in pool),
"target_pool_size": target_pool_size,
"preliminary_pool_count": len(preliminary_pool),
"eligible_pool_exhausted": eligible_pool_exhausted,
"estimate_acquisition_mode": "sharded_snapshot",
"economic_attempt_count": len(universe_rows),
"economically_evaluable_count": len(evaluable_rows),
"snapshot_id": snapshot_id,
"snapshot_verification_digest": verification_digest,
"snapshot_verification": verdict,
"snapshot_classification_totals": classification_totals,
"quality_probe": quality_probe_audit,
"exact_liquidity_target_count": len(liquidity_targets),
"exact_liquidity_eligible_count": len(ordered_liquidity_targets),
"exact_liquidity_valid_count": len(valid_liquidity_symbols),
"economic_candidate_universe_exhausted": True,
}
)
_write_json(audit_dir / "provider-prefilter-audit.json", discovery_audit)
_write_json(audit_dir / "snapshot-verification.json", verdict)
_write_json(audit_dir / "quality-probe-audit.json", quality_probe_audit)
_write_jsonl(audit_dir / "enriched-estimates.jsonl", probed_rows)
for lane, rows in _snapshot_lane_rows(final_input_rows, config).items():
_write_jsonl(audit_dir / f"lane-{lane}.jsonl", rows)
screen_config = dict(SCREEN_DEFAULTS)
screen_config.update(config)
screen_config["max_deep_dive_candidates"] = int(
config.get("full_snapshot_deep_dive_candidates", 5)
)
enumeration = dict(verdict.get("listing_enumeration") or {})
universe_decisions, candidate_decisions, audit, selected, queue = run_layered(
universe_rows,
pool,
screen_config,
analysis_as_of=analysis_as_of.isoformat(),
universe_source_ids=[f"snapshot:{snapshot_id}"],
candidate_source_ids=[f"snapshot:{snapshot_id}"],
candidate_generation_mode="sharded_snapshot",
retrieval_min_market_cap=float(enumeration["requested_min_market_cap"]),
retrieval_max_market_cap=float(enumeration["requested_max_market_cap"]),
requested_min_market_cap=float(config["min_market_cap"]),
requested_max_market_cap=float(config["max_market_cap"]),
retrieval_scope_explicit=True,
candidate_pool_exhausted=True,
provider_reported_total=int(enumeration.get("row_count") or 0),
pages_fetched=int(enumeration.get("query_count") or 0),
pagination_exhausted=enumeration.get("pagination_exhausted") is True,
band_audit=enumeration.get("bands") or [],
discovery_audit=discovery_audit,
)
universe_decisions = _attach_snapshot_attempts(
universe_decisions,
universe_rows,
snapshot_id=snapshot_id,
verification_digest=verification_digest,
)
universe_path = audit_dir / "universe-audit-results.jsonl"
candidate_path = audit_dir / "broad-screen-results.jsonl"
queue_path = audit_dir / "enrichment-queue.json"
universe_path.write_text(
"".join(_canonical_line(row) for row in universe_decisions), encoding="utf-8"
)
candidate_path.write_text(
"".join(_canonical_line(row) for row in candidate_decisions), encoding="utf-8"
)
_write_json(queue_path, queue)
audit["universe"].update(
{
"artifact_path": universe_path.name,
"artifact_sha256": hashlib.sha256(universe_path.read_bytes()).hexdigest(),
"snapshot_classification_totals": classification_totals,
}
)
audit["candidate_pool"].update(
{
"artifact_path": candidate_path.name,
"artifact_sha256": hashlib.sha256(candidate_path.read_bytes()).hexdigest(),
}
)
audit["enrichment"].update(
{
"artifact_path": queue_path.name,
"artifact_sha256": hashlib.sha256(queue_path.read_bytes()).hexdigest(),
}
)
audit["snapshot_verification"] = verdict
audit["snapshot_verification_digest"] = verification_digest
_write_json(audit_dir / "broad-screen-audit.json", audit)
selected_rows = {_symbol(row): row for row in pool if _symbol(row) in set(selected)}
packet_paths: list[str] = []
if include_packets:
packet_targets = _ordered_unique_symbols(selected)
packet_attempted: list[str] = []
packet_completed: set[str] = set()
packet_client = _OperationTrackingClient(client)
for symbol in packet_targets:
packet_attempted.append(symbol)
packet_client.last_operation = None
try:
packet_paths.append(
str(build_fmp_packet(packet_client, selected_rows[symbol], packet_dir))
)
except ApiCallBudgetExceeded as exc:
invalidated_audit = _invalidate_final_screen_audit(audit)
invalidated_audit["partial_run"] = {
"status": "provider_budget_exhausted",
"stage": "candidate_packets",
"commit_marker": "audit/partial-run-diagnostic.json",
"candidate_artifact_rewrite": "pending",
}
invalidated_candidates = _invalidate_candidate_decisions(candidate_decisions)
# Publish the non-authoritative audit first. If the process
# stops during the row rewrite, no audit can still authorize
# the old selected rows.
invalidated_audit["candidate_pool"]["artifact_sha256"] = None
_write_json(audit_dir / "broad-screen-audit.json", invalidated_audit)
candidate_sha = _write_jsonl(candidate_path, invalidated_candidates)
invalidated_audit["candidate_pool"]["decision_counts"] = {
status: sum(
1
for row in invalidated_candidates
if str((row.get("decision") or {}).get("status") or "") == status
)
for status in sorted(
{
str((row.get("decision") or {}).get("status") or "")
for row in invalidated_candidates
}
)
}
invalidated_audit["candidate_pool"]["artifact_sha256"] = candidate_sha
invalidated_audit["partial_run"]["candidate_artifact_rewrite"] = "complete"
_write_json(audit_dir / "broad-screen-audit.json", invalidated_audit)
return _persist_partial_snapshot_diagnostic(
output_dir=output_dir,
audit_dir=audit_dir,
snapshot_id=snapshot_id,
verification_digest=verification_digest,
stage="candidate_packets",
progress={
"target_symbols": packet_targets,
"attempted_symbols": packet_attempted,
"completed_symbols": [
value for value in packet_targets if value in packet_completed
],
"pending_symbols": [
value for value in packet_targets if value not in packet_completed
],
"interrupted_symbols": [symbol],
"interrupted_at": {
"symbol": symbol,
"operation": packet_client.last_operation or "build_fmp_packet",
"partial_packet_written": False,
},
},
error=str(exc),
client=client,
partial_rows=probed_rows,
preserved_artifact_paths=[
*(Path(path) for path in packet_paths),
audit_dir / "broad-screen-audit.json",
],
)
packet_completed.add(symbol)
ranking_scope = classify_ranking_scope(
economic_attempt_count=len(universe_rows),
listing_universe_count=len(universe_rows),
economic_scope_complete=bool(verdict.get("ready_for_screening")),
unresolved_queue_count=len(queue),
)
if audit.get("conclusion_scope") != "full_listing_universe":
ranking_scope = "diagnostic"
coverage = build_coverage_block(
ranking_scope=ranking_scope,
listing_universe_count=len(universe_rows),
economic_attempt_count=len(universe_rows),
economically_evaluable_count=len(evaluable_rows),
quality_probe_count=_coverage_count(quality_probe_audit.get("attempted")),
deep_dive_count=len(selected),
)
pool_status = str(audit.get("candidate_pool_status") or "")
if selected and pool_status == "sufficient" and ranking_scope == "final_marketwide":
status = "ready_for_underwriting"
action = "complete_primary_source_underwriting"
elif (
not selected
and pool_status == "no_qualifying_candidates"
and ranking_scope == "final_marketwide"
):
status = "no_candidates_in_marketwide_snapshot"
action = "publish_marketwide_no_candidates"
else:
status = "needs_enrichment"
action = "repair_snapshot_screening_contract"
common = {
"runtime": runtime_metadata(),
"run_id": output_dir.name,
"status": status,
"stage": "screen-full-snapshot",
"analysis_as_of": analysis_as_of.isoformat(),
"conclusion_scope": audit.get("conclusion_scope"),
"economic_screen_scope_complete": bool(verdict.get("ready_for_screening")),
"estimate_acquisition_mode": "sharded_snapshot",
"snapshot_id": snapshot_id,
"snapshot_verification_digest": verification_digest,
"snapshot_verification": verdict,
"snapshot_classification_totals": classification_totals,
**coverage,
"estimate_seed_count": len(universe_rows),
"estimate_seed_coverage_pct": 100.0,
"valid_estimate_count": len(evaluable_rows),
"valid_estimate_coverage_pct": round(len(evaluable_rows) / len(universe_rows) * 100.0, 6),
"provider_prefilter_pool_count": len(pool),
"quality_probe_count": _coverage_count(quality_probe_audit.get("attempted")),
"exact_liquidity_target_count": len(liquidity_targets),
"selected_symbols": selected,
"selected_candidates": [compact_candidate(selected_rows[symbol]) for symbol in selected],
"enrichment_queue_count": len(queue),
"candidate_pool_status": pool_status,
"selection_outcome": audit.get("selection_outcome"),
"provider_diagnostics": client.diagnostics(),
}
next_action = {
**common,
"action": action,
"symbols": selected,
"user_confirmation_required": False,
"do_not_read_bulk_provider_payloads_into_model_context": True,
"required_after_underwriting": [
"manage_run_state.py assemble",
"evaluate_candidates.py --strict --require-final",
"prepublish_audit.py",
"bundle_run_artifacts.py",
],
}
_write_json(output_dir / "NEXT_ACTION.json", next_action)
summary = {
**common,
"artifacts": {
"run_summary": "run-summary.json",
"next_action": "NEXT_ACTION.json",
"snapshot_verification": "audit/snapshot-verification.json",
"provider_prefilter_audit": "audit/provider-prefilter-audit.json",
"broad_screen_audit": "audit/broad-screen-audit.json",
"candidate_packets": [str(Path(path).relative_to(output_dir)) for path in packet_paths],
},
}
_write_json(output_dir / "run-summary.json", summary)
return PipelineResult(
summary=summary,
exit_code=0
if status in {"ready_for_underwriting", "no_candidates_in_marketwide_snapshot"}
else 2,
)
def execute_collect_estimates(
client: FMPClient,
config: Mapping[str, Any],
*,
analysis_as_of: datetime,
snapshot_dir: Path,
shard_index: int,
shard_count: int,
resume: bool,
collection_started_at: datetime | None = None,
) -> PipelineResult:
"""Collect minimal FY1-FY3 estimates for one deterministic universe shard.
First invocation freezes the listing universe into ``snapshot_dir``
(``snapshot-manifest.json`` + ``universe.jsonl``); every later shard run
writes into that frozen snapshot only. Each attempted symbol is
normalized and classified (``evaluable / no_estimates / negative_eps /
unit_mismatch / excluded``) so that, once every shard is complete, the
classification counts sum exactly to the frozen universe — the
precondition ``screen-full-snapshot`` will demand before ever emitting
``ranking_scope: final_marketwide``. Budget exhaustion mid-shard leaves
the shard honestly ``partial`` (exit code 3) and is resumable.
"""
validate_current_snapshot_collection(
analysis_as_of=analysis_as_of,
config=config,
collection_started_at=collection_started_at,
)
if shard_count <= 0 or not (0 <= shard_index < shard_count):
raise ValueError("shard_index must satisfy 0 <= shard_index < shard_count")
if (snapshot_dir / snapshot_store.MANIFEST_NAME).exists():
manifest = snapshot_store.load_manifest(snapshot_dir)
if int(manifest.get("shard_count") or 0) != shard_count:
raise ValueError(
f"snapshot at {snapshot_dir} was created with shard_count="
f"{manifest.get('shard_count')}; got {shard_count}"
)
# Re-verify the freeze before any API access or shard append: a
# swapped universe.jsonl must never collect under the frozen id.
universe_rows = snapshot_store.load_verified_universe(snapshot_dir, manifest)
else:
universe_rows, enumeration_audit = collect_listing_universe(
client,
min_market_cap=float(config["min_market_cap"]),
max_market_cap=float(config["max_market_cap"]),
min_price=float(config["min_price"]),
page_limit=int(config["company_screener_limit"]),
minimum_band_width=float(config["minimum_market_cap_band_width"]),
maximum_depth=int(config["maximum_market_cap_band_depth"]),
)
if not universe_rows:
raise ValueError("FMP company screener returned no normalized listing rows")
profile_overrides = {
str(key).upper(): str(value)
for key, value in (config.get("sector_profile_overrides") or {}).items()
}
for row in universe_rows:
pinned = profile_overrides.get(str(row.get("symbol", "")).upper())
if pinned:
row["sector_profile_type"] = pinned
manifest = snapshot_store.create_snapshot(
snapshot_dir,
universe_rows,
shard_count=shard_count,
as_of=analysis_as_of,
enumeration_audit=enumeration_audit,
)
shard_listings = [
row
for row in universe_rows
if snapshot_store.stable_shard(_symbol(row), shard_count) == shard_index
]
existing = snapshot_store.load_shard_rows(snapshot_dir, shard_index)
if existing and not resume:
raise ValueError(
f"shard {shard_index} already has {len(existing)} rows; pass --resume to continue"
)
done = {str(row.get("symbol")) for row in existing}
pending = [row for row in shard_listings if _symbol(row) not in done]
source_id = f"fmp-analyst-estimates-{analysis_as_of.date().isoformat()}"
calls_before = int(client.diagnostics().get("api_calls_made") or 0)
buffered: list[dict[str, Any]] = []
fetch_failed_symbols: list[str] = []
budget_exhausted = False
for listing in pending:
symbol = _symbol(listing)
try:
fetched = client.get_analyst_estimates_detailed(symbol, period="annual", limit=6)
except ApiCallBudgetExceeded:
budget_exhausted = True
break
if str(fetched.get("status")) == "failed":
# The client reports provider failures explicitly (HTTP errors,
# offline misses, HTTP-200 error payloads, schema failures);
# such a symbol stays UNCOLLECTED — a failure classified as
# no_estimates would satisfy the marketwide invariant without
# ever fetching anything.
fetch_failed_symbols.append(symbol)
continue
estimates = [dict(row) for row in (fetched.get("rows") or [])]
served_from_cache = bool(fetched.get("served_from_cache"))
epoch = fetched.get("retrieved_at")
# The stamp is the time the ADOPTED payload was actually fetched
# over HTTP (a cache hit keeps the entry's creation time, reported
# by the client for the exact response it returned). Unknown
# provenance stays null — never back-filled with "now" (fail closed
# for PR B's staleness gate).
retrieved_at = (
datetime.fromtimestamp(float(epoch), tz=timezone.utc).isoformat()
if isinstance(epoch, (int, float))
else None
)
[normalized] = normalize_estimate_frame(
[listing],
{symbol: estimates},
analysis_as_of=analysis_as_of,
source_id=source_id,
config=config,
)
record = dict(normalized)
record["snapshot_classification"] = snapshot_store.classify_symbol(
listing,
normalized,
requires_unit_reconciliation=requires_unit_reconciliation,
minimum_plausible_forward_pe=float(config.get("minimum_plausible_forward_pe", 2.0)),
)
record["snapshot_shard"] = shard_index
record["snapshot_retrieved_at"] = retrieved_at
record["snapshot_normalization_as_of"] = analysis_as_of.astimezone(timezone.utc).isoformat()
record["snapshot_served_from_cache"] = served_from_cache
buffered.append(record)
if len(buffered) >= 25:
snapshot_store.append_shard_rows(snapshot_dir, shard_index, buffered)
buffered = []
if buffered:
snapshot_store.append_shard_rows(snapshot_dir, shard_index, buffered)
all_rows = snapshot_store.load_shard_rows(snapshot_dir, shard_index)
classified: dict[str, int] = {}
retrieved_stamps: list[str] = []
retrieval_time_unknown = 0
normalization_stamps: list[str] = []
normalization_time_unknown = 0
for row in all_rows:
name = str(row.get("snapshot_classification") or "no_estimates")
classified[name] = classified.get(name, 0) + 1
stamp = row.get("snapshot_retrieved_at")
if isinstance(stamp, str) and stamp:
retrieved_stamps.append(stamp)
else:
retrieval_time_unknown += 1
normalization_stamp = row.get("snapshot_normalization_as_of")
if isinstance(normalization_stamp, str) and normalization_stamp:
normalization_stamps.append(normalization_stamp)
else:
normalization_time_unknown += 1
collected_this_run = len(all_rows) - len(existing)
shard_complete = (
len(all_rows) >= len(shard_listings) and not budget_exhausted and not fetch_failed_symbols
)
previous_entry = manifest.get("shards", {}).get(str(shard_index)) or {}
previous_calls = int(previous_entry.get("calls_used") or 0)
calls_delta = int(client.diagnostics().get("api_calls_made") or 0) - calls_before
if collected_this_run > 0 or not previous_entry.get("as_of"):
shard_as_of = analysis_as_of.astimezone(timezone.utc).isoformat()
else:
# A run that collected nothing must not refresh the shard's
# freshness stamp — PR B's staleness gate reads it.
shard_as_of = str(previous_entry.get("as_of"))
manifest = snapshot_store.update_shard(
snapshot_dir,
manifest,
shard_index,
status="complete" if shard_complete else "partial",
as_of=shard_as_of,
calls_used=previous_calls + calls_delta,
classified=classified,
attempted=len(all_rows),
expected=len(shard_listings),
fetch_failed=len(fetch_failed_symbols),
oldest_retrieved_at=min(retrieved_stamps) if retrieved_stamps else None,
newest_retrieved_at=max(retrieved_stamps) if retrieved_stamps else None,
retrieval_time_unknown=retrieval_time_unknown,
oldest_normalization_as_of=min(normalization_stamps) if normalization_stamps else None,
newest_normalization_as_of=max(normalization_stamps) if normalization_stamps else None,
normalization_time_unknown=normalization_time_unknown,
)
if shard_complete:
status_label = "shard_complete"
elif budget_exhausted:
status_label = "shard_partial_budget"
elif fetch_failed_symbols:
status_label = "shard_partial_fetch_failures"
else:
status_label = "shard_partial"
summary = {
"runtime": runtime_metadata(),
"status": status_label,
"stage": "collect-estimates",
"snapshot_dir": str(snapshot_dir),
"shard_index": shard_index,
"shard_count": shard_count,
"shard_symbol_count": len(shard_listings),
"collected_this_run": collected_this_run,
"shard_classified": dict(sorted(classified.items())),
"budget_exhausted": budget_exhausted,
"fetch_failed_count": len(fetch_failed_symbols),
"fetch_failed_symbols": fetch_failed_symbols[:50],
"retrieval_time_unknown": retrieval_time_unknown,
"normalization_time_unknown": normalization_time_unknown,
"snapshot": snapshot_store.snapshot_status(manifest),
"provider_diagnostics": client.diagnostics(),
}
_write_json(snapshot_dir / f"shard-{shard_index}-summary.json", summary)
return PipelineResult(summary=summary, exit_code=0 if shard_complete else 3)
def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", type=Path)
parser.add_argument(
"--output-dir", type=Path, default=Path("reports/us-undervalued-growth-screener")
)
parser.add_argument("--analysis-as-of", help="ISO-8601 timestamp; defaults to now")
parser.add_argument("--api-key", help="Defaults to FMP_API_KEY")
parser.add_argument("--cache-path", type=Path)
parser.add_argument("--raw-store-dir", type=Path)
parser.add_argument("--max-api-calls", type=int)
parser.add_argument("--offline", action="store_true")
parser.add_argument("--skip-candidate-packets", action="store_true")
parser.add_argument(
"--stage",
choices=("discover", "collect-estimates", "screen-full-snapshot"),
default="discover",
help=(
"discover = the bounded pilot pipeline (default); collect-estimates = "
"one deterministic shard of the full-universe estimate snapshot; "
"screen-full-snapshot = current-only screen of a deeply verified snapshot"
),
)
parser.add_argument("--shard-index", type=int, help="collect-estimates: shard to collect")
parser.add_argument("--shard-count", type=int, help="collect-estimates: total shards")
parser.add_argument(
"--snapshot-dir",
type=Path,
help=(
"snapshot directory (collect default: <output-dir>/estimate-snapshot; "
"required for screen-full-snapshot)"
),
)
parser.add_argument(
"--resume", action="store_true", help="collect-estimates: continue a partial shard"
)
parser.add_argument("--version", action="store_true")
return parser.parse_args(argv)
def main(argv: Sequence[str] | None = None) -> int:
args = parse_args(argv)
if args.version:
print(json.dumps(runtime_metadata(), sort_keys=True))
return 0
try:
config = load_config(args.config)
analysis_as_of = (
datetime.fromisoformat(args.analysis_as_of.replace("Z", "+00:00"))
if args.analysis_as_of
else datetime.now(timezone.utc)
)
if analysis_as_of.tzinfo is None:
analysis_as_of = analysis_as_of.replace(tzinfo=timezone.utc)
prepared_snapshot: dict[str, Any] | None = None
collection_started_at: datetime | None = None
if args.stage == "collect-estimates":
if args.shard_index is None or args.shard_count is None:
raise ValueError(
"--shard-index and --shard-count are required for --stage collect-estimates"
)
collection_started_at = validate_current_snapshot_collection(
analysis_as_of=analysis_as_of,
config=config,
)
if args.stage == "screen-full-snapshot":
if args.snapshot_dir is None:
raise ValueError("--snapshot-dir is required for --stage screen-full-snapshot")
if args.resume or args.shard_index is not None or args.shard_count is not None:
raise ValueError(
"--resume/--shard-index/--shard-count are collect-estimates options"
)
# Read-only and intentionally before FMPClient construction: a bad
# snapshot must not create cache/raw/run artifacts.
prepared_snapshot = prepare_screen_full_snapshot(
args.snapshot_dir,
analysis_as_of=analysis_as_of,
config=config,
)
run_id = (
f"run-{analysis_as_of.astimezone(timezone.utc).strftime('%Y%m%d-%H%M%S')}"
f"-attempt-{uuid.uuid4().hex[:12]}"
)
output_dir = args.output_dir / run_id
if args.stage == "screen-full-snapshot":
_require_fresh_screen_output_dir(output_dir)
cache_cfg = dict(config.get("cache") or {})
cache_path = args.cache_path or Path(
str(cache_cfg.get("path", ".cache/us-garp/fmp-cache.sqlite3"))
)
if args.raw_store_dir is not None:
raw_store_dir = args.raw_store_dir
elif args.stage == "screen-full-snapshot":
raw_store_dir = args.output_dir / ".provider-raw" / run_id
else:
raw_store_dir = output_dir / "provider-raw"
if args.stage == "screen-full-snapshot" and _path_is_within(raw_store_dir, output_dir):
raise ValueError("screen-full-snapshot raw store must be outside the new run directory")
max_calls = args.max_api_calls or int(config["max_api_calls"])
with FMPClient(
api_key=args.api_key,
max_api_calls=max_calls,
cache_path=cache_path,
raw_store_dir=raw_store_dir,
offline=args.offline,
) as client:
if args.stage == "collect-estimates":
result = execute_collect_estimates(
client,
config,
analysis_as_of=analysis_as_of,
snapshot_dir=args.snapshot_dir or (args.output_dir / "estimate-snapshot"),
shard_index=int(args.shard_index),
shard_count=int(args.shard_count),
resume=bool(args.resume),
collection_started_at=collection_started_at,
)
elif args.stage == "screen-full-snapshot":
if prepared_snapshot is None: # defensive; preflight above is mandatory
raise ValueError("screen-full-snapshot preflight was not completed")
result = execute_screen_full_snapshot(
client,
config,
analysis_as_of=analysis_as_of,
output_dir=output_dir,
prepared_snapshot=prepared_snapshot,
include_packets=not args.skip_candidate_packets,
)
else:
result = execute_pipeline(
client,
config,
analysis_as_of=analysis_as_of,
output_dir=output_dir,
include_packets=not args.skip_candidate_packets,
)
encoded = json.dumps(result.summary, ensure_ascii=False, sort_keys=True)
max_bytes = int(config.get("compact_stdout_max_bytes", 20_000))
if len(encoded.encode("utf-8")) > max_bytes:
compact = dict(result.summary)
compact.pop("selected_candidates", None)
compact["stdout_compacted"] = True
compact["selected_candidate_details_path"] = str(output_dir / "run-summary.json")
encoded = json.dumps(compact, ensure_ascii=False, sort_keys=True)
print(encoded)
return result.exit_code
except ApiCallBudgetExceeded as exc:
print(
json.dumps(
{
"runtime": runtime_metadata(),
"status": "provider_budget_exhausted",
"error": str(exc),
},
sort_keys=True,
)
)
return 2
except (OSError, ValueError, json.JSONDecodeError) as exc:
print(
json.dumps(
{"runtime": runtime_metadata(), "status": "failed", "error": str(exc)},
sort_keys=True,
)
)
return 1
if __name__ == "__main__":
raise SystemExit(main())
scripts/screen_universe.py
#!/usr/bin/env python3
"""Tiered broad screen for US undervalued-growth candidates (contract v3.5).
The screener separates two coverage layers:
1. Listing-universe audit: symbol, exchange, active/common-stock status, price,
market capitalization, and liquidity for the full requested universe.
2. Economic candidate-pool screen: forward valuation and per-share growth for a
bounded, transparently generated pool. Full statements are *not* required
across the entire market; they are verified only for selected deep-dive names.
Rows with blocking missing data are never selected or ranked by an alphabetical
tie-break. Strong cyclicals may advance with a mid-cycle deep-dive requirement.
A run with no economically evaluable candidate pool is reported as
``insufficient_data`` rather than as ``no qualifying candidates``.
"""
from __future__ import annotations
import argparse
import csv
import hashlib
import json
import math
import sys
from collections.abc import Mapping, Sequence
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
try:
from skill_version import (
CONTRACT_REVISION,
RUNTIME_FINGERPRINT,
SCHEMA_VERSION,
SKILL_VERSION,
runtime_metadata,
)
except ModuleNotFoundError: # Supports importlib-based unit loading.
import importlib.util as _importlib_util
_version_path = Path(__file__).with_name("skill_version.py")
_version_spec = _importlib_util.spec_from_file_location("skill_version", _version_path)
if _version_spec is None or _version_spec.loader is None:
raise
_version_module = _importlib_util.module_from_spec(_version_spec)
_version_spec.loader.exec_module(_version_module)
CONTRACT_REVISION = _version_module.CONTRACT_REVISION
RUNTIME_FINGERPRINT = _version_module.RUNTIME_FINGERPRINT
SCHEMA_VERSION = _version_module.SCHEMA_VERSION
SKILL_VERSION = _version_module.SKILL_VERSION
runtime_metadata = _version_module.runtime_metadata
try:
from screening_semantics import normalize_forward_valuation, normalize_liquidity
except ModuleNotFoundError:
import importlib.util as _semantics_importlib_util
_semantics_path = Path(__file__).with_name("screening_semantics.py")
_semantics_spec = _semantics_importlib_util.spec_from_file_location(
"screening_semantics", _semantics_path
)
if _semantics_spec is None or _semantics_spec.loader is None:
raise
_semantics_module = _semantics_importlib_util.module_from_spec(_semantics_spec)
_semantics_spec.loader.exec_module(_semantics_module)
normalize_forward_valuation = _semantics_module.normalize_forward_valuation
normalize_liquidity = _semantics_module.normalize_liquidity
AUDIT_SCHEMA_VERSION = 3
DEFAULT_REQUESTED_MIN_MARKET_CAP = 500_000_000
DEFAULT_REQUESTED_MAX_MARKET_CAP = 20_000_000_000
ALLOWED_EXCHANGES = {"NYSE", "NASDAQ", "NYSE AMERICAN", "NYSEAMERICAN", "AMEX"}
CANDIDATE_GENERATION_MODES = {
"full_universe_fundamentals",
"provider_prefilter",
"sharded_snapshot",
"liquidity_stratified_estimates",
"available_fundamentals",
"user_supplied",
}
ESTIMATE_MODES = CANDIDATE_GENERATION_MODES - {"full_universe_fundamentals"}
SECTOR_PROFILES = {"bank", "insurance", "reit", "bdc", "mlp", "asset_manager"}
# ISIN prefixes that prove a US-domiciled security.
_US_ISIN_PREFIXES = ("US",)
def requires_unit_reconciliation(row: Mapping[str, Any]) -> bool:
"""Fail-closed unit-context gate shared by discovery and screening.
Valuation ratios are trustworthy only when the listing price and the
statement/estimate figures are provably in the same currency and share
unit. That is the case only for a row PROVEN domestic (country US, no
foreign ISIN, not an ADR/ADS, currency USD or unstated). Anything
else — a missing country, a non-USD currency, a non-US ISIN, or an
ADR/ADS flag — requires explicit ``unit_reconciliation_verified``
evidence: an unknown unit context is treated as unreconciled, never as
domestic (QFIN passed the old gate whenever the provider dropped the
country field).
"""
if row.get("unit_reconciliation_verified") is True:
return False
if row.get("is_adr") is True:
return True
country = (_text(row.get("country")) or "").upper()
if country != "US":
return True
currency = (_text(row.get("currency")) or "").upper()
if currency and currency != "USD":
return True
isin = (_text(row.get("isin")) or "").upper()
if isin and not isin.startswith(_US_ISIN_PREFIXES):
return True
return False
DEFAULTS: dict[str, Any] = {
"requested_min_market_cap": DEFAULT_REQUESTED_MIN_MARKET_CAP,
"requested_max_market_cap": DEFAULT_REQUESTED_MAX_MARKET_CAP,
"min_market_cap": 500_000_000,
"max_market_cap": 20_000_000_000,
"min_price": 5.0,
"min_average_daily_dollar_volume": 5_000_000,
"hard_min_average_daily_dollar_volume": 1_000_000,
"minimum_average_volume_period_days": 20,
"max_deep_dive_candidates": 5,
"selection_lane_quota_core_garp": 2,
"selection_lane_quota_high_growth": 1,
"selection_lane_quota_near_miss": 1,
"selection_lane_quota_cyclical": 1,
"maximum_selected_per_sector": 2,
"preferred_forward_pe": 20.0,
"high_growth_exception_max_forward_pe": 30.0,
"near_miss_max_forward_pe": 22.0,
"preferred_ev_to_fcf": 20.0,
"preferred_fcf_yield_pct": 5.0,
"minimum_revenue_growth_pct": 8.0,
"minimum_per_share_growth_pct": 12.0,
"near_miss_min_per_share_growth_pct": 8.0,
"high_growth_exception_growth_pct": 20.0,
"minimum_roic_pct": 8.0,
"preferred_max_dilution_pct": 3.0,
"preferred_max_net_debt_to_ebitda": 2.5,
"hard_max_net_debt_to_ebitda": 4.0,
"maximum_forward_pe_for_economic_screen": 60.0,
"minimum_listing_data_coverage_pct": 95.0,
"minimum_discovery_analyst_count": 2,
"maximum_enrichment_attempts": 60,
"sector_review_selection_penalty": 5.0,
"near_miss_selection_penalty": 4.0,
"missing_deep_dive_field_penalty": 1.5,
"forward_pe_reconciliation_tolerance_pct": 5.0,
"maximum_forward_eps_dispersion_pct": 100.0,
"max_estimate_age_days": 45,
"maximum_fy1_horizon_days": 430,
}
class ScreenError(ValueError):
"""Raised when screening input or configuration is invalid."""
def _mapping(value: Any) -> dict[str, Any]:
return dict(value) if isinstance(value, Mapping) else {}
def _text(value: Any) -> str | None:
if isinstance(value, str):
result = value.strip()
return result or None
return None
def _number(value: Any) -> float | None:
if isinstance(value, bool):
return None
if isinstance(value, (int, float)):
result = float(value)
return result if math.isfinite(result) else None
if isinstance(value, str):
try:
result = float(value.replace(",", "").strip())
except ValueError:
return None
return result if math.isfinite(result) else None
return None
def _integer(value: Any) -> int | None:
number = _number(value)
return int(number) if number is not None and number.is_integer() else None
def _bool(value: Any) -> bool | None:
if isinstance(value, bool):
return value
if isinstance(value, str):
lowered = value.strip().lower()
if lowered in {"true", "1", "yes", "y"}:
return True
if lowered in {"false", "0", "no", "n"}:
return False
return None
def _load_rows(path: Path) -> list[dict[str, Any]]:
suffix = path.suffix.lower()
if suffix == ".csv":
with path.open(newline="", encoding="utf-8-sig") as handle:
return [dict(row) for row in csv.DictReader(handle)]
if suffix in {".jsonl", ".ndjson"}:
rows: list[dict[str, Any]] = []
for line_no, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
if not line.strip():
continue
value = json.loads(line)
if not isinstance(value, Mapping):
raise ScreenError(f"line {line_no} is not an object")
rows.append(dict(value))
return rows
value = json.loads(path.read_text(encoding="utf-8"))
if isinstance(value, list):
return [_mapping(row) for row in value]
if isinstance(value, Mapping) and isinstance(value.get("rows"), list):
return [_mapping(row) for row in value["rows"]]
raise ScreenError("input must be a JSON array, {rows:[...]}, JSONL, or CSV")
def _metric(row: Mapping[str, Any], *keys: str) -> float | None:
for key in keys:
value = _number(row.get(key))
if value is not None:
return value
return None
def _symbol(row: Mapping[str, Any]) -> str:
return (_text(row.get("symbol")) or _text(row.get("ticker")) or "UNKNOWN").upper()
def _exchange(row: Mapping[str, Any]) -> str:
return (_text(row.get("exchange")) or _text(row.get("exchangeShortName")) or "").upper()
def _liquidity(row: Mapping[str, Any], config: Mapping[str, Any]) -> dict[str, Any]:
return normalize_liquidity(
row,
minimum_period_days=int(config.get("minimum_average_volume_period_days", 20)),
)
def _adv(row: Mapping[str, Any], config: Mapping[str, Any]) -> float | None:
return _liquidity(row, config).get("value")
def _active(row: Mapping[str, Any]) -> bool | None:
if "is_actively_trading" in row:
return _bool(row.get("is_actively_trading"))
return _bool(row.get("isActivelyTrading"))
def _common(row: Mapping[str, Any]) -> bool | None:
if "is_common_stock" in row:
return _bool(row.get("is_common_stock"))
if "common_stock" in row:
return _bool(row.get("common_stock"))
return None
def _listing_decision(row: Mapping[str, Any], config: Mapping[str, Any]) -> dict[str, Any]:
symbol = _symbol(row)
exchange = _exchange(row)
price = _metric(row, "price", "last")
market_cap = _metric(row, "market_cap", "marketCap")
liquidity = _liquidity(row, config)
adv = liquidity.get("value")
active = _active(row)
common = _common(row)
reasons: list[str] = []
status = "in_scope"
if exchange not in ALLOWED_EXCHANGES:
status, reasons = "out_of_scope", ["exchange_out_of_scope"]
elif active is False:
status, reasons = "excluded", ["inactive_symbol"]
elif common is False:
status, reasons = "excluded", ["not_common_stock"]
elif price is None:
status, reasons = "listing_data_incomplete", ["price_unavailable"]
elif price < float(config["min_price"]):
status, reasons = "out_of_scope", ["price_below_minimum"]
elif market_cap is None:
status, reasons = "listing_data_incomplete", ["market_cap_unavailable"]
elif market_cap < float(config["min_market_cap"]) or market_cap > float(
config["max_market_cap"]
):
status, reasons = "out_of_scope", ["market_cap_out_of_range"]
elif liquidity.get("valid_for_screen") is not True:
status, reasons = (
"liquidity_review",
list(liquidity.get("reasons") or ["average_liquidity_evidence_required"]),
)
elif adv < float(config["hard_min_average_daily_dollar_volume"]):
status, reasons = "excluded", ["extreme_illiquidity"]
elif adv < float(config["min_average_daily_dollar_volume"]):
status, reasons = "liquidity_review", ["liquidity_below_preferred"]
listing_complete = (
symbol != "UNKNOWN" and bool(exchange) and price is not None and market_cap is not None
)
return {
"symbol": symbol,
"company_name": _text(row.get("company_name"))
or _text(row.get("companyName"))
or _text(row.get("name")),
"exchange": exchange,
"sector": _text(row.get("sector")),
"industry": _text(row.get("industry")),
"listing_decision": {"status": status, "reasons": reasons},
"listing_data_complete": listing_complete,
"eligible_for_candidate_generation": status in {"in_scope", "liquidity_review"},
"metrics": {
"price": price,
"market_cap": market_cap,
"average_daily_dollar_volume": adv,
"average_daily_dollar_volume_method": liquidity.get("method"),
"average_volume_period_days": liquidity.get("period_days"),
"liquidity_source_ids": liquidity.get("source_ids", []),
"liquidity_valid_for_screen": liquidity.get("valid_for_screen"),
"is_actively_trading": active,
"is_common_stock": common,
},
}
def _candidate_decision(
row: Mapping[str, Any], config: Mapping[str, Any], mode: str, analysis_as_of: str
) -> dict[str, Any]:
"""Classify one discovery row without treating guidelines as hard cutoffs.
The broad screen answers two separate questions:
1. Is there enough forward valuation/growth evidence to prioritize the name?
2. What additional sector, cyclicality, and quality work is required before ranking?
A cyclical normalization requirement is therefore a deep-dive requirement,
not an automatic rejection. Missing bank/REIT/BDC/MLP-specific valuation or
auto-dealer floorplan leverage remains blocking because the general-company
multiples are not decision-useful for those profiles.
"""
symbol = _symbol(row)
exchange = _exchange(row)
price = _metric(row, "price", "last")
market_cap = _metric(row, "market_cap", "marketCap")
liquidity = _liquidity(row, config)
adv = liquidity.get("value")
active = _active(row)
common = _common(row)
revenue_growth = _metric(row, "revenue_growth_pct", "revenue_growth", "fy1_revenue_growth_pct")
eps_growth = _metric(row, "eps_growth_pct", "forward_eps_growth_pct", "fy1_eps_growth_pct")
fcf_ps_growth = _metric(row, "fcf_per_share_growth_pct")
explicit_per_share_growth = _metric(row, "per_share_growth_pct")
per_share_values = [
value
for value in (eps_growth, fcf_ps_growth, explicit_per_share_growth)
if value is not None
]
per_share_growth = max(per_share_values) if per_share_values else None
fcf = _metric(row, "standard_fcf", "free_cash_flow")
roic = _metric(row, "roic_pct", "return_on_invested_capital_pct")
forward = normalize_forward_valuation(
row,
price=price,
analysis_as_of=analysis_as_of,
max_age_days=int(config.get("max_estimate_age_days", 45)),
reconciliation_tolerance_pct=float(
config.get("forward_pe_reconciliation_tolerance_pct", 5.0)
),
maximum_dispersion_pct=float(config.get("maximum_forward_eps_dispersion_pct", 100.0)),
maximum_fy1_horizon_days=int(config.get("maximum_fy1_horizon_days", 430)),
)
forward_pe = _number(forward.get("forward_pe"))
ev_to_fcf = _metric(row, "ev_to_fcf")
fcf_yield = _metric(row, "fcf_yield_pct")
dilution = _metric(row, "diluted_share_growth_pct", "dilution_pct")
leverage = _metric(row, "net_debt_to_ebitda")
analyst_count = (
_integer(forward.get("analyst_count"))
or _integer(row.get("analyst_count"))
or _integer(row.get("fy1_analyst_count"))
)
profile_type = (_text(row.get("sector_profile_type")) or "general").lower()
cyclicality = int(_metric(row, "cyclicality_score") or 1)
normalized_metric = _metric(row, "normalized_eps", "normalized_fcf_per_share")
hard: list[str] = []
blocking_review: list[str] = []
nonblocking_review: list[str] = []
missing_discovery: list[str] = []
hard_fail: list[str] = []
guideline_misses: list[str] = []
deep_dive_requirements: list[str] = []
if exchange not in ALLOWED_EXCHANGES:
hard.append("exchange_out_of_scope")
if active is False:
hard.append("inactive_symbol")
if common is False:
hard.append("not_common_stock")
if price is None or price < float(config["min_price"]):
hard.append("price_below_minimum")
if (
market_cap is None
or market_cap < float(config["min_market_cap"])
or market_cap > float(config["max_market_cap"])
):
hard.append("market_cap_out_of_range")
if liquidity.get("valid_for_screen") is not True:
missing_discovery.extend(
list(liquidity.get("reasons") or ["average_liquidity_evidence_required"])
)
elif adv < float(config["hard_min_average_daily_dollar_volume"]):
hard.append("extreme_illiquidity")
elif adv < float(config["min_average_daily_dollar_volume"]):
deep_dive_requirements.append("liquidity_below_preferred")
# Sector profiles whose standard company multiples are not comparable need
# their own valuation metric before discovery scoring is meaningful.
if profile_type in SECTOR_PROFILES:
sector_multiple = _metric(
row, "sector_forward_multiple", "p_to_tbv", "p_to_affo", "p_to_book"
)
sector_growth = _metric(
row,
"sector_per_share_growth_pct",
"affo_per_share_growth_pct",
"tbv_per_share_growth_pct",
)
if sector_multiple is None or sector_growth is None:
blocking_review.append("sector_specific_valuation_required")
if profile_type == "auto_dealership":
adjusted = _metric(row, "sector_adjusted_net_debt_to_ebitda")
floorplan_excluded = _bool(row.get("floorplan_debt_excluded"))
if adjusted is None or floorplan_excluded is not True:
blocking_review.append("sector_adjusted_leverage_required")
else:
leverage = adjusted
# Unit-context gate (fail closed): a USD listing price against
# local-currency (or differently ratioed ADS) statements produces
# meaningless ratios until unit reconciliation is verified — and an
# UNKNOWN context (missing country, non-USD currency, non-US ISIN,
# ADR/ADS) counts as unreconciled, never as domestic.
if requires_unit_reconciliation(row) or (
"foreign_private_issuer_review" in (row.get("provider_prefilter_flags") or [])
and _bool(row.get("unit_reconciliation_verified")) is not True
):
blocking_review.append("unit_reconciliation_required")
# Unit-anomaly circuit breakers (any issuer): impossible "cheapness" is a
# suspected currency/ADS-unit mismatch, never deep value (QFIN: forward
# P/E 0.45x, FCF yield 94% from CNY EPS against a USD ADS price).
latest_actual_eps = _metric(row, "latest_actual_eps", "fy0_actual_eps")
unit_anomaly = bool(
(
forward_pe is not None
and 0 < forward_pe < float(config.get("minimum_plausible_forward_pe", 2.0))
)
or (
fcf_yield is not None
and fcf_yield > float(config.get("maximum_plausible_fcf_yield_pct", 50.0))
)
or (
price is not None
and price > 0
and latest_actual_eps is not None
and latest_actual_eps > 2.0 * price
)
)
if unit_anomaly:
hard_fail.append("unit_mismatch_suspected")
if cyclicality > 2 and normalized_metric is None:
nonblocking_review.append("mid_cycle_normalization_required")
deep_dive_requirements.append("mid_cycle_normalization_required")
estimate_mode = mode in ESTIMATE_MODES
min_analysts = int(config.get("minimum_discovery_analyst_count", 2))
if forward.get("valid") is not True:
missing_discovery.extend(list(forward.get("reasons") or ["forward_valuation_invalid"]))
if per_share_growth is None:
missing_discovery.append("per_share_growth_unavailable")
if estimate_mode and (analyst_count is None or analyst_count < min_analysts):
missing_discovery.append("estimate_breadth_below_discovery_minimum")
all_fundamentals = {
"revenue_growth": revenue_growth,
"per_share_growth": per_share_growth,
"standard_fcf": fcf,
"roic": roic,
"forward_pe": forward_pe,
"leverage": leverage,
"dilution": dilution,
}
if estimate_mode:
if revenue_growth is None:
deep_dive_requirements.append("revenue_growth_to_verify")
for key in ("standard_fcf", "roic", "leverage", "dilution"):
if all_fundamentals[key] is None:
deep_dive_requirements.append(f"{key}_to_verify")
else:
for key, value in all_fundamentals.items():
if value is None:
missing_discovery.append(f"{key}_unavailable")
discovery_evaluable = not hard and not blocking_review and not missing_discovery
# The prompt calls these figures guidelines rather than mechanical gates.
if revenue_growth is not None and revenue_growth < float(config["minimum_revenue_growth_pct"]):
guideline_misses.append("revenue_growth_below_guideline")
if per_share_growth is not None and per_share_growth < float(
config["minimum_per_share_growth_pct"]
):
guideline_misses.append("per_share_growth_below_guideline")
if roic is not None and roic < float(config["minimum_roic_pct"]):
guideline_misses.append("roic_below_guideline")
if leverage is not None and leverage > float(config["preferred_max_net_debt_to_ebitda"]):
guideline_misses.append("leverage_above_guideline")
# Only severe, economically disqualifying findings are broad-screen fails.
if fcf is not None and fcf <= 0:
hard_fail.append("non_positive_standard_fcf")
if roic is not None and roic < 0:
hard_fail.append("negative_roic")
if (
leverage is not None
and leverage > float(config.get("hard_max_net_debt_to_ebitda", 4.0))
# General-company net debt/EBITDA is meaningless for balance-sheet
# businesses (mortgage REITs, banks, insurers, BDCs, MLPs, asset
# managers): a mortgage REIT at 13x is normal, not a hard failure.
# Those profiles are already routed to
# sector_specific_valuation_required until sector metrics exist.
and profile_type not in SECTOR_PROFILES
):
hard_fail.append("excessive_leverage")
if forward_pe is not None and forward_pe > float(
config.get("maximum_forward_pe_for_economic_screen", 60.0)
):
hard_fail.append("extreme_forward_valuation")
if (
revenue_growth is not None
and revenue_growth < 0
and per_share_growth is not None
and per_share_growth < 0
):
hard_fail.append("negative_forward_growth")
normal_value = (
(forward_pe is not None and forward_pe <= float(config["preferred_forward_pe"]))
or (ev_to_fcf is not None and ev_to_fcf <= float(config["preferred_ev_to_fcf"]))
or (fcf_yield is not None and fcf_yield >= float(config["preferred_fcf_yield_pct"]))
)
if estimate_mode:
high_growth_exception = all(
[
forward_pe is not None
and forward_pe <= float(config["high_growth_exception_max_forward_pe"]),
per_share_growth is not None
and per_share_growth >= float(config["high_growth_exception_growth_pct"]),
]
)
else:
high_growth_exception = all(
[
forward_pe is not None
and forward_pe <= float(config["high_growth_exception_max_forward_pe"]),
per_share_growth is not None
and per_share_growth >= float(config["high_growth_exception_growth_pct"]),
roic is not None and roic >= 10,
fcf is not None and fcf > 0,
dilution is not None and dilution <= float(config["preferred_max_dilution_pct"]),
leverage is not None
and leverage <= float(config["preferred_max_net_debt_to_ebitda"]),
]
)
standard_growth_pass = per_share_growth is not None and per_share_growth >= float(
config["minimum_per_share_growth_pct"]
)
near_miss = all(
[
normal_value,
forward_pe is not None
and forward_pe <= float(config.get("near_miss_max_forward_pe", 22.0)),
per_share_growth is not None
and per_share_growth >= float(config.get("near_miss_min_per_share_growth_pct", 8.0)),
revenue_growth is None or revenue_growth >= 0,
]
)
economic_pass = normal_value and standard_growth_pass
if forward_pe is not None and not normal_value and not high_growth_exception:
hard_fail.append("valuation_not_supported_by_growth")
exhaustion_reason = _text(row.get("enrichment_exhaustion_reason"))
exhaustion_sources = row.get("enrichment_source_ids")
exhaustion_evidenced = (
_bool(row.get("enrichment_exhausted")) is True
and bool(exhaustion_reason)
and isinstance(exhaustion_sources, list)
and bool(exhaustion_sources)
and all(isinstance(value, str) and value.strip() for value in exhaustion_sources)
)
if hard:
status = "excluded"
selection_eligible = False
resolution = "resolved"
elif (blocking_review or missing_discovery) and exhaustion_evidenced:
status = "unavailable_after_enrichment"
selection_eligible = False
resolution = "resolved"
elif blocking_review or missing_discovery:
status = "needs_enrichment"
selection_eligible = False
resolution = "unresolved"
elif hard_fail:
status = "screened_out"
selection_eligible = False
resolution = "resolved"
elif high_growth_exception or economic_pass or near_miss:
selection_eligible = True
resolution = "resolved"
if nonblocking_review:
status = "sector_review_required"
elif high_growth_exception and not normal_value:
status = "passed_exception"
elif near_miss and not economic_pass:
status = "near_miss_review"
else:
status = "passed"
else:
status = "screened_out"
selection_eligible = False
resolution = "resolved"
if not hard_fail:
hard_fail.append("growth_and_valuation_combination_below_review_threshold")
valuation_score = (
max(0.0, 35.0 - min(float(forward_pe or 35.0), 100.0)) if forward_pe is not None else 0.0
)
if fcf_yield is not None:
valuation_score += max(0.0, min(20.0, fcf_yield * 2.0))
growth_score = max(0.0, min(35.0, per_share_growth or 0.0)) + max(
0.0, min(15.0, revenue_growth or 0.0)
)
quality_score = max(0.0, min(20.0, roic or 0.0)) if roic is not None else 0.0
dilution_score = (
max(0.0, 10.0 - max(0.0, dilution or 0.0) * 2.0) if dilution is not None else 0.0
)
leverage_score = (
max(0.0, 10.0 - max(0.0, leverage or 0.0) * 2.0) if leverage is not None else 0.0
)
breadth_score = min(10.0, float(analyst_count or 0) * 2.0)
broad_score = (
valuation_score
+ growth_score
+ quality_score
+ dilution_score
+ leverage_score
+ breadth_score
)
broad_score -= max(0, cyclicality - 2) * 2.0
broad_score -= len(deep_dive_requirements) * float(
config.get("missing_deep_dive_field_penalty", 1.5)
)
if nonblocking_review:
broad_score -= float(config.get("sector_review_selection_penalty", 5.0))
if near_miss and not economic_pass:
broad_score -= float(config.get("near_miss_selection_penalty", 4.0))
if per_share_growth is not None and per_share_growth > 100:
broad_score -= 10.0 # likely base-effect or near-zero denominator; verify before ranking
broad_score = max(0.0, broad_score)
completeness_count = sum(value is not None for value in all_fundamentals.values())
fundamental_complete = completeness_count == len(all_fundamentals)
enrichment_attempted = _bool(row.get("enrichment_attempted")) is True or any(
value is not None
for value in (forward_pe, per_share_growth, revenue_growth, fcf, roic, leverage, dilution)
)
enrichment_resolved = resolution == "resolved"
priority_score = broad_score
if high_growth_exception:
priority_score += 6.0
if status == "near_miss_review":
priority_score -= 2.0
if status == "sector_review_required":
priority_score -= max(0, cyclicality - 2) * 2.0
return {
"symbol": symbol,
"company_name": _text(row.get("company_name"))
or _text(row.get("companyName"))
or _text(row.get("name")),
"exchange": exchange,
"sector": _text(row.get("sector")),
"industry": _text(row.get("industry")),
"candidate_generation_mode": mode,
"decision": {
"status": status,
"preselection_status": status,
"hard_reasons": sorted(set(hard)),
"review_reasons": sorted(set(blocking_review + nonblocking_review + missing_discovery)),
"blocking_review_reasons": sorted(set(blocking_review + missing_discovery)),
"guideline_misses": sorted(set(guideline_misses)),
"screen_fail_reasons": sorted(set(hard_fail)),
"deep_dive_requirements": sorted(set(deep_dive_requirements)),
"exception_admitted": high_growth_exception and not normal_value,
"selection_eligible": selection_eligible,
"resolution": resolution,
"enrichment_exhausted": exhaustion_evidenced,
"enrichment_exhaustion_reason": exhaustion_reason,
"enrichment_source_ids": exhaustion_sources
if isinstance(exhaustion_sources, list)
else [],
},
"broad_score": round(broad_score, 4) if discovery_evaluable else None,
"deep_dive_priority_score": round(priority_score, 4)
if discovery_evaluable and selection_eligible
else None,
"discovery_evaluable": discovery_evaluable,
"selection_eligible": selection_eligible,
"enrichment_attempted": enrichment_attempted,
"enrichment_resolved": enrichment_resolved,
"fundamental_completeness_count": completeness_count,
"fundamental_complete": fundamental_complete,
"metrics": {
"price": price,
"market_cap": market_cap,
"average_daily_dollar_volume": adv,
"average_daily_dollar_volume_method": liquidity.get("method"),
"average_volume": liquidity.get("average_volume"),
"average_volume_period_days": liquidity.get("period_days"),
"liquidity_source_ids": liquidity.get("source_ids", []),
"liquidity_valid_for_screen": liquidity.get("valid_for_screen"),
"liquidity_validation_reasons": liquidity.get("reasons", []),
"revenue_growth_pct": revenue_growth,
"per_share_growth_pct": per_share_growth,
"standard_fcf": fcf,
"roic_pct": roic,
"forward_pe": forward_pe,
"forward_eps": forward.get("forward_eps"),
"forward_pe_period": forward.get("period"),
"forward_fiscal_year": forward.get("fiscal_year"),
"forward_period_end": forward.get("period_end"),
"forward_horizon_days": forward.get("horizon_days"),
"forward_metric_origin": forward.get("origin"),
"forward_estimate_as_of": forward.get("estimate_as_of"),
"forward_estimate_source_ids": forward.get("source_ids", []),
"forward_eps_dispersion_pct": forward.get("dispersion_pct"),
"forward_validation_reasons": forward.get("reasons", []),
"ev_to_fcf": ev_to_fcf,
"fcf_yield_pct": fcf_yield,
"dilution_pct": dilution,
"net_debt_to_ebitda": leverage,
"analyst_count": analyst_count,
"cyclicality_score": cyclicality,
"growth_pattern": _text(row.get("growth_pattern")),
"sector_profile_type": profile_type,
},
}
def _canonical_line(row: Mapping[str, Any]) -> str:
return json.dumps(dict(row), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n"
def _merge_pool_rows(
universe_rows: Sequence[Mapping[str, Any]], pool_rows: Sequence[Mapping[str, Any]]
) -> tuple[list[dict[str, Any]], list[str]]:
universe_index = {_symbol(row): dict(row) for row in universe_rows if _symbol(row) != "UNKNOWN"}
merged: list[dict[str, Any]] = []
missing: list[str] = []
seen: set[str] = set()
for raw in pool_rows:
symbol = _symbol(raw)
if symbol in seen:
raise ScreenError(f"candidate pool contains duplicate symbol {symbol}")
seen.add(symbol)
base = universe_index.get(symbol)
if base is None:
missing.append(symbol)
merged.append(dict(raw))
continue
combined = dict(base)
combined.update(dict(raw))
combined["symbol"] = symbol
merged.append(combined)
return merged, sorted(missing)
def _enrichment_queue(rows: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]:
"""Return only unresolved rows, ordered by GARP information value.
Liquidity is a tiebreaker, not the main ranking axis. A name with a credible
forward valuation/growth combination is enriched before a liquid but
economically implausible base-effect name.
"""
queue: list[dict[str, Any]] = []
for row in rows:
if row.get("enrichment_resolved") is True:
continue
status = _text(_mapping(row.get("decision")).get("status")) or "needs_enrichment"
metrics = _mapping(row.get("metrics"))
forward_pe = _number(metrics.get("forward_pe"))
per_share_growth = _number(metrics.get("per_share_growth_pct"))
revenue_growth = _number(metrics.get("revenue_growth_pct"))
analyst_count = _integer(metrics.get("analyst_count")) or 0
cyclicality = _integer(metrics.get("cyclicality_score")) or 1
reasons = _mapping(row.get("decision")).get("review_reasons") or []
score = 0.0
if forward_pe is not None:
score += max(-20.0, 35.0 - min(forward_pe, 80.0))
else:
score -= 12.0
if per_share_growth is not None:
score += max(-10.0, min(35.0, per_share_growth))
if per_share_growth > 100:
score -= 12.0
else:
score -= 12.0
if revenue_growth is not None:
score += max(-10.0, min(15.0, revenue_growth))
score += min(10.0, analyst_count * 1.5)
score -= max(0, cyclicality - 2) * 2.0
score -= len(reasons) * 1.5
queue.append(
{
"symbol": row.get("symbol"),
"status": status,
"reasons": reasons,
"enrichment_priority_score": round(score, 4),
"forward_pe": forward_pe,
"per_share_growth_pct": per_share_growth,
"revenue_growth_pct": revenue_growth,
"analyst_count": analyst_count,
"cyclicality_score": cyclicality,
"growth_pattern": _text(row.get("growth_pattern")),
"average_daily_dollar_volume": metrics.get("average_daily_dollar_volume"),
"fundamental_completeness_count": row.get("fundamental_completeness_count"),
}
)
queue.sort(
key=lambda row: (
-float(row.get("enrichment_priority_score") or 0.0),
-int(row.get("analyst_count") or 0),
-float(row.get("average_daily_dollar_volume") or 0.0),
str(row.get("symbol") or ""),
)
)
return queue
def _validate_candidate_pool_scope(
*,
candidate_generation_mode: str,
universe_rows: Sequence[Mapping[str, Any]],
candidate_rows: Sequence[Mapping[str, Any]],
candidate_pool_covers_in_scope: bool,
discovery_audit: Mapping[str, Any] | None,
requested_min_market_cap: float,
requested_max_market_cap: float,
) -> tuple[bool, str, dict[str, Any], list[str]]:
"""Verify what population the economic candidate pool represents.
Full-universe mode must cover every in-scope listing. A bounded discovery
pool may support a final *scoped* ranking only when its generation audit is
reproducible, uses validated average liquidity, and its stratification plan
spans the user's full requested market-cap range. A convenient single band
is not an acceptable substitute for the user's request.
"""
audit = dict(discovery_audit or {})
candidate_symbols = sorted(
{_symbol(row) for row in candidate_rows if _symbol(row) != "UNKNOWN"}
)
universe_symbols = sorted({_symbol(row) for row in universe_rows if _symbol(row) != "UNKNOWN"})
reasons: list[str] = []
if candidate_generation_mode == "full_universe_fundamentals":
valid = candidate_pool_covers_in_scope
if not valid:
reasons.append("full-universe candidate pool does not cover every in-scope listing")
return (
valid,
"full_listing_universe",
{
"valid": valid,
"selection_method": "full_universe_fundamentals",
"input_row_count": len(universe_rows),
"selected_count": len(candidate_rows),
"selected_symbols": candidate_symbols,
"coverage_plan": {
"coverage_plan_valid": valid,
"user_requested_range_spanned": valid,
},
},
reasons,
)
if candidate_generation_mode == "sharded_snapshot":
verification = _mapping(audit.get("snapshot_verification"))
digest = _text(verification.get("snapshot_verification_digest"))
classifications = _mapping(audit.get("snapshot_classification_totals"))
classified_total = sum(
value for raw in classifications.values() if (value := _integer(raw)) is not None
)
economic_attempt_count = _integer(audit.get("economic_attempt_count"))
evaluable_count = _integer(audit.get("economically_evaluable_count"))
input_count = _integer(audit.get("input_row_count"))
selected_count = _integer(audit.get("selected_count"))
declared_symbols = sorted(
str(value).upper()
for value in (audit.get("selected_symbols") or [])
if str(value).strip()
)
source_ids = audit.get("source_ids")
target_pool_size = _integer(audit.get("target_pool_size"))
preliminary_pool_count = _integer(audit.get("preliminary_pool_count"))
eligible_pool_exhausted = audit.get("eligible_pool_exhausted") is True
runtime = _mapping(audit.get("runtime"))
expected_runtime = runtime_metadata()
if audit.get("selection_method") != "sharded_snapshot_multilane":
reasons.append("sharded-snapshot audit has an incompatible selection_method")
if audit.get("valid") is not True:
reasons.append("sharded-snapshot generation audit is not valid")
if input_count != len(universe_rows):
reasons.append("sharded-snapshot input_row_count does not match frozen universe")
if economic_attempt_count != len(universe_rows):
reasons.append("sharded-snapshot economic attempts do not cover frozen universe")
if classified_total != len(universe_rows):
reasons.append("sharded-snapshot classifications do not cover frozen universe")
allowed_classifications = {
"evaluable",
"no_estimates",
"negative_eps",
"unit_mismatch",
"excluded",
}
if set(classifications) - allowed_classifications:
reasons.append("sharded-snapshot audit has an unknown classification")
if evaluable_count is None or evaluable_count < len(candidate_rows):
reasons.append("sharded-snapshot evaluable count is smaller than its final pool")
if any(row.get("snapshot_classification") != "evaluable" for row in candidate_rows):
reasons.append("sharded-snapshot final pool contains a non-evaluable row")
if any(row.get("quality_probe_attempted") is not True for row in candidate_rows):
reasons.append("sharded-snapshot final pool contains an unprobed row")
if selected_count != len(candidate_rows) or declared_symbols != candidate_symbols:
reasons.append("sharded-snapshot selected rows do not match its generation audit")
if target_pool_size is None or not 30 <= target_pool_size <= 50:
reasons.append("sharded-snapshot target_pool_size must be between 30 and 50")
elif len(candidate_rows) < target_pool_size and not (
eligible_pool_exhausted
and preliminary_pool_count is not None
and preliminary_pool_count < target_pool_size
):
reasons.append("a short sharded-snapshot pool must prove eligible-pool exhaustion")
if not (
verification.get("ready_for_screening") is True
and verification.get("classification_matches_universe") is True
and _integer(verification.get("classified_total")) == len(universe_rows)
and digest is not None
and len(digest) == 64
):
reasons.append("sharded-snapshot verification is missing or not screen-ready")
if audit.get("snapshot_verification_digest") != digest:
reasons.append("sharded-snapshot digest does not match its verification verdict")
if (
not isinstance(source_ids, list)
or not source_ids
or not all(isinstance(value, str) and value.strip() for value in source_ids)
):
reasons.append("sharded-snapshot audit requires source_ids")
if not _text(audit.get("artifact_path")) or not _text(audit.get("artifact_sha256")):
reasons.append("sharded-snapshot audit requires a bound pool artifact")
for key in (
"skill_name",
"skill_version",
"schema_version",
"contract_revision",
"runtime_fingerprint",
):
if runtime.get(key) != expected_runtime.get(key):
reasons.append(f"sharded-snapshot runtime {key} is stale or mismatched")
valid = not reasons
normalized = dict(audit)
normalized.update(
{
"valid": valid,
"coverage_scope": "full_listing_universe",
"actual_input_row_count": len(universe_rows),
"actual_selected_count": len(candidate_rows),
"actual_selected_symbols": candidate_symbols,
"universe_symbol_count": len(universe_symbols),
}
)
return valid, "full_listing_universe", normalized, reasons
expected_methods = {
"liquidity_stratified_estimates": {
"sector_market_cap_stratified_liquidity",
"sector_market_cap_stratified_validated_liquidity",
},
"provider_prefilter": {"provider_prefilter", "provider_screen"},
"available_fundamentals": {
"available_fundamentals",
"sector_market_cap_stratified_validated_liquidity",
},
"user_supplied": {"user_supplied"},
}
method = (_text(audit.get("selection_method")) or "").lower()
declared_symbols = sorted(
str(value).upper() for value in (audit.get("selected_symbols") or []) if str(value).strip()
)
input_count = _integer(audit.get("input_row_count"))
selected_count = _integer(audit.get("selected_count"))
source_ids = audit.get("source_ids")
artifact_sha = _text(audit.get("artifact_sha256"))
artifact_path = _text(audit.get("artifact_path"))
if method not in expected_methods.get(candidate_generation_mode, set()):
reasons.append("candidate-pool generation audit has an incompatible selection_method")
if input_count != len(universe_rows):
reasons.append(
"candidate-pool generation audit input_row_count does not match listing frame"
)
if selected_count != len(candidate_rows):
reasons.append(
"candidate-pool generation audit selected_count does not match candidate pool"
)
if declared_symbols != candidate_symbols:
reasons.append(
"candidate-pool generation audit selected_symbols do not match candidate pool"
)
if (
not isinstance(source_ids, list)
or not source_ids
or not all(isinstance(value, str) and value.strip() for value in source_ids)
):
reasons.append("candidate-pool generation audit requires source_ids")
if not artifact_sha or not artifact_path:
reasons.append("candidate-pool generation audit requires artifact_path and artifact_sha256")
if audit.get("valid") is not True:
reasons.append("candidate-pool generation audit is not valid")
runtime = _mapping(audit.get("runtime"))
expected_runtime = runtime_metadata()
for key in (
"skill_name",
"skill_version",
"schema_version",
"contract_revision",
"runtime_fingerprint",
):
if runtime.get(key) != expected_runtime.get(key):
reasons.append(f"candidate-pool generation runtime {key} is stale or mismatched")
if candidate_generation_mode == "provider_prefilter":
if audit.get("provider_exhausted") is not True:
reasons.append("provider-prefilter audit requires provider_exhausted=true")
coverage_plan = _mapping(audit.get("coverage_plan"))
if coverage_plan.get("multi_lane_provider_prefilter") is not True:
reasons.append("provider-prefilter audit must combine multiple opportunity lanes")
if (_integer(audit.get("lane_coverage_count")) or 0) < 3:
reasons.append(
"provider-prefilter audit requires at least three represented opportunity lanes"
)
if audit.get("pool_adequate") is not True:
reasons.append(
"provider-prefilter pool does not meet the minimum breadth or exhaustion rule"
)
liquidity_validation = _mapping(audit.get("liquidity_validation"))
if liquidity_validation.get("basis_validated") is not True:
reasons.append("provider-prefilter liquidity basis is not validated")
if candidate_generation_mode in {"liquidity_stratified_estimates", "available_fundamentals"}:
liquidity_validation = _mapping(audit.get("liquidity_validation"))
if liquidity_validation.get("basis_validated") is not True:
reasons.append("discovery-pool liquidity basis is not validated")
if (_integer(liquidity_validation.get("minimum_window_days")) or 0) < int(
DEFAULTS.get("minimum_average_volume_period_days", 20)
):
reasons.append("discovery-pool liquidity window is below the minimum")
coverage_plan = _mapping(audit.get("coverage_plan"))
if coverage_plan.get("coverage_plan_valid") is not True:
reasons.append("discovery-pool coverage plan is not valid")
if coverage_plan.get("user_requested_range_spanned") is not True:
reasons.append("discovery pool does not span the user-requested market-cap range")
if coverage_plan.get("market_cap_buckets_cover_user_requested_range") is not True:
reasons.append(
"discovery-pool market-cap buckets do not cover the user-requested range"
)
if coverage_plan.get("single_band_only") is True:
reasons.append("single-band discovery pool cannot stand in for the requested universe")
scope = _mapping(audit.get("scope"))
if _number(scope.get("user_requested_min_market_cap")) != requested_min_market_cap:
reasons.append("discovery audit requested lower market-cap bound does not match")
if _number(scope.get("user_requested_max_market_cap")) != requested_max_market_cap:
reasons.append("discovery audit requested upper market-cap bound does not match")
if scope.get("user_requested_scope_complete") is not True:
reasons.append("discovery audit does not preserve the user-requested scope")
if scope.get("scope_valid") is not True:
reasons.append("discovery audit scope is invalid")
scope_name = {
"liquidity_stratified_estimates": "stratified_discovery_pool",
"provider_prefilter": "provider_prefilter",
"available_fundamentals": "bounded_available_fundamentals",
"user_supplied": "user_supplied",
}.get(candidate_generation_mode, "bounded_candidate_pool")
valid = not reasons
normalized = dict(audit)
normalized.update(
{
"valid": valid,
"coverage_scope": scope_name,
"actual_input_row_count": len(universe_rows),
"actual_selected_count": len(candidate_rows),
"actual_selected_symbols": candidate_symbols,
"universe_symbol_count": len(universe_symbols),
}
)
return valid, scope_name, normalized, reasons
def _selection_lane(row: Mapping[str, Any]) -> str:
"""Assign one deterministic research lane to an economically reviewable row.
Lanes prevent a low-cost global sort from filling the entire deep-dive budget
with one style. They preserve the prompt's four useful opportunity types:
core GARP, high-growth exceptions, quality near misses, and cyclicals that
require mid-cycle normalization.
"""
decision = _mapping(row.get("decision"))
preselection = (
_text(decision.get("preselection_status")) or _text(decision.get("status")) or ""
).lower()
metrics = _mapping(row.get("metrics"))
cyclicality = int(_number(metrics.get("cyclicality_score")) or 1)
if cyclicality >= 3 or "mid_cycle_normalization_required" in set(
decision.get("deep_dive_requirements") or []
):
return "cyclical_normalization"
if preselection == "passed_exception" or decision.get("exception_admitted") is True:
return "high_growth_exception"
if preselection == "near_miss_review":
return "quality_near_miss"
growth_pattern = (
_text(metrics.get("growth_pattern")) or _text(row.get("growth_pattern")) or ""
).lower()
if growth_pattern == "trough_recovery":
# FY1 sits below the latest actual: the FY1->FY3 CAGR is a recovery,
# not compounding growth, so the name is reviewed as a near miss.
return "quality_near_miss"
return "core_garp"
def _select_multilane(
selectable: Sequence[Mapping[str, Any]],
*,
limit: int,
config: Mapping[str, Any],
) -> tuple[list[dict[str, Any]], dict[str, int]]:
"""Select a diversified deep-dive set using deterministic lane quotas.
Quotas are minimum targets, not hard reservations: unused slots are
backfilled by global priority. A sector cap prevents one industry from
consuming the entire research budget unless there are no alternatives.
"""
ranked = [dict(row) for row in selectable]
ranked.sort(
key=lambda row: (
-float(row.get("deep_dive_priority_score") or 0.0),
-float(row.get("broad_score") or 0.0),
-int(row.get("fundamental_completeness_count") or 0),
-float(_mapping(row.get("metrics")).get("analyst_count") or 0),
str(row.get("symbol") or ""),
)
)
quotas = {
"core_garp": int(config.get("selection_lane_quota_core_garp", 2)),
"high_growth_exception": int(config.get("selection_lane_quota_high_growth", 1)),
"quality_near_miss": int(config.get("selection_lane_quota_near_miss", 1)),
"cyclical_normalization": int(config.get("selection_lane_quota_cyclical", 1)),
}
max_per_sector = max(1, int(config.get("maximum_selected_per_sector", 2)))
selected: list[dict[str, Any]] = []
selected_symbols: set[str] = set()
sector_counts: dict[str, int] = {}
lane_counts: dict[str, int] = {key: 0 for key in quotas}
def can_add(row: Mapping[str, Any], *, ignore_sector_cap: bool = False) -> bool:
symbol = str(row.get("symbol") or "")
sector = (_text(row.get("sector")) or "UNKNOWN").upper()
if not symbol or symbol in selected_symbols:
return False
return ignore_sector_cap or sector_counts.get(sector, 0) < max_per_sector
def add(row: Mapping[str, Any]) -> None:
item = dict(row)
lane = _selection_lane(item)
item["selection_lane"] = lane
selected.append(item)
selected_symbols.add(str(item.get("symbol") or ""))
sector = (_text(item.get("sector")) or "UNKNOWN").upper()
sector_counts[sector] = sector_counts.get(sector, 0) + 1
lane_counts[lane] = lane_counts.get(lane, 0) + 1
plan_total = sum(max(0, value) for value in quotas.values())
if limit < plan_total:
# Budget smaller than the lane plan (e.g. 3 deep dives vs a 2/1/1/1
# plan): a lane-first fill would hand every slot to the first lanes in
# plan order and never reach the cyclical lane even when its names
# rank highest. Walk the priority order instead and treat each lane
# quota as a cap, so every lane's best name competes on priority.
for row in ranked:
if len(selected) >= limit:
break
lane = _selection_lane(row)
if lane_counts.get(lane, 0) >= max(0, quotas[lane]):
continue
if can_add(row):
add(row)
else:
for lane in (
"core_garp",
"high_growth_exception",
"quality_near_miss",
"cyclical_normalization",
):
target = max(0, quotas[lane])
for row in ranked:
if len(selected) >= limit or lane_counts.get(lane, 0) >= target:
break
if _selection_lane(row) == lane and can_add(row):
add(row)
for row in ranked:
if len(selected) >= limit:
break
if can_add(row):
add(row)
# Do not leave budget unused solely because all remaining names share a
# sector. The cap is a diversification preference, not a hidden filter.
for row in ranked:
if len(selected) >= limit:
break
if can_add(row, ignore_sector_cap=True):
add(row)
return selected, lane_counts
def run_layered(
universe_rows: Sequence[Mapping[str, Any]],
candidate_rows: Sequence[Mapping[str, Any]],
config: Mapping[str, Any],
*,
analysis_as_of: str,
universe_source_ids: Sequence[str],
candidate_source_ids: Sequence[str],
candidate_generation_mode: str,
retrieval_min_market_cap: float,
retrieval_max_market_cap: float,
requested_min_market_cap: float | None = None,
requested_max_market_cap: float | None = None,
retrieval_scope_explicit: bool = True,
scope_override_authorized: bool = False,
scope_reduction_reason: str | None = None,
user_scope_evidence: str | None = None,
candidate_pool_exhausted: bool = False,
provider_reported_total: int | None = None,
pages_fetched: int | None = None,
pagination_exhausted: bool = False,
band_audit: Sequence[Mapping[str, Any]] | None = None,
discovery_audit: Mapping[str, Any] | None = None,
) -> tuple[
list[dict[str, Any]], list[dict[str, Any]], dict[str, Any], list[str], list[dict[str, Any]]
]:
if candidate_generation_mode not in CANDIDATE_GENERATION_MODES:
raise ScreenError(f"invalid candidate generation mode: {candidate_generation_mode}")
if not universe_rows:
raise ScreenError("listing universe is empty")
universe_decisions = [_listing_decision(row, config) for row in universe_rows]
merged_candidates, pool_not_in_universe = _merge_pool_rows(universe_rows, candidate_rows)
candidate_decisions = [
_candidate_decision(row, config, candidate_generation_mode, analysis_as_of)
for row in merged_candidates
]
selectable = [
row
for row in candidate_decisions
if row.get("selection_eligible") is True
and row.get("discovery_evaluable") is True
and row.get("deep_dive_priority_score") is not None
]
limit = int(config["max_deep_dive_candidates"])
selected_rows, selected_lane_counts = _select_multilane(selectable, limit=limit, config=config)
selected = {str(row["symbol"]) for row in selected_rows}
selected_lane_by_symbol = {
str(row["symbol"]): str(row.get("selection_lane")) for row in selected_rows
}
selected_set_payload = {
"analysis_as_of": analysis_as_of,
"max_deep_dive_candidates": limit,
"selected_symbols": sorted(selected),
}
selected_set_sha256 = hashlib.sha256(
json.dumps(selected_set_payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
# Every economically evaluable row receives a broad-screen disposition.
for row in candidate_decisions:
decision = _mapping(row.get("decision"))
status = _text(decision.get("status")) or "needs_enrichment"
if row.get("selection_eligible") is True:
decision["preselection_status"] = status
decision["status"] = "selected" if row["symbol"] in selected else "deferred_by_budget"
if row["symbol"] in selected:
lane = selected_lane_by_symbol.get(row["symbol"], _selection_lane(row))
decision["selection_lane"] = lane
decision["selection_reason"] = (
f"selected by deterministic multi-lane deep-dive plan: {lane}"
)
else:
decision["selection_lane"] = _selection_lane(row)
decision["selection_reason"] = (
"economically reviewable but outside the bounded multi-lane deep-dive budget"
)
row["decision"] = decision
universe_counts: dict[str, int] = {}
listing_complete = 0
in_scope_count = 0
for row in universe_decisions:
status = row["listing_decision"]["status"]
universe_counts[status] = universe_counts.get(status, 0) + 1
listing_complete += int(bool(row["listing_data_complete"]))
in_scope_count += int(status in {"in_scope", "liquidity_review"})
discovery_evaluable_count = sum(
int(row.get("discovery_evaluable") is True) for row in candidate_decisions
)
selection_eligible_count = sum(
int(row.get("selection_eligible") is True) for row in candidate_decisions
)
fundamental_complete_count = sum(
int(row.get("fundamental_complete") is True) for row in candidate_decisions
)
enrichment_attempted_count = sum(
int(row.get("enrichment_attempted") is True) for row in candidate_decisions
)
enrichment_resolved_count = sum(
int(row.get("enrichment_resolved") is True) for row in candidate_decisions
)
queue = _enrichment_queue(candidate_decisions)
unresolved_count = len(queue)
all_rows_resolved = unresolved_count == 0 and enrichment_resolved_count == len(
candidate_decisions
)
requested_min = float(
requested_min_market_cap
if requested_min_market_cap is not None
else DEFAULT_REQUESTED_MIN_MARKET_CAP
)
requested_max = float(
requested_max_market_cap
if requested_max_market_cap is not None
else DEFAULT_REQUESTED_MAX_MARKET_CAP
)
retrieval_covers_requested = (
retrieval_min_market_cap <= requested_min and retrieval_max_market_cap >= requested_max
)
normalized_bands = [dict(row) for row in (band_audit or []) if isinstance(row, Mapping)]
normalized_band_ranges: list[tuple[float, float]] = []
bands_well_formed = bool(normalized_bands)
disallowed_listing_filter_terms = (
"share volume",
"volume>",
"volume >=",
"avgvolume",
"average volume",
'"min_volume"',
'"minimum_volume"',
'"average_volume"',
'"avg_volume"',
)
def retrieval_filter_text(value: Any) -> str:
if isinstance(value, Mapping):
return json.dumps(dict(value), sort_keys=True, separators=(",", ":")).lower()
if isinstance(value, list):
return json.dumps(value, sort_keys=True, separators=(",", ":")).lower()
return (_text(value) or "").lower()
provider_listing_prefiltered = any(
any(
term in retrieval_filter_text(row.get("retrieval_filters"))
for term in disallowed_listing_filter_terms
)
for row in normalized_bands
)
for row in normalized_bands:
band_min = _number(row.get("min_market_cap"))
band_max = _number(row.get("max_market_cap"))
rows_fetched = _integer(row.get("rows_fetched"))
if (
band_min is None
or band_max is None
or band_min >= band_max
or rows_fetched is None
or rows_fetched < 0
or _bool(row.get("provider_exhausted")) is not True
):
bands_well_formed = False
continue
normalized_band_ranges.append((band_min, band_max))
normalized_band_ranges.sort()
bands_cover_requested = False
if bands_well_formed and normalized_band_ranges:
cursor = retrieval_min_market_cap
bands_cover_requested = True
for band_min, band_max in normalized_band_ranges:
if band_max < retrieval_min_market_cap or band_min > retrieval_max_market_cap:
continue
clipped_min = max(band_min, retrieval_min_market_cap)
clipped_max = min(band_max, retrieval_max_market_cap)
if clipped_min > cursor:
bands_cover_requested = False
break
cursor = max(cursor, clipped_max)
bands_cover_requested = bands_cover_requested and cursor >= retrieval_max_market_cap
bands_verified = bands_well_formed and bands_cover_requested
pagination_verified = bool(pagination_exhausted or bands_verified)
provider_total_consistent = (
provider_reported_total is None or len(universe_rows) >= provider_reported_total
)
enumeration_verified = pagination_verified and provider_total_consistent
scope_reduced = not retrieval_covers_requested
scope_reduction_disclosed = (not scope_reduced) or bool(_text(scope_reduction_reason))
executed_scope_complete = bool(retrieval_scope_explicit and enumeration_verified)
user_requested_scope_complete = bool(
retrieval_covers_requested and executed_scope_complete and not provider_listing_prefiltered
)
scope_reasons: list[str] = []
if not retrieval_scope_explicit:
scope_reasons.append("retrieval_scope_not_explicitly_recorded")
if scope_reduced:
scope_reasons.append("executed_scope_narrower_than_user_request")
if not scope_reduction_disclosed:
scope_reasons.append("scope_reduction_reason_required")
if not scope_override_authorized:
scope_reasons.append("reduced_scope_requires_explicit_user_authorization")
if not enumeration_verified:
scope_reasons.append("executed_scope_enumeration_not_verified")
if provider_listing_prefiltered:
scope_reasons.append("provider_listing_query_applied_share_volume_prefilter")
if provider_reported_total is not None and not provider_total_consistent:
scope_reasons.append("provider_reported_total_exceeds_rows_fetched")
listing_coverage_pct = listing_complete / len(universe_decisions) * 100.0
fundamental_pct = (
fundamental_complete_count / len(candidate_decisions) * 100.0
if candidate_decisions
else 0.0
)
discovery_pct = (
discovery_evaluable_count / len(candidate_decisions) * 100.0 if candidate_decisions else 0.0
)
resolution_pct = (
enrichment_resolved_count / len(candidate_decisions) * 100.0 if candidate_decisions else 0.0
)
in_scope_symbols = {
str(row.get("symbol") or "").upper()
for row in universe_decisions
if _text(_mapping(row.get("listing_decision")).get("status"))
in {"in_scope", "liquidity_review"}
}
candidate_symbols = {str(row.get("symbol") or "").upper() for row in candidate_decisions}
missing_in_scope_candidate_symbols = sorted(in_scope_symbols - candidate_symbols)
candidate_pool_covers_in_scope = not missing_in_scope_candidate_symbols
pool_scope_verified, conclusion_scope, normalized_discovery_audit, pool_scope_reasons = (
_validate_candidate_pool_scope(
candidate_generation_mode=candidate_generation_mode,
universe_rows=universe_rows,
candidate_rows=merged_candidates,
candidate_pool_covers_in_scope=candidate_pool_covers_in_scope,
discovery_audit=discovery_audit,
requested_min_market_cap=requested_min,
requested_max_market_cap=requested_max,
)
)
if scope_reduced and conclusion_scope == "full_listing_universe":
conclusion_scope = "executed_listing_universe"
if provider_listing_prefiltered and conclusion_scope in {
"full_listing_universe",
"stratified_discovery_pool",
}:
conclusion_scope = "provider_prefilter"
bounded_sampling_ready = bool(
conclusion_scope
in {
"stratified_discovery_pool",
"provider_prefilter",
"bounded_available_fundamentals",
"user_supplied",
}
and pool_scope_verified
and retrieval_covers_requested
and not scope_reduced
)
screening_scope_ready = bool(
user_requested_scope_complete
or bounded_sampling_ready
or (scope_override_authorized and executed_scope_complete)
)
# Full enumeration and bounded-pool readiness are separate concepts.
scope_complete = user_requested_scope_complete
candidate_pool_exhaustion_verified = bool(
candidate_pool_exhausted
and all_rows_resolved
and pool_scope_verified
and screening_scope_ready
)
enrichment_status = "complete" if candidate_pool_exhaustion_verified else "pending"
pool_complete = enrichment_status == "complete"
candidate_counts: dict[str, int] = {}
for row in candidate_decisions:
status = row["decision"]["status"]
candidate_counts[status] = candidate_counts.get(status, 0) + 1
marketwide_snapshot_exhausted = bool(
candidate_generation_mode == "sharded_snapshot" and pool_scope_verified
)
if selected and pool_complete:
candidate_pool_status = "sufficient"
elif selected:
candidate_pool_status = "sufficient_pending_enrichment"
elif pool_complete and (discovery_evaluable_count > 0 or marketwide_snapshot_exhausted):
candidate_pool_status = (
"no_qualifying_candidates"
if conclusion_scope == "full_listing_universe"
else "no_qualifying_candidates_in_bounded_pool"
)
else:
candidate_pool_status = "insufficient_data"
selection_outcome = {
"sufficient": "selected",
"sufficient_pending_enrichment": "selected_pending_enrichment",
"no_qualifying_candidates": "no_candidates",
"no_qualifying_candidates_in_bounded_pool": "no_candidates_in_bounded_pool",
"insufficient_data": "insufficient_data",
}[candidate_pool_status]
if not candidate_decisions and marketwide_snapshot_exhausted:
next_action = "publish_no_candidates"
elif not candidate_decisions:
next_action = "build_discovery_pool"
elif not pool_scope_verified:
next_action = "verify_candidate_pool_generation"
elif queue:
next_action = "enrich_queue"
elif not candidate_pool_exhausted:
next_action = "verify_candidate_pool_exhaustion"
elif selected:
next_action = "proceed_to_deep_dive"
else:
next_action = "publish_no_candidates"
audit = {
"audit_schema_version": AUDIT_SCHEMA_VERSION,
"contract_revision": CONTRACT_REVISION,
"runtime": runtime_metadata(),
"conclusion_scope": conclusion_scope,
"generated_at": datetime.now(timezone.utc).isoformat(),
"analysis_as_of": analysis_as_of,
"candidate_generation_mode": candidate_generation_mode,
"candidate_pool_status": candidate_pool_status,
"selection_outcome": selection_outcome,
"selected_symbols": sorted(selected),
"deep_dive_plan": {
"selected_symbols": sorted(selected),
"selected_count": len(selected),
"all_selected_must_be_resolved": True,
"budget_locked": True,
"budget_change_requires_rescreen": True,
"user_confirmation_required": False,
"user_continue_instruction_allowed": False,
"selected_set_is_committed": True,
"max_deep_dive_candidates": limit,
"selected_set_sha256": selected_set_sha256,
"commitment_payload": selected_set_payload,
"selection_method": "deterministic_multi_lane",
"lane_counts": selected_lane_counts,
"maximum_selected_per_sector": int(config.get("maximum_selected_per_sector", 2)),
},
"filters": dict(config),
"scope": {
"requested_min_market_cap": requested_min,
"requested_max_market_cap": requested_max,
"user_requested_scope": {
"min_market_cap": requested_min,
"max_market_cap": requested_max,
"source": "explicit_user_request" if scope_override_authorized else "skill_default",
},
"retrieval_min_market_cap": retrieval_min_market_cap,
"retrieval_max_market_cap": retrieval_max_market_cap,
"retrieval_scope_explicit": retrieval_scope_explicit,
"scope_override_authorized": scope_override_authorized,
"scope_reduced": scope_reduced,
"scope_reduction_mode": (
"user_authorized" if scope_override_authorized else "invalid_internal_narrowing"
)
if scope_reduced
else "none",
"scope_reduction_disclosed": scope_reduction_disclosed,
"scope_reduction_reason": scope_reduction_reason,
"user_scope_evidence": user_scope_evidence,
"user_requested_scope_complete": user_requested_scope_complete,
"executed_scope_complete": executed_scope_complete,
"bounded_sampling_ready": bounded_sampling_ready,
"screening_scope_ready": screening_scope_ready,
"executed_scope": {
"min_market_cap": retrieval_min_market_cap,
"max_market_cap": retrieval_max_market_cap,
},
"scope_complete": scope_complete,
"reasons": scope_reasons,
"enumeration": {
"verified": enumeration_verified,
"provider_reported_total": provider_reported_total,
"rows_fetched": len(universe_rows),
"pages_fetched": pages_fetched,
"pagination_exhausted": pagination_exhausted,
"band_audit": normalized_bands,
"bands_well_formed": bands_well_formed,
"bands_cover_executed_range": bands_cover_requested,
"bands_cover_requested_range": bands_cover_requested
if user_requested_scope_complete
else False,
"bands_verified": bands_verified,
"provider_listing_prefiltered": provider_listing_prefiltered,
"full_listing_enumeration_verified": enumeration_verified
and not provider_listing_prefiltered,
},
},
"universe": {
"row_count": len(universe_decisions),
"decision_counts": universe_counts,
"listing_data_complete_count": listing_complete,
"listing_data_complete_pct": round(listing_coverage_pct, 4),
"in_scope_count": in_scope_count,
"source_ids": list(universe_source_ids),
},
"candidate_pool": {
"row_count": len(candidate_decisions),
"decision_counts": candidate_counts,
"discovery_evaluable_count": discovery_evaluable_count,
"discovery_evaluable_pct": round(discovery_pct, 4),
"selection_eligible_count": selection_eligible_count,
"selected_count": len(selected),
"in_scope_covered_count": len(in_scope_symbols & candidate_symbols),
"in_scope_missing_count": len(missing_in_scope_candidate_symbols),
"in_scope_missing_symbols": missing_in_scope_candidate_symbols,
"coverage_complete": pool_scope_verified,
"coverage_scope": conclusion_scope,
"listing_coverage_complete": candidate_pool_covers_in_scope,
"generation_audit": normalized_discovery_audit,
"generation_review_reasons": pool_scope_reasons,
"fundamental_complete_count": fundamental_complete_count,
"fundamental_complete_pct": round(fundamental_pct, 4),
"source_ids": list(candidate_source_ids),
"symbols_not_in_universe": pool_not_in_universe,
},
"enrichment": {
"status": enrichment_status,
"next_action": next_action,
"discovery_pool_required": not bool(candidate_decisions),
"attempted_count": enrichment_attempted_count,
"resolved_count": enrichment_resolved_count,
"unresolved_count": unresolved_count,
"resolution_pct": round(resolution_pct, 4),
"all_rows_resolved": all_rows_resolved,
"maximum_attempts": int(config.get("maximum_enrichment_attempts", 60)),
"candidate_pool_exhaustion_declared": candidate_pool_exhausted,
"candidate_pool_exhausted": candidate_pool_exhaustion_verified,
"candidate_pool_covers_in_scope": candidate_pool_covers_in_scope,
"candidate_pool_scope_verified": pool_scope_verified,
"conclusion_scope": conclusion_scope,
"queue_count": len(queue),
"queue_symbols": [str(row["symbol"]) for row in queue],
},
"coverage_interpretation": {
"listing_universe_audited": True,
"universe_enumeration_verified": enumeration_verified,
"full_listing_enumeration_verified": enumeration_verified
and not provider_listing_prefiltered,
"provider_listing_prefiltered": provider_listing_prefiltered,
"full_universe_fundamentals_claimed": candidate_generation_mode
== "full_universe_fundamentals",
"candidate_pool_fundamentals_only": candidate_generation_mode
!= "full_universe_fundamentals",
"full_market_fundamental_coverage_is_not_a_completion_gate": True,
"no_candidates_requires_zero_unresolved_rows_and_exhausted_pool": True,
"bounded_pool_final_ranking_allowed_when_generation_is_audited": True,
"market_wide_no_candidates_requires_full_listing_universe_coverage": True,
},
"row_count": len(candidate_decisions),
"decision_counts": candidate_counts,
"source_ids": list(dict.fromkeys(list(universe_source_ids) + list(candidate_source_ids))),
}
return universe_decisions, candidate_decisions, audit, sorted(selected), queue
def run(
rows: Sequence[Mapping[str, Any]],
config: Mapping[str, Any],
*,
analysis_as_of: str,
source_ids: Sequence[str],
universe_query_min_cap: float | None = None,
universe_query_max_cap: float | None = None,
) -> tuple[list[dict[str, Any]], dict[str, Any], list[str], list[dict[str, Any]]]:
"""Backward-compatible in-memory full-fundamentals screen used by tests."""
query_min = float(
config["min_market_cap"] if universe_query_min_cap is None else universe_query_min_cap
)
query_max = float(
config["max_market_cap"] if universe_query_max_cap is None else universe_query_max_cap
)
_, decisions, audit, selected, queue = run_layered(
rows,
rows,
config,
analysis_as_of=analysis_as_of,
universe_source_ids=source_ids,
candidate_source_ids=source_ids,
candidate_generation_mode="full_universe_fundamentals",
retrieval_min_market_cap=query_min,
retrieval_max_market_cap=query_max,
retrieval_scope_explicit=True,
candidate_pool_exhausted=True,
provider_reported_total=len(rows),
pages_fetched=1,
pagination_exhausted=True,
)
return decisions, audit, selected, queue
def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Audit the full listing universe and screen a bounded economic candidate pool."
)
parser.add_argument("--input", type=Path, required=True, help="Full listing universe.")
parser.add_argument(
"--candidate-pool",
type=Path,
help="Enriched candidate pool; otherwise available rows are used.",
)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--analysis-as-of", required=True)
parser.add_argument(
"--source-id",
action="append",
required=True,
help="Universe/listing source ID; repeatable.",
)
parser.add_argument(
"--candidate-source-id", action="append", help="Candidate/estimate source ID; repeatable."
)
parser.add_argument("--candidate-generation-mode", choices=sorted(CANDIDATE_GENERATION_MODES))
parser.add_argument("--retrieval-min-market-cap", type=float)
parser.add_argument("--retrieval-max-market-cap", type=float)
parser.add_argument("--user-requested-min-market-cap", type=float)
parser.add_argument("--user-requested-max-market-cap", type=float)
parser.add_argument("--user-scope-override-authorized", action="store_true")
parser.add_argument("--user-scope-evidence")
parser.add_argument("--scope-reduction-reason")
parser.add_argument(
"--allow-reduced-scope",
action="store_true",
help="Deprecated alias; never implies user authorization",
)
parser.add_argument("--candidate-pool-exhausted", action="store_true")
parser.add_argument("--provider-reported-total", type=int)
parser.add_argument("--pages-fetched", type=int)
parser.add_argument("--pagination-exhausted", action="store_true")
parser.add_argument(
"--band-audit",
type=Path,
help="JSON array of retrieval bands with provider_exhausted flags",
)
parser.add_argument(
"--discovery-audit",
type=Path,
help="Generation audit for a bounded discovery/prefilter pool",
)
parser.add_argument("--config", type=Path)
parser.add_argument("--max-deep-dives", type=int)
return parser.parse_args(argv)
def main(argv: Sequence[str] | None = None) -> int:
raw_argv = list(argv) if argv is not None else sys.argv[1:]
if "--version" in raw_argv:
print(json.dumps(runtime_metadata(), sort_keys=True))
return 0
args = parse_args(raw_argv)
try:
datetime.fromisoformat(args.analysis_as_of.replace("Z", "+00:00"))
universe_rows = _load_rows(args.input)
config = dict(DEFAULTS)
if args.config:
extra = json.loads(args.config.read_text(encoding="utf-8"))
if not isinstance(extra, Mapping):
raise ScreenError("config must be an object")
config.update(extra)
if args.max_deep_dives is not None:
if args.max_deep_dives <= 0:
raise ScreenError("max deep dives must be positive")
config["max_deep_dive_candidates"] = args.max_deep_dives
if args.candidate_pool:
candidate_rows = _load_rows(args.candidate_pool)
mode = args.candidate_generation_mode or "provider_prefilter"
else:
candidate_rows = [
row
for row in universe_rows
if any(
_metric(row, key) is not None
for key in (
"forward_pe",
"fy1_pe",
"ntm_pe",
"eps_growth_pct",
"forward_eps_growth_pct",
)
)
]
mode = args.candidate_generation_mode or (
"full_universe_fundamentals"
if len(candidate_rows) == len(universe_rows)
else "available_fundamentals"
)
candidate_source_ids = args.candidate_source_id or args.source_id
explicit_scope = (
args.retrieval_min_market_cap is not None and args.retrieval_max_market_cap is not None
)
# Config files may tune the executed retrieval/screening range, but
# cannot rewrite the user's default request. Only explicit authorized
# CLI arguments may change the request scope.
requested_min = float(DEFAULT_REQUESTED_MIN_MARKET_CAP)
requested_max = float(DEFAULT_REQUESTED_MAX_MARKET_CAP)
if (
args.user_requested_min_market_cap is not None
or args.user_requested_max_market_cap is not None
):
if not args.user_scope_override_authorized or not args.user_scope_evidence:
raise ScreenError(
"changing the user-requested scope requires --user-scope-override-authorized and --user-scope-evidence"
)
requested_min = float(
args.user_requested_min_market_cap
if args.user_requested_min_market_cap is not None
else requested_min
)
requested_max = float(
args.user_requested_max_market_cap
if args.user_requested_max_market_cap is not None
else requested_max
)
if requested_min >= requested_max:
raise ScreenError("user-requested market-cap bounds are invalid")
retrieval_min = (
args.retrieval_min_market_cap
if args.retrieval_min_market_cap is not None
else float(config["min_market_cap"])
)
retrieval_max = (
args.retrieval_max_market_cap
if args.retrieval_max_market_cap is not None
else float(config["max_market_cap"])
)
discovery_audit: dict[str, Any] = {}
if args.discovery_audit:
discovery_value = json.loads(args.discovery_audit.read_text(encoding="utf-8"))
if not isinstance(discovery_value, Mapping):
raise ScreenError("discovery audit must be a JSON object")
discovery_audit = dict(discovery_value)
band_audit_rows: list[dict[str, Any]] = []
if args.band_audit:
band_value = json.loads(args.band_audit.read_text(encoding="utf-8"))
if not isinstance(band_value, list) or not all(
isinstance(row, Mapping) for row in band_value
):
raise ScreenError("band audit must be a JSON array of objects")
band_audit_rows = [dict(row) for row in band_value]
universe_decisions, candidate_decisions, audit, selected, queue = run_layered(
universe_rows,
candidate_rows,
config,
analysis_as_of=args.analysis_as_of,
universe_source_ids=args.source_id,
candidate_source_ids=candidate_source_ids,
candidate_generation_mode=mode,
retrieval_min_market_cap=retrieval_min,
retrieval_max_market_cap=retrieval_max,
requested_min_market_cap=requested_min,
requested_max_market_cap=requested_max,
retrieval_scope_explicit=explicit_scope,
scope_override_authorized=args.user_scope_override_authorized,
scope_reduction_reason=args.scope_reduction_reason,
user_scope_evidence=args.user_scope_evidence,
candidate_pool_exhausted=args.candidate_pool_exhausted,
provider_reported_total=args.provider_reported_total,
pages_fetched=args.pages_fetched,
pagination_exhausted=args.pagination_exhausted,
band_audit=band_audit_rows,
discovery_audit=discovery_audit,
)
args.output_dir.mkdir(parents=True, exist_ok=True)
universe_artifact = args.output_dir / "universe-audit-results.jsonl"
candidate_artifact = args.output_dir / "broad-screen-results.jsonl"
enrichment_artifact = args.output_dir / "enrichment-queue.json"
universe_artifact.write_text(
"".join(_canonical_line(row) for row in universe_decisions), encoding="utf-8"
)
candidate_artifact.write_text(
"".join(_canonical_line(row) for row in candidate_decisions), encoding="utf-8"
)
enrichment_artifact.write_text(
json.dumps(queue, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
audit["universe"].update(
{
"artifact_path": universe_artifact.name,
"artifact_sha256": hashlib.sha256(universe_artifact.read_bytes()).hexdigest(),
}
)
audit["candidate_pool"].update(
{
"artifact_path": candidate_artifact.name,
"artifact_sha256": hashlib.sha256(candidate_artifact.read_bytes()).hexdigest(),
}
)
audit["enrichment"].update(
{
"artifact_path": enrichment_artifact.name,
"artifact_sha256": hashlib.sha256(enrichment_artifact.read_bytes()).hexdigest(),
}
)
audit_path = args.output_dir / "broad-screen-audit.json"
audit_path.write_text(
json.dumps(audit, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
shortlist_path = args.output_dir / "broad-screen-shortlist.json"
shortlist_path.write_text(
json.dumps(
{
"runtime": runtime_metadata(),
"candidate_generation_mode": mode,
"candidate_pool_status": audit["candidate_pool_status"],
"selection_outcome": audit["selection_outcome"],
"selected_symbols": selected,
"deep_dive_plan": audit.get("deep_dive_plan", {}),
},
ensure_ascii=False,
indent=2,
)
+ "\n",
encoding="utf-8",
)
except (OSError, ValueError, json.JSONDecodeError, ScreenError) as exc:
print(f"ERROR: {exc}", file=sys.stderr)
return 1
for path in (
universe_artifact,
candidate_artifact,
enrichment_artifact,
audit_path,
shortlist_path,
):
print(f"wrote: {path}")
incomplete_reasons: list[str] = []
scope_section = _mapping(audit.get("scope"))
if scope_section.get("screening_scope_ready") is not True:
incomplete_reasons.append(
"requested scope is neither fully enumerated nor covered by an audited full-range stratified pool"
)
if audit.get("candidate_pool_status") in {"insufficient_data", "sufficient_pending_enrichment"}:
incomplete_reasons.append("candidate pool needs enrichment or exhaustion verification")
if incomplete_reasons:
print(
"INCOMPLETE: "
+ "; ".join(incomplete_reasons)
+ "; build/enrich the bounded discovery pool and rerun with explicit retrieval bounds",
file=sys.stderr,
)
return 2
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/screening_semantics.py
#!/usr/bin/env python3
"""Shared fail-closed semantics for broad-screen liquidity and estimates.
The discovery layer must not confuse one-session share volume with average daily
*dollar* volume. Likewise, a distant outer-year estimate must never be labelled
as the current NTM/FY1 metric merely because it makes a P/E look low. All
helpers in this module return normalized evidence plus explicit reason codes;
callers may explain those reasons, but may not override them.
"""
from __future__ import annotations
import math
from collections.abc import Mapping
from datetime import datetime, timezone
from typing import Any
VALID_ADDV_METHODS = {
"provider_average_dollar_volume",
"price_x_provider_average_volume",
"price_x_20d_average_volume",
"price_x_30d_average_volume",
"price_x_60d_average_volume",
"price_x_90d_average_volume",
}
INVALID_SINGLE_SESSION_METHODS = {
"price_x_single_session_volume",
"single_session_dollar_volume",
"single_session_volume",
"price_x_volume",
}
VALID_CURRENT_FORWARD_PERIODS = {"NTM", "FY1"}
VALID_FORWARD_ORIGINS = {
"computed_from_price_and_fy1_eps",
"computed_from_price_and_ntm_eps",
"provider_fy1_pe_with_eps_reconciliation",
"provider_ntm_pe_with_eps_reconciliation",
}
def _number(value: Any) -> float | None:
if isinstance(value, bool):
return None
if isinstance(value, (int, float)):
result = float(value)
return result if math.isfinite(result) else None
if isinstance(value, str):
try:
result = float(value.replace(",", "").strip())
except ValueError:
return None
return result if math.isfinite(result) else None
return None
def _integer(value: Any) -> int | None:
number = _number(value)
return int(number) if number is not None and number.is_integer() else None
def _text(value: Any) -> str | None:
if isinstance(value, str):
value = value.strip()
return value or None
return None
def _bool(value: Any) -> bool | None:
if isinstance(value, bool):
return value
if isinstance(value, str):
lowered = value.strip().lower()
if lowered in {"true", "1", "yes", "y"}:
return True
if lowered in {"false", "0", "no", "n"}:
return False
return None
def _list_of_strings(value: Any) -> list[str]:
if not isinstance(value, list):
return []
return [item.strip() for item in value if isinstance(item, str) and item.strip()]
def _parse_datetime(value: Any) -> datetime | None:
text = _text(value)
if not text:
return None
try:
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
except ValueError:
return None
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed
def normalize_liquidity(
row: Mapping[str, Any],
*,
minimum_period_days: int = 20,
) -> dict[str, Any]:
"""Return auditable average-daily-dollar-volume evidence.
A raw ``volume`` value is deliberately ignored. To be valid, liquidity
must carry an allowed averaging method, a window of at least
``minimum_period_days``, and at least one source ID. Provider camelCase
aliases are accepted only for the *average* volume field.
"""
price = _number(row.get("price"))
if price is None:
price = _number(row.get("last"))
explicit_addv = _number(row.get("average_daily_dollar_volume"))
if explicit_addv is None:
explicit_addv = _number(row.get("avg_dollar_volume"))
average_volume = _number(row.get("average_volume"))
if average_volume is None:
average_volume = _number(row.get("avg_volume"))
if average_volume is None:
average_volume = _number(row.get("avgVolume"))
if average_volume is None:
average_volume = _number(row.get("averageVolume"))
method = (
_text(row.get("average_daily_dollar_volume_method"))
or _text(row.get("liquidity_method"))
or ""
).lower()
period_days = (
_integer(row.get("average_volume_period_days"))
or _integer(row.get("liquidity_window_days"))
or _integer(row.get("average_daily_dollar_volume_period_days"))
)
source_ids = _list_of_strings(row.get("liquidity_source_ids"))
if not source_ids:
source_ids = _list_of_strings(row.get("average_daily_dollar_volume_source_ids"))
reasons: list[str] = []
value: float | None = None
if method in INVALID_SINGLE_SESSION_METHODS:
reasons.append("single_session_volume_not_valid_for_addv")
elif method not in VALID_ADDV_METHODS:
reasons.append("average_liquidity_method_missing_or_invalid")
elif period_days is None or period_days < minimum_period_days:
reasons.append("average_liquidity_window_below_minimum")
elif method == "provider_average_dollar_volume":
if explicit_addv is None or explicit_addv <= 0:
reasons.append("average_daily_dollar_volume_unavailable_or_non_positive")
else:
value = explicit_addv
else:
if price is None or price <= 0 or average_volume is None or average_volume <= 0:
reasons.append("price_or_average_volume_unavailable_or_non_positive")
else:
value = price * average_volume
if explicit_addv is not None:
tolerance = max(abs(value), 1.0) * 0.05
if abs(explicit_addv - value) > tolerance:
reasons.append("average_daily_dollar_volume_reconciliation_failed")
if not source_ids:
reasons.append("liquidity_source_ids_required")
valid = value is not None and value > 0 and not reasons
return {
"value": value if valid else None,
"valid_for_screen": valid,
"method": method or None,
"period_days": period_days,
"source_ids": source_ids,
"reasons": sorted(set(reasons)),
"price": price,
"average_volume": average_volume,
"raw_single_session_volume_present": _number(row.get("volume")) is not None,
}
def normalize_forward_valuation(
row: Mapping[str, Any],
*,
price: float | None,
analysis_as_of: str | None,
max_age_days: int = 45,
reconciliation_tolerance_pct: float = 5.0,
maximum_dispersion_pct: float = 100.0,
maximum_fy1_horizon_days: int = 430,
) -> dict[str, Any]:
"""Validate that the current multiple is genuinely NTM or FY1.
``forward_pe`` alone is never sufficient. The row must identify the
current period (NTM/FY1), its period end, the estimate date, source IDs, a
positive matching EPS, and the origin of the multiple. Distant FY2/FY3
estimates, pre-operating companies, zero-crossing ranges, and discontinuous
annual series fail closed.
"""
reasons: list[str] = []
normalization_status = (_text(row.get("estimate_normalization_status")) or "").lower()
normalization_reasons = _list_of_strings(row.get("estimate_normalization_reasons"))
if normalization_status and normalization_status != "valid":
reasons.append("estimate_normalization_not_valid")
reasons.extend(normalization_reasons)
elif normalization_status == "valid" and normalization_reasons:
reasons.append("estimate_normalization_status_inconsistent")
reasons.extend(normalization_reasons)
if _number(row.get("ntm_pe")) is not None:
pe = _number(row.get("ntm_pe"))
period = "NTM"
eps = _number(row.get("ntm_eps")) or _number(row.get("forward_eps"))
inferred_origin = "provider_ntm_pe_with_eps_reconciliation"
elif _number(row.get("fy1_pe")) is not None:
pe = _number(row.get("fy1_pe"))
period = "FY1"
eps = _number(row.get("fy1_eps")) or _number(row.get("forward_eps"))
inferred_origin = "provider_fy1_pe_with_eps_reconciliation"
else:
pe = _number(row.get("forward_pe"))
period = (
_text(row.get("forward_pe_period")) or _text(row.get("forward_metric_period")) or ""
).upper() or None
eps = _number(row.get("forward_eps"))
if period == "NTM" and eps is None:
eps = _number(row.get("ntm_eps"))
if period == "FY1" and eps is None:
eps = _number(row.get("fy1_eps"))
inferred_origin = None
origin = (
_text(row.get("forward_metric_origin"))
or _text(row.get("forward_pe_origin"))
or inferred_origin
)
fiscal_year = _text(row.get("forward_fiscal_year")) or _text(row.get("fy1_fiscal_year"))
period_end_text = (
_text(row.get("forward_period_end"))
or _text(row.get("forward_metric_period_end"))
or _text(row.get("fy1_period_end"))
)
period_end = _parse_datetime(period_end_text)
estimate_as_of_text = _text(row.get("forward_estimate_as_of")) or _text(
row.get("estimate_as_of")
)
estimate_as_of = _parse_datetime(estimate_as_of_text)
analysis_dt = _parse_datetime(analysis_as_of)
source_ids = _list_of_strings(row.get("forward_estimate_source_ids"))
if not source_ids:
source_ids = _list_of_strings(row.get("forward_metric_source_ids"))
if not source_ids:
source_ids = _list_of_strings(row.get("enrichment_source_ids"))
low = _number(row.get("forward_eps_low"))
high = _number(row.get("forward_eps_high"))
dispersion_pct: float | None = None
horizon_days: float | None = None
if pe is None or pe <= 0:
reasons.append("forward_pe_unavailable_or_non_positive")
if period not in VALID_CURRENT_FORWARD_PERIODS:
reasons.append("forward_pe_period_must_be_ntm_or_fy1")
if eps is None or eps <= 0:
reasons.append("current_forward_eps_unavailable_or_non_positive")
if origin not in VALID_FORWARD_ORIGINS:
reasons.append("forward_metric_origin_missing_or_invalid")
if period == "FY1" and not fiscal_year:
reasons.append("fy1_fiscal_year_required")
if period_end is None:
reasons.append("current_forward_period_end_required")
elif analysis_dt is not None:
horizon_days = (period_end - analysis_dt).total_seconds() / 86400.0
if horizon_days <= 0:
reasons.append("current_forward_period_must_end_after_analysis")
elif horizon_days > maximum_fy1_horizon_days:
reasons.append("current_forward_period_is_not_fy1_or_ntm")
operating_stage = (_text(row.get("operating_stage")) or "").lower().replace(" ", "_")
if _bool(row.get("is_pre_operating")) is True or operating_stage in {
"pre_operating",
"pre-revenue",
"pre_revenue",
"development_stage",
}:
reasons.append("pre_operating_company_not_pe_evaluable")
if row.get("estimate_series_contiguous") is False:
reasons.append("annual_estimate_series_not_contiguous")
if estimate_as_of is None:
reasons.append("forward_estimate_as_of_required")
elif analysis_dt is not None:
age_days = (analysis_dt - estimate_as_of).total_seconds() / 86400.0
if age_days < -1e-6:
reasons.append("forward_estimate_after_analysis_as_of")
elif age_days > max_age_days:
reasons.append("forward_estimate_stale")
if not source_ids:
reasons.append("forward_estimate_source_ids_required")
implied: float | None = None
if (
price is not None
and price > 0
and pe is not None
and pe > 0
and eps is not None
and eps > 0
):
implied = price / eps
diff_pct = abs(implied - pe) / max(abs(pe), 1e-9) * 100.0
if diff_pct > reconciliation_tolerance_pct:
reasons.append("forward_pe_does_not_reconcile_to_price_and_eps")
else:
reasons.append("price_eps_multiple_reconciliation_unavailable")
if low is not None and high is not None:
if low > high:
reasons.append("forward_eps_range_invalid")
elif low <= 0 <= high:
reasons.append("forward_eps_range_crosses_zero")
elif eps is not None and eps != 0:
dispersion_pct = abs(high - low) / abs(eps) * 100.0
if dispersion_pct > maximum_dispersion_pct:
reasons.append("forward_eps_dispersion_excessive")
valid = not reasons
analyst_count = (
_integer(row.get("forward_eps_analyst_count"))
or _integer(row.get("fy1_analyst_count"))
or _integer(row.get("analyst_count"))
)
return {
"valid": valid,
"forward_pe": pe if valid else None,
"forward_eps": eps if valid else None,
"period": period,
"fiscal_year": fiscal_year,
"period_end": period_end_text,
"horizon_days": horizon_days,
"estimate_as_of": estimate_as_of_text,
"source_ids": source_ids,
"origin": origin,
"analyst_count": analyst_count,
"eps_low": low,
"eps_high": high,
"dispersion_pct": dispersion_pct,
"implied_forward_pe": implied,
"reasons": sorted(set(reasons)),
}
scripts/skill_version.py
#!/usr/bin/env python3
"""Single source of truth for the installed skill/runtime contract."""
from __future__ import annotations
SKILL_NAME = "us-undervalued-growth-screener"
SKILL_VERSION = "3.6.1"
SCHEMA_VERSION = 3
CONTRACT_REVISION = "3.5"
# Change this whenever contract semantics change. It intentionally differs from
# the human version so stale/cached runtimes are detected in serialized audits.
RUNTIME_FINGERPRINT = "ug-v3.6.1-claude-code-direct-fmp-20260830"
def runtime_metadata() -> dict[str, object]:
return {
"skill_name": SKILL_NAME,
"skill_version": SKILL_VERSION,
"schema_version": SCHEMA_VERSION,
"contract_revision": CONTRACT_REVISION,
"runtime_fingerprint": RUNTIME_FINGERPRINT,
}
scripts/tests/test_artifact_paths.py
"""Discovery artifacts must resolve from one run root after set-screening-audit."""
from __future__ import annotations
import hashlib
import json
import sys
import tempfile
import unittest
from pathlib import Path
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
if str(SCRIPTS_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPTS_DIR))
import manage_run_state as RUN_STATE # noqa: E402
import prepublish_audit as PREPUBLISH # noqa: E402
ASSETS_DIR = SCRIPTS_DIR.parent / "assets"
class ArtifactPathTests(unittest.TestCase):
def _init_and_attach(self, run_dir: Path) -> dict:
payload = json.loads((ASSETS_DIR / "candidate-input.example.json").read_text())
self.assertEqual(
RUN_STATE.main(
[
"init",
"--run-dir",
str(run_dir),
"--analysis-as-of",
payload["analysis_as_of"],
"--price-as-of",
payload["price_basis"]["as_of"],
"--session",
"regular_close",
"--price-source-id",
"quote-20260821",
"--config",
str(ASSETS_DIR / "screening-config.example.json"),
"--market-context",
str(ASSETS_DIR / "market-context.example.json"),
"--global-sources",
str(ASSETS_DIR / "global-sources.example.json"),
]
),
0,
)
self.assertEqual(
RUN_STATE.main(
[
"set-screening-audit",
"--run-dir",
str(run_dir),
"--audit",
str(ASSETS_DIR / "broad-screen-audit.example.json"),
"--universe-artifact",
str(ASSETS_DIR / "universe-audit-results.example.jsonl"),
"--candidate-artifact",
str(ASSETS_DIR / "broad-screen-results.example.jsonl"),
]
),
0,
)
return json.loads((run_dir / "audit" / "broad-screen-audit.json").read_text())
def test_enrichment_and_generation_artifacts_are_copied_under_audit(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
run_dir = Path(temp_dir) / "run"
audit = self._init_and_attach(run_dir)
enrichment = audit["enrichment"]
generation = audit["candidate_pool"].get("generation_audit") or {}
sections = [audit["universe"], audit["candidate_pool"], enrichment]
if generation.get("artifact_path"):
sections.append(generation)
for section in sections:
path = section["artifact_path"]
self.assertTrue(path.startswith("audit/"), path)
target = run_dir / path
self.assertTrue(target.is_file(), path)
self.assertEqual(
hashlib.sha256(target.read_bytes()).hexdigest(), section["artifact_sha256"]
)
def test_prepublish_resolves_every_artifact_from_the_run_root(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
run_dir = Path(temp_dir) / "run"
audit = self._init_and_attach(run_dir)
errors: list[str] = []
PREPUBLISH._verify_artifact(audit["universe"], run_dir, "universe", errors)
PREPUBLISH._verify_artifact(audit["candidate_pool"], run_dir, "candidate_pool", errors)
PREPUBLISH._verify_artifact(audit["enrichment"], run_dir, "enrichment", errors)
generation = audit["candidate_pool"].get("generation_audit") or {}
if generation.get("artifact_path"):
PREPUBLISH._verify_artifact(generation, run_dir, "generation", errors)
self.assertEqual(errors, [])
def test_prepublish_accepts_audit_relative_bare_names(self) -> None:
# A discovery audit that was never passed through set-screening-audit
# still references bare file names relative to audit/.
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
(root / "audit").mkdir()
body = b'{"a": 1}\n'
(root / "audit" / "enrichment-queue.json").write_bytes(body)
section = {
"artifact_path": "enrichment-queue.json",
"artifact_sha256": hashlib.sha256(body).hexdigest(),
}
errors: list[str] = []
PREPUBLISH._verify_artifact(section, root, "enrichment", errors)
self.assertEqual(errors, [])
missing: list[str] = []
PREPUBLISH._verify_artifact(
{"artifact_path": "nope.json", "artifact_sha256": "0" * 64}, root, "x", missing
)
self.assertEqual(missing, ["x artifact is missing: nope.json"])
if __name__ == "__main__":
unittest.main()
scripts/tests/test_budget_diagnostics.py
"""Budget refusal must not invent provider spend or destroy selection history."""
import copy
import sys
from datetime import datetime, timezone
from pathlib import Path
from unittest import mock
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import run_pipeline as PIPELINE
@pytest.mark.parametrize("operation", ["get_key_metrics_ttm", "get_income_statement"])
@pytest.mark.parametrize(
"consumed,refused", [(0, True), (1, True), (0, False), (1, False), (2, False)]
)
def test_probe_counts_provider_budget_units(operation, consumed, refused):
class Client:
api_calls_made = 7 # Prior stages must not leak into this stage's audit.
def call(self, name):
self.api_calls_made += consumed if name == operation else 1
if name == operation and refused:
raise PIPELINE.ApiCallBudgetExceeded("test budget stop")
return []
def get_key_metrics_ttm(self, symbol):
return self.call("get_key_metrics_ttm")
def get_income_statement(self, symbol, **kwargs):
return self.call("get_income_statement")
client = Client()
_, audit = PIPELINE.apply_quality_probe(
client,
[{"symbol": "AAA"}],
target_symbols=["AAA"],
source_id="test",
analysis_as_of=datetime(2026, 9, 19, tzinfo=timezone.utc),
return_partial_on_budget=True,
)
income_reached = operation == "get_income_statement" or not refused
assert audit["actual_eps_calls"] == (
consumed if operation == "get_income_statement" else int(income_reached)
)
assert audit["calls_used"] == client.api_calls_made - 7
assert audit["budget_exhausted"] is refused
assert audit["attempted_symbols"] == ["AAA"]
def test_real_client_refuses_first_income_call_without_counting_it(tmp_path):
client = PIPELINE.FMPClient(
api_key="test", # pragma: allowlist secret - mocked transport only
max_api_calls=1,
cache_path=tmp_path / "cache.sqlite",
rate_limit_delay=0,
)
response = mock.Mock(status_code=200)
response.json.return_value = []
try:
with mock.patch.object(client.session, "get", return_value=response) as get:
_, audit = PIPELINE.apply_quality_probe(
client,
[{"symbol": "AAA"}],
target_symbols=["AAA"],
source_id="test",
analysis_as_of=datetime(2026, 9, 19, tzinfo=timezone.utc),
return_partial_on_budget=True,
)
assert get.call_count == 1
assert audit["actual_eps_calls"] == 0
assert audit["calls_used"] == 1
assert audit["interrupted_at"]["operation"] == "get_income_statement"
finally:
client.close()
@pytest.mark.parametrize("status", ["selected", "rejected", "deferred_by_budget"])
def test_invalidation_preserves_original_audit_without_selection_authority(status):
rows = [
{
"symbol": "AAA",
"selection_eligible": status == "selected",
"selection_lane": "growth",
"decision": {
"status": status,
"selection_eligible": status == "selected",
"preselection_status": "eligible",
"selection_lane": "growth",
"selection_reason": {"reason": ["original"]},
},
}
]
before = copy.deepcopy(rows)
result = PIPELINE._invalidate_candidate_decisions(rows)
assert rows == before
assert result[0]["prior_decision"] == before[0]["decision"]
assert result[0]["prior_selection"] == {
"selection_eligible": status == "selected",
"selection_lane": "growth",
}
assert result[0]["decision"] == {"status": "deferred_by_budget", "selection_eligible": False}
assert result[0]["selection_eligible"] is False
assert "selection_lane" not in result[0]
assert PIPELINE._invalidate_candidate_decisions(result) == result
rows[0]["decision"]["selection_reason"]["reason"].append("changed")
assert result[0]["prior_decision"] == before[0]["decision"]
@pytest.mark.parametrize("refused", [False, True])
def test_client_without_counter_uses_completed_logical_calls(refused):
class Client:
def get_key_metrics_ttm(self, symbol):
return []
def get_income_statement(self, symbol, **kwargs):
if refused:
raise PIPELINE.ApiCallBudgetExceeded("test budget stop")
return []
_, audit = PIPELINE.apply_quality_probe(
Client(),
[{"symbol": "AAA"}],
target_symbols=["AAA"],
source_id="test",
analysis_as_of=datetime(2026, 9, 19, tzinfo=timezone.utc),
return_partial_on_budget=True,
)
assert audit["actual_eps_calls"] == int(not refused)
assert audit["calls_used"] == 1 + int(not refused)
scripts/tests/test_budget_transport_attempts.py
"""Timeouts must consume dispatch budget like any other transport attempt.
Issue #607: the global counter used to advance only after session.get
returned a response, so a ReadTimeout cost zero budget and stable->v3
fallback plus later symbols could transport far beyond max_api_calls.
"""
import sys
from pathlib import Path
from unittest import mock
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import fmp_client as CLIENT
def _client(tmp_path, *, max_api_calls=1, rate_limit_delay=0):
return CLIENT.FMPClient(
api_key="test", # pragma: allowlist secret - mocked transport only
max_api_calls=max_api_calls,
cache_path=tmp_path / "cache.sqlite",
rate_limit_delay=rate_limit_delay,
)
def _timeout_client(tmp_path, **kwargs):
client = _client(tmp_path, **kwargs)
transport = mock.Mock(side_effect=CLIENT.requests.ReadTimeout("fictional timeout"))
return client, transport
def test_timeout_consumes_exact_budget(tmp_path):
"""AC-1: window budget of one allows exactly one dispatch attempt.
The stable attempt is timed out and therefore consumed; the stable->v3
fallback then meets the exhausted-budget guard and raises instead of
transporting (issue repro: pre-fix `6 0 1`, post-fix `1 1 0`).
"""
client, transport = _timeout_client(tmp_path)
with mock.patch.object(client.session, "get", transport):
with pytest.raises(CLIENT.ApiCallBudgetExceeded):
client.get_profile("AAA")
assert transport.call_count == 1
assert client.api_calls_made == 1
assert client.remaining_calls == 0
def test_next_symbol_cannot_bypass_cap(tmp_path):
"""AC-3: a second symbol cannot dispatch after the budget is exhausted."""
client, transport = _timeout_client(tmp_path)
with mock.patch.object(client.session, "get", transport):
with pytest.raises(CLIENT.ApiCallBudgetExceeded):
client.get_profile("AAA")
with pytest.raises(CLIENT.ApiCallBudgetExceeded):
client.get_profile("BBB")
assert transport.call_count == 1
assert client.api_calls_made == 1
def test_fallback_within_remaining_budget_returns_none(tmp_path):
"""AC-2a: two timed-out attempts exhaust a budget of two; no raise yet.
get_profile returns None (stable and v3 both dispatched and consumed);
a subsequent symbol with a zeroed remaining budget then raises.
"""
client, transport = _timeout_client(tmp_path, max_api_calls=2)
with mock.patch.object(client.session, "get", transport):
profile = client.get_profile("AAA")
assert profile is None
assert transport.call_count == 2
assert client.api_calls_made == 2
with pytest.raises(CLIENT.ApiCallBudgetExceeded):
client.get_profile("BBB")
assert transport.call_count == 2
def test_pretripped_stable_refusal_consumes_zero(tmp_path):
"""AC-4 refusal leg + AC-2b: breaker-tripped stable URL refuses; only the
v3 URL dispatches a counted HTTP-500 attempt (counted failures keep the
accounting identical before and after #607).
"""
client = _client(tmp_path, max_api_calls=2)
client._disabled_endpoints.add(f"{client.STABLE_URL}/profile")
transport = mock.Mock(return_value=mock.Mock(status_code=500, json=lambda: []))
with mock.patch.object(client.session, "get", transport):
profile = client.get_profile("AAA")
assert profile is None
assert transport.call_count == 1
assert client.api_calls_made == 1
assert remaining(client) == 1
def test_cache_hit_consumes_zero(tmp_path):
"""AC-4 cache leg: a cached payload is returned without dispatching."""
client = _client(tmp_path, max_api_calls=1)
key = client.cache.make_key(
f"{client.STABLE_URL}/profile", {"symbol": "AAA", "apikey": client.api_key}
)
client.cache.put(key, [{"symbol": "AAA", "price": 12.0}])
transport = mock.Mock(side_effect=AssertionError("transport must not be touched"))
with mock.patch.object(client.session, "get", transport):
profile = client.get_profile("AAA")
assert profile == {"symbol": "AAA", "price": 12.0}
assert client.api_calls_made == 0
assert remaining(client) == 1
def test_paces_next_call_after_timeout(tmp_path):
"""A consumed attempt advances last_call_time and paces the next dispatch,
even when the attempt failed with a transport exception."""
client = _client(tmp_path, max_api_calls=2, rate_limit_delay=1.0)
transport = mock.Mock(side_effect=CLIENT.requests.ReadTimeout("fictional timeout"))
with (
mock.patch.object(client.session, "get", transport),
mock.patch.object(CLIENT.time, "sleep") as sleep,
):
# Stable attempt (last_call_time init 0 -> no pacing), then the v3
# fallback dispatch sees elapsed ~0 << delay and must sleep.
assert client.get_profile("AAA") is None
paced = [call.args[0] for call in sleep.call_args_list if call.args]
assert paced and all(0 < value <= 1.0 for value in paced)
assert transport.call_count == 2
assert client.api_calls_made == 2
assert client.last_call_time > 0
def test_success_path_for_three_symbols_counts_three(tmp_path):
"""Regression: normal 200 flows keep consuming exactly one per response."""
client = _client(tmp_path, max_api_calls=3)
payload = mock.Mock(status_code=200, json=lambda: [{"symbol": "S"}])
transport = mock.Mock(side_effect=lambda *_args, **_kwargs: payload)
with mock.patch.object(client.session, "get", transport):
for symbol in ("AAA", "BBB", "CCC"):
assert client.get_profile(symbol) == {"symbol": "S"}
assert client.cache_hits == 0
# The three responses were cached; a re-read is served from cache
# before the (now exhausted) budget guard is reached.
assert client.get_profile("AAA") == {"symbol": "S"}
assert client.cache_hits == 1
assert transport.call_count == 3
assert client.api_calls_made == 3
def remaining(client):
return client.max_api_calls - client.api_calls_made
scripts/tests/test_capability_cache.py
"""Tests for the FMPClient endpoint capability cache (workstream C, v3.6.1).
Covers: a 402/403 on a bulk endpoint is persisted to the SQLite cache under
``capability:<endpoint>``; a fresh second client instance pointed at the same
cache path pre-disables that endpoint and skips the HTTP call entirely; a
stale (TTL-expired) capability entry is re-probed; and the
``respect_capability_cache=False`` flag disables the whole mechanism.
"""
from __future__ import annotations
import importlib.util
import sys
import tempfile
import time
import unittest
from pathlib import Path
from typing import Any
SCRIPTS = Path(__file__).resolve().parents[1]
if str(SCRIPTS) not in sys.path:
sys.path.insert(0, str(SCRIPTS))
def load_module(name: str, path: Path):
spec = importlib.util.spec_from_file_location(name, path)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
sys.modules[name] = module
spec.loader.exec_module(module)
return module
FMP = load_module("fmp_client_capability_cache_test", SCRIPTS / "fmp_client.py")
class FakeResponse:
def __init__(self, status_code: int, payload: Any):
self.status_code = status_code
self._payload = payload
def json(self):
return self._payload
class SequenceSession:
def __init__(self, responses: list[FakeResponse]):
self.responses = list(responses)
self.calls: list[tuple[str, dict[str, Any]]] = []
def get(self, url: str, params=None, timeout=30):
self.calls.append((url, dict(params or {})))
if not self.responses:
raise AssertionError("unexpected HTTP request")
return self.responses.pop(0)
def close(self):
pass
def make_client(cache_path: Path, **kwargs) -> FMP.FMPClient:
defaults = dict(
api_key="key", # pragma: allowlist secret
cache_path=cache_path,
max_api_calls=5,
rate_limit_delay=0,
)
defaults.update(kwargs)
return FMP.FMPClient(**defaults)
class CapabilityCacheTests(unittest.TestCase):
def test_first_run_records_402_into_capability_cache(self) -> None:
with tempfile.TemporaryDirectory() as temp:
cache_path = Path(temp) / "cache.sqlite3"
client = make_client(cache_path)
client.session = SequenceSession([FakeResponse(402, {"error": "plan"})])
rows = client.get_bulk_dataset("some-bulk-endpoint")
self.assertEqual(rows, [])
self.assertEqual(client.api_calls_made, 1)
client.close()
cache = FMP.SQLiteJsonCache(cache_path)
key = "capability:https://financialmodelingprep.com/stable/some-bulk-endpoint"
stored = cache.get(key, ttl_seconds=30 * 86_400)
self.assertIsNotNone(stored)
self.assertEqual(stored["status"], 402)
self.assertEqual(stored["ttl_days"], 30)
self.assertIn("checked_at", stored)
cache.close()
def test_second_client_instance_skips_call_on_cached_402(self) -> None:
with tempfile.TemporaryDirectory() as temp:
cache_path = Path(temp) / "cache.sqlite3"
first = make_client(cache_path)
first.session = SequenceSession([FakeResponse(402, {"error": "plan"})])
first.get_bulk_dataset("some-bulk-endpoint")
first.close()
second = make_client(cache_path)
second.session = SequenceSession([]) # any GET call raises AssertionError
rows = second.get_bulk_dataset("some-bulk-endpoint")
self.assertEqual(rows, [])
self.assertEqual(second.api_calls_made, 0)
self.assertEqual(second.capability_cache_hits, 1)
self.assertIn(
"https://financialmodelingprep.com/stable/some-bulk-endpoint",
second._disabled_endpoints,
)
second.close()
def test_403_is_also_cached_and_pre_disables(self) -> None:
with tempfile.TemporaryDirectory() as temp:
cache_path = Path(temp) / "cache.sqlite3"
first = make_client(cache_path)
first.session = SequenceSession([FakeResponse(403, {"error": "forbidden"})])
first.get_bulk_dataset("another-endpoint")
first.close()
second = make_client(cache_path)
second.session = SequenceSession([])
rows = second.get_bulk_dataset("another-endpoint")
self.assertEqual(rows, [])
self.assertEqual(second.api_calls_made, 0)
second.close()
def test_ttl_expiry_re_probes(self) -> None:
with tempfile.TemporaryDirectory() as temp:
cache_path = Path(temp) / "cache.sqlite3"
first = make_client(cache_path)
first.session = SequenceSession([FakeResponse(402, {"error": "plan"})])
first.get_bulk_dataset("some-bulk-endpoint")
first.close()
# Manually age the stored capability row past the 30-day TTL.
cache = FMP.SQLiteJsonCache(cache_path)
key = "capability:https://financialmodelingprep.com/stable/some-bulk-endpoint"
cache._connection.execute(
"UPDATE responses SET created_at = ? WHERE cache_key = ?",
(time.time() - 31 * 86_400, key),
)
cache._connection.commit()
cache.close()
second = make_client(cache_path)
second.session = SequenceSession([FakeResponse(200, [{"symbol": "ACME"}])])
rows = second.get_bulk_dataset("some-bulk-endpoint")
self.assertEqual(rows, [{"symbol": "ACME"}])
self.assertEqual(second.api_calls_made, 1)
second.close()
def test_respect_capability_cache_false_disables_mechanism(self) -> None:
with tempfile.TemporaryDirectory() as temp:
cache_path = Path(temp) / "cache.sqlite3"
first = make_client(cache_path)
first.session = SequenceSession([FakeResponse(402, {"error": "plan"})])
first.get_bulk_dataset("some-bulk-endpoint")
first.close()
second = make_client(cache_path, respect_capability_cache=False)
second.session = SequenceSession([FakeResponse(200, [{"symbol": "ACME"}])])
rows = second.get_bulk_dataset("some-bulk-endpoint")
self.assertEqual(rows, [{"symbol": "ACME"}])
self.assertEqual(second.api_calls_made, 1)
self.assertEqual(second.capability_cache_hits, 0)
second.close()
def test_quality_probe_or_other_endpoints_unaffected_on_200(self) -> None:
with tempfile.TemporaryDirectory() as temp:
cache_path = Path(temp) / "cache.sqlite3"
client = make_client(cache_path)
client.session = SequenceSession([FakeResponse(200, [{"symbol": "ACME"}])])
rows = client.get_bulk_dataset("healthy-endpoint")
self.assertEqual(rows, [{"symbol": "ACME"}])
self.assertEqual(client.capability_cache_hits, 0)
client.close()
def test_remaining_calls_property(self) -> None:
with tempfile.TemporaryDirectory() as temp:
client = make_client(Path(temp) / "cache.sqlite3", max_api_calls=5)
self.assertEqual(client.remaining_calls, 5)
client.session = SequenceSession([FakeResponse(200, [{"symbol": "A"}])])
client.get_bulk_dataset("healthy-endpoint")
self.assertEqual(client.remaining_calls, 4)
client.close()
def test_diagnostics_exposes_capability_cache_hits(self) -> None:
with tempfile.TemporaryDirectory() as temp:
cache_path = Path(temp) / "cache.sqlite3"
first = make_client(cache_path)
first.session = SequenceSession([FakeResponse(402, {"error": "plan"})])
first.get_bulk_dataset("some-bulk-endpoint")
first.close()
second = make_client(cache_path)
second.session = SequenceSession([])
second.get_bulk_dataset("some-bulk-endpoint")
diag = second.diagnostics()
self.assertEqual(diag["capability_cache_hits"], 1)
second.close()
if __name__ == "__main__":
unittest.main()
scripts/tests/test_checkpoint_counts.py
"""Real screening outputs must cross the checkpoint count/completion boundary."""
from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
from zipfile import ZipFile
import pytest
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
if str(SCRIPTS_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPTS_DIR))
import build_discovery_pool as DISCOVERY
import manage_run_state as RUN_STATE
import screen_universe as SCREEN
ASSETS = SCRIPTS_DIR.parent / "assets"
AS_OF = "2026-08-22T14:00:00-07:00"
def _listing(symbol: str, *, eligible: bool) -> dict:
pe = 16.0 if eligible else 80.0
return {
"symbol": symbol,
"country": "US",
"currency": "USD",
"exchange": "NASDAQ",
"sector": "Technology",
"industry": "Software",
"is_actively_trading": True,
"is_common_stock": True,
"price": 40.0,
"market_cap": 3_500_000_000,
"average_daily_dollar_volume": 12_000_000.0,
"average_daily_dollar_volume_method": "provider_average_dollar_volume",
"average_volume_period_days": 20,
"liquidity_source_ids": ["fictional-listing"],
"revenue_growth_pct": 14.0 if eligible else -5.0,
"eps_growth_pct": 22.0 if eligible else -10.0,
"standard_fcf": 250_000_000.0,
"roic_pct": 18.0,
"forward_pe": pe,
"forward_eps": 40.0 / pe,
"forward_pe_period": "FY1",
"forward_fiscal_year": "FY2027",
"forward_period_end": "2027-06-30",
"forward_estimate_as_of": AS_OF,
"forward_estimate_source_ids": ["fictional-estimate"],
"forward_metric_origin": "computed_from_price_and_fy1_eps",
"analyst_count": 6,
"dilution_pct": 1.0,
"net_debt_to_ebitda": 1.0,
"fcf_yield_pct": 6.0,
}
def _produce(
root: Path, *, bounded=False, eligible=False, exhausted=True, selection_limit=None
) -> Path:
"""Run original production CLIs, including their artifact/hash writers."""
listing = root / "listing.jsonl"
listing.write_text(
"".join(
json.dumps(_listing(symbol, eligible=eligible)) + "\n" for symbol in ("AAA", "BBB")
),
encoding="utf-8",
)
output = root / "screen"
args = [
"--input",
str(listing),
"--output-dir",
str(output),
"--analysis-as-of",
AS_OF,
"--source-id",
"fictional-listing",
"--retrieval-min-market-cap",
"500000000",
"--retrieval-max-market-cap",
"20000000000",
"--provider-reported-total",
"2",
"--pages-fetched",
"1",
"--pagination-exhausted",
]
if bounded:
assert (
DISCOVERY.main(
[
"--input",
str(listing),
"--output-dir",
str(output),
"--source-id",
"fictional-listing",
"--max-pool",
"1",
]
)
== 0
)
args += [
"--candidate-pool",
str(output / "discovery-pool.jsonl"),
"--discovery-audit",
str(output / "discovery-audit.json"),
"--candidate-generation-mode",
"liquidity_stratified_estimates",
]
if exhausted:
args.append("--candidate-pool-exhausted")
if selection_limit is not None:
args += ["--max-deep-dives", str(selection_limit)]
assert SCREEN.main(args) in (0, 2)
return output / "broad-screen-audit.json"
def _init(root: Path) -> Path:
run = root / "checkpoint"
assert (
RUN_STATE.main(
[
"init",
"--run-dir",
str(run),
"--analysis-as-of",
AS_OF,
"--price-as-of",
AS_OF,
"--session",
"regular_close",
"--price-source-id",
"fictional-quote",
"--market-context",
str(ASSETS / "market-context.example.json"),
"--global-sources",
str(ASSETS / "global-sources.example.json"),
]
)
== 0
)
return run
def _attach(run: Path, audit: Path) -> int:
return RUN_STATE.main(
[
"set-screening-audit",
"--run-dir",
str(run),
"--audit",
str(audit),
]
)
@pytest.mark.parametrize("bounded", [False, True], ids=["full", "bounded"])
def test_zero_selection_producer_to_complete_snapshot(tmp_path, bounded):
audit_path = _produce(tmp_path, bounded=bounded)
audit = json.loads(audit_path.read_text())
status = "no_qualifying_candidates" + ("_in_bounded_pool" if bounded else "")
assert audit["candidate_pool_status"] == status
assert audit["deep_dive_plan"]["selected_count"] == 0
assert audit["selected_symbols"] == []
assert audit["enrichment"]["candidate_pool_exhausted"] is True
run = _init(tmp_path)
assert _attach(run, audit_path) == 0
assert RUN_STATE.main(["set-status", "--run-dir", str(run), "complete"]) == 0
output = tmp_path / "final.json"
assert RUN_STATE.main(["assemble", "--run-dir", str(run), "--output", str(output)]) == 0
snapshot = json.loads(output.read_text())
assert snapshot["run_metadata"]["status"] == "complete"
assert snapshot["run_metadata"]["selected_symbols"] == []
assert snapshot["candidates"] == []
assert snapshot["screening_audit"]["candidate_pool_status"] == status
assert snapshot["screening_audit"]["conclusion_scope"] == (
"stratified_discovery_pool" if bounded else "full_listing_universe"
)
@pytest.mark.parametrize(
"count",
[None, -1, True, False, 2.0, 2.9, float("nan"), float("inf"), "2", "bad", [2], {"value": 2}],
ids=[
"null",
"negative",
"true",
"false",
"float",
"fractional",
"nan",
"infinity",
"numeric-string",
"text",
"array",
"object",
],
)
def test_invalid_count_fails_cleanly_without_changing_state(tmp_path, capsys, count):
audit_path = _produce(tmp_path, eligible=True)
audit = json.loads(audit_path.read_text())
assert audit["deep_dive_plan"]["selected_count"] == 2
audit["deep_dive_plan"]["selected_count"] = count
audit_path.write_text(json.dumps(audit), encoding="utf-8")
run = _init(tmp_path)
before = (run / "run_state.json").read_bytes()
capsys.readouterr()
assert _attach(run, audit_path) == 1
assert "deep_dive_plan.selected_count" in capsys.readouterr().err
assert (run / "run_state.json").read_bytes() == before
@pytest.mark.parametrize("missing", [False, True], ids=["mismatched", "missing"])
def test_missing_or_inconsistent_count_is_rejected(tmp_path, missing):
audit_path = _produce(tmp_path, eligible=True)
audit = json.loads(audit_path.read_text())
if missing:
del audit["deep_dive_plan"]["selected_count"]
else:
audit["deep_dive_plan"]["selected_count"] = 1
audit_path.write_text(json.dumps(audit), encoding="utf-8")
run = _init(tmp_path)
before = (run / "run_state.json").read_bytes()
assert _attach(run, audit_path) == 1
assert (run / "run_state.json").read_bytes() == before
@pytest.mark.parametrize("count", [False, True])
def test_boolean_count_is_rejected_even_when_numerically_equal(tmp_path, count):
audit_path = _produce(tmp_path, eligible=count, selection_limit=1)
audit = json.loads(audit_path.read_text())
assert audit["deep_dive_plan"]["selected_count"] == int(count)
audit["deep_dive_plan"]["selected_count"] = count
audit_path.write_text(json.dumps(audit), encoding="utf-8")
run = _init(tmp_path)
assert _attach(run, audit_path) == 1
def test_positive_selection_still_requires_verified_candidates(tmp_path):
audit_path = _produce(tmp_path, eligible=True)
run = _init(tmp_path)
assert _attach(run, audit_path) == 0
state = json.loads((run / "run_state.json").read_text())
assert state["unprocessed_candidates"] == ["AAA", "BBB"]
assert RUN_STATE.main(["set-status", "--run-dir", str(run), "complete"]) == 1
def test_zero_count_does_not_authorize_unexhausted_pool_completion(tmp_path):
audit_path = _produce(tmp_path, exhausted=False)
audit = json.loads(audit_path.read_text())
assert audit["candidate_pool_status"] == "insufficient_data"
assert audit["deep_dive_plan"]["selected_count"] == 0
run = _init(tmp_path)
assert _attach(run, audit_path) == 0
assert RUN_STATE.main(["set-status", "--run-dir", str(run), "complete"]) == 1
assert (
RUN_STATE.main(
[
"assemble",
"--run-dir",
str(run),
"--output",
str(tmp_path / "invalid.json"),
]
)
== 1
)
assert not (tmp_path / "invalid.json").exists()
@pytest.mark.parametrize("tamper", ["artifact", "commitment"])
def test_zero_count_keeps_artifact_and_commitment_guards(tmp_path, capsys, tamper):
audit_path = _produce(tmp_path)
audit = json.loads(audit_path.read_text())
if tamper == "artifact":
artifact = audit_path.parent / audit["candidate_pool"]["artifact_path"]
artifact.write_bytes(artifact.read_bytes() + b" ")
expected = "artifact SHA-256"
else:
audit["deep_dive_plan"]["selected_set_sha256"] = "0" * 64
audit_path.write_text(json.dumps(audit), encoding="utf-8")
expected = "selected_set_sha256"
run = _init(tmp_path)
before = (run / "run_state.json").read_bytes()
capsys.readouterr()
assert _attach(run, audit_path) == 1
assert expected in capsys.readouterr().err
assert (run / "run_state.json").read_bytes() == before
@pytest.mark.parametrize("bounded", [False, True], ids=["full", "bounded"])
def test_packaged_cli_completes_zero_candidate_run(tmp_path, bounded):
audit = _produce(tmp_path, bounded=bounded)
run = _init(tmp_path)
destination = tmp_path / "unpacked"
package = SCRIPTS_DIR.parents[2] / "skill-packages/us-undervalued-growth-screener.skill"
with ZipFile(package) as archive:
archive.extractall(destination)
script = destination / "us-undervalued-growth-screener/scripts/manage_run_state.py"
assert script.read_bytes() == (SCRIPTS_DIR / "manage_run_state.py").read_bytes()
output = tmp_path / "packaged-final.json"
commands = [
["set-screening-audit", "--run-dir", str(run), "--audit", str(audit)],
["set-status", "--run-dir", str(run), "complete"],
["assemble", "--run-dir", str(run), "--output", str(output)],
]
for command in commands:
result = subprocess.run(
[sys.executable, str(script), *command],
capture_output=True,
text=True,
check=False,
cwd=tmp_path,
)
assert result.returncode == 0, result.stderr
snapshot = json.loads(output.read_text())
assert snapshot["run_metadata"]["status"] == "complete"
assert snapshot["candidates"] == []
scripts/tests/test_claude_code_pipeline.py
from __future__ import annotations
import importlib.util
import json
import tempfile
import unittest
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
SCRIPTS = Path(__file__).resolve().parents[1]
import sys
if str(SCRIPTS) not in sys.path:
sys.path.insert(0, str(SCRIPTS))
def load_module(name: str, path: Path):
spec = importlib.util.spec_from_file_location(name, path)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
import sys
sys.modules[name] = module
spec.loader.exec_module(module)
return module
FMP = load_module("fmp_client_direct_test", SCRIPTS / "fmp_client.py")
PIPELINE = load_module("run_pipeline_direct_test", SCRIPTS / "run_pipeline.py")
class FakeResponse:
def __init__(self, status_code: int, payload: Any):
self.status_code = status_code
self._payload = payload
def json(self):
return self._payload
class SequenceSession:
def __init__(self, responses: list[FakeResponse]):
self.responses = list(responses)
self.calls: list[tuple[str, dict[str, Any]]] = []
def get(self, url: str, params=None, timeout=30):
self.calls.append((url, dict(params or {})))
if not self.responses:
raise AssertionError("unexpected HTTP request")
return self.responses.pop(0)
def close(self):
pass
class FMPClientTests(unittest.TestCase):
def test_persistent_cache_and_raw_store_do_not_expose_api_key(self) -> None:
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
cache = root / "cache.sqlite3"
raw = root / "raw"
client = FMP.FMPClient(
api_key="top-secret-key", # pragma: allowlist secret
cache_path=cache,
raw_store_dir=raw,
max_api_calls=5,
rate_limit_delay=0,
)
session = SequenceSession(
[FakeResponse(200, [{"symbol": "ACME", "companyName": "Acme"}])]
)
client.session = session
profile = client.get_profile("ACME")
self.assertEqual(profile["symbol"], "ACME")
self.assertEqual(client.api_calls_made, 1)
client.close()
raw_files = list(raw.glob("*.json"))
self.assertEqual(len(raw_files), 1)
raw_text = raw_files[0].read_text(encoding="utf-8")
self.assertNotIn("top-secret-key", raw_text)
self.assertNotIn("apikey", json.loads(raw_text)["params"])
offline = FMP.FMPClient(
api_key=None,
offline=True,
cache_path=cache,
raw_store_dir=None,
max_api_calls=5,
rate_limit_delay=0,
)
cached = offline.get_profile("ACME")
self.assertEqual(cached["companyName"], "Acme")
self.assertEqual(offline.api_calls_made, 0)
self.assertEqual(offline.cache_hits, 1)
offline.close()
def test_stable_failure_falls_back_to_v3(self) -> None:
with tempfile.TemporaryDirectory() as temp:
client = FMP.FMPClient(
api_key="key", # pragma: allowlist secret
cache_path=Path(temp) / "cache.sqlite3",
max_api_calls=5,
rate_limit_delay=0,
)
client.session = SequenceSession(
[
FakeResponse(403, {"error": "plan"}),
FakeResponse(200, [{"symbol": "ACME", "companyName": "Acme"}]),
]
)
profile = client.get_profile("ACME")
self.assertEqual(profile["symbol"], "ACME")
self.assertEqual(client.api_calls_made, 2)
client.close()
def test_call_budget_is_enforced(self) -> None:
with tempfile.TemporaryDirectory() as temp:
client = FMP.FMPClient(
api_key="key", # pragma: allowlist secret
cache_path=Path(temp) / "cache.sqlite3",
max_api_calls=1,
rate_limit_delay=0,
)
client.session = SequenceSession(
[
FakeResponse(200, [{"symbol": "A", "companyName": "A"}]),
]
)
self.assertIsNotNone(client.get_profile("A"))
with self.assertRaises(FMP.ApiCallBudgetExceeded):
client.get_profile("B")
client.close()
class FakePipelineClient:
def __init__(self):
self.api_calls_made = 0
self.max_api_calls = 999
self.cache_hits = 0
self.symbol_rows: dict[str, dict[str, Any]] = {}
sectors = [
("Technology", "Software"),
("Industrials", "Machinery"),
("Basic Materials", "Chemicals"),
("Consumer Defensive", "Packaged Foods"),
]
counter = 0
for exchange in PIPELINE.EXCHANGES:
for lane_idx, (sector, industry) in enumerate(sectors):
counter += 1
symbol = f"T{counter:02d}"
self.symbol_rows[symbol] = {
"symbol": symbol,
"companyName": f"Test {symbol}",
"exchangeShortName": exchange,
"sector": sector,
"industry": industry,
"price": (20.0, 50.0, 30.0, 45.0)[lane_idx],
"marketCap": 700_000_000 + counter * 500_000_000,
"volume": 600_000,
"isEtf": False,
"isFund": False,
"isActivelyTrading": True,
"currency": "USD",
"country": "US",
"lane_idx": lane_idx,
}
def get_company_screener(self, *, exchange, min_market_cap, max_market_cap, min_price, limit):
self.api_calls_made += 1
return [
{k: v for k, v in row.items() if k != "lane_idx"}
for row in self.symbol_rows.values()
if row["exchangeShortName"] == exchange
and min_market_cap < row["marketCap"] < max_market_cap
and row["price"] > min_price
][:limit]
def get_bulk_dataset(self, endpoint: str, **params):
self.api_calls_made += 1
if endpoint == "analyst-estimates-bulk":
year = int(params["year"])
rows = []
for symbol, source in self.symbol_rows.items():
idx = source["lane_idx"]
fy1 = 1.5 + idx * 0.2
growth = (1.18, 1.32, 1.10, 1.20)[idx]
offset = year - 2026
eps = fy1 * (growth**offset)
revenue_growth = (1.11, 1.24, 1.05, 1.12)[idx]
revenue = 500_000_000 * (revenue_growth**offset)
rows.append(
{
"symbol": symbol,
"date": f"{year}-12-31",
"fiscalYear": str(year),
"epsAvg": eps,
"epsLow": eps * 0.9,
"epsHigh": eps * 1.1,
"revenueAvg": revenue,
"numAnalystsEps": 6,
"numAnalystsRevenue": 6,
}
)
return rows
if endpoint == "eod-bulk":
return [
{"symbol": symbol, "volume": 500_000 + idx * 50_000}
for idx, symbol in enumerate(sorted(self.symbol_rows))
]
if endpoint == "ratios-ttm-bulk":
return [
{"symbol": symbol, "freeCashFlowYield": 0.07, "netDebtToEBITDA": 1.2}
for symbol in self.symbol_rows
]
if endpoint == "key-metrics-ttm-bulk":
return [
{"symbol": symbol, "returnOnInvestedCapital": 0.16, "enterpriseValueOverFCF": 14.0}
for symbol in self.symbol_rows
]
if endpoint == "income-statement-growth-bulk":
return [
{"symbol": symbol, "growthRevenue": 0.12, "growthEPS": 0.18}
for symbol in self.symbol_rows
]
return []
def get_analyst_estimates(self, symbol, *, period="annual", limit=6):
raise AssertionError("bulk estimate path should be used")
def get_key_metrics_ttm(self, symbol):
self.api_calls_made += 1
return [
{
"symbol": symbol,
"returnOnInvestedCapitalTTM": 0.16,
"freeCashFlowYieldTTM": 0.07,
"evToFreeCashFlowTTM": 14.0,
"netDebtToEBITDATTM": 1.2,
"stockBasedCompensationToRevenueTTM": 0.02,
}
]
def get_historical_prices(self, symbol, *, from_date, to_date):
return [{"date": f"2026-08-{day:02d}", "volume": 500_000} for day in range(1, 25)]
def diagnostics(self):
return {
"api_calls_made": self.api_calls_made,
"max_api_calls": self.max_api_calls,
"cache_hits": self.cache_hits,
"failure_count": 0,
"failure_samples": [],
"disabled_endpoint_count": 0,
"offline": False,
}
class DirectPipelineTests(unittest.TestCase):
def test_direct_pipeline_keeps_provider_payloads_on_disk_and_emits_compact_summary(
self,
) -> None:
config = PIPELINE.load_config(None)
config.update(
{
"company_screener_limit": 100,
"pre_enrichment_limit": 20,
"exact_liquidity_limit": 20,
"provider_prefilter_pool_size": 12,
"provider_prefilter_minimum_pool": 8,
"provider_prefilter_per_lane": 4,
"max_deep_dive_candidates": 3,
"bulk_estimate_minimum_coverage_pct": 10.0,
}
)
with tempfile.TemporaryDirectory() as temp:
output = Path(temp) / "run-direct"
result = PIPELINE.execute_pipeline(
FakePipelineClient(),
config,
analysis_as_of=datetime(2026, 8, 29, 17, tzinfo=timezone.utc),
output_dir=output,
include_packets=False,
)
self.assertEqual(result.exit_code, 0)
self.assertEqual(result.summary["status"], "ready_for_underwriting")
self.assertEqual(result.summary["runtime"]["skill_version"], "3.6.1")
self.assertEqual(result.summary["estimate_acquisition_mode"], "analyst_estimates_bulk")
self.assertTrue(result.summary["listing_enumeration_verified"])
self.assertGreaterEqual(len(result.summary["selected_symbols"]), 1)
self.assertLess(len(json.dumps(result.summary).encode("utf-8")), 20_000)
self.assertTrue((output / "run-summary.json").is_file())
self.assertTrue((output / "NEXT_ACTION.json").is_file())
self.assertTrue((output / "audit" / "provider-prefilter-audit.json").is_file())
self.assertTrue((output / "audit" / "broad-screen-audit.json").is_file())
next_action = json.loads((output / "NEXT_ACTION.json").read_text(encoding="utf-8"))
self.assertFalse(next_action["user_confirmation_required"])
self.assertTrue(next_action["do_not_read_bulk_provider_payloads_into_model_context"])
def test_adaptive_enumeration_splits_saturated_band(self) -> None:
class SaturatedClient:
def get_company_screener(
self, *, exchange, min_market_cap, max_market_cap, min_price, limit
):
width = max_market_cap - min_market_cap
count = limit if width > 100_000_000 else max(1, limit - 1)
return [
{
"symbol": f"{exchange[:1]}{int(min_market_cap)}{idx}",
"companyName": "Test",
"exchangeShortName": exchange,
"price": 10,
"marketCap": min_market_cap + (idx + 1) * max(width / (count + 1), 1),
"isEtf": False,
"isFund": False,
"isActivelyTrading": True,
}
for idx in range(count)
]
rows, audit = PIPELINE.collect_listing_universe(
SaturatedClient(),
min_market_cap=500_000_000,
max_market_cap=1_000_000_000,
min_price=5,
page_limit=4,
minimum_band_width=50_000_000,
maximum_depth=8,
)
self.assertTrue(audit["enumeration_verified"])
self.assertGreater(audit["query_count"], len(PIPELINE.EXCHANGES))
self.assertTrue(rows)
if __name__ == "__main__":
unittest.main()
scripts/tests/test_evaluate_candidates.py
from __future__ import annotations
import hashlib
import importlib.util
import json
import subprocess
import sys
import tempfile
import unittest
from copy import deepcopy
from datetime import datetime, timezone
from pathlib import Path
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
ASSETS_DIR = Path(__file__).resolve().parents[2] / "assets"
EXAMPLE_PATH = ASSETS_DIR / "candidate-input.example.json"
AUDIT_ROWS_PATH = ASSETS_DIR / "broad-screen-results.example.jsonl"
UNIVERSE_AUDIT_ROWS_PATH = ASSETS_DIR / "universe-audit-results.example.jsonl"
UNIVERSE_AUDIT_PATH = UNIVERSE_AUDIT_ROWS_PATH
MARKET_CONTEXT_PATH = ASSETS_DIR / "market-context.example.json"
GLOBAL_SOURCES_PATH = ASSETS_DIR / "global-sources.example.json"
def load_module(name: str, path: Path):
spec = importlib.util.spec_from_file_location(name, path)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
EVALUATOR = load_module("evaluate_candidates", SCRIPTS_DIR / "evaluate_candidates.py")
RUN_STATE = load_module("manage_run_state", SCRIPTS_DIR / "manage_run_state.py")
CONTRACT = load_module("research_contract_test", SCRIPTS_DIR / "research_contract.py")
SCREEN = load_module("screen_universe_test", SCRIPTS_DIR / "screen_universe.py")
DISCOVERY = load_module("build_discovery_pool_test", SCRIPTS_DIR / "build_discovery_pool.py")
NORMALIZER = load_module("normalize_estimates_test", SCRIPTS_DIR / "normalize_estimates.py")
VERSION = load_module("skill_version_test", SCRIPTS_DIR / "skill_version.py")
PREPUBLISH = load_module("prepublish_audit_test", SCRIPTS_DIR / "prepublish_audit.py")
BUNDLER = load_module("bundle_run_artifacts_test", SCRIPTS_DIR / "bundle_run_artifacts.py")
PROVIDER_POOL = load_module(
"build_provider_prefilter_pool_test", SCRIPTS_DIR / "build_provider_prefilter_pool.py"
)
ORIGINAL_SCREEN_RUN = SCREEN.run
ORIGINAL_SCREEN_RUN_LAYERED = SCREEN.run_layered
ORIGINAL_DISCOVERY_BUILD_POOL = DISCOVERY.build_pool
def _prepare_screen_rows(
rows: list[dict], *, analysis_as_of: str = "2026-08-22T14:00:00-07:00"
) -> list[dict]:
"""Upgrade legacy test rows to the current discovery input contract.
Production code remains fail-closed. Only general regression fixtures are
normalized here; v3.4 rejection tests call the original functions directly.
"""
prepared = deepcopy(rows)
for row in prepared:
metrics = row.get("metrics") if isinstance(row.get("metrics"), dict) else {}
def first(*keys, row=row, metrics=metrics):
for key in keys:
if row.get(key) is not None:
return row.get(key)
if metrics.get(key) is not None:
return metrics.get(key)
return None
addv = first("average_daily_dollar_volume", "avg_dollar_volume")
avg_volume = first("average_volume", "avg_volume")
if addv is not None:
row.setdefault("average_daily_dollar_volume", addv)
row.setdefault("average_daily_dollar_volume_method", "provider_average_dollar_volume")
row.setdefault("average_volume_period_days", 20)
row.setdefault("liquidity_source_ids", ["listing-source"])
elif avg_volume is not None:
row.setdefault("average_volume", avg_volume)
row.setdefault("average_daily_dollar_volume_method", "price_x_provider_average_volume")
row.setdefault("average_volume_period_days", 20)
row.setdefault("liquidity_source_ids", ["listing-source"])
pe = first("forward_pe")
price = first("price", "last")
if (
isinstance(pe, (int, float))
and pe > 0
and isinstance(price, (int, float))
and price > 0
):
row.setdefault("forward_pe", pe)
row.setdefault("forward_eps", price / pe)
row.setdefault("forward_pe_period", "FY1")
row.setdefault("forward_fiscal_year", "FY2027")
row.setdefault("forward_period_end", "2027-06-30")
row.setdefault("forward_estimate_as_of", analysis_as_of)
row.setdefault("forward_estimate_source_ids", ["estimate-source"])
row.setdefault("forward_metric_origin", "provider_fy1_pe_with_eps_reconciliation")
row.setdefault("analyst_count", first("analyst_count", "fy1_analyst_count") or 4)
return prepared
def _screen_run_layered_with_contract(universe_rows, candidate_rows, config, *args, **kwargs):
analysis_as_of = kwargs.get("analysis_as_of", "2026-08-22T14:00:00-07:00")
prepared_universe = _prepare_screen_rows(list(universe_rows), analysis_as_of=analysis_as_of)
prepared_candidates = _prepare_screen_rows(list(candidate_rows), analysis_as_of=analysis_as_of)
mode = kwargs.get("candidate_generation_mode", "full_universe_fundamentals")
if mode != "full_universe_fundamentals":
audit = deepcopy(kwargs.get("discovery_audit") or {})
audit.setdefault("runtime", VERSION.runtime_metadata())
audit.setdefault("valid", True)
audit.setdefault("selection_method", "sector_market_cap_stratified_validated_liquidity")
audit.setdefault("input_row_count", len(prepared_universe))
audit.setdefault("selected_count", len(prepared_candidates))
audit.setdefault(
"selected_symbols",
sorted(str(row.get("symbol") or "").upper() for row in prepared_candidates),
)
audit.setdefault("source_ids", ["listing-source"])
audit.setdefault("artifact_path", "discovery-pool.jsonl")
audit.setdefault("artifact_sha256", "synthetic-test-sha")
audit.setdefault(
"liquidity_validation", {"basis_validated": True, "minimum_window_days": 20}
)
audit.setdefault(
"coverage_plan",
{
"coverage_plan_valid": True,
"user_requested_range_spanned": True,
"market_cap_buckets_cover_user_requested_range": True,
"single_band_only": False,
},
)
audit.setdefault(
"scope",
{
"user_requested_min_market_cap": 500_000_000,
"user_requested_max_market_cap": 20_000_000_000,
"user_requested_scope_complete": True,
"scope_valid": True,
},
)
kwargs["discovery_audit"] = audit
return ORIGINAL_SCREEN_RUN_LAYERED(
prepared_universe, prepared_candidates, config, *args, **kwargs
)
SCREEN.run_layered = _screen_run_layered_with_contract
def _screen_run_with_contract(rows, config, *args, **kwargs):
analysis_as_of = kwargs.get("analysis_as_of", "2026-08-22T14:00:00-07:00")
return ORIGINAL_SCREEN_RUN(
_prepare_screen_rows(list(rows), analysis_as_of=analysis_as_of), config, *args, **kwargs
)
SCREEN.run = _screen_run_with_contract
def _discovery_build_pool_with_contract(rows, *args, **kwargs):
return ORIGINAL_DISCOVERY_BUILD_POOL(_prepare_screen_rows(list(rows)), *args, **kwargs)
DISCOVERY.build_pool = _discovery_build_pool_with_contract
def canonical_jsonl(rows: list[dict]) -> bytes:
return b"".join(
(json.dumps(row, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode(
"utf-8"
)
for row in rows
)
def deep_dive_plan(selected: list[str], analysis_as_of: str, *, maximum: int = 3) -> dict:
payload = {
"analysis_as_of": analysis_as_of,
"max_deep_dive_candidates": maximum,
"selected_symbols": sorted(selected),
}
digest = hashlib.sha256(
json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
return {
"selected_symbols": sorted(selected),
"selected_count": len(selected),
"all_selected_must_be_resolved": True,
"budget_locked": True,
"budget_change_requires_rescreen": True,
"user_confirmation_required": False,
"user_continue_instruction_allowed": False,
"selected_set_is_committed": True,
"max_deep_dive_candidates": maximum,
"selected_set_sha256": digest,
"commitment_payload": payload,
}
class FixtureMixin:
def setUp(self) -> None:
self.payload = json.loads(EXAMPLE_PATH.read_text(encoding="utf-8"))
self.audit_rows = [
json.loads(line)
for line in AUDIT_ROWS_PATH.read_text(encoding="utf-8").splitlines()
if line.strip()
]
self.universe_audit_rows = [
json.loads(line)
for line in UNIVERSE_AUDIT_ROWS_PATH.read_text(encoding="utf-8").splitlines()
if line.strip()
]
self.universe_rows = [
json.loads(line)
for line in UNIVERSE_AUDIT_PATH.read_text(encoding="utf-8").splitlines()
if line.strip()
]
def candidate(self, symbol: str) -> dict:
for row in self.payload["candidates"]:
if row.get("identity", {}).get("symbol") == symbol:
return row
raise AssertionError(f"missing candidate {symbol}")
def audit_row(self, symbol: str) -> dict:
for row in self.audit_rows:
if row.get("symbol") == symbol:
return deepcopy(row)
raise AssertionError(f"missing audit row {symbol}")
def universe_audit_row(self, symbol: str) -> dict:
for row in self.universe_audit_rows:
if row.get("symbol") == symbol:
return deepcopy(row)
raise AssertionError(f"missing universe audit row {symbol}")
def universe_row(self, symbol: str) -> dict:
for row in self.universe_rows:
if row.get("symbol") == symbol:
return deepcopy(row)
raise AssertionError(f"missing universe row {symbol}")
def single(self, symbol: str) -> dict:
payload = deepcopy(self.payload)
payload["candidates"] = [deepcopy(self.candidate(symbol))]
candidate_row = self.audit_row(symbol)
universe_row = self.universe_audit_row(symbol)
decision = candidate_row["decision"]["status"]
selected = [symbol] if decision == "selected" else []
universe_rows = [universe_row]
candidate_rows = [candidate_row]
universe_sha = hashlib.sha256(canonical_jsonl(universe_rows)).hexdigest()
candidate_sha = hashlib.sha256(canonical_jsonl(candidate_rows)).hexdigest()
attempted = candidate_row.get("enrichment_attempted") is True
resolved = candidate_row.get("enrichment_resolved") is True
unresolved = not resolved
evaluable = candidate_row.get("discovery_evaluable") is True
selection_eligible = candidate_row.get("selection_eligible") is True
exhausted = resolved
if selected:
pool_status = "sufficient" if resolved else "sufficient_pending_enrichment"
outcome = "selected" if resolved else "selected_pending_enrichment"
elif unresolved:
pool_status = "insufficient_data"
outcome = "insufficient_data"
else:
pool_status = "no_qualifying_candidates"
outcome = "no_candidates"
payload["screening_audit"] = {
"audit_schema_version": 3,
"contract_revision": VERSION.CONTRACT_REVISION,
"runtime": VERSION.runtime_metadata(),
"analysis_as_of": payload["analysis_as_of"],
"generated_at": payload["analysis_as_of"],
"candidate_generation_mode": "full_universe_fundamentals",
"candidate_pool_status": pool_status,
"selection_outcome": outcome,
"selected_symbols": selected,
"deep_dive_plan": deep_dive_plan(selected, payload["analysis_as_of"], maximum=3),
"filters": {
"test_fixture": True,
"minimum_listing_data_coverage_pct": 95.0,
"max_deep_dive_candidates": 3,
},
"source_ids": ["universe-fmp-20260822"],
"scope": {
"requested_min_market_cap": 500_000_000,
"requested_max_market_cap": 20_000_000_000,
"retrieval_min_market_cap": 500_000_000,
"retrieval_max_market_cap": 20_000_000_000,
"retrieval_scope_explicit": True,
"scope_override_authorized": False,
"scope_reduction_reason": None,
"user_scope_evidence": None,
"user_requested_scope": {
"min_market_cap": 500_000_000,
"max_market_cap": 20_000_000_000,
"source": "skill_default",
},
"scope_reduced": False,
"scope_reduction_mode": "none",
"scope_reduction_disclosed": True,
"user_requested_scope_complete": True,
"executed_scope_complete": True,
"bounded_sampling_ready": False,
"screening_scope_ready": True,
"executed_scope": {
"min_market_cap": 500_000_000,
"max_market_cap": 20_000_000_000,
},
"scope_complete": True,
"reasons": [],
"enumeration": {
"verified": True,
"provider_reported_total": 1,
"rows_fetched": 1,
"pages_fetched": 1,
"pagination_exhausted": True,
"band_audit": [],
"bands_well_formed": False,
"bands_cover_executed_range": False,
"bands_cover_requested_range": False,
"bands_verified": False,
},
},
"universe": {
"row_count": 1,
"decision_counts": {universe_row["listing_decision"]["status"]: 1},
"listing_data_complete_count": int(
universe_row.get("listing_data_complete") is True
),
"listing_data_complete_pct": 100.0
if universe_row.get("listing_data_complete") is True
else 0.0,
"in_scope_count": int(
universe_row["listing_decision"]["status"] in {"in_scope", "liquidity_review"}
),
"source_ids": ["universe-fmp-20260822"],
"embedded_rows": universe_rows,
"artifact_sha256": universe_sha,
},
"candidate_pool": {
"row_count": 1,
"decision_counts": {decision: 1},
"discovery_evaluable_count": int(evaluable),
"discovery_evaluable_pct": 100.0 if evaluable else 0.0,
"selection_eligible_count": int(selection_eligible),
"selected_count": len(selected),
"in_scope_covered_count": 1,
"in_scope_missing_count": 0,
"in_scope_missing_symbols": [],
"coverage_complete": True,
"coverage_scope": "full_listing_universe",
"listing_coverage_complete": True,
"generation_audit": {
"runtime": VERSION.runtime_metadata(),
"valid": True,
"selection_method": "full_universe_fundamentals",
"input_row_count": 1,
"selected_count": 1,
"selected_symbols": [symbol],
"liquidity_validation": {
"basis_validated": True,
"minimum_window_days": 20,
},
"coverage_plan": {
"coverage_plan_valid": True,
"user_requested_range_spanned": True,
"market_cap_buckets_cover_user_requested_range": True,
"single_band_only": False,
},
},
"generation_review_reasons": [],
"fundamental_complete_count": int(
candidate_row.get("fundamental_complete") is True
),
"fundamental_complete_pct": 100.0
if candidate_row.get("fundamental_complete") is True
else 0.0,
"source_ids": ["universe-fmp-20260822"],
"symbols_not_in_universe": [],
"embedded_rows": candidate_rows,
"artifact_sha256": candidate_sha,
},
"enrichment": {
"status": "complete" if resolved else "pending",
"next_action": (
"proceed_to_deep_dive"
if selected and resolved
else "continue_enrichment"
if unresolved
else "publish_no_candidates"
),
"discovery_pool_required": False,
"attempted_count": int(attempted),
"resolved_count": int(resolved),
"unresolved_count": int(unresolved),
"resolution_pct": 100.0 if resolved else 0.0,
"all_rows_resolved": resolved,
"maximum_attempts": 60,
"candidate_pool_exhaustion_declared": exhausted,
"candidate_pool_exhausted": exhausted,
"candidate_pool_covers_in_scope": True,
"candidate_pool_scope_verified": True,
"conclusion_scope": "full_listing_universe",
"queue_count": int(unresolved),
"queue_symbols": [symbol] if unresolved else [],
},
}
payload["screening_funnel"] = {
"universe_count": 1,
"listing_in_scope_count": int(
universe_row["listing_decision"]["status"] in {"in_scope", "liquidity_review"}
),
"candidate_pool_count": 1,
"discovery_evaluable_count": int(evaluable),
"deep_dive_selected_count": len(selected),
"preflight_passed_count": len(selected),
"deep_dive_completed_count": len(selected),
}
payload["run_metadata"]["status"] = (
"complete" if pool_status in {"sufficient", "no_qualifying_candidates"} else "partial"
)
payload["run_metadata"]["selected_symbols"] = selected
payload["run_metadata"]["unprocessed_candidates"] = [] if resolved else [symbol]
return payload
def evaluate(self, payload: dict, *, strict: bool = True) -> dict:
return EVALUATOR.evaluate_snapshot(payload, strict=strict, artifact_root=ASSETS_DIR)
class EvaluatorTests(FixtureMixin, unittest.TestCase):
def test_example_strict_statuses(self) -> None:
report = self.evaluate(self.payload)
self.assertEqual([row["symbol"] for row in report["ranked_candidates"]], ["ACME"])
self.assertEqual({row["symbol"] for row in report["review_required"]}, {"CYCLE", "STALE"})
self.assertEqual([row["symbol"] for row in report["screened_out"]], ["SCREEN"])
self.assertEqual({row["symbol"] for row in report["excluded"]}, {"MNA", "OTCX"})
self.assertEqual(report["ranked_candidates"][0]["data_quality_score"], 100)
self.assertEqual(report["ranking_status"], "final")
self.assertTrue(report["contract"]["valid"])
def test_constant_multiple_scenario_uses_forward_metric(self) -> None:
report = self.evaluate(self.single("ACME"))
acme = report["ranked_candidates"][0]
self.assertEqual(acme["valuation"]["current_period_kind"], "fy1")
year_three = acme["valuation"]["constant_multiple"]["year_3"]
stress = acme["valuation"]["multiple_contraction"]["year_3"]
self.assertAlmostEqual(year_three["implied_price"], 66.6667, places=3)
self.assertAlmostEqual(year_three["upside_pct"], 66.6667, places=3)
self.assertAlmostEqual(stress["implied_price"], 53.3333, places=3)
self.assertAlmostEqual(stress["upside_pct"], 33.3333, places=3)
def test_ttm_current_metric_blocks_formal_scenario(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
candidate["identity"]["symbol"] = "TTMX"
candidate["valuation_case"]["periods"]["current"]["period_kind"] = "ttm"
report = self.evaluate(payload)
row = report["review_required"][0]
self.assertFalse(row["valuation"]["formal_forward_scenario_valid"])
self.assertIsNone(row["valuation"]["constant_multiple"]["year_3"])
self.assertTrue(any("NTM or FY1" in reason for reason in row["review_reasons"]))
def test_one_analyst_year3_cannot_be_sole_ranked_horizon(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
candidate["identity"]["symbol"] = "THIN"
candidate["valuation_case"]["periods"]["year_2"]["metric"] = None
candidate["valuation_case"]["periods"]["year_3"]["analyst_count"] = 1
report = self.evaluate(payload)
row = report["review_required"][0]
self.assertFalse(row["valuation"]["rankable_horizons"]["year_3"])
self.assertTrue(any("no rankable" in reason.lower() for reason in row["review_reasons"]))
def test_sum_four_discrete_cash_flow_is_canonical(self) -> None:
report = self.evaluate(self.single("ACME"))
cash = report["ranked_candidates"][0]["financial_metrics"]["cash_flow_ttm"]
self.assertEqual(cash["operating_cash_flow"], 210_000_000)
self.assertEqual(cash["capex_cash_outflow"], 30_000_000)
self.assertEqual(cash["standard_fcf"], 180_000_000)
self.assertEqual(len(cash["used_periods"]), 4)
def test_negative_capex_sign_is_fail_closed(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
candidate["identity"]["symbol"] = "NEGCF"
candidate["financials"]["cash_flow_periods"][0]["capex_cash_outflow"] = -7_000_000
report = self.evaluate(payload)
row = report["review_required"][0]
self.assertTrue(
any("capex_cash_outflow must be non-negative" in warning for warning in row["warnings"])
)
self.assertTrue(
any("missing/invalid OCF or capex" in reason for reason in row["review_reasons"])
)
def test_ytd_reconstruction(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
candidate["identity"]["symbol"] = "YTD"
candidate["financials"]["cash_flow_ttm"].update(
{
"method": "fy_plus_current_ytd_minus_prior_ytd",
"operating_cash_flow": 230_000_000,
"capex_cash_outflow": 36_000_000,
"standard_fcf": 194_000_000,
}
)
candidate["financials"]["ttm_reconstruction"] = {
"latest_fy": {
"period": "FY2025",
"operating_cash_flow": 200_000_000,
"capex_cash_outflow": 30_000_000,
"source_ids": ["acme-10q"],
},
"current_ytd": {
"period": "H1 FY2026",
"operating_cash_flow": 120_000_000,
"capex_cash_outflow": 18_000_000,
"source_ids": ["acme-10q"],
},
"prior_ytd": {
"period": "H1 FY2025",
"operating_cash_flow": 90_000_000,
"capex_cash_outflow": 12_000_000,
"source_ids": ["acme-10q"],
},
}
sec_source = next(source for source in candidate["sources"] if source["id"] == "acme-10q")
for support_path in (
"financials.ttm_reconstruction.latest_fy",
"financials.ttm_reconstruction.current_ytd",
"financials.ttm_reconstruction.prior_ytd",
):
if support_path not in sec_source["supports"]:
sec_source["supports"].append(support_path)
report = self.evaluate(payload)
cash = report["ranked_candidates"][0]["financial_metrics"]["cash_flow_ttm"]
self.assertEqual(cash["operating_cash_flow"], 230_000_000)
self.assertEqual(cash["capex_cash_outflow"], 36_000_000)
self.assertEqual(cash["standard_fcf"], 194_000_000)
def test_company_adjusted_fcf_is_not_used_as_standard(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
candidate["financials"]["cash_flow_ttm"]["company_adjusted_fcf"] = 999_000_000
report = self.evaluate(payload)
metrics = report["ranked_candidates"][0]["financial_metrics"]
self.assertEqual(metrics["standard_fcf"], 180_000_000)
self.assertEqual(metrics["company_adjusted_fcf"], 999_000_000)
def test_ev_to_fcf_uses_corporate_cash_and_marketable_securities(self) -> None:
report = self.evaluate(self.single("ACME"))
metrics = report["ranked_candidates"][0]["financial_metrics"]
self.assertEqual(metrics["cash_definition"], "corporate_cash_plus_marketable_securities")
self.assertAlmostEqual(metrics["enterprise_value"], 4_920_000_000, delta=1)
self.assertAlmostEqual(metrics["ev_to_fcf"], 27.3333, places=3)
def test_cash_inconsistency_without_reconciliation_blocks(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
candidate["identity"]["symbol"] = "CASHX"
candidate["latest_earnings"]["quarter"]["metrics"].update(
{
"cash_and_equivalents": 100_000_000,
"marketable_securities": 20_000_000,
"total_debt": 100_000_000,
}
)
report = self.evaluate(payload)
row = report["review_required"][0]
self.assertTrue(
any("cash_and_equivalents differs" in reason for reason in row["review_reasons"])
)
self.assertLessEqual(row["data_quality_score"], 70)
def test_consistent_adjusted_basis_with_reconciliation_can_rank(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
candidate["identity"]["symbol"] = "ADJBASIS"
for key in ("current", "year_2", "year_3"):
candidate["valuation_case"]["periods"][key]["metric_basis"] = "company_adjusted"
candidate["forecast_bridge"]["periods"]["year_2"]["metric_basis"] = "company_adjusted"
candidate["forecast_bridge"]["periods"]["year_3"]["metric_basis"] = "company_adjusted"
candidate["gaap_reconciliation"] = {
"periods": {
"current": {
"gaap_metric": 2.8,
"adjustments": [
{"label": "Recurring amortization", "amount": 0.2, "recurring": True}
],
"source_ids": ["acme-10q", "analyst-model"],
},
"year_2": {
"gaap_metric": 4.0,
"adjustments": [
{"label": "Recurring amortization", "amount": 0.2, "recurring": True}
],
"source_ids": ["consensus-20260822", "analyst-model"],
},
"year_3": {
"gaap_metric": 4.8,
"adjustments": [
{"label": "Recurring amortization", "amount": 0.2, "recurring": True}
],
"source_ids": ["consensus-20260822", "analyst-model"],
},
}
}
# Driver model must independently produce GAAP EPS and then add the
# same after-tax per-share adjustment used by the reconciliation.
for key, gaap_eps, adjustment_per_share in (("year_2", 4.0, 0.2), ("year_3", 4.8, 0.2)):
period = candidate["valuation_case"]["periods"][key]
bridge = candidate["forecast_bridge"]["periods"][key]
drivers = bridge["drivers"]
shares = float(drivers["diluted_shares"])
tax_rate = float(drivers["tax_rate_pct"]) / 100.0
net_interest_income = float(drivers.get("net_interest_income", 0.0))
gaap_net_income = gaap_eps * shares
pretax_income = gaap_net_income / (1.0 - tax_rate)
operating_income = pretax_income - net_interest_income
drivers["operating_margin_pct"] = operating_income / float(drivers["revenue"]) * 100.0
drivers["after_tax_adjustments"] = adjustment_per_share * shares
bridge.setdefault("driver_provenance", {})["after_tax_adjustments"] = {
"origin": "analyst_assumption",
"source_ids": ["analyst-model"],
"target_solved": False,
}
bridge["metric_numerator"] = float(period["metric"]) * shares
bridge["metric_denominator"] = shares
for peer in candidate["peers"]:
peer["metric_basis"] = "company_adjusted"
candidate["valuation_case"]["peer_median_multiple"]["metric_basis"] = "company_adjusted"
report = self.evaluate(payload)
row = report["ranked_candidates"][0]
self.assertTrue(row["valuation"]["metric_basis_consistent"])
self.assertTrue(row["valuation"]["gaap_reconciliation_valid"])
def test_metric_basis_mismatch_blocks_scenario(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
candidate["identity"]["symbol"] = "MIXED"
candidate["valuation_case"]["periods"]["year_3"]["metric_basis"] = "company_adjusted"
report = self.evaluate(payload)
row = report["review_required"][0]
self.assertFalse(row["valuation"]["metric_basis_consistent"])
self.assertIsNone(row["valuation"]["constant_multiple"]["year_3"])
def test_self_attestation_does_not_create_quality_evidence(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
candidate["identity"]["symbol"] = "CLAIM"
candidate["research_completeness"] = {key: True for key in EVALUATOR.QUALITY_WEIGHTS}
candidate.pop("forecast_bridge")
candidate["evidence"] = {"identity.price": ["quote-20260821"]}
report = self.evaluate(payload)
row = report["review_required"][0]
self.assertFalse(row["data_quality_details"]["forecast_bridge_verified"]["awarded"])
self.assertTrue(
any("forecast bridge" in reason.lower() for reason in row["review_reasons"])
)
def test_commercial_biopharma_concentration_is_derived_and_loe_stressed(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
candidate["identity"]["symbol"] = "BIO"
candidate["sector_profile"] = {
"type": "commercial_biopharma",
"kpis": {
"top_product_revenue": 82_700_000,
"total_revenue": 100_000_000,
"nearest_material_loe_date": "2028-08-22",
"source_ids": ["acme-10q"],
},
}
report = self.evaluate(payload)
row = report["ranked_candidates"][0]
self.assertAlmostEqual(row["sector_profile"]["top_product_revenue_pct"], 82.7, places=1)
self.assertTrue(row["sector_profile"]["concentration_derived"])
self.assertEqual(
[x["multiple"] for x in row["sector_profile"]["loe_stress_scenarios"]], [6.0, 8.0]
)
self.assertGreater(row["penalties"]["sector_specific_risk"], 0)
def test_payments_profile_requires_sourced_cash_separation(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
candidate["identity"]["symbol"] = "PAY"
candidate["sector_profile"] = {
"type": "payments",
"kpis": {"gross_profit_to_tpv_pct": 0.72, "gross_profit_to_tpv_prior_pct": 1.07},
}
report = self.evaluate(payload)
row = report["review_required"][0]
self.assertTrue(any("sourced corporate cash" in reason for reason in row["review_reasons"]))
self.assertTrue(any("take rate declined" in warning for warning in row["warnings"]))
def test_auto_dealer_requires_floorplan_adjustment(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
candidate["identity"]["symbol"] = "AUTO"
candidate["sector_profile"] = {
"type": "auto_dealership",
"kpis": {"source_ids": ["acme-10q"]},
}
report = self.evaluate(payload)
row = report["review_required"][0]
self.assertTrue(any("floorplan debt" in reason for reason in row["review_reasons"]))
def test_completed_mna_is_hard_excluded(self) -> None:
report = self.evaluate(self.single("MNA"))
self.assertTrue(
any(
"M&A status is completed" in reason
for reason in report["excluded"][0]["hard_exclusion_reasons"]
)
)
def test_stale_event_price_is_review_required(self) -> None:
report = self.evaluate(self.single("STALE"))
self.assertTrue(
any(
"predates the latest earnings" in reason
for reason in report["review_required"][0]["review_reasons"]
)
)
def test_cyclical_normalization_is_evidence_based(self) -> None:
report = self.evaluate(self.single("CYCLE"))
row = report["review_required"][0]
self.assertTrue(
any("normalization is required" in reason for reason in row["review_reasons"])
)
def test_minimum_upside_gate(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
candidate["identity"]["symbol"] = "LOWUP"
candidate["valuation_case"]["periods"]["year_2"]["metric"] = 3.1
candidate["valuation_case"]["periods"]["year_3"]["metric"] = 3.3
candidate["forecast_bridge"]["periods"]["year_2"]["metric_numerator"] = 387_500_000
candidate["forecast_bridge"]["periods"]["year_3"]["metric_numerator"] = 405_900_000
report = self.evaluate(payload)
self.assertTrue(
any(
"constant-multiple upside" in reason
for reason in report["review_required"][0]["review_reasons"]
)
)
def test_score_penalties_are_explicit(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
candidate["identity"]["symbol"] = "DILUTE"
candidate["financials"]["sbc_ttm"] = 180_000_000
candidate["financials"]["histories"]["diluted_shares"] = [
{"date": "2023-12-31", "period": "FY2023", "value": 100_000_000},
{"date": "2024-12-31", "period": "FY2024", "value": 112_000_000},
{"date": "2025-12-31", "period": "FY2025", "value": 125_000_000},
]
report = self.evaluate(payload)
candidates = (
report["ranked_candidates"] + report.get("conditional", []) + report["review_required"]
)
result = next(row for row in candidates if row["symbol"] == "DILUTE")
self.assertEqual(result["penalties"]["sbc"], 5)
self.assertEqual(result["penalties"]["dilution"], 8)
self.assertEqual(result["total_penalty"], 13)
def test_completed_no_candidates_run_is_final_without_ranked_names(self) -> None:
payload = self.single("SCREEN")
payload["candidates"] = []
payload["run_metadata"]["status"] = "complete"
report = self.evaluate(payload)
self.assertEqual(report["ranking_status"], "final")
self.assertTrue(report["contract"]["valid"])
self.assertEqual(report["contract"]["selection_outcome"], "no_candidates")
self.assertEqual(report["ranked_candidates"], [])
self.assertEqual(
report["final_three"],
{"highest_conviction": None, "most_undervalued": None, "largest_upside": None},
)
def test_partial_run_is_provisional_and_has_no_final_three(self) -> None:
payload = self.single("ACME")
payload["run_metadata"]["status"] = "partial"
payload["run_metadata"]["unprocessed_candidates"] = ["OTHER"]
report = self.evaluate(payload)
self.assertEqual(report["ranking_status"], "provisional")
self.assertIsNone(report["final_three"]["highest_conviction"])
def test_markdown_contains_full_contract_sections_and_no_placeholder(self) -> None:
report = self.evaluate(self.single("ACME"))
markdown = EVALUATOR.render_markdown(report, language="ja")
for heading in [
"#### 1. 基本情報",
"#### 5. 直近決算",
"#### 7. 同業他社比較",
"#### 9. 倍率据え置きシナリオ",
"#### 13. 投資仮説の無効化条件",
"## H. 最終選定3銘柄",
"## I. 情報源台帳",
"## J. 未解決データと全体警告",
]:
self.assertIn(heading, markdown)
self.assertNotIn("Replace this", markdown)
self.assertNotIn("placeholder", markdown.lower())
def test_cli_writes_json_and_markdown(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
exit_code = EVALUATOR.main(
[
"--input",
str(EXAMPLE_PATH),
"--artifact-root",
str(ASSETS_DIR),
"--output-dir",
temp_dir,
"--language",
"ja",
"--strict",
]
)
self.assertEqual(exit_code, 0)
self.assertEqual(len(list(Path(temp_dir).glob("*.json"))), 1)
self.assertEqual(len(list(Path(temp_dir).glob("*.md"))), 1)
def test_cli_require_final_returns_two_for_partial_run(self) -> None:
payload = self.single("ACME")
payload["run_metadata"]["status"] = "partial"
payload["run_metadata"]["unprocessed_candidates"] = ["OTHER"]
with tempfile.TemporaryDirectory() as temp_dir:
input_path = Path(temp_dir) / "partial.json"
input_path.write_text(json.dumps(payload), encoding="utf-8")
output_dir = Path(temp_dir) / "out"
exit_code = EVALUATOR.main(
[
"--input",
str(input_path),
"--artifact-root",
str(ASSETS_DIR),
"--output-dir",
str(output_dir),
"--strict",
"--require-final",
]
)
self.assertEqual(exit_code, 2)
self.assertEqual(len(list(output_dir.glob("*.json"))), 1)
self.assertEqual(len(list(output_dir.glob("*.md"))), 1)
class ContractTests(FixtureMixin, unittest.TestCase):
def validate(self, payload: dict) -> tuple[dict, dict]:
return CONTRACT.validate_and_normalize_snapshot(payload, artifact_root=ASSETS_DIR)
def test_market_context_placeholder_is_blocked_and_quality_capped(self) -> None:
payload = self.single("ACME")
payload["market_context"]["summary"] = "Replace this placeholder"
normalized, contract = self.validate(payload)
self.assertFalse(contract["market_context_valid"])
caps = normalized["candidates"][0]["_contract"]["quality_caps"]
self.assertTrue(any(cap["cap"] == 70 for cap in caps))
def test_market_context_after_analysis_is_blocked(self) -> None:
payload = self.single("ACME")
payload["market_context"]["as_of"] = "2026-08-23T12:00:00-07:00"
_, contract = self.validate(payload)
self.assertFalse(contract["valid"])
self.assertTrue(
any("after analysis_as_of" in reason for reason in contract["review_reasons"])
)
def test_supports_must_be_array(self) -> None:
payload = self.single("ACME")
payload["candidates"][0]["sources"][0]["supports"] = "latest_earnings;cash"
normalized, _ = self.validate(payload)
row_contract = normalized["candidates"][0]["_contract"]
self.assertFalse(row_contract["source_schema_valid"])
self.assertTrue(
any("supports must be" in reason for reason in row_contract["review_reasons"])
)
def test_source_kind_tier_mismatch_is_blocked(self) -> None:
payload = self.single("ACME")
payload["candidates"][0]["sources"][1]["tier"] = 3
normalized, _ = self.validate(payload)
reasons = normalized["candidates"][0]["_contract"]["review_reasons"]
self.assertTrue(any("incompatible with tier" in reason for reason in reasons))
def test_third_party_transcript_cannot_be_company_ir(self) -> None:
payload = self.single("ACME")
source = payload["candidates"][0]["sources"][1]
source["title"] = "Motley Fool transcript"
source["kind"] = "third_party_transcript"
source["tier"] = 2
normalized, _ = self.validate(payload)
reasons = normalized["candidates"][0]["_contract"]["review_reasons"]
self.assertTrue(any("incompatible with tier" in reason for reason in reasons))
def test_mixed_quarter_and_full_year_period_is_blocked(self) -> None:
payload = self.single("ACME")
payload["candidates"][0]["latest_earnings"]["quarter"]["period"] = "Q4 FY2026 / FY2026"
normalized, _ = self.validate(payload)
reasons = normalized["candidates"][0]["_contract"]["review_reasons"]
self.assertTrue(any("must not combine" in reason for reason in reasons))
def test_separate_quarter_and_full_year_records_are_accepted(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
candidate["latest_earnings"]["full_year"] = {
"period_type": "full_year",
"period": "FY2025",
"period_end": "2025-12-31",
"published_at": "2026-02-15T16:05:00-05:00",
"source_ids": ["acme-10q", "acme-ir"],
"metrics": {"revenue": 880_000_000, "gaap_eps": 2.55},
"guidance": [],
"key_kpis": [],
"one_time_items": [],
}
normalized, _ = self.validate(payload)
contract = normalized["candidates"][0]["_contract"]
self.assertTrue(contract["latest_earnings_valid"])
self.assertIsNotNone(normalized["candidates"][0]["latest_earnings_records"]["quarter"])
self.assertIsNotNone(normalized["candidates"][0]["latest_earnings_records"]["full_year"])
def test_audit_hash_mismatch_blocks(self) -> None:
payload = deepcopy(self.payload)
payload["screening_audit"]["universe"]["artifact_sha256"] = "0" * 64
_, contract = self.validate(payload)
self.assertFalse(contract["screening_audit_valid"])
self.assertTrue(any("artifact_sha256" in reason for reason in contract["review_reasons"]))
def test_valid_audit_passes(self) -> None:
_, contract = self.validate(self.payload)
self.assertTrue(contract["screening_audit_valid"])
self.assertEqual(contract["selected_symbols"], ["ACME", "CYCLE", "STALE"])
class BroadScreenTests(FixtureMixin, unittest.TestCase):
def test_high_growth_exception_is_selected(self) -> None:
row = {
"symbol": "EXLSX",
"country": "US",
"company_name": "High Growth",
"exchange": "NASDAQ",
"is_actively_trading": True,
"is_common_stock": True,
"price": 30,
"market_cap": 5_000_000_000,
"average_daily_dollar_volume": 20_000_000,
"revenue_growth_pct": 18,
"eps_growth_pct": 25,
"standard_fcf": 200_000_000,
"roic_pct": 16,
"forward_pe": 25,
"ev_to_fcf": 24,
"fcf_yield_pct": 4.2,
"dilution_pct": 1,
"net_debt_to_ebitda": 1.0,
}
decisions, _, selected, _ = SCREEN.run(
[row],
dict(SCREEN.DEFAULTS),
analysis_as_of="2026-08-22T14:00:00-07:00",
source_ids=["u"],
)
self.assertEqual(selected, ["EXLSX"])
self.assertTrue(decisions[0]["decision"]["exception_admitted"])
def test_auto_dealer_without_floorplan_adjustment_is_review_required(self) -> None:
row = {
"symbol": "GPIX",
"country": "US",
"company_name": "Dealer",
"exchange": "NYSE",
"is_actively_trading": True,
"is_common_stock": True,
"price": 300,
"market_cap": 8_000_000_000,
"average_daily_dollar_volume": 20_000_000,
"revenue_growth_pct": 10,
"eps_growth_pct": 15,
"standard_fcf": 500_000_000,
"roic_pct": 12,
"forward_pe": 8,
"dilution_pct": -2,
"net_debt_to_ebitda": 7,
"sector_profile_type": "auto_dealership",
}
decisions, _, selected, _ = SCREEN.run(
[row],
dict(SCREEN.DEFAULTS),
analysis_as_of="2026-08-22T14:00:00-07:00",
source_ids=["u"],
)
self.assertEqual(selected, [])
self.assertEqual(decisions[0]["decision"]["status"], "needs_enrichment")
self.assertIn(
"sector_adjusted_leverage_required", decisions[0]["decision"]["review_reasons"]
)
self.assertIsNone(decisions[0]["broad_score"])
def test_all_universe_rows_are_audited_and_budget_defers(self) -> None:
rows = []
for index in range(7):
rows.append(
{
"symbol": f"S{index}",
"country": "US",
"exchange": "NASDAQ",
"is_actively_trading": True,
"is_common_stock": True,
"price": 20,
"market_cap": 1_000_000_000,
"average_daily_dollar_volume": 10_000_000,
"revenue_growth_pct": 12,
"eps_growth_pct": 18 + index,
"standard_fcf": 100_000_000,
"roic_pct": 15,
"forward_pe": 15,
"fcf_yield_pct": 6,
"dilution_pct": 1,
"net_debt_to_ebitda": 1,
}
)
config = dict(SCREEN.DEFAULTS)
config["max_deep_dive_candidates"] = 5
decisions, audit, selected, _ = SCREEN.run(
rows, config, analysis_as_of="2026-08-22T14:00:00-07:00", source_ids=["u"]
)
self.assertEqual(len(decisions), 7)
self.assertEqual(len(selected), 5)
self.assertEqual(audit["candidate_pool"]["decision_counts"]["deferred_by_budget"], 2)
def test_missing_fundamentals_are_not_shortlisted(self) -> None:
rows = [
{
"symbol": "MISSB",
"country": "US",
"exchange": "NASDAQ",
"sector": "Technology",
"is_actively_trading": True,
"is_common_stock": True,
"price": 20,
"market_cap": 3_000_000_000,
"average_daily_dollar_volume": 50_000_000,
},
{
"symbol": "MISSA",
"country": "US",
"exchange": "NYSE",
"sector": "Industrials",
"is_actively_trading": True,
"is_common_stock": True,
"price": 30,
"market_cap": 7_000_000_000,
"average_daily_dollar_volume": 40_000_000,
},
]
_, decisions, audit, selected, _ = SCREEN.run_layered(
rows,
rows,
dict(SCREEN.DEFAULTS),
analysis_as_of="2026-08-22T14:00:00-07:00",
universe_source_ids=["u"],
candidate_source_ids=["e"],
candidate_generation_mode="liquidity_stratified_estimates",
retrieval_min_market_cap=500_000_000,
retrieval_max_market_cap=20_000_000_000,
)
self.assertEqual(selected, [])
self.assertTrue(all(row["decision"]["status"] == "needs_enrichment" for row in decisions))
self.assertTrue(all(row["broad_score"] is None for row in decisions))
self.assertEqual(audit["selection_outcome"], "insufficient_data")
def test_empty_candidate_pool_is_insufficient_data_not_no_candidates(self) -> None:
rows = [
{
"symbol": "ONLY",
"country": "US",
"exchange": "NASDAQ",
"sector": "Technology",
"is_actively_trading": True,
"is_common_stock": True,
"price": 20,
"market_cap": 3_000_000_000,
"average_daily_dollar_volume": 20_000_000,
}
]
_, decisions, audit, selected, _ = SCREEN.run_layered(
rows,
[],
dict(SCREEN.DEFAULTS),
analysis_as_of="2026-08-22T14:00:00-07:00",
universe_source_ids=["u"],
candidate_source_ids=["e"],
candidate_generation_mode="liquidity_stratified_estimates",
retrieval_min_market_cap=500_000_000,
retrieval_max_market_cap=20_000_000_000,
)
self.assertEqual(decisions, [])
self.assertEqual(selected, [])
self.assertEqual(audit["selection_outcome"], "insufficient_data")
def test_evaluable_rows_can_be_selected_while_pool_exhaustion_is_pending(self) -> None:
rows = [
{
"symbol": "EARLY",
"country": "US",
"exchange": "NASDAQ",
"sector": "Technology",
"is_actively_trading": True,
"is_common_stock": True,
"price": 20,
"market_cap": 3_000_000_000,
"average_daily_dollar_volume": 20_000_000,
"revenue_growth_pct": 12,
"eps_growth_pct": 20,
"forward_pe": 15,
"analyst_count": 4,
"enrichment_attempted": True,
}
]
_, decisions, audit, selected, _ = SCREEN.run_layered(
rows,
rows,
dict(SCREEN.DEFAULTS),
analysis_as_of="2026-08-22T14:00:00-07:00",
universe_source_ids=["u"],
candidate_source_ids=["e"],
candidate_generation_mode="liquidity_stratified_estimates",
retrieval_min_market_cap=500_000_000,
retrieval_max_market_cap=20_000_000_000,
)
self.assertEqual(selected, ["EARLY"])
self.assertEqual(decisions[0]["decision"]["status"], "selected")
self.assertIsNotNone(decisions[0]["broad_score"])
self.assertEqual(audit["candidate_pool_status"], "sufficient_pending_enrichment")
self.assertEqual(audit["selection_outcome"], "selected_pending_enrichment")
self.assertEqual(audit["selected_symbols"], ["EARLY"])
self.assertFalse(audit["enrichment"]["candidate_pool_exhausted"])
def test_genuine_no_candidates_requires_assessable_rows(self) -> None:
rows = [
{
"symbol": "FAIL",
"country": "US",
"exchange": "NASDAQ",
"sector": "Technology",
"is_actively_trading": True,
"is_common_stock": True,
"price": 20,
"market_cap": 3_000_000_000,
"average_daily_dollar_volume": 20_000_000,
"revenue_growth_pct": 1,
"eps_growth_pct": 2,
"standard_fcf": 100_000_000,
"roic_pct": 5,
"forward_pe": 35,
"fcf_yield_pct": 3,
"dilution_pct": 1,
"net_debt_to_ebitda": 1,
}
]
_, decisions, audit, selected, _ = SCREEN.run_layered(
rows,
rows,
dict(SCREEN.DEFAULTS),
analysis_as_of="2026-08-22T14:00:00-07:00",
universe_source_ids=["u"],
candidate_source_ids=["u"],
candidate_generation_mode="full_universe_fundamentals",
retrieval_min_market_cap=500_000_000,
retrieval_max_market_cap=20_000_000_000,
candidate_pool_exhausted=True,
provider_reported_total=1,
pages_fetched=1,
pagination_exhausted=True,
)
self.assertEqual(decisions[0]["decision"]["status"], "screened_out")
self.assertEqual(selected, [])
self.assertEqual(audit["selection_outcome"], "no_candidates")
self.assertTrue(audit["enrichment"]["all_rows_resolved"])
def test_review_only_rows_are_never_alphabetically_selected(self) -> None:
rows = []
for symbol in ("AGNC", "AHR", "ALLY", "AMH"):
rows.append(
{
"symbol": symbol,
"country": "US",
"exchange": "NASDAQ",
"sector": "Financial Services",
"sector_profile_type": "reit",
"is_actively_trading": True,
"is_common_stock": True,
"price": 20,
"market_cap": 12_000_000_000,
"average_daily_dollar_volume": 20_000_000,
}
)
_, decisions, _, selected, _ = SCREEN.run_layered(
rows,
rows,
dict(SCREEN.DEFAULTS),
analysis_as_of="2026-08-22T14:00:00-07:00",
universe_source_ids=["u"],
candidate_source_ids=["e"],
candidate_generation_mode="liquidity_stratified_estimates",
retrieval_min_market_cap=500_000_000,
retrieval_max_market_cap=20_000_000_000,
)
self.assertEqual(selected, [])
self.assertTrue(all(row["decision"]["status"] == "needs_enrichment" for row in decisions))
self.assertTrue(all(row["broad_score"] is None for row in decisions))
def test_reduced_scope_is_disclosed_and_not_completion_ready(self) -> None:
row = {
"symbol": "READY",
"country": "US",
"exchange": "NASDAQ",
"is_actively_trading": True,
"is_common_stock": True,
"price": 20,
"market_cap": 12_000_000_000,
"average_daily_dollar_volume": 20_000_000,
"revenue_growth_pct": 12,
"eps_growth_pct": 18,
"standard_fcf": 100_000_000,
"roic_pct": 15,
"forward_pe": 15,
"fcf_yield_pct": 6,
"dilution_pct": 1,
"net_debt_to_ebitda": 1,
}
_, _, audit, _, _ = SCREEN.run_layered(
[row],
[row],
dict(SCREEN.DEFAULTS),
analysis_as_of="2026-08-22T14:00:00-07:00",
universe_source_ids=["u"],
candidate_source_ids=["u"],
candidate_generation_mode="full_universe_fundamentals",
retrieval_min_market_cap=10_000_000_000,
retrieval_max_market_cap=20_000_000_000,
)
self.assertFalse(audit["scope"]["scope_complete"])
def test_discovery_pool_is_stratified_and_deterministic(self) -> None:
rows = []
for sector in ("Technology", "Healthcare", "Industrials"):
for index, cap in enumerate((700_000_000, 3_000_000_000, 12_000_000_000)):
rows.append(
{
"symbol": f"{sector[0]}{index}",
"country": "US",
"exchange": "NASDAQ",
"sector": sector,
"is_actively_trading": True,
"is_common_stock": True,
"price": 20,
"market_cap": cap,
"average_daily_dollar_volume": 10_000_000 + index,
}
)
first, audit1 = DISCOVERY.build_pool(
rows,
min_market_cap=500_000_000,
max_market_cap=20_000_000_000,
min_price=5,
hard_min_adv=1_000_000,
max_pool=6,
per_cell=1,
)
second, audit2 = DISCOVERY.build_pool(
list(reversed(rows)),
min_market_cap=500_000_000,
max_market_cap=20_000_000_000,
min_price=5,
hard_min_adv=1_000_000,
max_pool=6,
per_cell=1,
)
self.assertEqual([r["symbol"] for r in first], [r["symbol"] for r in second])
self.assertGreaterEqual(len({r["sector"] for r in first}), 2)
self.assertEqual(audit1["selection_method"], audit2["selection_method"])
def test_discovery_partial_layer_uses_liquidity_not_sector_alphabetical_order(self) -> None:
rows = [
{
"symbol": "ALOWV",
"country": "US",
"exchange": "NASDAQ",
"sector": "Aerospace",
"is_actively_trading": True,
"is_common_stock": True,
"price": 20,
"market_cap": 3_000_000_000,
"average_daily_dollar_volume": 2_000_000,
},
{
"symbol": "BMID",
"country": "US",
"exchange": "NASDAQ",
"sector": "Banks",
"is_actively_trading": True,
"is_common_stock": True,
"price": 20,
"market_cap": 3_000_000_000,
"average_daily_dollar_volume": 20_000_000,
},
{
"symbol": "ZHIGH",
"country": "US",
"exchange": "NASDAQ",
"sector": "Zoology",
"is_actively_trading": True,
"is_common_stock": True,
"price": 20,
"market_cap": 3_000_000_000,
"average_daily_dollar_volume": 50_000_000,
},
]
pool, _ = DISCOVERY.build_pool(
rows,
min_market_cap=500_000_000,
max_market_cap=20_000_000_000,
min_price=5,
hard_min_adv=1_000_000,
max_pool=2,
per_cell=1,
)
self.assertEqual([row["symbol"] for row in pool], ["ZHIGH", "BMID"])
def test_cli_refuses_listing_only_input_without_candidate_pool(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
listing = root / "listing.jsonl"
listing.write_text(
json.dumps(
{
"symbol": "ONLY",
"country": "US",
"exchange": "NASDAQ",
"is_actively_trading": True,
"is_common_stock": True,
"price": 20,
"market_cap": 2_000_000_000,
"average_daily_dollar_volume": 10_000_000,
}
)
+ "\n",
encoding="utf-8",
)
code = SCREEN.main(
[
"--input",
str(listing),
"--output-dir",
str(root / "out"),
"--analysis-as-of",
"2026-08-22T14:00:00-07:00",
"--source-id",
"listing-source",
"--retrieval-min-market-cap",
"500000000",
"--retrieval-max-market-cap",
"20000000000",
"--provider-reported-total",
"1",
"--pages-fetched",
"1",
"--pagination-exhausted",
]
)
self.assertEqual(code, 2)
audit = json.loads(
(root / "out" / "broad-screen-audit.json").read_text(encoding="utf-8")
)
self.assertEqual(audit["enrichment"]["next_action"], "build_discovery_pool")
self.assertTrue(audit["enrichment"]["discovery_pool_required"])
def test_cli_refuses_implicit_retrieval_scope_even_with_evaluable_pool(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
listing = root / "listing.jsonl"
listing.write_text(
json.dumps(
{
"symbol": "READY",
"country": "US",
"exchange": "NASDAQ",
"is_actively_trading": True,
"is_common_stock": True,
"price": 20,
"market_cap": 2_000_000_000,
"average_daily_dollar_volume": 10_000_000,
"revenue_growth_pct": 12,
"eps_growth_pct": 18,
"standard_fcf": 100_000_000,
"roic_pct": 15,
"forward_pe": 15,
"fcf_yield_pct": 6,
"dilution_pct": 1,
"net_debt_to_ebitda": 1,
}
)
+ "\n",
encoding="utf-8",
)
code = SCREEN.main(
[
"--input",
str(listing),
"--output-dir",
str(root / "out"),
"--analysis-as-of",
"2026-08-22T14:00:00-07:00",
"--source-id",
"listing-source",
]
)
self.assertEqual(code, 2)
def test_insufficient_data_outcome_is_not_completion_ready(self) -> None:
payload = self.single("ACME")
row = payload["screening_audit"]["candidate_pool"]["embedded_rows"][0]
row["decision"]["status"] = "needs_enrichment"
row["decision"]["resolution"] = "unresolved"
row["decision"]["selection_eligible"] = False
row["broad_score"] = None
row["discovery_evaluable"] = False
row["fundamental_complete"] = False
row["enrichment_attempted"] = False
row["enrichment_resolved"] = False
row["selection_eligible"] = False
audit = payload["screening_audit"]
pool = audit["candidate_pool"]
audit["selected_symbols"] = []
audit["deep_dive_plan"] = deep_dive_plan([], payload["analysis_as_of"], maximum=3)
audit["candidate_pool_status"] = "insufficient_data"
audit["selection_outcome"] = "insufficient_data"
pool["decision_counts"] = {"needs_enrichment": 1}
pool["discovery_evaluable_count"] = 0
pool["discovery_evaluable_pct"] = 0.0
pool["fundamental_complete_count"] = 0
pool["fundamental_complete_pct"] = 0.0
pool["selection_eligible_count"] = 0
pool["selected_count"] = 0
pool["artifact_sha256"] = hashlib.sha256(canonical_jsonl([row])).hexdigest()
audit["enrichment"] = {
"status": "pending",
"next_action": "continue_enrichment",
"discovery_pool_required": False,
"attempted_count": 0,
"resolved_count": 0,
"unresolved_count": 1,
"resolution_pct": 0.0,
"all_rows_resolved": False,
"queue_count": 1,
"queue_symbols": ["ACME"],
"maximum_attempts": 60,
"candidate_pool_exhaustion_declared": False,
"candidate_pool_exhausted": False,
"candidate_pool_covers_in_scope": True,
"candidate_pool_scope_verified": True,
"conclusion_scope": "full_listing_universe",
}
payload["screening_funnel"]["discovery_evaluable_count"] = 0
payload["screening_funnel"]["deep_dive_selected_count"] = 0
payload["screening_funnel"]["preflight_passed_count"] = 0
payload["screening_funnel"]["deep_dive_completed_count"] = 0
payload["run_metadata"]["status"] = "partial"
payload["run_metadata"]["selected_symbols"] = []
payload["run_metadata"]["unprocessed_candidates"] = []
payload["candidates"] = []
_, contract = CONTRACT.validate_and_normalize_snapshot(payload, artifact_root=ASSETS_DIR)
self.assertTrue(contract["screening_audit_valid"])
self.assertFalse(contract["screening_completion_ready"])
self.assertEqual(contract["selection_outcome"], "insufficient_data")
class RegressionV33Tests(FixtureMixin, unittest.TestCase):
def _run_layered(
self,
universe_rows: list[dict],
candidate_rows: list[dict],
*,
exhausted: bool = False,
mode: str = "liquidity_stratified_estimates",
):
return SCREEN.run_layered(
universe_rows,
candidate_rows,
dict(SCREEN.DEFAULTS),
analysis_as_of="2026-08-22T14:00:00-07:00",
universe_source_ids=["listing-source"],
candidate_source_ids=["estimate-source"],
candidate_generation_mode=mode,
retrieval_min_market_cap=500_000_000,
retrieval_max_market_cap=20_000_000_000,
candidate_pool_exhausted=exhausted,
provider_reported_total=len(universe_rows),
pages_fetched=1,
pagination_exhausted=True,
)
def test_twenty_one_attempts_and_six_evaluable_do_not_override_unresolved_queue(self) -> None:
rows: list[dict] = []
for index in range(6):
rows.append(
{
"symbol": f"FAIL{index}",
"country": "US",
"exchange": "NASDAQ",
"sector": "Technology",
"is_actively_trading": True,
"is_common_stock": True,
"price": 20,
"market_cap": 3_000_000_000,
"average_daily_dollar_volume": 20_000_000,
"revenue_growth_pct": 1,
"eps_growth_pct": 2,
"forward_pe": 80,
"analyst_count": 5,
"enrichment_attempted": True,
}
)
for index in range(15):
rows.append(
{
"symbol": f"WAIT{index}",
"country": "US",
"exchange": "NYSE",
"sector": "Industrials",
"is_actively_trading": True,
"is_common_stock": True,
"price": 25,
"market_cap": 4_000_000_000,
"average_daily_dollar_volume": 15_000_000,
"revenue_growth_pct": 12,
"analyst_count": 4,
"enrichment_attempted": True,
}
)
_, decisions, audit, selected, queue = self._run_layered(rows, rows, exhausted=False)
self.assertEqual(audit["enrichment"]["attempted_count"], 21)
self.assertEqual(audit["candidate_pool"]["discovery_evaluable_count"], 6)
self.assertEqual(audit["enrichment"]["unresolved_count"], 15)
self.assertEqual(len(queue), 15)
self.assertEqual(selected, [])
self.assertEqual(audit["candidate_pool_status"], "insufficient_data")
self.assertEqual(audit["selection_outcome"], "insufficient_data")
self.assertFalse(audit["enrichment"]["all_rows_resolved"])
self.assertFalse(audit["enrichment"]["candidate_pool_exhausted"])
self.assertTrue(any(row["decision"]["status"] == "needs_enrichment" for row in decisions))
def test_high_growth_cyclicals_advance_to_mid_cycle_deep_dive(self) -> None:
rows = [
{
"symbol": "MODX",
"country": "US",
"exchange": "NYSE",
"sector": "Industrials",
"is_actively_trading": True,
"is_common_stock": True,
"price": 120,
"market_cap": 6_000_000_000,
"average_daily_dollar_volume": 30_000_000,
"revenue_growth_pct": 24.38,
"eps_growth_pct": 46.28,
"forward_pe": 25.86,
"analyst_count": 6,
"cyclicality_score": 4,
},
{
"symbol": "MKSX",
"country": "US",
"exchange": "NASDAQ",
"sector": "Technology",
"is_actively_trading": True,
"is_common_stock": True,
"price": 150,
"market_cap": 10_000_000_000,
"average_daily_dollar_volume": 25_000_000,
"revenue_growth_pct": 20.08,
"eps_growth_pct": 23.17,
"forward_pe": 21.25,
"analyst_count": 9,
"cyclicality_score": 4,
},
]
_, decisions, audit, selected, _ = self._run_layered(rows, rows, exhausted=False)
self.assertEqual(set(selected), {"MODX", "MKSX"})
self.assertEqual(audit["candidate_pool_status"], "sufficient_pending_enrichment")
for row in decisions:
self.assertEqual(row["decision"]["preselection_status"], "sector_review_required")
self.assertIn(
"mid_cycle_normalization_required", row["decision"]["deep_dive_requirements"]
)
self.assertTrue(row["selection_eligible"])
self.assertIsNotNone(row["broad_score"])
def test_growth_guidelines_are_not_mechanical_cutoffs(self) -> None:
row = {
"symbol": "BFAMX",
"country": "US",
"exchange": "NYSE",
"sector": "Consumer Defensive",
"is_actively_trading": True,
"is_common_stock": True,
"price": 100,
"market_cap": 8_000_000_000,
"average_daily_dollar_volume": 20_000_000,
"revenue_growth_pct": 5.5,
"eps_growth_pct": 15.92,
"forward_pe": 14.26,
"analyst_count": 5,
"cyclicality_score": 2,
}
_, decisions, _, selected, _ = self._run_layered([row], [row], exhausted=False)
self.assertEqual(selected, ["BFAMX"])
self.assertEqual(decisions[0]["decision"]["status"], "selected")
self.assertIn(
"revenue_growth_below_guideline", decisions[0]["decision"]["guideline_misses"]
)
self.assertEqual(decisions[0]["decision"]["screen_fail_reasons"], [])
def test_enrichment_queue_prioritizes_garp_information_value_over_liquidity(self) -> None:
rows = [
{
"symbol": "GARP",
"country": "US",
"exchange": "NASDAQ",
"sector": "Technology",
"is_actively_trading": True,
"is_common_stock": True,
"price": 50,
"market_cap": 5_000_000_000,
"average_daily_dollar_volume": 5_000_000,
"revenue_growth_pct": 18,
"eps_growth_pct": 25,
"forward_pe": 22,
"analyst_count": 1,
"enrichment_attempted": True,
},
{
"symbol": "BASE",
"country": "US",
"exchange": "NYSE",
"sector": "Materials",
"is_actively_trading": True,
"is_common_stock": True,
"price": 20,
"market_cap": 5_000_000_000,
"average_daily_dollar_volume": 200_000_000,
"revenue_growth_pct": 100,
"eps_growth_pct": 800,
"forward_pe": 3474,
"analyst_count": 1,
"enrichment_attempted": True,
},
]
_, _, audit, _, queue = self._run_layered(rows, rows, exhausted=False)
self.assertEqual([row["symbol"] for row in queue], ["GARP", "BASE"])
self.assertEqual(audit["enrichment"]["next_action"], "enrich_queue")
def test_full_cap_range_without_pagination_evidence_is_not_scope_complete(self) -> None:
row = {
"symbol": "READY",
"country": "US",
"exchange": "NASDAQ",
"is_actively_trading": True,
"is_common_stock": True,
"price": 20,
"market_cap": 2_000_000_000,
"average_daily_dollar_volume": 10_000_000,
"revenue_growth_pct": 12,
"eps_growth_pct": 18,
"forward_pe": 15,
"analyst_count": 4,
}
_, _, audit, _, _ = SCREEN.run_layered(
[row],
[row],
dict(SCREEN.DEFAULTS),
analysis_as_of="2026-08-22T14:00:00-07:00",
universe_source_ids=["u"],
candidate_source_ids=["e"],
candidate_generation_mode="liquidity_stratified_estimates",
retrieval_min_market_cap=500_000_000,
retrieval_max_market_cap=20_000_000_000,
)
self.assertFalse(audit["scope"]["scope_complete"])
self.assertIn("executed_scope_enumeration_not_verified", audit["scope"]["reasons"])
def test_unavailable_after_enrichment_requires_reason_and_sources(self) -> None:
base = {
"symbol": "MISSING",
"country": "US",
"exchange": "NASDAQ",
"sector": "Technology",
"is_actively_trading": True,
"is_common_stock": True,
"price": 20,
"market_cap": 2_000_000_000,
"average_daily_dollar_volume": 10_000_000,
"revenue_growth_pct": 12,
"analyst_count": 4,
"enrichment_attempted": True,
"enrichment_exhausted": True,
"enrichment_exhaustion_reason": "forward EPS was not published by available providers",
}
decisions, _, _, _ = SCREEN.run(
[base],
dict(SCREEN.DEFAULTS),
analysis_as_of="2026-08-22T14:00:00-07:00",
source_ids=["e"],
)
self.assertEqual(decisions[0]["decision"]["status"], "needs_enrichment")
evidenced = dict(base)
evidenced["enrichment_source_ids"] = ["e"]
decisions, _, _, _ = SCREEN.run(
[evidenced],
dict(SCREEN.DEFAULTS),
analysis_as_of="2026-08-22T14:00:00-07:00",
source_ids=["e"],
)
self.assertEqual(decisions[0]["decision"]["status"], "unavailable_after_enrichment")
self.assertEqual(decisions[0]["decision"]["resolution"], "resolved")
def test_stale_market_valuation_source_blocks_market_context(self) -> None:
payload = self.single("ACME")
for source in payload["global_sources"]:
if source["id"] == "market-valuation-20260822":
source["published_at"] = "2026-05-28T00:00:00-04:00"
source["data_as_of"] = "2026-05-28T00:00:00-04:00"
_, contract = CONTRACT.validate_and_normalize_snapshot(payload, artifact_root=ASSETS_DIR)
self.assertFalse(contract["market_context_valid"])
self.assertTrue(
any(
"market_forward_pe lacks a source within" in reason
for reason in contract["review_reasons"]
)
)
def test_tampered_no_candidates_with_unresolved_queue_is_rejected(self) -> None:
payload = self.single("SCREEN")
payload["candidates"] = []
audit = payload["screening_audit"]
row = audit["candidate_pool"]["embedded_rows"][0]
row["decision"]["status"] = "needs_enrichment"
row["decision"]["resolution"] = "unresolved"
row["decision"]["selection_eligible"] = False
row["enrichment_resolved"] = False
row["selection_eligible"] = False
row["discovery_evaluable"] = False
row["broad_score"] = None
audit["candidate_pool"]["decision_counts"] = {"needs_enrichment": 1}
audit["candidate_pool"]["discovery_evaluable_count"] = 0
audit["candidate_pool"]["discovery_evaluable_pct"] = 0.0
audit["candidate_pool"]["selection_eligible_count"] = 0
audit["candidate_pool"]["artifact_sha256"] = hashlib.sha256(
canonical_jsonl([row])
).hexdigest()
audit["candidate_pool_status"] = "no_qualifying_candidates"
audit["selection_outcome"] = "no_candidates"
audit["enrichment"].update(
{
"status": "pending",
"next_action": "enrich_queue",
"resolved_count": 0,
"unresolved_count": 1,
"resolution_pct": 0.0,
"all_rows_resolved": False,
"candidate_pool_exhausted": False,
"queue_count": 1,
"queue_symbols": ["SCREEN"],
}
)
payload["run_metadata"]["status"] = "complete"
_, contract = CONTRACT.validate_and_normalize_snapshot(payload, artifact_root=ASSETS_DIR)
self.assertFalse(contract["screening_audit_valid"])
self.assertFalse(contract["screening_completion_ready"])
self.assertTrue(
any(
"no_qualifying_candidates" in reason or "completion" in reason
for reason in contract["review_reasons"]
)
)
def test_runtime_fingerprint_rejects_stale_cached_skill(self) -> None:
payload = self.single("ACME")
payload["runtime"]["contract_revision"] = "3.2"
with self.assertRaises(CONTRACT.ContractError):
CONTRACT.validate_and_normalize_snapshot(payload, artifact_root=ASSETS_DIR)
def test_reverse_engineered_numerator_does_not_replace_driver_bridge(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
# Keep metric_numerator / shares tied to consensus, but break the
# independent operating-driver model. This was the v3.2 circular bug.
candidate["forecast_bridge"]["periods"]["year_2"]["drivers"]["operating_margin_pct"] = 25.0
report = self.evaluate(payload)
row = report["review_required"][0]
self.assertFalse(row["valuation"]["forecast_bridge_valid"])
self.assertLessEqual(row["data_quality_score"], 60)
self.assertTrue(
any("driver-derived forecast" in reason for reason in row["review_reasons"])
)
def test_adjusted_bridge_must_reconcile_gaap_eps_and_adjustments(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
for key in ("current", "year_2", "year_3"):
candidate["valuation_case"]["periods"][key]["metric_basis"] = "company_adjusted"
for key, gaap_eps in (("year_2", 4.0), ("year_3", 4.8)):
bridge = candidate["forecast_bridge"]["periods"][key]
bridge["metric_basis"] = "company_adjusted"
drivers = bridge["drivers"]
shares = float(drivers["diluted_shares"])
tax_rate = float(drivers["tax_rate_pct"]) / 100.0
net_interest_income = float(drivers.get("net_interest_income", 0.0))
gaap_net_income = gaap_eps * shares
pretax_income = gaap_net_income / (1.0 - tax_rate)
drivers["operating_margin_pct"] = (
(pretax_income - net_interest_income) / float(drivers["revenue"]) * 100.0
)
# Deliberately wrong: reconciliation requires $0.20/share.
drivers["after_tax_adjustments"] = 0.05 * shares
candidate["gaap_reconciliation"] = {
"periods": {
"current": {
"gaap_metric": 2.8,
"adjustments": [{"label": "Amortization", "amount": 0.2, "recurring": True}],
"source_ids": ["acme-10q", "analyst-model"],
},
"year_2": {
"gaap_metric": 4.0,
"adjustments": [{"label": "Amortization", "amount": 0.2, "recurring": True}],
"source_ids": ["consensus-20260822", "analyst-model"],
},
"year_3": {
"gaap_metric": 4.8,
"adjustments": [{"label": "Amortization", "amount": 0.2, "recurring": True}],
"source_ids": ["consensus-20260822", "analyst-model"],
},
}
}
for peer in candidate["peers"]:
peer["metric_basis"] = "company_adjusted"
candidate["valuation_case"]["peer_median_multiple"]["metric_basis"] = "company_adjusted"
report = self.evaluate(payload)
row = report["review_required"][0]
self.assertFalse(row["valuation"]["forecast_bridge_valid"])
self.assertTrue(
any("adjusted forecast bridge" in reason for reason in row["review_reasons"])
)
def test_ordinary_cash_is_normalized_from_cash_and_equivalents(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
cash = candidate["financials"]["cash_classification"]
cash["cash_and_equivalents"] = cash.pop("corporate_cash")
report = self.evaluate(payload)
row = report["ranked_candidates"][0]
self.assertEqual(row["financial_metrics"]["corporate_cash"], 160_000_000)
self.assertEqual(
row["financial_metrics"]["cash_classification_method"],
"derived_from_reported_cash_and_equivalents",
)
def test_peak_profit_risk_forces_normalization_even_with_low_cyclicality_score(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
candidate["identity"]["symbol"] = "PEAKX"
candidate["cyclicality"].update(
{
"score": 2,
"position": "late_cycle",
"peak_profit_risk": True,
"normalization_required": False,
"normalization": None,
}
)
report = self.evaluate(payload)
row = report["review_required"][0]
self.assertTrue(row["cyclicality"]["normalization_required"])
self.assertFalse(row["cyclicality"]["normalization_valid"])
self.assertLessEqual(row["data_quality_score"], 65)
def test_biopharma_alias_triggers_loe_stress(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
candidate["identity"]["symbol"] = "BIOALIAS"
candidate["sector_profile"] = {
"type": "biopharma",
"kpis": {
"top_product_revenue": 800_000_000,
"total_revenue": 1_000_000_000,
"nearest_material_loe_date": "2029-12-31",
"source_ids": ["acme-10q", "acme-ir"],
},
}
report = self.evaluate(payload)
row = report["ranked_candidates"][0]
self.assertEqual(row["sector_profile"]["type"], "commercial_biopharma")
self.assertTrue(row["sector_profile"]["type_normalized"])
self.assertEqual(row["sector_profile"]["top_product_revenue_pct"], 80.0)
self.assertEqual(
{item["multiple"] for item in row["sector_profile"]["loe_stress_scenarios"]}, {6.0, 8.0}
)
def test_missing_roic_and_ebitda_evidence_caps_quality(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
candidate["evidence"].pop("financials.roic_pct", None)
candidate["evidence"].pop("financials.ebitda_ttm", None)
for source in candidate["sources"]:
source["supports"] = [
field
for field in source.get("supports", [])
if field not in {"financials.roic_pct", "financials.ebitda_ttm"}
]
report = self.evaluate(payload)
row = report["review_required"][0]
self.assertLessEqual(row["data_quality_score"], 65)
self.assertFalse(
row["data_quality_details"]["roic_ebitda_evidence_verified"]["evidence_ok"]
)
def test_ttm_cash_flow_periods_require_source_ids(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
candidate["financials"]["cash_flow_periods"][0]["source_ids"] = []
report = self.evaluate(payload)
row = report["review_required"][0]
self.assertLessEqual(row["data_quality_score"], 65)
self.assertTrue(any("cash-flow period" in reason for reason in row["review_reasons"]))
def test_valid_bounded_pool_can_publish_final_ranking_scope(self) -> None:
universe = [
{
"symbol": "ACME",
"country": "US",
"exchange": "NASDAQ",
"sector": "Technology",
"is_actively_trading": True,
"is_common_stock": True,
"price": 40,
"market_cap": 5_000_000_000,
"average_daily_dollar_volume": 45_000_000,
},
{
"symbol": "OTHER",
"country": "US",
"exchange": "NYSE",
"sector": "Industrials",
"is_actively_trading": True,
"is_common_stock": True,
"price": 30,
"market_cap": 3_000_000_000,
"average_daily_dollar_volume": 20_000_000,
},
]
pool = [
{
**universe[0],
"revenue_growth_pct": 14,
"eps_growth_pct": 22,
"forward_pe": 13.33,
"analyst_count": 8,
}
]
discovery_audit = {
"selection_method": "sector_market_cap_stratified_liquidity",
"input_row_count": 2,
"selected_count": 1,
"selected_symbols": ["ACME"],
"source_ids": ["listing-source"],
"artifact_path": "discovery-pool.jsonl",
"artifact_sha256": "abc123",
}
_, _, audit, selected, queue = SCREEN.run_layered(
universe,
pool,
dict(SCREEN.DEFAULTS),
analysis_as_of="2026-08-22T14:00:00-07:00",
universe_source_ids=["listing-source"],
candidate_source_ids=["estimate-source"],
candidate_generation_mode="liquidity_stratified_estimates",
retrieval_min_market_cap=500_000_000,
retrieval_max_market_cap=20_000_000_000,
candidate_pool_exhausted=True,
provider_reported_total=2,
pages_fetched=1,
pagination_exhausted=True,
discovery_audit=discovery_audit,
)
self.assertEqual(selected, ["ACME"])
self.assertEqual(queue, [])
self.assertEqual(audit["candidate_pool_status"], "sufficient")
self.assertEqual(audit["candidate_pool"]["coverage_scope"], "stratified_discovery_pool")
self.assertFalse(audit["candidate_pool"]["listing_coverage_complete"])
self.assertTrue(audit["candidate_pool"]["coverage_complete"])
self.assertTrue(audit["enrichment"]["candidate_pool_exhausted"])
def test_bounded_no_candidates_conclusion_is_scoped(self) -> None:
universe = [
{
"symbol": "FAIL",
"country": "US",
"exchange": "NASDAQ",
"sector": "Technology",
"is_actively_trading": True,
"is_common_stock": True,
"price": 20,
"market_cap": 2_000_000_000,
"average_daily_dollar_volume": 10_000_000,
},
{
"symbol": "OTHER",
"country": "US",
"exchange": "NYSE",
"sector": "Industrials",
"is_actively_trading": True,
"is_common_stock": True,
"price": 30,
"market_cap": 3_000_000_000,
"average_daily_dollar_volume": 20_000_000,
},
]
pool = [
{
**universe[0],
"revenue_growth_pct": -5,
"eps_growth_pct": -10,
"forward_pe": 80,
"analyst_count": 5,
}
]
discovery_audit = {
"selection_method": "sector_market_cap_stratified_liquidity",
"input_row_count": 2,
"selected_count": 1,
"selected_symbols": ["FAIL"],
"source_ids": ["listing-source"],
"artifact_path": "discovery-pool.jsonl",
"artifact_sha256": "abc123",
}
_, _, audit, selected, queue = SCREEN.run_layered(
universe,
pool,
dict(SCREEN.DEFAULTS),
analysis_as_of="2026-08-22T14:00:00-07:00",
universe_source_ids=["listing-source"],
candidate_source_ids=["estimate-source"],
candidate_generation_mode="liquidity_stratified_estimates",
retrieval_min_market_cap=500_000_000,
retrieval_max_market_cap=20_000_000_000,
candidate_pool_exhausted=True,
provider_reported_total=2,
pages_fetched=1,
pagination_exhausted=True,
discovery_audit=discovery_audit,
)
self.assertEqual(selected, [])
self.assertEqual(queue, [])
self.assertEqual(audit["candidate_pool_status"], "no_qualifying_candidates_in_bounded_pool")
self.assertEqual(audit["selection_outcome"], "no_candidates_in_bounded_pool")
self.assertEqual(audit["conclusion_scope"], "stratified_discovery_pool")
def test_runtime_version_flags_are_consistent_across_scripts(self) -> None:
expected = VERSION.runtime_metadata()
for module in (
SCREEN,
PROVIDER_POOL,
DISCOVERY,
NORMALIZER,
EVALUATOR,
RUN_STATE,
CONTRACT,
PREPUBLISH,
BUNDLER,
):
self.assertEqual(module.runtime_metadata(), expected)
def test_markdown_surfaces_broad_stage_review_and_screened_out_rows(self) -> None:
report = self.evaluate(self.payload)
report["ranking_status"] = "provisional"
report["broad_screen"] = {
"counts": {
"selected": 0,
"deferred_by_budget": 0,
"review_required": 1,
"screened_out": 1,
"excluded": 0,
"unavailable_after_enrichment": 0,
},
"selected": [],
"deferred_by_budget": [],
"review_required": [
{
"symbol": "MODX",
"country": "US",
"company_name": "Modine-like Cyclical",
"status": "sector_review_required",
"review_reasons": ["mid_cycle_normalization_required"],
"deep_dive_requirements": ["mid_cycle_normalization_required"],
"metrics": {"forward_pe": 25.86, "per_share_growth_pct": 46.28},
}
],
"screened_out": [
{
"symbol": "FAILX",
"country": "US",
"company_name": "Extreme Valuation",
"status": "screened_out",
"screen_fail_reasons": ["extreme_forward_valuation"],
"metrics": {"forward_pe": 80.0, "per_share_growth_pct": 2.0},
}
],
"excluded": [],
"unavailable_after_enrichment": [],
}
markdown = EVALUATOR.render_markdown(report, language="ja")
self.assertIn("MODX", markdown)
self.assertIn("mid_cycle_normalization_required", markdown)
self.assertIn("FAILX", markdown)
self.assertIn("extreme_forward_valuation", markdown)
def test_no_candidates_requires_candidate_pool_coverage_of_all_in_scope_symbols(self) -> None:
universe = [
{
"symbol": "FAILA",
"country": "US",
"exchange": "NASDAQ",
"sector": "Technology",
"is_actively_trading": True,
"is_common_stock": True,
"price": 20,
"market_cap": 2_000_000_000,
"average_daily_dollar_volume": 10_000_000,
},
{
"symbol": "UNSEEN",
"country": "US",
"exchange": "NYSE",
"sector": "Industrials",
"is_actively_trading": True,
"is_common_stock": True,
"price": 25,
"market_cap": 3_000_000_000,
"average_daily_dollar_volume": 12_000_000,
},
]
pool = [
{
**universe[0],
"revenue_growth_pct": 1,
"eps_growth_pct": 2,
"forward_pe": 80,
"analyst_count": 5,
}
]
_, _, audit, selected, _ = self._run_layered(universe, pool, exhausted=True)
self.assertEqual(selected, [])
self.assertTrue(audit["candidate_pool"]["coverage_complete"])
self.assertFalse(audit["candidate_pool"]["listing_coverage_complete"])
self.assertEqual(audit["candidate_pool"]["in_scope_missing_symbols"], ["UNSEEN"])
self.assertEqual(audit["candidate_pool"]["coverage_scope"], "stratified_discovery_pool")
self.assertTrue(audit["enrichment"]["candidate_pool_exhaustion_declared"])
self.assertTrue(audit["enrichment"]["candidate_pool_exhausted"])
self.assertEqual(audit["candidate_pool_status"], "no_qualifying_candidates_in_bounded_pool")
self.assertEqual(audit["enrichment"]["next_action"], "publish_no_candidates")
def test_band_audit_must_cover_requested_market_cap_range_without_gaps(self) -> None:
row = {
"symbol": "READY",
"country": "US",
"exchange": "NASDAQ",
"sector": "Technology",
"is_actively_trading": True,
"is_common_stock": True,
"price": 20,
"market_cap": 2_000_000_000,
"average_daily_dollar_volume": 10_000_000,
"revenue_growth_pct": 12,
"eps_growth_pct": 18,
"forward_pe": 15,
"analyst_count": 4,
}
_, _, audit, _, _ = SCREEN.run_layered(
[row],
[row],
dict(SCREEN.DEFAULTS),
analysis_as_of="2026-08-22T14:00:00-07:00",
universe_source_ids=["u"],
candidate_source_ids=["e"],
candidate_generation_mode="full_universe_fundamentals",
retrieval_min_market_cap=500_000_000,
retrieval_max_market_cap=20_000_000_000,
candidate_pool_exhausted=True,
band_audit=[
{
"min_market_cap": 500_000_000,
"max_market_cap": 1_000_000_000,
"rows_fetched": 1,
"provider_exhausted": True,
},
{
"min_market_cap": 2_000_000_000,
"max_market_cap": 20_000_000_000,
"rows_fetched": 1,
"provider_exhausted": True,
},
],
)
enum = audit["scope"]["enumeration"]
self.assertTrue(enum["bands_well_formed"])
self.assertFalse(enum["bands_cover_requested_range"])
self.assertFalse(enum["bands_verified"])
self.assertFalse(audit["scope"]["scope_complete"])
class RegressionV34Tests(FixtureMixin, unittest.TestCase):
ANALYSIS_AS_OF = "2026-08-22T14:00:00-07:00"
def valid_discovery_row(self, symbol: str = "VALID") -> dict:
return {
"symbol": symbol,
"country": "US",
"exchange": "NASDAQ",
"sector": "Technology",
"is_actively_trading": True,
"is_common_stock": True,
"price": 40.0,
"market_cap": 3_500_000_000,
"average_daily_dollar_volume": 12_000_000.0,
"average_daily_dollar_volume_method": "provider_average_dollar_volume",
"average_volume_period_days": 20,
"liquidity_source_ids": ["listing-source"],
"revenue_growth_pct": 14.0,
"eps_growth_pct": 22.0,
"standard_fcf": 250_000_000.0,
"roic_pct": 18.0,
"forward_pe": 16.0,
"forward_eps": 2.5,
"forward_pe_period": "FY1",
"forward_fiscal_year": "FY2027",
"forward_period_end": "2027-06-30",
"forward_estimate_as_of": self.ANALYSIS_AS_OF,
"forward_estimate_source_ids": ["estimate-source"],
"forward_metric_origin": "computed_from_price_and_fy1_eps",
"analyst_count": 6,
"dilution_pct": 1.0,
"net_debt_to_ebitda": 1.0,
"fcf_yield_pct": 6.0,
}
def run_original(
self, universe: list[dict], pool: list[dict], *, config=None, exhausted=False, **kwargs
):
cfg = dict(SCREEN.DEFAULTS)
if config:
cfg.update(config)
return ORIGINAL_SCREEN_RUN_LAYERED(
universe,
pool,
cfg,
analysis_as_of=self.ANALYSIS_AS_OF,
universe_source_ids=["listing-source"],
candidate_source_ids=["estimate-source"],
candidate_generation_mode=kwargs.pop("mode", "full_universe_fundamentals"),
retrieval_min_market_cap=kwargs.pop("retrieval_min_market_cap", 500_000_000),
retrieval_max_market_cap=kwargs.pop("retrieval_max_market_cap", 20_000_000_000),
retrieval_scope_explicit=True,
candidate_pool_exhausted=exhausted,
provider_reported_total=len(universe),
pages_fetched=1,
pagination_exhausted=True,
**kwargs,
)
def test_single_session_volume_is_not_accepted_as_addv(self) -> None:
row = self.valid_discovery_row("SINGLE")
row["average_daily_dollar_volume_method"] = "price_x_single_session_volume"
row["average_volume_period_days"] = 1
row["average_volume"] = 600_000
_, decisions, _, selected, queue = self.run_original([row], [row], exhausted=False)
self.assertEqual(selected, [])
self.assertEqual(decisions[0]["decision"]["status"], "needs_enrichment")
self.assertIn(
"single_session_volume_not_valid_for_addv",
decisions[0]["decision"]["blocking_review_reasons"],
)
self.assertEqual([item["symbol"] for item in queue], ["SINGLE"])
def test_twenty_day_provider_average_liquidity_is_accepted(self) -> None:
row = self.valid_discovery_row("AVG20")
_, decisions, audit, selected, queue = self.run_original([row], [row], exhausted=True)
self.assertEqual(selected, ["AVG20"])
self.assertEqual(queue, [])
self.assertTrue(decisions[0]["metrics"]["liquidity_valid_for_screen"])
self.assertEqual(decisions[0]["metrics"]["average_volume_period_days"], 20)
self.assertEqual(audit["candidate_pool_status"], "sufficient")
def test_generic_forward_pe_without_ntm_or_fy1_metadata_is_rejected(self) -> None:
row = self.valid_discovery_row("NOHORIZON")
for key in (
"forward_eps",
"forward_pe_period",
"forward_fiscal_year",
"forward_period_end",
"forward_estimate_as_of",
"forward_estimate_source_ids",
):
row.pop(key, None)
_, decisions, _, selected, queue = self.run_original([row], [row], exhausted=False)
self.assertEqual(selected, [])
reasons = decisions[0]["decision"]["blocking_review_reasons"]
self.assertIn("forward_pe_period_must_be_ntm_or_fy1", reasons)
self.assertIn("current_forward_eps_unavailable_or_non_positive", reasons)
self.assertEqual([item["symbol"] for item in queue], ["NOHORIZON"])
def test_outer_year_frmi_like_pe_is_not_a_current_forward_multiple(self) -> None:
row = self.valid_discovery_row("FRMIX")
row.update(
{
"forward_pe": 4.0,
"forward_eps": 10.0,
"forward_pe_period": "FY1",
"forward_fiscal_year": "FY2030",
"forward_period_end": "2030-12-31",
"forward_eps_low": -0.23,
"forward_eps_high": 15.99,
"analyst_count": 2,
"operating_stage": "pre_operating",
}
)
_, decisions, _, selected, queue = self.run_original([row], [row], exhausted=False)
self.assertEqual(selected, [])
reasons = decisions[0]["decision"]["blocking_review_reasons"]
self.assertIn("current_forward_period_is_not_fy1_or_ntm", reasons)
self.assertIn("forward_eps_range_crosses_zero", reasons)
self.assertIn("pre_operating_company_not_pe_evaluable", reasons)
self.assertEqual([item["symbol"] for item in queue], ["FRMIX"])
def test_executed_scope_cannot_replace_default_user_requested_scope(self) -> None:
row = self.valid_discovery_row("BAND")
_, _, audit, _, _ = self.run_original(
[row],
[row],
config={"min_market_cap": 3_000_000_000, "max_market_cap": 4_000_000_000},
retrieval_min_market_cap=3_000_000_000,
retrieval_max_market_cap=4_000_000_000,
exhausted=True,
scope_reduction_reason="bounded automatically to finish within the execution budget",
)
self.assertEqual(audit["scope"]["user_requested_scope"]["min_market_cap"], 500_000_000)
self.assertEqual(audit["scope"]["user_requested_scope"]["max_market_cap"], 20_000_000_000)
self.assertEqual(audit["scope"]["executed_scope"]["min_market_cap"], 3_000_000_000)
self.assertEqual(audit["scope"]["executed_scope"]["max_market_cap"], 4_000_000_000)
self.assertFalse(audit["scope"]["user_requested_scope_complete"])
self.assertIn("executed_scope_narrower_than_user_request", audit["scope"]["reasons"])
self.assertFalse(audit["scope"]["scope_override_authorized"])
def test_next_action_requires_every_selected_symbol_without_user_choice(self) -> None:
state = {
"status": "partial",
"screening_audit": {
"candidate_pool_status": "sufficient",
"selection_outcome": "selected",
"selected_symbols": ["JOYY", "MWH", "FSM", "EXK", "FRMI"],
"enrichment": {"queue_count": 0, "queue_symbols": [], "all_rows_resolved": True},
},
"candidates": {
"JOYY": {"stage": "verified"},
"MWH": {"stage": "verified"},
},
"unprocessed_candidates": ["EXK", "FRMI", "FSM"],
}
action = RUN_STATE._next_action_payload(state)
self.assertEqual(action["action"], "complete_all_selected_deep_dives")
self.assertEqual(action["symbols"], ["EXK", "FRMI", "FSM"])
self.assertFalse(action["user_confirmation_required"])
self.assertIn("Never ask the user", action["instruction"])
self.assertIn("rerunning the deterministic broad screen", action["instruction"])
def test_normalizer_emits_auditable_fy1_and_resolves_valid_row(self) -> None:
listing = self.valid_discovery_row("VALIDFY1")
listing.pop("forward_pe", None)
listing.pop("forward_eps", None)
listing.pop("forward_pe_period", None)
listing.pop("forward_fiscal_year", None)
listing.pop("forward_period_end", None)
listing.pop("forward_estimate_as_of", None)
listing.pop("forward_estimate_source_ids", None)
listing.pop("forward_metric_origin", None)
estimates = [
{
"symbol": "VALIDFY1",
"country": "US",
"date": "2026-12-31",
"fiscalYear": "2026",
"epsAvg": 2.5,
"epsLow": 2.3,
"epsHigh": 2.7,
"revenueAvg": 1_000_000_000,
"numAnalystsEps": 5,
"numAnalystsRevenue": 5,
},
{
"symbol": "VALIDFY1",
"country": "US",
"date": "2027-12-31",
"fiscalYear": "2027",
"epsAvg": 3.0,
"epsLow": 2.8,
"epsHigh": 3.2,
"revenueAvg": 1_150_000_000,
"numAnalystsEps": 5,
"numAnalystsRevenue": 5,
},
{
"symbol": "VALIDFY1",
"country": "US",
"date": "2028-12-31",
"fiscalYear": "2028",
"epsAvg": 3.6,
"epsLow": 3.3,
"epsHigh": 3.9,
"revenueAvg": 1_320_000_000,
"numAnalystsEps": 4,
"numAnalystsRevenue": 4,
},
]
row = NORMALIZER.normalize_symbol(
"VALIDFY1",
estimates,
listing,
analysis_as_of=datetime(2026, 8, 22, tzinfo=timezone.utc),
estimate_as_of=datetime(2026, 8, 21, tzinfo=timezone.utc),
source_ids=["estimate-source"],
minimum_analysts=2,
max_dispersion_pct=100.0,
max_fy1_horizon_days=430,
forward_pe_tolerance_pct=3.0,
)
self.assertEqual(row["estimate_normalization_status"], "valid")
self.assertTrue(row["enrichment_resolved"])
self.assertFalse(row["enrichment_exhausted"])
self.assertEqual(row["forward_pe_period"], "FY1")
self.assertEqual(row["forward_fiscal_year"], "2026")
self.assertEqual(row["forward_metric_origin"], "computed_from_price_and_fy1_eps")
self.assertEqual(row["forward_estimate_source_ids"], ["estimate-source"])
self.assertAlmostEqual(row["forward_pe"], listing["price"] / 2.5, places=5)
_, decisions, _, selected, queue = self.run_original([row], [row], exhausted=True)
self.assertEqual(selected, ["VALIDFY1"])
self.assertEqual(queue, [])
self.assertTrue(decisions[0]["metrics"]["forward_validation_reasons"] == [])
def test_normalizer_rejects_outer_year_and_clears_current_forward_fields(self) -> None:
listing = self.valid_discovery_row("FRMIRAW")
listing.update({"is_pre_operating": True, "operating_stage": "pre_operating"})
estimates = [
{
"symbol": "FRMIRAW",
"country": "US",
"date": "2028-12-31",
"fiscalYear": "2028",
"epsAvg": 1.35,
"epsLow": -0.23,
"epsHigh": 2.93,
"revenueAvg": 10_000_000,
"numAnalystsEps": 2,
"numAnalystsRevenue": 2,
},
{
"symbol": "FRMIRAW",
"country": "US",
"date": "2030-12-31",
"fiscalYear": "2030",
"epsAvg": 8.0,
"epsLow": 0.30,
"epsHigh": 15.99,
"revenueAvg": 500_000_000,
"numAnalystsEps": 2,
"numAnalystsRevenue": 2,
},
]
row = NORMALIZER.normalize_symbol(
"FRMIRAW",
estimates,
listing,
analysis_as_of=datetime(2026, 8, 28, tzinfo=timezone.utc),
estimate_as_of=datetime(2026, 8, 27, tzinfo=timezone.utc),
source_ids=["estimate-source"],
minimum_analysts=2,
max_dispersion_pct=100.0,
max_fy1_horizon_days=430,
forward_pe_tolerance_pct=3.0,
)
self.assertEqual(row["estimate_normalization_status"], "unavailable")
self.assertTrue(row["enrichment_resolved"])
self.assertTrue(row["enrichment_exhausted"])
self.assertIsNone(row["forward_pe"])
self.assertIsNone(row["forward_eps"])
self.assertIsNotNone(row["raw_forward_candidate"]["computed_pe"])
reasons = row["estimate_normalization_reasons"]
self.assertIn("invalid_fy1_horizon", reasons)
self.assertIn("pre_operating_company", reasons)
self.assertIn("fy1_eps_range_crosses_zero", reasons)
self.assertIn("annual_estimate_series_not_contiguous", reasons)
_, decisions, _, selected, queue = self.run_original([row], [row], exhausted=True)
self.assertEqual(selected, [])
self.assertEqual(queue, [])
self.assertEqual(decisions[0]["decision"]["status"], "unavailable_after_enrichment")
self.assertIn(
"estimate_normalization_not_valid",
decisions[0]["decision"]["blocking_review_reasons"],
)
def test_forward_validator_honors_failed_normalization_status(self) -> None:
row = self.valid_discovery_row("TAMPER")
row["estimate_normalization_status"] = "unavailable"
row["estimate_normalization_reasons"] = ["invalid_fy1_horizon"]
evidence = SCREEN.normalize_forward_valuation(
row,
price=row["price"],
analysis_as_of=self.ANALYSIS_AS_OF,
)
self.assertFalse(evidence["valid"])
self.assertIn("estimate_normalization_not_valid", evidence["reasons"])
self.assertIn("invalid_fy1_horizon", evidence["reasons"])
def test_normalizer_requires_explicit_estimate_as_of(self) -> None:
with self.assertRaises(SystemExit):
NORMALIZER.parse_args(
[
"--estimates",
"estimates.json",
"--listing-input",
"listing.jsonl",
"--analysis-as-of",
self.ANALYSIS_AS_OF,
"--source-id",
"estimate-source",
"--output",
"normalized.jsonl",
]
)
class RunStateTests(FixtureMixin, unittest.TestCase):
def test_checkpoint_requires_all_selected_verified_before_complete(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
run_dir = root / "run-1"
self.assertEqual(
RUN_STATE.main(
[
"init",
"--run-dir",
str(run_dir),
"--analysis-as-of",
self.payload["analysis_as_of"],
"--price-as-of",
self.payload["price_basis"]["as_of"],
"--session",
"regular_close",
"--price-source-id",
"quote-20260821",
"--base-commit",
"759151ee92e0eed9a7f2ddf4d6b0e4d6707df409", # pragma: allowlist secret
"--config",
str(ASSETS_DIR / "screening-config.example.json"),
"--market-context",
str(MARKET_CONTEXT_PATH),
"--global-sources",
str(GLOBAL_SOURCES_PATH),
]
),
0,
)
self.assertEqual(
RUN_STATE.main(
[
"set-screening-audit",
"--run-dir",
str(run_dir),
"--audit",
str(ASSETS_DIR / "broad-screen-audit.example.json"),
"--universe-artifact",
str(UNIVERSE_AUDIT_PATH),
"--candidate-artifact",
str(AUDIT_ROWS_PATH),
]
),
0,
)
self.assertEqual(
RUN_STATE.main(["set-status", "--run-dir", str(run_dir), "complete"]), 1
)
def test_checkpoint_resume_and_assemble_schema_v3(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
run_dir = root / "run-1"
self.assertEqual(
RUN_STATE.main(
[
"init",
"--run-dir",
str(run_dir),
"--analysis-as-of",
self.payload["analysis_as_of"],
"--price-as-of",
self.payload["price_basis"]["as_of"],
"--session",
"regular_close",
"--price-source-id",
"quote-20260821",
"--base-commit",
"759151ee92e0eed9a7f2ddf4d6b0e4d6707df409", # pragma: allowlist secret
"--config",
str(ASSETS_DIR / "screening-config.example.json"),
"--market-context",
str(MARKET_CONTEXT_PATH),
"--global-sources",
str(GLOBAL_SOURCES_PATH),
]
),
0,
)
self.assertEqual(
RUN_STATE.main(
[
"set-screening-audit",
"--run-dir",
str(run_dir),
"--audit",
str(ASSETS_DIR / "broad-screen-audit.example.json"),
"--universe-artifact",
str(UNIVERSE_AUDIT_PATH),
"--candidate-artifact",
str(AUDIT_ROWS_PATH),
]
),
0,
)
for symbol in ("ACME", "CYCLE", "STALE"):
candidate_file = root / f"{symbol}.json"
candidate_file.write_text(json.dumps(self.candidate(symbol)), encoding="utf-8")
self.assertEqual(
RUN_STATE.main(
[
"save-candidate",
"--run-dir",
str(run_dir),
"--candidate",
str(candidate_file),
"--stage",
"verified",
]
),
0,
)
self.assertEqual(
RUN_STATE.main(
[
"set-funnel",
"--run-dir",
str(run_dir),
"--preflight-passed-count",
"3",
]
),
0,
)
self.assertEqual(
RUN_STATE.main(["set-status", "--run-dir", str(run_dir), "complete"]), 0
)
assembled = root / "assembled.json"
self.assertEqual(
RUN_STATE.main(
[
"assemble",
"--run-dir",
str(run_dir),
"--output",
str(assembled),
]
),
0,
)
snapshot = json.loads(assembled.read_text(encoding="utf-8"))
self.assertEqual(snapshot["schema_version"], 3)
self.assertEqual(snapshot["run_metadata"]["status"], "complete")
self.assertEqual(
{c["identity"]["symbol"] for c in snapshot["candidates"]},
{"ACME", "CYCLE", "STALE"},
)
report = EVALUATOR.evaluate_snapshot(snapshot, strict=True, artifact_root=run_dir)
self.assertEqual(report["ranking_status"], "final")
self.assertEqual(RUN_STATE.main(["status", "--run-dir", str(run_dir)]), 0)
def test_manage_run_state_version_cli(self):
completed = subprocess.run(
[sys.executable, str(SCRIPTS_DIR / "manage_run_state.py"), "--version"],
check=False,
capture_output=True,
text=True,
)
self.assertEqual(completed.returncode, 0, completed.stderr)
payload = json.loads(completed.stdout)
self.assertEqual(payload["skill_version"], "3.6.1")
self.assertEqual(payload["contract_revision"], "3.5")
self.assertEqual(
payload["runtime_fingerprint"], "ug-v3.6.1-claude-code-direct-fmp-20260830"
)
class RegressionV35Tests(FixtureMixin, unittest.TestCase):
def test_provider_prefilter_builder_combines_four_lanes(self) -> None:
universe = [
json.loads(line)
for line in (ASSETS_DIR / "universe-input.example.jsonl").read_text().splitlines()
if line.strip()
]
base = universe[0]
lane_rows = {}
for lane, suffix, pe, growth in (
("core_garp", "C", 14.0, 18.0),
("high_growth_exception", "H", 27.0, 32.0),
("quality_near_miss", "N", 11.0, 10.0),
("cyclical_normalization", "Y", 16.0, 22.0),
):
rows = []
for index in range(2):
row = deepcopy(base)
row["symbol"] = f"{suffix}{index}"
row["price"] = 28.0 + index
row["forward_pe"] = pe
row["forward_eps"] = row["price"] / pe
row["eps_growth_pct"] = growth
row["revenue_growth_pct"] = max(4.0, growth / 2.0)
row["analyst_count"] = 5
row["cyclicality_score"] = 4 if lane == "cyclical_normalization" else 1
rows.append(row)
lane_rows[lane] = rows
pool, audit = PROVIDER_POOL.build_pool(
universe_rows=universe,
lane_rows=lane_rows,
analysis_as_of="2026-08-22T14:00:00-07:00",
source_ids=["universe-fmp-20260822"],
per_lane=2,
max_pool=8,
minimum_pool=6,
requested_min_market_cap=500_000_000,
requested_max_market_cap=20_000_000_000,
provider_exhausted=False,
)
self.assertTrue(audit["valid"])
self.assertEqual(audit["lane_coverage_count"], 4)
self.assertEqual(len(pool), 8)
self.assertEqual(
{lane for row in pool for lane in row["provider_prefilter_lanes"]},
PROVIDER_POOL.ALLOWED_LANES,
)
def test_multilane_selection_preserves_opportunity_types(self) -> None:
def row(symbol, lane, priority, sector):
pre = {
"core": "passed",
"growth": "passed_exception",
"near": "near_miss_review",
"cycle": "passed",
}[lane]
cyclicality = 4 if lane == "cycle" else 1
requirements = ["mid_cycle_normalization_required"] if lane == "cycle" else []
return {
"symbol": symbol,
"country": "US",
"sector": sector,
"deep_dive_priority_score": priority,
"broad_score": priority,
"fundamental_completeness_count": 8,
"metrics": {"analyst_count": 5, "cyclicality_score": cyclicality},
"decision": {
"preselection_status": pre,
"status": pre,
"exception_admitted": lane == "growth",
"deep_dive_requirements": requirements,
},
}
rows = [
row("CORE", "core", 100, "Technology"),
row("GROW", "growth", 95, "Healthcare"),
row("NEAR", "near", 90, "Consumer"),
row("CYCL", "cycle", 85, "Materials"),
row("CORE2", "core", 80, "Technology"),
]
config = {
"selection_lane_quota_core_garp": 1,
"selection_lane_quota_high_growth": 1,
"selection_lane_quota_near_miss": 1,
"selection_lane_quota_cyclical": 1,
"maximum_selected_per_sector": 1,
}
selected, counts = SCREEN._select_multilane(rows, limit=4, config=config)
self.assertEqual({row["symbol"] for row in selected}, {"CORE", "GROW", "NEAR", "CYCL"})
self.assertEqual(counts["core_garp"], 1)
self.assertEqual(counts["high_growth_exception"], 1)
self.assertEqual(counts["quality_near_miss"], 1)
self.assertEqual(counts["cyclical_normalization"], 1)
def test_candidate_serializer_preserves_liquidity_provenance(self) -> None:
row = json.loads(
(ASSETS_DIR / "enriched-candidate-pool.example.jsonl").read_text().splitlines()[0]
)
config = json.loads((ASSETS_DIR / "screening-config.example.json").read_text())
decision = SCREEN._candidate_decision(
row, config, "liquidity_stratified_estimates", "2026-08-22T14:00:00-07:00"
)
metrics = decision["metrics"]
self.assertIn("average_volume", metrics)
self.assertEqual(metrics["liquidity_source_ids"], ["universe-fmp-20260822"])
self.assertTrue(metrics["liquidity_valid_for_screen"])
def test_provider_share_volume_prefilter_cannot_claim_full_listing_scope(self) -> None:
universe = [
json.loads((ASSETS_DIR / "universe-input.example.jsonl").read_text().splitlines()[0])
]
config = json.loads((ASSETS_DIR / "screening-config.example.json").read_text())
_, _, audit, _, _ = ORIGINAL_SCREEN_RUN_LAYERED(
universe,
universe,
config,
analysis_as_of="2026-08-22T14:00:00-07:00",
universe_source_ids=["universe-fmp-20260822"],
candidate_source_ids=["universe-fmp-20260822"],
candidate_generation_mode="full_universe_fundamentals",
retrieval_min_market_cap=500_000_000,
retrieval_max_market_cap=20_000_000_000,
requested_min_market_cap=500_000_000,
requested_max_market_cap=20_000_000_000,
provider_reported_total=1,
pages_fetched=1,
pagination_exhausted=True,
candidate_pool_exhausted=True,
band_audit=[
{
"min_market_cap": 500_000_000,
"max_market_cap": 20_000_000_000,
"rows_fetched": 1,
"provider_exhausted": True,
"retrieval_filters": {"min_volume": 500_000},
}
],
)
self.assertTrue(audit["scope"]["enumeration"]["provider_listing_prefiltered"])
self.assertFalse(audit["scope"]["enumeration"]["full_listing_enumeration_verified"])
self.assertFalse(audit["scope"]["user_requested_scope_complete"])
def test_dks_like_low_fcf_candidate_is_not_formally_eligible(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
candidate["identity"]["symbol"] = "LOWFCF"
candidate["financials"]["roic_pct"] = 8.1
candidate["financials"]["ebitda_ttm"] = 100_000_000
candidate["financials"]["total_debt"] = 500_000_000
candidate["financials"]["cash_classification"]["corporate_cash"] = 150_000_000
candidate["financials"]["cash_classification"]["cash_and_equivalents"] = 150_000_000
candidate["financials"]["cash_classification"]["marketable_securities"] = 0
for item in candidate["financials"]["cash_flow_periods"]:
item["operating_cash_flow"] = 10_000_000
item["capex_cash_outflow"] = 9_000_000
candidate["financials"]["cash_flow_ttm"].update(
{
"operating_cash_flow": 40_000_000,
"capex_cash_outflow": 36_000_000,
"standard_fcf": 4_000_000,
}
)
candidate["financials"]["sbc_ttm"] = 5_000_000
report = self.evaluate(payload)
self.assertEqual(report["ranked_candidates"], [])
row = report["review_required"][0]
self.assertIn("severely_weak_fcf_support", row["severe_quality_gate_failures"])
def test_provider_prefilter_skips_invalid_liquidity_without_poisoning_valid_pool(self) -> None:
valid = {
"symbol": "GOOD",
"country": "US",
"price": 25.0,
"market_cap": 2_000_000_000,
"average_volume": 500_000,
"average_volume_period_days": 20,
"average_daily_dollar_volume_method": "price_x_provider_average_volume",
"liquidity_source_ids": ["quote"],
"forward_pe": 14.0,
"forward_eps": 25.0 / 14.0,
"forward_pe_period": "FY1",
"forward_fiscal_year": "FY2027",
"forward_period_end": "2027-12-31",
"forward_estimate_as_of": "2026-08-22T14:00:00-07:00",
"forward_estimate_source_ids": ["estimate"],
"analyst_count": 5,
"eps_growth_pct": 18.0,
"revenue_growth_pct": 10.0,
}
invalid = {"symbol": "BAD", "price": 10.0, "volume": 3_000_000}
lanes = {
"core_garp": [valid, invalid],
"high_growth_exception": [{**valid, "symbol": "HIGH", "eps_growth_pct": 30.0}],
"quality_near_miss": [
{**valid, "symbol": "NEAR", "forward_pe": 12.0, "forward_eps": 25.0 / 12.0}
],
"cyclical_normalization": [{**valid, "symbol": "CYCLE", "cyclicality_score": 4}],
}
pool, audit = PROVIDER_POOL.build_pool(
universe_rows=[valid],
lane_rows=lanes,
analysis_as_of="2026-08-22T14:00:00-07:00",
source_ids=["provider"],
per_lane=1,
max_pool=4,
minimum_pool=4,
requested_min_market_cap=500_000_000,
requested_max_market_cap=20_000_000_000,
provider_exhausted=False,
)
self.assertTrue(audit["valid"])
self.assertEqual(audit["invalid_liquidity_symbols"], ["BAD"])
self.assertNotIn("BAD", [row["symbol"] for row in pool])
def test_target_solved_driver_bridge_is_rejected(self) -> None:
payload = self.single("ACME")
bridge = payload["candidates"][0]["forecast_bridge"]["periods"]["year_2"]
bridge["driver_provenance"]["operating_margin_pct"]["target_solved"] = True
report = self.evaluate(payload)
self.assertEqual(report["ranked_candidates"], [])
self.assertTrue(
any(
"target-solved" in reason
for reason in report["review_required"][0]["review_reasons"]
)
)
def test_sec_browse_page_is_not_primary_filing_evidence(self) -> None:
payload = self.single("ACME")
source = next(row for row in payload["candidates"][0]["sources"] if row["id"] == "acme-10q")
source["url"] = "https://www.sec.gov/edgar/browse/?CIK=1000000"
normalized, contract = CONTRACT.validate_and_normalize_snapshot(
payload, artifact_root=ASSETS_DIR
)
candidate_contract = normalized["candidates"][0]["_contract"]
self.assertFalse(candidate_contract["source_schema_valid"])
self.assertTrue(
any("accession-specific" in reason for reason in candidate_contract["review_reasons"])
)
def test_generic_cash_flow_support_does_not_verify_discrete_periods(self) -> None:
payload = self.single("ACME")
source = next(row for row in payload["candidates"][0]["sources"] if row["id"] == "acme-10q")
source["supports"] = [
value
for value in source["supports"]
if not value.startswith("financials.cash_flow_periods.")
]
self.assertIn("financials.cash_flow_ttm", source["supports"])
report = self.evaluate(payload)
self.assertEqual(report["ranked_candidates"], [])
self.assertTrue(
any(
"cash-flow period" in reason
for reason in report["review_required"][0]["review_reasons"]
)
)
def test_recent_spin_off_routes_to_review_not_hard_exclusion(self) -> None:
payload = self.single("ACME")
candidate = payload["candidates"][0]
candidate["identity"]["symbol"] = "SPIN"
candidate["identity"]["special_case"] = "recent_spin_off"
report = self.evaluate(payload)
self.assertEqual(report["excluded"], [])
self.assertEqual(report["review_required"][0]["symbol"], "SPIN")
self.assertTrue(
any(
"corporate transition" in reason.lower()
for reason in report["review_required"][0]["review_reasons"]
)
)
def test_final_three_categories_can_remain_null(self) -> None:
report = self.evaluate(self.single("ACME"))
self.assertIsNotNone(report["final_three"]["highest_conviction"])
self.assertIsNone(report["final_three"]["most_undervalued"])
def test_prepublish_audit_and_bundle_are_self_contained(self) -> None:
report = self.evaluate(self.payload)
markdown = EVALUATOR.render_markdown(report, language="ja")
self.assertTrue(
PREPUBLISH.audit_publication(
report, report_markdown=markdown, artifact_root=ASSETS_DIR
)["valid"]
)
with tempfile.TemporaryDirectory() as temp_dir:
run_dir = Path(temp_dir) / "run"
run_dir.mkdir()
for name in (
"universe-audit-results.example.jsonl",
"broad-screen-results.example.jsonl",
"enrichment-queue.example.json",
):
(run_dir / name).write_bytes((ASSETS_DIR / name).read_bytes())
report_path = run_dir / "final.json"
md_path = run_dir / "final.md"
report_path.write_text(json.dumps(report, ensure_ascii=False), encoding="utf-8")
md_path.write_text(markdown, encoding="utf-8")
output = Path(temp_dir) / "bundle.zip"
result = BUNDLER.build_bundle(run_dir, report_path, md_path, output)
self.assertTrue(output.is_file())
self.assertGreaterEqual(result["file_count"], 5)
from zipfile import ZipFile
with ZipFile(output) as archive:
names = set(archive.namelist())
self.assertIn("BUNDLE_MANIFEST.json", names)
self.assertIn("universe-audit-results.example.jsonl", names)
self.assertIn("broad-screen-results.example.jsonl", names)
def test_prepublish_audit_rejects_continue_language(self) -> None:
report = self.evaluate(self.payload)
markdown = EVALUATOR.render_markdown(report, language="ja") + "\n次のターンで続けます。\n"
audit = PREPUBLISH.audit_publication(
report, report_markdown=markdown, artifact_root=ASSETS_DIR
)
self.assertFalse(audit["valid"])
self.assertTrue(any("unfinished-run phrase" in error for error in audit["errors"]))
if __name__ == "__main__":
unittest.main()
scripts/tests/test_growth_basis.py
from __future__ import annotations
import importlib.util
import unittest
from datetime import datetime, timezone
from pathlib import Path
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
def load_module(name: str, path: Path):
spec = importlib.util.spec_from_file_location(name, path)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
NORMALIZER = load_module(
"normalize_estimates_growth_basis_test", SCRIPTS_DIR / "normalize_estimates.py"
)
ANALYSIS_AS_OF = datetime(2026, 8, 22, tzinfo=timezone.utc)
ESTIMATE_AS_OF = datetime(2026, 8, 21, tzinfo=timezone.utc)
def _listing(symbol: str, price: float = 40.0) -> dict:
return {"symbol": symbol, "price": price}
def _row(symbol: str, date: str, fiscal_year: str, eps_avg: float, **extra) -> dict:
base = {
"symbol": symbol,
"date": date,
"fiscalYear": fiscal_year,
"epsAvg": eps_avg,
"epsLow": eps_avg - 0.1,
"epsHigh": eps_avg + 0.1,
"revenueAvg": 1_000_000_000,
"numAnalystsEps": 5,
"numAnalystsRevenue": 5,
}
base.update(extra)
return base
def _normalize(symbol: str, estimates: list[dict], listing: dict | None = None) -> dict:
return NORMALIZER.normalize_symbol(
symbol,
estimates,
listing if listing is not None else _listing(symbol),
analysis_as_of=ANALYSIS_AS_OF,
estimate_as_of=ESTIMATE_AS_OF,
source_ids=["estimate-source"],
minimum_analysts=2,
max_dispersion_pct=100.0,
max_fy1_horizon_days=430,
forward_pe_tolerance_pct=3.0,
)
class GrowthBasisTests(unittest.TestCase):
def test_yelp_like_series_is_trough_recovery(self) -> None:
estimates = [
_row(
"YELP",
"2025-12-31",
"2025",
2.24,
isActual=True,
publishedDate="2026-02-27 16:05:00",
),
_row("YELP", "2026-12-31", "2026", 1.81),
_row("YELP", "2027-12-31", "2027", 2.53),
_row("YELP", "2028-12-31", "2028", 2.61),
]
row = _normalize("YELP", estimates)
self.assertAlmostEqual(row["latest_actual_eps"], 2.24, places=6)
self.assertTrue(row["fy1_eps_below_latest_actual"])
self.assertAlmostEqual(
row["current_year_growth_pct"], (1.81 / 2.24 - 1.0) * 100.0, places=4
)
self.assertEqual(row["growth_pattern"], "trough_recovery")
# existing field name stays populated identically (FY1->FY3 CAGR)
self.assertIsNotNone(row["eps_growth_fy1_to_fy3_pct"])
self.assertAlmostEqual(row["eps_growth_fy1_to_fy3_pct"], row["eps_growth_pct"], places=6)
# actual(2.24) -> FY3(2.61) CAGR over ~3 years ~= +5.2%/yr (spec value)
self.assertIsNotNone(row["eps_growth_actual_to_fy3_pct"])
self.assertAlmostEqual(row["eps_growth_actual_to_fy3_pct"], 5.227, places=2)
self.assertEqual(row["growth_basis_source_ids"], ["estimate-source"])
def test_steady_growth_series(self) -> None:
estimates = [
_row(
"STDY",
"2025-12-31",
"2025",
2.00,
isActual=True,
publishedDate="2026-02-27 16:05:00",
),
_row("STDY", "2026-12-31", "2026", 2.20),
_row("STDY", "2027-12-31", "2027", 2.42),
_row("STDY", "2028-12-31", "2028", 2.66),
]
row = _normalize("STDY", estimates)
self.assertAlmostEqual(row["latest_actual_eps"], 2.00, places=6)
self.assertFalse(row["fy1_eps_below_latest_actual"])
self.assertEqual(row["growth_pattern"], "steady")
def test_declining_series(self) -> None:
estimates = [
_row(
"DECL",
"2025-12-31",
"2025",
3.00,
isActual=True,
publishedDate="2026-02-27 16:05:00",
),
_row("DECL", "2026-12-31", "2026", 2.80),
_row("DECL", "2027-12-31", "2027", 2.60),
_row("DECL", "2028-12-31", "2028", 2.40),
]
row = _normalize("DECL", estimates)
self.assertEqual(row["growth_pattern"], "declining")
self.assertTrue(row["fy1_eps_below_latest_actual"])
def test_missing_actual_yields_none_and_unknown(self) -> None:
estimates = [
_row("NEWCO", "2026-12-31", "2026", 1.00),
_row("NEWCO", "2027-12-31", "2027", 1.30),
_row("NEWCO", "2028-12-31", "2028", 1.69),
]
row = _normalize("NEWCO", estimates)
self.assertIsNone(row["latest_actual_eps"])
self.assertIsNone(row["fy1_eps_below_latest_actual"])
self.assertIsNone(row["current_year_growth_pct"])
self.assertIsNone(row["eps_growth_actual_to_fy3_pct"])
self.assertEqual(row["growth_pattern"], "unknown")
def test_never_divides_by_zero_or_roots_negative(self) -> None:
# latest actual EPS is non-positive; must not raise and must yield None
# for actual-based derivatives instead of dividing by zero / taking a
# fractional power of a negative number.
estimates = [
_row(
"NEG",
"2025-12-31",
"2025",
-1.50,
isActual=True,
publishedDate="2026-02-27 16:05:00",
),
_row("NEG", "2026-12-31", "2026", 1.00),
_row("NEG", "2027-12-31", "2027", 1.30),
_row("NEG", "2028-12-31", "2028", 1.69),
]
row = _normalize("NEG", estimates)
self.assertAlmostEqual(row["latest_actual_eps"], -1.50, places=6)
self.assertIsNone(row["eps_growth_actual_to_fy3_pct"])
self.assertIsNone(row["current_year_growth_pct"])
self.assertIsNone(row["fy1_eps_below_latest_actual"])
self.assertEqual(row["growth_pattern"], "unknown")
def test_accelerating_growth_labeled_accelerating(self) -> None:
estimates = [
_row(
"ACC",
"2025-12-31",
"2025",
1.00,
isActual=True,
publishedDate="2026-02-27 16:05:00",
),
_row("ACC", "2026-12-31", "2026", 1.20),
_row("ACC", "2027-12-31", "2027", 1.60),
_row("ACC", "2028-12-31", "2028", 2.30),
]
row = _normalize("ACC", estimates)
self.assertFalse(row["fy1_eps_below_latest_actual"])
# fy3 (2.30) grows faster than fy1->fy2 pace -> accelerating, not just steady
self.assertIn(row["growth_pattern"], {"accelerating", "steady"})
if __name__ == "__main__":
unittest.main()
scripts/tests/test_pool_floor_and_sectors.py
"""Round-4 review P0s: pool-floor waiver scope; sector-profile inference and routing."""
from __future__ import annotations
import json
import sys
import unittest
from pathlib import Path
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
if str(SCRIPTS_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPTS_DIR))
import build_provider_prefilter_pool as POOL # noqa: E402
import run_pipeline as PIPELINE # noqa: E402
import screen_universe as SCREEN # noqa: E402
ASSETS_DIR = SCRIPTS_DIR.parent / "assets"
def _lane_row(symbol: str, addv: float = 20_000_000.0) -> dict:
return {
"symbol": symbol,
"sector": "Technology",
"market_cap": 2_000_000_000,
"price": 20.0,
"forward_pe": 15.0,
"eps_growth_pct": 15.0,
"revenue_growth_pct": 10.0,
"analyst_count": 5,
"cyclicality_score": 2,
"average_daily_dollar_volume": addv,
"average_daily_dollar_volume_method": "provider_average_dollar_volume",
"average_volume_period_days": 20,
"liquidity_source_ids": ["liq-src"],
}
def _build(minimum_pool: int, scope: str | None, count: int = 3):
rows = [_lane_row(f"SY{i}") for i in range(count)]
return POOL.build_pool(
universe_rows=rows,
lane_rows={"core_garp": rows, "high_growth_exception": rows, "quality_near_miss": rows},
analysis_as_of="2026-08-30T18:00:00+00:00",
source_ids=["estimate-source"],
per_lane=8,
max_pool=30,
minimum_pool=minimum_pool,
requested_min_market_cap=500_000_000,
requested_max_market_cap=20_000_000_000,
provider_exhausted=True,
provider_exhausted_scope=scope,
)
class PoolFloorScopeTests(unittest.TestCase):
def test_estimate_seed_exhaustion_does_not_waive_the_floor(self) -> None:
# The round-4 reviewer's reproduction: 30-row pool, floor 100,
# scope=estimate_seed used to come back valid.
_, audit = _build(minimum_pool=100, scope="estimate_seed")
self.assertFalse(audit["pool_adequate"])
self.assertFalse(audit["valid"])
self.assertFalse(audit["pool_floor_waived"])
def test_full_universe_exhaustion_still_waives_the_floor(self) -> None:
_, audit = _build(minimum_pool=100, scope="economic_candidate_universe")
self.assertTrue(audit["pool_adequate"])
self.assertTrue(audit["pool_floor_waived"])
def test_missing_scope_fails_closed(self) -> None:
# Round-5 hardening: an unstated exhaustion scope proves nothing, so
# it no longer earns the legacy full waiver.
_, audit = _build(minimum_pool=100, scope=None)
self.assertFalse(audit["pool_adequate"])
self.assertFalse(audit["pool_floor_waived"])
def test_floor_met_needs_no_waiver(self) -> None:
_, audit = _build(minimum_pool=2, scope="estimate_seed")
self.assertTrue(audit["pool_adequate"])
self.assertFalse(audit["pool_floor_waived"])
class SectorProfileInferenceTests(unittest.TestCase):
CASES = [
("Real Estate", "REIT - Mortgage", "reit"),
("Real Estate", "REIT - Diversified", "reit"),
("Financial Services", "Insurance - Property & Casualty", "insurance"),
("Financial Services", "Banks - Regional", "bank"),
("Financial Services", "Asset Management", "asset_manager"),
("Financial Services", "Business Development Company", "bdc"),
("Energy", "Oil & Gas MLP", "mlp"),
("Consumer Cyclical", "Auto & Truck Dealerships", "auto_dealership"),
("Consumer Cyclical", "Auto - Dealerships", "auto_dealership"),
("Financial Services", "Investment - Banking & Investment Services", "capital_markets"),
("Financial Services", "Banks - Diversified", "bank"),
("Technology", "Software - Application", "general"),
(None, None, "general"),
]
def test_inference_matches_screen_universe_profiles(self) -> None:
for sector, industry, expected in self.CASES:
with self.subTest(sector=sector, industry=industry):
self.assertEqual(PIPELINE.infer_sector_profile_type(sector, industry), expected)
# every BLOCKED profile we emit must be one screen_universe gates on;
# capital_markets and auto_dealership have their own handling.
emitted = (
set(PIPELINE.INDUSTRY_PROFILE_MAP.values())
| {profile for _, profile in PIPELINE._INDUSTRY_PREFIX_RULES}
| {"bdc", "mlp", "reit"}
)
self.assertTrue(
emitted <= (SCREEN.SECTOR_PROFILES | {"auto_dealership", "capital_markets"})
)
def test_normalize_listing_carries_the_profile(self) -> None:
row = {
"symbol": "ABR",
"companyName": "Arbor Realty Trust",
"price": 5.06,
"marketCap": 970_000_000,
"sector": "Real Estate",
"industry": "REIT - Mortgage",
}
out = PIPELINE.normalize_listing(row, "NYSE")
self.assertEqual(out["sector_profile_type"], "reit")
def test_reit_without_sector_metrics_routes_to_sector_review(self) -> None:
# A mortgage REIT must hit sector_specific_valuation_required, not the
# general-company leverage gate (ABR: net debt/EBITDA 13x was excluded
# as excessive_leverage while typed "general").
base = json.loads(
(ASSETS_DIR / "enriched-candidate-pool.example.jsonl").read_text().splitlines()[0]
)
config = json.loads((ASSETS_DIR / "screening-config.example.json").read_text())
base["sector_profile_type"] = "reit"
base["net_debt_to_ebitda"] = 13.2
decision = SCREEN._candidate_decision(
base, config, "liquidity_stratified_estimates", "2026-08-22T14:00:00-07:00"
)
d = decision["decision"]
reasons = (
list(d.get("blocking_review_reasons") or [])
+ list(d.get("review_reasons") or [])
+ list(d.get("hard_reasons") or [])
+ list(d.get("screen_fail_reasons") or [])
)
self.assertIn("sector_specific_valuation_required", reasons)
self.assertNotIn("excessive_leverage", d.get("hard_reasons") or [])
self.assertNotIn("excessive_leverage", d.get("screen_fail_reasons") or [])
class SectorExhaustionTests(unittest.TestCase):
def test_reit_without_sector_metrics_is_declared_exhausted(self) -> None:
rows = [
{"symbol": "ABR", "sector_profile_type": "reit"},
{"symbol": "GEN", "sector_profile_type": "general"},
{"symbol": "AFO", "sector_profile_type": "reit", "p_to_affo": 9.5},
{"symbol": "HLI", "sector_profile_type": "capital_markets"},
]
out = PIPELINE.mark_sector_profile_exhaustion(rows, source_id="fmp-key-metrics-ttm-x")
by = {r["symbol"]: r for r in out}
self.assertTrue(by["ABR"]["enrichment_exhausted"])
self.assertIn("reit", by["ABR"]["enrichment_exhaustion_reason"])
self.assertEqual(by["ABR"]["enrichment_source_ids"], ["fmp-key-metrics-ttm-x"])
self.assertNotIn("enrichment_exhausted", by["GEN"])
self.assertNotIn("enrichment_exhausted", by["AFO"]) # has a sector metric
# Round-8 review: capital_markets is valued on ordinary multiples —
# it must never be declared sector-enrichment exhausted.
self.assertNotIn("enrichment_exhausted", by["HLI"])
def test_exhausted_reit_resolves_as_unavailable_after_enrichment(self) -> None:
base = json.loads(
(ASSETS_DIR / "enriched-candidate-pool.example.jsonl").read_text().splitlines()[0]
)
config = json.loads((ASSETS_DIR / "screening-config.example.json").read_text())
base["sector_profile_type"] = "reit"
base["net_debt_to_ebitda"] = 13.2
[marked] = PIPELINE.mark_sector_profile_exhaustion(
[base], source_id="fmp-key-metrics-ttm-x"
)
decision = SCREEN._candidate_decision(
marked, config, "liquidity_stratified_estimates", "2026-08-22T14:00:00-07:00"
)
d = decision["decision"]
self.assertEqual(d.get("status"), "unavailable_after_enrichment")
self.assertEqual(d.get("resolution"), "resolved")
self.assertIn("sector_specific_valuation_required", d.get("blocking_review_reasons") or [])
if __name__ == "__main__":
unittest.main()
scripts/tests/test_ranking_scope.py
"""Round-6 review: honest ranking scope — a 4% economic sample is never a market ranking."""
from __future__ import annotations
import sys
import unittest
from pathlib import Path
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
if str(SCRIPTS_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPTS_DIR))
import evaluate_candidates as EVAL # noqa: E402
import run_pipeline as PIPELINE # noqa: E402
class ClassifyRankingScopeTests(unittest.TestCase):
def test_seed_sample_is_scoped_never_marketwide(self) -> None:
# The reviewer's case: 180 attempted of 2,371 listed (98 evaluable).
self.assertEqual(
PIPELINE.classify_ranking_scope(
economic_attempt_count=180,
listing_universe_count=2371,
economic_scope_complete=False,
unresolved_queue_count=0,
),
"final_scoped",
)
def test_unresolved_queue_is_diagnostic(self) -> None:
self.assertEqual(
PIPELINE.classify_ranking_scope(
economic_attempt_count=180,
listing_universe_count=2371,
economic_scope_complete=False,
unresolved_queue_count=5,
),
"diagnostic",
)
def test_full_universe_attempt_is_marketwide(self) -> None:
self.assertEqual(
PIPELINE.classify_ranking_scope(
economic_attempt_count=2371,
listing_universe_count=2371,
economic_scope_complete=True,
unresolved_queue_count=0,
),
"final_marketwide",
)
def test_zero_attempts_fail_closed_to_diagnostic(self) -> None:
# Round-7 review: a run that attempted nothing proved nothing about
# any subset — never final_scoped.
self.assertEqual(
PIPELINE.classify_ranking_scope(
economic_attempt_count=0,
listing_universe_count=2371,
economic_scope_complete=False,
unresolved_queue_count=0,
),
"diagnostic",
)
self.assertEqual(
PIPELINE.classify_ranking_scope(
economic_attempt_count=180,
listing_universe_count=0,
economic_scope_complete=False,
unresolved_queue_count=0,
),
"diagnostic",
)
def test_full_attempt_without_scope_completeness_stays_scoped(self) -> None:
# Attempt counts alone are not enough: the exact-count economic
# completeness verdict must agree.
self.assertEqual(
PIPELINE.classify_ranking_scope(
economic_attempt_count=2371,
listing_universe_count=2371,
economic_scope_complete=False,
unresolved_queue_count=0,
),
"final_scoped",
)
def _audit(
*,
universe: int = 2371,
seeds: int = 180,
covered: int = 0,
mode: str = "per_symbol_fallback",
unresolved: int = 0,
probe: int = 35,
evaluable: int = 98,
snapshot_verified: bool = False,
) -> dict:
return {
"universe": {"row_count": universe},
"enrichment": {"unresolved_count": unresolved},
"candidate_pool": {
"generation_audit": {
"estimate_acquisition_mode": mode,
"economic_attempt_count": seeds,
"economically_evaluable_count": evaluable,
"seed_audit": {"seed_limit_effective": seeds},
"bulk_estimate_audit": {
"covered_symbol_count": covered,
"universe_symbol_count": universe,
},
"quality_probe": {"attempted": [f"S{i}" for i in range(probe)]},
"snapshot_verification": {
"ready_for_screening": snapshot_verified,
"snapshot_verification_digest": "a" * 64 if snapshot_verified else None,
"classification_matches_universe": snapshot_verified,
"classified_total": universe if snapshot_verified else 0,
},
}
},
}
class DeriveRankingScopeTests(unittest.TestCase):
def test_run16_shape_is_final_scoped_with_coverage(self) -> None:
info = EVAL._derive_ranking_scope(_audit(), deep_dive_count=3)
self.assertEqual(info["ranking_scope"], "final_scoped")
self.assertEqual(info["economic_attempt_count"], 180)
self.assertAlmostEqual(info["economic_attempt_coverage_pct"], 7.591734, places=4)
self.assertEqual(info["economically_evaluable_count"], 98)
self.assertAlmostEqual(info["economically_evaluable_coverage_pct"], 4.133277, places=4)
self.assertEqual(info["quality_probe_count"], 35)
self.assertEqual(info["deep_dive_count"], 3)
self.assertAlmostEqual(info["deep_dive_coverage_pct"], 0.126529, places=4)
def test_unresolved_queue_marks_diagnostic(self) -> None:
info = EVAL._derive_ranking_scope(_audit(unresolved=4), deep_dive_count=0)
self.assertEqual(info["ranking_scope"], "diagnostic")
def test_zero_attempts_derive_as_diagnostic(self) -> None:
info = EVAL._derive_ranking_scope(_audit(seeds=0, evaluable=0), deep_dive_count=0)
self.assertEqual(info["ranking_scope"], "diagnostic")
def test_missing_attempt_count_fails_closed(self) -> None:
# Round-8 review: the evaluator must copy discovery's ACTUAL count,
# and a missing field is a diagnostic, never a reconstructed limit.
audit = _audit()
del audit["candidate_pool"]["generation_audit"]["economic_attempt_count"]
info = EVAL._derive_ranking_scope(audit, deep_dive_count=3)
self.assertEqual(info["ranking_scope"], "diagnostic")
def test_attempts_exceeding_universe_fail_closed(self) -> None:
# The reviewer's reproduction: 50-name universe, seed LIMIT 180 —
# 360% coverage must never appear as a scoped conclusion.
info = EVAL._derive_ranking_scope(_audit(universe=50, seeds=180), deep_dive_count=3)
self.assertEqual(info["ranking_scope"], "diagnostic")
def test_bulk_full_coverage_is_marketwide(self) -> None:
info = EVAL._derive_ranking_scope(
_audit(mode="analyst_estimates_bulk", covered=2371), deep_dive_count=3
)
self.assertEqual(info["ranking_scope"], "final_marketwide")
self.assertEqual(info["economic_attempt_count"], 2371)
def test_partial_bulk_coverage_stays_scoped(self) -> None:
info = EVAL._derive_ranking_scope(
_audit(mode="analyst_estimates_bulk", covered=600), deep_dive_count=3
)
self.assertEqual(info["ranking_scope"], "final_scoped")
self.assertEqual(info["economic_attempt_count"], 600)
def test_verified_sharded_snapshot_is_marketwide(self) -> None:
info = EVAL._derive_ranking_scope(
_audit(
mode="sharded_snapshot",
seeds=2371,
evaluable=1800,
snapshot_verified=True,
),
deep_dive_count=5,
)
self.assertEqual(info["ranking_scope"], "final_marketwide")
def test_unverified_sharded_snapshot_is_diagnostic(self) -> None:
info = EVAL._derive_ranking_scope(
_audit(mode="sharded_snapshot", seeds=2371, evaluable=1800),
deep_dive_count=5,
)
self.assertEqual(info["ranking_scope"], "diagnostic")
def test_marketwide_markdown_drops_scoped_pilot_warning(self) -> None:
report = {
"ranking_scope": "final_marketwide",
"ranking_status": "final",
"strict_mode": True,
"analysis_as_of": "2026-09-05T12:00:00+00:00",
"coverage": {
"listing_universe_count": 2371,
"economic_attempt_count": 2371,
},
"ranked_candidates": [],
"conditional": [],
"review_required": [],
"screened_out": [],
"excluded": [],
}
markdown = EVAL.render_markdown(report, language="en")
self.assertNotIn("Scoped Pilot", markdown)
self.assertNotIn("NOT a market-wide ranking", markdown)
report["ranking_scope"] = "final_scoped"
report["coverage"]["economic_attempt_count"] = 180
markdown = EVAL.render_markdown(report, language="en")
self.assertIn("Scoped Pilot", markdown)
self.assertIn("NOT a market-wide ranking", markdown)
if __name__ == "__main__":
unittest.main()
scripts/tests/test_scope_and_actuals.py
"""P0 regressions from the v3.6.1 review: economic scope completeness and verified actual EPS."""
from __future__ import annotations
import sys
import unittest
from datetime import datetime, timezone
from pathlib import Path
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
if str(SCRIPTS_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPTS_DIR))
import normalize_estimates as NORMALIZER # noqa: E402
import run_pipeline as PIPELINE # noqa: E402
AS_OF = datetime(2026, 8, 22, tzinfo=timezone.utc)
class EconomicScopeTests(unittest.TestCase):
def _complete(self, mode: str, covered: int, universe: int) -> bool:
return PIPELINE.economic_scope_complete(
estimate_acquisition_mode=mode,
covered_symbol_count=covered,
universe_symbol_count=universe,
)
def test_99pct_coverage_is_not_complete(self) -> None:
self.assertFalse(self._complete("analyst_estimates_bulk", 99, 100))
def test_9999_of_10000_is_not_complete(self) -> None:
self.assertFalse(self._complete("analyst_estimates_bulk", 9999, 10000))
def test_exact_full_coverage_is_complete(self) -> None:
self.assertTrue(self._complete("analyst_estimates_bulk", 2371, 2371))
def test_per_symbol_fallback_is_never_complete(self) -> None:
self.assertFalse(self._complete("bounded_per_symbol_fallback", 2371, 2371))
def test_empty_universe_is_not_complete(self) -> None:
self.assertFalse(self._complete("analyst_estimates_bulk", 0, 0))
def test_completeness_is_not_configurable(self) -> None:
# No ratio knob: the function takes no config, and the old key is gone
# from DEFAULT_CONFIG, so a user cannot lower the bar back to 20%.
import inspect
params = inspect.signature(PIPELINE.economic_scope_complete).parameters
self.assertNotIn("config", params)
self.assertNotIn("bulk_coverage_pct", params)
self.assertNotIn("economic_scope_complete_minimum_coverage_pct", PIPELINE.DEFAULT_CONFIG)
def _row(date: str, fiscal_year: str, eps: float, **extra) -> dict:
base = {
"symbol": "T",
"date": date,
"fiscalYear": fiscal_year,
"epsAvg": eps,
"epsLow": eps - 0.1,
"epsHigh": eps + 0.1,
"revenueAvg": 1_000_000_000,
"numAnalystsEps": 5,
"numAnalystsRevenue": 5,
}
base.update(extra)
return base
def _normalize(rows):
return NORMALIZER.normalize_symbol(
"T",
rows,
{"symbol": "T", "price": 40.0},
analysis_as_of=AS_OF,
estimate_as_of=datetime(2026, 8, 21, tzinfo=timezone.utc),
source_ids=["estimate-source"],
minimum_analysts=2,
max_dispersion_pct=100.0,
max_fy1_horizon_days=430,
forward_pe_tolerance_pct=3.0,
)
class VerifiedActualTests(unittest.TestCase):
SERIES = [
_row("2025-12-31", "2025", 2.0), # unmarked prior-year consensus row
_row("2026-12-31", "2026", 1.8),
_row("2027-12-31", "2027", 2.5),
_row("2028-12-31", "2028", 2.6),
]
def test_unmarked_prior_year_estimate_is_not_an_actual(self) -> None:
out = _normalize(self.SERIES)
self.assertIsNone(out["latest_actual_eps"])
self.assertIsNone(out["fy1_eps_below_latest_actual"])
# ...but it IS the same-basis consensus reference for the growth pattern
self.assertEqual(out["fy0_consensus_eps"], 2.0)
self.assertTrue(out["fy1_eps_below_fy0_consensus"])
self.assertEqual(out["growth_pattern"], "trough_recovery")
self.assertEqual(out["growth_pattern_basis"], "consensus_same_basis")
def test_no_prior_year_row_means_unknown_pattern(self) -> None:
out = _normalize(self.SERIES[1:])
self.assertIsNone(out["fy0_consensus_eps"])
self.assertEqual(out["growth_pattern"], "unknown")
self.assertEqual(out["growth_pattern_basis"], "unknown")
def test_marked_actual_after_analysis_as_of_is_ignored(self) -> None:
rows = list(self.SERIES) + [_row("2027-12-31", "2027", 9.0, isActual=True)]
out = _normalize(rows)
self.assertIsNone(out["latest_actual_eps"])
self.assertEqual(out["fy0_consensus_eps"], 2.0) # look-ahead row never becomes FY0
def test_marked_actual_with_publication_timestamp_is_used(self) -> None:
rows = [
_row("2025-12-31", "2025", 2.24, isActual=True, publishedDate="2026-02-27 16:05:00")
] + self.SERIES[1:]
out = _normalize(rows)
self.assertEqual(out["latest_actual_eps"], 2.24)
self.assertTrue(out["latest_actual_verified"])
self.assertEqual(out["latest_actual_basis"], "provider_marked_actual")
self.assertEqual(out["latest_actual_source_ids"], ["estimate-source"])
self.assertEqual(out["growth_pattern"], "trough_recovery")
def test_marked_actual_without_publication_timestamp_is_rejected(self) -> None:
rows = [_row("2025-12-31", "2025", 2.24, isActual=True)] + self.SERIES[1:]
out = _normalize(rows)
self.assertIsNone(out["latest_actual_eps"])
self.assertFalse(out["latest_actual_verified"])
self.assertIsNone(out["latest_actual_basis"])
self.assertEqual(out["latest_actual_source_ids"], [])
def test_historical_as_of_between_period_end_and_publication(self) -> None:
# Period ended 2025-12-31, published 2026-02-27, but we are replaying
# as of 2026-01-05: the actual was NOT public yet -> no actual fields.
rows = [
_row("2025-12-31", "2025", 2.24, isActual=True, publishedDate="2026-02-27 16:05:00"),
_row("2026-12-31", "2026", 1.8),
_row("2027-12-31", "2027", 2.5),
_row("2028-12-31", "2028", 2.6),
]
out = NORMALIZER.normalize_symbol(
"T",
rows,
{"symbol": "T", "price": 40.0},
analysis_as_of=datetime(2026, 1, 5, tzinfo=timezone.utc),
estimate_as_of=datetime(2026, 1, 4, tzinfo=timezone.utc),
source_ids=["estimate-source"],
minimum_analysts=2,
max_dispersion_pct=100.0,
max_fy1_horizon_days=430,
forward_pe_tolerance_pct=3.0,
)
self.assertIsNone(out["latest_actual_eps"])
self.assertFalse(out["latest_actual_verified"])
def test_apply_verified_actual_eps_rederives_growth_basis(self) -> None:
base = _normalize(self.SERIES)
out = NORMALIZER.apply_verified_actual_eps(
base,
actual_eps=2.24,
period_end="2025-12-31",
analysis_as_of=AS_OF,
source_ids=["fmp-income-statement-annual-2026-08-22"],
)
self.assertTrue(out["latest_actual_verified"])
self.assertEqual(out["latest_actual_eps"], 2.24)
self.assertEqual(out["latest_actual_basis"], "gaap_diluted")
self.assertTrue(out["fy1_eps_below_latest_actual"])
self.assertAlmostEqual(
out["current_year_growth_pct_vs_gaap_actual"], (1.8 / 2.24 - 1) * 100, places=4
)
# consensus FY0 2.0 vs GAAP 2.24 -> 10.7% gap: same basis, no flag
self.assertFalse(out["estimate_basis_likely_adjusted"])
self.assertEqual(out["growth_pattern"], "trough_recovery") # unchanged, consensus basis
self.assertEqual(
out["latest_actual_source_ids"], ["fmp-income-statement-annual-2026-08-22"]
)
def test_apply_verified_actual_eps_fails_closed_on_future_or_missing(self) -> None:
base = _normalize(self.SERIES)
future = NORMALIZER.apply_verified_actual_eps(
base, actual_eps=2.24, period_end="2027-12-31", analysis_as_of=AS_OF, source_ids=["x"]
)
self.assertFalse(future["latest_actual_verified"])
self.assertIsNone(future["latest_actual_eps"])
missing = NORMALIZER.apply_verified_actual_eps(
base, actual_eps=None, period_end=None, analysis_as_of=AS_OF, source_ids=[]
)
self.assertIsNone(missing["latest_actual_eps"])
# consensus-basis pattern is untouched by a missing/invalid actual
self.assertEqual(missing["growth_pattern"], base["growth_pattern"])
def test_gaap_actual_far_below_consensus_flags_basis_mismatch(self) -> None:
# DOCS-like: consensus FY0 1.52 (non-GAAP), GAAP actual 0.98, FY1 1.35
rows = [
_row("2026-03-31", "2026", 1.52),
_row("2027-03-31", "2027", 1.35),
_row("2028-03-31", "2028", 1.55),
_row("2029-03-31", "2029", 1.75),
]
base = _normalize(rows)
self.assertTrue(base["fy1_eps_below_fy0_consensus"])
self.assertEqual(base["growth_pattern"], "trough_recovery")
out = NORMALIZER.apply_verified_actual_eps(
base, actual_eps=0.98, period_end="2026-03-31", analysis_as_of=AS_OF, source_ids=["is"]
)
self.assertTrue(out["estimate_basis_likely_adjusted"])
self.assertFalse(out["fy1_eps_below_latest_actual"]) # 1.35 > 0.98 on mixed bases
self.assertEqual(out["growth_pattern"], "trough_recovery") # same-basis verdict kept
class VerifiedAnnualActualParsingTests(unittest.TestCase):
def test_rejects_filings_accepted_after_analysis_as_of(self) -> None:
statements = [
{
"period": "FY",
"date": "2025-12-31",
"acceptedDate": "2026-02-27 16:05:00",
"epsDiluted": 2.24,
},
{
"period": "FY",
"date": "2026-12-31",
"acceptedDate": "2027-02-25 16:05:00",
"epsDiluted": 9.0,
},
{
"period": "Q2",
"date": "2026-06-30",
"acceptedDate": "2026-08-07 16:05:00",
"epsDiluted": 0.57,
},
]
eps, end = PIPELINE._verified_annual_actual(statements, analysis_as_of=AS_OF)
self.assertEqual((eps, end), (2.24, "2025-12-31"))
def test_filing_date_fallback_is_gone_and_date_only_rejected(self) -> None:
date_only = [
{"period": "FY", "date": "2025-12-31", "acceptedDate": "2026-02-27", "epsDiluted": 2.24}
]
self.assertEqual(
PIPELINE._verified_annual_actual(date_only, analysis_as_of=AS_OF), (None, None)
)
filing_only = [
{"period": "FY", "date": "2025-12-31", "filingDate": "2026-02-27", "epsDiluted": 2.24}
]
self.assertEqual(
PIPELINE._verified_annual_actual(filing_only, analysis_as_of=AS_OF), (None, None)
)
def test_period_ended_but_not_yet_filed_is_rejected(self) -> None:
statements = [
{
"period": "FY",
"date": "2026-06-30",
"acceptedDate": "2026-08-30 16:05:00",
"epsDiluted": 3.0,
}
]
eps, end = PIPELINE._verified_annual_actual(statements, analysis_as_of=AS_OF)
self.assertEqual((eps, end), (None, None))
def test_probe_wires_actual_eps_and_counts_calls(self) -> None:
class Client:
def __init__(self):
self.calls = []
def get_key_metrics_ttm(self, symbol):
self.calls.append(("km", symbol))
return [{"freeCashFlowYieldTTM": 0.05}]
def get_income_statement(self, symbol, *, period="annual", limit=6):
self.calls.append(("is", symbol))
return [
{
"period": "FY",
"date": "2025-12-31",
"acceptedDate": "2026-02-27 16:05:00",
"epsDiluted": 2.24,
}
]
client = Client()
rows = [_normalize(VerifiedActualTests.SERIES)]
out, audit = PIPELINE.apply_quality_probe(
client,
rows,
target_symbols=["T"],
source_id="fmp-key-metrics-ttm-2026-08-22",
analysis_as_of=AS_OF,
actual_source_id="fmp-income-statement-annual-2026-08-22",
)
self.assertEqual(out[0]["latest_actual_eps"], 2.24)
self.assertEqual(out[0]["growth_pattern"], "trough_recovery")
self.assertEqual(audit["actual_eps_resolved"], ["T"])
self.assertEqual(audit["calls_used"], 2)
if __name__ == "__main__":
unittest.main()
scripts/tests/test_screening_audit_transaction.py
"""set-screening-audit replaces a run's audit bundle all-or-nothing (Issue #642)."""
from __future__ import annotations
import hashlib
import json
import os
import shutil
import stat
import subprocess
import sys
from pathlib import Path
import pytest
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
if str(SCRIPTS_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPTS_DIR))
import bundle_run_artifacts as BUNDLE # noqa: E402
import manage_run_state as RUN_STATE # noqa: E402
import prepublish_audit as PREPUBLISH # noqa: E402
ASSETS = SCRIPTS_DIR.parent / "assets"
UNIVERSE_TARGET = "audit/universe-audit-results.jsonl"
CANDIDATE_TARGET = "audit/broad-screen-results.jsonl"
COMMITTED_TARGETS = [
UNIVERSE_TARGET,
CANDIDATE_TARGET,
"audit/broad-screen-audit.json",
"events.ndjson",
"run_state.json",
]
def _sha(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def _snapshot(run: Path) -> dict[str, bytes]:
return {
path.relative_to(run).as_posix(): path.read_bytes()
for path in sorted(run.rglob("*"))
if path.is_file()
}
def _attach_args(run: Path, audit: Path, universe: Path, candidate: Path) -> list[str]:
return [
"set-screening-audit",
"--run-dir",
str(run),
"--audit",
str(audit),
"--universe-artifact",
str(universe),
"--candidate-artifact",
str(candidate),
]
def _init(run: Path) -> None:
example = json.loads((ASSETS / "candidate-input.example.json").read_text(encoding="utf-8"))
args = [
"init",
"--run-dir",
str(run),
"--analysis-as-of",
example["analysis_as_of"],
"--price-as-of",
example["price_basis"]["as_of"],
"--session",
"regular_close",
"--price-source-id",
"quote-20260821",
"--config",
str(ASSETS / "screening-config.example.json"),
"--market-context",
str(ASSETS / "market-context.example.json"),
"--global-sources",
str(ASSETS / "global-sources.example.json"),
]
assert RUN_STATE.main(args) == 0
class Replacement:
"""An editable copy of the bundled audit whose universe bytes differ from the example."""
def __init__(self, root: Path) -> None:
self.dir = root / "replacement"
shutil.copytree(ASSETS, self.dir)
self.audit_path = self.dir / "broad-screen-audit.example.json"
self.audit = json.loads(self.audit_path.read_text(encoding="utf-8"))
self.universe = self.dir / "universe-audit-results.example.jsonl"
self.candidate = self.dir / "broad-screen-results.example.jsonl"
self.universe.write_bytes(self.universe.read_bytes() + b"\n")
self.audit["universe"]["artifact_sha256"] = _sha(self.universe)
def args(self, run: Path) -> list[str]:
self.audit_path.write_text(json.dumps(self.audit), encoding="utf-8")
return _attach_args(run, self.audit_path, self.universe, self.candidate)
@pytest.fixture
def attached_run(tmp_path: Path) -> Path:
run = tmp_path / "run"
_init(run)
args = _attach_args(
run,
ASSETS / "broad-screen-audit.example.json",
ASSETS / "universe-audit-results.example.jsonl",
ASSETS / "broad-screen-results.example.jsonl",
)
assert RUN_STATE.main(args) == 0
assert RUN_STATE.main(["status", "--run-dir", str(run), "--strict"]) == 0
return run
@pytest.fixture
def replacement(tmp_path: Path) -> Replacement:
return Replacement(tmp_path)
def _bad_candidate_sha(item: Replacement) -> None:
item.audit["candidate_pool"]["artifact_sha256"] = "0" * 64
def _selected_wrong_type(item: Replacement) -> None:
item.audit["selected_symbols"] = "ACME"
def _plan_mismatch(item: Replacement) -> None:
item.audit["deep_dive_plan"]["selected_symbols"] = ["OTHER"]
def _plan_symbol_not_text(item: Replacement) -> None:
item.audit["deep_dive_plan"]["selected_symbols"] = [1]
def _plan_budget_not_numeric(item: Replacement) -> None:
item.audit["deep_dive_plan"]["max_deep_dive_candidates"] = "many"
def _funnel_count_not_numeric(item: Replacement) -> None:
item.audit["universe"]["row_count"] = "many"
def _bad_enrichment_sha(item: Replacement) -> None:
item.audit["enrichment"]["artifact_sha256"] = "0" * 64
def _bad_pool_status(item: Replacement) -> None:
item.audit["candidate_pool_status"] = "unknown"
REJECTIONS = {
"bad-second-artifact-sha": (_bad_candidate_sha, "candidate-pool artifact SHA-256"),
"selected-symbols-wrong-type": (_selected_wrong_type, "selected_symbols"),
"deep-dive-plan-mismatch": (_plan_mismatch, "deep_dive_plan.selected_symbols"),
"deep-dive-symbol-not-text": (_plan_symbol_not_text, "malformed"),
"deep-dive-budget-not-numeric": (_plan_budget_not_numeric, "malformed"),
"funnel-count-not-numeric": (_funnel_count_not_numeric, "malformed"),
"bad-enrichment-sha": (_bad_enrichment_sha, "enrichment artifact SHA-256"),
"bad-pool-status": (_bad_pool_status, "candidate_pool_status"),
}
@pytest.mark.parametrize("name", REJECTIONS)
def test_rejected_replacement_changes_nothing(attached_run, replacement, capsys, name):
mutate, message = REJECTIONS[name]
mutate(replacement)
before = _snapshot(attached_run)
capsys.readouterr()
assert RUN_STATE.main(replacement.args(attached_run)) == 1
assert message in capsys.readouterr().err
assert _snapshot(attached_run) == before
assert RUN_STATE.main(["status", "--run-dir", str(attached_run), "--strict"]) == 0
def test_rejected_replacement_keeps_stored_hashes_valid(attached_run, replacement):
_bad_candidate_sha(replacement)
assert RUN_STATE.main(replacement.args(attached_run)) == 1
state = json.loads((attached_run / "run_state.json").read_text(encoding="utf-8"))
audit = state["screening_audit"]
assert _sha(attached_run / UNIVERSE_TARGET) == audit["universe"]["artifact_sha256"]
assert _sha(attached_run / CANDIDATE_TARGET) == audit["candidate_pool"]["artifact_sha256"]
def test_unreadable_audit_is_a_controlled_error(attached_run, replacement, capsys):
replacement.audit_path.write_bytes(b"\xff\xfe not utf-8")
before = _snapshot(attached_run)
capsys.readouterr()
args = _attach_args(
attached_run, replacement.audit_path, replacement.universe, replacement.candidate
)
assert RUN_STATE.main(args) == 1
assert "malformed" in capsys.readouterr().err
assert _snapshot(attached_run) == before
def test_copy_failure_preserves_checkpoint(attached_run, replacement, monkeypatch, capsys):
real_copyfile = shutil.copyfile
def fail_on_candidate(source, destination, **kwargs):
if Path(source) == replacement.candidate:
raise PermissionError("cannot read candidate artifact")
return real_copyfile(source, destination, **kwargs)
args = replacement.args(attached_run)
before = _snapshot(attached_run)
monkeypatch.setattr(RUN_STATE.shutil, "copyfile", fail_on_candidate)
capsys.readouterr()
assert RUN_STATE.main(args) == 1
assert "cannot read candidate artifact" in capsys.readouterr().err
assert _snapshot(attached_run) == before
def _fail_replace_once(monkeypatch, destination: Path, error: BaseException) -> None:
real_replace = os.replace
armed = {"value": True}
def replace(source, target):
if armed["value"] and Path(target) == destination:
armed["value"] = False
raise error
return real_replace(source, target)
monkeypatch.setattr(RUN_STATE.os, "replace", replace)
@pytest.mark.parametrize("target", COMMITTED_TARGETS)
def test_commit_failure_restores_every_file(attached_run, replacement, monkeypatch, capsys, target):
args = replacement.args(attached_run)
before = _snapshot(attached_run)
_fail_replace_once(monkeypatch, attached_run / target, OSError("disk full"))
capsys.readouterr()
assert RUN_STATE.main(args) == 1
err = capsys.readouterr().err
assert "disk full" in err
assert "restored" in err
assert _snapshot(attached_run) == before
assert not [path for path in attached_run.iterdir() if ".set-screening-audit" in path.name]
def test_commit_failure_on_first_attach_removes_new_files(tmp_path, monkeypatch):
run = tmp_path / "run"
_init(run)
before = _snapshot(run)
_fail_replace_once(monkeypatch, run / "run_state.json", OSError("disk full"))
args = _attach_args(
run,
ASSETS / "broad-screen-audit.example.json",
ASSETS / "universe-audit-results.example.jsonl",
ASSETS / "broad-screen-results.example.jsonl",
)
assert RUN_STATE.main(args) == 1
assert _snapshot(run) == before
def test_interrupt_during_commit_restores_and_propagates(attached_run, replacement, monkeypatch):
args = replacement.args(attached_run)
before = _snapshot(attached_run)
_fail_replace_once(monkeypatch, attached_run / "run_state.json", KeyboardInterrupt())
with pytest.raises(KeyboardInterrupt):
RUN_STATE.main(args)
assert _snapshot(attached_run) == before
def test_blocked_target_fails_cleanly_through_real_cli(attached_run, replacement):
# A real mid-commit failure: both artifacts are replaced before this path is reached.
audit_file = attached_run / "audit" / "broad-screen-audit.json"
audit_file.unlink()
audit_file.mkdir()
args = replacement.args(attached_run)
before = _snapshot(attached_run)
result = subprocess.run(
[sys.executable, str(SCRIPTS_DIR / "manage_run_state.py"), *args],
capture_output=True,
text=True,
check=False,
)
assert result.returncode == 1
assert result.stderr.startswith("ERROR: ")
assert "Traceback" not in result.stderr
assert _snapshot(attached_run) == before
def test_failed_rollback_reports_and_keeps_backups(attached_run, replacement, monkeypatch, capsys):
args = replacement.args(attached_run)
original_universe = (attached_run / UNIVERSE_TARGET).read_bytes()
real_replace = os.replace
run_state = attached_run / "run_state.json"
universe = attached_run / UNIVERSE_TARGET
def replace(source, target):
if Path(target) == run_state:
raise OSError("disk full")
if Path(target) == universe and Path(source).name.startswith("backup-"):
raise OSError("restore denied")
return real_replace(source, target)
monkeypatch.setattr(RUN_STATE.os, "replace", replace)
capsys.readouterr()
assert RUN_STATE.main(args) == 1
err = capsys.readouterr().err
assert "restore denied" in err
# Each failed restore names the backup file that holds the prior bytes.
assert "universe-audit-results.jsonl: restore denied" in err
assert "backup-0.tmp" in err
kept = [path for path in attached_run.iterdir() if ".set-screening-audit" in path.name]
assert len(kept) == 1
assert str(kept[0]) in err
backups = [path.read_bytes() for path in kept[0].glob("backup-*")]
assert original_universe in backups
# Kept backups must never leak into a published audit bundle.
assert not [path for path in kept[0].rglob("*") if BUNDLE._include(path, attached_run)]
@pytest.mark.parametrize(
"artifact_name",
["universe-audit-results.jsonl", "broad-screen-audit.json"],
ids=["other-artifact", "audit-document"],
)
def test_artifact_name_collision_is_rejected(attached_run, replacement, capsys, artifact_name):
# Letting one artifact overwrite another would break the stored SHA of the first.
enrichment = replacement.dir / artifact_name
enrichment.write_bytes(b"[]\n")
replacement.audit["enrichment"]["artifact_path"] = artifact_name
replacement.audit["enrichment"]["artifact_sha256"] = _sha(enrichment)
before = _snapshot(attached_run)
capsys.readouterr()
assert RUN_STATE.main(replacement.args(attached_run)) == 1
assert artifact_name in capsys.readouterr().err
assert _snapshot(attached_run) == before
def test_valid_replacement_stores_matching_hashes(attached_run, replacement):
events_before = (attached_run / "events.ndjson").read_bytes()
assert RUN_STATE.main(replacement.args(attached_run)) == 0
state = json.loads((attached_run / "run_state.json").read_text(encoding="utf-8"))
audit = state["screening_audit"]
saved = json.loads((attached_run / "audit/broad-screen-audit.json").read_text(encoding="utf-8"))
assert saved == audit
assert (attached_run / UNIVERSE_TARGET).read_bytes() == replacement.universe.read_bytes()
assert _sha(attached_run / UNIVERSE_TARGET) == audit["universe"]["artifact_sha256"]
errors: list[str] = []
for label in ("universe", "candidate_pool", "enrichment"):
PREPUBLISH._verify_artifact(audit[label], attached_run, label, errors)
assert errors == []
events = (attached_run / "events.ndjson").read_bytes()
assert events.startswith(events_before)
assert json.loads(events[len(events_before) :])["event"] == "screening_audit_attached"
assert RUN_STATE.main(["status", "--run-dir", str(attached_run), "--strict"]) == 0
assert not [path for path in attached_run.iterdir() if ".set-screening-audit" in path.name]
def test_reattaching_from_the_runs_own_artifacts_succeeds(attached_run):
# Source and destination are the same file; staging must not trip over that.
args = _attach_args(
attached_run,
ASSETS / "broad-screen-audit.example.json",
attached_run / UNIVERSE_TARGET,
attached_run / CANDIDATE_TARGET,
)
assert RUN_STATE.main(args) == 0
assert RUN_STATE.main(["status", "--run-dir", str(attached_run), "--strict"]) == 0
def _staging_dirs(run: Path) -> list[Path]:
return [path for path in run.iterdir() if ".set-screening-audit" in path.name]
def _lost_names(run: Path, before: dict[str, bytes]) -> list[str]:
"""Names whose prior bytes are neither live nor in a kept backup."""
backups = [
path.read_bytes() for staging in _staging_dirs(run) for path in staging.glob("backup-*")
]
lost = []
for name, data in before.items():
live = run / name
if not (live.is_file() and live.read_bytes() == data) and data not in backups:
lost.append(name)
return lost
def test_interrupt_during_restore_keeps_prior_bytes(attached_run, replacement, monkeypatch):
args = replacement.args(attached_run)
before = _snapshot(attached_run)
real_replace = os.replace
run_state = attached_run / "run_state.json"
events = attached_run / "events.ndjson"
def replace(source, target):
if Path(target) == run_state:
raise OSError("disk full")
if Path(target) == events and Path(source).name.startswith("backup-"):
raise KeyboardInterrupt() # the first restore that really moves a file
return real_replace(source, target)
monkeypatch.setattr(RUN_STATE.os, "replace", replace)
with pytest.raises(KeyboardInterrupt):
RUN_STATE.main(args)
assert _lost_names(attached_run, before) == []
# The interrupt must not skip the remaining files.
for name, data in before.items():
if name != "events.ndjson":
assert (attached_run / name).read_bytes() == data
assert len(_staging_dirs(attached_run)) == 1
@pytest.mark.parametrize("target", COMMITTED_TARGETS)
def test_interrupt_after_rename_restores_snapshot(attached_run, replacement, monkeypatch, target):
args = replacement.args(attached_run)
before = _snapshot(attached_run)
real_replace = os.replace
live = attached_run / target
armed = {"value": True}
def replace(source, destination):
result = real_replace(source, destination)
if armed["value"] and Path(destination) == live:
armed["value"] = False
raise KeyboardInterrupt()
return result
monkeypatch.setattr(RUN_STATE.os, "replace", replace)
with pytest.raises(KeyboardInterrupt):
RUN_STATE.main(args)
assert _snapshot(attached_run) == before
def test_interrupt_after_rename_of_new_file_removes_it(tmp_path, monkeypatch):
run = tmp_path / "run"
_init(run)
before = _snapshot(run)
real_replace = os.replace
live = run / UNIVERSE_TARGET
def replace(source, destination):
result = real_replace(source, destination)
if Path(destination) == live:
raise KeyboardInterrupt()
return result
monkeypatch.setattr(RUN_STATE.os, "replace", replace)
args = _attach_args(
run,
ASSETS / "broad-screen-audit.example.json",
ASSETS / "universe-audit-results.example.jsonl",
ASSETS / "broad-screen-results.example.jsonl",
)
with pytest.raises(KeyboardInterrupt):
RUN_STATE.main(args)
assert _snapshot(run) == before
def _run_cli(args: list[str]) -> subprocess.CompletedProcess:
return subprocess.run(
[sys.executable, str(SCRIPTS_DIR / "manage_run_state.py"), *args],
capture_output=True,
text=True,
check=False,
)
def _assert_controlled_error(result: subprocess.CompletedProcess) -> None:
assert result.returncode == 1
assert result.stderr.startswith("ERROR: ")
assert "Traceback" not in result.stderr
def test_infinite_count_is_a_controlled_error(attached_run, replacement):
# json.loads accepts the bare token Infinity; int(inf) raises OverflowError.
replacement.audit["universe"]["row_count"] = 987654321
text = json.dumps(replacement.audit).replace("987654321", "Infinity", 1)
replacement.audit_path.write_text(text, encoding="utf-8")
assert "Infinity" in text
before = _snapshot(attached_run)
args = _attach_args(
attached_run, replacement.audit_path, replacement.universe, replacement.candidate
)
result = _run_cli(args)
_assert_controlled_error(result)
assert _snapshot(attached_run) == before
def test_deeply_nested_audit_is_a_controlled_error(attached_run, replacement):
replacement.audit_path.write_text("[" * 200000 + "]" * 200000, encoding="utf-8")
before = _snapshot(attached_run)
args = _attach_args(
attached_run, replacement.audit_path, replacement.universe, replacement.candidate
)
result = _run_cli(args)
_assert_controlled_error(result)
assert _snapshot(attached_run) == before
def test_list_budget_is_a_controlled_error_through_cli(attached_run, replacement):
replacement.audit["deep_dive_plan"]["max_deep_dive_candidates"] = [1]
before = _snapshot(attached_run)
result = _run_cli(replacement.args(attached_run))
_assert_controlled_error(result)
assert "malformed" in result.stderr
assert _snapshot(attached_run) == before
@pytest.mark.parametrize(
"artifact_name", ["UNIVERSE-AUDIT-RESULTS.JSONL", "BROAD-SCREEN-AUDIT.JSON"]
)
def test_case_only_name_collision_is_rejected(attached_run, replacement, capsys, artifact_name):
# On a case-insensitive filesystem these names would overwrite each other.
enrichment = replacement.dir / artifact_name
enrichment.write_bytes(b"[]\n")
replacement.audit["enrichment"]["artifact_path"] = artifact_name
replacement.audit["enrichment"]["artifact_sha256"] = _sha(enrichment)
before = _snapshot(attached_run)
capsys.readouterr()
assert RUN_STATE.main(replacement.args(attached_run)) == 1
assert "collides" in capsys.readouterr().err
assert _snapshot(attached_run) == before
def test_replaced_files_keep_their_mode(attached_run, replacement):
for target in COMMITTED_TARGETS:
(attached_run / target).chmod(0o600)
assert RUN_STATE.main(replacement.args(attached_run)) == 0
for target in COMMITTED_TARGETS:
assert stat.S_IMODE((attached_run / target).stat().st_mode) == 0o600, target
def test_new_files_keep_default_mode(tmp_path):
run = tmp_path / "run"
_init(run)
reference = tmp_path / "reference"
reference.write_text("x", encoding="utf-8")
args = _attach_args(
run,
ASSETS / "broad-screen-audit.example.json",
ASSETS / "universe-audit-results.example.jsonl",
ASSETS / "broad-screen-results.example.jsonl",
)
assert RUN_STATE.main(args) == 0
expected = stat.S_IMODE(reference.stat().st_mode)
assert stat.S_IMODE((run / UNIVERSE_TARGET).stat().st_mode) == expected
@pytest.mark.skipif(not hasattr(os, "chflags"), reason="needs BSD file flags")
def test_immutable_live_file_leaves_no_staging(attached_run, replacement, capsys):
args = replacement.args(attached_run)
before = _snapshot(attached_run)
events = attached_run / "events.ndjson"
os.chflags(events, stat.UF_IMMUTABLE)
capsys.readouterr()
try:
assert RUN_STATE.main(args) == 1
assert _snapshot(attached_run) == before
# copyfile does not carry the flag, so the backups can be removed.
assert _staging_dirs(attached_run) == []
finally:
os.chflags(events, 0)
for staging in _staging_dirs(attached_run):
for path in staging.rglob("*"):
os.chflags(path, 0)
def test_unremovable_staging_is_reported_without_implying_cleanup(
attached_run, replacement, monkeypatch, capsys
):
real_rmtree = shutil.rmtree
monkeypatch.setattr(RUN_STATE.shutil, "rmtree", lambda *args, **kwargs: None)
capsys.readouterr()
assert RUN_STATE.main(replacement.args(attached_run)) == 0
leftovers = _staging_dirs(attached_run)
assert len(leftovers) == 1
err = capsys.readouterr().err
assert "warning" in err.lower()
assert str(leftovers[0]) in err
real_rmtree(leftovers[0])
def _share_artifact(item: Replacement, generation_path: str, generation_file: Path) -> Path:
shared = item.dir / "shared-artifact.jsonl"
shared.write_bytes(b"[]\n")
item.audit["enrichment"].update({"artifact_path": shared.name, "artifact_sha256": _sha(shared)})
generation = dict(item.audit["candidate_pool"].get("generation_audit") or {})
generation.update(artifact_path=generation_path, artifact_sha256=_sha(generation_file))
item.audit["candidate_pool"]["generation_audit"] = generation
return shared
def test_enrichment_and_generation_may_share_one_file(attached_run, replacement):
# Same file and same SHA under two labels is one artifact, not a collision.
shared = replacement.dir / "shared-artifact.jsonl"
shared.write_bytes(b"[]\n")
_share_artifact(replacement, shared.name, shared)
assert RUN_STATE.main(replacement.args(attached_run)) == 0
state = json.loads((attached_run / "run_state.json").read_text(encoding="utf-8"))
audit = state["screening_audit"]
assert audit["enrichment"]["artifact_path"] == "audit/shared-artifact.jsonl"
generation = audit["candidate_pool"]["generation_audit"]
assert generation["artifact_path"] == "audit/shared-artifact.jsonl"
assert (attached_run / "audit/shared-artifact.jsonl").read_bytes() == b"[]\n"
def test_same_name_with_different_content_is_rejected(attached_run, replacement, capsys):
other = replacement.dir / "other" / "shared-artifact.jsonl"
other.parent.mkdir()
other.write_bytes(b"[1]\n")
_share_artifact(replacement, "other/shared-artifact.jsonl", other)
before = _snapshot(attached_run)
capsys.readouterr()
assert RUN_STATE.main(replacement.args(attached_run)) == 1
assert "shared-artifact.jsonl" in capsys.readouterr().err
assert _snapshot(attached_run) == before
def test_verification_hashes_the_staged_copy(attached_run, replacement, monkeypatch):
original = replacement.universe.read_bytes()
real_copyfile = shutil.copyfile
def copy_then_change_source(source, destination, **kwargs):
result = real_copyfile(source, destination, **kwargs)
if Path(source) == replacement.universe:
replacement.universe.write_bytes(b"changed after copy\n")
return result
args = replacement.args(attached_run)
monkeypatch.setattr(RUN_STATE.shutil, "copyfile", copy_then_change_source)
assert RUN_STATE.main(args) == 0
assert (attached_run / UNIVERSE_TARGET).read_bytes() == original
def test_success_updates_timestamp_and_status(attached_run, replacement):
state_path = attached_run / "run_state.json"
state = json.loads(state_path.read_text(encoding="utf-8"))
state["status"] = "complete"
state["updated_at"] = "2000-01-01T00:00:00+00:00"
state_path.write_text(json.dumps(state), encoding="utf-8")
assert RUN_STATE.main(replacement.args(attached_run)) == 0
after = json.loads(state_path.read_text(encoding="utf-8"))
assert after["status"] == "partial"
assert after["updated_at"] > "2000-01-01T00:00:00+00:00"
@pytest.mark.parametrize("changed_audit", [False, True], ids=["same-audit", "new-evidence"])
def test_verified_checkpoint_rejects_audit_reattachment(
attached_run, replacement, tmp_path, capsys, changed_audit
):
example = json.loads((ASSETS / "candidate-input.example.json").read_text(encoding="utf-8"))
candidate = tmp_path / "ACME.json"
candidate.write_text(json.dumps(example["candidates"][0]), encoding="utf-8")
assert (
RUN_STATE.main(
[
"save-candidate",
"--run-dir",
str(attached_run),
"--candidate",
str(candidate),
"--stage",
"verified",
]
)
== 0
)
assert RUN_STATE.main(["status", "--run-dir", str(attached_run), "--strict"]) == 0
before = _snapshot(attached_run)
args = (
replacement.args(attached_run)
if changed_audit
else _attach_args(
attached_run,
ASSETS / "broad-screen-audit.example.json",
ASSETS / "universe-audit-results.example.jsonl",
ASSETS / "broad-screen-results.example.jsonl",
)
)
capsys.readouterr()
assert RUN_STATE.main(args) == 1
error = capsys.readouterr().err
assert "ERROR:" in error
assert "verified candidates" in error
assert "new run" in error
assert _snapshot(attached_run) == before
assert not _staging_dirs(attached_run)
assert RUN_STATE.main(["status", "--run-dir", str(attached_run), "--strict"]) == 0
def test_draft_checkpoint_allows_audit_replacement(attached_run, replacement, tmp_path):
example = json.loads((ASSETS / "candidate-input.example.json").read_text(encoding="utf-8"))
candidate = tmp_path / "ACME.json"
candidate.write_text(json.dumps(example["candidates"][0]), encoding="utf-8")
assert (
RUN_STATE.main(
[
"save-candidate",
"--run-dir",
str(attached_run),
"--candidate",
str(candidate),
"--stage",
"draft",
]
)
== 0
)
saved = (attached_run / "candidates" / "ACME.json").read_bytes()
assert RUN_STATE.main(replacement.args(attached_run)) == 0
assert (attached_run / "candidates" / "ACME.json").read_bytes() == saved
state = json.loads((attached_run / "run_state.json").read_text(encoding="utf-8"))
assert state["candidates"]["ACME"]["stage"] == "draft"
assert "ACME" in state["unprocessed_candidates"]
assert state["screening_funnel"]["deep_dive_completed_count"] == 0
assert RUN_STATE.main(["status", "--run-dir", str(attached_run), "--strict"]) == 0
def _save(run: Path, candidate: Path, stage: str, *extra: str) -> int:
return RUN_STATE.main(
[
"save-candidate",
"--run-dir",
str(run),
"--candidate",
str(candidate),
"--stage",
stage,
*extra,
]
)
def _candidate_file(tmp_path: Path, symbol: str = "ACME") -> Path:
example = json.loads((ASSETS / "candidate-input.example.json").read_text(encoding="utf-8"))
data = dict(example["candidates"][0])
data["identity"] = {**data["identity"], "symbol": symbol}
candidate = tmp_path / f"{symbol}.json"
candidate.write_text(json.dumps(data), encoding="utf-8")
return candidate
def _pending_audit(item: Replacement) -> None:
item.audit["candidate_pool_status"] = "sufficient_pending_enrichment"
item.audit["selection_outcome"] = "selected_pending_enrichment"
def test_pending_pool_workflow_is_recoverable(tmp_path, capsys):
run = tmp_path / "run"
_init(run)
pending = Replacement(tmp_path)
_pending_audit(pending)
assert RUN_STATE.main(pending.args(run)) == 0
candidate = _candidate_file(tmp_path)
assert _save(run, candidate, "draft") == 0
before = _snapshot(run)
capsys.readouterr()
assert _save(run, candidate, "verified") == 1
error = capsys.readouterr().err
assert "ERROR:" in error
assert "sufficient_pending_enrichment" in error
assert "draft" in error
assert _snapshot(run) == before
args = _attach_args(
run,
ASSETS / "broad-screen-audit.example.json",
ASSETS / "universe-audit-results.example.jsonl",
ASSETS / "broad-screen-results.example.jsonl",
)
assert RUN_STATE.main(args) == 0
for symbol in ("ACME", "CYCLE", "STALE"):
assert _save(run, _candidate_file(tmp_path, symbol), "verified") == 0
assert RUN_STATE.main(["set-status", "--run-dir", str(run), "complete"]) == 0
def test_verified_candidate_cannot_be_downgraded_to_draft(attached_run, tmp_path, capsys):
candidate = _candidate_file(tmp_path)
assert _save(attached_run, candidate, "verified") == 0
before = _snapshot(attached_run)
capsys.readouterr()
assert _save(attached_run, candidate, "draft") == 1
error = capsys.readouterr().err
assert "ERROR:" in error
assert "downgrade" in error
assert _snapshot(attached_run) == before
def test_reattach_guard_blocks_verified_candidate_outside_selection(
attached_run, replacement, tmp_path, capsys
):
outside = _candidate_file(tmp_path, "OUTSIDE")
assert _save(attached_run, outside, "verified", "--allow-unselected") == 0
before = _snapshot(attached_run)
capsys.readouterr()
assert RUN_STATE.main(replacement.args(attached_run)) == 1
assert "verified candidates" in capsys.readouterr().err
assert _snapshot(attached_run) == before
scripts/tests/test_seed_and_probe.py
"""Tests for v3.6.1 workstream A: seed apportionment, dynamic seed limit,
quality probe, FCF prefilter exclusion, cyclicality keywords, and the honest
scope fields surfaced through run-summary.json.
"""
from __future__ import annotations
import importlib.util
import random
import sys
import tempfile
import unittest
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
SCRIPTS = Path(__file__).resolve().parents[1]
if str(SCRIPTS) not in sys.path:
sys.path.insert(0, str(SCRIPTS))
def load_module(name: str, path: Path):
spec = importlib.util.spec_from_file_location(name, path)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
sys.modules[name] = module
spec.loader.exec_module(module)
return module
PIPELINE = load_module("run_pipeline_seed_probe_test", SCRIPTS / "run_pipeline.py")
POOL = load_module("build_pool_seed_probe_test", SCRIPTS / "build_provider_prefilter_pool.py")
def _listing_row(
symbol: str,
*,
sector: str,
market_cap: float,
price: float = 20.0,
volume: float = 100_000.0,
) -> dict[str, Any]:
return {
"symbol": symbol,
"sector": sector,
"market_cap": market_cap,
"price": price,
"volume": volume,
}
class QuotaApportionmentTests(unittest.TestCase):
def test_quota_sums_to_limit_and_is_order_independent(self) -> None:
sizes = {
("technology", "500000000-1000000000"): 40,
("industrials", "1000000000-2000000000"): 12,
("healthcare", "2000000000-5000000000"): 3,
("energy", "5000000000-10000000000"): 1,
}
quotas_a = PIPELINE._apportion_quota(sizes, 30)
# Rebuild the same sizes dict with a shuffled insertion order; the
# apportionment must not depend on Python dict iteration order.
items = list(sizes.items())
random.Random(7).shuffle(items)
quotas_b = PIPELINE._apportion_quota(dict(items), 30)
self.assertEqual(sum(quotas_a.values()), 30)
self.assertEqual(quotas_a, quotas_b)
for key, count in quotas_a.items():
self.assertGreaterEqual(count, 1)
self.assertLessEqual(count, sizes[key])
def test_quota_below_cell_count_allocates_one_per_largest_cells(self) -> None:
sizes = {
("a", "bucket"): 10,
("b", "bucket"): 5,
("c", "bucket"): 1,
}
quotas = PIPELINE._apportion_quota(sizes, 2)
self.assertEqual(sum(quotas.values()), 2)
self.assertEqual(quotas[("a", "bucket")], 1)
self.assertEqual(quotas[("b", "bucket")], 1)
self.assertEqual(quotas[("c", "bucket")], 0)
class DiversifiedSeedTests(unittest.TestCase):
def _rows(self) -> list[dict[str, Any]]:
rows = []
for idx in range(40):
sector = "Technology" if idx % 2 == 0 else "Industrials"
rows.append(
_listing_row(
f"SYM{idx:03d}",
sector=sector,
market_cap=700_000_000 + idx * 10_000_000,
price=10.0 + (idx % 5),
volume=100_000 + idx * 1_000,
)
)
return rows
def test_within_cell_ranking_prefers_dollar_volume_never_ticker(self) -> None:
rows = self._rows()
selected_a, audit_a = PIPELINE.diversified_seed(rows, 10, run_salt="2026-08-29")
shuffled = list(rows)
random.Random(11).shuffle(shuffled)
selected_b, audit_b = PIPELINE.diversified_seed(shuffled, 10, run_salt="2026-08-29")
symbols_a = sorted(PIPELINE._symbol(row) for row in selected_a)
symbols_b = sorted(PIPELINE._symbol(row) for row in selected_b)
self.assertEqual(symbols_a, symbols_b)
self.assertEqual(audit_a["cell_count"], audit_b["cell_count"])
self.assertEqual(audit_a["quota_method"], "sqrt_hamilton")
self.assertEqual(audit_a["alphabetic_tie_break_used_count"], 0)
def test_ties_use_hash_not_ticker(self) -> None:
# Two rows in the same cell, identical price/volume/market-cap/missing
# counts: the tie can only be broken by the salted hash.
rows = [
_listing_row(
"ZZZZ", sector="Technology", market_cap=800_000_000, price=10, volume=1000
),
_listing_row(
"AAAA", sector="Technology", market_cap=800_000_000, price=10, volume=1000
),
]
selected, audit = PIPELINE.diversified_seed(rows, 1, run_salt="salt-1")
self.assertEqual(len(selected), 1)
self.assertEqual(audit["hash_tie_break_used_count"], 1)
# A different salt may pick the other row, but must remain deterministic
# for a fixed salt across shuffled input order.
reselected, _ = PIPELINE.diversified_seed(list(reversed(rows)), 1, run_salt="salt-1")
self.assertEqual(PIPELINE._symbol(selected[0]), PIPELINE._symbol(reselected[0]))
def test_economic_metrics_available_flips_selection_basis(self) -> None:
rows = self._rows()
for row in rows:
row["eps_growth_pct"] = 12.0
selected, audit = PIPELINE.diversified_seed(rows, 10, run_salt="2026-08-29")
self.assertTrue(audit["economic_metrics_available_for_seed"])
self.assertEqual(audit["seed_selection_basis"], "stratified_economic_score")
rows_no_econ = self._rows()
_, audit_no_econ = PIPELINE.diversified_seed(rows_no_econ, 10, run_salt="2026-08-29")
self.assertFalse(audit_no_econ["economic_metrics_available_for_seed"])
self.assertEqual(audit_no_econ["seed_selection_basis"], "stratified_liquidity_proxy")
class DynamicSeedLimitTests(unittest.TestCase):
def test_effective_limit_is_bounded_by_remaining_budget(self) -> None:
effective = PIPELINE.compute_effective_seed_limit(
pre_enrichment_limit=180,
seed_limit_cap=200,
max_api_calls=350,
api_calls_made=0,
quality_probe_limit=35,
exact_liquidity_limit=40,
candidate_packet_reserve_calls=30,
retry_reserve_calls=25,
)
# reserved = 35*2+40+30+25 = 165 (probe = key metrics + annual income
# statement); remaining = 350; 350-165 = 185; min(180, 200, 185) = 180
self.assertEqual(effective, 180)
def test_effective_limit_shrinks_with_calls_already_made(self) -> None:
effective = PIPELINE.compute_effective_seed_limit(
pre_enrichment_limit=180,
seed_limit_cap=200,
max_api_calls=350,
api_calls_made=165,
quality_probe_limit=35,
exact_liquidity_limit=40,
candidate_packet_reserve_calls=30,
retry_reserve_calls=25,
)
# remaining = 185; reserved = 165; 185-165 = 20
self.assertEqual(effective, 20)
def test_effective_limit_below_20_raises(self) -> None:
with self.assertRaises(ValueError) as ctx:
PIPELINE.compute_effective_seed_limit(
pre_enrichment_limit=180,
seed_limit_cap=200,
max_api_calls=350,
api_calls_made=166,
quality_probe_limit=35,
exact_liquidity_limit=40,
candidate_packet_reserve_calls=30,
retry_reserve_calls=25,
)
self.assertIn("estimate seed budget insufficient", str(ctx.exception))
class QualityProbeTests(unittest.TestCase):
class FakeClient:
def __init__(self, payloads: dict[str, list[dict[str, Any]]]):
self.payloads = payloads
self.calls: list[str] = []
def get_key_metrics_ttm(self, symbol: str):
self.calls.append(symbol)
return self.payloads.get(symbol, [])
def test_probe_maps_fields_and_tolerates_failing_symbol(self) -> None:
client = self.FakeClient(
{
"AAA": [
{
"returnOnInvestedCapitalTTM": 0.20,
"freeCashFlowYieldTTM": 0.08,
"evToFreeCashFlowTTM": 12.5,
"netDebtToEBITDATTM": 1.5,
"stockBasedCompensationToRevenueTTM": 0.03,
# revenue = EV / (EV/Sales) = 1.2B / 2.0 = 600M
"enterpriseValueTTM": 1_200_000_000,
"evToSalesTTM": 2.0,
}
],
# BBB has no payload -> unresolved, must not raise.
}
)
rows = [{"symbol": "AAA", "market_cap": 1_000_000_000}, {"symbol": "BBB"}]
output, audit = PIPELINE.apply_quality_probe(
client, rows, target_symbols=["AAA", "BBB"], source_id="fmp-key-metrics-ttm-2026-08-29"
)
by_symbol = {row["symbol"]: row for row in output}
self.assertEqual(by_symbol["AAA"]["roic_pct"], 20.0)
self.assertEqual(by_symbol["AAA"]["fcf_yield_pct"], 8.0)
self.assertEqual(by_symbol["AAA"]["ev_to_fcf"], 12.5)
self.assertEqual(by_symbol["AAA"]["net_debt_to_ebitda"], 1.5)
self.assertEqual(by_symbol["AAA"]["sbc_revenue_pct"], 3.0)
# (FCF - SBC) / mktcap = 8% - 3% * 600M / 1,000M = 8% - 1.8% = 6.2%
self.assertAlmostEqual(by_symbol["AAA"]["sbc_adjusted_fcf_yield_pct"], 6.2, places=6)
self.assertTrue(by_symbol["AAA"]["quality_probe_resolved"])
self.assertFalse(by_symbol["BBB"]["quality_probe_resolved"])
self.assertTrue(by_symbol["BBB"]["quality_probe_attempted"])
self.assertEqual(audit["calls_used"], 2)
self.assertEqual(sorted(audit["resolved"]), ["AAA"])
def test_rows_outside_target_symbols_are_untouched(self) -> None:
client = self.FakeClient({})
rows = [{"symbol": "CCC"}]
output, audit = PIPELINE.apply_quality_probe(
client, rows, target_symbols=[], source_id="src"
)
self.assertFalse(output[0]["quality_probe_attempted"])
self.assertEqual(audit["calls_used"], 0)
self.assertEqual(client.calls, [])
class FcfPrefilterTests(unittest.TestCase):
def _liquid_row(self, symbol: str, **overrides: Any) -> dict[str, Any]:
row = {
"symbol": symbol,
"price": 20.0,
"average_volume": 500_000.0,
"average_daily_dollar_volume_method": "price_x_20d_average_volume",
"average_volume_period_days": 20,
"liquidity_source_ids": ["fmp-historical-eod-2026-08-29"],
"forward_pe": 15.0,
"eps_growth_pct": 15.0,
"revenue_growth_pct": 12.0,
"analyst_count": 5,
"roic_pct": 12.0,
"cyclicality_score": 2,
}
row.update(overrides)
return row
def test_low_fcf_yield_excludes_row_from_non_high_growth_lanes(self) -> None:
row = self._liquid_row(
"AEIS", fcf_yield_pct=0.5, quality_probe_resolved=True, net_debt_to_ebitda=1.0
)
pool, audit = POOL.build_pool(
universe_rows=[row],
lane_rows={
"core_garp": [row],
"high_growth_exception": [],
"quality_near_miss": [],
"cyclical_normalization": [],
},
analysis_as_of="2026-08-29T00:00:00+00:00",
source_ids=["src"],
per_lane=10,
max_pool=10,
minimum_pool=1,
requested_min_market_cap=500_000_000,
requested_max_market_cap=20_000_000_000,
provider_exhausted=True,
)
self.assertNotIn("AEIS", {item["symbol"] for item in pool})
self.assertIn("AEIS", audit["fcf_prefilter_excluded_symbols"])
def test_low_fcf_yield_in_high_growth_lane_is_tagged_not_excluded(self) -> None:
row = self._liquid_row(
"GROW", fcf_yield_pct=0.5, quality_probe_resolved=True, net_debt_to_ebitda=1.0
)
pool, audit = POOL.build_pool(
universe_rows=[row],
lane_rows={
"core_garp": [],
"high_growth_exception": [row],
"quality_near_miss": [],
"cyclical_normalization": [],
},
analysis_as_of="2026-08-29T00:00:00+00:00",
source_ids=["src"],
per_lane=10,
max_pool=10,
minimum_pool=1,
requested_min_market_cap=500_000_000,
requested_max_market_cap=20_000_000_000,
provider_exhausted=True,
)
by_symbol = {item["symbol"]: item for item in pool}
self.assertIn("GROW", by_symbol)
self.assertIn("weak_fcf_support", by_symbol["GROW"].get("provider_prefilter_flags", []))
self.assertNotIn("GROW", audit["fcf_prefilter_excluded_symbols"])
def test_unresolved_probe_does_not_trigger_exclusion(self) -> None:
row = self._liquid_row("UNSET", fcf_yield_pct=0.1, quality_probe_resolved=False)
pool, audit = POOL.build_pool(
universe_rows=[row],
lane_rows={
"core_garp": [row],
"high_growth_exception": [],
"quality_near_miss": [],
"cyclical_normalization": [],
},
analysis_as_of="2026-08-29T00:00:00+00:00",
source_ids=["src"],
per_lane=10,
max_pool=10,
minimum_pool=1,
requested_min_market_cap=500_000_000,
requested_max_market_cap=20_000_000_000,
provider_exhausted=True,
)
self.assertIn("UNSET", {item["symbol"] for item in pool})
self.assertEqual(audit["fcf_prefilter_excluded_symbols"], [])
class CyclicalityTests(unittest.TestCase):
def test_gold_miner_scores_four(self) -> None:
self.assertEqual(PIPELINE.classify_cyclicality("Basic Materials", "Gold"), 4)
def test_mining_keywords_score_four(self) -> None:
# "Coal" is intentionally excluded here: it already scores 5 via the
# dry-bulk/steel/coal/iron-ore tier, checked before the score-4 tier.
for industry in ("Copper", "Silver", "Uranium", "Other Precious Metals"):
self.assertEqual(
PIPELINE.classify_cyclicality("Basic Materials", industry), 4, industry
)
def test_coal_scores_five_via_higher_tier(self) -> None:
self.assertEqual(PIPELINE.classify_cyclicality("Energy", "Coal"), 5)
def test_semiconductor_equipment_scores_three(self) -> None:
self.assertEqual(PIPELINE.classify_cyclicality("Technology", "Semiconductor Equipment"), 3)
def test_aluminum_scores_three(self) -> None:
self.assertEqual(PIPELINE.classify_cyclicality("Basic Materials", "Aluminum"), 3)
class LaneTroughRecoveryTests(unittest.TestCase):
def test_trough_recovery_moves_row_out_of_core_garp_into_near_miss(self) -> None:
config = PIPELINE.load_config(None)
row = {
"forward_pe": 15.0,
"eps_growth_pct": 15.0,
"revenue_growth_pct": 12.0,
"cyclicality_score": 2,
"growth_pattern": "trough_recovery",
}
lanes = PIPELINE.lane_memberships(row, config)
self.assertNotIn("core_garp", lanes)
self.assertIn("quality_near_miss", lanes)
def test_missing_growth_pattern_is_treated_as_none(self) -> None:
config = PIPELINE.load_config(None)
row = {
"forward_pe": 15.0,
"eps_growth_pct": 15.0,
"revenue_growth_pct": 12.0,
"cyclicality_score": 2,
}
lanes = PIPELINE.lane_memberships(row, config)
self.assertIn("core_garp", lanes)
class ForeignPrivateIssuerTests(unittest.TestCase):
def test_non_us_country_flags_fpi(self) -> None:
self.assertTrue(PIPELINE._is_foreign_private_issuer({"country": "GB"}))
def test_us_country_is_not_fpi(self) -> None:
self.assertFalse(PIPELINE._is_foreign_private_issuer({"country": "US"}))
def test_isin_prefix_takes_precedence(self) -> None:
self.assertTrue(
PIPELINE._is_foreign_private_issuer({"isin": "GB0002634946", "country": "US"})
)
self.assertFalse(
PIPELINE._is_foreign_private_issuer({"isin": "US0378331005", "country": "GB"})
)
class ScopeFieldsTests(unittest.TestCase):
def test_run_summary_carries_honest_scope_fields(self) -> None:
from test_claude_code_pipeline import FakePipelineClient
config = PIPELINE.load_config(None)
config.update(
{
"company_screener_limit": 100,
"pre_enrichment_limit": 20,
"exact_liquidity_limit": 20,
"provider_prefilter_pool_size": 12,
"provider_prefilter_minimum_pool": 8,
"provider_prefilter_per_lane": 4,
"max_deep_dive_candidates": 3,
"bulk_estimate_minimum_coverage_pct": 10.0,
}
)
with tempfile.TemporaryDirectory() as temp:
output = Path(temp) / "run-direct"
result = PIPELINE.execute_pipeline(
FakePipelineClient(),
config,
analysis_as_of=datetime(2026, 8, 29, 17, tzinfo=timezone.utc),
output_dir=output,
include_packets=False,
)
summary = result.summary
for key in (
"listing_enumeration_complete",
"economic_screen_scope_complete",
"listing_universe_count",
"estimate_seed_count",
"estimate_seed_coverage_pct",
"valid_estimate_count",
"valid_estimate_coverage_pct",
"scope_complete_deprecated_note",
):
self.assertIn(key, summary, key)
self.assertTrue(summary["economic_screen_scope_complete"])
self.assertEqual(summary["listing_enumeration_complete"], summary["scope_complete"])
next_action = __import__("json").loads(
(output / "NEXT_ACTION.json").read_text(encoding="utf-8")
)
for key in (
"listing_enumeration_complete",
"economic_screen_scope_complete",
"listing_universe_count",
"estimate_seed_count",
"valid_estimate_count",
):
self.assertIn(key, next_action, key)
discovery_audit = __import__("json").loads(
(output / "audit" / "provider-prefilter-audit.json").read_text(encoding="utf-8")
)
self.assertTrue(discovery_audit["listing_provider_exhausted"])
self.assertTrue(discovery_audit["estimate_seed_exhausted"])
self.assertIn("quality_probe", discovery_audit)
self.assertEqual(discovery_audit["provider_exhausted_scope"], "estimate_seed")
if __name__ == "__main__":
unittest.main()
scripts/tests/test_selection_lanes.py
"""v3.6.1 selection-lane behaviour in screen_universe."""
from __future__ import annotations
import importlib.util
import sys
import unittest
from pathlib import Path
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
if str(SCRIPTS_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPTS_DIR))
_spec = importlib.util.spec_from_file_location(
"screen_universe", SCRIPTS_DIR / "screen_universe.py"
)
SCREEN = importlib.util.module_from_spec(_spec)
assert _spec.loader is not None
_spec.loader.exec_module(SCREEN)
def _row(symbol, lane, priority, sector, *, growth_pattern=None):
pre = {
"core": "passed",
"growth": "passed_exception",
"near": "near_miss_review",
"cycle": "passed",
}[lane]
cyclicality = 4 if lane == "cycle" else 1
requirements = ["mid_cycle_normalization_required"] if lane == "cycle" else []
metrics = {"analyst_count": 5, "cyclicality_score": cyclicality}
if growth_pattern:
metrics["growth_pattern"] = growth_pattern
return {
"symbol": symbol,
"sector": sector,
"deep_dive_priority_score": priority,
"broad_score": priority,
"fundamental_completeness_count": 8,
"metrics": metrics,
"decision": {
"preselection_status": pre,
"status": pre,
"exception_admitted": lane == "growth",
"deep_dive_requirements": requirements,
},
}
PLAN = {
"selection_lane_quota_core_garp": 2,
"selection_lane_quota_high_growth": 1,
"selection_lane_quota_near_miss": 1,
"selection_lane_quota_cyclical": 1,
"maximum_selected_per_sector": 2,
}
class SelectionLaneTests(unittest.TestCase):
def test_trough_recovery_is_never_core_garp(self) -> None:
row = _row("YELP", "core", 90, "Communication", growth_pattern="trough_recovery")
self.assertEqual(SCREEN._selection_lane(row), "quality_near_miss")
steady = _row("STDY", "core", 90, "Communication", growth_pattern="steady")
self.assertEqual(SCREEN._selection_lane(steady), "core_garp")
def test_small_budget_ranks_by_priority_with_lane_caps(self) -> None:
# Three top-priority cyclicals, then a core name, then near-miss names.
rows = [
_row("BTG", "cycle", 105, "Materials"),
_row("FSM", "cycle", 102, "Materials"),
_row("CENX", "cycle", 99, "Materials"),
_row("YELP", "core", 100, "Communication"),
_row("JAZZ", "near", 84, "Healthcare"),
_row("DOCS", "near", 83, "Healthcare"),
]
selected, counts = SCREEN._select_multilane(rows, limit=3, config=PLAN)
symbols = [row["symbol"] for row in selected]
# Best cyclical takes its single-slot cap; the lane-first fill used to
# hand all three slots to core/near-miss and defer BTG/FSM/CENX.
self.assertIn("BTG", symbols)
self.assertIn("YELP", symbols)
self.assertEqual(counts["cyclical_normalization"], 1)
self.assertNotIn("FSM", symbols) # cyclical cap is 1
self.assertEqual(len(symbols), 3)
def test_full_budget_keeps_lane_first_plan(self) -> None:
rows = [
_row("C1", "core", 100, "Technology"),
_row("C2", "core", 99, "Industrials"),
_row("G1", "growth", 98, "Healthcare"),
_row("N1", "near", 97, "Consumer"),
_row("Y1", "cycle", 96, "Materials"),
_row("C3", "core", 95, "Energy"),
]
selected, counts = SCREEN._select_multilane(rows, limit=5, config=PLAN)
self.assertEqual({row["symbol"] for row in selected}, {"C1", "C2", "G1", "N1", "Y1"})
self.assertEqual(counts["core_garp"], 2)
self.assertEqual(counts["cyclical_normalization"], 1)
def test_small_budget_is_deterministic_and_respects_sector_cap(self) -> None:
rows = [
_row("A", "cycle", 100, "Materials"),
_row("B", "core", 99, "Materials"),
_row("C", "core", 98, "Materials"), # third Materials name: sector cap 2
_row("D", "near", 97, "Healthcare"),
]
first, _ = SCREEN._select_multilane(rows, limit=3, config=PLAN)
second, _ = SCREEN._select_multilane(list(reversed(rows)), limit=3, config=PLAN)
self.assertEqual([r["symbol"] for r in first], [r["symbol"] for r in second])
self.assertEqual({r["symbol"] for r in first}, {"A", "B", "D"})
if __name__ == "__main__":
unittest.main()
scripts/tests/test_snapshot_collection.py
"""v3.7 PR A: sharded estimate snapshot + shared coverage semantics."""
from __future__ import annotations
import contextlib
import sys
import tempfile
import time
import unittest
from collections.abc import Iterator
from datetime import datetime, timedelta, timezone
from pathlib import Path
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
if str(SCRIPTS_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPTS_DIR))
import coverage_semantics as COV # noqa: E402
import estimate_snapshot as SNAP # noqa: E402
import pytest
import run_pipeline as PIPELINE # noqa: E402
from fmp_client import ApiCallBudgetExceeded # noqa: E402
from screen_universe import requires_unit_reconciliation # noqa: E402
AS_OF = datetime.now(timezone.utc).replace(microsecond=0)
@pytest.fixture(autouse=True)
def current_analysis_basis(monkeypatch):
"""Root collection can precede this suite by more than the live 5-minute window."""
monkeypatch.setitem(globals(), "AS_OF", datetime.now(timezone.utc).replace(microsecond=0))
def _listing(symbol: str, **overrides) -> dict:
row = {
"symbol": symbol,
"company_name": f"{symbol} Corp",
"exchange": "NASDAQ",
"sector": "Technology",
"industry": "Software - Application",
"price": 20.0,
"market_cap": 2_000_000_000,
"volume": 1_000_000,
"is_actively_trading": True,
"is_common_stock": True,
"common_stock": True,
"currency": "USD",
"country": "US",
"isin": None,
"is_adr": False,
"sector_profile_type": "general",
}
row.update(overrides)
return row
def _estimate_rows(fy1: float = 2.0) -> list[dict]:
rows = []
for offset, eps in ((0, fy1), (1, fy1 * 1.15), (2, fy1 * 1.3)):
year = 2026 + offset
rows.append(
{
"date": f"{year}-12-31",
"fiscalYear": str(year),
"epsAvg": eps,
"epsHigh": eps * 1.1,
"epsLow": eps * 0.9,
"revenueAvg": 1_000_000_000 * (1 + 0.1 * offset),
"numAnalystsEps": 4,
"numAnalystsRevenue": 4,
}
)
return rows
class FakeClient:
def __init__(self, estimates: dict[str, list[dict]], budget: int | None = None):
self._estimates = estimates
self._budget = budget
self.calls = 0
def get_analyst_estimates(self, symbol: str, *, period: str = "annual", limit: int = 6):
if self._budget is not None and self.calls >= self._budget:
raise ApiCallBudgetExceeded("budget exhausted")
self.calls += 1
return self._estimates.get(symbol, [])
def get_analyst_estimates_detailed(
self, symbol: str, *, period: str = "annual", limit: int = 6
) -> dict:
rows = self.get_analyst_estimates(symbol, period=period, limit=limit)
return {
"rows": rows,
"status": "ok" if rows else "empty",
"served_from_cache": False,
"retrieved_at": time.time(),
}
def diagnostics(self) -> dict:
return {"api_calls_made": self.calls, "cache_hits": 0, "failure_count": 0}
def _config() -> dict:
return dict(PIPELINE.DEFAULT_CONFIG)
def _enumeration_audit(universe: list[dict]) -> dict:
return {
"method": "adaptive_market_cap_bands",
"retrieval_scope_explicit": True,
"pagination_exhausted": True,
"requested_exchanges": ["NASDAQ", "NYSE", "AMEX"],
"retrieved_exchanges": ["NASDAQ", "NYSE", "AMEX"],
"requested_min_market_cap": 500_000_000,
"requested_max_market_cap": 20_000_000_000,
"min_price": 5.0,
"page_limit": 1000,
"query_count": 3,
"row_count": len(universe),
"bands": [
{
"exchange": exchange,
"min_market_cap": 500_000_000,
"max_market_cap": 20_000_000_000,
"row_count": 0,
"rows_fetched": 0,
"provider_exhausted": True,
"depth": 0,
}
for exchange in ("NASDAQ", "NYSE", "AMEX")
],
"saturated_leaf_count": 0,
"enumeration_verified": True,
}
class StableShardTests(unittest.TestCase):
def test_shard_assignment_is_deterministic_and_case_insensitive(self) -> None:
self.assertEqual(SNAP.stable_shard("AAPL", 8), SNAP.stable_shard(" aapl ", 8))
first = [SNAP.stable_shard(f"SYM{i}", 8) for i in range(200)]
second = [SNAP.stable_shard(f"SYM{i}", 8) for i in range(200)]
self.assertEqual(first, second)
self.assertTrue(all(0 <= shard < 8 for shard in first))
# every shard gets some members over a reasonable universe
self.assertEqual(len(set(first)), 8)
class ClassifySymbolTests(unittest.TestCase):
def _classify(self, listing: dict, normalized: dict) -> str:
return SNAP.classify_symbol(
listing, normalized, requires_unit_reconciliation=requires_unit_reconciliation
)
def test_precedence(self) -> None:
base = _listing("T")
self.assertEqual(
self._classify(_listing("T", is_common_stock=False), {"estimate_periods": []}),
"excluded",
)
self.assertEqual(
self._classify(_listing("T", country="CN"), {"estimate_periods": []}),
"unit_mismatch",
)
self.assertEqual(self._classify(base, {"estimate_periods": []}), "no_estimates")
self.assertEqual(
self._classify(base, {"estimate_periods": [{}], "fy1_eps": -0.5}), "negative_eps"
)
self.assertEqual(
self._classify(base, {"estimate_periods": [{}], "fy1_eps": 2.0}), "evaluable"
)
def test_implausible_forward_pe_is_unit_mismatch(self) -> None:
verdict = self._classify(
_listing("T"), {"estimate_periods": [{}], "fy1_eps": 20.0, "forward_pe": 0.45}
)
self.assertEqual(verdict, "unit_mismatch")
class CollectEstimatesStageTests(unittest.TestCase):
def _universe(self) -> list[dict]:
rows = [_listing(f"SYM{i}") for i in range(20)]
rows.append(_listing("FRGN", country="CN", currency="CNY"))
rows.append(_listing("NOEST"))
return rows
def _snapshot(self, tmp: str) -> tuple[Path, list[dict]]:
snapshot_dir = Path(tmp) / "snap"
universe = self._universe()
SNAP.create_snapshot(snapshot_dir, universe, shard_count=2, as_of=AS_OF)
return snapshot_dir, universe
def _estimates_for(self, universe: list[dict]) -> dict[str, list[dict]]:
estimates = {row["symbol"]: _estimate_rows() for row in universe}
estimates["NOEST"] = []
return estimates
def test_full_shard_collection_classifies_and_completes(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir, universe = self._snapshot(tmp)
estimates = self._estimates_for(universe)
for shard in (0, 1):
result = PIPELINE.execute_collect_estimates(
FakeClient(estimates),
_config(),
analysis_as_of=AS_OF,
snapshot_dir=snapshot_dir,
shard_index=shard,
shard_count=2,
resume=False,
)
self.assertEqual(result.exit_code, 0)
self.assertEqual(result.summary["status"], "shard_complete")
status = SNAP.snapshot_status(SNAP.load_manifest(snapshot_dir))
self.assertTrue(status["all_shards_complete"])
self.assertTrue(status["classification_matches_universe"])
self.assertEqual(status["universe_count"], len(universe))
totals = status["classified_totals"]
self.assertEqual(totals["unit_mismatch"], 1) # FRGN
self.assertEqual(totals["no_estimates"], 1) # NOEST
self.assertEqual(totals["evaluable"], len(universe) - 2)
def test_budget_exhaustion_is_partial_and_resumable(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir, universe = self._snapshot(tmp)
estimates = self._estimates_for(universe)
shard0 = [row for row in universe if SNAP.stable_shard(row["symbol"], 2) == 0]
partial = PIPELINE.execute_collect_estimates(
FakeClient(estimates, budget=3),
_config(),
analysis_as_of=AS_OF,
snapshot_dir=snapshot_dir,
shard_index=0,
shard_count=2,
resume=False,
)
self.assertEqual(partial.exit_code, 3)
self.assertEqual(partial.summary["status"], "shard_partial_budget")
self.assertTrue(partial.summary["budget_exhausted"])
manifest = SNAP.load_manifest(snapshot_dir)
self.assertEqual(manifest["shards"]["0"]["status"], "partial")
self.assertEqual(manifest["shards"]["0"]["attempted"], 3)
# a second run without --resume must refuse
with self.assertRaises(ValueError):
PIPELINE.execute_collect_estimates(
FakeClient(estimates),
_config(),
analysis_as_of=AS_OF,
snapshot_dir=snapshot_dir,
shard_index=0,
shard_count=2,
resume=False,
)
resumed = PIPELINE.execute_collect_estimates(
FakeClient(estimates),
_config(),
analysis_as_of=AS_OF,
snapshot_dir=snapshot_dir,
shard_index=0,
shard_count=2,
resume=True,
)
self.assertEqual(resumed.exit_code, 0)
manifest = SNAP.load_manifest(snapshot_dir)
entry = manifest["shards"]["0"]
self.assertEqual(entry["status"], "complete")
self.assertEqual(entry["attempted"], len(shard0))
rows = SNAP.load_shard_rows(snapshot_dir, 0)
self.assertEqual(len({row["symbol"] for row in rows}), len(shard0))
def test_shard_count_mismatch_is_refused(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir, _ = self._snapshot(tmp)
with self.assertRaises(ValueError):
PIPELINE.execute_collect_estimates(
FakeClient({}),
_config(),
analysis_as_of=AS_OF,
snapshot_dir=snapshot_dir,
shard_index=0,
shard_count=4,
resume=False,
)
class CoverageSemanticsSharedTests(unittest.TestCase):
def test_pipeline_and_evaluator_share_one_implementation(self) -> None:
import evaluate_candidates as EVAL
self.assertIs(PIPELINE.classify_ranking_scope, COV.classify_ranking_scope)
# the evaluator delegates to the shared derive function
self.assertIn(
"derive_ranking_scope_from_audit", EVAL._derive_ranking_scope.__code__.co_names
)
def test_validate_coverage_block(self) -> None:
block = COV.build_coverage_block(
ranking_scope="final_scoped",
listing_universe_count=2371,
economic_attempt_count=180,
economically_evaluable_count=98,
quality_probe_count=35,
deep_dive_count=3,
)
self.assertEqual(COV.validate_coverage_block(block), [])
broken = dict(block)
broken["economic_attempt_count"] = 5000
problems = COV.validate_coverage_block(broken)
self.assertTrue(any("exceeds listing_universe_count" in p for p in problems))
broken2 = dict(block)
broken2["ranking_scope"] = "complete"
self.assertTrue(COV.validate_coverage_block(broken2))
broken3 = dict(block)
del broken3["economically_evaluable_count"]
self.assertTrue(COV.validate_coverage_block(broken3))
class OfflineMissClient:
"""Simulates offline mode with an empty cache: an explicit failed status."""
def get_analyst_estimates_detailed(
self, symbol: str, *, period: str = "annual", limit: int = 6
) -> dict:
return {"rows": [], "status": "failed", "served_from_cache": False, "retrieved_at": None}
def diagnostics(self) -> dict:
return {"api_calls_made": 0, "cache_hits": 0, "failure_count": 0}
class HttpFailClient:
"""Simulates HTTP failures: calls made, failures recorded, failed status."""
def __init__(self) -> None:
self.calls = 0
def get_analyst_estimates_detailed(
self, symbol: str, *, period: str = "annual", limit: int = 6
) -> dict:
self.calls += 1
return {"rows": [], "status": "failed", "served_from_cache": False, "retrieved_at": None}
def diagnostics(self) -> dict:
return {"api_calls_made": self.calls, "cache_hits": 0, "failure_count": self.calls}
class ProviderFailureTests(unittest.TestCase):
"""Round-2 review P0: provider failures must never classify as no_estimates."""
def _run(self, client, snapshot_dir, resume=False):
return PIPELINE.execute_collect_estimates(
client,
_config(),
analysis_as_of=AS_OF,
snapshot_dir=snapshot_dir,
shard_index=0,
shard_count=1,
resume=resume,
)
def test_offline_empty_cache_is_fetch_failure_not_no_estimates(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = Path(tmp) / "snap"
SNAP.create_snapshot(
snapshot_dir, [_listing("AAA"), _listing("BBB")], shard_count=1, as_of=AS_OF
)
result = self._run(OfflineMissClient(), snapshot_dir)
self.assertEqual(result.exit_code, 3)
self.assertEqual(result.summary["status"], "shard_partial_fetch_failures")
self.assertEqual(result.summary["fetch_failed_count"], 2)
self.assertEqual(result.summary["shard_classified"], {})
status = SNAP.snapshot_status(SNAP.load_manifest(snapshot_dir))
self.assertFalse(status["all_shards_complete"])
self.assertFalse(status["classification_matches_universe"])
def test_http_failures_are_fetch_failures(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = Path(tmp) / "snap"
SNAP.create_snapshot(snapshot_dir, [_listing("AAA")], shard_count=1, as_of=AS_OF)
result = self._run(HttpFailClient(), snapshot_dir)
self.assertEqual(result.exit_code, 3)
self.assertEqual(result.summary["fetch_failed_symbols"], ["AAA"])
manifest = SNAP.load_manifest(snapshot_dir)
self.assertEqual(manifest["shards"]["0"]["fetch_failed"], 1)
self.assertEqual(manifest["shards"]["0"]["status"], "partial")
def test_successful_empty_response_is_still_no_estimates(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = Path(tmp) / "snap"
SNAP.create_snapshot(snapshot_dir, [_listing("AAA")], shard_count=1, as_of=AS_OF)
result = self._run(FakeClient({"AAA": []}), snapshot_dir)
self.assertEqual(result.exit_code, 0)
self.assertEqual(result.summary["shard_classified"], {"no_estimates": 1})
class FrozenUniverseIntegrityTests(unittest.TestCase):
"""Round-2 review P0: a swapped universe.jsonl must be refused."""
def test_tampered_universe_is_refused_before_collection(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = Path(tmp) / "snap"
SNAP.create_snapshot(snapshot_dir, [_listing("AAA")], shard_count=1, as_of=AS_OF)
tampered = _listing("ZZZ")
import json as _json
(snapshot_dir / SNAP.UNIVERSE_NAME).write_text(
_json.dumps(tampered, sort_keys=True) + "\n", encoding="utf-8"
)
client = FakeClient({"ZZZ": _estimate_rows()})
with self.assertRaises(ValueError):
PIPELINE.execute_collect_estimates(
client,
_config(),
analysis_as_of=AS_OF,
snapshot_dir=snapshot_dir,
shard_index=0,
shard_count=1,
resume=False,
)
self.assertEqual(client.calls, 0) # refused before any API access
def test_count_mismatch_is_refused(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = Path(tmp) / "snap"
SNAP.create_snapshot(
snapshot_dir, [_listing("AAA"), _listing("BBB")], shard_count=1, as_of=AS_OF
)
content = (snapshot_dir / SNAP.UNIVERSE_NAME).read_text().splitlines()
(snapshot_dir / SNAP.UNIVERSE_NAME).write_text(content[0] + "\n", encoding="utf-8")
with self.assertRaises(ValueError):
SNAP.load_verified_universe(snapshot_dir, SNAP.load_manifest(snapshot_dir))
class FreshnessStampTests(unittest.TestCase):
"""Round-2 review P0: zero-collection runs must not refresh shard freshness."""
def test_zero_collect_resume_preserves_as_of(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = Path(tmp) / "snap"
SNAP.create_snapshot(snapshot_dir, [_listing("AAA")], shard_count=1, as_of=AS_OF)
estimates = {"AAA": _estimate_rows()}
first = PIPELINE.execute_collect_estimates(
FakeClient(estimates),
_config(),
analysis_as_of=AS_OF,
snapshot_dir=snapshot_dir,
shard_index=0,
shard_count=1,
resume=False,
)
self.assertEqual(first.exit_code, 0)
original_as_of = SNAP.load_manifest(snapshot_dir)["shards"]["0"]["as_of"]
later = AS_OF + timedelta(minutes=1)
second = PIPELINE.execute_collect_estimates(
FakeClient(estimates),
_config(),
analysis_as_of=later,
snapshot_dir=snapshot_dir,
shard_index=0,
shard_count=1,
resume=True,
)
self.assertEqual(second.summary["collected_this_run"], 0)
entry = SNAP.load_manifest(snapshot_dir)["shards"]["0"]
self.assertEqual(entry["as_of"], original_as_of)
self.assertIsNotNone(entry["oldest_retrieved_at"])
self.assertIsNotNone(entry["newest_retrieved_at"])
def test_rows_carry_actual_retrieval_stamps(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = Path(tmp) / "snap"
SNAP.create_snapshot(snapshot_dir, [_listing("AAA")], shard_count=1, as_of=AS_OF)
PIPELINE.execute_collect_estimates(
FakeClient({"AAA": _estimate_rows()}),
_config(),
analysis_as_of=AS_OF,
snapshot_dir=snapshot_dir,
shard_index=0,
shard_count=1,
resume=False,
)
[row] = SNAP.load_shard_rows(snapshot_dir, 0)
self.assertIn("snapshot_retrieved_at", row)
self.assertFalse(row["snapshot_served_from_cache"])
class NegativeEpsIntegrationTests(unittest.TestCase):
"""Round-2 review P1: negative_eps must be reachable through the real normalizer."""
def test_negative_consensus_classifies_negative_eps(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = Path(tmp) / "snap"
SNAP.create_snapshot(snapshot_dir, [_listing("NEG")], shard_count=1, as_of=AS_OF)
result = PIPELINE.execute_collect_estimates(
FakeClient({"NEG": _estimate_rows(fy1=-1.0)}),
_config(),
analysis_as_of=AS_OF,
snapshot_dir=snapshot_dir,
shard_index=0,
shard_count=1,
resume=False,
)
self.assertEqual(result.summary["shard_classified"], {"negative_eps": 1})
class _StubResponse:
def __init__(self, status_code: int, payload) -> None:
self.status_code = status_code
self._payload = payload
def json(self):
return self._payload
class Http200ErrorPayloadTests(unittest.TestCase):
"""Round-3 review P0: HTTP-200 error objects are failures, never data."""
@contextlib.contextmanager
def _tmpdir(self) -> Iterator[str]:
"""Yield a temporary directory, closing the SQLite caches opened in it first.
Windows cannot delete the cache file while its connection is open. Only
the cache is closed because the tests replace the HTTP session with a stub.
"""
self._open_clients = []
with tempfile.TemporaryDirectory() as tmp:
try:
yield tmp
finally:
for client in self._open_clients:
client.cache.close()
def _client(self, tmp: str, *, offline: bool = False, session_get=None):
from fmp_client import FMPClient
client = FMPClient(
api_key="test-key", # pragma: allowlist secret
max_api_calls=10,
cache_path=Path(tmp) / "cache.sqlite",
raw_store_dir=Path(tmp) / "raw",
offline=offline,
)
self._open_clients.append(client)
if session_get is not None:
import types
client.session = types.SimpleNamespace(get=session_get)
return client
def _stable_key(self, client, symbol: str) -> str:
from fmp_client import SQLiteJsonCache
return SQLiteJsonCache.make_key(
f"{client.STABLE_URL}/analyst-estimates",
{"symbol": symbol, "period": "annual", "limit": 6},
)
def test_http200_error_object_is_failure_and_never_cached(self) -> None:
payload = {"Error Message": "plan limit reached"}
with self._tmpdir() as tmp:
client = self._client(
tmp, session_get=lambda url, params=None, timeout=None: _StubResponse(200, payload)
)
self.assertEqual(client.get_analyst_estimates("AAA"), [])
diag = client.diagnostics()
self.assertGreaterEqual(diag["failure_count"], 1)
self.assertEqual(diag["cache_hits"], 0)
self.assertIsNone(client.cache.get(self._stable_key(client, "AAA"), 10**9))
def test_previously_cached_error_object_is_purged_and_failed(self) -> None:
with self._tmpdir() as tmp:
client = self._client(tmp, offline=True)
key = self._stable_key(client, "AAA")
client.cache.put(key, {"Error Message": "poisoned"})
result = client.get_analyst_estimates_detailed("AAA")
self.assertEqual(result["status"], "failed")
self.assertGreaterEqual(client.diagnostics()["failure_count"], 1)
# the poisoned entry is purged so an online retry reaches HTTP
self.assertIsNone(client.cache.get(key, 10**9))
def test_error_object_inside_list_is_failure_and_never_cached(self) -> None:
# Round-4 review P0: [{"Error Message": ...}] hid the outage one
# level down and was accepted (and cached) as data.
payload = [{"Error Message": "plan limit reached"}]
with self._tmpdir() as tmp:
client = self._client(
tmp, session_get=lambda url, params=None, timeout=None: _StubResponse(200, payload)
)
result = client.get_analyst_estimates_detailed("AAA")
self.assertEqual(result["status"], "failed")
self.assertEqual(result["rows"], [])
self.assertGreaterEqual(client.diagnostics()["failure_count"], 1)
self.assertIsNone(client.cache.get(self._stable_key(client, "AAA"), 10**9))
def test_poisoned_cache_is_purged_and_refetched_online(self) -> None:
good = _estimate_rows()
with self._tmpdir() as tmp:
client = self._client(
tmp, session_get=lambda url, params=None, timeout=None: _StubResponse(200, good)
)
key = self._stable_key(client, "AAA")
client.cache.put(key, [{"Error Message": "poisoned"}])
result = client.get_analyst_estimates_detailed("AAA")
self.assertEqual(result["status"], "ok")
self.assertFalse(result["served_from_cache"])
self.assertEqual(len(result["rows"]), len(good))
self.assertGreaterEqual(client.diagnostics()["failure_count"], 1)
self.assertIsNotNone(client.cache.get(key, 10**9)) # replaced with valid data
def test_nonempty_unusable_payload_is_failure_not_no_estimates(self) -> None:
# Round-5 review P0: a non-empty payload that cannot yield a single
# valid estimate period is unusable noise, not "no estimates".
for payload in (
[{"date": "not-a-date", "epsAvg": None}],
[{"date": "2026-12-31", "epsAvg": None, "revenueAvg": None}],
# round-6: truncation let a garbage suffix through
[{"date": "2026-12-31garbage", "epsAvg": 2.0}],
# round-6: an analyst COUNT is not an estimate value
[{"date": "2026-12-31", "epsAvg": None, "revenueAvg": None, "numAnalystsEps": 4}],
):
with self._tmpdir() as tmp:
client = self._client(
tmp,
session_get=lambda url, params=None, timeout=None, p=payload: _StubResponse(
200, p
),
)
result = client.get_analyst_estimates_detailed("AAA")
self.assertEqual(result["status"], "failed", payload)
self.assertIsNone(client.cache.get(self._stable_key(client, "AAA"), 10**9))
def test_stable_failure_with_v3_empty_success_is_no_estimates(self) -> None:
# Round-4 review P1: stable HTTP 500 followed by a SUCCESSFUL empty
# v3 response is a genuine empty consensus, not a fetch failure.
def _get(url, params=None, timeout=None):
if "/stable/" in url:
return _StubResponse(500, None)
return _StubResponse(200, [])
with self._tmpdir() as tmp:
snapshot_dir = Path(tmp) / "snap"
SNAP.create_snapshot(snapshot_dir, [_listing("AAA")], shard_count=1, as_of=AS_OF)
client = self._client(tmp, session_get=_get)
result = PIPELINE.execute_collect_estimates(
client,
_config(),
analysis_as_of=AS_OF,
snapshot_dir=snapshot_dir,
shard_index=0,
shard_count=1,
resume=False,
)
self.assertEqual(result.exit_code, 0)
self.assertEqual(result.summary["status"], "shard_complete")
self.assertEqual(result.summary["shard_classified"], {"no_estimates": 1})
def test_cached_error_object_becomes_fetch_failure_in_collect(self) -> None:
with self._tmpdir() as tmp:
snapshot_dir = Path(tmp) / "snap"
SNAP.create_snapshot(snapshot_dir, [_listing("AAA")], shard_count=1, as_of=AS_OF)
client = self._client(tmp, offline=True)
client.cache.put(self._stable_key(client, "AAA"), {"Error Message": "poisoned"})
result = PIPELINE.execute_collect_estimates(
client,
_config(),
analysis_as_of=AS_OF,
snapshot_dir=snapshot_dir,
shard_index=0,
shard_count=1,
resume=False,
)
self.assertEqual(result.summary["status"], "shard_partial_fetch_failures")
self.assertEqual(result.summary["shard_classified"], {})
class CacheProvenanceTests(unittest.TestCase):
"""Round-3 review P1: cache-served data keeps its original fetch time."""
def test_stable_http_failure_with_v3_cache_hit_keeps_cache_time(self) -> None:
import sqlite3 as _sqlite3
import time as _time
from fmp_client import FMPClient, SQLiteJsonCache
# One hour ago: old enough to differ from "now", young enough to
# survive the estimates cache TTL.
fixed_created_at = float(int(_time.time() - 3600.0))
with tempfile.TemporaryDirectory() as tmp, contextlib.ExitStack() as stack:
snapshot_dir = Path(tmp) / "snap"
SNAP.create_snapshot(snapshot_dir, [_listing("CPV")], shard_count=1, as_of=AS_OF)
client = FMPClient(
api_key="test-key", # pragma: allowlist secret
max_api_calls=10,
cache_path=Path(tmp) / "cache.sqlite",
raw_store_dir=Path(tmp) / "raw",
)
# Close the SQLite cache before the directory is removed (Windows locks it).
stack.callback(client.cache.close)
import types
client.session = types.SimpleNamespace(
get=lambda url, params=None, timeout=None: _StubResponse(500, None)
)
v3_key = SQLiteJsonCache.make_key(
f"{client.V3_URL}/analyst-estimates/CPV", {"period": "annual", "limit": 6}
)
client.cache.put(v3_key, _estimate_rows())
with contextlib.closing(_sqlite3.connect(str(client.cache.path))) as connection:
connection.execute(
"UPDATE responses SET created_at = ? WHERE cache_key = ?",
(fixed_created_at, v3_key),
)
connection.commit()
result = PIPELINE.execute_collect_estimates(
client,
_config(),
analysis_as_of=AS_OF,
snapshot_dir=snapshot_dir,
shard_index=0,
shard_count=1,
resume=False,
)
self.assertEqual(result.summary["shard_classified"], {"evaluable": 1})
[row] = SNAP.load_shard_rows(snapshot_dir, 0)
self.assertTrue(row["snapshot_served_from_cache"])
expected = datetime.fromtimestamp(fixed_created_at, tz=timezone.utc).isoformat()
self.assertEqual(row["snapshot_retrieved_at"], expected)
def test_unknown_cache_provenance_stays_null(self) -> None:
class UnknownCacheClient:
def __init__(self, estimates):
self._estimates = estimates
self.hits = 0
def get_analyst_estimates_detailed(
self, symbol: str, *, period: str = "annual", limit: int = 6
) -> dict:
self.hits += 1
return {
"rows": self._estimates.get(symbol, []),
"status": "ok",
"served_from_cache": True,
"retrieved_at": None,
}
def diagnostics(self) -> dict:
return {"api_calls_made": 0, "cache_hits": self.hits, "failure_count": 0}
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = Path(tmp) / "snap"
SNAP.create_snapshot(snapshot_dir, [_listing("AAA")], shard_count=1, as_of=AS_OF)
result = PIPELINE.execute_collect_estimates(
UnknownCacheClient({"AAA": _estimate_rows()}),
_config(),
analysis_as_of=AS_OF,
snapshot_dir=snapshot_dir,
shard_index=0,
shard_count=1,
resume=False,
)
[row] = SNAP.load_shard_rows(snapshot_dir, 0)
self.assertIsNone(row["snapshot_retrieved_at"])
self.assertTrue(row["snapshot_served_from_cache"])
self.assertEqual(result.summary["retrieval_time_unknown"], 1)
entry = SNAP.load_manifest(snapshot_dir)["shards"]["0"]
self.assertEqual(entry["retrieval_time_unknown"], 1)
self.assertIsNone(entry["oldest_retrieved_at"])
# Round-4 review: unknown provenance blocks screening readiness
# even when collection itself is complete.
status = SNAP.snapshot_status(SNAP.load_manifest(snapshot_dir))
self.assertTrue(status["all_shards_complete"])
self.assertFalse(status["freshness_provenance_complete"])
self.assertFalse(status["collection_ready"])
class SnapshotReadinessTests(unittest.TestCase):
"""Round-4 review P1: readiness = collection + classification + provenance."""
def test_clean_complete_snapshot_is_ready(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = Path(tmp) / "snap"
SNAP.create_snapshot(snapshot_dir, [_listing("AAA")], shard_count=1, as_of=AS_OF)
PIPELINE.execute_collect_estimates(
FakeClient({"AAA": _estimate_rows()}),
_config(),
analysis_as_of=AS_OF,
snapshot_dir=snapshot_dir,
shard_index=0,
shard_count=1,
resume=False,
)
status = SNAP.snapshot_status(SNAP.load_manifest(snapshot_dir))
self.assertTrue(status["all_shards_complete"])
self.assertTrue(status["classification_matches_universe"])
self.assertEqual(status["retrieval_time_unknown_total"], 0)
self.assertEqual(status["fetch_failed_total"], 0)
self.assertTrue(status["freshness_provenance_complete"])
self.assertTrue(status["collection_ready"])
self.assertNotIn("ready_for_screening", status) # verify_snapshot only
self.assertIsNotNone(status["oldest_retrieved_at"])
self.assertIsNotNone(status["newest_retrieved_at"])
def test_fetch_failures_block_readiness(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = Path(tmp) / "snap"
SNAP.create_snapshot(snapshot_dir, [_listing("AAA")], shard_count=1, as_of=AS_OF)
PIPELINE.execute_collect_estimates(
OfflineMissClient(),
_config(),
analysis_as_of=AS_OF,
snapshot_dir=snapshot_dir,
shard_index=0,
shard_count=1,
resume=False,
)
status = SNAP.snapshot_status(SNAP.load_manifest(snapshot_dir))
self.assertGreaterEqual(status["fetch_failed_total"], 1)
self.assertFalse(status["collection_ready"])
class SnapshotVerificationTests(unittest.TestCase):
"""Round-5 review P0: readiness must be proven against the ACTUAL files."""
def _build_clean(self, tmp: str) -> Path:
snapshot_dir = Path(tmp) / "snap"
universe = [_listing("AAA"), _listing("BBB")]
SNAP.create_snapshot(
snapshot_dir,
universe,
shard_count=1,
as_of=AS_OF,
enumeration_audit=_enumeration_audit(universe),
)
result = PIPELINE.execute_collect_estimates(
FakeClient({row["symbol"]: _estimate_rows() for row in universe}),
_config(),
analysis_as_of=AS_OF,
snapshot_dir=snapshot_dir,
shard_index=0,
shard_count=1,
resume=False,
)
assert result.exit_code == 0
return snapshot_dir
def _verify(self, snapshot_dir: Path, **overrides) -> dict:
kwargs = {
"screening_as_of": datetime.now(timezone.utc),
"max_staleness_days": 7.0,
}
kwargs.update(overrides)
return SNAP.verify_snapshot(snapshot_dir, **kwargs)
def test_clean_snapshot_verifies_and_is_ready(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
verdict = self._verify(self._build_clean(tmp))
self.assertTrue(verdict["contents_verified"], verdict["problems"])
self.assertTrue(verdict["staleness_ok"])
self.assertTrue(verdict["ready_for_screening"])
def test_swapped_shard_symbol_is_detected(self) -> None:
# The reviewer's reproduction: freeze AAA+BBB, then swap AAA -> ZZZ
# inside the shard file. Manifest counters still balance.
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = self._build_clean(tmp)
path = SNAP.shard_path(snapshot_dir, 0)
path.write_text(path.read_text().replace('"symbol": "AAA"', '"symbol": "ZZZ"'))
self.assertTrue(
SNAP.snapshot_status(SNAP.load_manifest(snapshot_dir))["collection_ready"]
)
verdict = self._verify(snapshot_dir)
self.assertFalse(verdict["contents_verified"])
self.assertFalse(verdict["ready_for_screening"])
text = " ".join(verdict["problems"])
self.assertIn("ZZZ", text)
self.assertIn("AAA", text) # AAA reported as never collected
def test_missing_shard_file_is_detected(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = self._build_clean(tmp)
SNAP.shard_path(snapshot_dir, 0).unlink()
verdict = self._verify(snapshot_dir)
self.assertFalse(verdict["ready_for_screening"])
self.assertTrue(any("file missing" in p for p in verdict["problems"]))
def test_duplicate_symbol_is_detected(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = self._build_clean(tmp)
[row, *_] = SNAP.load_shard_rows(snapshot_dir, 0)
SNAP.append_shard_rows(snapshot_dir, 0, [row])
verdict = self._verify(snapshot_dir)
self.assertFalse(verdict["ready_for_screening"])
self.assertTrue(any("duplicate symbol" in p for p in verdict["problems"]))
def test_future_data_blocks_historical_screening(self) -> None:
# Round-6 review P0: a 2026-collected snapshot must never be "ready"
# for a 2020 screening_as_of — that is look-ahead.
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = self._build_clean(tmp)
verdict = self._verify(
snapshot_dir,
screening_as_of=datetime(2020, 1, 1, tzinfo=timezone.utc),
max_staleness_days=36500.0,
)
self.assertTrue(verdict["contents_verified"])
self.assertFalse(verdict["no_future_retrievals"])
self.assertFalse(verdict["ready_for_screening"])
def test_missing_shard_sha_is_detected(self) -> None:
# Round-6 review P1: an unpinned shard (pre-SHA collection) must not
# verify; a zero-collect --resume backfills the SHA.
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = self._build_clean(tmp)
manifest = SNAP.load_manifest(snapshot_dir)
del manifest["shards"]["0"]["shard_sha256"]
SNAP.write_manifest(snapshot_dir, manifest)
verdict = self._verify(snapshot_dir)
self.assertFalse(verdict["ready_for_screening"])
self.assertTrue(any("shard_sha256" in p for p in verdict["problems"]))
resumed = PIPELINE.execute_collect_estimates(
FakeClient({}),
_config(),
analysis_as_of=AS_OF,
snapshot_dir=snapshot_dir,
shard_index=0,
shard_count=1,
resume=True,
)
self.assertEqual(resumed.summary["collected_this_run"], 0)
verdict = self._verify(snapshot_dir)
self.assertTrue(verdict["ready_for_screening"], verdict["problems"])
def test_stale_snapshot_is_not_ready(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = self._build_clean(tmp)
future = datetime.now(timezone.utc) + timedelta(days=30)
verdict = self._verify(snapshot_dir, screening_as_of=future, max_staleness_days=1.0)
self.assertTrue(verdict["contents_verified"])
self.assertFalse(verdict["staleness_ok"])
self.assertFalse(verdict["ready_for_screening"])
if __name__ == "__main__":
unittest.main()
scripts/tests/test_snapshot_screening.py
"""v3.7 PR B: verified full-snapshot screening and market-wide scope."""
from __future__ import annotations
import hashlib
import io
import json
import sys
import tempfile
import unittest
from contextlib import redirect_stdout
from datetime import datetime, timedelta, timezone
from pathlib import Path
from unittest import mock
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
if str(SCRIPTS_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPTS_DIR))
import estimate_snapshot as SNAP # noqa: E402
import run_pipeline as PIPELINE # noqa: E402
def _listing(symbol: str, index: int) -> dict:
sectors = (
("Technology", "Software - Application"),
("Industrials", "Industrial Distribution"),
("Basic Materials", "Chemicals"),
("Consumer Defensive", "Packaged Foods"),
)
sector, industry = sectors[index % len(sectors)]
return {
"symbol": symbol,
"company_name": f"{symbol} Corp",
"exchange": "NASDAQ",
"sector": sector,
"industry": industry,
"price": 20.0,
"market_cap": 1_000_000_000 + index * 50_000_000,
"volume": 500_000,
"is_actively_trading": True,
"is_common_stock": True,
"common_stock": True,
"currency": "USD",
"country": "US",
"isin": None,
"is_adr": False,
"sector_profile_type": "general",
}
def _estimate_rows() -> list[dict]:
return [
{
"date": f"{year}-12-31",
"fiscalYear": str(year),
"epsAvg": eps,
"epsHigh": eps * 1.1,
"epsLow": eps * 0.9,
"revenueAvg": revenue,
"numAnalystsEps": 5,
"numAnalystsRevenue": 5,
}
for year, eps, revenue in (
(2026, 2.0, 1_000_000_000),
(2027, 2.4, 1_120_000_000),
(2028, 2.8, 1_250_000_000),
)
]
def _enumeration_audit(universe: list[dict], *, row_count: int | None = None) -> dict:
bands = [
{
"exchange": exchange,
"min_market_cap": 500_000_000,
"max_market_cap": 20_000_000_000,
"row_count": sum(row["exchange"] == exchange for row in universe),
"rows_fetched": sum(row["exchange"] == exchange for row in universe),
"provider_exhausted": True,
"depth": 0,
}
for exchange in ("NASDAQ", "NYSE", "AMEX")
]
return {
"method": "adaptive_market_cap_bands",
"retrieval_scope_explicit": True,
"pagination_exhausted": True,
"requested_exchanges": ["NASDAQ", "NYSE", "AMEX"],
"retrieved_exchanges": ["AMEX", "NASDAQ", "NYSE"],
"requested_min_market_cap": 500_000_000,
"requested_max_market_cap": 20_000_000_000,
"min_price": 5.0,
"page_limit": 1000,
"query_count": 3,
"row_count": len(universe) if row_count is None else row_count,
"bands": bands,
"saturated_leaf_count": 0,
"enumeration_verified": True,
}
def _build_snapshot(
root: Path,
*,
now: datetime,
count: int = 8,
with_estimates: bool = True,
enumeration_row_count: int | None = None,
enumeration_band_range: tuple[float, float] | None = None,
) -> Path:
snapshot_dir = root / "snapshot"
universe = [_listing(f"S{i:02d}", i) for i in range(count)]
enumeration_audit = _enumeration_audit(universe, row_count=enumeration_row_count)
if enumeration_band_range is not None:
for band in enumeration_audit["bands"]:
band["min_market_cap"], band["max_market_cap"] = enumeration_band_range
manifest = SNAP.create_snapshot(
snapshot_dir,
universe,
shard_count=1,
as_of=now,
enumeration_audit=enumeration_audit,
)
normalized = PIPELINE.normalize_estimate_frame(
universe,
{row["symbol"]: (_estimate_rows() if with_estimates else []) for row in universe},
analysis_as_of=now,
source_id="fixture-estimates",
config=PIPELINE.load_config(None),
)
stamp = now.astimezone(timezone.utc).isoformat()
records = []
classified: dict[str, int] = {}
for row in normalized:
record = dict(row)
classification = SNAP.classify_symbol(
row,
row,
requires_unit_reconciliation=PIPELINE.requires_unit_reconciliation,
)
record.update(
{
"snapshot_classification": classification,
"snapshot_shard": 0,
"snapshot_retrieved_at": stamp,
"snapshot_normalization_as_of": stamp,
"snapshot_served_from_cache": False,
}
)
records.append(record)
classified[classification] = classified.get(classification, 0) + 1
SNAP.append_shard_rows(snapshot_dir, 0, records)
SNAP.update_shard(
snapshot_dir,
manifest,
0,
status="complete",
as_of=stamp,
calls_used=count,
classified=classified,
attempted=count,
expected=count,
oldest_retrieved_at=stamp,
newest_retrieved_at=stamp,
oldest_normalization_as_of=stamp,
newest_normalization_as_of=stamp,
)
return snapshot_dir
class FakeScreenClient:
def __init__(self) -> None:
self.api_calls_made = 0
self.max_api_calls = 999
def get_historical_prices(self, symbol, *, from_date, to_date):
self.api_calls_made += 1
return [{"date": f"2026-08-{day:02d}", "volume": 600_000} for day in range(1, 26)]
def get_key_metrics_ttm(self, symbol):
self.api_calls_made += 1
return [
{
"returnOnInvestedCapitalTTM": 0.16,
"freeCashFlowYieldTTM": 0.07,
"evToFreeCashFlowTTM": 14.0,
"netDebtToEBITDATTM": 1.2,
"stockBasedCompensationToRevenueTTM": 0.02,
}
]
def diagnostics(self):
return {
"api_calls_made": self.api_calls_made,
"max_api_calls": self.max_api_calls,
"cache_hits": 0,
"failure_count": 0,
"failure_samples": [],
"disabled_endpoint_count": 0,
"offline": False,
}
class BackfillScreenClient(FakeScreenClient):
def __init__(self, *, failed_history_calls: int) -> None:
super().__init__()
self.failed_history_calls = failed_history_calls
self.history_calls = 0
def get_historical_prices(self, symbol, *, from_date, to_date):
self.api_calls_made += 1
self.history_calls += 1
if self.history_calls <= self.failed_history_calls:
return []
return [{"date": f"2026-08-{day:02d}", "volume": 600_000} for day in range(1, 26)]
class BudgetOnLiquidityClient(FakeScreenClient):
def __init__(self, *, successful_history_calls: int) -> None:
super().__init__()
self.successful_history_calls = successful_history_calls
self.history_calls = 0
def get_historical_prices(self, symbol, *, from_date, to_date):
self.api_calls_made += 1
self.history_calls += 1
if self.history_calls > self.successful_history_calls:
raise PIPELINE.ApiCallBudgetExceeded("budget exhausted during liquidity")
return [{"date": f"2026-08-{day:02d}", "volume": 600_000} for day in range(1, 26)]
class BudgetOnQualityClient(FakeScreenClient):
def __init__(self, *, successful_quality_calls: int) -> None:
super().__init__()
self.successful_quality_calls = successful_quality_calls
self.quality_calls = 0
def get_key_metrics_ttm(self, symbol):
self.api_calls_made += 1
self.quality_calls += 1
if self.quality_calls > self.successful_quality_calls:
raise PIPELINE.ApiCallBudgetExceeded("budget exhausted during quality probe")
return [
{
"returnOnInvestedCapitalTTM": 0.16,
"freeCashFlowYieldTTM": 0.07,
"evToFreeCashFlowTTM": 14.0,
"netDebtToEBITDATTM": 1.2,
"stockBasedCompensationToRevenueTTM": 0.02,
}
]
class BudgetOnActualEpsClient(FakeScreenClient):
def __init__(self, *, successful_actual_calls: int) -> None:
super().__init__()
self.successful_actual_calls = successful_actual_calls
self.actual_calls = 0
def get_income_statement(self, symbol, *, period="annual", limit=2):
self.api_calls_made += 1
self.actual_calls += 1
if self.actual_calls > self.successful_actual_calls:
raise PIPELINE.ApiCallBudgetExceeded("budget exhausted during actual EPS")
return []
class BudgetOnPacketClient(FakeScreenClient):
def __init__(self, *, successful_packets: int) -> None:
super().__init__()
self.successful_packets = successful_packets
self.packet_profiles = 0
def get_profile(self, symbol):
self.packet_profiles += 1
self.api_calls_made += 1
if self.packet_profiles > self.successful_packets:
raise PIPELINE.ApiCallBudgetExceeded("budget exhausted during packet")
return {}
def get_quotes(self, symbols):
self.api_calls_made += 1
return {symbol: {} for symbol in symbols}
def get_analyst_estimates(self, symbol, *, period="annual", limit=6):
self.api_calls_made += 1
return []
def get_income_statement(self, symbol, *, period="annual", limit=6):
self.api_calls_made += 1
return []
def get_balance_sheet(self, symbol, *, period="annual", limit=6):
self.api_calls_made += 1
return []
def get_cash_flow(self, symbol, *, period="annual", limit=6):
self.api_calls_made += 1
return []
def get_ratios_ttm(self, symbol):
self.api_calls_made += 1
return []
def get_stock_peers(self, symbol):
self.api_calls_made += 1
return []
class VerifiedSnapshotBundleTests(unittest.TestCase):
def _replace_shard_payload(self, snapshot_dir: Path, payload: bytes) -> Path:
path = SNAP.shard_path(snapshot_dir, 0)
path.write_bytes(payload)
manifest = SNAP.load_manifest(snapshot_dir)
manifest["shards"]["0"]["shard_sha256"] = hashlib.sha256(payload).hexdigest()
SNAP.write_manifest(snapshot_dir, manifest)
return path
def _load_with_tracked_shard_read(
self,
snapshot_dir: Path,
*,
now: datetime,
read_error: OSError | None = None,
) -> tuple[dict, int]:
target = SNAP.shard_path(snapshot_dir, 0)
original_read_bytes = Path.read_bytes
shard_reads = 0
def controlled_read_bytes(path: Path) -> bytes:
nonlocal shard_reads
if path == target:
shard_reads += 1
if read_error is not None:
raise read_error
return original_read_bytes(path)
with mock.patch.object(Path, "read_bytes", controlled_read_bytes):
bundle = SNAP.load_verified_snapshot(
snapshot_dir,
screening_as_of=now,
max_staleness_days=7,
)
return bundle, shard_reads
def _assert_failed_shard_bundle(
self,
bundle: dict,
*,
path: Path,
diagnostic: str,
expected_sha: str | None,
) -> None:
verdict = bundle["verdict"]
self.assertFalse(verdict["contents_verified"])
self.assertFalse(verdict["ready_for_screening"])
self.assertEqual(bundle["rows"], [])
self.assertTrue(
any(f"shard 0 ({path}): {diagnostic}" in problem for problem in verdict["problems"]),
verdict["problems"],
)
self.assertEqual(verdict["verification_binding"]["shard_sha256"]["0"], expected_sha)
def test_verified_rows_and_digest_come_from_one_verified_read(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = _build_snapshot(Path(tmp), now=now)
bundle = SNAP.load_verified_snapshot(
snapshot_dir,
screening_as_of=now,
max_staleness_days=7,
)
self.assertTrue(bundle["verdict"]["ready_for_screening"])
self.assertEqual(len(bundle["rows"]), 8)
self.assertEqual(len(bundle["verification_digest"]), 64)
self.assertEqual(
bundle["verification_digest"],
bundle["verdict"]["snapshot_verification_digest"],
)
def test_tampered_shard_returns_no_consumable_rows(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = _build_snapshot(Path(tmp), now=now)
path = SNAP.shard_path(snapshot_dir, 0)
path.write_text(path.read_text().replace('"symbol": "S00"', '"symbol": "BAD"'))
bundle = SNAP.load_verified_snapshot(
snapshot_dir,
screening_as_of=now,
max_staleness_days=7,
)
self.assertFalse(bundle["verdict"]["ready_for_screening"])
self.assertEqual(bundle["rows"], [])
def test_malformed_json_shard_returns_failed_verdict_with_actual_sha(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
payload = b'{"broken"\n'
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = _build_snapshot(Path(tmp), now=now)
path = self._replace_shard_payload(snapshot_dir, payload)
bundle, shard_reads = self._load_with_tracked_shard_read(snapshot_dir, now=now)
self.assertEqual(shard_reads, 1)
self._assert_failed_shard_bundle(
bundle,
path=path,
diagnostic="invalid JSON",
expected_sha=hashlib.sha256(payload).hexdigest(),
)
def test_invalid_utf8_shard_returns_failed_verdict_with_actual_sha(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
payload = b"\xff\n"
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = _build_snapshot(Path(tmp), now=now)
path = self._replace_shard_payload(snapshot_dir, payload)
bundle, shard_reads = self._load_with_tracked_shard_read(snapshot_dir, now=now)
self.assertEqual(shard_reads, 1)
self._assert_failed_shard_bundle(
bundle,
path=path,
diagnostic="invalid UTF-8",
expected_sha=hashlib.sha256(payload).hexdigest(),
)
def test_non_object_shard_row_returns_failed_verdict_with_actual_sha(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
payload = b"[]\n"
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = _build_snapshot(Path(tmp), now=now)
path = self._replace_shard_payload(snapshot_dir, payload)
bundle, shard_reads = self._load_with_tracked_shard_read(snapshot_dir, now=now)
self.assertEqual(shard_reads, 1)
self._assert_failed_shard_bundle(
bundle,
path=path,
diagnostic="row is not a JSON object",
expected_sha=hashlib.sha256(payload).hexdigest(),
)
def test_unreadable_shard_returns_failed_verdict_with_null_sha(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = _build_snapshot(Path(tmp), now=now)
path = SNAP.shard_path(snapshot_dir, 0)
bundle, shard_reads = self._load_with_tracked_shard_read(
snapshot_dir,
now=now,
read_error=OSError("synthetic unreadable shard"),
)
self.assertEqual(shard_reads, 1)
self._assert_failed_shard_bundle(
bundle,
path=path,
diagnostic="read failed",
expected_sha=None,
)
def test_subset_enumeration_proof_is_rejected(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = _build_snapshot(Path(tmp), now=now, count=8, enumeration_row_count=9)
bundle = SNAP.load_verified_snapshot(
snapshot_dir,
screening_as_of=now,
max_staleness_days=7,
)
self.assertFalse(bundle["verdict"]["listing_enumeration_verified"])
self.assertFalse(bundle["verdict"]["ready_for_screening"])
self.assertTrue(
any("complete requested scope" in p for p in bundle["verdict"]["problems"])
)
def test_gapped_enumeration_bands_are_rejected(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = _build_snapshot(
Path(tmp),
now=now,
enumeration_band_range=(1_000_000_000, 2_000_000_000),
)
bundle = SNAP.load_verified_snapshot(
snapshot_dir,
screening_as_of=now,
max_staleness_days=7,
)
self.assertFalse(bundle["verdict"]["listing_enumeration_verified"])
self.assertFalse(bundle["verdict"]["ready_for_screening"])
def test_old_normalization_basis_is_rejected_even_with_fresh_retrieval(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
old = (now - timedelta(days=30)).isoformat()
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = _build_snapshot(Path(tmp), now=now)
path = SNAP.shard_path(snapshot_dir, 0)
rows = SNAP.load_shard_rows(snapshot_dir, 0)
for row in rows:
row["snapshot_normalization_as_of"] = old
path.write_text(
"".join(json.dumps(row, sort_keys=True) + "\n" for row in rows),
encoding="utf-8",
)
manifest = SNAP.load_manifest(snapshot_dir)
entry = manifest["shards"]["0"]
SNAP.update_shard(
snapshot_dir,
manifest,
0,
status="complete",
as_of=entry["as_of"],
calls_used=entry["calls_used"],
classified=entry["classified"],
attempted=entry["attempted"],
expected=entry["expected"],
oldest_retrieved_at=entry["oldest_retrieved_at"],
newest_retrieved_at=entry["newest_retrieved_at"],
oldest_normalization_as_of=old,
newest_normalization_as_of=old,
)
bundle = SNAP.load_verified_snapshot(
snapshot_dir,
screening_as_of=now,
max_staleness_days=7,
)
self.assertFalse(bundle["verdict"]["normalization_current"])
self.assertFalse(bundle["verdict"]["ready_for_screening"])
class FullSnapshotStageTests(unittest.TestCase):
def _config(self) -> dict:
config = PIPELINE.load_config(None)
config.update(
{
"full_snapshot_pool_size": 30,
"full_snapshot_deep_dive_candidates": 5,
"full_snapshot_max_staleness_days": 7,
"full_snapshot_screening_clock_skew_seconds": 300,
}
)
return config
def _assert_partial_diagnostic(self, output: Path, prepared: dict, stage: str) -> dict:
diagnostic_path = output / "audit" / "partial-run-diagnostic.json"
self.assertTrue(diagnostic_path.is_file())
diagnostic = json.loads(diagnostic_path.read_text())
self.assertEqual(diagnostic["status"], "provider_budget_exhausted")
self.assertEqual(diagnostic["stage"], stage)
self.assertEqual(diagnostic["snapshot_id"], prepared["verdict"]["snapshot_id"])
self.assertEqual(
diagnostic["snapshot_verification_digest"], prepared["verification_digest"]
)
self.assertTrue(diagnostic["no_final_marketwide_conclusion"])
self.assertTrue(diagnostic["commit_marker"])
partial_record = diagnostic["artifacts"]["partial_enriched_estimates"]
partial_path = output / partial_record["path"]
self.assertTrue(partial_path.is_file())
self.assertEqual(
partial_record["sha256"], hashlib.sha256(partial_path.read_bytes()).hexdigest()
)
self.assertEqual(partial_record["row_count"], len(partial_path.read_text().splitlines()))
for key in ("run_summary", "next_action"):
record = diagnostic["artifacts"][key]
path = output / record["path"]
self.assertTrue(path.is_file())
self.assertEqual(record["sha256"], hashlib.sha256(path.read_bytes()).hexdigest())
summary = json.loads((output / "run-summary.json").read_text())
next_action = json.loads((output / "NEXT_ACTION.json").read_text())
for artifact in (summary, next_action):
self.assertTrue(artifact["no_final_marketwide_conclusion"])
self.assertNotIn("selected_symbols", artifact)
self.assertNotEqual(artifact.get("ranking_scope"), "final_marketwide")
progress = diagnostic["progress"]
self.assertEqual(
progress["pending_symbols"],
[
symbol
for symbol in progress["target_symbols"]
if symbol not in progress["completed_symbols"]
],
)
self.assertTrue(all(symbol == symbol.upper() for symbol in progress["target_symbols"]))
self.assertEqual(len(progress["target_symbols"]), len(set(progress["target_symbols"])))
return diagnostic
def _snapshot_bytes(self, snapshot_dir: Path) -> tuple[bytes, bytes]:
return (
(snapshot_dir / SNAP.MANIFEST_NAME).read_bytes(),
SNAP.shard_path(snapshot_dir, 0).read_bytes(),
)
def test_liquidity_budget_exhaustion_persists_progress_after_partial_work(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
snapshot_dir = _build_snapshot(root, now=now)
prepared = PIPELINE.prepare_screen_full_snapshot(
snapshot_dir,
analysis_as_of=now,
config=self._config(),
screening_started_at=now,
)
before = self._snapshot_bytes(snapshot_dir)
output = root / "run"
result = PIPELINE.execute_screen_full_snapshot(
BudgetOnLiquidityClient(successful_history_calls=2),
self._config(),
analysis_as_of=now,
output_dir=output,
prepared_snapshot=prepared,
include_packets=False,
)
self.assertEqual(result.exit_code, 2)
diagnostic = self._assert_partial_diagnostic(output, prepared, "liquidity")
progress = diagnostic["progress"]
self.assertEqual(len(progress["completed_symbols"]), 2)
self.assertEqual(len(progress["interrupted_symbols"]), 1)
self.assertEqual(len(progress["attempted_symbols"]), 3)
self.assertEqual(before, self._snapshot_bytes(snapshot_dir))
self.assertFalse((output / "audit" / "broad-screen-audit.json").exists())
def test_quality_probe_budget_exhaustion_persists_progress_after_partial_work(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
snapshot_dir = _build_snapshot(root, now=now)
prepared = PIPELINE.prepare_screen_full_snapshot(
snapshot_dir,
analysis_as_of=now,
config=self._config(),
screening_started_at=now,
)
before = self._snapshot_bytes(snapshot_dir)
output = root / "run"
result = PIPELINE.execute_screen_full_snapshot(
BudgetOnQualityClient(successful_quality_calls=1),
self._config(),
analysis_as_of=now,
output_dir=output,
prepared_snapshot=prepared,
include_packets=False,
)
self.assertEqual(result.exit_code, 2)
diagnostic = self._assert_partial_diagnostic(output, prepared, "quality_probe")
progress = diagnostic["progress"]
self.assertEqual(len(progress["completed_symbols"]), 1)
self.assertEqual(len(progress["interrupted_symbols"]), 1)
self.assertTrue((output / "audit" / "quality-probe-audit.json").is_file())
self.assertEqual(before, self._snapshot_bytes(snapshot_dir))
def test_quality_probe_actual_eps_budget_exhaustion_keeps_current_partial_row(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
snapshot_dir = _build_snapshot(root, now=now)
prepared = PIPELINE.prepare_screen_full_snapshot(
snapshot_dir,
analysis_as_of=now,
config=self._config(),
screening_started_at=now,
)
output = root / "run"
result = PIPELINE.execute_screen_full_snapshot(
BudgetOnActualEpsClient(successful_actual_calls=1),
self._config(),
analysis_as_of=now,
output_dir=output,
prepared_snapshot=prepared,
include_packets=False,
)
self.assertEqual(result.exit_code, 2)
diagnostic = self._assert_partial_diagnostic(output, prepared, "quality_probe")
self.assertEqual(
diagnostic["progress"]["interrupted_at"]["operation"],
"get_income_statement",
)
rows = {
json.loads(line)["symbol"]: json.loads(line)
for line in (output / "audit" / "enriched-estimates.partial.jsonl")
.read_text()
.splitlines()
}
interrupted = diagnostic["progress"]["interrupted_symbols"][0]
self.assertTrue(rows[interrupted]["quality_probe_resolved"])
self.assertFalse(rows[interrupted]["quality_probe_complete"])
def test_zero_progress_budget_exhaustion_is_still_fail_closed(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
snapshot_dir = _build_snapshot(root, now=now)
prepared = PIPELINE.prepare_screen_full_snapshot(
snapshot_dir,
analysis_as_of=now,
config=self._config(),
screening_started_at=now,
)
output = root / "run"
result = PIPELINE.execute_screen_full_snapshot(
BudgetOnLiquidityClient(successful_history_calls=0),
self._config(),
analysis_as_of=now,
output_dir=output,
prepared_snapshot=prepared,
include_packets=False,
)
self.assertEqual(result.exit_code, 2)
diagnostic = self._assert_partial_diagnostic(output, prepared, "liquidity")
self.assertEqual(diagnostic["progress"]["completed_symbols"], [])
self.assertEqual(len(diagnostic["progress"]["interrupted_symbols"]), 1)
def test_packet_budget_exhaustion_invalidates_nested_selection_and_keeps_packets(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
snapshot_dir = _build_snapshot(root, now=now)
prepared = PIPELINE.prepare_screen_full_snapshot(
snapshot_dir,
analysis_as_of=now,
config=self._config(),
screening_started_at=now,
)
output = root / "run"
result = PIPELINE.execute_screen_full_snapshot(
BudgetOnPacketClient(successful_packets=1),
self._config(),
analysis_as_of=now,
output_dir=output,
prepared_snapshot=prepared,
include_packets=True,
)
self.assertEqual(result.exit_code, 2)
diagnostic = self._assert_partial_diagnostic(output, prepared, "candidate_packets")
self.assertEqual(diagnostic["progress"]["interrupted_at"]["operation"], "get_profile")
preserved = diagnostic["artifacts"]["preserved"]
self.assertTrue(any("candidate-packets" in item["path"] for item in preserved))
broad = json.loads((output / "audit" / "broad-screen-audit.json").read_text())
self.assertEqual(broad["status"], "incomplete")
self.assertEqual(broad["conclusion_scope"], "diagnostic")
self.assertEqual(broad["ranking_scope"], "diagnostic")
self.assertEqual(broad["selection_outcome"], "incomplete_budget_exhausted")
self.assertEqual(broad["selected_symbols"], [])
self.assertEqual(broad["deep_dive_plan"]["selected_symbols"], [])
self.assertEqual(broad["deep_dive_plan"]["selected_count"], 0)
self.assertIsNone(broad["deep_dive_plan"]["selected_set_sha256"])
self.assertIsNone(broad["deep_dive_plan"]["commitment_payload"])
self.assertFalse(broad["deep_dive_plan"]["selected_set_is_committed"])
self.assertEqual(broad["candidate_pool"]["selected_count"], 0)
self.assertEqual(broad["candidate_pool"]["status"], "incomplete")
self.assertEqual(broad["candidate_pool"]["generation_audit"]["selected_symbols"], [])
self.assertEqual(
broad["candidate_pool"]["generation_audit"]["actual_selected_symbols"], []
)
self.assertEqual(broad["candidate_pool"]["generation_audit"]["selected_count"], 0)
self.assertEqual(
broad["candidate_pool"]["generation_audit"]["actual_selected_count"], 0
)
self.assertEqual(
broad["candidate_pool"]["generation_audit"]["lane_selected_counts"], {}
)
self.assertFalse(broad["candidate_pool"]["generation_audit"]["valid"])
self.assertEqual(
hashlib.sha256(
(output / "audit" / "broad-screen-results.jsonl").read_bytes()
).hexdigest(),
broad["candidate_pool"]["artifact_sha256"],
)
candidate_rows = [
json.loads(line)
for line in (output / "audit" / "broad-screen-results.jsonl")
.read_text()
.splitlines()
]
self.assertTrue(all(row["decision"]["status"] != "selected" for row in candidate_rows))
self.assertTrue(
all(row["decision"]["selection_eligible"] is False for row in candidate_rows)
)
self.assertTrue(
all("preselection_status" not in row["decision"] for row in candidate_rows)
)
self.assertTrue(all("prior_decision" in row for row in candidate_rows))
self.assertTrue(
any(row["prior_decision"]["status"] == "selected" for row in candidate_rows)
)
interrupted = diagnostic["progress"]["interrupted_symbols"][0]
self.assertFalse(
(output / "candidate-packets" / f"{interrupted}.fmp-packet.json").exists()
)
def test_full_snapshot_rejects_reused_output_directory(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
snapshot_dir = _build_snapshot(root, now=now)
prepared = PIPELINE.prepare_screen_full_snapshot(
snapshot_dir,
analysis_as_of=now,
config=self._config(),
screening_started_at=now,
)
output = root / "run"
output.mkdir()
sentinel = output / "run-summary.json"
sentinel.write_text('{"status":"ready_for_underwriting"}\n')
with self.assertRaisesRegex(ValueError, "must be new and empty"):
PIPELINE.execute_screen_full_snapshot(
FakeScreenClient(),
self._config(),
analysis_as_of=now,
output_dir=output,
prepared_snapshot=prepared,
include_packets=False,
)
self.assertEqual(sentinel.read_text(), '{"status":"ready_for_underwriting"}\n')
def test_complete_current_snapshot_emits_consistent_marketwide_audit(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
snapshot_dir = _build_snapshot(root, now=now)
prepared = PIPELINE.prepare_screen_full_snapshot(
snapshot_dir,
analysis_as_of=now,
config=self._config(),
screening_started_at=now,
)
output = root / "run"
result = PIPELINE.execute_screen_full_snapshot(
FakeScreenClient(),
self._config(),
analysis_as_of=now,
output_dir=output,
prepared_snapshot=prepared,
include_packets=False,
)
self.assertEqual(result.exit_code, 0)
self.assertEqual(result.summary["ranking_scope"], "final_marketwide")
self.assertEqual(result.summary["conclusion_scope"], "full_listing_universe")
self.assertEqual(result.summary["estimate_acquisition_mode"], "sharded_snapshot")
self.assertEqual(result.summary["economic_attempt_count"], 8)
self.assertEqual(result.summary["listing_universe_count"], 8)
self.assertEqual(
result.summary["snapshot_verification_digest"], prepared["verification_digest"]
)
broad = json.loads((output / "audit" / "broad-screen-audit.json").read_text())
self.assertEqual(broad["conclusion_scope"], "full_listing_universe")
self.assertEqual(broad["deep_dive_plan"]["max_deep_dive_candidates"], 5)
universe_audit = [
json.loads(line)
for line in (output / "audit" / "universe-audit-results.jsonl")
.read_text()
.splitlines()
]
self.assertEqual(
{row["symbol"] for row in universe_audit}, {f"S{i:02d}" for i in range(8)}
)
self.assertEqual(
{row["estimate_attempt"]["classification"] for row in universe_audit},
{"evaluable"},
)
self.assertTrue(
all(
row["estimate_attempt"]["snapshot_verification_digest"]
== prepared["verification_digest"]
for row in universe_audit
)
)
selected = set(result.summary["selected_symbols"])
final_pool = {
json.loads(line)["symbol"]: json.loads(line)
for line in (output / "audit" / "provider-prefilter-pool.jsonl")
.read_text()
.splitlines()
}
self.assertTrue(selected)
self.assertTrue(
all(final_pool[symbol]["quality_probe_attempted"] for symbol in selected)
)
def test_historical_as_of_is_rejected_before_enrichment(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
historical = now - timedelta(days=30)
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = _build_snapshot(Path(tmp), now=historical)
with self.assertRaisesRegex(ValueError, "current-only"):
PIPELINE.prepare_screen_full_snapshot(
snapshot_dir,
analysis_as_of=historical,
config=self._config(),
screening_started_at=now,
)
def test_snapshot_scope_must_cover_current_screen_request(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
config = self._config()
config["min_market_cap"] = 100_000_000
config["max_market_cap"] = 100_000_000_000
with tempfile.TemporaryDirectory() as tmp:
snapshot_dir = _build_snapshot(Path(tmp), now=now)
with self.assertRaisesRegex(ValueError, "does not cover"):
PIPELINE.prepare_screen_full_snapshot(
snapshot_dir,
analysis_as_of=now,
config=config,
screening_started_at=now,
)
def test_fifty_name_pool_is_not_silently_limited_to_forty(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
config = self._config()
config["full_snapshot_pool_size"] = 50
config["exact_liquidity_limit"] = 40
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
snapshot_dir = _build_snapshot(root, now=now, count=55)
prepared = PIPELINE.prepare_screen_full_snapshot(
snapshot_dir,
analysis_as_of=now,
config=config,
screening_started_at=now,
)
result = PIPELINE.execute_screen_full_snapshot(
FakeScreenClient(),
config,
analysis_as_of=now,
output_dir=root / "run",
prepared_snapshot=prepared,
include_packets=False,
)
self.assertEqual(result.summary["exact_liquidity_target_count"], 50)
self.assertEqual(result.summary["provider_prefilter_pool_count"], 50)
self.assertEqual(len(result.summary["selected_symbols"]), 5)
def test_liquidity_failures_backfill_until_target_is_reached(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
config = self._config()
config["full_snapshot_pool_size"] = 50
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
snapshot_dir = _build_snapshot(root, now=now, count=75)
prepared = PIPELINE.prepare_screen_full_snapshot(
snapshot_dir,
analysis_as_of=now,
config=config,
screening_started_at=now,
)
client = BackfillScreenClient(failed_history_calls=25)
result = PIPELINE.execute_screen_full_snapshot(
client,
config,
analysis_as_of=now,
output_dir=root / "run",
prepared_snapshot=prepared,
include_packets=False,
)
self.assertEqual(client.history_calls, 75)
self.assertEqual(result.summary["exact_liquidity_target_count"], 75)
self.assertEqual(result.summary["provider_prefilter_pool_count"], 50)
self.assertEqual(result.summary["ranking_scope"], "final_marketwide")
def test_all_symbols_classified_no_estimates_is_marketwide_no_candidates(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
snapshot_dir = _build_snapshot(
root,
now=now,
count=3,
with_estimates=False,
)
prepared = PIPELINE.prepare_screen_full_snapshot(
snapshot_dir,
analysis_as_of=now,
config=self._config(),
screening_started_at=now,
)
result = PIPELINE.execute_screen_full_snapshot(
FakeScreenClient(),
self._config(),
analysis_as_of=now,
output_dir=root / "run",
prepared_snapshot=prepared,
include_packets=False,
)
self.assertEqual(result.exit_code, 0)
self.assertEqual(result.summary["status"], "no_candidates_in_marketwide_snapshot")
self.assertEqual(result.summary["ranking_scope"], "final_marketwide")
self.assertEqual(result.summary["snapshot_classification_totals"]["no_estimates"], 3)
def test_cli_invalid_snapshot_creates_no_cache_raw_or_run_tree(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
output = root / "reports"
cache = root / "cache" / "fmp.sqlite3"
raw = root / "raw"
rc = PIPELINE.main(
[
"--stage",
"screen-full-snapshot",
"--snapshot-dir",
str(root / "missing"),
"--analysis-as-of",
now.isoformat(),
"--output-dir",
str(output),
"--cache-path",
str(cache),
"--raw-store-dir",
str(raw),
]
)
self.assertEqual(rc, 1)
self.assertFalse(output.exists())
self.assertFalse(cache.exists())
self.assertFalse(raw.exists())
def test_cli_malformed_shard_fails_before_creating_output_state(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
snapshot_dir = _build_snapshot(root, now=now)
payload = b'{"broken"\n'
path = SNAP.shard_path(snapshot_dir, 0)
path.write_bytes(payload)
manifest = SNAP.load_manifest(snapshot_dir)
manifest["shards"]["0"]["shard_sha256"] = hashlib.sha256(payload).hexdigest()
SNAP.write_manifest(snapshot_dir, manifest)
output = root / "reports"
cache = root / "cache" / "fmp.sqlite3"
raw = root / "raw"
stdout = io.StringIO()
with redirect_stdout(stdout):
rc = PIPELINE.main(
[
"--stage",
"screen-full-snapshot",
"--snapshot-dir",
str(snapshot_dir),
"--analysis-as-of",
now.isoformat(),
"--output-dir",
str(output),
"--cache-path",
str(cache),
"--raw-store-dir",
str(raw),
]
)
result = json.loads(stdout.getvalue())
self.assertEqual(rc, 1)
self.assertEqual(result["status"], "failed")
self.assertNotIn("ranking_scope", result)
self.assertFalse(output.exists())
self.assertFalse(cache.exists())
self.assertFalse(raw.exists())
def test_cli_historical_collection_basis_is_rejected_before_side_effects(self) -> None:
now = datetime.now(timezone.utc).replace(microsecond=0)
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
output = root / "reports"
cache = root / "cache" / "fmp.sqlite3"
raw = root / "raw"
snapshot = root / "snapshot"
rc = PIPELINE.main(
[
"--stage",
"collect-estimates",
"--shard-index",
"0",
"--shard-count",
"1",
"--snapshot-dir",
str(snapshot),
"--analysis-as-of",
(now - timedelta(days=30)).isoformat(),
"--output-dir",
str(output),
"--cache-path",
str(cache),
"--raw-store-dir",
str(raw),
]
)
self.assertEqual(rc, 1)
self.assertFalse(output.exists())
self.assertFalse(cache.exists())
self.assertFalse(raw.exists())
self.assertFalse(snapshot.exists())
if __name__ == "__main__":
unittest.main()
scripts/tests/test_unit_reconciliation.py
"""Round-5 review P0s: FPI unit reconciliation, unit-anomaly gates, ET filing clock."""
from __future__ import annotations
import json
import sys
import unittest
from datetime import datetime, timezone
from pathlib import Path
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
if str(SCRIPTS_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPTS_DIR))
import normalize_estimates as NORM # noqa: E402
import run_pipeline as PIPELINE # noqa: E402
import screen_universe as SCREEN # noqa: E402
ASSETS_DIR = SCRIPTS_DIR.parent / "assets"
AS_OF_TEXT = "2026-08-22T14:00:00-07:00"
def _base_row() -> dict:
return json.loads(
(ASSETS_DIR / "enriched-candidate-pool.example.jsonl").read_text().splitlines()[0]
)
def _config() -> dict:
return json.loads((ASSETS_DIR / "screening-config.example.json").read_text())
def _decide(row: dict) -> dict:
return SCREEN._candidate_decision(row, _config(), "liquidity_stratified_estimates", AS_OF_TEXT)[
"decision"
]
class UnitReconciliationGateTests(unittest.TestCase):
def test_foreign_issuer_without_verification_is_blocked(self) -> None:
row = _base_row()
row["country"] = "CN"
d = _decide(row)
self.assertIn("unit_reconciliation_required", d.get("blocking_review_reasons") or [])
self.assertFalse(d.get("selection_eligible"))
def test_verified_foreign_issuer_passes_the_gate(self) -> None:
row = _base_row()
row["country"] = "CN"
row["unit_reconciliation_verified"] = True
d = _decide(row)
self.assertNotIn("unit_reconciliation_required", d.get("blocking_review_reasons") or [])
def test_qfin_like_ratios_are_unit_mismatch_not_deep_value(self) -> None:
# QFIN as shipped: $8.80 ADS price against CNY-denominated EPS/FCF.
row = _base_row()
row["country"] = "CN"
row["price"] = 8.80
row["fcf_yield_pct"] = 93.72
row["latest_actual_eps"] = 85.58
d = _decide(row)
self.assertIn("unit_mismatch_suspected", d.get("screen_fail_reasons") or [])
self.assertFalse(d.get("selection_eligible"))
def test_domestic_eps_above_twice_price_is_stopped(self) -> None:
row = _base_row()
row["latest_actual_eps"] = float(row.get("price") or 20.0) * 3.0
d = _decide(row)
self.assertIn("unit_mismatch_suspected", d.get("screen_fail_reasons") or [])
self.assertFalse(d.get("selection_eligible"))
def test_domestic_implausible_fcf_yield_is_stopped(self) -> None:
row = _base_row()
row["fcf_yield_pct"] = 61.0
d = _decide(row)
self.assertIn("unit_mismatch_suspected", d.get("screen_fail_reasons") or [])
self.assertFalse(d.get("selection_eligible"))
def test_unverified_foreign_issuer_resolves_after_exhaustion(self) -> None:
row = _base_row()
row["country"] = "CN"
[marked] = PIPELINE.mark_unit_reconciliation_exhaustion([row], source_id="fmp-probe-x")
self.assertTrue(marked["enrichment_exhausted"])
self.assertIn("ADS-unit", marked["enrichment_exhaustion_reason"])
d = _decide(marked)
self.assertEqual(d.get("status"), "unavailable_after_enrichment")
self.assertFalse(d.get("selection_eligible"))
def test_exhaustion_marker_leaves_domestic_and_verified_rows_alone(self) -> None:
rows = [
{"symbol": "USA", "country": "US"},
{"symbol": "OKF", "country": "CN", "unit_reconciliation_verified": True},
]
out = PIPELINE.mark_unit_reconciliation_exhaustion(rows, source_id="src")
for row in out:
self.assertNotIn("enrichment_exhausted", row)
class UnitContextFailClosedTests(unittest.TestCase):
"""Round-8 review: an UNKNOWN unit context is unreconciled, never domestic."""
def test_missing_country_with_cny_currency_is_blocked(self) -> None:
row = _base_row()
row["country"] = None
row["currency"] = "CNY"
d = _decide(row)
self.assertIn("unit_reconciliation_required", d.get("blocking_review_reasons") or [])
self.assertFalse(d.get("selection_eligible"))
def test_missing_country_with_foreign_isin_is_blocked(self) -> None:
row = _base_row()
row["country"] = None
row["currency"] = None
row["isin"] = "KYG123456789"
d = _decide(row)
self.assertIn("unit_reconciliation_required", d.get("blocking_review_reasons") or [])
self.assertFalse(d.get("selection_eligible"))
def test_adr_flag_is_blocked_even_with_us_country(self) -> None:
row = _base_row()
row["country"] = "US"
row["is_adr"] = True
d = _decide(row)
self.assertIn("unit_reconciliation_required", d.get("blocking_review_reasons") or [])
def test_proven_domestic_row_passes_the_gate(self) -> None:
row = _base_row()
row["country"] = "US"
row["currency"] = "USD"
row["isin"] = "US1564311082"
d = _decide(row)
self.assertNotIn("unit_reconciliation_required", d.get("blocking_review_reasons") or [])
def test_requires_unit_reconciliation_unit_cases(self) -> None:
req = SCREEN.requires_unit_reconciliation
self.assertTrue(req({})) # nothing proven -> fail closed
self.assertTrue(req({"country": "US", "currency": "CNY"}))
self.assertTrue(req({"country": "US", "isin": "KYG8464W1069"}))
self.assertTrue(req({"country": "US", "is_adr": True}))
self.assertFalse(req({"country": "US"}))
self.assertFalse(req({"country": "US", "currency": "USD", "isin": "US1564311082"}))
self.assertFalse(req({"country": "CN", "unit_reconciliation_verified": True}))
def test_normalize_listing_keeps_unit_identity_signals(self) -> None:
out = PIPELINE.normalize_listing(
{
"symbol": "QFIN",
"companyName": "Qifu Technology",
"price": 8.8,
"marketCap": 1_200_000_000,
"country": "CN",
"currency": "CNY",
"isin": "KYG8464W1069",
"isAdr": True,
},
"NASDAQ",
)
self.assertEqual(out["isin"], "KYG8464W1069")
self.assertTrue(out["is_adr"])
self.assertTrue(SCREEN.requires_unit_reconciliation(out))
class AcceptedDateTimezoneTests(unittest.TestCase):
def _statements(self, accepted: str) -> list[dict]:
return [{"period": "FY", "date": "2025-12-31", "acceptedDate": accepted, "epsDiluted": 2.0}]
def test_summer_acceptance_is_eastern_not_utc(self) -> None:
# 17:23:10 ET (EDT) == 21:23:10 UTC. At 16:00 ET (20:00 UTC) the
# filing is NOT yet public; the old UTC reading leaked it 4h early.
statements = self._statements("2026-04-03 17:23:10")
before = datetime(2026, 4, 3, 20, 0, tzinfo=timezone.utc)
after = datetime(2026, 4, 3, 21, 30, tzinfo=timezone.utc)
self.assertEqual(
PIPELINE._verified_annual_actual(statements, analysis_as_of=before), (None, None)
)
self.assertEqual(
PIPELINE._verified_annual_actual(statements, analysis_as_of=after),
(2.0, "2025-12-31"),
)
def test_winter_acceptance_uses_est_offset(self) -> None:
# 17:00 ET (EST) == 22:00 UTC.
statements = self._statements("2026-01-15 17:00:00")
before = datetime(2026, 1, 15, 21, 30, tzinfo=timezone.utc)
after = datetime(2026, 1, 15, 22, 30, tzinfo=timezone.utc)
self.assertEqual(
PIPELINE._verified_annual_actual(statements, analysis_as_of=before), (None, None)
)
self.assertEqual(
PIPELINE._verified_annual_actual(statements, analysis_as_of=after),
(2.0, "2025-12-31"),
)
class PublishedBeforeTimezoneTests(unittest.TestCase):
def _record(self, published: str) -> dict:
return {
"is_actual": True,
"period_end": "2026-03-31T00:00:00+00:00",
"published_at": published,
"eps_avg": 1.0,
}
def test_naive_publish_stamp_is_eastern(self) -> None:
rec = self._record("2026-04-03 17:23:10")
early = datetime(2026, 4, 3, 20, 0, tzinfo=timezone.utc) # 16:00 ET
late = datetime(2026, 4, 3, 22, 0, tzinfo=timezone.utc)
self.assertIsNone(NORM._latest_actual_period([rec], early))
self.assertIsNotNone(NORM._latest_actual_period([rec], late))
def test_date_only_publish_stamp_counts_after_the_whole_day(self) -> None:
rec = self._record("2026-04-03")
same_day = datetime(2026, 4, 3, 12, 0, tzinfo=timezone.utc)
next_day = datetime(2026, 4, 4, 6, 0, tzinfo=timezone.utc) # past 23:59:59 EDT
self.assertIsNone(NORM._latest_actual_period([rec], same_day))
self.assertIsNotNone(NORM._latest_actual_period([rec], next_day))
class TaxonomyRound5Tests(unittest.TestCase):
def test_fmp_auto_dealership_label_is_mapped(self) -> None:
# 14 dealers (ABG, AN, KMX, LAD, PAG, ...) sat in "general" because the
# old needle never matched FMP's actual "Auto - Dealerships" label.
self.assertEqual(
PIPELINE.infer_sector_profile_type("Consumer Cyclical", "Auto - Dealerships"),
"auto_dealership",
)
def test_investment_banks_are_not_deposit_banks(self) -> None:
profile = PIPELINE.infer_sector_profile_type(
"Financial Services", "Investment - Banking & Investment Services"
)
self.assertEqual(profile, "capital_markets")
# advisory firms are valued on ordinary earnings multiples: never
# routed to the deposit-bank (P/TBV, NIM) blocking gate.
self.assertNotIn("capital_markets", SCREEN.SECTOR_PROFILES)
def test_bdc_detected_from_company_name_within_asset_management(self) -> None:
self.assertEqual(
PIPELINE.infer_sector_profile_type(
"Financial Services", "Asset Management", "Golub Capital BDC, Inc."
),
"bdc",
)
self.assertEqual(
PIPELINE.infer_sector_profile_type(
"Financial Services", "Asset Management", "Prospect Business Development"
),
"bdc",
)
def test_override_config_key_exists_for_unmarked_bdcs(self) -> None:
self.assertIn("sector_profile_overrides", PIPELINE.DEFAULT_CONFIG)
if __name__ == "__main__":
unittest.main()
SKILL.md
---
name: us-undervalued-growth-screener
description: Autonomously screen NYSE, Nasdaq, and NYSE American operating-company stocks for undervalued-growth/GARP opportunities using forward same-basis valuation, driver-derived EPS/FCF forecasts, primary-source financial verification, SBC and dilution controls, sector and cycle normalization, auditable candidate-pool coverage, and fail-closed final reporting. Use when asked to find, screen, rank, or refresh US undervalued-growth stocks, including minimal requests with no ticker list or parameters.
---
# US Undervalued Growth Screener — v3.6 (Claude Code Direct-FMP)
## Overview
Run an end-to-end US undervalued-growth/GARP screen from a minimal request. Find companies whose EPS or FCF per share can compound enough to support attractive two- to three-year returns **without assuming multiple expansion**, while controlling for accounting basis, forecast construction, SBC, dilution, leverage, cyclicality, corporate actions, peer context, source freshness, and evidence quality.
**Claude Code is the preferred execution environment.** In Claude Code, run the local direct-FMP pipeline once. The Python process performs bulk retrieval, persistent caching, FY1 normalization, liquidity calculation, four-lane discovery, and deterministic broad screening while keeping raw FMP payloads on disk and out of the model context. Claude reads only the compact run summary and selected candidate packets, then completes SEC/IR underwriting and the existing strict evaluation sequence.
Treat a request such as **“use this skill to screen for undervalued-growth stocks” as complete**. Resolve defaults, collect current data, choose a viable acquisition path, checkpoint the work, repair obtainable blockers, and return the finished result in the same task. Never ask the user to supply a ticker list, API-plan details, output path, or a separate “continue” instruction unless the user explicitly narrows the scope.
**Deep-dive budget and lane coverage:** The bounded direct-FMP path defaults to `max_deep_dive_candidates: 3`. Three selected names cannot represent all four research lanes. The default lane targets are core GARP 2, high-growth exception 1, quality near miss 1, and cyclical normalization 1 (five slots total). A three-name run prioritizes candidates across lanes; it does not promise one name per lane. For a five-slot lane-first selection, set `max_deep_dive_candidates: 5` in a local copy of `assets/claude-code-config.example.json` and rerun the bounded pipeline with that config. Eligible candidates and diversification preferences still determine actual lane representation. The separate full-snapshot path requires `full_snapshot_deep_dive_candidates: 5`. See `references/claude-code-execution.md` for selection and budget-change details.
## Non-Negotiable Runtime Preflight
Before reading or reusing any prior run artifact, verify the installed runtime:
```bash
python3 skills/us-undervalued-growth-screener/scripts/run_pipeline.py --version
python3 skills/us-undervalued-growth-screener/scripts/screen_universe.py --version
python3 skills/us-undervalued-growth-screener/scripts/build_discovery_pool.py --version
python3 skills/us-undervalued-growth-screener/scripts/build_provider_prefilter_pool.py --version
python3 skills/us-undervalued-growth-screener/scripts/normalize_estimates.py --version
python3 skills/us-undervalued-growth-screener/scripts/manage_run_state.py --version
python3 skills/us-undervalued-growth-screener/scripts/evaluate_candidates.py --version
python3 skills/us-undervalued-growth-screener/scripts/prepublish_audit.py --version
python3 skills/us-undervalued-growth-screener/scripts/bundle_run_artifacts.py --version
```
Every command must report the same metadata:
```text
skill_version = 3.6.1
schema_version = 3
contract_revision = 3.5
runtime_fingerprint = ug-v3.6.1-claude-code-direct-fmp-20260830
```
Discard and regenerate any audit, checkpoint, or snapshot whose runtime metadata differs. Do not mix scripts, assets, or run artifacts from v3.1 through v3.5. A stale or cached same-name skill is a hard execution failure, not a warning.
## Autonomous Completion Contract
For a minimal request, perform all of the following without handing control back to the user:
1. Fix `analysis_as_of` and the latest completed US regular-session close.
2. Collect current market context with field-level source support and freshness checks.
3. In Claude Code, invoke `run_pipeline.py` instead of issuing bulk FMP MCP calls. Keep provider payloads on disk and expose only compact summaries to the model.
4. Audit the requested NYSE/Nasdaq/NYSE American listing universe through adaptive, exhausted market-cap bands when provider responses saturate.
5. Build a reproducible economic candidate pool. Do not require complete financial statements across the whole market.
6. Distinguish **enrichment attempted** from **enrichment resolved**.
7. Apply the deterministic broad-screen script. Do not replace its statuses with ad hoc LLM cutoffs.
8. Select up to three economically plausible deep-dive candidates in Claude Code through deterministic multi-lane sampling: core GARP, high-growth exceptions, quality near misses, and cyclicals requiring normalization. Apply a two-name sector cap when alternatives exist. Growth thresholds remain guidelines, not isolated hard gates.
9. Resolve every row in the chosen candidate pool or record sourced exhaustion evidence.
10. Perform corporate-action preflight and primary-source underwriting for every selected symbol.
11. Save every selected symbol as a verified candidate record, including `review_required`, `screened_out`, and `excluded` outcomes.
12. Assemble the schema-v3 / contract-v3.5 snapshot.
13. Run `evaluate_candidates.py --strict --require-final`.
14. Apply the final quality eligibility gate; route weak-cash-flow, low-ROIC, overleveraged, heavily dilutive, fragile-low-case, or severe-LOE names to `conditional` or `review_required`.
15. Run `prepublish_audit.py`; repair every obtainable blocker and rerun.
16. Run `bundle_run_artifacts.py` to produce a self-contained audit ZIP containing every referenced artifact.
17. Present a formal ranking and downloadable bundle only when all gates pass.
An exit code of `2` means **continue the same execution**: enrich the queue, verify the pool-generation audit, complete selected deep dives, or repair contract failures. It never means “ask the user to say continue.” After attaching the audit, run `manage_run_state.py next-action`; execute the returned action and every returned symbol. Never ask whether to process two versus five selected names. To change the budget, rerun the broad screen first so omitted names become `deferred_by_budget`.
## Completion Semantics
### Final ranking from an audited bounded pool
A reproducibly generated bounded pool may support a **scoped final ranking** when:
```text
candidate-pool generation audit is valid
candidate_pool_exhausted = true
all_rows_resolved = true
unresolved_count = 0
queue_count = 0
all selected symbols have verified candidate records
strict final evaluation passes
```
Label the conclusion scope, such as `stratified_discovery_pool` or `provider_prefilter`. Do not imply that unexamined market listings were economically screened.
### Market-wide “no qualifying candidates”
A market-wide no-candidates conclusion requires all of the above plus full in-scope economic coverage:
```text
conclusion_scope = full_listing_universe
candidate_pool covers every in-scope listing
in_scope_missing_count = 0
at least one row was economically assessable
selected_symbols = []
```
### Bounded-pool “no qualifying candidates”
A bounded pool may conclude only:
```text
no qualifying candidates in the audited bounded pool
```
It must not claim that the entire US small/mid-cap market has no qualifying company.
Read `references/autonomous-execution.md` before a live minimal-request run.
## When to Use
Use this skill to:
- Discover and rank US-listed undervalued-growth or GARP stocks.
- Screen operating-company common stocks on NYSE, Nasdaq, and NYSE American.
- Test whether EPS or FCF-per-share growth alone supports roughly 30%–50% upside over two to three years.
- Refresh a prior screen after earnings, guidance, filings, corporate actions, or estimate revisions.
- Compare candidates on forward valuation, growth durability, ROIC, standard FCF, SBC, dilution, peers, cycle risk, and sector-specific KPIs.
Do not use it for:
- A generic single-ticker report after a stock is already selected; use `us-stock-analysis`.
- Pure dividend, momentum, technical-pattern, pre-revenue biotechnology, or merger-arbitrage screening.
- Automatic order placement.
## Prerequisites
- Python 3.9 or later.
- `requests` for the generated direct-FMP client; deterministic evaluation and audit scripts otherwise use the standard library.
- `FMP_API_KEY` in the environment for Claude Code direct mode. Never commit or print the key.
- Current SEC, company-IR, and macro sources accessible for selected-company underwriting.
- Writable `reports/` and `.cache/` directories.
- No specific paid FMP plan is assumed. Bulk endpoint failures fall back to bounded per-symbol enrichment and are disclosed in diagnostics.
## Default Scope
Unless the user specifies otherwise:
- Exchanges: NYSE, Nasdaq, NYSE American.
- Security type: active operating-company common stock.
- Market-cap focus: USD 500 million–20 billion.
- Minimum price: USD 5.
- Preferred average daily dollar volume: USD 5 million; hard floor USD 1 million.
- Claude Code provider-prefilter pool: target 30 symbols after code-side economics and verified liquidity; bounded per-symbol fallback may inspect up to 80 names without loading their provider payloads into model context.
- Deep-dive budget: three symbols by default on the bounded Claude Code path, selected from four deterministic research lanes; three slots cannot represent all four lanes. Once selected, every symbol must be resolved; lower the budget only by rerunning the broad screen so omitted names become `deferred_by_budget`.
- Maximum ranked output: ten, though the verified deep-dive set may be smaller.
- Minimum formal constant-multiple upside: 30% over a supported two- or three-year horizon.
- Multiple-contraction stress: current forward multiple reduced by 20%.
- Minimum analysts for a rankable consensus horizon: three, unless a fully sourced independent forecast is constructed.
### Immutable request scope and bounded execution
For a minimal request, the user-requested market-cap scope is always USD 500M–20B. Never rewrite the run config so a convenient 3–4B band becomes the requested scope. Record `user_requested_scope` and `executed_scope` separately. A narrower executed scope is incomplete unless the user explicitly requested it; context or tool-budget pressure is not authorization. Stream listing pages/bands to JSONL rather than loading every row into the conversation context.
### Liquidity evidence
Never calculate ADDV from one session's volume. Candidate generation requires a provider average-dollar-volume measure or `price × average_volume` with an explicit averaging window of at least 20 trading days and source IDs. Rows lacking valid average-liquidity evidence remain `needs_enrichment` and cannot enter the discovery pool.
### Current forward horizon
A current P/E must be explicitly NTM or FY1 and must reconcile to a positive current forward EPS, price, fiscal-year/period metadata, estimate date, analyst count, and source IDs. Generic outer-year P/E values are invalid. A missing FY1/NTM row, a range crossing zero, or extreme forecast dispersion sends the name to enrichment or `unavailable_after_enrichment`; it must never appear as a low-P/E selection.
## Result-Quality Controls
### Deterministic multi-lane candidate discovery
Do not let one global score fill the research budget with a single style or sector. Allocate the default five deep dives across these lanes when qualifying names exist:
- two core GARP names,
- one high-growth exception with Forward P/E up to 30x,
- one quality near miss whose low valuation may compensate for sub-guideline headline growth,
- one cyclical candidate that requires an explicit mid-cycle model.
Backfill unused slots by deterministic priority and limit selections to two per sector when alternatives exist. Report each selected name's `selection_lane`. The LLM may explain these decisions but may not replace them with ad hoc cutoffs.
### Formal eligibility quality gate
A name that passes the upside test is not automatically `eligible`. Formal ranking also requires:
- final score at least 70,
- SBC-adjusted FCF yield at least 3% **or** EV/FCF at most 30x,
- ROIC at least 8%,
- Net Debt/EBITDA no more than 3.0x when applicable,
- diluted-share CAGR no more than 5%,
- at least 15% upside under a supported low-consensus case,
- no severe LOE tail loss worse than the configured threshold.
One or two ordinary failures may produce `conditional`; severe FCF or LOE failures and broader weakness produce `review_required`. Never label a low-P/E/high-EV-FCF transition story as a formal winner merely because average-consensus EPS implies 30% upside.
### Independent forecast-driver evidence
Every ranked year-2/year-3 bridge must use `construction_method=independent_driver_model`. Each revenue, margin, interest, tax, share-count, FCF, and adjustment driver needs an origin and source IDs. Mark any driver solved backwards from target EPS with `target_solved=true`; this fails the bridge. A residual adjustment that merely forces the model to consensus is not independent validation.
### Publish only self-contained results
The final report is not publishable until `prepublish_audit.py` verifies all referenced audit files, counts, hashes, scenario arithmetic, candidate statuses, final-three labels, and absence of unfinished-run language. Package the entire run directory with `bundle_run_artifacts.py`; a ZIP containing only Markdown/JSON summaries is incomplete.
## Data Acquisition Strategy
### Preferred Claude Code path — direct FMP, compact model context
Run one local command:
```bash
python3 skills/us-undervalued-growth-screener/scripts/run_pipeline.py \
--config skills/us-undervalued-growth-screener/assets/claude-code-config.example.json \
--output-dir reports/us-undervalued-growth-screener
```
The generated FMP client reuses the repository's central `scripts/fmp_client/` source-of-truth pattern. It writes raw responses and a persistent SQLite cache to disk. Do **not** paste or read the raw provider trees into the language-model context. Read only `run-summary.json`, `NEXT_ACTION.json`, `audit/broad-screen-audit.json`, and the compact packets under `candidate-packets/`.
For a market-wide run on a plan without bulk estimates, collect every frozen
universe shard first, then screen the verified snapshot:
```bash
python3 skills/us-undervalued-growth-screener/scripts/run_pipeline.py \
--stage collect-estimates --shard-index 0 --shard-count 8 \
--snapshot-dir .cache/us-garp/snapshot-current \
--config skills/us-undervalued-growth-screener/assets/claude-code-config.example.json \
--resume
python3 skills/us-undervalued-growth-screener/scripts/run_pipeline.py \
--stage screen-full-snapshot \
--snapshot-dir .cache/us-garp/snapshot-current \
--config skills/us-undervalued-growth-screener/assets/claude-code-config.example.json \
--output-dir reports/us-undervalued-growth-screener
```
Run each shard once without `--resume`; use `--resume` only to continue an
existing partial shard. Both collection and screening are current-only:
collection fixes the estimate-normalization basis, while screening adds exact
liquidity and TTM quality evidence. The first collection binds the exhausted
listing-enumeration audit into the snapshot manifest. Screening performs a
read-only content/freshness preflight before creating provider or run
artifacts, carries a digest binding the enumeration proof and every verified
shard into downstream audits, backfills liquidity failures until 50 valid
names or full eligible exhaustion, and commits five probed deep-dive names. Read
`references/full-universe-snapshot.md` for the full collection and screening
contract.
If the provider call budget is exhausted during screen enrichment, the command
exits 2 and writes `audit/partial-run-diagnostic.json` plus
`audit/enriched-estimates.partial.jsonl`. The diagnostic is the last atomic
commit marker and records the snapshot ID/digest, stage, deterministic
attempted/completed/pending/interrupted symbols, interrupted provider
operation, provider counters, and hashes for the partial artifacts. Read a
partial artifact only when that marker exists and its recorded hashes and row
counts verify; a missing or mismatched marker makes the partial output
non-authoritative. `screen-full-snapshot` has no `--resume`: rerun the same
verified snapshot after restoring the provider budget, allowing new cache/raw
records from successful calls and reusing existing cache/raw data without
deleting or rewriting it. A packet-stage budget failure also marks
`broad-screen-audit.json` incomplete/diagnostic, clears all nested selection
commitment fields, and rewrites **all** broad-screen rows (including rejected
rows) to `deferred_by_budget` with selection eligibility disabled. The original
`decision` is preserved in `prior_decision`, and original top-level selection
fields in `prior_selection`; these archives are audit-only, never selection
authority, and repeated invalidation retains the first archive. Neither the
audit nor any partial output is a final market-wide conclusion. Quality-probe
`calls_used` and `actual_eps_calls` count FMP provider-budget units: cached or
unsent calls cost zero, and consumed retry responses remain counted. Each retry uses a new empty run directory. The CLI adds a unique
attempt suffix and keeps raw responses in a sibling attempt directory, while
direct callers must not reuse a non-empty output directory.
The direct runner first attempts bulk ratios, key metrics, estimate, and EOD datasets. It falls back to bounded per-symbol enrichment only when bulk access is unavailable; a plan-gated (402/403) bulk endpoint is remembered in the cache for 30 days so later runs do not spend calls re-probing it. On the fallback path the estimate seed is a stratified sector × market-cap sample (√-weighted Hamilton quota, liquidity-ranked within cells, hash tie-break) whose size is derived from the remaining call budget; `audit/seed-audit.json` states the selection basis. Before pool selection, the top lane candidates receive a `key-metrics-ttm` quality probe (ROIC, FCF yield, EV/FCF, leverage, SBC) and a probe-resolved row with SBC-adjusted FCF yield below 1% cannot enter any lane except `high_growth_exception`. Exact 20-day ADDV work is prioritized by the four economic lanes, not by ticker order. Read `references/claude-code-execution.md` and `references/migration-v3.6-to-v3.6.1.md` for the full execution contract.
### Layer 1 — Listing-Universe Audit
Collect only fields suitable for broad coverage:
- symbol and exchange,
- active/common-stock status,
- price and quote timestamp,
- market capitalization,
- average daily dollar volume or enough data to calculate it,
- sector and industry when available.
Do not substitute a share-volume floor for an ADDV threshold. If an API can filter only by share volume, fetch broadly, calculate `price × average volume`, and apply the dollar-liquidity rule locally.
Target at least 95% listing-field completeness. Prove enumeration using provider totals and exhausted pagination or exhausted gap-free market-cap bands. Matching only the requested lower and upper market-cap endpoints is not proof of completeness.
### Layer 2 — Economic Candidate Pool
Choose the best route in this order:
1. Multi-lane provider prefilter built with `build_provider_prefilter_pool.py`. Retrieve broad rows separately for core GARP, P/E 21–30 high growth, low-P/E quality near misses, and cyclicals requiring normalization; union and audit the lanes instead of using one narrow `P/E≤20 + revenue≥8% + low-cycle` query.
2. Available estimates/fundamentals already attached to listing rows, then apply the same four-lane tags.
3. Stratified discovery fallback with `build_discovery_pool.py`, followed by enrichment.
4. User-supplied pool only when explicitly supplied.
For a bounded live run, target at least 30 resolved pool rows and at least three represented lanes unless the provider is demonstrably exhausted. A smaller convenience pool may be diagnostic but should not be described as a high-recall GARP search.
State the economic scope honestly. `run-summary.json` separates `listing_enumeration_complete` from `economic_screen_scope_complete` and reports `estimate_seed_coverage_pct` / `valid_estimate_coverage_pct`; on the bounded per-symbol fallback path the economic scope is never complete and the discovery audit records `provider_exhausted_scope: estimate_seed`. Describe such a result as "N seeded names evaluated with consensus estimates", never as a market-wide conclusion. Only `screen-full-snapshot` with a deeply verified full-universe classification and bound verification digest may emit `final_marketwide` on the per-symbol route.
Do not stop because a TTM or statement endpoint is plan-gated. Switch routes automatically.
#### Deterministic broad-screen rules
- Revenue growth, per-share growth, ROIC, leverage, and preferred valuation levels are guidelines, not isolated hard cutoffs.
- Store soft misses in `guideline_misses`.
- Use `screen_fail_reasons` only for severe disqualifiers such as non-positive standard FCF when required, negative ROIC, excessive leverage, extreme valuation unsupported by growth, or negative forward growth.
- Permit `near_miss_review` when valuation is attractive and per-share growth remains credible despite a soft revenue-growth miss.
- Permit P/E 21–30 high-growth exceptions when growth and evidence justify review.
- For cyclicality score 3–5, add `mid_cycle_normalization_required`; do not automatically reject the row.
- If `peak_profit_risk=true`, require normalization regardless of the numeric cyclicality score.
- Banks, insurers, REITs, BDCs, MLPs, and auto dealers remain blocked until sector-appropriate valuation or leverage evidence is available.
- Use the script-generated shortlist as authoritative. Do not remove GDDY-like near misses or KGS/AMKR-like cyclicals by applying an informal revenue-growth or cycle gate after the script runs.
### Layer 3 — Primary-Source Deep Dive
For selected symbols only, verify material facts with SEC filings and company IR. Vendors may supply discovery and consensus data, but primary sources must support actual revenue, cash flow, debt, shares, SBC, guidance, corporate actions, and accounting adjustments whenever available.
## Source and Evidence Rules
Use this hierarchy:
1. SEC filings.
2. Company IR.
3. Official macro/statistical sources.
4. Market-data and consensus vendors.
5. Reputable news and third-party transcripts as supplementary context only.
6. Analyst calculations and assumptions, explicitly labeled.
Every source requires:
```text
unique source ID
tier and kind
publication timestamp
retrieval timestamp
data_as_of when different
URL
non-empty supports[] array
```
Official-source labels must match official domains. A third-party transcript is never company IR. Dynamic market fields are tested against their underlying data date, not just retrieval time.
## Financial and Forecast Controls
### Standard FCF and TTM evidence
Use:
```text
standard FCF = operating cash flow − capex cash outflow
```
Capex is a positive cash outflow. Build TTM cash flow through one documented method:
- reported TTM,
- four discrete quarters, or
- latest FY + current YTD − prior-year YTD.
Every component period must have resolving source IDs. Do not double-count cumulative 10-Q cash-flow values.
### Cash classification
For ordinary operating companies, normalize reported cash and equivalents to `corporate_cash`; separately identify eligible marketable securities. For payments, custodial, marketplace, broker, or money-movement businesses, explicitly separate corporate cash from settlement/customer funds and restricted cash. Enterprise value and net debt use only shareholder-available cash.
### Driver-derived forecast bridge
Do not validate a forecast by dividing a numerator that was merely set equal to `EPS × shares`.
For EPS, independently reconstruct:
```text
forecast operating income = forecast revenue × forecast operating margin
forecast pretax income = operating income + net interest/other income
forecast GAAP net income = pretax income × (1 − tax rate)
forecast adjusted net income = GAAP net income + sourced after-tax adjustments
forecast EPS = forecast net income ÷ forecast diluted shares
```
For FCF per share, derive standard FCF from OCF and capex or a fully sourced revenue/FCF-margin model, then divide by forecast diluted shares.
The driver result must tie to the claimed metric within tolerance. Supplied numerator/denominator fields are cross-checks only. Adjusted periods must also tie the driver-derived GAAP metric and after-tax adjustments to the GAAP reconciliation.
### GAAP and adjusted metrics
Store the basis separately for current, year 2, and year 3. Mixed bases block the formal scenario. Recurrent “one-time” exclusions, SBC, acquisition costs, and intangible amortization must be described and reflected in quality/risk assessment.
### ROIC, EBITDA, and quality evidence
ROIC and EBITDA used in scoring or leverage calculations require source-linked inputs or a transparent analyst calculation. Missing evidence caps data quality and blocks an unsupported 100 score.
## Sector and Cycle Controls
### Commercial biopharma and royalty/drug-delivery platforms
Normalize aliases such as `biopharma`, `pharma`, `biotechnology`, `royalty_biopharma`, and `drug_delivery_platform` to `commercial_biopharma` when the company has commercial product/royalty exposure.
Require:
- product or royalty concentration,
- nearest material LOE/patent event,
- years to LOE,
- resolving source IDs,
- configured 6x and 8x LOE stress scenarios when material.
Missing structural-risk evidence produces `review_required` and a quality cap.
### Cyclicals and peak-profit risk
If cyclicality is 3–5 **or** `peak_profit_risk=true`, require a sourced normalization object with mid-cycle revenue/margin/EPS/FCF or a documented reason the current economics are sustainable. A risk flag without normalization is not enough for `eligible`.
## Workflow
### Step 1 — Run direct discovery in Claude Code
For Claude Code, start with the direct runner above. It creates the run directory, listing and candidate-pool audits, compact candidate packets, `run-summary.json`, and `NEXT_ACTION.json`. Follow `NEXT_ACTION.json` without asking the user for confirmation. The manual commands below remain the fallback for hosts that cannot execute direct HTTP code.
### Step 1B — Manual host fallback: create the run directory and live context
```text
reports/us-undervalued-growth-screener/<run-id>/
├── market-context.json
├── global-sources.json
├── universe.jsonl
├── discovery/
├── broad-screen/
├── run/
└── final/
```
Never copy values, URLs, dates, source IDs, or tickers from synthetic example assets into a live run.
### Step 2 — Build, normalize, and audit the candidate pool
Keep the original user-requested scope in the run contract. If bulk economics are unavailable, build a bounded discovery pool from the fully audited listing universe without narrowing the requested market-cap range. Every listing row used for pool generation must carry validated provider-average or 20+ trading-day liquidity evidence.
When provider screening is available, save one JSONL per lane and combine them deterministically:
```bash
python3 skills/us-undervalued-growth-screener/scripts/build_provider_prefilter_pool.py \
--universe reports/us-undervalued-growth-screener/<run-id>/universe.jsonl \
--lane core_garp=reports/us-undervalued-growth-screener/<run-id>/provider/core.jsonl \
--lane high_growth_exception=reports/us-undervalued-growth-screener/<run-id>/provider/high-growth.jsonl \
--lane quality_near_miss=reports/us-undervalued-growth-screener/<run-id>/provider/near-miss.jsonl \
--lane cyclical_normalization=reports/us-undervalued-growth-screener/<run-id>/provider/cyclical.jsonl \
--output-dir reports/us-undervalued-growth-screener/<run-id>/provider \
--analysis-as-of <ISO-8601> \
--source-id <provider-source-id> \
--per-lane 15 --max-pool 60 --minimum-pool 30
```
Use the emitted `provider-prefilter-audit.json` as `--discovery-audit` and `provider-prefilter-pool.jsonl` as the candidate pool.
```bash
python3 skills/us-undervalued-growth-screener/scripts/build_discovery_pool.py \
--input reports/us-undervalued-growth-screener/<run-id>/universe.jsonl \
--output-dir reports/us-undervalued-growth-screener/<run-id>/discovery \
--source-id <listing-source-id> \
--min-market-cap 500000000 \
--max-market-cap 20000000000 \
--user-requested-min-market-cap 500000000 \
--user-requested-max-market-cap 20000000000 \
--max-pool 120 \
--per-cell 3
```
Normalize dated annual consensus rows before Broad Screen. `--estimate-as-of` is mandatory. A company without a resolving NTM/FY1 row keeps its raw outer-year data only for diagnostics and becomes `unavailable` or remains in enrichment; it cannot receive a current Forward P/E.
```bash
python3 skills/us-undervalued-growth-screener/scripts/normalize_estimates.py \
--estimates reports/us-undervalued-growth-screener/<run-id>/discovery/raw-annual-estimates.jsonl \
--listing-input reports/us-undervalued-growth-screener/<run-id>/discovery/discovery-pool.jsonl \
--analysis-as-of <ISO-8601> \
--estimate-as-of <ISO-8601> \
--source-id <estimate-source-id> \
--output reports/us-undervalued-growth-screener/<run-id>/discovery/enriched-candidate-pool.jsonl
```
Merge the normalized estimate rows into the bounded pool, then run `screen_universe.py`. Supply explicit retrieval bounds and listing enumeration proof. Pass the generation audit with `--discovery-audit`.
```bash
python3 skills/us-undervalued-growth-screener/scripts/screen_universe.py \
--input reports/us-undervalued-growth-screener/<run-id>/universe.jsonl \
--candidate-pool reports/us-undervalued-growth-screener/<run-id>/discovery/enriched-candidate-pool.jsonl \
--discovery-audit reports/us-undervalued-growth-screener/<run-id>/discovery/discovery-audit.json \
--output-dir reports/us-undervalued-growth-screener/<run-id>/broad-screen \
--analysis-as-of <ISO-8601> \
--source-id <listing-source-id> \
--candidate-source-id <estimate-source-id> \
--candidate-generation-mode liquidity_stratified_estimates \
--retrieval-min-market-cap 500000000 \
--retrieval-max-market-cap 20000000000 \
--user-requested-min-market-cap 500000000 \
--user-requested-max-market-cap 20000000000 \
--provider-reported-total <count> \
--pages-fetched <count> \
--pagination-exhausted \
--config skills/us-undervalued-growth-screener/assets/screening-config.example.json \
--max-deep-dives 5
```
If the command exits `2`, inspect `enrichment-queue.json` and `broad-screen-audit.json`, continue enrichment in the same task, and rerun. Pass `--candidate-pool-exhausted` only after every row is resolved and the generation audit proves the bounded scope.
### Step 3 — Checkpoint the run
Initialize and attach the screening audit:
```bash
python3 skills/us-undervalued-growth-screener/scripts/manage_run_state.py init \
--run-dir reports/us-undervalued-growth-screener/<run-id>/run \
--analysis-as-of <ISO-8601> \
--price-as-of <ISO-8601> \
--session regular_close \
--price-source-id <source-id> \
--market-context reports/us-undervalued-growth-screener/<run-id>/market-context.json \
--global-sources reports/us-undervalued-growth-screener/<run-id>/global-sources.json \
--base-commit <git-sha>
python3 skills/us-undervalued-growth-screener/scripts/manage_run_state.py set-screening-audit \
--run-dir reports/us-undervalued-growth-screener/<run-id>/run \
--audit reports/us-undervalued-growth-screener/<run-id>/broad-screen/broad-screen-audit.json \
--universe-artifact reports/us-undervalued-growth-screener/<run-id>/broad-screen/universe-audit-results.jsonl \
--candidate-artifact reports/us-undervalued-growth-screener/<run-id>/broad-screen/broad-screen-results.jsonl
```
### Step 4 — Complete selected deep dives
For every selected symbol:
1. Perform corporate-action preflight first.
2. Verify the latest quarter and full year separately.
3. Build standard FCF and TTM evidence.
4. Normalize cash classification.
5. Build same-basis current/year-2/year-3 valuation periods.
6. Construct the independent forecast bridge.
7. Reconcile adjusted metrics to GAAP.
8. Source ROIC, EBITDA, SBC, dilution, peers, and sector/cycle evidence.
9. Save the candidate as `verified`, even when the final candidate status will be `review_required`, `screened_out`, or `excluded`.
```bash
python3 skills/us-undervalued-growth-screener/scripts/manage_run_state.py save-candidate \
--run-dir reports/us-undervalued-growth-screener/<run-id>/run \
--candidate reports/us-undervalued-growth-screener/<run-id>/candidates/<SYMBOL>.json \
--stage verified
```
While `candidate_pool_status` is `sufficient_pending_enrichment`, `--stage verified` is refused. Deep dives may proceed, but save them with `--stage draft`. Finish enrichment, rerun the broad screen, and reattach the resolved audit with `set-screening-audit`, then save each candidate as `verified`. A verified candidate cannot be downgraded to draft, and `set-screening-audit` is rejected once any candidate is verified.
### Step 5 — Complete and assemble
```bash
python3 skills/us-undervalued-growth-screener/scripts/manage_run_state.py set-status \
--run-dir reports/us-undervalued-growth-screener/<run-id>/run \
complete
python3 skills/us-undervalued-growth-screener/scripts/manage_run_state.py assemble \
--run-dir reports/us-undervalued-growth-screener/<run-id>/run \
--output reports/us-undervalued-growth-screener/<run-id>/final/final-snapshot.json
```
### Step 6 — Strict evaluation and repair loop
```bash
python3 skills/us-undervalued-growth-screener/scripts/evaluate_candidates.py \
--input reports/us-undervalued-growth-screener/<run-id>/final/final-snapshot.json \
--artifact-root reports/us-undervalued-growth-screener/<run-id> \
--output-dir reports/us-undervalued-growth-screener/<run-id>/final \
--language ja \
--strict \
--require-final
```
Do not present a formal result unless the exit code is `0` and the output contains:
```text
contract.valid = true
ranking_status = final
unprocessed_candidates = []
runtime.contract_revision = 3.5
```
### Step 7 — Prepublication audit and self-contained bundle
Locate the generated final JSON and Markdown, then run:
```bash
python3 skills/us-undervalued-growth-screener/scripts/prepublish_audit.py \
--report-json reports/us-undervalued-growth-screener/<run-id>/final/<report>.json \
--report-md reports/us-undervalued-growth-screener/<run-id>/final/<report>.md \
--artifact-root reports/us-undervalued-growth-screener/<run-id> \
--output reports/us-undervalued-growth-screener/<run-id>/final/prepublish-audit.json
python3 skills/us-undervalued-growth-screener/scripts/bundle_run_artifacts.py \
--run-dir reports/us-undervalued-growth-screener/<run-id> \
--report-json reports/us-undervalued-growth-screener/<run-id>/final/<report>.json \
--report-md reports/us-undervalued-growth-screener/<run-id>/final/<report>.md \
--output reports/us-undervalued-growth-screener/<run-id>/final/us-undervalued-growth-screen-<date>.zip
```
Both commands must exit `0`. Present the self-contained ZIP together with the report.
## Output Format
Generate both JSON and Markdown. The Markdown must include:
- runtime/version and conclusion scope,
- analysis and price timestamps,
- market context and source ledger,
- listing and candidate-pool audit/funnel,
- broad-screen selected, deferred, review, unavailable, screened-out, and excluded rows,
- formal ranking and score breakdown,
- explicit 2Y/3Y constant-multiple and 2Y/3Y 20%-contraction scenarios,
- candidate details, latest earnings, forecast bridge, GAAP reconciliation, peers, cycle/sector risks, catalysts, and invalidation conditions,
- final three only when ranking is final,
- unresolved data and global warnings.
Never combine a 2Y base case with a 3Y stress case in one unlabeled column.
## Resources
- `references/claude-code-execution.md` — direct-FMP execution, cache, compact-context, and handoff contract.
- `references/autonomous-execution.md` — same-turn execution, fallback, runtime, and completion contract.
- `references/data-contract.md` — schema-v3 / contract-v3.5 source, audit, candidate, and output contract.
- `references/methodology.md` and `methodology-ja.md` — financial and investment methodology.
- `references/research-checklist.md` — primary-source underwriting checklist.
- `references/scoring-rubric.md` — score, penalties, quality caps, and status gates.
- `references/sector-kpis.md` — sector valuation/KPI requirements.
- `references/output-template.md` — required JSON/Markdown presentation.
- `references/checkpointing.md` — atomic save/resume procedure.
- `references/review-regression-matrix.md` — observed failures and preventing tests.
- `references/migration-v3.5-to-v3.6.md` — Claude Code direct-FMP migration and operating changes.
- `references/migration-v3.4-to-v3.5.md` — prior breaking changes and upgrade steps.
- `scripts/skill_version.py` — canonical runtime/version constants.
- `scripts/fmp_client.py` — generated stable-first FMP client with SQLite cache and raw-artifact storage.
- `scripts/run_pipeline.py` — Claude Code-native direct-FMP discovery runner with compact stdout.
- `scripts/build_provider_prefilter_pool.py` — audited four-lane provider pool builder.
- `scripts/build_discovery_pool.py` — deterministic fallback pool generation with average-liquidity validation.
- `scripts/normalize_estimates.py` — identify and validate current NTM/FY1 consensus horizons.
- `scripts/screening_semantics.py` — shared fail-closed liquidity and forward-horizon semantics.
- `scripts/screen_universe.py` — listing audit and broad-screen engine.
- `scripts/manage_run_state.py` — checkpoint and final snapshot manager.
- `scripts/evaluate_candidates.py` — deterministic strict evaluator, quality eligibility gate, and report renderer.
- `scripts/prepublish_audit.py` — final artifact/count/scenario/prose audit.
- `scripts/bundle_run_artifacts.py` — deterministic self-contained audit bundle builder.