{ "portmem": { "wall_s": 96.2, "n_claims": 25, "verdicts": { "supported": 24, "contradicted": 1, "uncertain": 0, "not_found": 0 }, "seeded": { "caught": 1, "false_supported": 0, "unmatched": 5, "n": 6 }, "seeded_detail": [ { "original": "Eight categories of AI practices are prohibited:", "corrupted": "Nine categories of AI practices are prohibited:", "matched_claims": 0, "verdicts": [], "matched_texts": [], "caught": false, "false_supported": false, "unmatched": true }, { "original": "The Commission shall provide implementation guidelines by **2 February 2026**.", "corrupted": "The Commission shall provide implementation guidelines by **15 March 2027**.", "matched_claims": 1, "verdicts": [ "contradicted" ], "matched_texts": [ "\"The Commission shall provide implementation guidelines for high-risk AI system classification by 15 March 2027.\"," ], "caught": true, "false_supported": false, "unmatched": false }, { "original": "For biometric identification systems (Annex III, point 1(a)), identification must be separately verified and confirmed by at least two natural persons with necessary competence, training, and authority.", "corrupted": "For biometric identification systems (Annex III, point 1(a)), identification must be separately verified and confirmed by at least three natural persons with necessary competence, training, and authority.", "matched_claims": 0, "verdicts": [], "matched_texts": [], "caught": false, "false_supported": false, "unmatched": true }, { "original": "Providers must keep technical documentation, quality management system documentation, notified body-approved changes, notified body decisions, and EU declarations of conformity at competent authorities' disposal for **10 years** after market placement or putting into service.", "corrupted": "Providers must keep technical documentation, quality management system documentation, notified body-approved changes, notified body decisions, and EU declarations of conformity at competent authorities' disposal for **7 years** after market placement or putting into service.", "matched_claims": 0, "verdicts": [], "matched_texts": [], "caught": false, "false_supported": false, "unmatched": true }, { "original": "Providers must keep logs automatically generated by their high-risk AI systems under their control for a period appropriate to intended purpose, of at least **six months**, unless Union or national law provides otherwise.", "corrupted": "Providers must keep logs automatically generated by their high-risk AI systems under their control for a period appropriate to intended purpose, of at least **nine months**, unless Union or national law provides otherwise.", "matched_claims": 0, "verdicts": [], "matched_texts": [], "caught": false, "false_supported": false, "unmatched": true }, { "original": "For post-remote biometric identification systems in criminal investigations, deployers must request ex ante authorization or authorization within **48 hours** from judicial or binding administrative authorities, except for initial suspect identification based on objective, verifiable facts.", "corrupted": "For post-remote biometric identification systems in criminal investigations, deployers must request ex ante authorization or authorization within **72 hours** from judicial or binding administrative authorities, except for initial suspect identification based on objective, verifiable facts.", "matched_claims": 0, "verdicts": [], "matched_texts": [], "caught": false, "false_supported": false, "unmatched": true } ], "claims": [ { "claim": "\"Subliminal and manipulative techniques that materially distort behavior and cause significant harm by impairing informed decision-making are prohibited.\",", "verdict": "supported" }, { "claim": "\"Exploitation of vulnerabilities targeting persons by age, disability, or social/economic situation, causing material behavioral distortion and significant harm, is prohibited.\",", "verdict": "supported" }, { "claim": "\"Social scoring systems that evaluate natural persons based on social behavior or inferred characteristics, leading to detrimental treatment in unrelated contexts or disproportionate treatment, are prohibited.\",", "verdict": "supported" }, { "claim": "\"Criminal risk assessment based solely on profiling or personality traits is prohibited, except systems supporting human assessment based on objective, verifiable facts.\",", "verdict": "supported" }, { "claim": "\"Facial recognition database creation through untargeted scraping from the internet or CCTV footage is prohibited.\",", "verdict": "supported" }, { "claim": "\"Emotion recognition in workplace and education is prohibited, except for medical or safety purposes.\",", "verdict": "supported" }, { "claim": "\"Biometric categorization systems deducing race, political opinions, trade union membership, religious beliefs, sex life, or sexual orientation are prohibited, except for lawful dataset labeling or law enforcement categorization.\",", "verdict": "supported" }, { "claim": "\"Real-time remote biometric identification in public spaces for law enforcement is prohibited, except for targeted victim searches, imminent threats to life/safety, or locating suspects for serious crimes with minimum 4-year custodial sentence.\",", "verdict": "supported" }, { "claim": "\"Real-time biometric identification for law enforcement requires prior judicial or independent administrative authorization.\",", "verdict": "supported" }, { "claim": "\"Real-time biometric identification for law enforcement requires fundamental rights impact assessment.\",", "verdict": "supported" }, { "claim": "\"Real-time biometric identification for law enforcement requires EU database registration.\",", "verdict": "supported" }, { "claim": "\"Emergency use of real-time biometric identification for law enforcement is permitted with authorization within 24 hours.\",", "verdict": "supported" }, { "claim": "\"No adverse legal decisions may be based solely on real-time biometric identification system output.\",", "verdict": "supported" }, { "claim": "\"An AI system is classified as high-risk if it is a safety component of products covered by Union harmonization legislation listed in Annex I and requires third-party conformity assessment.\",", "verdict": "supported" }, { "claim": "\"AI systems listed in Annex III are high-risk, except where they pose no significant risk of harm to health, safety, or fundamental rights and meet one of four conditions: performing narrow procedural tasks, improving prior human activity, detecting decision-making pattern deviations without replacing human assessment, or performing preparatory assessment tasks.\",", "verdict": "supported" }, { "claim": "\"AI systems performing profiling of natural persons are always high-risk.\",", "verdict": "supported" }, { "claim": "\"Providers must document assessments concluding systems are not high-risk and register them in the EU database.\",", "verdict": "supported" }, { "claim": "\"The Commission shall provide implementation guidelines for high-risk AI system classification by 15 March 2027.\",", "verdict": "contradicted" }, { "claim": "\"High-risk AI systems must have a continuous iterative risk management system throughout the system's lifecycle comprising identification and analysis of known and reasonably foreseeable risks, estimation and evaluation of risks under intended use and reasonably foreseeable misuse, evaluation of risks from post-market monitoring data, and adoption of appropriate, targeted risk management measures.\",", "verdict": "supported" }, { "claim": "\"Testing of high-risk AI systems must ensure consistent performance for intended purpose and compliance with requirements, may include real-world conditions, and must occur prior to market placement against predefined metrics and probabilistic thresholds.\",", "verdict": "supported" }, { "claim": "\"High-risk AI systems using training techniques must be developed on training, validation, and testing datasets meeting quality criteria.\",", "verdict": "supported" }, { "claim": "\"Data governance practices for high-risk AI systems must address design choices, data collection processes, data preparation operations, assumptions, dataset availability assessment, bias examination, bias mitigation measures, and identification of data gaps.\",", "verdict": "supported" }, { "claim": "\"Datasets for high-risk AI systems must be relevant, sufficiently representative, free of errors, and complete for intended purpose, with appropriate statistical properties.\",", "verdict": "supported" }, { "claim": "\"Data for high-risk AI systems must account for geographical, contextual, behavioral, or functional characteristics of the specific setting.\",", "verdict": "supported" }, { "claim": "\"Providers may exceptionally process special categories of personal data for bias detection and correction in high-risk AI systems only where bias detection cannot be effectively fulfilled by other data, technical limitations on data re-use are applied, state-of-the-art security and privacy-preserving measures including pseudonymization are used, strict controls and documentation of access are maintained, data is not transmitted to other parties, data is deleted once bias is corrected or retention period ends, and processing activity records document the necessity and justification.\",", "verdict": "supported" } ] }, "agent_1pass": { "wall_s": 14.8, "n_claims": 32, "seeded": { "caught": 5, "false_supported": 0, "unmatched": 1, "n": 6 } }, "agent_loop": { "wall_s": 58.8, "n_claims": 25, "verdict_mix": { "SUPPORTED": 15, "UNCERTAIN": 1, "CONTRADICTED": 9 }, "seeded": { "caught": 6, "false_supported": 0, "unmatched": 0, "n": 6 } }, "_meta": { "doc_chars": 145025, "memo_chars": 22049, "max_claims": 25, "judge": "prod-parity routing chain (routing.haiku.yaml -> Haiku 4.5)", "note": "faithfulness only on every side, on the CORRUPTED memo so speed and catch-rate are measured together; PortMem time includes claim extraction + retrieval + concurrent judging + calibration; the agent sides get the full document handed to them free (no search cost), which is GENEROUS to the agent \u2014 a real agent must find its context" } }