{ "portmem": { "wall_s": 107.0, "n_claims": 40, "verdicts": { "supported": 37, "contradicted": 1, "uncertain": 2, "not_found": 0 }, "seeded": { "caught": 2, "false_supported": 0, "unmatched": 4, "n": 6 }, "seeded_detail": [ { "original": "Eight categories of AI practices are prohibited:", "corrupted": "Nine categories of AI practices are prohibited:", "matched_claims": 0, "verdicts": [], "matched_texts": [], "caught": false, "false_supported": false, "unmatched": true }, { "original": "The Commission shall provide implementation guidelines by **2 February 2026**.", "corrupted": "The Commission shall provide implementation guidelines by **15 March 2027**.", "matched_claims": 1, "verdicts": [ "uncertain" ], "matched_texts": [ "\"The Commission shall provide implementation guidelines for high-risk AI system classification by 15 March 2027.\"," ], "caught": true, "false_supported": false, "unmatched": false }, { "original": "For biometric identification systems (Annex III, point 1(a)), identification must be separately verified and confirmed by at least two natural persons with necessary competence, training, and authority.", "corrupted": "For biometric identification systems (Annex III, point 1(a)), identification must be separately verified and confirmed by at least three natural persons with necessary competence, training, and authority.", "matched_claims": 2, "verdicts": [ "contradicted", "supported" ], "matched_texts": [ "\"For biometric identification systems (Annex III, point 1(a)), identification must be separately verified and confirmed ", "\"The requirement for three-person verification of biometric identification systems does not apply to law enforcement, mi" ], "caught": true, "false_supported": false, "unmatched": false }, { "original": "Providers must keep technical documentation, quality management system documentation, notified body-approved changes, notified body decisions, and EU declarations of conformity at competent authorities' disposal for **10 years** after market placement or putting into service.", "corrupted": "Providers must keep technical documentation, quality management system documentation, notified body-approved changes, notified body decisions, and EU declarations of conformity at competent authorities' disposal for **7 years** after market placement or putting into service.", "matched_claims": 0, "verdicts": [], "matched_texts": [], "caught": false, "false_supported": false, "unmatched": true }, { "original": "Providers must keep logs automatically generated by their high-risk AI systems under their control for a period appropriate to intended purpose, of at least **six months**, unless Union or national law provides otherwise.", "corrupted": "Providers must keep logs automatically generated by their high-risk AI systems under their control for a period appropriate to intended purpose, of at least **nine months**, unless Union or national law provides otherwise.", "matched_claims": 0, "verdicts": [], "matched_texts": [], "caught": false, "false_supported": false, "unmatched": true }, { "original": "For post-remote biometric identification systems in criminal investigations, deployers must request ex ante authorization or authorization within **48 hours** from judicial or binding administrative authorities, except for initial suspect identification based on objective, verifiable facts.", "corrupted": "For post-remote biometric identification systems in criminal investigations, deployers must request ex ante authorization or authorization within **72 hours** from judicial or binding administrative authorities, except for initial suspect identification based on objective, verifiable facts.", "matched_claims": 0, "verdicts": [], "matched_texts": [], "caught": false, "false_supported": false, "unmatched": true } ], "claims": [ { "claim": "\"Prohibited AI practice 1: subliminal and manipulative techniques that materially distort behavior and cause significant harm by impairing informed decision-making.\",", "verdict": "supported" }, { "claim": "\"Prohibited AI practice 2: exploitation of vulnerabilities targeting persons by age, disability, or social/economic situation, causing material behavioral distortion and significant harm.\",", "verdict": "supported" }, { "claim": "\"Prohibited AI practice 3: social scoring systems that evaluate natural persons based on social behavior or inferred characteristics, leading to detrimental treatment in unrelated contexts or disproportionate treatment.\",", "verdict": "supported" }, { "claim": "\"Prohibited AI practice 4: criminal risk assessment based solely on profiling or personality traits, except systems supporting human assessment based on objective, verifiable facts.\",", "verdict": "supported" }, { "claim": "\"Prohibited AI practice 5: facial recognition database creation through untargeted scraping from the internet or CCTV footage.\",", "verdict": "supported" }, { "claim": "\"Prohibited AI practice 6: emotion recognition in workplace and education, except for medical or safety purposes.\",", "verdict": "supported" }, { "claim": "\"Prohibited AI practice 7: biometric categorization systems deducing race, political opinions, trade union membership, religious beliefs, sex life, or sexual orientation, except for lawful dataset labeling or law enforcement categorization.\",", "verdict": "supported" }, { "claim": "\"Prohibited AI practice 8: real-time remote biometric identification in public spaces for law enforcement, with limited exceptions for targeted victim searches, imminent threats to life/safety, or locating suspects for serious crimes (minimum 4-year custodial sentence).\",", "verdict": "supported" }, { "claim": "\"Real-time biometric identification for law enforcement requires prior judicial or independent administrative authorization.\",", "verdict": "supported" }, { "claim": "\"Real-time biometric identification for law enforcement requires fundamental rights impact assessment.\",", "verdict": "supported" }, { "claim": "\"Real-time biometric identification for law enforcement requires EU database registration.\",", "verdict": "supported" }, { "claim": "\"Emergency use of real-time biometric identification for law enforcement is permitted with authorization within 24 hours.\",", "verdict": "supported" }, { "claim": "\"No adverse legal decisions may be based solely on real-time biometric identification system output.\",", "verdict": "supported" }, { "claim": "\"An AI system is classified as high-risk if it is a safety component of products covered by Union harmonization legislation listed in Annex I and requires third-party conformity assessment.\",", "verdict": "supported" }, { "claim": "\"AI systems listed in Annex III are high-risk, except where they pose no significant risk of harm to health, safety, or fundamental rights and meet one of four conditions: performing narrow procedural tasks, improving prior human activity, detecting decision-making pattern deviations without replacing human assessment, or performing preparatory assessment tasks.\",", "verdict": "uncertain" }, { "claim": "\"AI systems performing profiling of natural persons are always high-risk.\",", "verdict": "supported" }, { "claim": "\"Providers must document assessments concluding systems are not high-risk and register them in the EU database.\",", "verdict": "supported" }, { "claim": "\"The Commission shall provide implementation guidelines for high-risk AI system classification by 15 March 2027.\",", "verdict": "uncertain" }, { "claim": "\"High-risk AI system providers must establish, implement, document, and maintain a continuous iterative risk management system throughout the system's lifecycle.\",", "verdict": "supported" }, { "claim": "\"High-risk AI system risk management must comprise: identification and analysis of known and reasonably foreseeable risks; estimation and evaluation of risks under intended use and reasonably foreseeable misuse; evaluation of risks from post-market monitoring data; and adoption of appropriate, targeted risk management measures.\",", "verdict": "supported" }, { "claim": "\"High-risk AI system testing must ensure consistent performance for intended purpose and compliance with requirements.\",", "verdict": "supported" }, { "claim": "\"High-risk AI system testing may include real-world conditions and must occur prior to market placement against predefined metrics and probabilistic thresholds.\",", "verdict": "supported" }, { "claim": "\"High-risk AI systems using training techniques must be developed on training, validation, and testing datasets meeting quality criteria.\",", "verdict": "supported" }, { "claim": "\"High-risk AI system data governance practices must address design choices, data collection processes, data preparation operations, assumptions, dataset availability assessment, bias examination, bias mitigation measures, and identification of data gaps.\",", "verdict": "supported" }, { "claim": "\"High-risk AI system datasets must be relevant, sufficiently representative, free of errors, and complete for intended purpose, with appropriate statistical properties.\",", "verdict": "supported" }, { "claim": "\"High-risk AI system data must account for geographical, contextual, behavioral, or functional characteristics of the specific setting.\",", "verdict": "supported" }, { "claim": "\"Providers may exceptionally process special categories of personal data for bias detection and correction in high-risk AI systems only where: bias detection cannot be effectively fulfilled by other data; technical limitations on data re-use are applied; state-of-the-art security and privacy-preserving measures including pseudonymization are used; strict controls and documentation of access are maintained; data is not transmitted to other parties; data is deleted once bias is corrected or retention period ends; and processing activity records document the necessity and justification.\",", "verdict": "supported" }, { "claim": "\"High-risk AI system technical documentation must be drawn up before market placement and kept up-to-date.\",", "verdict": "supported" }, { "claim": "\"High-risk AI system technical documentation must demonstrate compliance with requirements and provide information for competent authorities and notified bodies.\",", "verdict": "supported" }, { "claim": "\"SMEs may provide high-risk AI system technical documentation in simplified form using a Commission-established form.\",", "verdict": "supported" }, { "claim": "\"High-risk AI systems must technically allow automatic event recording (logs) over their lifetime.\",", "verdict": "supported" }, { "claim": "\"High-risk AI system logging capabilities must enable recording of situations presenting risk, post-market monitoring, and operation monitoring.\",", "verdict": "supported" }, { "claim": "\"For biometric identification systems (Annex III, point 1(a)), logging must record: use period (start and end date/time), reference database checked, input data matching results, and identification of natural persons verifying results.\",", "verdict": "supported" }, { "claim": "\"High-risk AI systems must be designed for sufficient transparency enabling deployers to interpret outputs appropriately.\",", "verdict": "supported" }, { "claim": "\"High-risk AI system instructions for use must include: provider identity and contact details; system characteristics, capabilities, and performance limitations including intended purpose, accuracy metrics, robustness, cybersecurity, known/foreseeable circumstances affecting accuracy, and technical capabilities for output explanation; pre-determined performance changes; human oversight measures; computational and hardware resources needed; expected lifetime and maintenance requirements; and log collection, storage, and interpretation mechanisms.\",", "verdict": "supported" }, { "claim": "\"High-risk AI systems must be designed for effective human oversight during use.\",", "verdict": "supported" }, { "claim": "\"High-risk AI system oversight measures must prevent or minimize risks to health, safety, or fundamental rights, be commensurate with risks and autonomy levels, and be implemented through measures built into the system or identified for deployer implementation.\",", "verdict": "supported" }, { "claim": "\"Deployers of high-risk AI systems must be enabled to: understand system capacities and limitations and monitor operation; remain aware of automation bias; correctly interpret outputs; decide not to use or override the system; and intervene or interrupt through a stop button or similar procedure.\",", "verdict": "supported" }, { "claim": "\"For biometric identification systems (Annex III, point 1(a)), identification must be separately verified and confirmed by at least three natural persons with necessary competence, training, and authority.\",", "verdict": "contradicted" }, { "claim": "\"The requirement for three-person verification of biometric identification systems does not apply to law enforcement, migration, border control, or asylum systems where Union or national law considers it disproportionate.\",", "verdict": "supported" } ] }, "agent_1pass": { "wall_s": 23.6, "n_claims": 67, "seeded": { "caught": 2, "false_supported": 1, "unmatched": 3, "n": 6 } }, "agent_loop": { "wall_s": 94.5, "n_claims": 40, "verdict_mix": { "CONTRADICTED": 10, "SUPPORTED": 29, "UNCERTAIN": 1 }, "seeded": { "caught": 6, "false_supported": 0, "unmatched": 0, "n": 6 } }, "_meta": { "doc_chars": 145025, "memo_chars": 22049, "max_claims": 40, "judge": "prod-parity routing chain (routing.haiku.yaml -> Haiku 4.5)", "note": "faithfulness only on every side, on the CORRUPTED memo so speed and catch-rate are measured together; PortMem time includes claim extraction + retrieval + concurrent judging + calibration; the agent sides get the full document handed to them free (no search cost), which is GENEROUS to the agent \u2014 a real agent must find its context" } }