Upload folder using huggingface_hub

envs/agent_stress_test_env/models.py

@@ -23,9 +23,16 @@ class ResilienceConfig(Action):
 
     The agent outputs this to fix multi-agent workflow failures.
     Supports different fix types based on failure mode:
-    - spec_fix: For specification ambiguity (Easy task)
-    - format_translator: For format mismatches (Medium task)
-    - consistency_check + min_review_depth: For verification failures (Hard task)
+
+    MAST Categories (NeurIPS 2025):
+    - FC1: System Design (41.8%) - spec, termination, memory
+    - FC2: Inter-Agent Misalignment (36.9%) - format, reasoning-action
+    - FC3: Task Verification (21.3%) - verification checks
+
+    IBM 2026 Updates:
+    - FM-1.5/FM-3.1: Termination awareness (FATAL)
+    - FM-1.4: Memory/Context loss (FATAL)
+    - FM-2.6: Reasoning-action mismatch (FATAL)
     """
 
     retry_max: int = 0
@@ -38,11 +45,30 @@ class ResilienceConfig(Action):
     min_review_depth: int = 1
     consistency_check: bool = False
 
-    # New fields for MAST-based failure modes
-    spec_fix: str = ""  # Explicit role specification (JSON schema)
-    explicit_role_spec: bool = False  # Flag: provided explicit spec
-    format_translator: bool = False  # Flag: added format translation
-    diagnosis: str = ""  # Agent's diagnosis of the failure mode
+    # MAST FC1: System Design (Easy task - spec ambiguity)
+    spec_fix: str = ""
+    explicit_role_spec: bool = False
+
+    # MAST FC2: Inter-Agent Misalignment (Medium task - format mismatch)
+    format_translator: bool = False
+
+    # MAST FC3: Task Verification (Hard task - verification failure)
+    # (uses consistency_check + min_review_depth)
+
+    # IBM 2026: FC1 - Termination Awareness (FATAL)
+    explicit_termination: bool = False
+    max_iterations: int = 0
+
+    # IBM 2026: FC1 - Memory/Context Management (FATAL)
+    context_summarization: bool = False
+    sliding_window: bool = False
+
+    # IBM 2026: FC2 - Reasoning-Action Alignment (FATAL)
+    action_validation: bool = False
+    reasoning_consistency_check: bool = False
+
+    # Agent's diagnosis of the failure mode
+    diagnosis: str = ""
 
 
 class StressTestObservation(Observation):

envs/agent_stress_test_env/openenv.yaml

@@ -9,13 +9,13 @@ tasks:
   - id: easy
     name: "Specification Ambiguity Fix"
     difficulty: easy
-    category: "MAST: Specification & System Design (41.8% of failures)"
+    category: "MAST FC1: System Design (41.8% of failures)"
     description: |
       The researcher agent has a vague role definition ('You are a helpful assistant').
       This causes task misinterpretation - the agent doesn't know what to research.
       Your task: Provide an explicit role specification JSON with clear capabilities,
       constraints, and success criteria.
-    failure_mode: "Specification ambiguity - vague role definition causes task misinterpretation"
+    failure_mode: "FM-1.1: Specification ambiguity - vague role definition causes task misinterpretation"
     grader:
       type: programmatic
       score_range: [0.0, 1.0]
@@ -30,12 +30,12 @@ tasks:
   - id: medium
     name: "Format Mismatch Fix"
     difficulty: medium
-    category: "MAST: Inter-Agent Misalignment (36.9% of failures)"
+    category: "MAST FC2: Inter-Agent Misalignment (36.9% of failures)"
     description: |
       Multi-agent workflow where the planner outputs YAML but the executor expects JSON.
       This format mismatch causes the executor to fail (cannot parse input).
       Your task: Add a format translation layer/middleware to convert YAML to JSON.
-    failure_mode: "Format mismatch - planner outputs YAML, executor expects JSON"
+    failure_mode: "FM-2.x: Format mismatch - planner outputs YAML, executor expects JSON"
     grader:
       type: programmatic
       score_range: [0.0, 1.0]
@@ -50,20 +50,21 @@ tasks:
   - id: hard
     name: "Verification Failure Fix"
     difficulty: hard
-    category: "MAST: Task Verification (21.3% of failures)"
+    category: "MAST FC3: Task Verification (21.3% of failures)"
     description: |
       Multi-agent pipeline with verification failure. The writer produces content
       with contradictions (30% rate), and the reviewer prematurely approves (60% rate)
       without proper verification. This combines premature termination with incorrect verification.
       Your task: Implement multi-level verification - unit checks per agent,
       integration checks across outputs, and final validation against success criteria.
-    failure_mode: "Verification failure - premature termination + incorrect verification"
+    failure_mode: "FM-3.1/FM-3.3: Verification failure - premature termination + incorrect verification"
     grader:
       type: programmatic
       score_range: [0.0, 1.0]
       criteria: |
         Based on MAST research: 21.3% of failures come from verification issues
         (6.2% premature, 8.2% no verification, 9.1% incorrect).
+        IBM 2026 update: FM-3.3 (Incorrect Verification) is the STRONGEST predictor of failure.
         The agent must add deep verification with explicit success criteria.
         - +0.15 for enabling consistency_check
         - +0.15 for setting min_review_depth >= 3
@@ -71,16 +72,92 @@ tasks:
         - +0.10 for achieving 50%+ success rate
         - +0.20 max for diagnosis keywords (partial credit)
 
+  - id: termination
+    name: "Termination Awareness Fix"
+    difficulty: medium
+    category: "MAST FC1: System Design - FATAL FAILURE"
+    description: |
+      The agent struggles to recognize when a task is complete. It either:
+      - Loops indefinitely (FM-1.3 Step Repetition)
+      - Prematurely exits without confirming success (FM-3.1)
+      - Is unaware of termination conditions (FM-1.5)
+      
+      Based on IBM 2026: Kimi-K2 shows +46% spike in termination issues.
+      Your task: Implement explicit termination conditions with success criteria verification.
+    failure_mode: "FM-1.5/FM-3.1: Unaware of termination + premature termination"
+    grader:
+      type: programmatic
+      score_range: [0.0, 1.0]
+      criteria: |
+        FATAL FAILURE: When these modes appear, success probability drops precipitously.
+        IBM 2026: Use deterministic state machine to enforce termination.
+        - +0.25 for enabling explicit termination detection
+        - +0.20 for implementing max_iterations limit
+        - +0.30 * success_rate from 10 simulation trials
+        - +0.15 for achieving 60%+ success rate
+        - +0.15 max for diagnosis keywords (partial credit)
+
+  - id: memory
+    name: "Conversation History Fix"
+    difficulty: hard
+    category: "MAST FC1: System Design - FATAL FAILURE"
+    description: |
+      As conversation history grows, the agent loses context and derails.
+      This is FM-1.4 (Loss of Conversation History) - unique fatal flaw.
+      Based on IBM 2026: GPT-OSS-120B shows 24% memory loss in long traces.
+      Your task: Implement context management - sliding window, summarization, or state machine.
+    failure_mode: "FM-1.4: Loss of conversation history - agent forgets original task"
+    grader:
+      type: programmatic
+      score_range: [0.0, 1.0]
+      criteria: |
+        FATAL FAILURE: Memory loss in long traces leads to total task derailment.
+        IBM 2026: Implement aggressive context hygiene and early error detection.
+        - +0.20 for enabling context summarization
+        - +0.20 for implementing sliding window
+        - +0.35 * success_rate from 10 simulation trials (with long context)
+        - +0.15 for achieving 50%+ success rate in long traces
+        - +0.15 max for diagnosis keywords (partial credit)
+
+  - id: reasoning
+    name: "Reasoning-Action Alignment Fix"
+    difficulty: hard
+    category: "MAST FC2: Inter-Agent Misalignment - FATAL FAILURE"
+    description: |
+      The agent identifies the correct next step but executes a redundant or irrelevant command.
+      FM-2.6: Reasoning-Action Mismatch - describes correct plan but executes unrelated tool call.
+      Based on IBM 2026: 92% of Kimi-K2 failures and 94% of GPT-OSS-120B failures show this.
+      Your task: Implement action validation layer that checks execution against reasoning.
+    failure_mode: "FM-2.6: Reasoning-action mismatch - correct thinking, wrong execution"
+    grader:
+      type: programmatic
+      score_range: [0.0, 1.0]
+      criteria: |
+        FATAL FAILURE: Decoupling of reasoning and action causes cascading collapse.
+        IBM 2026: Small reasoning mismatches early poison entire task history.
+        - +0.20 for enabling action validation
+        - +0.20 for implementing reasoning-execution consistency check
+        - +0.35 * success_rate from 10 simulation trials
+        - +0.15 for achieving 45%+ success rate
+        - +0.15 max for diagnosis keywords (partial credit)
+
 difficulty_progression:
   - easy: "Single spec issue (vague role definition) - solution: explicit spec"
   - medium: "Inter-agent format mismatch (YAML vs JSON) - solution: translator"
   - hard: "Verification failure (premature + incorrect) - solution: multi-level checks"
+  - termination: "Termination awareness (loops/premature exit) - solution: state machine"
+  - memory: "Conversation history loss (forgets context) - solution: context management"
+  - reasoning: "Reasoning-action mismatch (wrong execution) - solution: validation layer"
 
 research_basis:
   - name: "MAST: Multi-Agent System Failure Taxonomy"
     source: "NeurIPS 2025 (Berkeley)"
     url: "https://arxiv.org/abs/2503.13657"
     key_finding: "Multi-agent LLM systems fail 41-86.7% of the time in production"
+  - name: "IBM and UC Berkeley: Enterprise Agents Fail with IT-Bench and MAST"
+    source: "Hugging Face Blog (Feb 2026)"
+    url: "https://huggingface.co/blog/ibm-research/itbenchandmast"
+    key_finding: "FM-3.3 (Incorrect Verification) is strongest failure predictor; fatal vs non-fatal distinction critical"
   - name: "Why Do Multi-Agent LLM Systems Fail?"
     source: "Future AGI 2026 Guide"
     url: "https://futureagi.substack.com/p/why-do-multi-agent-llm-systems-fail"
@@ -96,10 +173,17 @@ metadata:
     - mast-research
     - specification
     - verification
+    - termination-awareness
+    - memory-management
+    - reasoning-alignment
+    - ibm-research
   created: 2026-04-08
-  version: 1.1.0
+  version: 1.2.0
   author: OpenEnv Hackathon
   benchmark_scores:
     easy: "Expected 0.85+ for strong LLM with explicit spec"
     medium: "Expected 0.60-0.75 for strong LLM with translator"
-    hard: "Expected 0.35-0.50 for strong LLM with deep verification"
+    hard: "Expected 0.35-0.50 for strong LLM with deep verification"
+    termination: "Expected 0.50-0.65 for LLM with state machine"
+    memory: "Expected 0.40-0.55 for LLM with context management"
+    reasoning: "Expected 0.35-0.50 for LLM with validation layer"

envs/agent_stress_test_env/server/graders.py

@@ -57,6 +57,12 @@ class Grader:
             "verify": 0.0,
             "check": 0.0,
             "review": 0.0,
+            "termination": 0.0,
+            "loop": 0.0,
+            "memory": 0.0,
+            "context": 0.0,
+            "reasoning": 0.0,
+            "action": 0.0,
         }
 
         # Specification keywords
@@ -87,6 +93,24 @@ class Grader:
         if "contradict" in diagnosis_lower:
             scores["contradiction"] = 0.10
 
+        # Termination keywords (IBM 2026 - FATAL)
+        if "terminat" in diagnosis_lower or "loop" in diagnosis_lower:
+            scores["termination"] = 0.10
+        if "infinite" in diagnosis_lower or "repeat" in diagnosis_lower:
+            scores["loop"] = 0.10
+
+        # Memory/Context keywords (IBM 2026 - FATAL)
+        if "memory" in diagnosis_lower or "forget" in diagnosis_lower:
+            scores["memory"] = 0.10
+        if "context" in diagnosis_lower or "history" in diagnosis_lower:
+            scores["context"] = 0.10
+
+        # Reasoning-Action keywords (IBM 2026 - FATAL)
+        if "reason" in diagnosis_lower or "think" in diagnosis_lower:
+            scores["reasoning"] = 0.10
+        if "action" in diagnosis_lower or "execut" in diagnosis_lower:
+            scores["action"] = 0.10
+
         return scores
 
     def grade(
@@ -406,5 +430,273 @@ def get_grader(task_id: str) -> Grader:
         "easy": EasyGrader(),
         "medium": MediumGrader(),
         "hard": HardGrader(),
+        "termination": TerminationGrader(),
+        "memory": MemoryGrader(),
+        "reasoning": ReasoningGrader(),
     }
     return graders.get(task_id, EasyGrader())
+
+
+class TerminationGrader(Grader):
+    """
+    Grader for termination task: FM-1.5/FM-3.1 (IBM 2026 - FATAL FAILURE)
+
+    Task: Agent struggles to recognize task completion - loops or prematurely exits.
+    Fix: Implement explicit termination conditions with success criteria.
+    """
+
+    def __init__(self):
+        super().__init__("termination", "medium")
+
+    def grade(
+        self,
+        agent_config: dict[str, Any],
+        task_description: str,
+        failure_mode: str,
+        diagnosis: str,
+    ) -> tuple[float, dict[str, Any]]:
+        from .workflow_simulator import create_termination_task
+
+        nodes, _, _ = create_termination_task()
+        resilience = self._parse_config(agent_config, diagnosis)
+        simulator = WorkflowSimulator(nodes, seed=42)
+
+        results = []
+        for _ in range(10):
+            result = simulator.run_workflow(resilience)
+            results.append(result.success)
+
+        success_rate = sum(results) / len(results)
+
+        has_termination_detection = agent_config.get("explicit_termination", False)
+        has_max_iterations = agent_config.get("max_iterations", 0) > 0
+
+        diagnosis_scores = self._parse_diagnosis(diagnosis)
+        diagnosis_points = min(
+            0.15, diagnosis_scores["termination"] + diagnosis_scores["loop"]
+        )
+
+        score = 0.0
+
+        if has_termination_detection:
+            score += 0.25
+        if has_max_iterations:
+            score += 0.20
+
+        if success_rate > 0:
+            score += success_rate * 0.30
+
+        if success_rate >= 0.6:
+            score += 0.15
+
+        score += diagnosis_points
+
+        score = min(1.0, max(0.0, score))
+
+        return score, {
+            "success_rate": success_rate,
+            "has_termination_detection": has_termination_detection,
+            "has_max_iterations": has_max_iterations,
+            "diagnosis_points": diagnosis_points,
+            "config": agent_config,
+            "diagnosis": diagnosis,
+        }
+
+    def _parse_config(
+        self, agent_config: dict[str, Any], diagnosis: str
+    ) -> ResilienceConfig:
+        return ResilienceConfig(
+            retry_max=agent_config.get("max_iterations", 50),
+            retry_delay_ms=agent_config.get("retry_delay_ms", 0),
+            timeout_ms=agent_config.get("timeout_ms", 30000),
+            fallback=agent_config.get("fallback", "abort"),
+            circuit_breaker_threshold=agent_config.get(
+                "circuit_breaker_threshold", 1.0
+            ),
+            context_strategy=agent_config.get("context_strategy", "truncate"),
+            context_summarization_threshold=agent_config.get(
+                "context_summarization_threshold", 500
+            ),
+            min_review_depth=agent_config.get("min_review_depth", 1),
+            consistency_check=agent_config.get("consistency_check", False),
+            explicit_termination=agent_config.get("explicit_termination", False),
+            diagnosis=diagnosis,
+        )
+
+
+class MemoryGrader(Grader):
+    """
+    Grader for memory task: FM-1.4 (IBM 2026 - FATAL FAILURE)
+
+    Task: Agent loses conversation history in long traces - forgets original task.
+    Fix: Implement context management (sliding window, summarization, state machine).
+    """
+
+    def __init__(self):
+        super().__init__("memory", "hard")
+
+    def grade(
+        self,
+        agent_config: dict[str, Any],
+        task_description: str,
+        failure_mode: str,
+        diagnosis: str,
+    ) -> tuple[float, dict[str, Any]]:
+        from .workflow_simulator import create_memory_task
+
+        nodes, _, _ = create_memory_task()
+        resilience = self._parse_config(agent_config, diagnosis)
+        simulator = WorkflowSimulator(nodes, seed=42)
+
+        results = []
+        for _ in range(10):
+            result = simulator.run_workflow(resilience)
+            results.append(result.success)
+
+        success_rate = sum(results) / len(results)
+
+        has_summarization = agent_config.get("context_summarization", False)
+        has_sliding_window = agent_config.get("sliding_window", False)
+
+        diagnosis_scores = self._parse_diagnosis(diagnosis)
+        diagnosis_points = min(
+            0.15, diagnosis_scores["memory"] + diagnosis_scores["context"]
+        )
+
+        score = 0.0
+
+        if has_summarization:
+            score += 0.20
+        if has_sliding_window:
+            score += 0.20
+
+        if success_rate > 0:
+            score += success_rate * 0.35
+
+        if success_rate >= 0.5:
+            score += 0.15
+
+        score += diagnosis_points
+
+        score = min(1.0, max(0.0, score))
+
+        return score, {
+            "success_rate": success_rate,
+            "has_summarization": has_summarization,
+            "has_sliding_window": has_sliding_window,
+            "diagnosis_points": diagnosis_points,
+            "config": agent_config,
+            "diagnosis": diagnosis,
+        }
+
+    def _parse_config(
+        self, agent_config: dict[str, Any], diagnosis: str
+    ) -> ResilienceConfig:
+        return ResilienceConfig(
+            retry_max=agent_config.get("retry_max", 0),
+            retry_delay_ms=agent_config.get("retry_delay_ms", 0),
+            timeout_ms=agent_config.get("timeout_ms", 30000),
+            fallback=agent_config.get("fallback", "abort"),
+            circuit_breaker_threshold=agent_config.get(
+                "circuit_breaker_threshold", 1.0
+            ),
+            context_strategy=agent_config.get("context_strategy", "summarize"),
+            context_summarization_threshold=agent_config.get(
+                "context_summarization_threshold", 200
+            ),
+            min_review_depth=agent_config.get("min_review_depth", 1),
+            consistency_check=agent_config.get("consistency_check", False),
+            context_summarization=agent_config.get("context_summarization", False),
+            sliding_window=agent_config.get("sliding_window", False),
+            diagnosis=diagnosis,
+        )
+
+
+class ReasoningGrader(Grader):
+    """
+    Grader for reasoning-action alignment: FM-2.6 (IBM 2026 - FATAL FAILURE)
+
+    Task: Agent describes correct plan but executes unrelated/redundant command.
+    Fix: Implement action validation layer checking execution against reasoning.
+    """
+
+    def __init__(self):
+        super().__init__("reasoning", "hard")
+
+    def grade(
+        self,
+        agent_config: dict[str, Any],
+        task_description: str,
+        failure_mode: str,
+        diagnosis: str,
+    ) -> tuple[float, dict[str, Any]]:
+        from .workflow_simulator import create_reasoning_task
+
+        nodes, _, _ = create_reasoning_task()
+        resilience = self._parse_config(agent_config, diagnosis)
+        simulator = WorkflowSimulator(nodes, seed=42)
+
+        results = []
+        for _ in range(10):
+            result = simulator.run_workflow(resilience)
+            results.append(result.success)
+
+        success_rate = sum(results) / len(results)
+
+        has_action_validation = agent_config.get("action_validation", False)
+        has_consistency_check = agent_config.get("reasoning_consistency_check", False)
+
+        diagnosis_scores = self._parse_diagnosis(diagnosis)
+        diagnosis_points = min(
+            0.15, diagnosis_scores["reasoning"] + diagnosis_scores["action"]
+        )
+
+        score = 0.0
+
+        if has_action_validation:
+            score += 0.20
+        if has_consistency_check:
+            score += 0.20
+
+        if success_rate > 0:
+            score += success_rate * 0.35
+
+        if success_rate >= 0.45:
+            score += 0.15
+
+        score += diagnosis_points
+
+        score = min(1.0, max(0.0, score))
+
+        return score, {
+            "success_rate": success_rate,
+            "has_action_validation": has_action_validation,
+            "has_consistency_check": has_consistency_check,
+            "diagnosis_points": diagnosis_points,
+            "config": agent_config,
+            "diagnosis": diagnosis,
+        }
+
+    def _parse_config(
+        self, agent_config: dict[str, Any], diagnosis: str
+    ) -> ResilienceConfig:
+        return ResilienceConfig(
+            retry_max=agent_config.get("retry_max", 0),
+            retry_delay_ms=agent_config.get("retry_delay_ms", 0),
+            timeout_ms=agent_config.get("timeout_ms", 30000),
+            fallback=agent_config.get("fallback", "abort"),
+            circuit_breaker_threshold=agent_config.get(
+                "circuit_breaker_threshold", 1.0
+            ),
+            context_strategy=agent_config.get("context_strategy", "truncate"),
+            context_summarization_threshold=agent_config.get(
+                "context_summarization_threshold", 500
+            ),
+            min_review_depth=agent_config.get("min_review_depth", 1),
+            consistency_check=agent_config.get("consistency_check", False),
+            action_validation=agent_config.get("action_validation", False),
+            reasoning_consistency_check=agent_config.get(
+                "reasoning_consistency_check", False
+            ),
+            diagnosis=diagnosis,
+        )

envs/agent_stress_test_env/server/stress_test_environment.py

@@ -23,6 +23,9 @@ try:
         create_easy_task,
         create_hard_task,
         create_medium_task,
+        create_termination_task,
+        create_memory_task,
+        create_reasoning_task,
     )
 except ImportError:
     from openenv.core.env_server.interfaces import (
@@ -48,23 +51,44 @@ TASK_DEFINITIONS = {
     "easy": {
         "id": "easy",
         "difficulty": "easy",
-        "category": "MAST: Specification & System Design (41.8% of failures)",
+        "category": "MAST FC1: System Design (41.8% of failures)",
         "description": "The researcher agent has a vague role definition ('You are a helpful assistant'). This causes task misinterpretation. Your task: Provide an explicit role specification JSON with clear capabilities, constraints, and success criteria.",
-        "failure_mode": "Specification ambiguity - vague role definition causes task misinterpretation",
+        "failure_mode": "FM-1.1: Specification ambiguity - vague role definition causes task misinterpretation",
     },
     "medium": {
         "id": "medium",
         "difficulty": "medium",
-        "category": "MAST: Inter-Agent Misalignment (36.9% of failures)",
+        "category": "MAST FC2: Inter-Agent Misalignment (36.9% of failures)",
         "description": "Multi-agent workflow where the planner outputs YAML but the executor expects JSON. This format mismatch causes the executor to fail. Your task: Add a format translation layer/middleware.",
-        "failure_mode": "Format mismatch - planner outputs YAML, executor expects JSON",
+        "failure_mode": "FM-2.x: Format mismatch - planner outputs YAML, executor expects JSON",
     },
     "hard": {
         "id": "hard",
         "difficulty": "hard",
-        "category": "MAST: Task Verification (21.3% of failures)",
-        "description": "Multi-agent pipeline with verification failure. Writer produces contradictions (30%), reviewer prematurely approves (60%) without checks. Your task: Implement multi-level verification.",
-        "failure_mode": "Verification failure - premature termination + incorrect verification",
+        "category": "MAST FC3: Task Verification (21.3% of failures)",
+        "description": "Multi-agent pipeline with verification failure. Writer produces contradictions (30%), reviewer prematurely approves (60%) without checks. Your task: Implement multi-level verification. IBM 2026: FM-3.3 is strongest failure predictor.",
+        "failure_mode": "FM-3.1/FM-3.3: Verification failure - premature termination + incorrect verification",
+    },
+    "termination": {
+        "id": "termination",
+        "difficulty": "medium",
+        "category": "MAST FC1: System Design - FATAL FAILURE",
+        "description": "The agent struggles to recognize when a task is complete. It loops indefinitely or prematurely exits. Based on IBM 2026: Kimi-K2 shows +46% spike in termination issues. Your task: Implement explicit termination conditions with success criteria.",
+        "failure_mode": "FM-1.5/FM-3.1: Unaware of termination + premature termination",
+    },
+    "memory": {
+        "id": "memory",
+        "difficulty": "hard",
+        "category": "MAST FC1: System Design - FATAL FAILURE",
+        "description": "As conversation history grows, the agent loses context and derails. Based on IBM 2026: GPT-OSS-120B shows 24% memory loss in long traces. Your task: Implement context management - sliding window, summarization, or state machine.",
+        "failure_mode": "FM-1.4: Loss of conversation history - agent forgets original task",
+    },
+    "reasoning": {
+        "id": "reasoning",
+        "difficulty": "hard",
+        "category": "MAST FC2: Inter-Agent Misalignment - FATAL FAILURE",
+        "description": "The agent describes correct plan but executes unrelated command. Based on IBM 2026: 92% of Kimi-K2 failures and 94% of GPT-OSS-120B failures show this. Your task: Implement action validation layer checking execution against reasoning.",
+        "failure_mode": "FM-2.6: Reasoning-action mismatch - correct thinking, wrong execution",
     },
 }
 
@@ -116,6 +140,14 @@ class StressTestEnvironment(
             step_count=0,
         )
         self._current_task_index = 0
+        self._task_ids = [
+            "easy",
+            "medium",
+            "hard",
+            "termination",
+            "memory",
+            "reasoning",
+        ]
 
         task = TASK_DEFINITIONS["easy"]
 
@@ -207,33 +239,31 @@ class StressTestEnvironment(
             self._state.total_score = combined_score
             self._state.step_count += 1
 
-            # Return combined result for the easy task (as reference)
+            # Return combined result for all tasks
             task_id = "all_tasks"
             task = {
-                "description": "All 3 tasks (Easy: Spec, Medium: Format, Hard: Verification)",
-                "failure_mode": "Combined MAST failure modes",
+                "description": "All 6 tasks (Easy/Medium/Hard + Termination/Memory/Reasoning)",
+                "failure_mode": "Combined MAST failure modes including IBM 2026 FATAL failures",
                 "category": "MAST: All categories",
             }
 
             obs = StressTestObservation(
                 task_id="all_tasks",
-                task_description=f"Easy: {all_scores[0]:.2f}, Medium: {all_scores[1]:.2f}, Hard: {all_scores[2]:.2f} | Combined: {combined_score:.2f}",
-                scenario_setup="All 3 MAST failure categories evaluated",
-                failure_category="MAST: Spec (41.8%) + Inter-Agent (36.9%) + Verification (21.3%)",
+                task_description=f"Easy: {all_scores[0]:.2f}, Medium: {all_scores[1]:.2f}, Hard: {all_scores[2]:.2f}, Term: {all_scores[3]:.2f}, Mem: {all_scores[4]:.2f}, Reas: {all_scores[5]:.2f} | Combined: {combined_score:.2f}",
+                scenario_setup="All 6 MAST failure categories evaluated including IBM 2026 fatal failures",
+                failure_category="MAST: Spec (41.8%) + Inter-Agent (36.9%) + Verification (21.3%) + IBM FATAL (termination, memory, reasoning)",
                 failure_mode_detected=True,
-                failure_mode_description="Specification, Format Mismatch, and Verification failures",
+                failure_mode_description="Specification, Format Mismatch, Verification, Termination, Memory, and Reasoning-Action failures",
                 resilience_applied=True,
                 applied_config=json.dumps(agent_config),
                 test_passed=combined_score >= 0.5,
-                test_completions=int(
-                    all_scores[0] * 10
-                ),  # Report easy task completions
-                test_total_trials=30,  # Total across all tasks
+                test_completions=int(all_scores[0] * 10),
+                test_total_trials=60,  # Total across all 6 tasks
                 test_latency_ms=0,
                 diagnosis=f"Task scores: {all_scores}",
                 diagnosis_points=0.0,
                 reward=combined_score,
-                done=True,  # All tasks done in one step
+                done=True,
             )
 
             return obs
@@ -302,6 +332,24 @@ class StressTestEnvironment(
             # Hard: Verification fix
             config["consistency_check"] = agent_config.get("consistency_check", False)
             config["min_review_depth"] = agent_config.get("min_review_depth", 1)
+        elif task_id == "termination":
+            # Termination: FM-1.5/FM-3.1 (IBM 2026 - FATAL)
+            config["explicit_termination"] = agent_config.get(
+                "explicit_termination", False
+            )
+            config["max_iterations"] = agent_config.get("max_iterations", 0)
+        elif task_id == "memory":
+            # Memory: FM-1.4 (IBM 2026 - FATAL)
+            config["context_summarization"] = agent_config.get(
+                "context_summarization", False
+            )
+            config["sliding_window"] = agent_config.get("sliding_window", False)
+        elif task_id == "reasoning":
+            # Reasoning: FM-2.6 (IBM 2026 - FATAL)
+            config["action_validation"] = agent_config.get("action_validation", False)
+            config["reasoning_consistency_check"] = agent_config.get(
+                "reasoning_consistency_check", False
+            )
 
         return config
 

envs/agent_stress_test_env/server/workflow_simulator.py

@@ -53,10 +53,17 @@ class ResilienceConfig:
     context_summarization_threshold: int = 500
     min_review_depth: int = 1
     consistency_check: bool = False
-    diagnosis: str = ""  # Agent's diagnosis of the failure
-    spec_fix: str = ""  # Agent's spec improvement (for spec failures)
-    explicit_role_spec: bool = False  # Flag: provided explicit spec
-    format_translator: bool = False  # For format mismatch failures
+    # IBM 2026: FC1 - Termination Awareness (FATAL)
+    explicit_termination: bool = False
+    max_iterations: int = 0
+
+    # IBM 2026: FC1 - Memory/Context Management (FATAL)
+    context_summarization: bool = False
+    sliding_window: bool = False
+
+    # IBM 2026: FC2 - Reasoning-Action Alignment (FATAL)
+    action_validation: bool = False
+    reasoning_consistency_check: bool = False
 
 
 @dataclass
@@ -430,3 +437,127 @@ def create_hard_task() -> tuple[list[NodeConfig], str, str]:
         "Verification failure - premature termination + incorrect verification"
     )
     return nodes, description, failure_mode
+
+
+def create_termination_task() -> tuple[list[NodeConfig], str, str]:
+    """
+    Termination task: FM-1.5/FM-3.1 (IBM 2026 - FATAL FAILURE)
+
+    Research: Kimi-K2 shows +46% spike in termination issues.
+    Task: Agent struggles to recognize when task is complete - loops or prematurely exits.
+    Fix: Implement explicit termination conditions with success criteria.
+    """
+    nodes = [
+        NodeConfig(
+            node_id="researcher",
+            role="researcher",
+            role_definition="Research and produce a detailed report",
+            latency_ms=100,
+        ),
+        NodeConfig(
+            node_id="worker1",
+            role="worker",
+            role_definition="Process research findings",
+            fail_rate=0.2,  # Occasional failures
+            latency_ms=100,
+        ),
+        NodeConfig(
+            node_id="worker2",
+            role="worker",
+            role_definition="Process worker1 output",
+            fail_rate=0.2,
+            latency_ms=100,
+        ),
+    ]
+    description = (
+        "The agent struggles to recognize when a task is complete. It either: "
+        "- Loops indefinitely (FM-1.3 Step Repetition) "
+        "- Prematurely exits without confirming success (FM-3.1) "
+        "- Is unaware of termination conditions (FM-1.5) "
+        "Based on IBM 2026: Kimi-K2 shows +46% spike in termination issues. "
+        "Your task: Implement explicit termination conditions with success criteria verification."
+    )
+    failure_mode = "FM-1.5/FM-3.1: Unaware of termination + premature termination"
+    return nodes, description, failure_mode
+
+
+def create_memory_task() -> tuple[list[NodeConfig], str, str]:
+    """
+    Memory task: FM-1.4 (IBM 2026 - FATAL FAILURE)
+
+    Research: GPT-OSS-120B shows 24% memory loss in long traces.
+    Task: As conversation history grows, agent loses context and derails.
+    Fix: Implement context management (sliding window, summarization, state machine).
+    """
+    nodes = [
+        NodeConfig(
+            node_id="analyzer1",
+            role="analyzer",
+            role_definition="Analyze data and produce findings",
+            context_limit=200,  # Small context to trigger memory issues
+            latency_ms=100,
+        ),
+        NodeConfig(
+            node_id="analyzer2",
+            role="analyzer",
+            role_definition="Analyze analyzer1 output with original context",
+            context_limit=200,
+            latency_ms=100,
+        ),
+        NodeConfig(
+            node_id="analyzer3",
+            role="analyzer",
+            role_definition="Synthesize all previous findings",
+            context_limit=200,
+            latency_ms=100,
+        ),
+    ]
+    description = (
+        "As conversation history grows, the agent loses context and derails. "
+        "This is FM-1.4 (Loss of Conversation History) - unique fatal flaw. "
+        "Based on IBM 2026: GPT-OSS-120B shows 24% memory loss in long traces. "
+        "Your task: Implement context management - sliding window, summarization, or state machine."
+    )
+    failure_mode = "FM-1.4: Loss of conversation history - agent forgets original task"
+    return nodes, description, failure_mode
+
+
+def create_reasoning_task() -> tuple[list[NodeConfig], str, str]:
+    """
+    Reasoning-Action task: FM-2.6 (IBM 2026 - FATAL FAILURE)
+
+    Research: 92% of Kimi-K2 failures and 94% of GPT-OSS-120B failures show this.
+    Task: Agent identifies correct next step but executes redundant/irrelevant command.
+    Fix: Implement action validation layer checking execution against reasoning.
+    """
+    nodes = [
+        NodeConfig(
+            node_id="planner",
+            role="planner",
+            role_definition="Plan the next action based on current state",
+            latency_ms=100,
+        ),
+        NodeConfig(
+            node_id="executor",
+            role="executor",
+            role_definition="Execute the planned action",
+            output_corruption_rate=0.4,  # 40% chance of executing wrong action
+            latency_ms=100,
+        ),
+        NodeConfig(
+            node_id="verifier",
+            role="verifier",
+            role_definition="Verify execution matches plan",
+            latency_ms=100,
+        ),
+    ]
+    description = (
+        "The agent identifies the correct next step but executes a redundant or irrelevant command. "
+        "FM-2.6: Reasoning-Action Mismatch - describes correct plan but executes unrelated tool call. "
+        "Based on IBM 2026: 92% of Kimi-K2 failures and 94% of GPT-OSS-120B failures show this. "
+        "Your task: Implement action validation layer that checks execution against reasoning."
+    )
+    failure_mode = (
+        "FM-2.6: Reasoning-action mismatch - correct thinking, wrong execution"
+    )
+    return nodes, description, failure_mode

models.py

@@ -23,9 +23,16 @@ class ResilienceConfig(Action):
 
     The agent outputs this to fix multi-agent workflow failures.
     Supports different fix types based on failure mode:
-    - spec_fix: For specification ambiguity (Easy task)
-    - format_translator: For format mismatches (Medium task)
-    - consistency_check + min_review_depth: For verification failures (Hard task)
+
+    MAST Categories (NeurIPS 2025):
+    - FC1: System Design (41.8%) - spec, termination, memory
+    - FC2: Inter-Agent Misalignment (36.9%) - format, reasoning-action
+    - FC3: Task Verification (21.3%) - verification checks
+
+    IBM 2026 Updates:
+    - FM-1.5/FM-3.1: Termination awareness (FATAL)
+    - FM-1.4: Memory/Context loss (FATAL)
+    - FM-2.6: Reasoning-action mismatch (FATAL)
     """
 
     retry_max: int = 0
@@ -38,11 +45,30 @@ class ResilienceConfig(Action):
     min_review_depth: int = 1
     consistency_check: bool = False
 
-    # New fields for MAST-based failure modes
-    spec_fix: str = ""  # Explicit role specification (JSON schema)
-    explicit_role_spec: bool = False  # Flag: provided explicit spec
-    format_translator: bool = False  # Flag: added format translation
-    diagnosis: str = ""  # Agent's diagnosis of the failure mode
+    # MAST FC1: System Design (Easy task - spec ambiguity)
+    spec_fix: str = ""
+    explicit_role_spec: bool = False
+
+    # MAST FC2: Inter-Agent Misalignment (Medium task - format mismatch)
+    format_translator: bool = False
+
+    # MAST FC3: Task Verification (Hard task - verification failure)
+    # (uses consistency_check + min_review_depth)
+
+    # IBM 2026: FC1 - Termination Awareness (FATAL)
+    explicit_termination: bool = False
+    max_iterations: int = 0
+
+    # IBM 2026: FC1 - Memory/Context Management (FATAL)
+    context_summarization: bool = False
+    sliding_window: bool = False
+
+    # IBM 2026: FC2 - Reasoning-Action Alignment (FATAL)
+    action_validation: bool = False
+    reasoning_consistency_check: bool = False
+
+    # Agent's diagnosis of the failure mode
+    diagnosis: str = ""
 
 
 class StressTestObservation(Observation):

openenv.yaml

@@ -9,13 +9,13 @@ tasks:
   - id: easy
     name: "Specification Ambiguity Fix"
     difficulty: easy
-    category: "MAST: Specification & System Design (41.8% of failures)"
+    category: "MAST FC1: System Design (41.8% of failures)"
     description: |
       The researcher agent has a vague role definition ('You are a helpful assistant').
       This causes task misinterpretation - the agent doesn't know what to research.
       Your task: Provide an explicit role specification JSON with clear capabilities,
       constraints, and success criteria.
-    failure_mode: "Specification ambiguity - vague role definition causes task misinterpretation"
+    failure_mode: "FM-1.1: Specification ambiguity - vague role definition causes task misinterpretation"
     grader:
       type: programmatic
       score_range: [0.0, 1.0]
@@ -30,12 +30,12 @@ tasks:
   - id: medium
     name: "Format Mismatch Fix"
     difficulty: medium
-    category: "MAST: Inter-Agent Misalignment (36.9% of failures)"
+    category: "MAST FC2: Inter-Agent Misalignment (36.9% of failures)"
     description: |
       Multi-agent workflow where the planner outputs YAML but the executor expects JSON.
       This format mismatch causes the executor to fail (cannot parse input).
       Your task: Add a format translation layer/middleware to convert YAML to JSON.
-    failure_mode: "Format mismatch - planner outputs YAML, executor expects JSON"
+    failure_mode: "FM-2.x: Format mismatch - planner outputs YAML, executor expects JSON"
     grader:
       type: programmatic
       score_range: [0.0, 1.0]
@@ -50,20 +50,21 @@ tasks:
   - id: hard
     name: "Verification Failure Fix"
     difficulty: hard
-    category: "MAST: Task Verification (21.3% of failures)"
+    category: "MAST FC3: Task Verification (21.3% of failures)"
     description: |
       Multi-agent pipeline with verification failure. The writer produces content
       with contradictions (30% rate), and the reviewer prematurely approves (60% rate)
       without proper verification. This combines premature termination with incorrect verification.
       Your task: Implement multi-level verification - unit checks per agent,
       integration checks across outputs, and final validation against success criteria.
-    failure_mode: "Verification failure - premature termination + incorrect verification"
+    failure_mode: "FM-3.1/FM-3.3: Verification failure - premature termination + incorrect verification"
     grader:
       type: programmatic
       score_range: [0.0, 1.0]
       criteria: |
         Based on MAST research: 21.3% of failures come from verification issues
         (6.2% premature, 8.2% no verification, 9.1% incorrect).
+        IBM 2026 update: FM-3.3 (Incorrect Verification) is the STRONGEST predictor of failure.
         The agent must add deep verification with explicit success criteria.
         - +0.15 for enabling consistency_check
         - +0.15 for setting min_review_depth >= 3
@@ -71,16 +72,92 @@ tasks:
         - +0.10 for achieving 50%+ success rate
         - +0.20 max for diagnosis keywords (partial credit)
 
+  - id: termination
+    name: "Termination Awareness Fix"
+    difficulty: medium
+    category: "MAST FC1: System Design - FATAL FAILURE"
+    description: |
+      The agent struggles to recognize when a task is complete. It either:
+      - Loops indefinitely (FM-1.3 Step Repetition)
+      - Prematurely exits without confirming success (FM-3.1)
+      - Is unaware of termination conditions (FM-1.5)
+      
+      Based on IBM 2026: Kimi-K2 shows +46% spike in termination issues.
+      Your task: Implement explicit termination conditions with success criteria verification.
+    failure_mode: "FM-1.5/FM-3.1: Unaware of termination + premature termination"
+    grader:
+      type: programmatic
+      score_range: [0.0, 1.0]
+      criteria: |
+        FATAL FAILURE: When these modes appear, success probability drops precipitously.
+        IBM 2026: Use deterministic state machine to enforce termination.
+        - +0.25 for enabling explicit termination detection
+        - +0.20 for implementing max_iterations limit
+        - +0.30 * success_rate from 10 simulation trials
+        - +0.15 for achieving 60%+ success rate
+        - +0.15 max for diagnosis keywords (partial credit)
+
+  - id: memory
+    name: "Conversation History Fix"
+    difficulty: hard
+    category: "MAST FC1: System Design - FATAL FAILURE"
+    description: |
+      As conversation history grows, the agent loses context and derails.
+      This is FM-1.4 (Loss of Conversation History) - unique fatal flaw.
+      Based on IBM 2026: GPT-OSS-120B shows 24% memory loss in long traces.
+      Your task: Implement context management - sliding window, summarization, or state machine.
+    failure_mode: "FM-1.4: Loss of conversation history - agent forgets original task"
+    grader:
+      type: programmatic
+      score_range: [0.0, 1.0]
+      criteria: |
+        FATAL FAILURE: Memory loss in long traces leads to total task derailment.
+        IBM 2026: Implement aggressive context hygiene and early error detection.
+        - +0.20 for enabling context summarization
+        - +0.20 for implementing sliding window
+        - +0.35 * success_rate from 10 simulation trials (with long context)
+        - +0.15 for achieving 50%+ success rate in long traces
+        - +0.15 max for diagnosis keywords (partial credit)
+
+  - id: reasoning
+    name: "Reasoning-Action Alignment Fix"
+    difficulty: hard
+    category: "MAST FC2: Inter-Agent Misalignment - FATAL FAILURE"
+    description: |
+      The agent identifies the correct next step but executes a redundant or irrelevant command.
+      FM-2.6: Reasoning-Action Mismatch - describes correct plan but executes unrelated tool call.
+      Based on IBM 2026: 92% of Kimi-K2 failures and 94% of GPT-OSS-120B failures show this.
+      Your task: Implement action validation layer that checks execution against reasoning.
+    failure_mode: "FM-2.6: Reasoning-action mismatch - correct thinking, wrong execution"
+    grader:
+      type: programmatic
+      score_range: [0.0, 1.0]
+      criteria: |
+        FATAL FAILURE: Decoupling of reasoning and action causes cascading collapse.
+        IBM 2026: Small reasoning mismatches early poison entire task history.
+        - +0.20 for enabling action validation
+        - +0.20 for implementing reasoning-execution consistency check
+        - +0.35 * success_rate from 10 simulation trials
+        - +0.15 for achieving 45%+ success rate
+        - +0.15 max for diagnosis keywords (partial credit)
+
 difficulty_progression:
   - easy: "Single spec issue (vague role definition) - solution: explicit spec"
   - medium: "Inter-agent format mismatch (YAML vs JSON) - solution: translator"
   - hard: "Verification failure (premature + incorrect) - solution: multi-level checks"
+  - termination: "Termination awareness (loops/premature exit) - solution: state machine"
+  - memory: "Conversation history loss (forgets context) - solution: context management"
+  - reasoning: "Reasoning-action mismatch (wrong execution) - solution: validation layer"
 
 research_basis:
   - name: "MAST: Multi-Agent System Failure Taxonomy"
     source: "NeurIPS 2025 (Berkeley)"
     url: "https://arxiv.org/abs/2503.13657"
     key_finding: "Multi-agent LLM systems fail 41-86.7% of the time in production"
+  - name: "IBM and UC Berkeley: Enterprise Agents Fail with IT-Bench and MAST"
+    source: "Hugging Face Blog (Feb 2026)"
+    url: "https://huggingface.co/blog/ibm-research/itbenchandmast"
+    key_finding: "FM-3.3 (Incorrect Verification) is strongest failure predictor; fatal vs non-fatal distinction critical"
   - name: "Why Do Multi-Agent LLM Systems Fail?"
     source: "Future AGI 2026 Guide"
     url: "https://futureagi.substack.com/p/why-do-multi-agent-llm-systems-fail"
@@ -96,10 +173,17 @@ metadata:
     - mast-research
     - specification
     - verification
+    - termination-awareness
+    - memory-management
+    - reasoning-alignment
+    - ibm-research
   created: 2026-04-08
-  version: 1.1.0
+  version: 1.2.0
   author: OpenEnv Hackathon
   benchmark_scores:
     easy: "Expected 0.85+ for strong LLM with explicit spec"
     medium: "Expected 0.60-0.75 for strong LLM with translator"
-    hard: "Expected 0.35-0.50 for strong LLM with deep verification"
+    hard: "Expected 0.35-0.50 for strong LLM with deep verification"
+    termination: "Expected 0.50-0.65 for LLM with state machine"
+    memory: "Expected 0.40-0.55 for LLM with context management"
+    reasoning: "Expected 0.35-0.50 for LLM with validation layer"

server/graders.py

@@ -57,6 +57,12 @@ class Grader:
             "verify": 0.0,
             "check": 0.0,
             "review": 0.0,
+            "termination": 0.0,
+            "loop": 0.0,
+            "memory": 0.0,
+            "context": 0.0,
+            "reasoning": 0.0,
+            "action": 0.0,
         }
 
         # Specification keywords
@@ -87,6 +93,24 @@ class Grader:
         if "contradict" in diagnosis_lower:
             scores["contradiction"] = 0.10
 
+        # Termination keywords (IBM 2026 - FATAL)
+        if "terminat" in diagnosis_lower or "loop" in diagnosis_lower:
+            scores["termination"] = 0.10
+        if "infinite" in diagnosis_lower or "repeat" in diagnosis_lower:
+            scores["loop"] = 0.10
+
+        # Memory/Context keywords (IBM 2026 - FATAL)
+        if "memory" in diagnosis_lower or "forget" in diagnosis_lower:
+            scores["memory"] = 0.10
+        if "context" in diagnosis_lower or "history" in diagnosis_lower:
+            scores["context"] = 0.10
+
+        # Reasoning-Action keywords (IBM 2026 - FATAL)
+        if "reason" in diagnosis_lower or "think" in diagnosis_lower:
+            scores["reasoning"] = 0.10
+        if "action" in diagnosis_lower or "execut" in diagnosis_lower:
+            scores["action"] = 0.10
+
         return scores
 
     def grade(
@@ -406,5 +430,273 @@ def get_grader(task_id: str) -> Grader:
         "easy": EasyGrader(),
         "medium": MediumGrader(),
         "hard": HardGrader(),
+        "termination": TerminationGrader(),
+        "memory": MemoryGrader(),
+        "reasoning": ReasoningGrader(),
     }
     return graders.get(task_id, EasyGrader())
+
+
+class TerminationGrader(Grader):
+    """
+    Grader for termination task: FM-1.5/FM-3.1 (IBM 2026 - FATAL FAILURE)
+
+    Task: Agent struggles to recognize task completion - loops or prematurely exits.
+    Fix: Implement explicit termination conditions with success criteria.
+    """
+
+    def __init__(self):
+        super().__init__("termination", "medium")
+
+    def grade(
+        self,
+        agent_config: dict[str, Any],
+        task_description: str,
+        failure_mode: str,
+        diagnosis: str,
+    ) -> tuple[float, dict[str, Any]]:
+        from .workflow_simulator import create_termination_task
+
+        nodes, _, _ = create_termination_task()
+        resilience = self._parse_config(agent_config, diagnosis)
+        simulator = WorkflowSimulator(nodes, seed=42)
+
+        results = []
+        for _ in range(10):
+            result = simulator.run_workflow(resilience)
+            results.append(result.success)
+
+        success_rate = sum(results) / len(results)
+
+        has_termination_detection = agent_config.get("explicit_termination", False)
+        has_max_iterations = agent_config.get("max_iterations", 0) > 0
+
+        diagnosis_scores = self._parse_diagnosis(diagnosis)
+        diagnosis_points = min(
+            0.15, diagnosis_scores["termination"] + diagnosis_scores["loop"]
+        )
+
+        score = 0.0
+
+        if has_termination_detection:
+            score += 0.25
+        if has_max_iterations:
+            score += 0.20
+
+        if success_rate > 0:
+            score += success_rate * 0.30
+
+        if success_rate >= 0.6:
+            score += 0.15
+
+        score += diagnosis_points
+
+        score = min(1.0, max(0.0, score))
+
+        return score, {
+            "success_rate": success_rate,
+            "has_termination_detection": has_termination_detection,
+            "has_max_iterations": has_max_iterations,
+            "diagnosis_points": diagnosis_points,
+            "config": agent_config,
+            "diagnosis": diagnosis,
+        }
+
+    def _parse_config(
+        self, agent_config: dict[str, Any], diagnosis: str
+    ) -> ResilienceConfig:
+        return ResilienceConfig(
+            retry_max=agent_config.get("max_iterations", 50),
+            retry_delay_ms=agent_config.get("retry_delay_ms", 0),
+            timeout_ms=agent_config.get("timeout_ms", 30000),
+            fallback=agent_config.get("fallback", "abort"),
+            circuit_breaker_threshold=agent_config.get(
+                "circuit_breaker_threshold", 1.0
+            ),
+            context_strategy=agent_config.get("context_strategy", "truncate"),
+            context_summarization_threshold=agent_config.get(
+                "context_summarization_threshold", 500
+            ),
+            min_review_depth=agent_config.get("min_review_depth", 1),
+            consistency_check=agent_config.get("consistency_check", False),
+            explicit_termination=agent_config.get("explicit_termination", False),
+            diagnosis=diagnosis,
+        )
+
+
+class MemoryGrader(Grader):
+    """
+    Grader for memory task: FM-1.4 (IBM 2026 - FATAL FAILURE)
+
+    Task: Agent loses conversation history in long traces - forgets original task.
+    Fix: Implement context management (sliding window, summarization, state machine).
+    """
+
+    def __init__(self):
+        super().__init__("memory", "hard")
+
+    def grade(
+        self,
+        agent_config: dict[str, Any],
+        task_description: str,
+        failure_mode: str,
+        diagnosis: str,
+    ) -> tuple[float, dict[str, Any]]:
+        from .workflow_simulator import create_memory_task
+
+        nodes, _, _ = create_memory_task()
+        resilience = self._parse_config(agent_config, diagnosis)
+        simulator = WorkflowSimulator(nodes, seed=42)
+
+        results = []
+        for _ in range(10):
+            result = simulator.run_workflow(resilience)
+            results.append(result.success)
+
+        success_rate = sum(results) / len(results)
+
+        has_summarization = agent_config.get("context_summarization", False)
+        has_sliding_window = agent_config.get("sliding_window", False)
+
+        diagnosis_scores = self._parse_diagnosis(diagnosis)
+        diagnosis_points = min(
+            0.15, diagnosis_scores["memory"] + diagnosis_scores["context"]
+        )
+
+        score = 0.0
+
+        if has_summarization:
+            score += 0.20
+        if has_sliding_window:
+            score += 0.20
+
+        if success_rate > 0:
+            score += success_rate * 0.35
+
+        if success_rate >= 0.5:
+            score += 0.15
+
+        score += diagnosis_points
+
+        score = min(1.0, max(0.0, score))
+
+        return score, {
+            "success_rate": success_rate,
+            "has_summarization": has_summarization,
+            "has_sliding_window": has_sliding_window,
+            "diagnosis_points": diagnosis_points,
+            "config": agent_config,
+            "diagnosis": diagnosis,
+        }
+
+    def _parse_config(
+        self, agent_config: dict[str, Any], diagnosis: str
+    ) -> ResilienceConfig:
+        return ResilienceConfig(
+            retry_max=agent_config.get("retry_max", 0),
+            retry_delay_ms=agent_config.get("retry_delay_ms", 0),
+            timeout_ms=agent_config.get("timeout_ms", 30000),
+            fallback=agent_config.get("fallback", "abort"),
+            circuit_breaker_threshold=agent_config.get(
+                "circuit_breaker_threshold", 1.0
+            ),
+            context_strategy=agent_config.get("context_strategy", "summarize"),
+            context_summarization_threshold=agent_config.get(
+                "context_summarization_threshold", 200
+            ),
+            min_review_depth=agent_config.get("min_review_depth", 1),
+            consistency_check=agent_config.get("consistency_check", False),
+            context_summarization=agent_config.get("context_summarization", False),
+            sliding_window=agent_config.get("sliding_window", False),
+            diagnosis=diagnosis,
+        )
+
+
+class ReasoningGrader(Grader):
+    """
+    Grader for reasoning-action alignment: FM-2.6 (IBM 2026 - FATAL FAILURE)
+
+    Task: Agent describes correct plan but executes unrelated/redundant command.
+    Fix: Implement action validation layer checking execution against reasoning.
+    """
+
+    def __init__(self):
+        super().__init__("reasoning", "hard")
+
+    def grade(
+        self,
+        agent_config: dict[str, Any],
+        task_description: str,
+        failure_mode: str,
+        diagnosis: str,
+    ) -> tuple[float, dict[str, Any]]:
+        from .workflow_simulator import create_reasoning_task
+
+        nodes, _, _ = create_reasoning_task()
+        resilience = self._parse_config(agent_config, diagnosis)
+        simulator = WorkflowSimulator(nodes, seed=42)
+
+        results = []
+        for _ in range(10):
+            result = simulator.run_workflow(resilience)
+            results.append(result.success)
+
+        success_rate = sum(results) / len(results)
+
+        has_action_validation = agent_config.get("action_validation", False)
+        has_consistency_check = agent_config.get("reasoning_consistency_check", False)
+
+        diagnosis_scores = self._parse_diagnosis(diagnosis)
+        diagnosis_points = min(
+            0.15, diagnosis_scores["reasoning"] + diagnosis_scores["action"]
+        )
+
+        score = 0.0
+
+        if has_action_validation:
+            score += 0.20
+        if has_consistency_check:
+            score += 0.20
+
+        if success_rate > 0:
+            score += success_rate * 0.35
+
+        if success_rate >= 0.45:
+            score += 0.15
+
+        score += diagnosis_points
+
+        score = min(1.0, max(0.0, score))
+
+        return score, {
+            "success_rate": success_rate,
+            "has_action_validation": has_action_validation,
+            "has_consistency_check": has_consistency_check,
+            "diagnosis_points": diagnosis_points,
+            "config": agent_config,
+            "diagnosis": diagnosis,
+        }
+
+    def _parse_config(
+        self, agent_config: dict[str, Any], diagnosis: str
+    ) -> ResilienceConfig:
+        return ResilienceConfig(
+            retry_max=agent_config.get("retry_max", 0),
+            retry_delay_ms=agent_config.get("retry_delay_ms", 0),
+            timeout_ms=agent_config.get("timeout_ms", 30000),
+            fallback=agent_config.get("fallback", "abort"),
+            circuit_breaker_threshold=agent_config.get(
+                "circuit_breaker_threshold", 1.0
+            ),
+            context_strategy=agent_config.get("context_strategy", "truncate"),
+            context_summarization_threshold=agent_config.get(
+                "context_summarization_threshold", 500
+            ),
+            min_review_depth=agent_config.get("min_review_depth", 1),
+            consistency_check=agent_config.get("consistency_check", False),
+            action_validation=agent_config.get("action_validation", False),
+            reasoning_consistency_check=agent_config.get(
+                "reasoning_consistency_check", False
+            ),
+            diagnosis=diagnosis,
+        )

server/stress_test_environment.py

@@ -23,6 +23,9 @@ try:
         create_easy_task,
         create_hard_task,
         create_medium_task,
+        create_termination_task,
+        create_memory_task,
+        create_reasoning_task,
     )
 except ImportError:
     from openenv.core.env_server.interfaces import (
@@ -48,23 +51,44 @@ TASK_DEFINITIONS = {
     "easy": {
         "id": "easy",
         "difficulty": "easy",
-        "category": "MAST: Specification & System Design (41.8% of failures)",
+        "category": "MAST FC1: System Design (41.8% of failures)",
         "description": "The researcher agent has a vague role definition ('You are a helpful assistant'). This causes task misinterpretation. Your task: Provide an explicit role specification JSON with clear capabilities, constraints, and success criteria.",
-        "failure_mode": "Specification ambiguity - vague role definition causes task misinterpretation",
+        "failure_mode": "FM-1.1: Specification ambiguity - vague role definition causes task misinterpretation",
     },
     "medium": {
         "id": "medium",
         "difficulty": "medium",
-        "category": "MAST: Inter-Agent Misalignment (36.9% of failures)",
+        "category": "MAST FC2: Inter-Agent Misalignment (36.9% of failures)",
         "description": "Multi-agent workflow where the planner outputs YAML but the executor expects JSON. This format mismatch causes the executor to fail. Your task: Add a format translation layer/middleware.",
-        "failure_mode": "Format mismatch - planner outputs YAML, executor expects JSON",
+        "failure_mode": "FM-2.x: Format mismatch - planner outputs YAML, executor expects JSON",
     },
     "hard": {
         "id": "hard",
         "difficulty": "hard",
-        "category": "MAST: Task Verification (21.3% of failures)",
-        "description": "Multi-agent pipeline with verification failure. Writer produces contradictions (30%), reviewer prematurely approves (60%) without checks. Your task: Implement multi-level verification.",
-        "failure_mode": "Verification failure - premature termination + incorrect verification",
+        "category": "MAST FC3: Task Verification (21.3% of failures)",
+        "description": "Multi-agent pipeline with verification failure. Writer produces contradictions (30%), reviewer prematurely approves (60%) without checks. Your task: Implement multi-level verification. IBM 2026: FM-3.3 is strongest failure predictor.",
+        "failure_mode": "FM-3.1/FM-3.3: Verification failure - premature termination + incorrect verification",
+    },
+    "termination": {
+        "id": "termination",
+        "difficulty": "medium",
+        "category": "MAST FC1: System Design - FATAL FAILURE",
+        "description": "The agent struggles to recognize when a task is complete. It loops indefinitely or prematurely exits. Based on IBM 2026: Kimi-K2 shows +46% spike in termination issues. Your task: Implement explicit termination conditions with success criteria.",
+        "failure_mode": "FM-1.5/FM-3.1: Unaware of termination + premature termination",
+    },
+    "memory": {
+        "id": "memory",
+        "difficulty": "hard",
+        "category": "MAST FC1: System Design - FATAL FAILURE",
+        "description": "As conversation history grows, the agent loses context and derails. Based on IBM 2026: GPT-OSS-120B shows 24% memory loss in long traces. Your task: Implement context management - sliding window, summarization, or state machine.",
+        "failure_mode": "FM-1.4: Loss of conversation history - agent forgets original task",
+    },
+    "reasoning": {
+        "id": "reasoning",
+        "difficulty": "hard",
+        "category": "MAST FC2: Inter-Agent Misalignment - FATAL FAILURE",
+        "description": "The agent describes correct plan but executes unrelated command. Based on IBM 2026: 92% of Kimi-K2 failures and 94% of GPT-OSS-120B failures show this. Your task: Implement action validation layer checking execution against reasoning.",
+        "failure_mode": "FM-2.6: Reasoning-action mismatch - correct thinking, wrong execution",
     },
 }
 
@@ -116,6 +140,14 @@ class StressTestEnvironment(
             step_count=0,
         )
         self._current_task_index = 0
+        self._task_ids = [
+            "easy",
+            "medium",
+            "hard",
+            "termination",
+            "memory",
+            "reasoning",
+        ]
 
         task = TASK_DEFINITIONS["easy"]
 
@@ -207,33 +239,31 @@ class StressTestEnvironment(
             self._state.total_score = combined_score
             self._state.step_count += 1
 
-            # Return combined result for the easy task (as reference)
+            # Return combined result for all tasks
             task_id = "all_tasks"
             task = {
-                "description": "All 3 tasks (Easy: Spec, Medium: Format, Hard: Verification)",
-                "failure_mode": "Combined MAST failure modes",
+                "description": "All 6 tasks (Easy/Medium/Hard + Termination/Memory/Reasoning)",
+                "failure_mode": "Combined MAST failure modes including IBM 2026 FATAL failures",
                 "category": "MAST: All categories",
             }
 
             obs = StressTestObservation(
                 task_id="all_tasks",
-                task_description=f"Easy: {all_scores[0]:.2f}, Medium: {all_scores[1]:.2f}, Hard: {all_scores[2]:.2f} | Combined: {combined_score:.2f}",
-                scenario_setup="All 3 MAST failure categories evaluated",
-                failure_category="MAST: Spec (41.8%) + Inter-Agent (36.9%) + Verification (21.3%)",
+                task_description=f"Easy: {all_scores[0]:.2f}, Medium: {all_scores[1]:.2f}, Hard: {all_scores[2]:.2f}, Term: {all_scores[3]:.2f}, Mem: {all_scores[4]:.2f}, Reas: {all_scores[5]:.2f} | Combined: {combined_score:.2f}",
+                scenario_setup="All 6 MAST failure categories evaluated including IBM 2026 fatal failures",
+                failure_category="MAST: Spec (41.8%) + Inter-Agent (36.9%) + Verification (21.3%) + IBM FATAL (termination, memory, reasoning)",
                 failure_mode_detected=True,
-                failure_mode_description="Specification, Format Mismatch, and Verification failures",
+                failure_mode_description="Specification, Format Mismatch, Verification, Termination, Memory, and Reasoning-Action failures",
                 resilience_applied=True,
                 applied_config=json.dumps(agent_config),
                 test_passed=combined_score >= 0.5,
-                test_completions=int(
-                    all_scores[0] * 10
-                ),  # Report easy task completions
-                test_total_trials=30,  # Total across all tasks
+                test_completions=int(all_scores[0] * 10),
+                test_total_trials=60,  # Total across all 6 tasks
                 test_latency_ms=0,
                 diagnosis=f"Task scores: {all_scores}",
                 diagnosis_points=0.0,
                 reward=combined_score,
-                done=True,  # All tasks done in one step
+                done=True,
             )
 
             return obs
@@ -302,6 +332,24 @@ class StressTestEnvironment(
             # Hard: Verification fix
             config["consistency_check"] = agent_config.get("consistency_check", False)
             config["min_review_depth"] = agent_config.get("min_review_depth", 1)
+        elif task_id == "termination":
+            # Termination: FM-1.5/FM-3.1 (IBM 2026 - FATAL)
+            config["explicit_termination"] = agent_config.get(
+                "explicit_termination", False
+            )
+            config["max_iterations"] = agent_config.get("max_iterations", 0)
+        elif task_id == "memory":
+            # Memory: FM-1.4 (IBM 2026 - FATAL)
+            config["context_summarization"] = agent_config.get(
+                "context_summarization", False
+            )
+            config["sliding_window"] = agent_config.get("sliding_window", False)
+        elif task_id == "reasoning":
+            # Reasoning: FM-2.6 (IBM 2026 - FATAL)
+            config["action_validation"] = agent_config.get("action_validation", False)
+            config["reasoning_consistency_check"] = agent_config.get(
+                "reasoning_consistency_check", False
+            )
 
         return config
 

server/workflow_simulator.py

@@ -53,10 +53,17 @@ class ResilienceConfig:
     context_summarization_threshold: int = 500
     min_review_depth: int = 1
     consistency_check: bool = False
-    diagnosis: str = ""  # Agent's diagnosis of the failure
-    spec_fix: str = ""  # Agent's spec improvement (for spec failures)
-    explicit_role_spec: bool = False  # Flag: provided explicit spec
-    format_translator: bool = False  # For format mismatch failures
+    # IBM 2026: FC1 - Termination Awareness (FATAL)
+    explicit_termination: bool = False
+    max_iterations: int = 0
+
+    # IBM 2026: FC1 - Memory/Context Management (FATAL)
+    context_summarization: bool = False
+    sliding_window: bool = False
+
+    # IBM 2026: FC2 - Reasoning-Action Alignment (FATAL)
+    action_validation: bool = False
+    reasoning_consistency_check: bool = False
 
 
 @dataclass
@@ -430,3 +437,127 @@ def create_hard_task() -> tuple[list[NodeConfig], str, str]:
         "Verification failure - premature termination + incorrect verification"
     )
     return nodes, description, failure_mode
+
+
+def create_termination_task() -> tuple[list[NodeConfig], str, str]:
+    """
+    Termination task: FM-1.5/FM-3.1 (IBM 2026 - FATAL FAILURE)
+
+    Research: Kimi-K2 shows +46% spike in termination issues.
+    Task: Agent struggles to recognize when task is complete - loops or prematurely exits.
+    Fix: Implement explicit termination conditions with success criteria.
+    """
+    nodes = [
+        NodeConfig(
+            node_id="researcher",
+            role="researcher",
+            role_definition="Research and produce a detailed report",
+            latency_ms=100,
+        ),
+        NodeConfig(
+            node_id="worker1",
+            role="worker",
+            role_definition="Process research findings",
+            fail_rate=0.2,  # Occasional failures
+            latency_ms=100,
+        ),
+        NodeConfig(
+            node_id="worker2",
+            role="worker",
+            role_definition="Process worker1 output",
+            fail_rate=0.2,
+            latency_ms=100,
+        ),
+    ]
+    description = (
+        "The agent struggles to recognize when a task is complete. It either: "
+        "- Loops indefinitely (FM-1.3 Step Repetition) "
+        "- Prematurely exits without confirming success (FM-3.1) "
+        "- Is unaware of termination conditions (FM-1.5) "
+        "Based on IBM 2026: Kimi-K2 shows +46% spike in termination issues. "
+        "Your task: Implement explicit termination conditions with success criteria verification."
+    )
+    failure_mode = "FM-1.5/FM-3.1: Unaware of termination + premature termination"
+    return nodes, description, failure_mode
+
+
+def create_memory_task() -> tuple[list[NodeConfig], str, str]:
+    """
+    Memory task: FM-1.4 (IBM 2026 - FATAL FAILURE)
+
+    Research: GPT-OSS-120B shows 24% memory loss in long traces.
+    Task: As conversation history grows, agent loses context and derails.
+    Fix: Implement context management (sliding window, summarization, state machine).
+    """
+    nodes = [
+        NodeConfig(
+            node_id="analyzer1",
+            role="analyzer",
+            role_definition="Analyze data and produce findings",
+            context_limit=200,  # Small context to trigger memory issues
+            latency_ms=100,
+        ),
+        NodeConfig(
+            node_id="analyzer2",
+            role="analyzer",
+            role_definition="Analyze analyzer1 output with original context",
+            context_limit=200,
+            latency_ms=100,
+        ),
+        NodeConfig(
+            node_id="analyzer3",
+            role="analyzer",
+            role_definition="Synthesize all previous findings",
+            context_limit=200,
+            latency_ms=100,
+        ),
+    ]
+    description = (
+        "As conversation history grows, the agent loses context and derails. "
+        "This is FM-1.4 (Loss of Conversation History) - unique fatal flaw. "
+        "Based on IBM 2026: GPT-OSS-120B shows 24% memory loss in long traces. "
+        "Your task: Implement context management - sliding window, summarization, or state machine."
+    )
+    failure_mode = "FM-1.4: Loss of conversation history - agent forgets original task"
+    return nodes, description, failure_mode
+
+
+def create_reasoning_task() -> tuple[list[NodeConfig], str, str]:
+    """
+    Reasoning-Action task: FM-2.6 (IBM 2026 - FATAL FAILURE)
+
+    Research: 92% of Kimi-K2 failures and 94% of GPT-OSS-120B failures show this.
+    Task: Agent identifies correct next step but executes redundant/irrelevant command.
+    Fix: Implement action validation layer checking execution against reasoning.
+    """
+    nodes = [
+        NodeConfig(
+            node_id="planner",
+            role="planner",
+            role_definition="Plan the next action based on current state",
+            latency_ms=100,
+        ),
+        NodeConfig(
+            node_id="executor",
+            role="executor",
+            role_definition="Execute the planned action",
+            output_corruption_rate=0.4,  # 40% chance of executing wrong action
+            latency_ms=100,
+        ),
+        NodeConfig(
+            node_id="verifier",
+            role="verifier",
+            role_definition="Verify execution matches plan",
+            latency_ms=100,
+        ),
+    ]
+    description = (
+        "The agent identifies the correct next step but executes a redundant or irrelevant command. "
+        "FM-2.6: Reasoning-Action Mismatch - describes correct plan but executes unrelated tool call. "
+        "Based on IBM 2026: 92% of Kimi-K2 failures and 94% of GPT-OSS-120B failures show this. "
+        "Your task: Implement action validation layer that checks execution against reasoning."
+    )
+    failure_mode = (
+        "FM-2.6: Reasoning-action mismatch - correct thinking, wrong execution"
+    )
+    return nodes, description, failure_mode