max token error recovery, browser concurrency fixes

.gitignore

@@ -216,6 +216,8 @@ memory/
 /tools/
 epics/
 outputs/
+errors/
+repl_state/
 logs/
 .leann/
 

docker/Dockerfile

@@ -7,7 +7,7 @@ FROM cyber-autoagent-tools:latest
 
 LABEL org.opencontainers.image.title="Cyber-AutoAgent"
 LABEL org.opencontainers.image.description="Autonomous AI agent for ethical security assessments"
-LABEL org.opencontainers.image.version="0.5.0"
+LABEL org.opencontainers.image.version="0.5.1"
 LABEL org.opencontainers.image.source="https://github.com/double16/cyber-autoagent"
 LABEL org.opencontainers.image.licenses="MIT"
 

docker/Dockerfile.tools

@@ -22,7 +22,7 @@ FROM kalilinux/kali-rolling
 
 LABEL org.opencontainers.image.title="Cyber-AutoAgent Tools"
 LABEL org.opencontainers.image.description="Autonomous AI agent for ethical security assessments"
-LABEL org.opencontainers.image.version="0.5.0"
+LABEL org.opencontainers.image.version="0.5.1"
 LABEL org.opencontainers.image.source="https://github.com/double16/cyber-autoagent"
 LABEL org.opencontainers.image.licenses="MIT"
 

docker/docker-compose.yml

@@ -90,6 +90,7 @@ services:
       - MEM0_EMBEDDING_MODEL=${MEM0_EMBEDDING_MODEL}
       - MEM0_LLM_MODEL=${MEM0_LLM_MODEL}
       - OPENSEARCH_HOST=${OPENSEARCH_HOST}
+      - MEM0_TELEMETRY=false
 
       # Observability configuration (auto-detected by default, forced true in compose mode)
       - ENABLE_OBSERVABILITY=${ENABLE_OBSERVABILITY:-true}

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "cyber-autoagent"
-version = "0.5.0"
+version = "0.5.1"
 description = "Cyber-AutoAgent: Autonomous AI agent for ethical security assessments"
 readme = "README.md"
 authors = [

pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+markers =
+    browser: Mark a test as requiring a browser

src/__init__.py

@@ -2,7 +2,7 @@
 Cyber-AutoAgent - Autonomous Cybersecurity Assessment Tool
 """
 
-__version__ = "0.5.0"
+__version__ = "0.5.1"
 __author__ = "Patrick Double"
 __credits__ = ["Aaron Brown (original author)"]
 __license__ = "MIT"

src/cyberautoagent.py

@@ -71,6 +71,7 @@
 from modules.tools import browser, channel_close_all
 from modules.tools.oast import close_oast_providers
 from modules.utils.telemetry import flush_traces
+from modules.utils.text_reducer import reduce_lines_lossy, collapse_first_repeated_sequence
 
 load_dotenv()
 
@@ -689,24 +690,25 @@ def _initial_prompt_accessor():
         step0_retry = 2
         # the number of consecutive action-less results
         actionless_step_count = 0
+        max_tokens_retry_count = 0
 
         # SDK-aligned execution loop with continuation support
         while not interrupted:
+            last_step = callback_handler.current_step
             try:
                 print_status(
                     f"Agent processing: {current_message[:100]}{' ...' if len(current_message) > 100 else ''}",
                     "THINKING",
                 )
                 logger.debug(f"Agent processing: {current_message}")
 
-                last_step = callback_handler.current_step
-
                 _strip_continue_messages(agent)
                 _ensure_prompt_within_budget(agent)
                 # Execute agent with current message
                 result = agent(current_message)
 
                 logger.debug(f"Agent result: {repr(result)}")
+                max_tokens_retry_count = 0
 
                 # Pass the metrics from the result to the callback handler
                 if (
@@ -821,12 +823,60 @@ def __init__(self, accumulated_usage):
 
             except Exception as error:
                 # Handle other termination scenarios
-                logger.debug("Termination exception", exc_info=error)
                 error_str = str(error).lower()
                 if isinstance(error, MaxTokensReachedException) or "maxtokensreached" in error_str or "max_tokens" in error_str:
+                    # if there is a response or reasoning text, reduce it and add as an assistant message, then try again
+                    if last_step != callback_handler.current_step:
+                        # progress was made
+                        max_tokens_retry_count = 0
+                    if callback_handler and max_tokens_retry_count < 2:
+                        # sometimes the max_tokens response includes the response, otherwise we'll look for reasoning text
+                        truncated_message = ""
+                        replace_last_message = False
+                        if agent.messages[-1].get("role", "") == "assistant":
+                            truncated_message = "".join([block.get("text", "") for block in agent.messages[-1].get("content", [])])
+                            replace_last_message = bool(truncated_message)
+                        if not truncated_message:
+                            truncated_message = "".join(callback_handler.reasoning_buffer).strip()
+                            truncated_message_prefix = truncated_message[:1000]
+                            replace_last_message = truncated_message and any([block.get("text", "").startswith(truncated_message_prefix) for block in agent.messages[-1].get("content", [])])
+                        reduced_text = reduce_lines_lossy(
+                            collapse_first_repeated_sequence(truncated_message),
+                            similarity_threshold=0.5, max_lines=40
+                        ).to_text().strip()
+                        max_tokens_retry_count += 1
+                        if reduced_text:
+                            reduced_message = {"role": "assistant", "content": [{"type": "text", "text": reduced_text}]}
+                            if replace_last_message:
+                                agent.messages[-1] = reduced_message
+                            else:
+                                agent.messages.append(reduced_message)
+                        reflection_snapshot = get_reflection_snapshot(
+                            current_step=callback_handler.current_step,
+                            max_steps=callback_handler.max_steps,
+                            plan_current_phase=None,
+                        )
+                        current_message = f"""<continue_instructions>
+You are continuing from a prior run that entered a repetitive reasoning loop.
+
+## CONSTRAINTS
+- Do NOT restate repeated points from the reduced notes.
+- Output must be structured, actionable, and short.
+- Avoid meta commentary about "looping" beyond what's required to recover.
+
+{reflection_snapshot}
+<continue_instructions>"""
+                        try:
+                            callback_handler._emit_accumulated_reasoning(force=True)
+                        except Exception:
+                            pass
+                        logger.warning("Model token limit reached, retrying with reduced text")
+                        continue
+
                     print_status(
                         "Token limit reached - generating final report", "WARNING"
                     )
+                    logger.debug("Termination exception", exc_info=error)
                     try:
                         if callback_handler:
                             callback_handler.emit_termination(
@@ -838,7 +888,10 @@ def __init__(self, accumulated_usage):
                             )
                     except Exception as max_tokens_finish_error:
                         logger.error("Failed to complete for token limit error", exc_info=max_tokens_finish_error)
-                elif "event loop cycle stop requested" in error_str:
+                    break
+
+                logger.debug("Termination exception", exc_info=error)
+                if "event loop cycle stop requested" in error_str:
                     # Extract the reason from the error message
                     reason_match = re.search(r"Reason: (.+?)(?:\\n|$)", str(error))
                     reason = (

src/modules/__init__.py

@@ -1,6 +1,6 @@
 """Cyber-AutoAgent modules package."""
 
-__version__ = "0.5.0"
+__version__ = "0.5.1"
 __author__ = "Patrick Double"
 __credits__ = ["Aaron Brown (original author)"]
 __license__ = "MIT"

src/modules/agents/patches.py

@@ -29,13 +29,26 @@
 @dataclass
 class _ToolUseIdStreamState:
     current_tool_use_id: Optional[str] = None
+    where_set: Optional[str] = None
+
+    def __call__(self, *, where: str, marker: str, id_factory: Optional[Callable[[str], str]]) -> str:
+        assert where
+        assert marker
+        assert id_factory
+        if self.where_set and self.where_set != where:
+            if not self.current_tool_use_id:
+                self.current_tool_use_id = id_factory(marker)
+                self.where_set = where
+        else:
+            self.current_tool_use_id = id_factory(marker)
+            self.where_set = where
+        return self.current_tool_use_id
 
 
 def patch_model_class_tool_use_id(
         model_cls: Type[Any],
         *,
-        is_bad_id: Optional[Callable[[Optional[str], Optional[str]], bool]] = None,
-        id_factory: Optional[Callable[[], str]] = None,
+        id_factory: Optional[Callable[[str], str]] = None,
         attr_prefix: str = "_tooluseid_class_patch",
 ) -> Type[Any]:
     """
@@ -63,17 +76,10 @@ def patch_model_class_tool_use_id(
             return model_cls
         raise TypeError(f"{model_cls.__name__} has no 'stream' method to patch")
 
-    if is_bad_id is None:
-        def is_bad_id(tool_use_id: Optional[str], tool_name: Optional[str]) -> bool:
-            # Treat missing/empty OR "id == tool name" as bad (your reported symptom)
-            if not tool_use_id:
-                return True
-            if tool_name and tool_use_id == tool_name:
-                return True
-            return False
-
+    # inline function supports unit tests
+    # marker: X - no toolUseId given, N - no tool_name given, E - toolUseId == tool_name, 'U' - unknown
     if id_factory is None:
-        id_factory = lambda: f"tooluse_{uuid4().hex}"
+        id_factory = lambda marker: f"tooluse_{marker or 'U'}-{uuid4().hex}"
 
     orig_stream = getattr(model_cls, "stream")
     setattr(model_cls, orig_attr, orig_stream)
@@ -82,24 +88,39 @@ def is_bad_id(tool_use_id: Optional[str], tool_name: Optional[str]) -> bool:
     async def stream_patched(self: Any, *args: Any, **kwargs: Any) -> AsyncIterator[dict]:
         state = _ToolUseIdStreamState()
 
+        def _patch_tool_use_id(event: dict[str, Any], where: str) -> None:
+            name = event.get("name")
+            tuid = event.get("toolUseId")
+            if not name and not tuid:
+                return
+            if not tuid:
+                marker = 'X'
+            elif not name:
+                marker = 'N'
+            elif tuid == name:
+                marker = 'E'
+            else:
+                return
+            if not name:
+                # in this case, toolUseId is the tool name, but no tool_name was given
+                event["name"] = tuid
+            tuid = state(where=where, marker=marker, id_factory=id_factory)
+            event["_toolUseId"] = event["toolUseId"] = tuid
+
         async for ev in orig_stream(self, *args, **kwargs):
+            # contentBlockStart and current_tool_use may come in any order, but we assume for a given provider the order is consistent
+
             # --- Pattern A: contentBlockStart -> toolUse ---
             cbs = ev.get("contentBlockStart")
             if isinstance(cbs, dict):
                 start = cbs.get("start")
                 if isinstance(start, dict):
                     tool_use = start.get("toolUse")
                     if isinstance(tool_use, dict):
-                        name = tool_use.get("name")
-                        tuid = tool_use.get("toolUseId")
-                        if is_bad_id(tuid, name):
-                            if not name:
-                                tool_use["name"] = tuid
-                            tuid = id_factory()
-                            tool_use["_toolUseId"] = tool_use["toolUseId"] = tuid
-                        state.current_tool_use_id = tuid
+                        _patch_tool_use_id(tool_use, "contentBlockStart")
 
             # --- Pattern B: contentBlockDelta -> toolUse (keep consistent) ---
+            # Assumption: contentBlockDelta do not overlap with concurrent tool uses
             cbd = ev.get("contentBlockDelta")
             if isinstance(cbd, dict):
                 delta = cbd.get("delta")
@@ -108,20 +129,13 @@ async def stream_patched(self: Any, *args: Any, **kwargs: Any) -> AsyncIterator[
                     if isinstance(dtu, dict):
                         name = dtu.get("name")
                         tuid = dtu.get("toolUseId")
-                        if (name or tuid) and is_bad_id(tuid, name) and state.current_tool_use_id:
+                        if (name or tuid) and state.current_tool_use_id:
                             dtu["_toolUseId"] = dtu["toolUseId"] = state.current_tool_use_id
 
             # --- Pattern C: Strands convenience field current_tool_use ---
             ctu = ev.get("current_tool_use")
             if isinstance(ctu, dict):
-                name = ctu.get("name")
-                tuid = ctu.get("toolUseId")
-                if is_bad_id(tuid, name):
-                    if not name:
-                        ctu["name"] = tuid
-                    tuid = state.current_tool_use_id or id_factory()
-                    ctu["_toolUseId"] = ctu["toolUseId"] = tuid
-                state.current_tool_use_id = tuid
+                _patch_tool_use_id(ctu, "current_tool_use")
 
             yield ev
 
@@ -154,15 +168,28 @@ def revert_tool_use_id(self, event: AfterToolCallEvent):
         tool_name = tool_use.get("name", "")
         tool_use_id = tool_use.get("toolUseId", "")
 
-        if tool_use_id.startswith("tooluse_") and tool_name:
-            # reverse the patch that set a generated ID because some models use toolUseId as the tool name !?!
+        tool_use_id_type = tool_use_id[:10]
+        if tool_use_id_type in ["tooluse_N-", "tooluse_X-", "tooluse_E-", "tooluse_U-"] and tool_name:
+            # reverse the patch that set a generated ID because some models use toolUseId as the tool name or no tool name at all !?!
+
+            reverted_tool_name = tool_name
+            if tool_use_id_type == "tooluse_E-":
+                reverted_tool_use_id = tool_name
+            elif tool_use_id_type == "tooluse_N-":
+                reverted_tool_use_id = tool_name
+                reverted_tool_name = ''
+            else:
+                reverted_tool_use_id = ''
+
             tool_use["_toolUseId"] = tool_use_id
-            tool_use["toolUseId"] = tool_name
+            tool_use["toolUseId"] = reverted_tool_use_id
+            tool_use["name"] = reverted_tool_name
 
             result = getattr(event, "result", None)
             if isinstance(result, dict) and "toolUseId" in result:
+                # there is no tool_name in "result"
                 result["_toolUseId"] = tool_use_id
-                result["toolUseId"] = tool_name
+                result["toolUseId"] = reverted_tool_use_id
 
 
 _OLLAMA_MODEL_TOKEN_USAGE_PATCH_ATTR = "_caa_ollama_usage_patch_v1"

src/modules/config/manager.py

@@ -166,7 +166,8 @@ def get_thinking_model_config(
             default_thinking_budget = 10000
 
         # Allow override via environment variables
-        max_tokens = self.getenv_int("MAX_TOKENS", default_max_tokens)
+        max_tokens_limit = self.getenv_int("MAX_TOKENS_REASONING_LIMIT", MAX_TOKENS_REASONING_LIMIT)
+        max_tokens = self.getenv_int("MAX_TOKENS", min(default_max_tokens, max_tokens_limit))
         thinking_budget = self.getenv_int("THINKING_BUDGET", default_thinking_budget)
 
         return {
@@ -692,6 +693,8 @@ def get_mem0_service_config(self, server: str, **overrides) -> Dict[str, Any]:
                     "azure_endpoint": self.getenv("AZURE_API_BASE"),
                     "api_version": self.getenv("AZURE_API_VERSION"),
                 }
+            elif mem0_provider == "ollama":
+                embedder_config["config"]["model"] = model_name
         elif server == "gemini":
             raise ValueError(f"Unsupported provider: {server}")
         elif server == "bedrock":
@@ -738,6 +741,8 @@ def get_mem0_service_config(self, server: str, **overrides) -> Dict[str, Any]:
                     "azure_endpoint": self.getenv("AZURE_API_BASE"),
                     "api_version": self.getenv("AZURE_API_VERSION"),
                 }
+            if mem0_llm_provider == "ollama":
+                llm_config["config"]["model"] = model_name
         elif server == "gemini":
             raise ValueError(f"Unsupported provider: {server}")
         elif server == "bedrock":

src/modules/config/providers/litellm_config.py

@@ -172,6 +172,8 @@ def align_litellm_defaults(
                     dims = 384
                 elif "titan" in embed_model and "v2" in embed_model:
                     dims = 1024
+                elif "mxbai-embed-large" in embed_model:
+                    dims = 1024
                 else:
                     dims = 1536
                     logger.warning(

src/modules/config/system/environment.py

@@ -73,6 +73,16 @@ def auto_setup(skip_mem0_cleanup: bool = False) -> List[str]:
     """Setup directories and discover available cyber tools"""
     # Disable Mem0 telemetry to prevent PostHog connection attempts
     os.environ.setdefault("MEM0_TELEMETRY", "false")
+    try:
+        import mem0.memory.telemetry as _mem0_telemetry
+        if hasattr(_mem0_telemetry, "MEM0_TELEMETRY"):
+            setattr(_mem0_telemetry, "MEM0_TELEMETRY", False)
+        if hasattr(_mem0_telemetry, "client_telemetry"):
+            client_telemetry = _mem0_telemetry.client_telemetry
+            if hasattr(client_telemetry, "posthog"):
+                client_telemetry.posthog.disabled = True
+    except Exception:
+        pass
 
     # Create necessary directories in proper locations
     try:

src/modules/config/system/environment.yaml

@@ -150,6 +150,7 @@ cyber_tools:
   # Basics
   netcat:
     description: "Network utility for reading/writing data"
+    command: "nc"
   curl:
     description: "HTTP client for web requests"
   tcpdump: