diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3cbc9c72..fc500d1a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -184,7 +184,9 @@ jobs: - name: Pull the stock sandbox image for integration tests if: matrix.test-type != 'unit' # Sandboxes run this unmodified MCR image and are prepared on the fly. - run: docker pull mcr.microsoft.com/devcontainers/python:3.11 + run: | + docker pull mcr.microsoft.com/devcontainers/python:3.11 + docker pull mcr.microsoft.com/azurelinux/base/core:3.0 - name: Check disk space before ollama installation if: matrix.test-type == 'ollama_local' diff --git a/docs/advanced/sandbox-images.md b/docs/advanced/sandbox-images.md new file mode 100644 index 00000000..d71f64b2 --- /dev/null +++ b/docs/advanced/sandbox-images.md @@ -0,0 +1,47 @@ +# Sandbox Images + +Every bot runs its commands inside a Docker sandbox created by `LocalDockerEnvironment`. The sandbox runs an **unmodified image from the Microsoft Container Registry**, and you choose it by passing `image`. Two images have constants in `microbots.environment.local_docker.images`: + +| Constant | Image | Use it for | +|----------|-------|------------| +| `DEBIAN_IMAGE` (default) | `mcr.microsoft.com/devcontainers/python:3.11` | General work. Debian with Python and a full developer toolset. | +| `AZURELINUX_IMAGE` | `mcr.microsoft.com/azurelinux/base/core:3.0` | Azure Linux and RPM work. Its `tdnf` repos package `mock`, `rpm-build`, and `golang`. | + +## How a stock image becomes a sandbox + +There is no custom image to build. When an environment starts, Microbots: + +1. Starts the image idle. +2. Copies its shell server (`local_docker/shell_server`) into `/opt/microbots`, owned by root. +3. Runs `shell_server/prepare.sh` as root. It installs anything missing (Python, `useradd`, `mount`, `git`) with `apt-get`, `tdnf`, `dnf`, or `yum`. It also creates the unprivileged `agent` account, removes the devcontainer's passwordless-sudo `vscode` user, and installs the server's dependencies (FastAPI, uvicorn, pydantic) into a root-owned venv. +4. Starts the shell server, which drops to the `agent` account before running any bot command. + +Preparation needs network access for package and PyPI downloads. It takes about 10 seconds on `DEBIAN_IMAGE` and about 40 seconds on `AZURELINUX_IMAGE`, which also installs Python and the user tools. + +## Choose an image + +Pass `image` to any bot or to `LocalDockerEnvironment`: + +```python +from microbots import WritingBot +from microbots.environment.local_docker.images import AZURELINUX_IMAGE + +bot = WritingBot( + model="azure-openai/your-deployment", + folder_to_mount="/path/to/repo", + image=AZURELINUX_IMAGE, +) +``` + +```python +from microbots.environment.local_docker.LocalDockerEnvironment import LocalDockerEnvironment + +env = LocalDockerEnvironment(port=8080, image="mcr.microsoft.com/devcontainers/base:debian") +``` + +- Any image works if it has `sh` and `bash`, plus either `python3` or one of the supported package managers. Images that already include the prerequisites start faster. +- A bot's `image` only applies to the environment the bot creates. If you pass your own `environment`, choose the image when you create it; passing both raises `ValueError`. + +## Tools and distros + +You never declare the sandbox's distribution. Tools read it from the running container's `/etc/os-release` and pick their [distro-specific commands](tools/tesseract_ocr_tool_use.md#distro-specific-commands) from it. A tool that only defines `debian` commands, such as `cscope` or `browser-use`, fails to install on the Azure Linux image with `UnsupportedDistroError`. diff --git a/mkdocs.yml b/mkdocs.yml index 071a91b7..7454a879 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -78,6 +78,7 @@ nav: - Advanced: - Bots: - CopilotBot: advanced/bots/copilot-bot.md + - "Sandbox Images": advanced/sandbox-images.md - "Authentication Setup": advanced/authentication.md - "Azure Managed Identity & Service Connection Setup": advanced/azure-managed-identity-setup.md - "Azure Pipelines Log Analyzer": advanced/azure-pipelines-log-analyzer.md diff --git a/src/microbots/MicroBot.py b/src/microbots/MicroBot.py index 341291c1..c9743d4e 100644 --- a/src/microbots/MicroBot.py +++ b/src/microbots/MicroBot.py @@ -19,6 +19,7 @@ from microbots.llm.llm import ToolCall, llm_output_format_str from microbots.tools.native_tools import OPENAI_NATIVE_TOOLS, RUN_COMMAND_TOOL, run_file_tool from microbots.tools.tool import ToolAbstract, get_tool_from_call +from microbots.environment.local_docker.images import DEFAULT_IMAGE from microbots.extras.mount import Mount, MountType from microbots.utils.logger import LogLevelEmoji from microbots.utils.network import get_free_port @@ -154,6 +155,9 @@ class MicroBot: A callable that returns a bearer token for Azure AD authentication. If not provided, it may be auto-created from environment variables. Defaults to None. + image : Optional[str] + Shell-server image of the auto-created environment, for example + ``images.AZURELINUX_IMAGE``. Defaults to None (``DEFAULT_IMAGE``). """ def __init__( @@ -165,6 +169,7 @@ def __init__( additional_tools: Optional[list[ToolAbstract]] = None, folder_to_mount: Optional[Mount] = None, token_provider: Optional[any] = None, + image: Optional[str] = None, ): """ Init function for MicroBot class. @@ -196,8 +201,24 @@ def __init__( A callable that returns a bearer token for Azure AD authentication. If not provided, it may be auto-created from environment variables. Defaults to None. + image : Optional[str] + Shell-server image of the auto-created environment, for example + ``images.AZURELINUX_IMAGE``. Defaults to None (``DEFAULT_IMAGE``). + Cannot be combined with ``environment``. + + Raises + ------ + ValueError + If both ``image`` and ``environment`` are provided. """ + if image is not None and environment is not None: + raise ValueError( + "Pass either `image` or `environment`, not both: the image of a " + "provided environment is chosen when that environment is created." + ) + self.image = image or DEFAULT_IMAGE + self.folder_to_mount = folder_to_mount # TODO : Need to check on the purpose of variable `mounted` @@ -488,6 +509,7 @@ def _create_environment(self, folder_to_mount: Optional[Mount]): self.environment = LocalDockerEnvironment( port=free_port, folder_to_mount=folder_to_mount, + image=self.image, ) def _create_llm(self): diff --git a/src/microbots/bot/AgentBoss.py b/src/microbots/bot/AgentBoss.py index 8e780a22..73cfd66d 100644 --- a/src/microbots/bot/AgentBoss.py +++ b/src/microbots/bot/AgentBoss.py @@ -27,6 +27,9 @@ class AgentBoss(MicroBot): is created automatically when *None*. additional_tools : Optional[list[ToolAbstract]] Extra tools to install alongside the built-in ``microbot_sub`` tool. + image : Optional[str] + Shell-server image of the auto-created environment, for example + ``images.AZURELINUX_IMAGE``. Defaults to None (Debian image). Workflow -------- @@ -45,6 +48,7 @@ def __init__( folder_to_mount: str, environment: Optional[any] = None, additional_tools: Optional[list[ToolAbstract]] = None, + image: Optional[str] = None, ): """ Create the bot with the ``microbot_sub`` tool and the repository mounted read-write. @@ -60,6 +64,9 @@ def __init__( is created automatically when *None*. additional_tools : Optional[list[ToolAbstract]] Extra tools to install alongside the built-in ``microbot_sub`` tool. + image : Optional[str] + Shell-server image of the auto-created environment, for example + ``images.AZURELINUX_IMAGE``. Defaults to None (Debian image). """ if additional_tools is None: additional_tools = [] @@ -115,6 +122,7 @@ def __init__( environment=environment, additional_tools=additional_tools, folder_to_mount=folder_mount_info, + image=image, ) def run( diff --git a/src/microbots/bot/BrowsingBot.py b/src/microbots/bot/BrowsingBot.py index 24672e31..3dfc338f 100644 --- a/src/microbots/bot/BrowsingBot.py +++ b/src/microbots/bot/BrowsingBot.py @@ -11,6 +11,25 @@ class BrowsingBot(MicroBot): + """ + A bot that searches the web with the ``browser`` tool. + + Parameters + ---------- + model : str + The model to use, in the format ``/``. + environment : Optional[Environment] + The execution environment for the bot. If not provided, a default + environment is created. Defaults to None. + additional_tools : Optional[list[ToolAbstract]] + Extra tools to install alongside the browser tool. Defaults to None. + token_provider : Optional[any] + A callable that returns a bearer token for Azure AD authentication. + Defaults to None. + image : Optional[str] + Shell-server image of the auto-created environment. The browser tool + only supports Debian-based images (the default). Defaults to None. + """ def __init__( self, @@ -18,6 +37,7 @@ def __init__( environment: Optional[Environment] = None, additional_tools: Optional[list[ToolAbstract]] = None, token_provider: Optional[any] = None, + image: Optional[str] = None, ): # validate init values before assigning bot_type = BotType.BROWSING_BOT @@ -32,6 +52,7 @@ def __init__( environment=environment, additional_tools=(additional_tools or []) + [BROWSER_USE_TOOL], token_provider=token_provider, + image=image, ) def run(self, task, timeout_in_seconds=200) -> BotRunResult: diff --git a/src/microbots/bot/LogAnalysisBot.py b/src/microbots/bot/LogAnalysisBot.py index b4192d4d..559acd6f 100644 --- a/src/microbots/bot/LogAnalysisBot.py +++ b/src/microbots/bot/LogAnalysisBot.py @@ -43,6 +43,9 @@ class LogAnalysisBot(MicroBot): token_provider : Optional[any] An optional callable that supplies authentication tokens for the model provider. Defaults to None. + image : Optional[str] + Shell-server image of the auto-created environment, for example + ``images.AZURELINUX_IMAGE``. Defaults to None (Debian image). Attributes ---------- @@ -76,6 +79,7 @@ def __init__( environment: Optional[any] = None, additional_tools: Optional[list[ToolAbstract]] = None, token_provider: Optional[any] = None, + image: Optional[str] = None, ): """ Initialize a LogAnalysisBot instance. @@ -97,6 +101,9 @@ def __init__( token_provider : Optional[any] An optional callable that supplies authentication tokens for the model provider. Defaults to None. + image : Optional[str] + Shell-server image of the auto-created environment, for example + ``images.AZURELINUX_IMAGE``. Defaults to None (Debian image). """ # validate init values before assigning bot_type = BotType.LOG_ANALYSIS_BOT @@ -124,6 +131,7 @@ def __init__( additional_tools=additional_tools or [], folder_to_mount=folder_mount_info, token_provider=token_provider, + image=image, ) def run( diff --git a/src/microbots/bot/ReadingBot.py b/src/microbots/bot/ReadingBot.py index 0c8029be..f4c74bdc 100644 --- a/src/microbots/bot/ReadingBot.py +++ b/src/microbots/bot/ReadingBot.py @@ -28,6 +28,9 @@ class ReadingBot(MicroBot): token_provider : Optional[any] A callable that returns a bearer token for Azure AD authentication. Defaults to None. + image : Optional[str] + Shell-server image of the auto-created environment, for example + ``images.AZURELINUX_IMAGE``. Defaults to None (Debian image). """ def __init__( @@ -37,6 +40,7 @@ def __init__( environment: Optional[any] = None, additional_tools: Optional[list[ToolAbstract]] = None, token_provider: Optional[any] = None, + image: Optional[str] = None, ): """ Create the bot with the repository mounted read-only. @@ -55,6 +59,9 @@ def __init__( token_provider : Optional[any] A callable that returns a bearer token for Azure AD authentication. Defaults to None. + image : Optional[str] + Shell-server image of the auto-created environment, for example + ``images.AZURELINUX_IMAGE``. Defaults to None (Debian image). """ # validate init values before assigning bot_type = BotType.READING_BOT @@ -82,4 +89,5 @@ def __init__( additional_tools=additional_tools or [], folder_to_mount=folder_mount_info, token_provider=token_provider, + image=image, ) diff --git a/src/microbots/bot/WritingBot.py b/src/microbots/bot/WritingBot.py index 5e72add2..51124b2a 100644 --- a/src/microbots/bot/WritingBot.py +++ b/src/microbots/bot/WritingBot.py @@ -28,6 +28,9 @@ class WritingBot(MicroBot): token_provider : Optional[any] A callable that returns a bearer token for Azure AD authentication. Defaults to None. + image : Optional[str] + Shell-server image of the auto-created environment, for example + ``images.AZURELINUX_IMAGE``. Defaults to None (Debian image). """ def __init__( @@ -37,6 +40,7 @@ def __init__( environment: Optional[any] = None, additional_tools: Optional[list[ToolAbstract]] = None, token_provider: Optional[any] = None, + image: Optional[str] = None, ): """ Create the bot with the repository mounted read-write. @@ -55,6 +59,9 @@ def __init__( token_provider : Optional[any] A callable that returns a bearer token for Azure AD authentication. Defaults to None. + image : Optional[str] + Shell-server image of the auto-created environment, for example + ``images.AZURELINUX_IMAGE``. Defaults to None (Debian image). """ # validate init values before assigning bot_type = BotType.WRITING_BOT @@ -87,4 +94,5 @@ def __init__( additional_tools=additional_tools or [], folder_to_mount=folder_mount_info, token_provider=token_provider, + image=image, ) diff --git a/src/microbots/environment/local_docker/images.py b/src/microbots/environment/local_docker/images.py index 21c7ecef..a960e4b5 100644 --- a/src/microbots/environment/local_docker/images.py +++ b/src/microbots/environment/local_docker/images.py @@ -15,4 +15,8 @@ # Debian with Python and a full developer toolset (git, curl, build tools). DEBIAN_IMAGE = "mcr.microsoft.com/devcontainers/python:3.11" +# Azure Linux 3. Minimal: Python and the user tools are installed on the fly +# (with tdnf), and its repos package RPM tooling such as mock and rpm-build. +AZURELINUX_IMAGE = "mcr.microsoft.com/azurelinux/base/core:3.0" + DEFAULT_IMAGE = DEBIAN_IMAGE diff --git a/test/bot/test_microbot.py b/test/bot/test_microbot.py index 088b4438..60411ff1 100644 --- a/test/bot/test_microbot.py +++ b/test/bot/test_microbot.py @@ -1074,3 +1074,49 @@ def test_llm_finds_symbols_using_cscope(self, cscope_tool, c_code_repo, openai_m f"LLM should have found that main() calls add_numbers. Got: {response.result}" finally: del bot + + +@pytest.mark.unit +class TestMicroBotImage: + """image selects the shell-server image of the auto-created environment.""" + + @pytest.mark.parametrize("image, expected", [ + (None, "mcr.microsoft.com/devcontainers/python:3.11"), + ("mcr.microsoft.com/azurelinux/base/core:3.0", "mcr.microsoft.com/azurelinux/base/core:3.0"), + ]) + def test_forwarded_to_auto_created_environment(self, image, expected): + with patch("microbots.MicroBot.LocalDockerEnvironment") as lde, \ + patch("microbots.MicroBot.get_free_port", return_value=4242), \ + patch.object(MicroBot, "_create_llm"): + bot = MicroBot(model="azure-openai/test-deploy", image=image) + + assert bot.image == expected + lde.assert_called_once_with(port=4242, folder_to_mount=None, image=expected) + + def test_rejects_image_with_provided_environment(self): + with patch.object(MicroBot, "_create_llm"), \ + pytest.raises(ValueError, match="Pass either `image` or `environment`"): + MicroBot(model="azure-openai/test-deploy", environment=MagicMock(), image="custom/shell:dev") + + def test_provided_environment_is_used_as_is(self): + env = MagicMock() + with patch.object(MicroBot, "_create_environment") as create_env, \ + patch.object(MicroBot, "_create_llm"): + bot = MicroBot(model="azure-openai/test-deploy", environment=env) + assert bot.environment is env + create_env.assert_not_called() + + @pytest.mark.parametrize("bot_name, kwargs", [ + ("ReadingBot", {"folder_to_mount": "/tmp/repo"}), + ("WritingBot", {"folder_to_mount": "/tmp/repo"}), + ("LogAnalysisBot", {"folder_to_mount": "/tmp/repo"}), + ("BrowsingBot", {}), + ("AgentBoss", {"folder_to_mount": "/tmp/repo"}), + ]) + @pytest.mark.parametrize("image", [None, "custom/shell:dev"]) + def test_subclasses_forward_image(self, bot_name, kwargs, image): + import microbots.bot as bots + + with patch.object(MicroBot, "__init__", return_value=None) as base_init: + getattr(bots, bot_name)(model="azure-openai/test-deploy", image=image, **kwargs) + assert base_init.call_args.kwargs["image"] == image diff --git a/test/environment/local_docker/test_azurelinux_environment.py b/test/environment/local_docker/test_azurelinux_environment.py new file mode 100644 index 00000000..be6de355 --- /dev/null +++ b/test/environment/local_docker/test_azurelinux_environment.py @@ -0,0 +1,69 @@ +"""Integration tests for the stock Azure Linux image (AZURELINUX_IMAGE). + +Requires Docker and network access: the image has no Python or user tools, +so LocalDockerEnvironment installs them with tdnf when it starts. +""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../src"))) + +from microbots.constants import DOCKER_WORKING_DIR, PermissionLabels +from microbots.environment.local_docker.LocalDockerEnvironment import LocalDockerEnvironment +from microbots.environment.local_docker.images import AZURELINUX_IMAGE +from microbots.extras.mount import Mount +from microbots.tools.internal_tool import Tool +from microbots.tools.tool import UnsupportedDistroError +from microbots.tools.tool_yaml_parser import parse_tool_definition +from microbots.utils.network import get_free_port + + +@pytest.mark.integration +@pytest.mark.docker +class TestAzureLinuxEnvironment: + + @pytest.fixture + def repo(self, tmp_path): + repo = tmp_path / "repo" + repo.mkdir() + (repo / "hello.txt").write_text("hello\n") + return repo + + @pytest.mark.parametrize("permission", [PermissionLabels.READ_ONLY, PermissionLabels.READ_WRITE]) + def test_unprivileged_shell_with_mount(self, repo, permission): + env = LocalDockerEnvironment( + port=get_free_port(), + image=AZURELINUX_IMAGE, + folder_to_mount=Mount(str(repo), f"{DOCKER_WORKING_DIR}/repo", permission), + ) + try: + assert env.get_distro_ids() == ["azurelinux"] + ret = env.execute("id -un && pwd && cat hello.txt && echo written > new.txt") + assert ret.return_code == 0, ret.stderr + assert ret.stdout.split() == ["agent", f"{DOCKER_WORKING_DIR}/repo", "hello"] + assert env.execute("tdnf --version").return_code == 0 + finally: + env.stop() + # READ_ONLY writes stay in the overlay; READ_WRITE writes reach the host. + assert (repo / "new.txt").exists() is (permission == PermissionLabels.READ_WRITE) + + def test_distro_specific_tool_commands(self): + env = LocalDockerEnvironment(port=get_free_port(), image=AZURELINUX_IMAGE) + try: + tool = Tool( + name="jq", + description="jq", + usage_instructions_to_llm="jq", + install_commands={"azurelinux": ["tdnf install -y jq"], "debian": ["apt-get install -y jq"]}, + verify_commands=["jq --version"], + ) + tool.install_tool(env) + tool.verify_tool_installation(env) + + with pytest.raises(UnsupportedDistroError): + parse_tool_definition("cscope.yaml").install_tool(env) + finally: + env.stop() diff --git a/test/environment/local_docker/test_images.py b/test/environment/local_docker/test_images.py index d54bf589..1921916b 100644 --- a/test/environment/local_docker/test_images.py +++ b/test/environment/local_docker/test_images.py @@ -11,7 +11,7 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../src"))) from microbots.environment.local_docker import shell_server -from microbots.environment.local_docker.images import DEBIAN_IMAGE, DEFAULT_IMAGE +from microbots.environment.local_docker.images import AZURELINUX_IMAGE, DEBIAN_IMAGE, DEFAULT_IMAGE pytestmark = pytest.mark.unit @@ -23,6 +23,10 @@ def test_default_image_is_stock_mcr_debian(): assert DEBIAN_IMAGE.startswith("mcr.microsoft.com/") +def test_azurelinux_image_is_stock_mcr(): + assert AZURELINUX_IMAGE == "mcr.microsoft.com/azurelinux/base/core:3.0" + + def test_prepare_script_is_executable_posix_sh(): assert os.access(PREPARE, os.X_OK) assert PREPARE.read_text().startswith("#!/bin/sh\n")