Use OpenAI computer-use helpers with E2B Desktop

npm i openai @e2b/desktop

pip install openai e2b-desktop

import base64
import time

from e2b_desktop import Sandbox
from openai import OpenAI
from openai.lib.computer import ComputerToolset, ComputerUseError

MODEL = "gpt-6-luna"
# OpenAI key names that e2b-desktop does not map on its own
KEYS = {"ARROWUP": "up", "ARROWDOWN": "down", "ARROWLEFT": "left", "ARROWRIGHT": "right",
        "PAGEUP": "page_up", "PAGEDOWN": "page_down"}


class E2BDesktopDriver:
    """Runs each action the model asks for on an E2B desktop."""

    def __init__(self, desktop: Sandbox):
        self.desktop = desktop

    def screenshot(self):
        png = self.desktop.screenshot()
        return {"type": "computer_screenshot",
                "image_url": "data:image/png;base64," + base64.b64encode(png).decode()}

    def click(self, action):
        no_held_keys(action)
        if action.button == "left":
            self.desktop.left_click(action.x, action.y)
        elif action.button == "right":
            self.desktop.right_click(action.x, action.y)
        elif action.button == "wheel":
            self.desktop.middle_click(action.x, action.y)
        else:
            raise ComputerUseError(f"Unsupported button: {action.button}")

    def double_click(self, action):
        no_held_keys(action)
        self.desktop.double_click(action.x, action.y)

    def drag(self, action):
        no_held_keys(action)
        start, *rest = action.path
        self.desktop.move_mouse(start.x, start.y)
        self.desktop.mouse_press()
        for point in rest:
            self.desktop.move_mouse(point.x, point.y)
        self.desktop.mouse_release()

    def move(self, action):
        self.desktop.move_mouse(action.x, action.y)

    def scroll(self, action):
        no_held_keys(action)
        if action.scroll_x:
            raise ComputerUseError("Horizontal scroll is not supported")
        self.desktop.move_mouse(action.x, action.y)
        # The model sends pixels; one wheel step scrolls about 100
        steps = max(1, abs(action.scroll_y) // 100)
        self.desktop.scroll("down" if action.scroll_y > 0 else "up", steps)

    def keypress(self, action):
        self.desktop.press([KEYS.get(key.upper(), key) for key in action.keys])

    def type(self, action):
        self.desktop.write(action.text)

    def wait(self, action):
        time.sleep(1)


def no_held_keys(action):
    if getattr(action, "keys", None):
        raise ComputerUseError("Holding keys during mouse actions is not supported")


def approve(action):
    print("action:", action.type)
    return True  # approve every action; put your own policy here


desktop = Sandbox.create(resolution=(1280, 800), timeout=300)
try:
    desktop.stream.start(require_auth=True)
    auth_key = desktop.stream.get_auth_key()
    print("Live view:", desktop.stream.get_url(auth_key=auth_key, view_only=True))

    desktop.files.write("/home/user/orders/priority-order.txt", "Order number: PO-7731\nPriority: high\n")
    desktop.files.write("/home/user/orders/standard-order.txt", "Order number: PO-1001\nPriority: standard\n")
    desktop.open("/home/user/orders")
    desktop.wait(3000)

    with OpenAI() as client:
        response = ComputerToolset(E2BDesktopDriver(desktop)).run(
            client,
            {
                "model": MODEL,
                "store": False,
                "tools": [{"type": "computer"}],
                "input": "Open priority-order.txt and reply with only its order number.",
            },
            max_steps=15,
            before_action=approve,
        )
    print("Answer:", response.output_text)
finally:
    desktop.kill()

import { Template, defaultBuildLogger } from 'e2b'

const template = Template()
  .fromTemplate('desktop')
  .setUser('root')
  .runCmd(['curl -fsSL https://deb.nodesource.com/setup_22.x | bash -', 'apt-get install -y nodejs'])
  .setUser('user')
  .setWorkdir('/home/user')
  // playwright-core only: Playwright drives the desktop's own Chrome over CDP
  .runCmd('mkdir -p /home/user/.exec_js && cd /home/user/.exec_js && npm install playwright-core@1.64.0')

await Template.build(template, 'openai-computer-use-exec-js', {
  cpuCount: 8,
  memoryMB: 8192,
  onBuildLogs: defaultBuildLogger(),
})

from e2b import Template, default_build_logger

# python-xlib replaces the python3-Xlib fork PyAutoGUI asks for: the desktop's
# X server listens only on an abstract socket, which python3-Xlib cannot reach.
PACKAGES = [
    "pyautogui==0.9.54", "mouseinfo==0.1.3", "pygetwindow==0.0.9", "pymsgbox==2.0.1",
    "pyperclip==1.11.0", "pyrect==0.2.0", "pyscreeze==1.0.1", "pytweening==1.2.0",
    "python-xlib==0.33", "six==1.17.0", "pillow==11.3.0",
]

template = (
    Template()
    .from_template("desktop")
    .set_user("root")
    .apt_install(["python3-tk"])  # importing PyAutoGUI exits without tkinter
    .run_cmd("pip install --no-deps " + " ".join(PACKAGES))
    .set_user("user")
    .set_workdir("/home/user")
)

Template.build(template, "openai-computer-use-exec-py", cpu_count=8, memory_mb=8192,
               on_build_logs=default_build_logger())