Direct control

Token costHigh

HarnessNone

ToolsNone

InputRGBProprioceptionLanguage

Raw action values

This method only measures models on their ability to autoregressively predict raw action values. Published results show that current models rarely finish a task this way.

In direct control, a multimodal LLM looks at the camera images and outputs the robot's actions itself. The arm runs those actions as the model writes them.

Example script

On each call, the model gets the task instruction, the robot's state, and the benchmark's color camera images.

LIBERORoboLab-120

# /// script
# requires-python = ">=3.10"
# dependencies = [
#     "manifold-sdk @ git+https://github.com/bifrostai/manifold-sdk",
#     "litellm>=1.60",
#     "tenacity",
#     "pillow>=10",
# ]
# ///
"""Serve a large language model that looks at the cameras and outputs the actions of the LIBERO robot."""

import argparse
import base64
import io
import json
import re
import time

import litellm
import numpy as np
from PIL import Image

import manifold
from manifold.benchmarks import LIBERO
from manifold.core.policy import PolicySignature
from manifold.core.sensor import CameraOrientation, Modality
from manifold.core.values import Action, Observation

PROMPT = """\
You control a Franka Panda arm in a simulator. Complete the task you are given.

You see two cameras: agentview faces the robot from the front of the table, and
wrist is mounted on the gripper. ee_pose is the gripper position x, y, z in
meters, its quaternion x, y, z, w, and the two finger positions (0.04 open, 0 closed).

Reply with the next {chunk} actions. Each action is 7 numbers between -1 and 1:
dx, dy, dz moves the gripper about 5 cm at 1 (+x away from the robot, +y to its
left, +z up); rx, ry, rz rotates it about 0.5 rad at 1; the last number opens
the gripper at -1 and closes it at 1.

Reply with JSON only: {{"thought": "<one sentence>", "actions": [[...], ...]}}
"""


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--model", required=True, help="LiteLLM model, or random")
    parser.add_argument("--base-url", help="endpoint of a self-hosted model")
    parser.add_argument("--chunk", type=int, default=10, help="actions per model call")
    parser.add_argument("--effort", help="reasoning effort, e.g. low")
    parser.add_argument("--header", action="append", default=[], help="extra request header, NAME=VALUE")
    args = parser.parse_args()

    # All LIBERO suites, such as libero-spatial, libero-plus and debug-libero, send this observation.
    signature = PolicySignature(
        action_space=LIBERO.embodiment.action,
        proprioception=LIBERO.embodiment.proprioception,
        # Color cameras only, upright: Manifold flips frames from a benchmark that sends them bottom row first.
        cameras=[
            c.model_copy(update={"orientation": CameraOrientation.UPRIGHT, "calibration": None})
            for c in LIBERO.sensors
            if c.modality == Modality.RGB
        ],
        chunk_size=args.chunk,
        execution_steps=args.chunk,
    )
    width = signature.action_space.expected_length()

    def predict(obs: Observation) -> Action:
        if args.model == "random":
            return Action.from_array(np.random.uniform(-1, 1, (args.chunk, width)))

        content = [{"type": "text", "text": f"Task: {obs.instruction}"}]
        for name, value in obs.state.items():
            content.append({"type": "text", "text": f"{name}: {[round(float(v), 3) for v in value]}"})
        for camera in signature.cameras:
            jpeg = io.BytesIO()
            Image.fromarray(obs.sensors[camera.name]).save(jpeg, format="JPEG")
            url = "data:image/jpeg;base64," + base64.b64encode(jpeg.getvalue()).decode()
            content.append({"type": "text", "text": f"{camera.name} camera:"})
            content.append({"type": "image_url", "image_url": {"url": url}})

        started = time.monotonic()
        response = litellm.completion(
            model=args.model,
            api_base=args.base_url,
            reasoning_effort=args.effort,
            extra_headers=dict(h.split("=", 1) for h in args.header),
            num_retries=3,
            max_tokens=4000,
            messages=[
                {"role": "system", "content": PROMPT.format(chunk=args.chunk)},
                {"role": "user", "content": content},
            ],
        )
        reply = response.choices[0].message.content or ""
        seconds, usage = time.monotonic() - started, response.usage

        try:
            parsed = json.loads(re.search(r"\{.*\}", reply, re.DOTALL)[0])
            actions = np.clip(np.array(parsed["actions"], dtype=float).reshape(args.chunk, width), -1, 1)
            print(f"{seconds:.1f} s, {usage.prompt_tokens} in, {usage.completion_tokens} out | {parsed.get('thought')}")
            return Action.from_array(actions)
        except Exception as error:
            # Zeros hold the arm still and leave the gripper as it is.
            print(f"{seconds:.1f} s | could not read the actions: {error}")
            return Action.from_array(np.zeros((args.chunk, width)))

    manifold.serve(predict, signature)


if __name__ == "__main__":
    main()

# /// script
# requires-python = ">=3.10"
# dependencies = [
#     "manifold-sdk @ git+https://github.com/bifrostai/manifold-sdk",
#     "litellm>=1.60",
#     "tenacity",
#     "pillow>=10",
# ]
# ///
"""Serve a large language model that looks at the cameras and outputs the actions of the RoboLab-120 robot."""

import argparse
import base64
import io
import json
import re
import time

import litellm
import numpy as np
from PIL import Image

import manifold
from manifold.benchmarks import ROBOLAB
from manifold.core.policy import PolicySignature
from manifold.core.sensor import Modality
from manifold.core.values import Action, Observation

PROMPT = """\
You control a Franka Panda arm with a Robotiq gripper in a simulator. Complete
the task you are given.

You see two cameras: over_shoulder_left looks at the table from behind and to the
left of the robot, and wrist is mounted on the gripper. joint_pos is the 7 joint
angles in radians, then the gripper (0 open, 1 closed). ee_pose is the gripper
position x, y, z in meters and its quaternion x, y, z, w.

Reply with the next {chunk} actions. Each action is 8 numbers: the 7 joint angles
in radians that the arm moves to, then the gripper, 0 to open it and 1 to close it.

Reply with JSON only: {{"thought": "<one sentence>", "actions": [[...], ...]}}
"""


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--model", required=True, help="LiteLLM model, or random")
    parser.add_argument("--base-url", help="endpoint of a self-hosted model")
    parser.add_argument("--chunk", type=int, default=10, help="actions per model call")
    parser.add_argument("--effort", help="reasoning effort, e.g. low")
    parser.add_argument("--header", action="append", default=[], help="extra request header, NAME=VALUE")
    args = parser.parse_args()

    # robolab-120 and debug-robolab-120 send this observation.
    signature = PolicySignature(
        action_space=ROBOLAB.embodiment.action,
        proprioception=ROBOLAB.embodiment.proprioception,
        # Color cameras only.
        cameras=[
            c.model_copy(update={"calibration": None})
            for c in ROBOLAB.sensors
            if c.modality == Modality.RGB
        ],
        chunk_size=args.chunk,
        execution_steps=args.chunk,
    )
    width = signature.action_space.expected_length()

    def predict(obs: Observation) -> Action:
        if args.model == "random":
            return Action.from_array(np.random.uniform(-1, 1, (args.chunk, width)))

        content = [{"type": "text", "text": f"Task: {obs.instruction}"}]
        for name, value in obs.state.items():
            content.append({"type": "text", "text": f"{name}: {[round(float(v), 3) for v in value]}"})
        for camera in signature.cameras:
            jpeg = io.BytesIO()
            Image.fromarray(obs.sensors[camera.name]).save(jpeg, format="JPEG")
            url = "data:image/jpeg;base64," + base64.b64encode(jpeg.getvalue()).decode()
            content.append({"type": "text", "text": f"{camera.name} camera:"})
            content.append({"type": "image_url", "image_url": {"url": url}})

        started = time.monotonic()
        response = litellm.completion(
            model=args.model,
            api_base=args.base_url,
            reasoning_effort=args.effort,
            extra_headers=dict(h.split("=", 1) for h in args.header),
            num_retries=3,
            max_tokens=4000,
            messages=[
                {"role": "system", "content": PROMPT.format(chunk=args.chunk)},
                {"role": "user", "content": content},
            ],
        )
        reply = response.choices[0].message.content or ""
        seconds, usage = time.monotonic() - started, response.usage

        try:
            parsed = json.loads(re.search(r"\{.*\}", reply, re.DOTALL)[0])
            actions = np.array(parsed["actions"], dtype=float).reshape(args.chunk, width)
            print(f"{seconds:.1f} s, {usage.prompt_tokens} in, {usage.completion_tokens} out | {parsed.get('thought')}")
            return Action.from_array(actions)
        except Exception as error:
            # The current joint angles and gripper hold the arm where it is.
            print(f"{seconds:.1f} s | could not read the actions: {error}")
            joints = np.asarray(obs.state["joint_pos"], dtype=float)
            hold = np.append(joints[:7], round(joints[7]))
            return Action.from_array(np.tile(hold, (args.chunk, 1)))

    manifold.serve(predict, signature)


if __name__ == "__main__":
    main()

Work	What the model outputs	Result
Large Language Models as General Pattern Machines, 2023	Low-level control values, as a sequence of numbers	Stabilizes CartPole
Prompt a Robot to Walk with Large Language Models, 2024	Target joint positions, from example steps in the prompt	GPT-4 walks the robot without fine-tuning
EmbodiedBench, 2025	Target end-effector poses in discrete bins, with object boxes and positions given	The best model averages 28.9%
Claude plays robotics, Anthropic, 2026	End-effector movements, as in this script	0 to 5.5% of tasks completed

Serve your policy

LLM as a Policy

Direct control

Example script

Further reading

Direct control ​

Example script ​

Further reading ​

Direct control

Example script

Further reading