openvibers/openvibe.link Commits

v0.36.0: the ai plugin passes tools and hands back the model's calls (#96)

OP OpenVibers committed 11 days ago · 640b59ba9b · 7 files, +166 −18

Contracts 0.277.0: an ai command may carry tools, tool_choice, tool
messages and an assistant message's tool_calls. The plugin (0.2.0) checks
each against the contract, passes them to the machine's own model server as
written, and answers with the calls the model asked for (an id made where
the server gave none; object arguments as JSON text). It never runs one.
Its capabilities say tools: true, so a caller sends tools only where they
can go.

Changes

ddiff --git a/STATUS.json b/STATUS.json
iindex 90ed865..870b89e 100644
---- a/STATUS.json
++++ b/STATUS.json
@@@ -5,8 +5,8 @@
"wave": "T15",
"plan": "Plan track T15 (Bot complete) and ADR-043 (Bot: devices, pairing, control and safety); the computer plugin and the job worker serve OpenVibe.Actor and the Codes job runner (tracks T14/T17)",
"kind": "client",
-- "version": "0.35.0",
-- "release": "v0.35.0: the Node applies the roles the network's machines hub gives it at each check-in (codes.worker starts the job worker it was installed with, desks.host and ai.host are checked against what the machine was installed for, an unknown role is ignored, and the local configuration always wins), and its status reports them back; a desk's first browser start in a cold sandbox gets 90 s. v0.34.0: on Windows a killed plugin takes every process it started with it (a Job Object with kill-on-close), as the process group does on Linux and macOS, so a bridge helper cannot keep a robot's link open after the Node stops or restarts its plugin; the Node reads what the coding tools remember (read-only, local, redacted) and `openvibe-node memory` uploads it to your own store; the heartbeat carries the machine's fleet of coding agents; the Codes job runner pin is v0.5.2",
++ "version": "0.36.0",
++ "release": "v0.36.0: the ai plugin (0.2.0) passes tools to the machine's own model server and hands back the calls the model asked for, never running one, so an agent on OpenVibe (Actor) can work on a model this machine serves; its capabilities say tools: true (contracts 0.277.0). v0.35.0: the Node applies the roles the network's machines hub gives it at each check-in (codes.worker starts the job worker it was installed with, desks.host and ai.host are checked against what the machine was installed for, an unknown role is ignored, and the local configuration always wins), and its status reports them back; a desk's first browser start in a cold sandbox gets 90 s. v0.34.0: on Windows a killed plugin takes every process it started with it (a Job Object with kill-on-close), as the process group does on Linux and macOS, so a bridge helper cannot keep a robot's link open after the Node stops or restarts its plugin; the Node reads what the coding tools remember (read-only, local, redacted) and `openvibe-node memory` uploads it to your own store; the heartbeat carries the machine's fleet of coding agents; the Codes job runner pin is v0.5.2",
"runtime": "go 1.24 (one static binary per platform); plugins in Python 3; an ESP32 Arduino library",
"database": "none",
"deployed": false,
ddiff --git a/plugins/ai/openvibe_ai/__init__.py b/plugins/ai/openvibe_ai/__init__.py
iindex 8789bb8..4990eae 100644
---- a/plugins/ai/openvibe_ai/__init__.py
++++ b/plugins/ai/openvibe_ai/__init__.py
@@@ -18,11 +18,17 @@ Config (the plugin's block in config.json):
* ``max_tokens``: the most a reply may be (default 2048, at most 32768).
* ``timeout_s``: how long one reply may take (default 180, at most 900).
--Command ``ai`` ``{"model", "messages": [{"role", "content"}], "max_tokens"?, "temperature"?}`` →
--``{model, message: {role: "assistant", content}, usage?, finish_reason?}``.
++Command ``ai`` ``{"model", "messages": [{"role", "content"}], "max_tokens"?, "temperature"?, "tools"?, "tool_choice"?}``
++→ ``{model, message: {role: "assistant", content, tool_calls?}, usage?, finish_reason?}``.
++
++Tools (contracts 0.277.0, plugin 0.2.0): the functions the model may ask for, passed to the model server as written.
++The plugin never runs one: it hands back the calls the model asked for, and the caller (an agent on OpenVibe) runs
++them and sends the next command with the assistant's message (its ``tool_calls``) and one ``tool`` message per call.
++``describe`` says ``tools: true`` so a caller knows this machine can.
"""
import json
++import re
import threading
import time
import urllib.error
@@@ -30,12 +36,81 @@ import urllib.request
from openvibe_plugin import Deferred, Fault, Plugin
--VERSION = "0.1.0"
++VERSION = "0.2.0"
DEFAULT_ENDPOINTS = ("http://127.0.0.1:11434", "http://127.0.0.1:1234", "http://127.0.0.1:8080")
MAX_REPLY_CHARS = 200 * 1024 # well inside the core's answer limit (512 KB)
MAX_MESSAGES = 200
MAX_CONTENT = 64 * 1024
--ROLES = ("system", "user", "assistant")
++ROLES = ("system", "user", "assistant", "tool")
++MAX_TOOLS = 64
++MAX_CALLS = 32
++MAX_TOOLS_BYTES = 256 * 1024
++NAME = re.compile(r"^[A-Za-z0-9_.-]{1,64}$")
++
++
++def _call(c):
++ """One tool call as the contract has it, or None."""
++ if not isinstance(c, dict) or c.get("type", "function") != "function":
++ return None
++ fn = c.get("function") if isinstance(c.get("function"), dict) else {}
++ name, args, cid = fn.get("name"), fn.get("arguments"), c.get("id")
++ if not isinstance(name, str) or not NAME.match(name):
++ return None
++ if isinstance(args, (dict, list)):
++ args = json.dumps(args) # Ollama may give the arguments as an object
++ if args is None:
++ args = "{}"
++ if not isinstance(args, str) or len(args) > MAX_CONTENT:
++ return None
++ if cid is not None and (not isinstance(cid, str) or not cid or len(cid) > 200):
++ return None
++ return {"id": cid, "type": "function", "function": {"name": name, "arguments": args}}
++
++
++def _message(m):
++ """A message as the model server gets it, or None when it breaks the contract."""
++ if not isinstance(m, dict) or m.get("role") not in ROLES or not isinstance(m.get("content"), str) or len(m["content"]) > MAX_CONTENT:
++ return None
++ out = {"role": m["role"], "content": m["content"]}
++ calls, call_id = m.get("tool_calls"), m.get("tool_call_id")
++ if calls is not None:
++ if m["role"] != "assistant" or not isinstance(calls, list) or not 0 < len(calls) <= MAX_CALLS:
++ return None
++ made = [_call(c) for c in calls]
++ if any(c is None or c["id"] is None for c in made):
++ return None
++ out["tool_calls"] = made
++ if m["role"] == "tool":
++ if not isinstance(call_id, str) or not call_id or len(call_id) > 200:
++ return None
++ out["tool_call_id"] = call_id
++ elif call_id is not None:
++ return None
++ return out
++
++
++def _tools(tools):
++ """The tools as the model server gets them, or None when they break the contract."""
++ if not isinstance(tools, list) or not 0 < len(tools) <= MAX_TOOLS:
++ return None
++ out = []
++ for t in tools:
++ fn = t.get("function") if isinstance(t, dict) else None
++ if not isinstance(fn, dict) or t.get("type") != "function" or not isinstance(fn.get("name"), str) or not NAME.match(fn["name"]):
++ return None
++ one = {"name": fn["name"]}
++ if fn.get("description") is not None:
++ if not isinstance(fn["description"], str) or len(fn["description"]) > 4096:
++ return None
++ one["description"] = fn["description"]
++ if fn.get("parameters") is not None:
++ if not isinstance(fn["parameters"], dict):
++ return None
++ one["parameters"] = fn["parameters"]
++ out.append({"type": "function", "function": one})
++ if len(json.dumps(out)) > MAX_TOOLS_BYTES:
++ return None
++ return out
def _int(v, name, lo, hi, default):
@@@ -112,7 +187,7 @@ class Ai(Plugin):
except Fault:
return {}
return {"ai": {"models": self.models, "ready": self.reachable and bool(self.models),
-- "max_tokens": self.max_tokens}}
++ "max_tokens": self.max_tokens, "tools": True}}
def setup(self, config):
self._configure(config)
@@@ -128,10 +203,17 @@ class Ai(Plugin):
messages = v.get("messages")
if not isinstance(model, str) or not model or len(model) > 200:
raise Fault("bad_value", "model is the name of a model this machine offers")
-- if not isinstance(messages, list) or not messages or len(messages) > MAX_MESSAGES or not all(
-- isinstance(m, dict) and m.get("role") in ROLES and isinstance(m.get("content"), str)
-- and len(m["content"]) <= MAX_CONTENT for m in messages):
-- raise Fault("bad_value", "messages are 1 to %d of {role: system|user|assistant, content: text}" % MAX_MESSAGES)
++ sent = [_message(m) for m in messages] if isinstance(messages, list) and 0 < len(messages) <= MAX_MESSAGES else None
++ if not sent or any(m is None for m in sent):
++ raise Fault("bad_value", "messages are 1 to %d of {role: system|user|assistant|tool, content: text}; tool_calls on an assistant message, tool_call_id on a tool one" % MAX_MESSAGES)
++ tools = None
++ if v.get("tools") is not None:
++ tools = _tools(v.get("tools"))
++ if tools is None:
++ raise Fault("bad_value", "tools are 1 to %d of {type: function, function: {name, description?, parameters?}}" % MAX_TOOLS)
++ tool_choice = v.get("tool_choice")
++ if tool_choice is not None and tool_choice not in ("auto", "none"):
++ raise Fault("bad_value", "tool_choice is auto or none")
if not self.reachable:
self._configure_again()
if not self.reachable:
@@@ -146,10 +228,12 @@ class Ai(Plugin):
raise Fault("bad_value", "temperature is 0 to 2")
if not self.busy.acquire(blocking=False):
raise Fault("not_ready", "the model is answering another request on this machine: try again in a moment")
-- body = {"model": model, "messages": [{"role": m["role"], "content": m["content"]} for m in messages],
-- "max_tokens": max_tokens, "stream": False}
++ body = {"model": model, "messages": sent, "max_tokens": max_tokens, "stream": False}
if temperature is not None:
body["temperature"] = temperature
++ if tools:
++ body["tools"] = tools
++ body["tool_choice"] = tool_choice or "auto"
threading.Thread(target=self._reply, args=(cmd.id, body), name="ai-reply", daemon=True).start()
raise Deferred()
@@@ -188,11 +272,22 @@ class Ai(Plugin):
choice = (doc.get("choices") or [{}])[0] if isinstance(doc, dict) else {}
msg = choice.get("message") if isinstance(choice, dict) else None
content = msg.get("content") if isinstance(msg, dict) else None
++ calls = []
++ if isinstance(msg, dict) and isinstance(msg.get("tool_calls"), list) and body.get("tools"):
++ for i, c in enumerate(msg["tool_calls"][:MAX_CALLS]):
++ made = _call(c)
++ if made is not None:
++ made["id"] = made["id"] or "call_%d" % (i + 1) # some servers give no id
++ calls.append(made)
if not isinstance(content, str):
-- return self.fail(cid, "hardware", "the model server gave no reply")
++ if not calls:
++ return self.fail(cid, "hardware", "the model server gave no reply")
++ content = ""
out = {"model": str(doc.get("model") or body["model"]),
"message": {"role": "assistant", "content": content[:MAX_REPLY_CHARS]},
"ms": int((self.clock() - started) * 1000)}
++ if calls:
++ out["message"]["tool_calls"] = calls
if isinstance(choice.get("finish_reason"), str):
out["finish_reason"] = choice["finish_reason"]
usage = doc.get("usage")
ddiff --git a/plugins/ai/pyproject.toml b/plugins/ai/pyproject.toml
iindex 664b26d..c573700 100644
---- a/plugins/ai/pyproject.toml
++++ b/plugins/ai/pyproject.toml
@@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "openvibe-ai"
--version = "0.1.0"
++version = "0.2.0"
description = "OpenVibe Node ai plugin: your own models (Ollama, llama.cpp, LM Studio) for your own OpenVibe work, relayed from OpenVibe with no open port"
requires-python = ">=3.9"
license = { text = "AGPL-3.0-or-later" }
ddiff --git a/plugins/ai/tests/test_ai.py b/plugins/ai/tests/test_ai.py
iindex 3d7b29c..c3944f6 100644
---- a/plugins/ai/tests/test_ai.py
++++ b/plugins/ai/tests/test_ai.py
@@@ -46,6 +46,12 @@ class FakeServer(http.server.BaseHTTPRequestHandler):
return self._send(404, {"error": "not found"})
if body.get("model") == "broken":
return self._send(500, {"error": "out of memory"})
++ if body.get("tools") and body["messages"][-1]["role"] != "tool":
++ # The model asks for tools: one call with an id and JSON text, one (as Ollama may) with neither.
++ calls = [{"id": "call_x", "type": "function", "function": {"name": body["tools"][0]["function"]["name"], "arguments": "{\"q\": \"Lisbon\"}"}},
++ {"type": "function", "function": {"name": "clock", "arguments": {"tz": "UTC"}}}]
++ return self._send(200, {"model": body["model"], "choices": [{"index": 0, "finish_reason": "tool_calls",
++ "message": {"role": "assistant", "content": None, "tool_calls": calls}}]})
last = body["messages"][-1]["content"]
self._send(200, {"model": body["model"], "choices": [{"index": 0, "finish_reason": "stop",
"message": {"role": "assistant", "content": "You said: " + last}}],
@@@ -110,6 +116,53 @@ class AiPluginTest(unittest.TestCase):
self.assertEqual(path, "/v1/chat/completions")
self.assertEqual((body["max_tokens"], body["temperature"], body["stream"]), (50, 0.2, False))
++ def test_tools_go_to_the_model_and_its_calls_come_back_never_run(self):
++ caps = self.boot()[0]["capabilities"]["ai"]
++ self.assertIs(caps["tools"], True, "the machine says it can pass tools")
++ tools = [{"type": "function", "function": {"name": "web_search", "description": "Search the web",
++ "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}}},
++ {"type": "function", "function": {"name": "clock"}}]
++ first = [{"role": "user", "content": "Weather in Lisbon?"}]
++ r = self.ask({"model": "qwen2.5:7b", "messages": first, "tools": tools})
++ self.assertEqual(r["op"], "ack", r)
++ msg = r["data"]["message"]
++ self.assertEqual(msg["content"], "", "a reply that only asks for tools has empty text")
++ self.assertEqual(msg["tool_calls"], [
++ {"id": "call_x", "type": "function", "function": {"name": "web_search", "arguments": '{"q": "Lisbon"}'}},
++ {"id": "call_2", "type": "function", "function": {"name": "clock", "arguments": '{"tz": "UTC"}'}},
++ ], "an id is made where the server gave none, and object arguments become JSON text")
++ self.assertEqual(r["data"]["finish_reason"], "tool_calls")
++ body = FakeServer.seen[-1][1]
++ self.assertEqual((body["tools"], body["tool_choice"]), (tools, "auto"), "the tools reach the model server as written")
++ # The caller ran them; the next command carries the calls and their results.
++ second = first + [msg, {"role": "tool", "tool_call_id": "call_x", "content": "19 C, sunny"},
++ {"role": "tool", "tool_call_id": "call_2", "content": "15:40"}]
++ r = self.ask({"model": "qwen2.5:7b", "messages": second, "tools": tools, "tool_choice": "auto"})
++ self.assertEqual(r["data"]["message"], {"role": "assistant", "content": "You said: 15:40"})
++ sent = FakeServer.seen[-1][1]["messages"]
++ self.assertEqual(sent[1]["tool_calls"][0]["id"], "call_x")
++ self.assertEqual((sent[2]["role"], sent[2]["tool_call_id"]), ("tool", "call_x"))
++ # Without tools, a reply's tool calls are never passed on (nothing asked for them).
++ r = self.ask({"model": "qwen2.5:7b", "messages": [{"role": "user", "content": "plain"}]})
++ self.assertNotIn("tool_calls", r["data"]["message"])
++ self.assertNotIn("tools", FakeServer.seen[-1][1])
++
++ def test_tool_messages_that_break_the_contract_are_refused(self):
++ self.boot()
++ bad = [
++ {"messages": [{"role": "user", "content": "x"}, {"role": "tool", "content": "orphan"}]},
++ {"messages": [{"role": "user", "content": "x", "tool_call_id": "c1"}]},
++ {"messages": [{"role": "user", "content": "x", "tool_calls": [{"id": "c", "type": "function", "function": {"name": "a", "arguments": "{}"}}]}]},
++ {"messages": [{"role": "assistant", "content": "", "tool_calls": [{"type": "function", "function": {"name": "a", "arguments": "{}"}}]}]},
++ {"messages": [{"role": "user", "content": "x"}], "tools": [{"type": "function", "function": {"name": "rm -rf"}}]},
++ {"messages": [{"role": "user", "content": "x"}], "tools": []},
++ {"messages": [{"role": "user", "content": "x"}], "tools": [{"type": "function", "function": {"name": "a"}}], "tool_choice": "required"},
++ ]
++ for v in bad:
++ r = self.ask({"model": "qwen2.5:7b", **v})
++ self.assertEqual((r["op"], r.get("fault_code")), ("nack", "bad_value"), v)
++ self.assertEqual(FakeServer.seen, [], "nothing reached the model server")
++
def test_it_says_how_fast_each_model_answers_here(self):
self.boot()
self.assertEqual(self.p.poll(), {"sensors": {"ai_busy": 0}}, "nothing measured before the first answer")
ddiff --git a/plugins/sdk/openvibe_plugin/__init__.py b/plugins/sdk/openvibe_plugin/__init__.py
iindex 82e139a..7448cd7 100644
---- a/plugins/sdk/openvibe_plugin/__init__.py
++++ b/plugins/sdk/openvibe_plugin/__init__.py
@@@ -24,7 +24,7 @@ __all__ = ["Plugin", "Fault", "Deferred", "Command", "StopInfo", "Runtime", "run
"RUNTIME_CAUSES", "SHARED_CLOCK", "SHARED_CLOCK_NAME", "TIMING_UNAVAILABLE", "__version__"]
# The Node release this runtime ships with (pyproject.toml carries the same).
--__version__ = "0.35.0"
++__version__ = "0.36.0"
PROTOCOL_VERSION = 1
HEARTBEAT_TIMEOUT_S = 1.0
DEFAULT_DEADLINE_MS = 300
ddiff --git a/plugins/sdk/pyproject.toml b/plugins/sdk/pyproject.toml
iindex 10c0aed..ad286d8 100644
---- a/plugins/sdk/pyproject.toml
++++ b/plugins/sdk/pyproject.toml
@@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "openvibe-plugin"
--version = "0.35.0"
++version = "0.36.0"
description = "Runtime for OpenVibe Node plugins (JSON lines over stdin/stdout, deadlines, heartbeat, e-stop)"
requires-python = ">=3.9"
license = { text = "AGPL-3.0-or-later" }
ddiff --git a/plugins/sdk/tests/test_runtime.py b/plugins/sdk/tests/test_runtime.py
iindex 9f2d1a7..5f2a9f1 100644
---- a/plugins/sdk/tests/test_runtime.py
++++ b/plugins/sdk/tests/test_runtime.py
@@@ -439,7 +439,7 @@ class RuntimeTest(unittest.TestCase):
self.assertEqual(rt.stop_infos[-1].halt_epoch, 1792900000500)
def test_the_package_reports_the_node_release(self):
-- self.assertEqual(openvibe_plugin.__version__, "0.35.0")
++ self.assertEqual(openvibe_plugin.__version__, "0.36.0")
with open(os.path.join(os.path.dirname(HERE), "pyproject.toml")) as f:
self.assertIn('version = "%s"' % openvibe_plugin.__version__, f.read())