openvibers/openvibe.link Commits
Memory: the Node uploads what your tools remember to your own store (#89)
Changes
@@@ -33,6 +33,7 @@ openvibe-node quit # end the node cleanly (nothing latched), e.g. from a sopenvibe-node approvals # the computer plugin at level ask: what waits; approve <id> / deny <id>openvibe-node tray # the same in your notification area, with Approve/Deny notifications (Linux, Windows)openvibe-node memory scan # what your coding tools remember: read-only, local, secrets removed++openvibe-node memory upload # send what they remember to your own memory store, after you say yes```## What is in here@@@ -72,7 +73,8 @@ task on openvibe.codes/run from any browser and have this machine do it ([docs/wDocs: [protocol](docs/protocol.md) (the wire format with OpenVibe.Bot), [plugins](docs/plugins.md) (the driverprotocol, the bundled drivers, and where Actor's computer-control plugin attaches), [install](docs/install.md),--[memory](docs/memory.md) (what the coding tools on a machine remember, and how the Node reads it).++[memory](docs/memory.md) (what the coding tools on a machine remember, how the Node reads it and how it is uploaded to++your own memory store).Decisions: [ADR-043](docs/ADR-043-bot-devices-and-control.md).## Development@@@ -108,7 +108,7 @@ func runWithStdin(args []string, stdin io.Reader, stdout, stderr io.Writer) intcmd, rest := args[0], args[1:]if cmd == "memory" {// The memory scan has its own flags, so it parses its own arguments.-- return cmdMemory(rest, stdout, stderr)++ return cmdMemory(rest, stdin, stdout, stderr)}if cmd == "desk-sandbox" { // its own flags, up to "--": a browser's flags after it are never the Node'sreturn cmdDeskSandbox(rest, stderr)@@@ -5,8 +5,6 @@ import ("flag""fmt""io"-- "os"-- "path/filepath""sort""strings"@@@ -15,16 +13,25 @@ import (// memoryUsage is what "openvibe-node memory" prints.const memoryUsage = "openvibe-node memory scan [--tools a,b] [--project DIR] [--json]\n" +++ "openvibe-node memory upload [--tools a,b] [--project DIR …] [--dry-run] [--yes]\n" +++ "openvibe-node memory status [--show]\n" +"\n" +-- " Reads the memory files of the coding tools on this machine and prints what it\n" +-- " found. It is read-only and local: nothing is written and nothing is sent\n" +-- " anywhere. Secrets are removed before anything is printed.\n" +++ " scan reads the memory files of the coding tools on this machine and\n" +++ " prints what it found. It is read-only and local: nothing is written\n" +++ " and nothing is sent anywhere. Secrets are removed before anything is\n" +++ " printed.\n" +++ " upload sends the items to your own memory store (OpenVibe.AI). It shows\n" +++ " what would leave the machine and asks before it sends anything.\n" +++ " status shows what your own memory store holds.\n" +"\n" +" --tools a,b only these adapters (claude-code, agents, gemini, cursor,\n" +" copilot, windsurf, cline, roo, kiro, continue, aider,\n" +" command-code, opencode)\n" +" --project DIR read this project as well; may be given more than once\n" +-- " --json print the items as JSON\n" +++ " --json print the items as JSON (scan)\n" +++ " --dry-run print the summary and send nothing (upload)\n" +++ " --yes do not ask before sending (upload)\n" +++ " --show print the items themselves (status)\n" +" --user-home DIR read this directory as the person's home (default: yours)\n"// stringList is a flag that may be given more than once.@@@ -43,8 +50,18 @@ func (s *stringList) Set(v string) error {}// cmdMemory runs the memory commands.--func cmdMemory(args []string, stdout, stderr io.Writer) int {-- if len(args) == 0 || args[0] != "scan" {++func cmdMemory(args []string, stdin io.Reader, stdout, stderr io.Writer) int {++ if len(args) == 0 {++ fmt.Fprint(stderr, memoryUsage)++ return 2++ }++ switch args[0] {++ case "upload":++ return cmdMemoryUpload(args[1:], stdin, stdout, stderr)++ case "status":++ return cmdMemoryStatus(args[1:], stdout, stderr)++ case "scan":++ default:fmt.Fprint(stderr, memoryUsage)return 2}@@@ -65,22 +82,12 @@ func cmdMemory(args []string, stdout, stderr io.Writer) int {fmt.Fprint(stderr, memoryUsage)return 2}-- home := *userHome-- if home == "" {-- h, err := os.UserHomeDir()-- if err != nil {-- fmt.Fprintln(stderr, "error: cannot find your home directory; pass --user-home")-- return 1-- }-- home = h-- }-- abs, err := filepath.Abs(home)-- if err != nil {-- fmt.Fprintln(stderr, "error:", err)-- return 1++ home, code := memoryHome(*userHome, stderr)++ if code != 0 {++ return code}res := memory.Scan(memory.Options{-- Home: abs,++ Home: home,Projects: projects,Tools: tools,})@@@ -44,7 +44,7 @@ func memoryFixture(t *testing.T) string {func TestMemoryScanPrintsEveryAdapter(t *testing.T) {home := memoryFixture(t)var stdout, stderr bytes.Buffer-- code := cmdMemory([]string{"scan", "--user-home", home}, &stdout, &stderr)++ code := cmdMemory([]string{"scan", "--user-home", home}, strings.NewReader(""), &stdout, &stderr)if code != 0 {t.Fatalf("the scan failed with %d: %s", code, stderr.String())}@@@ -79,7 +79,7 @@ func TestMemoryScanPrintsEveryAdapter(t *testing.T) {func TestMemoryScanJSON(t *testing.T) {home := memoryFixture(t)var stdout, stderr bytes.Buffer-- code := cmdMemory([]string{"scan", "--user-home", home, "--json"}, &stdout, &stderr)++ code := cmdMemory([]string{"scan", "--user-home", home, "--json"}, strings.NewReader(""), &stdout, &stderr)if code != 0 {t.Fatalf("the scan failed with %d: %s", code, stderr.String())}@@@ -112,7 +112,7 @@ func TestMemoryScanJSON(t *testing.T) {func TestMemoryScanToolsFilter(t *testing.T) {home := memoryFixture(t)var stdout, stderr bytes.Buffer-- code := cmdMemory([]string{"scan", "--user-home", home, "--tools", "cursor", "--json"}, &stdout, &stderr)++ code := cmdMemory([]string{"scan", "--user-home", home, "--tools", "cursor", "--json"}, strings.NewReader(""), &stdout, &stderr)if code != 0 {t.Fatalf("the scan failed with %d: %s", code, stderr.String())}@@@ -124,7 +124,7 @@ func TestMemoryScanToolsFilter(t *testing.T) {// TestMemoryScanUsage checks that a wrong use says how to use it.func TestMemoryScanUsage(t *testing.T) {var stdout, stderr bytes.Buffer-- if code := cmdMemory([]string{"list"}, &stdout, &stderr); code != 2 {++ if code := cmdMemory([]string{"list"}, strings.NewReader(""), &stdout, &stderr); code != 2 {t.Errorf("the command returned %d, want 2", code)}if !strings.Contains(stderr.String(), "memory scan") {@@@ -137,7 +137,7 @@ func TestMemoryScanIsReadOnly(t *testing.T) {home := memoryFixture(t)before := fixtureState(t, home)var stdout, stderr bytes.Buffer-- if code := cmdMemory([]string{"scan", "--user-home", home}, &stdout, &stderr); code != 0 {++ if code := cmdMemory([]string{"scan", "--user-home", home}, strings.NewReader(""), &stdout, &stderr); code != 0 {t.Fatalf("the scan failed with %d: %s", code, stderr.String())}after := fixtureState(t, home)nnew file mode 100644@@@ -0,0 +1,277 @@++package main++++import (++ "bufio"++ "context"++ "flag"++ "fmt"++ "io"++ "os"++ "path/filepath"++ "sort"++ "strings"++++ "github.com/OpenVibers/OpenVibe.Node/internal/memory"++ "github.com/OpenVibers/OpenVibe.Node/internal/memorystore"++ "github.com/OpenVibers/OpenVibe.Node/internal/memoryupload"++)++++// memoryUploadUsage is what "openvibe-node memory upload" prints.++const memoryUploadUsage = "openvibe-node memory upload [--tools a,b] [--project DIR …] [--dry-run] [--yes]\n" +++ "\n" +++ " Reads the memory files of the coding tools on this machine and sends the\n" +++ " items to your own memory store (OpenVibe.AI). It shows what it found and\n" +++ " what would leave the machine first, and asks before it sends anything.\n" +++ "\n" +++ " --tools a,b only these adapters\n" +++ " --project DIR read this project as well; may be given more than once\n" +++ " --dry-run print the summary and send nothing\n" +++ " --yes do not ask before sending\n" +++ " --endpoint URL the store (default " + memorystore.DefaultBaseURL + ")\n" +++ " --token-env NAME the variable the access token is read from (default " + memorystore.DefaultTokenEnv + ")\n" +++ " --user-home DIR read this directory as the person's home (default: yours)\n"++++// memoryStatusUsage is what "openvibe-node memory status" prints.++const memoryStatusUsage = "openvibe-node memory status [--show]\n" +++ "\n" +++ " Shows what your own memory store holds: how many items, per source and per\n" +++ " scope. The text of an item is only printed with --show.\n" +++ "\n" +++ " --show print the items themselves\n" +++ " --endpoint URL the store (default " + memorystore.DefaultBaseURL + ")\n" +++ " --token-env NAME the variable the access token is read from (default " + memorystore.DefaultTokenEnv + ")\n"++++// cmdMemoryUpload sends the items of the coding tools to the person's own++// memory store.++func cmdMemoryUpload(args []string, stdin io.Reader, stdout, stderr io.Writer) int {++ fs := flag.NewFlagSet("memory upload", flag.ContinueOnError)++ fs.SetOutput(stderr)++ fs.Usage = func() { fmt.Fprint(stderr, memoryUploadUsage) }++ var tools stringList++ var projects stringList++ fs.Var(&tools, "tools", "only these adapters")++ fs.Var(&projects, "project", "read this project as well")++ dryRun := fs.Bool("dry-run", false, "print the summary and send nothing")++ assumeYes := fs.Bool("yes", false, "do not ask before sending")++ endpoint := fs.String("endpoint", memorystore.DefaultBaseURL, "the memory store")++ tokenEnv := fs.String("token-env", memorystore.DefaultTokenEnv, "the variable the access token is read from")++ userHome := fs.String("user-home", "", "read this directory as the person's home")++ if err := fs.Parse(args); err != nil {++ return 2++ }++ if fs.NArg() > 0 {++ fmt.Fprintf(stderr, "unknown argument %q\n\n", fs.Arg(0))++ fmt.Fprint(stderr, memoryUploadUsage)++ return 2++ }++ home, code := memoryHome(*userHome, stderr)++ if code != 0 {++ return code++ }++ res := memory.Scan(memory.Options{Home: home, Projects: projects, Tools: tools})++ plan := memoryupload.Build(res)++++ printUploadSummary(plan, res, stdout)++ if *dryRun {++ fmt.Fprintln(stdout, "\nNothing was sent: this was a dry run.")++ return 0++ }++ if len(plan.Items) == 0 {++ fmt.Fprintln(stdout, "\nNothing to send.")++ return 0++ }++ if !*assumeYes && !confirmed(stdin, stdout, len(plan.Items)) {++ fmt.Fprintln(stdout, "\nNothing was sent.")++ return 0++ }++ token := strings.TrimSpace(os.Getenv(*tokenEnv))++ if token == "" {++ fmt.Fprintf(stderr, "error: %s is not set: it must hold your OpenVibe access token\n", *tokenEnv)++ return 1++ }++ client, err := memorystore.New(memorystore.Options{BaseURL: *endpoint, Token: token})++ if err != nil {++ fmt.Fprintln(stderr, "error:", err)++ return 1++ }++ res2, err := client.Upload(context.Background(), plan.Items)++ if err != nil {++ fmt.Fprintln(stderr, "error:", err)++ return 1++ }++ fmt.Fprintf(stdout, "\nSent: %d new, %d already there, %d refused.\n", res2.Created, res2.Deduplicated, res2.Refused)++ return 0++}++++// cmdMemoryStatus shows what the person's own memory store holds.++func cmdMemoryStatus(args []string, stdout, stderr io.Writer) int {++ fs := flag.NewFlagSet("memory status", flag.ContinueOnError)++ fs.SetOutput(stderr)++ fs.Usage = func() { fmt.Fprint(stderr, memoryStatusUsage) }++ show := fs.Bool("show", false, "print the items themselves")++ endpoint := fs.String("endpoint", memorystore.DefaultBaseURL, "the memory store")++ tokenEnv := fs.String("token-env", memorystore.DefaultTokenEnv, "the variable the access token is read from")++ if err := fs.Parse(args); err != nil {++ return 2++ }++ if fs.NArg() > 0 {++ fmt.Fprintf(stderr, "unknown argument %q\n\n", fs.Arg(0))++ fmt.Fprint(stderr, memoryStatusUsage)++ return 2++ }++ token := strings.TrimSpace(os.Getenv(*tokenEnv))++ if token == "" {++ fmt.Fprintf(stderr, "error: %s is not set: it must hold your OpenVibe access token\n", *tokenEnv)++ return 1++ }++ client, err := memorystore.New(memorystore.Options{BaseURL: *endpoint, Token: token})++ if err != nil {++ fmt.Fprintln(stderr, "error:", err)++ return 1++ }++ res, err := client.List(context.Background(), "", "")++ if err != nil {++ fmt.Fprintln(stderr, "error:", err)++ return 1++ }++ printMemoryStatus(res, *show, stdout)++ return 0++}++++// memoryHome gives the home directory to read, and the code to return when++// there is none.++func memoryHome(userHome string, stderr io.Writer) (string, int) {++ home := userHome++ if home == "" {++ h, err := os.UserHomeDir()++ if err != nil {++ fmt.Fprintln(stderr, "error: cannot find your home directory; pass --user-home")++ return "", 1++ }++ home = h++ }++ abs, err := filepath.Abs(home)++ if err != nil {++ fmt.Fprintln(stderr, "error:", err)++ return "", 1++ }++ return abs, 0++}++++// confirmed asks the person whether the items may be sent. Anything but a++// yes means no.++func confirmed(stdin io.Reader, stdout io.Writer, items int) bool {++ fmt.Fprintf(stdout, "\nSend these %d items to your memory store? [y/N] ", items)++ reader := bufio.NewReader(stdin)++ line, err := reader.ReadString('\n')++ if err != nil && line == "" {++ fmt.Fprintln(stdout)++ return false++ }++ answer := strings.ToLower(strings.TrimSpace(line))++ return answer == "y" || answer == "yes"++}++++// printUploadSummary shows what the scan found and what would leave the++// machine: every tool, every file, the items it gave, what was redacted and++// the scopes the items go to.++func printUploadSummary(plan memoryupload.Plan, res memory.Result, stdout io.Writer) {++ fmt.Fprintf(stdout, "Memory upload of %s\n", res.Home)++ if len(res.Projects) == 0 {++ fmt.Fprintln(stdout, "No project given: only the user-level files were read.")++ } else {++ fmt.Fprintf(stdout, "Projects: %s\n", strings.Join(res.Projects, ", "))++ }++ for _, ts := range plan.Summary.Tools {++ fmt.Fprintf(stdout, "\n%s (%s) — %d items\n", ts.Name, ts.ID, ts.Items)++ if ts.Items == 0 {++ // The tool was looked at and gave nothing: its paths are noise here.++ continue++ }++ for _, f := range ts.Files {++ switch {++ case f.Note != "":++ fmt.Fprintf(stdout, " - %s: %s\n", relativePath(f.Path, res), f.Note)++ case f.Found:++ fmt.Fprintf(stdout, " - %s: %d items\n", relativePath(f.Path, res), f.Items)++ default:++ fmt.Fprintf(stdout, " - %s: not there\n", relativePath(f.Path, res))++ }++ }++ if ts.Redacted > 0 {++ fmt.Fprintf(stdout, " %d items redacted (%d secrets removed)\n", ts.Redacted, ts.Redactions)++ }++ }++ fmt.Fprintf(stdout, "\nScopes: %s\n", strings.Join(plan.Summary.Scopes, ", "))++ fmt.Fprintf(stdout, "%d items to send", plan.Summary.Items)++ if plan.Summary.Redacted > 0 {++ fmt.Fprintf(stdout, ", %d redacted (%d secrets removed)", plan.Summary.Redacted, plan.Summary.Redactions)++ }++ if plan.Summary.Skipped > 0 {++ fmt.Fprintf(stdout, ", %d skipped (empty)", plan.Summary.Skipped)++ }++ fmt.Fprintln(stdout, ".")++}++++// relativePath gives a path as the summary shows it: relative to the folder++// it was scanned from, so the summary holds no absolute path.++func relativePath(path string, res memory.Result) string {++ root := res.Home++ for _, p := range res.Projects {++ if path == p || strings.HasPrefix(path, p+string(filepath.Separator)) {++ root = p++ break++ }++ }++ if root == "" {++ return path++ }++ rel, err := filepath.Rel(root, path)++ if err != nil || strings.HasPrefix(rel, "..") {++ return path++ }++ return filepath.ToSlash(rel)++}++++// printMemoryStatus shows what the store holds: the counts per source and per++// scope, and the items themselves only when they are asked for.++func printMemoryStatus(res memorystore.ListResult, show bool, stdout io.Writer) {++ bySource := map[string]int{}++ byScope := map[string]int{}++ for _, it := range res.Items {++ for _, s := range strings.Split(it.Source, ",") {++ if s = strings.TrimSpace(s); s != "" {++ bySource[s]++++ }++ }++ byScope[it.Scope]++++ }++ fmt.Fprintf(stdout, "Your memory store holds %d items.\n", res.Count)++ fmt.Fprintln(stdout, "\nPer source:")++ for _, s := range sortedKeys(bySource) {++ fmt.Fprintf(stdout, " %s: %d\n", s, bySource[s])++ }++ fmt.Fprintln(stdout, "\nPer scope:")++ for _, s := range sortedKeys(byScope) {++ fmt.Fprintf(stdout, " %s: %d\n", s, byScope[s])++ }++ if !show {++ fmt.Fprintln(stdout, "\nThe text of the items is not shown; use --show to read them.")++ return++ }++ fmt.Fprintln(stdout, "\nThe items:")++ for _, it := range res.Items {++ fmt.Fprintf(stdout, " - [%s] %s (%s)\n", it.Scope, it.Kind, it.Source)++ fmt.Fprintf(stdout, " %s\n", strings.TrimSpace(it.Text))++ }++}++++// sortedKeys gives the keys of a count map, sorted.++func sortedKeys(counts map[string]int) []string {++ out := make([]string, 0, len(counts))++ for k := range counts {++ out = append(out, k)++ }++ sort.Strings(out)++ return out++}nnew file mode 100644@@@ -0,0 +1,306 @@++package main++++import (++ "bytes"++ "encoding/json"++ "io"++ "net/http"++ "net/http/httptest"++ "os"++ "os/user"++ "path/filepath"++ "strings"++ "sync"++ "testing"++)++++// memoryStore is a fake memory store that records what it is sent.++type memoryStore struct {++ mu sync.Mutex++ requests [][]byte++ paths []string++ answer string++}++++// newMemoryStore starts a fake store.++func newMemoryStore(t *testing.T) (*memoryStore, *httptest.Server) {++ t.Helper()++ ms := &memoryStore{answer: `{"items":[],"count":0,"created":0,"deduped":0}`}++ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {++ raw, _ := io.ReadAll(r.Body)++ ms.mu.Lock()++ ms.requests = append(ms.requests, raw)++ ms.paths = append(ms.paths, r.URL.Path)++ answer := ms.answer++ ms.mu.Unlock()++ w.Header().Set("Content-Type", "application/json")++ _, _ = w.Write([]byte(answer))++ }))++ t.Cleanup(srv.Close)++ return ms, srv++}++++// sent gives the bodies the store received.++func (ms *memoryStore) sent() [][]byte {++ ms.mu.Lock()++ defer ms.mu.Unlock()++ return append([][]byte(nil), ms.requests...)++}++++// uploadFixture is the fixture home of the memory package and its project.++func uploadFixture(t *testing.T) (string, string) {++ t.Helper()++ wd, err := os.Getwd()++ if err != nil {++ t.Fatal(err)++ }++ root := filepath.Join(wd, "..", "..", "internal", "memory", "testdata", "claude-code")++ return filepath.Join(root, "home"), filepath.Join(root, "project")++}++++// TestMemoryUploadDryRunSendsNothing checks that a dry run shows the summary++// and sends nothing.++func TestMemoryUploadDryRunSendsNothing(t *testing.T) {++ home, project := uploadFixture(t)++ ms, srv := newMemoryStore(t)++ t.Setenv("OPENVIBE_TOKEN", "the-person-token")++ var stdout, stderr bytes.Buffer++ code := cmdMemory([]string{"upload", "--user-home", home, "--project", project, "--dry-run", "--endpoint", srv.URL},++ strings.NewReader(""), &stdout, &stderr)++ if code != 0 {++ t.Fatalf("the upload failed with %d: %s", code, stderr.String())++ }++ if len(ms.sent()) != 0 {++ t.Errorf("the store got %d requests, want none", len(ms.sent()))++ }++ out := stdout.String()++ for _, want := range []string{++ "Claude Code (claude-code)",++ "items to send",++ "Scopes:",++ "project:project",++ "Nothing was sent",++ } {++ if !strings.Contains(out, want) {++ t.Errorf("the output has no %q: %s", want, out)++ }++ }++}++++// TestMemoryUploadAsksBeforeSending checks that nothing is sent until the++// person says yes.++func TestMemoryUploadAsksBeforeSending(t *testing.T) {++ home, project := uploadFixture(t)++ ms, srv := newMemoryStore(t)++ t.Setenv("OPENVIBE_TOKEN", "the-person-token")++ var stdout, stderr bytes.Buffer++ code := cmdMemory([]string{"upload", "--user-home", home, "--project", project, "--endpoint", srv.URL},++ strings.NewReader("n\n"), &stdout, &stderr)++ if code != 0 {++ t.Fatalf("the upload failed with %d: %s", code, stderr.String())++ }++ if len(ms.sent()) != 0 {++ t.Errorf("the store got %d requests after a no, want none", len(ms.sent()))++ }++ if !strings.Contains(stdout.String(), "Nothing was sent") {++ t.Errorf("the output does not say that nothing was sent: %s", stdout.String())++ }++}++++// TestMemoryUploadSendsWhenThePersonSaysYes checks that a yes sends the items.++func TestMemoryUploadSendsWhenThePersonSaysYes(t *testing.T) {++ home, project := uploadFixture(t)++ ms, srv := newMemoryStore(t)++ t.Setenv("OPENVIBE_TOKEN", "the-person-token")++ var stdout, stderr bytes.Buffer++ code := cmdMemory([]string{"upload", "--user-home", home, "--project", project, "--endpoint", srv.URL},++ strings.NewReader("yes\n"), &stdout, &stderr)++ if code != 0 {++ t.Fatalf("the upload failed with %d: %s", code, stderr.String())++ }++ if len(ms.sent()) != 1 {++ t.Fatalf("the store got %d requests, want 1", len(ms.sent()))++ }++ if !strings.Contains(stdout.String(), "Sent:") {++ t.Errorf("the output does not say what was sent: %s", stdout.String())++ }++}++++// TestMemoryUploadYesSkipsTheQuestion checks that --yes sends without asking.++func TestMemoryUploadYesSkipsTheQuestion(t *testing.T) {++ home, project := uploadFixture(t)++ ms, srv := newMemoryStore(t)++ t.Setenv("OPENVIBE_TOKEN", "the-person-token")++ var stdout, stderr bytes.Buffer++ code := cmdMemory([]string{"upload", "--user-home", home, "--project", project, "--yes", "--endpoint", srv.URL},++ strings.NewReader(""), &stdout, &stderr)++ if code != 0 {++ t.Fatalf("the upload failed with %d: %s", code, stderr.String())++ }++ if len(ms.sent()) != 1 {++ t.Errorf("the store got %d requests, want 1", len(ms.sent()))++ }++ if strings.Contains(stdout.String(), "[y/N]") {++ t.Errorf("the output asked a question it did not need to ask: %s", stdout.String())++ }++}++++// TestMemoryUploadSendsNothingOfTheMachine checks the bodies that leave: no++// absolute path, no hostname, no name of the person.++func TestMemoryUploadSendsNothingOfTheMachine(t *testing.T) {++ home, project := uploadFixture(t)++ ms, srv := newMemoryStore(t)++ t.Setenv("OPENVIBE_TOKEN", "the-person-token")++ var stdout, stderr bytes.Buffer++ code := cmdMemory([]string{"upload", "--user-home", home, "--project", project, "--yes", "--endpoint", srv.URL},++ strings.NewReader(""), &stdout, &stderr)++ if code != 0 {++ t.Fatalf("the upload failed with %d: %s", code, stderr.String())++ }++ hostname, err := os.Hostname()++ if err != nil {++ hostname = ""++ }++ userName := ""++ if u, err := user.Current(); err == nil {++ userName = u.Username++ }++ for _, body := range ms.sent() {++ text := string(body)++ for _, secret := range []string{home, project, hostname, userName, os.Getenv("HOME")} {++ if secret == "" {++ continue++ }++ if strings.Contains(text, secret) {++ t.Errorf("the request holds %q, which is a fact about the machine: %s", secret, text)++ }++ }++ if strings.Contains(text, "/home/") {++ t.Errorf("the request holds an absolute path: %s", text)++ }++ }++}++++// TestMemoryUploadSendsTheItemsOfTheScan checks what the request holds: the++// text, the kind, the scope, the adapter and the place the item was written.++func TestMemoryUploadSendsTheItemsOfTheScan(t *testing.T) {++ home, project := uploadFixture(t)++ ms, srv := newMemoryStore(t)++ t.Setenv("OPENVIBE_TOKEN", "the-person-token")++ var stdout, stderr bytes.Buffer++ code := cmdMemory([]string{"upload", "--user-home", home, "--project", project, "--yes", "--endpoint", srv.URL},++ strings.NewReader(""), &stdout, &stderr)++ if code != 0 {++ t.Fatalf("the upload failed with %d: %s", code, stderr.String())++ }++ bodies := ms.sent()++ if len(bodies) != 1 {++ t.Fatalf("the store got %d requests, want 1", len(bodies))++ }++ var sent struct {++ Items []struct {++ Scope string `json:"scope"`++ Kind string `json:"kind"`++ Text string `json:"text"`++ Source string `json:"source"`++ SourceRef struct {++ Path string `json:"path"`++ Heading string `json:"heading"`++ } `json:"source_ref"`++ } `json:"items"`++ }++ if err := json.Unmarshal(bodies[0], &sent); err != nil {++ t.Fatalf("the request is not JSON: %v", err)++ }++ if len(sent.Items) == 0 {++ t.Fatal("the request holds no items")++ }++ scopes := map[string]bool{}++ for _, it := range sent.Items {++ scopes[it.Scope] = true++ if it.Text == "" {++ t.Error("an item has no text")++ }++ if it.Source != "claude-code" {++ t.Errorf("an item has the source %q, want the adapter id", it.Source)++ }++ if it.Kind == "" {++ t.Error("an item has no kind")++ }++ if strings.HasPrefix(it.SourceRef.Path, "/") {++ t.Errorf("an item has the absolute path %q", it.SourceRef.Path)++ }++ }++ if !scopes["user"] {++ t.Error("the request holds no item of the person")++ }++ if !scopes["project:project"] {++ t.Errorf("the request holds no item of the project: %v", scopes)++ }++}++++// TestMemoryUploadNeedsAToken checks that the command says which variable is++// missing.++func TestMemoryUploadNeedsAToken(t *testing.T) {++ home, _ := uploadFixture(t)++ _, srv := newMemoryStore(t)++ t.Setenv("OPENVIBE_TOKEN", "")++ var stdout, stderr bytes.Buffer++ code := cmdMemory([]string{"upload", "--user-home", home, "--yes", "--endpoint", srv.URL},++ strings.NewReader(""), &stdout, &stderr)++ if code != 1 {++ t.Fatalf("the upload returned %d, want 1", code)++ }++ if !strings.Contains(stderr.String(), "OPENVIBE_TOKEN") {++ t.Errorf("the error does not name the variable: %s", stderr.String())++ }++}++++// TestMemoryStatusCountsWithoutTheText checks what the status shows: the++// counts per source and per scope, and no text unless it is asked for.++func TestMemoryStatusCountsWithoutTheText(t *testing.T) {++ ms, srv := newMemoryStore(t)++ ms.answer = `{"items":[{"id":"mem_1","scope":"user","kind":"fact","text":"the secret of the day","source":"claude-code","status":"active"},++ {"id":"mem_2","scope":"project:my-project","kind":"fact","text":"another one","source":"cursor","status":"active"}],"count":2}`++ t.Setenv("OPENVIBE_TOKEN", "the-person-token")++ var stdout, stderr bytes.Buffer++ code := cmdMemory([]string{"status", "--endpoint", srv.URL}, strings.NewReader(""), &stdout, &stderr)++ if code != 0 {++ t.Fatalf("the status failed with %d: %s", code, stderr.String())++ }++ out := stdout.String()++ for _, want := range []string{"2 items", "claude-code: 1", "cursor: 1", "user: 1", "project:my-project: 1"} {++ if !strings.Contains(out, want) {++ t.Errorf("the output has no %q: %s", want, out)++ }++ }++ if strings.Contains(out, "the secret of the day") {++ t.Errorf("the output shows the text without --show: %s", out)++ }++}++++// TestMemoryStatusShowsTheTextWithShow checks that --show prints the items.++func TestMemoryStatusShowsTheTextWithShow(t *testing.T) {++ ms, srv := newMemoryStore(t)++ ms.answer = `{"items":[{"id":"mem_1","scope":"user","kind":"fact","text":"the secret of the day","source":"claude-code","status":"active"}],"count":1}`++ t.Setenv("OPENVIBE_TOKEN", "the-person-token")++ var stdout, stderr bytes.Buffer++ code := cmdMemory([]string{"status", "--show", "--endpoint", srv.URL}, strings.NewReader(""), &stdout, &stderr)++ if code != 0 {++ t.Fatalf("the status failed with %d: %s", code, stderr.String())++ }++ if !strings.Contains(stdout.String(), "the secret of the day") {++ t.Errorf("the output does not show the item: %s", stdout.String())++ }++}++++// TestMemoryUploadUsage checks that a wrong use says how to use it.++func TestMemoryUploadUsage(t *testing.T) {++ var stdout, stderr bytes.Buffer++ if code := cmdMemory([]string{"upload", "--nope"}, strings.NewReader(""), &stdout, &stderr); code != 2 {++ t.Errorf("the command returned %d, want 2", code)++ }++ if !strings.Contains(stderr.String(), "memory upload") {++ t.Errorf("the error does not say how to use it: %s", stderr.String())++ }++}@@@ -1,7 +1,7 @@# Memory: the Node reads what your tools rememberThe Node reads the memory files of the coding tools on your machine and tells--you what it found. It is **read-only, local and redacted**:++you what it found. The scan is **read-only, local and redacted**:- it opens the files it knows about and nothing else;- it never writes a file and never opens a network connection;@@@ -9,27 +9,39 @@ you what it found. It is **read-only, local and redacted**:- it never follows a symlink out of the home folder or the project folders itwas given.--This page is stage M1 of OpenVibe Memory: the readers. The store, the--encryption, the upload and the page come later.++The upload sends what it found to **your own memory store** (OpenVibe.AI),++after it has shown you what would leave the machine and after you have said++yes. The store is yours: it is sealed, only your own token opens it, and++nothing but the items themselves is sent.--## The command++This page is stage M1 of OpenVibe Memory (the readers) and stage M2a (the++upload). The page that shows your memory comes later.++++## The commands```shopenvibe-node memory scan [--tools a,b] [--project DIR …] [--json]++openvibe-node memory upload [--tools a,b] [--project DIR …] [--dry-run] [--yes]++openvibe-node memory status [--show]```| Flag | What it does ||---|---|| `--tools a,b` | only these adapters (default: all) || `--project DIR` | read this project as well; may be given more than once |--| `--json` | print the items as JSON |++| `--json` | print the items as JSON (scan) |++| `--dry-run` | print the summary and send nothing (upload) |++| `--yes` | do not ask before sending (upload) |++| `--show` | print the items themselves (status) |++| `--endpoint URL` | the memory store (default `https://ai.openvibe.services`) |++| `--token-env NAME` | the variable the access token is read from (default `OPENVIBE_TOKEN`) || `--user-home DIR` | read this directory as the person's home (default: yours) |Without `--project` only the user-level files are read: the files that holdyour own preferences across every project. With `--project` the project filesare read too, and their items carry the scope `project:<absolute path>`.--The command prints every adapter, every path it looked at and whether it was++The scan prints every adapter, every path it looked at and whether it wasthere, and how many items each file gave:```@@@ -153,6 +165,112 @@ different shape, and the test proves that none of them reaches an item. TheClaude Code fixture holds more: a key block split over two items, a key blockwith no END line, and an `.env` block.++## The upload++++```sh++openvibe-node memory upload [--tools a,b] [--project DIR …] [--dry-run] [--yes]++```++++The upload runs the same scan as `memory scan` — the same adapters, the same++paths, the same redaction — and sends what it found to **your own memory++store** (OpenVibe.AI).++++The token is your own OpenVibe access token, the same one the OpenVibe.Codes++CLI uses. It is read from the environment (`OPENVIBE_TOKEN`, or the variable++`--token-env` names), never from the config file, and it never appears in a++log line, in an error or in a URL.++++Nothing is sent before you have seen the summary and said yes. `--dry-run`++never sends anything at all:++++```++Memory upload of /home/you++Projects: /srv/work/demo++++Claude Code (claude-code) — 20 items++ - .claude/CLAUDE.md: 3 items++ - .claude/rules/sandbox.md: 6 items++ - CLAUDE.md: 3 items++ 2 items redacted (3 secrets removed)++++Scopes: project:demo, user++20 items to send, 2 redacted (3 secrets removed).++++Send these 20 items to your memory store? [y/N]++```++++### What leaves the machine++++| Field | What it is |++|---|---|++| `text` | the item, already redacted |++| `kind` | `preference`, `fact`, `project_rule`, `procedure`, `decision` or `lesson` |++| `scope` | `user`, or `project:<slug>`: the slug is the project's folder name, lowercased and slugged |++| `source` | the id of the adapter the item came from (`claude-code`, …) |++| `source_ref` | the path of the file relative to the folder it was scanned from, and the heading it was under |++| `confidence` | `1`: it is your own file |++++What a coding tool wrote is a suggestion until you accept it, so the items are++marked as written by a tool when they arrive at the store.++++### What never leaves the machine++++- **the absolute path of any file**: the path that is sent is relative to the++ folder that was scanned (`.claude/CLAUDE.md`, not `/home/you/.claude/CLAUDE.md`);++- **the name of the machine and the name of the person**;++- **the value of a secret**: the redaction runs before the item is counted,++ and the store scrubs again everything it receives;++- **an item whose text is empty** after the redaction;++- the files themselves, the settings of a tool and the chat history of a tool:++ the scan never reads them.++++### The store++++The items go to `POST https://ai.openvibe.services/api/v1/memory/items:batch`++in batches of 200, with your token as a Bearer. A store that is busy or broken++is tried again with a wait. The store scrubs the text again, refuses an item++that is nothing but a secret, and counts what it did:++++```++Sent: 18 new, 2 already there, 0 refused.++```++++An item the store would not take is counted as refused and the rest of the++batch is sent again: nothing is dropped without being said.++++## The status++++```sh++openvibe-node memory status [--show]++```++++It shows what your own store holds: how many items there are, per source and++per scope. The text of an item is only printed with `--show`.++++```++Your memory store holds 42 items.++++Per source:++ claude-code: 30++ cursor: 12++++Per scope:++ project:demo: 17++ user: 25++```++++## Deleting everything from one source++++The store has no soft delete: a delete removes the item. To remove everything++that came from one adapter, ask the store to delete that source:++++```sh++curl -X DELETE "https://ai.openvibe.services/api/v1/memory/items?source=claude-code" \++ -H "Authorization: Bearer $OPENVIBE_TOKEN"++```++++That removes every item of that source, for you only.++## The JSON```json@@@ -215,3 +333,16 @@ A file that was looked at but could not be used carries a `note`:- a file over 256 KB and a binary file are skipped;- the package does not import `net/http`: it never opens a networkconnection.++++`internal/memoryupload` and `internal/memorystore` have the tests of the++upload:++++- the scope of a project is the slug of its folder name, never an absolute++ path;++- the path that is sent is relative to the folder that was scanned;++- an item that is empty after the redaction never leaves the machine;++- a dry run sends nothing, and nothing is sent before the person says yes;++- the items go in batches of 200, and a busy store is tried again;++- the request that leaves holds no absolute path, no hostname and no name of++ the person;++- the token is never in an error message.nnew file mode 100644@@@ -0,0 +1,396 @@++// Package memorystore is the client of the person's own memory store: the++// OpenVibe.AI module behind /api/v1/memory.++//++// The store is scoped to one person: the person's own Network access token++// (the audience openvibe.network) is the only thing that opens it, and every++// query the server runs is scoped by the owner. This client sends the person's++// token as a Bearer and nothing else about the machine: the text of an item,++// its kind, its scope, the adapter it came from and where it was written.++//++// The token is a secret: it is never printed, never written to an error++// message and never put in a URL.++package memorystore++++import (++ "bytes"++ "context"++ "encoding/json"++ "errors"++ "fmt"++ "io"++ "math/rand"++ "net/http"++ "net/url"++ "strconv"++ "strings"++ "time"++)++++// DefaultBaseURL is where the store lives.++const DefaultBaseURL = "https://ai.openvibe.services"++++// BatchSize is how many items one request may carry. The server takes 200.++const BatchSize = 200++++// MaxAttempts is how many times a request is tried when the server is busy or++// broken.++const MaxAttempts = 4++++// MinTokenLength is the length a token must have before it is taken out of a++// message. A shorter one would match inside every other word.++const MinTokenLength = 8++++// DefaultTokenEnv is the environment variable the person's access token is++// read from. It is the same token the OpenVibe.Codes CLI uses.++const DefaultTokenEnv = "OPENVIBE_TOKEN"++++// SourceRef says where an item was written: the path of the file relative to++// the scanned root, and the heading it was under. It never holds an absolute++// path.++type SourceRef struct {++ Path string `json:"path,omitempty"`++ Heading string `json:"heading,omitempty"`++}++++// Item is one memory item on the wire.++type Item struct {++ // Scope is `user` or `project:<slug>`. Never an absolute path.++ Scope string `json:"scope"`++ // Kind is one of preference, fact, project_rule, procedure, decision, lesson.++ Kind string `json:"kind"`++ // Text is the item itself, already redacted.++ Text string `json:"text"`++ // Source is the id of the adapter that found it.++ Source string `json:"source"`++ // SourceRef says where it was written.++ SourceRef *SourceRef `json:"source_ref,omitempty"`++ // Confidence is how sure the item is, 0 to 1.++ Confidence float64 `json:"confidence,omitempty"`++ // NoLooks says the item was written by an AI, so it is a suggestion and++ // may not read as a fact about the person.++ NoLooks bool `json:"no_looks,omitempty"`++}++++// BatchResult is what the store answered about an import.++type BatchResult struct {++ // Created is how many items were new.++ Created int++ // Deduplicated is how many items were already there.++ Deduplicated int++ // Refused is how many items the store would not take.++ Refused int++}++++// ListItem is one item of the person's own list.++type ListItem struct {++ ID string `json:"id"`++ Scope string `json:"scope"`++ Kind string `json:"kind"`++ Text string `json:"text"`++ Source string `json:"source"`++ Status string `json:"status"`++ Ref *SourceRef `json:"source_ref,omitempty"`++}++++// ListResult is the person's own items.++type ListResult struct {++ Items []ListItem++ Count int++}++++// APIError is a refusal of the store.++type APIError struct {++ // Status is the HTTP status.++ Status int++ // Code is the problem code, e.g. `input.invalid`.++ Code string++ // Detail says what the store refused.++ Detail string++}++++func (e *APIError) Error() string {++ if e.Code == "" {++ return fmt.Sprintf("the memory store refused this with HTTP %d", e.Status)++ }++ return fmt.Sprintf("the memory store refused this with HTTP %d (%s): %s", e.Status, e.Code, e.Detail)++}++++// Retryable says whether trying again may help: the server is busy or broken.++func (e *APIError) Retryable() bool {++ return e.Status == http.StatusTooManyRequests || e.Status >= 500++}++++// ErrNoToken says there is no person token to open the store with.++var ErrNoToken = errors.New("no person token: set OPENVIBE_TOKEN (or the variable --token-env names) to your OpenVibe access token")++++// Client is the store client of one person.++type Client struct {++ baseURL string++ token string++ http *http.Client++ // Backoff waits before a retry. It is a field so the tests need not wait.++ backoff func(attempt int) time.Duration++}++++// Options configure a Client.++type Options struct {++ // BaseURL is the origin of the store. Empty means DefaultBaseURL.++ BaseURL string++ // Token is the person's access token.++ Token string++ // HTTP is the client to use. Empty means a client with a 30 s timeout.++ HTTP *http.Client++ // Backoff waits before a retry. Empty means an exponential wait with jitter.++ Backoff func(attempt int) time.Duration++}++++// New makes a client of the store.++func New(opts Options) (*Client, error) {++ base := strings.TrimSuffix(strings.TrimSpace(opts.BaseURL), "/")++ if base == "" {++ base = DefaultBaseURL++ }++ u, err := url.Parse(base)++ if err != nil || u.Scheme == "" || u.Host == "" {++ return nil, fmt.Errorf("the memory store address %q is not an URL", opts.BaseURL)++ }++ if u.Scheme != "https" && !(u.Scheme == "http" && isLoopback(u.Hostname())) {++ return nil, fmt.Errorf("refusing to talk to the memory store over %s", u.Scheme)++ }++ httpClient := opts.HTTP++ if httpClient == nil {++ httpClient = &http.Client{Timeout: 30 * time.Second}++ }++ c := &Client{++ baseURL: base,++ token: strings.TrimSpace(opts.Token),++ http: httpClient,++ backoff: opts.Backoff,++ }++ if c.backoff == nil {++ c.backoff = defaultBackoff++ }++ return c, nil++}++++// defaultBackoff waits a little longer every time, with jitter, so a busy++// store is not hammered by every machine at once.++func defaultBackoff(attempt int) time.Duration {++ base := 500 * time.Millisecond * time.Duration(1<<uint(attempt-1))++ if base > 8*time.Second {++ base = 8 * time.Second++ }++ return base/2 + time.Duration(rand.Int63n(int64(base/2)+1))++}++++// isLoopback tells whether a host is the machine itself.++func isLoopback(host string) bool {++ return host == "127.0.0.1" || host == "::1" || host == "localhost"++}++++// Upload sends the items to the store, in batches of 200.++//++// An item the store refuses is counted and the rest of the batch is sent++// again; a store that is busy or broken is tried again with a wait. The text++// of an item never appears in an error.++func (c *Client) Upload(ctx context.Context, items []Item) (BatchResult, error) {++ var out BatchResult++ for start := 0; start < len(items); start += BatchSize {++ end := start + BatchSize++ if end > len(items) {++ end = len(items)++ }++ res, refused, err := c.uploadChunk(ctx, items[start:end])++ out.Created += res.Created++ out.Deduplicated += res.Deduplicated++ out.Refused += refused++ if err != nil {++ return out, err++ }++ }++ return out, nil++}++++// uploadChunk sends one batch and returns what the store made of it. A refused++// item is dropped and the batch is sent again, at most a few times.++func (c *Client) uploadChunk(ctx context.Context, chunk []Item) (BatchResult, int, error) {++ refused := 0++ for attempt := 0; attempt < 10 && len(chunk) > 0; attempt++ {++ res, err := c.postBatch(ctx, chunk)++ if err == nil {++ res.Refused = refused++ return res, refused, nil++ }++ var apiErr *APIError++ if !errors.As(err, &apiErr) || !apiErr.Retryable() {++ index, ok := refusedIndex(apiErr)++ if !ok {++ return BatchResult{}, refused, err++ }++ // The store named the item it would not take: drop it and send the rest.++ refused++++ chunk = append(chunk[:index:index], chunk[index+1:]...)++ continue++ }++ if attempt == MaxAttempts-1 {++ return BatchResult{}, refused, err++ }++ select {++ case <-ctx.Done():++ return BatchResult{}, refused, ctx.Err()++ case <-time.After(c.backoff(attempt + 1)):++ }++ }++ return BatchResult{}, refused, errors.New("the memory store refused too many items")++}++++// refusedIndex finds the item the store named in its refusal.++func refusedIndex(err *APIError) (int, bool) {++ if err == nil {++ return 0, false++ }++ const marker = "items["++ at := strings.Index(err.Detail, marker)++ if at < 0 {++ return 0, false++ }++ rest := err.Detail[at+len(marker):]++ end := strings.Index(rest, "]")++ if end < 0 {++ return 0, false++ }++ index, convErr := strconv.Atoi(rest[:end])++ if convErr != nil || index < 0 {++ return 0, false++ }++ return index, true++}++++// batchResponse is what the store answers to an import.++type batchResponse struct {++ Created int `json:"created"`++ Deduped int `json:"deduped"`++ Count int `json:"count"`++}++++// postBatch sends one batch to the store.++func (c *Client) postBatch(ctx context.Context, chunk []Item) (BatchResult, error) {++ body, err := json.Marshal(map[string]any{"items": chunk})++ if err != nil {++ return BatchResult{}, fmt.Errorf("the items cannot be read: %w", err)++ }++ raw, status, err := c.do(ctx, http.MethodPost, "/api/v1/memory/items:batch", body)++ if err != nil {++ return BatchResult{}, err++ }++ if status/100 != 2 {++ return BatchResult{}, c.apiError(status, raw)++ }++ var res batchResponse++ if err := json.Unmarshal(raw, &res); err != nil {++ return BatchResult{}, errors.New("the memory store answered something that is not JSON")++ }++ out := BatchResult{Created: res.Created, Deduplicated: res.Deduped}++ if out.Created == 0 && out.Deduplicated == 0 && res.Count > 0 {++ // An older store counts the items and names them differently.++ out.Created = res.Count++ }++ return out, nil++}++++// List gives the person's own items.++func (c *Client) List(ctx context.Context, scope, query string) (ListResult, error) {++ path := "/api/v1/memory/items"++ params := url.Values{}++ if scope != "" {++ params.Set("scope", scope)++ }++ if query != "" {++ params.Set("q", query)++ }++ if len(params) > 0 {++ path += "?" + params.Encode()++ }++ raw, status, err := c.do(ctx, http.MethodGet, path, nil)++ if err != nil {++ return ListResult{}, err++ }++ if status/100 != 2 {++ return ListResult{}, c.apiError(status, raw)++ }++ var res struct {++ Items []ListItem `json:"items"`++ Count int `json:"count"`++ }++ if err := json.Unmarshal(raw, &res); err != nil {++ return ListResult{}, errors.New("the memory store answered something that is not JSON")++ }++ return ListResult{Items: res.Items, Count: res.Count}, nil++}++++// do sends one request with the person's token and gives back the body and the++// status. A network failure is an error that names the cause, never the token.++func (c *Client) do(ctx context.Context, method, path string, body []byte) ([]byte, int, error) {++ if c.token == "" {++ return nil, 0, ErrNoToken++ }++ var reader io.Reader++ if body != nil {++ reader = bytes.NewReader(body)++ }++ req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, reader)++ if err != nil {++ return nil, 0, fmt.Errorf("the request to the memory store cannot be made: %w", err)++ }++ req.Header.Set("Authorization", "Bearer "+c.token)++ req.Header.Set("Accept", "application/json")++ if body != nil {++ req.Header.Set("Content-Type", "application/json")++ }++ resp, err := c.http.Do(req)++ if err != nil {++ return nil, 0, fmt.Errorf("the memory store could not be reached: %s", scrub(err.Error(), c.token))++ }++ defer resp.Body.Close()++ raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))++ if err != nil {++ return nil, resp.StatusCode, fmt.Errorf("the memory store answered badly: %w", err)++ }++ return raw, resp.StatusCode, nil++}++++// problem is the RFC 9457 body of a refusal.++type problem struct {++ Status int `json:"status"`++ Code string `json:"code"`++ Title string `json:"title"`++ Detail string `json:"detail"`++}++++// apiError turns a refusal into an APIError.++func (c *Client) apiError(status int, raw []byte) error {++ var p problem++ if err := json.Unmarshal(raw, &p); err != nil {++ p.Detail = strings.TrimSpace(string(raw))++ }++ detail := p.Detail++ if detail == "" {++ detail = p.Title++ }++ if len(detail) > 400 {++ detail = detail[:400]++ }++ return &APIError{Status: status, Code: p.Code, Detail: scrub(detail, c.token)}++}++++// scrub removes the token from a text that is about to be printed.++//++// A short token is left alone: it would match inside every other word and++// turn the message into nonsense. A real access token is long, and a long one++// is worth taking out.++func scrub(text, token string) string {++ if len(token) < MinTokenLength {++ return text++ }++ return strings.ReplaceAll(text, token, "[redacted]")++}nnew file mode 100644@@@ -0,0 +1,309 @@++package memorystore++++import (++ "context"++ "encoding/json"++ "io"++ "net/http"++ "net/http/httptest"++ "strings"++ "sync"++ "testing"++ "time"++)++++// recorded is one request the fake store received.++type recorded struct {++ Method string++ Path string++ Auth string++ Body []byte++}++++// fakeStore is a store that answers what the test tells it to.++type fakeStore struct {++ mu sync.Mutex++ requests []recorded++ // answers is the list of answers, one per request; the last one repeats.++ answers []func(w http.ResponseWriter, r *http.Request)++}++++// newFakeStore starts a fake store.++func newFakeStore(t *testing.T, answers ...func(w http.ResponseWriter, r *http.Request)) (*fakeStore, *httptest.Server) {++ t.Helper()++ fs := &fakeStore{answers: answers}++ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {++ body := readBody(r)++ fs.mu.Lock()++ fs.requests = append(fs.requests, recorded{Method: r.Method, Path: r.URL.Path, Auth: r.Header.Get("Authorization"), Body: body})++ var answer func(w http.ResponseWriter, r *http.Request)++ if len(fs.answers) == 0 {++ answer = okAnswer++ } else if len(fs.requests) <= len(fs.answers) {++ answer = fs.answers[len(fs.requests)-1]++ } else {++ answer = fs.answers[len(fs.answers)-1]++ }++ fs.mu.Unlock()++ answer(w, r)++ }))++ t.Cleanup(srv.Close)++ return fs, srv++}++++// readBody reads a request body.++func readBody(r *http.Request) []byte {++ if r.Body == nil {++ return nil++ }++ raw, _ := io.ReadAll(r.Body)++ return raw++}++++// okAnswer answers with an empty import.++func okAnswer(w http.ResponseWriter, _ *http.Request) {++ w.Header().Set("Content-Type", "application/json")++ _, _ = w.Write([]byte(`{"items":[],"count":0,"created":0,"deduped":0}`))++}++++// seen gives the requests the fake store received.++func (fs *fakeStore) seen() []recorded {++ fs.mu.Lock()++ defer fs.mu.Unlock()++ return append([]recorded(nil), fs.requests...)++}++++// TestUploadBatchesInTwoHundred checks that the items go in batches of 200.++func TestUploadBatchesInTwoHundred(t *testing.T) {++ fs, srv := newFakeStore(t)++ client, err := New(Options{BaseURL: srv.URL, Token: "person-token", Backoff: func(int) time.Duration { return 0 }})++ if err != nil {++ t.Fatal(err)++ }++ items := make([]Item, 0, 401)++ for i := 0; i < 401; i++ {++ items = append(items, Item{Scope: "user", Kind: "fact", Text: "item", Source: "claude-code"})++ }++ res, err := client.Upload(context.Background(), items)++ if err != nil {++ t.Fatalf("the upload failed: %v", err)++ }++ reqs := fs.seen()++ if len(reqs) != 3 {++ t.Fatalf("the store got %d requests, want 3", len(reqs))++ }++ for i, want := range []int{200, 200, 1} {++ var body struct {++ Items []Item `json:"items"`++ }++ if err := json.Unmarshal(reqs[i].Body, &body); err != nil {++ t.Fatalf("the body is not JSON: %v", err)++ }++ if len(body.Items) != want {++ t.Errorf("request %d carried %d items, want %d", i, len(body.Items), want)++ }++ }++ if res.Created != 0 {++ t.Errorf("the store said %d items were created, want 0", res.Created)++ }++}++++// TestUploadCountsCreatedAndDeduped checks the counts of the store's answer.++func TestUploadCountsCreatedAndDeduped(t *testing.T) {++ _, srv := newFakeStore(t, func(w http.ResponseWriter, _ *http.Request) {++ w.Header().Set("Content-Type", "application/json")++ _, _ = w.Write([]byte(`{"items":[],"count":5,"created":3,"deduped":2}`))++ })++ client, err := New(Options{BaseURL: srv.URL, Token: "t", Backoff: func(int) time.Duration { return 0 }})++ if err != nil {++ t.Fatal(err)++ }++ res, err := client.Upload(context.Background(), []Item{{Scope: "user", Kind: "fact", Text: "x", Source: "claude-code"}})++ if err != nil {++ t.Fatalf("the upload failed: %v", err)++ }++ if res.Created != 3 || res.Deduplicated != 2 {++ t.Errorf("the counts are %d created and %d deduped, want 3 and 2", res.Created, res.Deduplicated)++ }++}++++// TestUploadRetriesOnABusyStore checks that a 429 and a 500 are tried again.++func TestUploadRetriesOnABusyStore(t *testing.T) {++ fs, srv := newFakeStore(t,++ func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusTooManyRequests) },++ func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) },++ okAnswer,++ )++ client, err := New(Options{BaseURL: srv.URL, Token: "t", Backoff: func(int) time.Duration { return 0 }})++ if err != nil {++ t.Fatal(err)++ }++ if _, err := client.Upload(context.Background(), []Item{{Scope: "user", Kind: "fact", Text: "x", Source: "claude-code"}}); err != nil {++ t.Fatalf("the upload failed: %v", err)++ }++ if len(fs.seen()) != 3 {++ t.Errorf("the store got %d requests, want 3 (the two refusals and the answer)", len(fs.seen()))++ }++}++++// TestUploadGivesUpOnABrokenStore checks that a store that stays broken ends++// the upload with an error.++func TestUploadGivesUpOnABrokenStore(t *testing.T) {++ _, srv := newFakeStore(t, func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusBadGateway) })++ client, err := New(Options{BaseURL: srv.URL, Token: "t", Backoff: func(int) time.Duration { return 0 }})++ if err != nil {++ t.Fatal(err)++ }++ _, err = client.Upload(context.Background(), []Item{{Scope: "user", Kind: "fact", Text: "x", Source: "claude-code"}})++ if err == nil {++ t.Fatal("the upload said it worked, but the store was broken")++ }++ var apiErr *APIError++ if !asAPIError(err, &apiErr) {++ t.Fatalf("the error is not an APIError: %v", err)++ }++ if !apiErr.Retryable() {++ t.Errorf("the error is not retryable: %v", err)++ }++}++++// TestUploadDropsTheItemTheStoreRefuses checks that one refused item does not++// stop the rest.++func TestUploadDropsTheItemTheStoreRefuses(t *testing.T) {++ refuse := func(w http.ResponseWriter, _ *http.Request) {++ w.Header().Set("Content-Type", "application/problem+json")++ w.WriteHeader(http.StatusUnprocessableEntity)++ _, _ = w.Write([]byte(`{"status":422,"code":"memory.secret_only","detail":"items[1] is only a secret and is refused"}`))++ }++ fs, srv := newFakeStore(t, refuse, okAnswer)++ client, err := New(Options{BaseURL: srv.URL, Token: "t", Backoff: func(int) time.Duration { return 0 }})++ if err != nil {++ t.Fatal(err)++ }++ items := []Item{++ {Scope: "user", Kind: "fact", Text: "one", Source: "claude-code"},++ {Scope: "user", Kind: "fact", Text: "two", Source: "claude-code"},++ }++ res, err := client.Upload(context.Background(), items)++ if err != nil {++ t.Fatalf("the upload failed: %v", err)++ }++ if res.Refused != 1 {++ t.Errorf("the client counted %d refused items, want 1", res.Refused)++ }++ reqs := fs.seen()++ if len(reqs) != 2 {++ t.Fatalf("the store got %d requests, want 2", len(reqs))++ }++ var body struct {++ Items []Item `json:"items"`++ }++ if err := json.Unmarshal(reqs[1].Body, &body); err != nil {++ t.Fatalf("the body is not JSON: %v", err)++ }++ if len(body.Items) != 1 || body.Items[0].Text != "one" {++ t.Errorf("the second request carried %+v, want only the first item", body.Items)++ }++}++++// TestUploadSendsTheTokenAsABearer checks how the client authenticates.++func TestUploadSendsTheTokenAsABearer(t *testing.T) {++ fs, srv := newFakeStore(t)++ client, err := New(Options{BaseURL: srv.URL, Token: "the-person-token", Backoff: func(int) time.Duration { return 0 }})++ if err != nil {++ t.Fatal(err)++ }++ if _, err := client.Upload(context.Background(), []Item{{Scope: "user", Kind: "fact", Text: "x", Source: "claude-code"}}); err != nil {++ t.Fatalf("the upload failed: %v", err)++ }++ reqs := fs.seen()++ if len(reqs) != 1 {++ t.Fatalf("the store got %d requests, want 1", len(reqs))++ }++ if reqs[0].Auth != "Bearer the-person-token" {++ t.Errorf("the request carried %q, want the person's token", reqs[0].Auth)++ }++ if reqs[0].Path != "/api/v1/memory/items:batch" {++ t.Errorf("the request went to %q, want the batch path", reqs[0].Path)++ }++}++++// TestUploadWithoutATokenSaysSo checks that a client with no token does not++// call the store at all.++func TestUploadWithoutATokenSaysSo(t *testing.T) {++ fs, srv := newFakeStore(t)++ client, err := New(Options{BaseURL: srv.URL, Token: " "})++ if err != nil {++ t.Fatal(err)++ }++ _, err = client.Upload(context.Background(), []Item{{Scope: "user", Kind: "fact", Text: "x", Source: "claude-code"}})++ if err == nil {++ t.Fatal("the upload worked without a token")++ }++ if !strings.Contains(err.Error(), "OPENVIBE_TOKEN") {++ t.Errorf("the error does not say which variable is missing: %v", err)++ }++ if len(fs.seen()) != 0 {++ t.Errorf("the store got %d requests, want none", len(fs.seen()))++ }++}++++// TestTheTokenNeverAppearsInAnError checks that the token is not in the++// message of a refusal.++func TestTheTokenNeverAppearsInAnError(t *testing.T) {++ _, srv := newFakeStore(t, func(w http.ResponseWriter, _ *http.Request) {++ w.Header().Set("Content-Type", "application/problem+json")++ w.WriteHeader(http.StatusUnauthorized)++ _, _ = w.Write([]byte(`{"status":401,"code":"token.invalid","detail":"the token the-person-token is not valid"}`))++ })++ client, err := New(Options{BaseURL: srv.URL, Token: "the-person-token", Backoff: func(int) time.Duration { return 0 }})++ if err != nil {++ t.Fatal(err)++ }++ _, err = client.Upload(context.Background(), []Item{{Scope: "user", Kind: "fact", Text: "x", Source: "claude-code"}})++ if err == nil {++ t.Fatal("the upload worked with a refused token")++ }++ if strings.Contains(err.Error(), "the-person-token") {++ t.Errorf("the error holds the token: %v", err)++ }++}++++// TestListGivesTheItems checks that the list is asked for and read.++func TestListGivesTheItems(t *testing.T) {++ _, srv := newFakeStore(t, func(w http.ResponseWriter, _ *http.Request) {++ w.Header().Set("Content-Type", "application/json")++ _, _ = w.Write([]byte(`{"items":[{"id":"mem_1","scope":"user","kind":"fact","text":"hello","source":"claude-code","status":"active"}],"count":1}`))++ })++ client, err := New(Options{BaseURL: srv.URL, Token: "t"})++ if err != nil {++ t.Fatal(err)++ }++ res, err := client.List(context.Background(), "", "")++ if err != nil {++ t.Fatalf("the list failed: %v", err)++ }++ if res.Count != 1 || len(res.Items) != 1 || res.Items[0].Text != "hello" {++ t.Errorf("the list is %+v, want one item with the text hello", res)++ }++}++++// TestNewRefusesAPlainAddress checks that the client does not talk to the++// store over plain http.++func TestNewRefusesAPlainAddress(t *testing.T) {++ if _, err := New(Options{BaseURL: "http://ai.example.com", Token: "t"}); err == nil {++ t.Error("the client accepted a plain http store")++ }++ if _, err := New(Options{BaseURL: "https://ai.example.com", Token: "t"}); err != nil {++ t.Errorf("the client refused an https store: %v", err)++ }++}++++// asAPIError is errors.As without the import, kept short for the tests.++func asAPIError(err error, target **APIError) bool {++ if e, ok := err.(*APIError); ok {++ *target = e++ return true++ }++ return false++}nnew file mode 100644@@@ -0,0 +1,196 @@++// Package memoryupload turns a memory scan into the items that go to the++// person's own memory store.++//++// The scan found the items on the machine; this package decides what may leave++// it. Only the text of an item, its kind, its scope, the adapter it came from++// and where it was written (relative to the scanned root) ever leave. The++// absolute path of the file, the name of the machine and the name of the++// person stay behind.++package memoryupload++++import (++ "path/filepath"++ "sort"++ "strings"++++ "github.com/OpenVibers/OpenVibe.Node/internal/memory"++ "github.com/OpenVibers/OpenVibe.Node/internal/memorystore"++)++++// MaxSlug is the length of a project slug. The store takes at most 64.++const MaxSlug = 64++++// FileSummary says what one file of a tool gave.++type FileSummary struct {++ // Path is the file, as the scan found it.++ Path string++ // Items is how many items it gave.++ Items int++ // Note says why the file was skipped, when it was.++ Note string++ // Found is true when the file was there.++ Found bool++}++++// ToolSummary says what one tool gave.++type ToolSummary struct {++ // ID is the id of the adapter.++ ID string++ // Name is how the tool is called.++ Name string++ // Files are the files that were looked at.++ Files []FileSummary++ // Items is how many items the tool gave.++ Items int++ // Redacted is how many of them held a secret.++ Redacted int++ // Redactions is how many secrets were removed.++ Redactions int++}++++// Summary is what the person is shown before anything is sent.++type Summary struct {++ // Tools are the tools that were read, in the order of the scan.++ Tools []ToolSummary++ // Scopes are the scopes the items go to, sorted.++ Scopes []string++ // Items is how many items will be sent.++ Items int++ // Redacted is how many items held a secret.++ Redacted int++ // Redactions is how many secrets were removed.++ Redactions int++ // Skipped is how many items stay behind because they are empty.++ Skipped int++}++++// Plan is what will be sent, and what the person is shown first.++type Plan struct {++ // Items are the items to send, in the order of the scan.++ Items []memorystore.Item++ // Summary is what the person is shown.++ Summary Summary++}++++// Build turns a scan into a plan.++//++// An item whose text is empty after the redaction never leaves the machine, so++// it is not in the plan and is counted as skipped. The scope of a project item++// is the slug of the project's folder name: never an absolute path.++func Build(res memory.Result) Plan {++ plan := Plan{Items: []memorystore.Item{}, Summary: Summary{Tools: []ToolSummary{}}}++ scopes := map[string]bool{}++ redactedBySource := map[string]int{}++ redactionsBySource := map[string]int{}++++ for _, ad := range res.Adapters {++ ts := ToolSummary{ID: ad.ID, Name: ad.Name, Files: []FileSummary{}}++ for _, f := range ad.Files {++ ts.Files = append(ts.Files, FileSummary{Path: f.Path, Items: f.Items, Note: f.Note, Found: f.Found})++ }++ ts.Items = ad.Items++ plan.Summary.Tools = append(plan.Summary.Tools, ts)++ }++++ for _, it := range res.Items {++ if it.Redacted {++ redactedBySource[it.Source]++++ redactionsBySource[it.Source] += it.Redactions++ }++ text := strings.TrimSpace(it.Text)++ if text == "" {++ plan.Summary.Skipped++++ continue++ }++ scope, root := scopeOf(it, res)++ scopes[scope] = true++ kind := string(it.Kind)++ if kind == "" {++ kind = string(memory.KindFact)++ }++ item := memorystore.Item{++ Scope: scope,++ Kind: kind,++ Text: text,++ Source: it.Source,++ Confidence: 1,++ // What a coding tool wrote is a suggestion until the person accepts it.++ NoLooks: true,++ }++ if ref := sourceRef(it, root); ref != nil {++ item.SourceRef = ref++ }++ plan.Items = append(plan.Items, item)++ }++++ for i := range plan.Summary.Tools {++ plan.Summary.Tools[i].Redacted = redactedBySource[plan.Summary.Tools[i].ID]++ plan.Summary.Tools[i].Redactions = redactionsBySource[plan.Summary.Tools[i].ID]++ }++ plan.Summary.Items = len(plan.Items)++ for _, ts := range plan.Summary.Tools {++ plan.Summary.Redacted += ts.Redacted++ plan.Summary.Redactions += ts.Redactions++ }++ for scope := range scopes {++ plan.Summary.Scopes = append(plan.Summary.Scopes, scope)++ }++ sort.Strings(plan.Summary.Scopes)++ return plan++}++++// scopeOf gives the scope an item goes to and the root its path is relative++// to: the home for the person, the project folder for a project.++func scopeOf(it memory.Item, res memory.Result) (string, string) {++ scope := string(it.Scope)++ if scope == "" || scope == string(memory.ScopeUser) {++ return string(memory.ScopeUser), res.Home++ }++ if abs, ok := strings.CutPrefix(scope, "project:"); ok {++ return "project:" + Slug(abs), abs++ }++ return string(memory.ScopeUser), res.Home++}++++// sourceRef says where the item was written: the path relative to the scanned++// root and the heading it was under. A path that stays inside the root is++// never absolute.++func sourceRef(it memory.Item, root string) *memorystore.SourceRef {++ ref := &memorystore.SourceRef{Heading: it.Heading}++ if root != "" && it.Path != "" {++ if rel, err := filepath.Rel(root, it.Path); err == nil && !strings.HasPrefix(rel, "..") {++ ref.Path = filepath.ToSlash(rel)++ }++ }++ if ref.Path == "" && ref.Heading == "" {++ return nil++ }++ return ref++}++++// Slug is the name a project is known by in the store: the folder's name,++// lowercased, with everything that is not a letter, a digit, a dot, a dash or++// an underscore turned into a dash. It is never an absolute path.++func Slug(absPath string) string {++ base := filepath.Base(filepath.Clean(absPath))++ var b strings.Builder++ for _, r := range strings.ToLower(base) {++ switch {++ case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '.', r == '-', r == '_':++ b.WriteRune(r)++ default:++ b.WriteRune('-')++ }++ }++ slug := b.String()++ // The store wants a slug that starts with a letter or a digit.++ slug = strings.TrimLeft(slug, "._-")++ if slug == "" {++ return "project"++ }++ if len(slug) > MaxSlug {++ slug = slug[:MaxSlug]++ slug = strings.TrimRight(slug, "._-")++ }++ return slug++}nnew file mode 100644@@@ -0,0 +1,153 @@++package memoryupload++++import (++ "strings"++ "testing"++++ "github.com/OpenVibers/OpenVibe.Node/internal/memory"++)++++// TestSlugIsTheFolderName checks the name a project is known by.++func TestSlugIsTheFolderName(t *testing.T) {++ cases := map[string]string{++ "/home/you/work/My Project": "my-project",++ "/home/you/work/demo": "demo",++ "/home/you/work/Ünïcode": "n-code",++ "/home/you/work/": "work",++ "/": "project",++ "/home/you/....": "project",++ }++ for path, want := range cases {++ if got := Slug(path); got != want {++ t.Errorf("the slug of %q is %q, want %q", path, got, want)++ }++ }++}++++// TestSlugFitsTheStore checks that a slug is what the store accepts: it++// starts with a letter or a digit and is not longer than 64 characters.++func TestSlugFitsTheStore(t *testing.T) {++ long := strings.Repeat("a", 200)++ slug := Slug("/home/you/" + long)++ if len(slug) > MaxSlug {++ t.Errorf("the slug is %d characters long, want at most %d", len(slug), MaxSlug)++ }++ for _, r := range slug {++ if !(r >= 'a' && r <= 'z' || r >= '0' && r <= '9' || r == '.' || r == '-' || r == '_') {++ t.Errorf("the slug holds %q, want only letters, digits, dots, dashes and underscores", r)++ }++ }++ if slug[0] == '.' || slug[0] == '-' || slug[0] == '_' {++ t.Errorf("the slug starts with %q, want a letter or a digit", slug[0])++ }++}++++// result is a small scan with a user item, a project item and an empty one.++func result() memory.Result {++ return memory.Result{++ Home: "/home/you",++ Projects: []string{"/home/you/work/My Project"},++ Adapters: []memory.AdapterReport{++ {ID: "claude-code", Name: "Claude Code", Items: 3, Files: []memory.FileReport{++ {Path: "/home/you/.claude/CLAUDE.md", Found: true, Items: 2},++ {Path: "/home/you/work/My Project/CLAUDE.md", Found: true, Items: 1},++ }},++ },++ Items: []memory.Item{++ {Text: "Prefer tabs", Kind: memory.KindPreference, Scope: memory.ScopeUser, Source: "claude-code",++ Path: "/home/you/.claude/CLAUDE.md", Heading: "Setup"},++ {Text: "The build takes 12 minutes", Kind: memory.KindProjectRule, Scope: memory.ScopeProject("/home/you/work/My Project"),++ Source: "claude-code", Path: "/home/you/work/My Project/CLAUDE.md", Heading: "Build"},++ {Text: " ", Kind: memory.KindFact, Scope: memory.ScopeUser, Source: "claude-code", Path: "/home/you/.claude/CLAUDE.md"},++ },++ }++}++++// TestBuildMakesTheScopes checks the scopes of the plan: the person, and a++// project by the slug of its folder name.++func TestBuildMakesTheScopes(t *testing.T) {++ plan := Build(result())++ if len(plan.Items) != 2 {++ t.Fatalf("the plan holds %d items, want 2 (the empty one stays behind)", len(plan.Items))++ }++ if plan.Items[0].Scope != "user" {++ t.Errorf("the first item has the scope %q, want user", plan.Items[0].Scope)++ }++ if plan.Items[1].Scope != "project:my-project" {++ t.Errorf("the second item has the scope %q, want project:my-project", plan.Items[1].Scope)++ }++ if plan.Summary.Skipped != 1 {++ t.Errorf("the plan skipped %d items, want 1", plan.Summary.Skipped)++ }++}++++// TestBuildNeverHoldsAnAbsolutePath checks that no path of the machine is in++// the plan: the paths are relative to the folder they were scanned from.++func TestBuildNeverHoldsAnAbsolutePath(t *testing.T) {++ plan := Build(result())++ if plan.Items[0].SourceRef == nil || plan.Items[0].SourceRef.Path != ".claude/CLAUDE.md" {++ t.Errorf("the first item has the path %+v, want .claude/CLAUDE.md", plan.Items[0].SourceRef)++ }++ if plan.Items[1].SourceRef == nil || plan.Items[1].SourceRef.Path != "CLAUDE.md" {++ t.Errorf("the second item has the path %+v, want CLAUDE.md", plan.Items[1].SourceRef)++ }++ if plan.Items[0].SourceRef.Heading != "Setup" {++ t.Errorf("the first item has the heading %q, want Setup", plan.Items[0].SourceRef.Heading)++ }++ for _, item := range plan.Items {++ if strings.Contains(item.SourceRef.Path, "/home/") {++ t.Errorf("the path %q holds the home folder", item.SourceRef.Path)++ }++ }++}++++// TestBuildCountsWhatWasRedacted checks the counts of the summary.++func TestBuildCountsWhatWasRedacted(t *testing.T) {++ res := result()++ res.Items[0].Redacted = true++ res.Items[0].Redactions = 2++ plan := Build(res)++ if plan.Summary.Redacted != 1 {++ t.Errorf("the summary counts %d redacted items, want 1", plan.Summary.Redacted)++ }++ if plan.Summary.Redactions != 2 {++ t.Errorf("the summary counts %d secrets, want 2", plan.Summary.Redactions)++ }++ if len(plan.Summary.Tools) != 1 || plan.Summary.Tools[0].Redacted != 1 {++ t.Errorf("the tool of the summary counts %+v, want one redacted item", plan.Summary.Tools)++ }++ if plan.Summary.Items != 2 {++ t.Errorf("the summary counts %d items to send, want 2", plan.Summary.Items)++ }++}++++// TestBuildMarksWhatAToolWrote checks that the items a coding tool wrote are++// marked as written by an AI, so the store keeps them as suggestions.++func TestBuildMarksWhatAToolWrote(t *testing.T) {++ plan := Build(result())++ for _, item := range plan.Items {++ if !item.NoLooks {++ t.Errorf("the item %q is not marked as written by a tool", item.Text)++ }++ if item.Confidence != 1 {++ t.Errorf("the item %q has the confidence %v, want 1", item.Text, item.Confidence)++ }++ if item.Source != "claude-code" {++ t.Errorf("the item %q has the source %q, want the adapter id", item.Text, item.Source)++ }++ }++}++++// TestBuildSortsTheScopes checks that the scopes of the summary are sorted.++func TestBuildSortsTheScopes(t *testing.T) {++ plan := Build(result())++ want := []string{"project:my-project", "user"}++ if len(plan.Summary.Scopes) != len(want) {++ t.Fatalf("the summary holds the scopes %v, want %v", plan.Summary.Scopes, want)++ }++ for i, scope := range want {++ if plan.Summary.Scopes[i] != scope {++ t.Errorf("the scopes are %v, want %v", plan.Summary.Scopes, want)++ }++ }++}
@@ -33,6 +33,7 @@ openvibe-node quit # end the node cleanly (nothing latched), e.g. from a s
@@ -33,6 +33,7 @@ openvibe-node quit # end the node cleanly (nothing latched), e.g. from a s
openvibe-node approvals # the computer plugin at level ask: what waits; approve <id> / deny <id>
openvibe-node approvals # the computer plugin at level ask: what waits; approve <id> / deny <id>
openvibe-node tray # the same in your notification area, with Approve/Deny notifications (Linux, Windows)
openvibe-node tray # the same in your notification area, with Approve/Deny notifications (Linux, Windows)
openvibe-node memory scan # what your coding tools remember: read-only, local, secrets removed
openvibe-node memory scan # what your coding tools remember: read-only, local, secrets removed
+openvibe-node memory upload # send what they remember to your own memory store, after you say yes
```
```
## What is in here
## What is in here
@@ -72,7 +73,8 @@ task on openvibe.codes/run from any browser and have this machine do it ([docs/w
@@ -72,7 +73,8 @@ task on openvibe.codes/run from any browser and have this machine do it ([docs/w
Docs: [protocol](docs/protocol.md) (the wire format with OpenVibe.Bot), [plugins](docs/plugins.md) (the driver
Docs: [protocol](docs/protocol.md) (the wire format with OpenVibe.Bot), [plugins](docs/plugins.md) (the driver
protocol, the bundled drivers, and where Actor's computer-control plugin attaches), [install](docs/install.md),
protocol, the bundled drivers, and where Actor's computer-control plugin attaches), [install](docs/install.md),
-[memory](docs/memory.md) (what the coding tools on a machine remember, and how the Node reads it).
+[memory](docs/memory.md) (what the coding tools on a machine remember, how the Node reads it and how it is uploaded to
+your own memory store).
Decisions: [ADR-043](docs/ADR-043-bot-devices-and-control.md).
Decisions: [ADR-043](docs/ADR-043-bot-devices-and-control.md).
## Development
## Development
@@ -108,7 +108,7 @@ func runWithStdin(args []string, stdin io.Reader, stdout, stderr io.Writer) int
@@ -108,7 +108,7 @@ func runWithStdin(args []string, stdin io.Reader, stdout, stderr io.Writer) int
cmd, rest := args[0], args[1:]
cmd, rest := args[0], args[1:]
if cmd == "memory" {
if cmd == "memory" {
// The memory scan has its own flags, so it parses its own arguments.
// The memory scan has its own flags, so it parses its own arguments.
- return cmdMemory(rest, stdout, stderr)
+ return cmdMemory(rest, stdin, stdout, stderr)
}
}
if cmd == "desk-sandbox" { // its own flags, up to "--": a browser's flags after it are never the Node's
if cmd == "desk-sandbox" { // its own flags, up to "--": a browser's flags after it are never the Node's
return cmdDeskSandbox(rest, stderr)
return cmdDeskSandbox(rest, stderr)
@@ -5,8 +5,6 @@ import (
@@ -5,8 +5,6 @@ import (
"flag"
"flag"
"fmt"
"fmt"
"io"
"io"
- "os"
- "path/filepath"
"sort"
"sort"
"strings"
"strings"
@@ -15,16 +13,25 @@ import (
@@ -15,16 +13,25 @@ import (
// memoryUsage is what "openvibe-node memory" prints.
// memoryUsage is what "openvibe-node memory" prints.
const memoryUsage = "openvibe-node memory scan [--tools a,b] [--project DIR] [--json]\n" +
const memoryUsage = "openvibe-node memory scan [--tools a,b] [--project DIR] [--json]\n" +
+ "openvibe-node memory upload [--tools a,b] [--project DIR …] [--dry-run] [--yes]\n" +
+ "openvibe-node memory status [--show]\n" +
"\n" +
"\n" +
- " Reads the memory files of the coding tools on this machine and prints what it\n" +
- " found. It is read-only and local: nothing is written and nothing is sent\n" +
- " anywhere. Secrets are removed before anything is printed.\n" +
+ " scan reads the memory files of the coding tools on this machine and\n" +
+ " prints what it found. It is read-only and local: nothing is written\n" +
+ " and nothing is sent anywhere. Secrets are removed before anything is\n" +
+ " printed.\n" +
+ " upload sends the items to your own memory store (OpenVibe.AI). It shows\n" +
+ " what would leave the machine and asks before it sends anything.\n" +
+ " status shows what your own memory store holds.\n" +
"\n" +
"\n" +
" --tools a,b only these adapters (claude-code, agents, gemini, cursor,\n" +
" --tools a,b only these adapters (claude-code, agents, gemini, cursor,\n" +
" copilot, windsurf, cline, roo, kiro, continue, aider,\n" +
" copilot, windsurf, cline, roo, kiro, continue, aider,\n" +
" command-code, opencode)\n" +
" command-code, opencode)\n" +
" --project DIR read this project as well; may be given more than once\n" +
" --project DIR read this project as well; may be given more than once\n" +
- " --json print the items as JSON\n" +
+ " --json print the items as JSON (scan)\n" +
+ " --dry-run print the summary and send nothing (upload)\n" +
+ " --yes do not ask before sending (upload)\n" +
+ " --show print the items themselves (status)\n" +
" --user-home DIR read this directory as the person's home (default: yours)\n"
" --user-home DIR read this directory as the person's home (default: yours)\n"
// stringList is a flag that may be given more than once.
// stringList is a flag that may be given more than once.
@@ -43,8 +50,18 @@ func (s *stringList) Set(v string) error {
@@ -43,8 +50,18 @@ func (s *stringList) Set(v string) error {
}
}
// cmdMemory runs the memory commands.
// cmdMemory runs the memory commands.
-func cmdMemory(args []string, stdout, stderr io.Writer) int {
- if len(args) == 0 || args[0] != "scan" {
+func cmdMemory(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ if len(args) == 0 {
+ fmt.Fprint(stderr, memoryUsage)
+ return 2
+ }
+ switch args[0] {
+ case "upload":
+ return cmdMemoryUpload(args[1:], stdin, stdout, stderr)
+ case "status":
+ return cmdMemoryStatus(args[1:], stdout, stderr)
+ case "scan":
+ default:
fmt.Fprint(stderr, memoryUsage)
fmt.Fprint(stderr, memoryUsage)
return 2
return 2
}
}
@@ -65,22 +82,12 @@ func cmdMemory(args []string, stdout, stderr io.Writer) int {
@@ -65,22 +82,12 @@ func cmdMemory(args []string, stdout, stderr io.Writer) int {
fmt.Fprint(stderr, memoryUsage)
fmt.Fprint(stderr, memoryUsage)
return 2
return 2
}
}
- home := *userHome
- if home == "" {
- h, err := os.UserHomeDir()
- if err != nil {
- fmt.Fprintln(stderr, "error: cannot find your home directory; pass --user-home")
- return 1
- }
- home = h
- }
- abs, err := filepath.Abs(home)
- if err != nil {
- fmt.Fprintln(stderr, "error:", err)
- return 1
+ home, code := memoryHome(*userHome, stderr)
+ if code != 0 {
+ return code
}
}
res := memory.Scan(memory.Options{
res := memory.Scan(memory.Options{
- Home: abs,
+ Home: home,
Projects: projects,
Projects: projects,
Tools: tools,
Tools: tools,
})
})
@@ -44,7 +44,7 @@ func memoryFixture(t *testing.T) string {
@@ -44,7 +44,7 @@ func memoryFixture(t *testing.T) string {
func TestMemoryScanPrintsEveryAdapter(t *testing.T) {
func TestMemoryScanPrintsEveryAdapter(t *testing.T) {
home := memoryFixture(t)
home := memoryFixture(t)
var stdout, stderr bytes.Buffer
var stdout, stderr bytes.Buffer
- code := cmdMemory([]string{"scan", "--user-home", home}, &stdout, &stderr)
+ code := cmdMemory([]string{"scan", "--user-home", home}, strings.NewReader(""), &stdout, &stderr)
if code != 0 {
if code != 0 {
t.Fatalf("the scan failed with %d: %s", code, stderr.String())
t.Fatalf("the scan failed with %d: %s", code, stderr.String())
}
}
@@ -79,7 +79,7 @@ func TestMemoryScanPrintsEveryAdapter(t *testing.T) {
@@ -79,7 +79,7 @@ func TestMemoryScanPrintsEveryAdapter(t *testing.T) {
func TestMemoryScanJSON(t *testing.T) {
func TestMemoryScanJSON(t *testing.T) {
home := memoryFixture(t)
home := memoryFixture(t)
var stdout, stderr bytes.Buffer
var stdout, stderr bytes.Buffer
- code := cmdMemory([]string{"scan", "--user-home", home, "--json"}, &stdout, &stderr)
+ code := cmdMemory([]string{"scan", "--user-home", home, "--json"}, strings.NewReader(""), &stdout, &stderr)
if code != 0 {
if code != 0 {
t.Fatalf("the scan failed with %d: %s", code, stderr.String())
t.Fatalf("the scan failed with %d: %s", code, stderr.String())
}
}
@@ -112,7 +112,7 @@ func TestMemoryScanJSON(t *testing.T) {
@@ -112,7 +112,7 @@ func TestMemoryScanJSON(t *testing.T) {
func TestMemoryScanToolsFilter(t *testing.T) {
func TestMemoryScanToolsFilter(t *testing.T) {
home := memoryFixture(t)
home := memoryFixture(t)
var stdout, stderr bytes.Buffer
var stdout, stderr bytes.Buffer
- code := cmdMemory([]string{"scan", "--user-home", home, "--tools", "cursor", "--json"}, &stdout, &stderr)
+ code := cmdMemory([]string{"scan", "--user-home", home, "--tools", "cursor", "--json"}, strings.NewReader(""), &stdout, &stderr)
if code != 0 {
if code != 0 {
t.Fatalf("the scan failed with %d: %s", code, stderr.String())
t.Fatalf("the scan failed with %d: %s", code, stderr.String())
}
}
@@ -124,7 +124,7 @@ func TestMemoryScanToolsFilter(t *testing.T) {
@@ -124,7 +124,7 @@ func TestMemoryScanToolsFilter(t *testing.T) {
// TestMemoryScanUsage checks that a wrong use says how to use it.
// TestMemoryScanUsage checks that a wrong use says how to use it.
func TestMemoryScanUsage(t *testing.T) {
func TestMemoryScanUsage(t *testing.T) {
var stdout, stderr bytes.Buffer
var stdout, stderr bytes.Buffer
- if code := cmdMemory([]string{"list"}, &stdout, &stderr); code != 2 {
+ if code := cmdMemory([]string{"list"}, strings.NewReader(""), &stdout, &stderr); code != 2 {
t.Errorf("the command returned %d, want 2", code)
t.Errorf("the command returned %d, want 2", code)
}
}
if !strings.Contains(stderr.String(), "memory scan") {
if !strings.Contains(stderr.String(), "memory scan") {
@@ -137,7 +137,7 @@ func TestMemoryScanIsReadOnly(t *testing.T) {
@@ -137,7 +137,7 @@ func TestMemoryScanIsReadOnly(t *testing.T) {
home := memoryFixture(t)
home := memoryFixture(t)
before := fixtureState(t, home)
before := fixtureState(t, home)
var stdout, stderr bytes.Buffer
var stdout, stderr bytes.Buffer
- if code := cmdMemory([]string{"scan", "--user-home", home}, &stdout, &stderr); code != 0 {
+ if code := cmdMemory([]string{"scan", "--user-home", home}, strings.NewReader(""), &stdout, &stderr); code != 0 {
t.Fatalf("the scan failed with %d: %s", code, stderr.String())
t.Fatalf("the scan failed with %d: %s", code, stderr.String())
}
}
after := fixtureState(t, home)
after := fixtureState(t, home)
new file mode 100644
new file mode 100644
@@ -0,0 +1,277 @@
@@ -0,0 +1,277 @@
+package main
+
+import (
+ "bufio"
+ "context"
+ "flag"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+
+ "github.com/OpenVibers/OpenVibe.Node/internal/memory"
+ "github.com/OpenVibers/OpenVibe.Node/internal/memorystore"
+ "github.com/OpenVibers/OpenVibe.Node/internal/memoryupload"
+)
+
+// memoryUploadUsage is what "openvibe-node memory upload" prints.
+const memoryUploadUsage = "openvibe-node memory upload [--tools a,b] [--project DIR …] [--dry-run] [--yes]\n" +
+ "\n" +
+ " Reads the memory files of the coding tools on this machine and sends the\n" +
+ " items to your own memory store (OpenVibe.AI). It shows what it found and\n" +
+ " what would leave the machine first, and asks before it sends anything.\n" +
+ "\n" +
+ " --tools a,b only these adapters\n" +
+ " --project DIR read this project as well; may be given more than once\n" +
+ " --dry-run print the summary and send nothing\n" +
+ " --yes do not ask before sending\n" +
+ " --endpoint URL the store (default " + memorystore.DefaultBaseURL + ")\n" +
+ " --token-env NAME the variable the access token is read from (default " + memorystore.DefaultTokenEnv + ")\n" +
+ " --user-home DIR read this directory as the person's home (default: yours)\n"
+
+// memoryStatusUsage is what "openvibe-node memory status" prints.
+const memoryStatusUsage = "openvibe-node memory status [--show]\n" +
+ "\n" +
+ " Shows what your own memory store holds: how many items, per source and per\n" +
+ " scope. The text of an item is only printed with --show.\n" +
+ "\n" +
+ " --show print the items themselves\n" +
+ " --endpoint URL the store (default " + memorystore.DefaultBaseURL + ")\n" +
+ " --token-env NAME the variable the access token is read from (default " + memorystore.DefaultTokenEnv + ")\n"
+
+// cmdMemoryUpload sends the items of the coding tools to the person's own
+// memory store.
+func cmdMemoryUpload(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ fs := flag.NewFlagSet("memory upload", flag.ContinueOnError)
+ fs.SetOutput(stderr)
+ fs.Usage = func() { fmt.Fprint(stderr, memoryUploadUsage) }
+ var tools stringList
+ var projects stringList
+ fs.Var(&tools, "tools", "only these adapters")
+ fs.Var(&projects, "project", "read this project as well")
+ dryRun := fs.Bool("dry-run", false, "print the summary and send nothing")
+ assumeYes := fs.Bool("yes", false, "do not ask before sending")
+ endpoint := fs.String("endpoint", memorystore.DefaultBaseURL, "the memory store")
+ tokenEnv := fs.String("token-env", memorystore.DefaultTokenEnv, "the variable the access token is read from")
+ userHome := fs.String("user-home", "", "read this directory as the person's home")
+ if err := fs.Parse(args); err != nil {
+ return 2
+ }
+ if fs.NArg() > 0 {
+ fmt.Fprintf(stderr, "unknown argument %q\n\n", fs.Arg(0))
+ fmt.Fprint(stderr, memoryUploadUsage)
+ return 2
+ }
+ home, code := memoryHome(*userHome, stderr)
+ if code != 0 {
+ return code
+ }
+ res := memory.Scan(memory.Options{Home: home, Projects: projects, Tools: tools})
+ plan := memoryupload.Build(res)
+
+ printUploadSummary(plan, res, stdout)
+ if *dryRun {
+ fmt.Fprintln(stdout, "\nNothing was sent: this was a dry run.")
+ return 0
+ }
+ if len(plan.Items) == 0 {
+ fmt.Fprintln(stdout, "\nNothing to send.")
+ return 0
+ }
+ if !*assumeYes && !confirmed(stdin, stdout, len(plan.Items)) {
+ fmt.Fprintln(stdout, "\nNothing was sent.")
+ return 0
+ }
+ token := strings.TrimSpace(os.Getenv(*tokenEnv))
+ if token == "" {
+ fmt.Fprintf(stderr, "error: %s is not set: it must hold your OpenVibe access token\n", *tokenEnv)
+ return 1
+ }
+ client, err := memorystore.New(memorystore.Options{BaseURL: *endpoint, Token: token})
+ if err != nil {
+ fmt.Fprintln(stderr, "error:", err)
+ return 1
+ }
+ res2, err := client.Upload(context.Background(), plan.Items)
+ if err != nil {
+ fmt.Fprintln(stderr, "error:", err)
+ return 1
+ }
+ fmt.Fprintf(stdout, "\nSent: %d new, %d already there, %d refused.\n", res2.Created, res2.Deduplicated, res2.Refused)
+ return 0
+}
+
+// cmdMemoryStatus shows what the person's own memory store holds.
+func cmdMemoryStatus(args []string, stdout, stderr io.Writer) int {
+ fs := flag.NewFlagSet("memory status", flag.ContinueOnError)
+ fs.SetOutput(stderr)
+ fs.Usage = func() { fmt.Fprint(stderr, memoryStatusUsage) }
+ show := fs.Bool("show", false, "print the items themselves")
+ endpoint := fs.String("endpoint", memorystore.DefaultBaseURL, "the memory store")
+ tokenEnv := fs.String("token-env", memorystore.DefaultTokenEnv, "the variable the access token is read from")
+ if err := fs.Parse(args); err != nil {
+ return 2
+ }
+ if fs.NArg() > 0 {
+ fmt.Fprintf(stderr, "unknown argument %q\n\n", fs.Arg(0))
+ fmt.Fprint(stderr, memoryStatusUsage)
+ return 2
+ }
+ token := strings.TrimSpace(os.Getenv(*tokenEnv))
+ if token == "" {
+ fmt.Fprintf(stderr, "error: %s is not set: it must hold your OpenVibe access token\n", *tokenEnv)
+ return 1
+ }
+ client, err := memorystore.New(memorystore.Options{BaseURL: *endpoint, Token: token})
+ if err != nil {
+ fmt.Fprintln(stderr, "error:", err)
+ return 1
+ }
+ res, err := client.List(context.Background(), "", "")
+ if err != nil {
+ fmt.Fprintln(stderr, "error:", err)
+ return 1
+ }
+ printMemoryStatus(res, *show, stdout)
+ return 0
+}
+
+// memoryHome gives the home directory to read, and the code to return when
+// there is none.
+func memoryHome(userHome string, stderr io.Writer) (string, int) {
+ home := userHome
+ if home == "" {
+ h, err := os.UserHomeDir()
+ if err != nil {
+ fmt.Fprintln(stderr, "error: cannot find your home directory; pass --user-home")
+ return "", 1
+ }
+ home = h
+ }
+ abs, err := filepath.Abs(home)
+ if err != nil {
+ fmt.Fprintln(stderr, "error:", err)
+ return "", 1
+ }
+ return abs, 0
+}
+
+// confirmed asks the person whether the items may be sent. Anything but a
+// yes means no.
+func confirmed(stdin io.Reader, stdout io.Writer, items int) bool {
+ fmt.Fprintf(stdout, "\nSend these %d items to your memory store? [y/N] ", items)
+ reader := bufio.NewReader(stdin)
+ line, err := reader.ReadString('\n')
+ if err != nil && line == "" {
+ fmt.Fprintln(stdout)
+ return false
+ }
+ answer := strings.ToLower(strings.TrimSpace(line))
+ return answer == "y" || answer == "yes"
+}
+
+// printUploadSummary shows what the scan found and what would leave the
+// machine: every tool, every file, the items it gave, what was redacted and
+// the scopes the items go to.
+func printUploadSummary(plan memoryupload.Plan, res memory.Result, stdout io.Writer) {
+ fmt.Fprintf(stdout, "Memory upload of %s\n", res.Home)
+ if len(res.Projects) == 0 {
+ fmt.Fprintln(stdout, "No project given: only the user-level files were read.")
+ } else {
+ fmt.Fprintf(stdout, "Projects: %s\n", strings.Join(res.Projects, ", "))
+ }
+ for _, ts := range plan.Summary.Tools {
+ fmt.Fprintf(stdout, "\n%s (%s) — %d items\n", ts.Name, ts.ID, ts.Items)
+ if ts.Items == 0 {
+ // The tool was looked at and gave nothing: its paths are noise here.
+ continue
+ }
+ for _, f := range ts.Files {
+ switch {
+ case f.Note != "":
+ fmt.Fprintf(stdout, " - %s: %s\n", relativePath(f.Path, res), f.Note)
+ case f.Found:
+ fmt.Fprintf(stdout, " - %s: %d items\n", relativePath(f.Path, res), f.Items)
+ default:
+ fmt.Fprintf(stdout, " - %s: not there\n", relativePath(f.Path, res))
+ }
+ }
+ if ts.Redacted > 0 {
+ fmt.Fprintf(stdout, " %d items redacted (%d secrets removed)\n", ts.Redacted, ts.Redactions)
+ }
+ }
+ fmt.Fprintf(stdout, "\nScopes: %s\n", strings.Join(plan.Summary.Scopes, ", "))
+ fmt.Fprintf(stdout, "%d items to send", plan.Summary.Items)
+ if plan.Summary.Redacted > 0 {
+ fmt.Fprintf(stdout, ", %d redacted (%d secrets removed)", plan.Summary.Redacted, plan.Summary.Redactions)
+ }
+ if plan.Summary.Skipped > 0 {
+ fmt.Fprintf(stdout, ", %d skipped (empty)", plan.Summary.Skipped)
+ }
+ fmt.Fprintln(stdout, ".")
+}
+
+// relativePath gives a path as the summary shows it: relative to the folder
+// it was scanned from, so the summary holds no absolute path.
+func relativePath(path string, res memory.Result) string {
+ root := res.Home
+ for _, p := range res.Projects {
+ if path == p || strings.HasPrefix(path, p+string(filepath.Separator)) {
+ root = p
+ break
+ }
+ }
+ if root == "" {
+ return path
+ }
+ rel, err := filepath.Rel(root, path)
+ if err != nil || strings.HasPrefix(rel, "..") {
+ return path
+ }
+ return filepath.ToSlash(rel)
+}
+
+// printMemoryStatus shows what the store holds: the counts per source and per
+// scope, and the items themselves only when they are asked for.
+func printMemoryStatus(res memorystore.ListResult, show bool, stdout io.Writer) {
+ bySource := map[string]int{}
+ byScope := map[string]int{}
+ for _, it := range res.Items {
+ for _, s := range strings.Split(it.Source, ",") {
+ if s = strings.TrimSpace(s); s != "" {
+ bySource[s]++
+ }
+ }
+ byScope[it.Scope]++
+ }
+ fmt.Fprintf(stdout, "Your memory store holds %d items.\n", res.Count)
+ fmt.Fprintln(stdout, "\nPer source:")
+ for _, s := range sortedKeys(bySource) {
+ fmt.Fprintf(stdout, " %s: %d\n", s, bySource[s])
+ }
+ fmt.Fprintln(stdout, "\nPer scope:")
+ for _, s := range sortedKeys(byScope) {
+ fmt.Fprintf(stdout, " %s: %d\n", s, byScope[s])
+ }
+ if !show {
+ fmt.Fprintln(stdout, "\nThe text of the items is not shown; use --show to read them.")
+ return
+ }
+ fmt.Fprintln(stdout, "\nThe items:")
+ for _, it := range res.Items {
+ fmt.Fprintf(stdout, " - [%s] %s (%s)\n", it.Scope, it.Kind, it.Source)
+ fmt.Fprintf(stdout, " %s\n", strings.TrimSpace(it.Text))
+ }
+}
+
+// sortedKeys gives the keys of a count map, sorted.
+func sortedKeys(counts map[string]int) []string {
+ out := make([]string, 0, len(counts))
+ for k := range counts {
+ out = append(out, k)
+ }
+ sort.Strings(out)
+ return out
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,306 @@
@@ -0,0 +1,306 @@
+package main
+
+import (
+ "bytes"
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "os/user"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+)
+
+// memoryStore is a fake memory store that records what it is sent.
+type memoryStore struct {
+ mu sync.Mutex
+ requests [][]byte
+ paths []string
+ answer string
+}
+
+// newMemoryStore starts a fake store.
+func newMemoryStore(t *testing.T) (*memoryStore, *httptest.Server) {
+ t.Helper()
+ ms := &memoryStore{answer: `{"items":[],"count":0,"created":0,"deduped":0}`}
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ raw, _ := io.ReadAll(r.Body)
+ ms.mu.Lock()
+ ms.requests = append(ms.requests, raw)
+ ms.paths = append(ms.paths, r.URL.Path)
+ answer := ms.answer
+ ms.mu.Unlock()
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(answer))
+ }))
+ t.Cleanup(srv.Close)
+ return ms, srv
+}
+
+// sent gives the bodies the store received.
+func (ms *memoryStore) sent() [][]byte {
+ ms.mu.Lock()
+ defer ms.mu.Unlock()
+ return append([][]byte(nil), ms.requests...)
+}
+
+// uploadFixture is the fixture home of the memory package and its project.
+func uploadFixture(t *testing.T) (string, string) {
+ t.Helper()
+ wd, err := os.Getwd()
+ if err != nil {
+ t.Fatal(err)
+ }
+ root := filepath.Join(wd, "..", "..", "internal", "memory", "testdata", "claude-code")
+ return filepath.Join(root, "home"), filepath.Join(root, "project")
+}
+
+// TestMemoryUploadDryRunSendsNothing checks that a dry run shows the summary
+// and sends nothing.
+func TestMemoryUploadDryRunSendsNothing(t *testing.T) {
+ home, project := uploadFixture(t)
+ ms, srv := newMemoryStore(t)
+ t.Setenv("OPENVIBE_TOKEN", "the-person-token")
+ var stdout, stderr bytes.Buffer
+ code := cmdMemory([]string{"upload", "--user-home", home, "--project", project, "--dry-run", "--endpoint", srv.URL},
+ strings.NewReader(""), &stdout, &stderr)
+ if code != 0 {
+ t.Fatalf("the upload failed with %d: %s", code, stderr.String())
+ }
+ if len(ms.sent()) != 0 {
+ t.Errorf("the store got %d requests, want none", len(ms.sent()))
+ }
+ out := stdout.String()
+ for _, want := range []string{
+ "Claude Code (claude-code)",
+ "items to send",
+ "Scopes:",
+ "project:project",
+ "Nothing was sent",
+ } {
+ if !strings.Contains(out, want) {
+ t.Errorf("the output has no %q: %s", want, out)
+ }
+ }
+}
+
+// TestMemoryUploadAsksBeforeSending checks that nothing is sent until the
+// person says yes.
+func TestMemoryUploadAsksBeforeSending(t *testing.T) {
+ home, project := uploadFixture(t)
+ ms, srv := newMemoryStore(t)
+ t.Setenv("OPENVIBE_TOKEN", "the-person-token")
+ var stdout, stderr bytes.Buffer
+ code := cmdMemory([]string{"upload", "--user-home", home, "--project", project, "--endpoint", srv.URL},
+ strings.NewReader("n\n"), &stdout, &stderr)
+ if code != 0 {
+ t.Fatalf("the upload failed with %d: %s", code, stderr.String())
+ }
+ if len(ms.sent()) != 0 {
+ t.Errorf("the store got %d requests after a no, want none", len(ms.sent()))
+ }
+ if !strings.Contains(stdout.String(), "Nothing was sent") {
+ t.Errorf("the output does not say that nothing was sent: %s", stdout.String())
+ }
+}
+
+// TestMemoryUploadSendsWhenThePersonSaysYes checks that a yes sends the items.
+func TestMemoryUploadSendsWhenThePersonSaysYes(t *testing.T) {
+ home, project := uploadFixture(t)
+ ms, srv := newMemoryStore(t)
+ t.Setenv("OPENVIBE_TOKEN", "the-person-token")
+ var stdout, stderr bytes.Buffer
+ code := cmdMemory([]string{"upload", "--user-home", home, "--project", project, "--endpoint", srv.URL},
+ strings.NewReader("yes\n"), &stdout, &stderr)
+ if code != 0 {
+ t.Fatalf("the upload failed with %d: %s", code, stderr.String())
+ }
+ if len(ms.sent()) != 1 {
+ t.Fatalf("the store got %d requests, want 1", len(ms.sent()))
+ }
+ if !strings.Contains(stdout.String(), "Sent:") {
+ t.Errorf("the output does not say what was sent: %s", stdout.String())
+ }
+}
+
+// TestMemoryUploadYesSkipsTheQuestion checks that --yes sends without asking.
+func TestMemoryUploadYesSkipsTheQuestion(t *testing.T) {
+ home, project := uploadFixture(t)
+ ms, srv := newMemoryStore(t)
+ t.Setenv("OPENVIBE_TOKEN", "the-person-token")
+ var stdout, stderr bytes.Buffer
+ code := cmdMemory([]string{"upload", "--user-home", home, "--project", project, "--yes", "--endpoint", srv.URL},
+ strings.NewReader(""), &stdout, &stderr)
+ if code != 0 {
+ t.Fatalf("the upload failed with %d: %s", code, stderr.String())
+ }
+ if len(ms.sent()) != 1 {
+ t.Errorf("the store got %d requests, want 1", len(ms.sent()))
+ }
+ if strings.Contains(stdout.String(), "[y/N]") {
+ t.Errorf("the output asked a question it did not need to ask: %s", stdout.String())
+ }
+}
+
+// TestMemoryUploadSendsNothingOfTheMachine checks the bodies that leave: no
+// absolute path, no hostname, no name of the person.
+func TestMemoryUploadSendsNothingOfTheMachine(t *testing.T) {
+ home, project := uploadFixture(t)
+ ms, srv := newMemoryStore(t)
+ t.Setenv("OPENVIBE_TOKEN", "the-person-token")
+ var stdout, stderr bytes.Buffer
+ code := cmdMemory([]string{"upload", "--user-home", home, "--project", project, "--yes", "--endpoint", srv.URL},
+ strings.NewReader(""), &stdout, &stderr)
+ if code != 0 {
+ t.Fatalf("the upload failed with %d: %s", code, stderr.String())
+ }
+ hostname, err := os.Hostname()
+ if err != nil {
+ hostname = ""
+ }
+ userName := ""
+ if u, err := user.Current(); err == nil {
+ userName = u.Username
+ }
+ for _, body := range ms.sent() {
+ text := string(body)
+ for _, secret := range []string{home, project, hostname, userName, os.Getenv("HOME")} {
+ if secret == "" {
+ continue
+ }
+ if strings.Contains(text, secret) {
+ t.Errorf("the request holds %q, which is a fact about the machine: %s", secret, text)
+ }
+ }
+ if strings.Contains(text, "/home/") {
+ t.Errorf("the request holds an absolute path: %s", text)
+ }
+ }
+}
+
+// TestMemoryUploadSendsTheItemsOfTheScan checks what the request holds: the
+// text, the kind, the scope, the adapter and the place the item was written.
+func TestMemoryUploadSendsTheItemsOfTheScan(t *testing.T) {
+ home, project := uploadFixture(t)
+ ms, srv := newMemoryStore(t)
+ t.Setenv("OPENVIBE_TOKEN", "the-person-token")
+ var stdout, stderr bytes.Buffer
+ code := cmdMemory([]string{"upload", "--user-home", home, "--project", project, "--yes", "--endpoint", srv.URL},
+ strings.NewReader(""), &stdout, &stderr)
+ if code != 0 {
+ t.Fatalf("the upload failed with %d: %s", code, stderr.String())
+ }
+ bodies := ms.sent()
+ if len(bodies) != 1 {
+ t.Fatalf("the store got %d requests, want 1", len(bodies))
+ }
+ var sent struct {
+ Items []struct {
+ Scope string `json:"scope"`
+ Kind string `json:"kind"`
+ Text string `json:"text"`
+ Source string `json:"source"`
+ SourceRef struct {
+ Path string `json:"path"`
+ Heading string `json:"heading"`
+ } `json:"source_ref"`
+ } `json:"items"`
+ }
+ if err := json.Unmarshal(bodies[0], &sent); err != nil {
+ t.Fatalf("the request is not JSON: %v", err)
+ }
+ if len(sent.Items) == 0 {
+ t.Fatal("the request holds no items")
+ }
+ scopes := map[string]bool{}
+ for _, it := range sent.Items {
+ scopes[it.Scope] = true
+ if it.Text == "" {
+ t.Error("an item has no text")
+ }
+ if it.Source != "claude-code" {
+ t.Errorf("an item has the source %q, want the adapter id", it.Source)
+ }
+ if it.Kind == "" {
+ t.Error("an item has no kind")
+ }
+ if strings.HasPrefix(it.SourceRef.Path, "/") {
+ t.Errorf("an item has the absolute path %q", it.SourceRef.Path)
+ }
+ }
+ if !scopes["user"] {
+ t.Error("the request holds no item of the person")
+ }
+ if !scopes["project:project"] {
+ t.Errorf("the request holds no item of the project: %v", scopes)
+ }
+}
+
+// TestMemoryUploadNeedsAToken checks that the command says which variable is
+// missing.
+func TestMemoryUploadNeedsAToken(t *testing.T) {
+ home, _ := uploadFixture(t)
+ _, srv := newMemoryStore(t)
+ t.Setenv("OPENVIBE_TOKEN", "")
+ var stdout, stderr bytes.Buffer
+ code := cmdMemory([]string{"upload", "--user-home", home, "--yes", "--endpoint", srv.URL},
+ strings.NewReader(""), &stdout, &stderr)
+ if code != 1 {
+ t.Fatalf("the upload returned %d, want 1", code)
+ }
+ if !strings.Contains(stderr.String(), "OPENVIBE_TOKEN") {
+ t.Errorf("the error does not name the variable: %s", stderr.String())
+ }
+}
+
+// TestMemoryStatusCountsWithoutTheText checks what the status shows: the
+// counts per source and per scope, and no text unless it is asked for.
+func TestMemoryStatusCountsWithoutTheText(t *testing.T) {
+ ms, srv := newMemoryStore(t)
+ ms.answer = `{"items":[{"id":"mem_1","scope":"user","kind":"fact","text":"the secret of the day","source":"claude-code","status":"active"},
+ {"id":"mem_2","scope":"project:my-project","kind":"fact","text":"another one","source":"cursor","status":"active"}],"count":2}`
+ t.Setenv("OPENVIBE_TOKEN", "the-person-token")
+ var stdout, stderr bytes.Buffer
+ code := cmdMemory([]string{"status", "--endpoint", srv.URL}, strings.NewReader(""), &stdout, &stderr)
+ if code != 0 {
+ t.Fatalf("the status failed with %d: %s", code, stderr.String())
+ }
+ out := stdout.String()
+ for _, want := range []string{"2 items", "claude-code: 1", "cursor: 1", "user: 1", "project:my-project: 1"} {
+ if !strings.Contains(out, want) {
+ t.Errorf("the output has no %q: %s", want, out)
+ }
+ }
+ if strings.Contains(out, "the secret of the day") {
+ t.Errorf("the output shows the text without --show: %s", out)
+ }
+}
+
+// TestMemoryStatusShowsTheTextWithShow checks that --show prints the items.
+func TestMemoryStatusShowsTheTextWithShow(t *testing.T) {
+ ms, srv := newMemoryStore(t)
+ ms.answer = `{"items":[{"id":"mem_1","scope":"user","kind":"fact","text":"the secret of the day","source":"claude-code","status":"active"}],"count":1}`
+ t.Setenv("OPENVIBE_TOKEN", "the-person-token")
+ var stdout, stderr bytes.Buffer
+ code := cmdMemory([]string{"status", "--show", "--endpoint", srv.URL}, strings.NewReader(""), &stdout, &stderr)
+ if code != 0 {
+ t.Fatalf("the status failed with %d: %s", code, stderr.String())
+ }
+ if !strings.Contains(stdout.String(), "the secret of the day") {
+ t.Errorf("the output does not show the item: %s", stdout.String())
+ }
+}
+
+// TestMemoryUploadUsage checks that a wrong use says how to use it.
+func TestMemoryUploadUsage(t *testing.T) {
+ var stdout, stderr bytes.Buffer
+ if code := cmdMemory([]string{"upload", "--nope"}, strings.NewReader(""), &stdout, &stderr); code != 2 {
+ t.Errorf("the command returned %d, want 2", code)
+ }
+ if !strings.Contains(stderr.String(), "memory upload") {
+ t.Errorf("the error does not say how to use it: %s", stderr.String())
+ }
+}
@@ -1,7 +1,7 @@
@@ -1,7 +1,7 @@
# Memory: the Node reads what your tools remember
# Memory: the Node reads what your tools remember
The Node reads the memory files of the coding tools on your machine and tells
The Node reads the memory files of the coding tools on your machine and tells
-you what it found. It is **read-only, local and redacted**:
+you what it found. The scan is **read-only, local and redacted**:
- it opens the files it knows about and nothing else;
- it opens the files it knows about and nothing else;
- it never writes a file and never opens a network connection;
- it never writes a file and never opens a network connection;
@@ -9,27 +9,39 @@ you what it found. It is **read-only, local and redacted**:
@@ -9,27 +9,39 @@ you what it found. It is **read-only, local and redacted**:
- it never follows a symlink out of the home folder or the project folders it
- it never follows a symlink out of the home folder or the project folders it
was given.
was given.
-This page is stage M1 of OpenVibe Memory: the readers. The store, the
-encryption, the upload and the page come later.
+The upload sends what it found to **your own memory store** (OpenVibe.AI),
+after it has shown you what would leave the machine and after you have said
+yes. The store is yours: it is sealed, only your own token opens it, and
+nothing but the items themselves is sent.
-## The command
+This page is stage M1 of OpenVibe Memory (the readers) and stage M2a (the
+upload). The page that shows your memory comes later.
+
+## The commands
```sh
```sh
openvibe-node memory scan [--tools a,b] [--project DIR …] [--json]
openvibe-node memory scan [--tools a,b] [--project DIR …] [--json]
+openvibe-node memory upload [--tools a,b] [--project DIR …] [--dry-run] [--yes]
+openvibe-node memory status [--show]
```
```
| Flag | What it does |
| Flag | What it does |
|---|---|
|---|---|
| `--tools a,b` | only these adapters (default: all) |
| `--tools a,b` | only these adapters (default: all) |
| `--project DIR` | read this project as well; may be given more than once |
| `--project DIR` | read this project as well; may be given more than once |
-| `--json` | print the items as JSON |
+| `--json` | print the items as JSON (scan) |
+| `--dry-run` | print the summary and send nothing (upload) |
+| `--yes` | do not ask before sending (upload) |
+| `--show` | print the items themselves (status) |
+| `--endpoint URL` | the memory store (default `https://ai.openvibe.services`) |
+| `--token-env NAME` | the variable the access token is read from (default `OPENVIBE_TOKEN`) |
| `--user-home DIR` | read this directory as the person's home (default: yours) |
| `--user-home DIR` | read this directory as the person's home (default: yours) |
Without `--project` only the user-level files are read: the files that hold
Without `--project` only the user-level files are read: the files that hold
your own preferences across every project. With `--project` the project files
your own preferences across every project. With `--project` the project files
are read too, and their items carry the scope `project:<absolute path>`.
are read too, and their items carry the scope `project:<absolute path>`.
-The command prints every adapter, every path it looked at and whether it was
+The scan prints every adapter, every path it looked at and whether it was
there, and how many items each file gave:
there, and how many items each file gave:
```
```
@@ -153,6 +165,112 @@ different shape, and the test proves that none of them reaches an item. The
@@ -153,6 +165,112 @@ different shape, and the test proves that none of them reaches an item. The
Claude Code fixture holds more: a key block split over two items, a key block
Claude Code fixture holds more: a key block split over two items, a key block
with no END line, and an `.env` block.
with no END line, and an `.env` block.
+## The upload
+
+```sh
+openvibe-node memory upload [--tools a,b] [--project DIR …] [--dry-run] [--yes]
+```
+
+The upload runs the same scan as `memory scan` — the same adapters, the same
+paths, the same redaction — and sends what it found to **your own memory
+store** (OpenVibe.AI).
+
+The token is your own OpenVibe access token, the same one the OpenVibe.Codes
+CLI uses. It is read from the environment (`OPENVIBE_TOKEN`, or the variable
+`--token-env` names), never from the config file, and it never appears in a
+log line, in an error or in a URL.
+
+Nothing is sent before you have seen the summary and said yes. `--dry-run`
+never sends anything at all:
+
+```
+Memory upload of /home/you
+Projects: /srv/work/demo
+
+Claude Code (claude-code) — 20 items
+ - .claude/CLAUDE.md: 3 items
+ - .claude/rules/sandbox.md: 6 items
+ - CLAUDE.md: 3 items
+ 2 items redacted (3 secrets removed)
+
+Scopes: project:demo, user
+20 items to send, 2 redacted (3 secrets removed).
+
+Send these 20 items to your memory store? [y/N]
+```
+
+### What leaves the machine
+
+| Field | What it is |
+|---|---|
+| `text` | the item, already redacted |
+| `kind` | `preference`, `fact`, `project_rule`, `procedure`, `decision` or `lesson` |
+| `scope` | `user`, or `project:<slug>`: the slug is the project's folder name, lowercased and slugged |
+| `source` | the id of the adapter the item came from (`claude-code`, …) |
+| `source_ref` | the path of the file relative to the folder it was scanned from, and the heading it was under |
+| `confidence` | `1`: it is your own file |
+
+What a coding tool wrote is a suggestion until you accept it, so the items are
+marked as written by a tool when they arrive at the store.
+
+### What never leaves the machine
+
+- **the absolute path of any file**: the path that is sent is relative to the
+ folder that was scanned (`.claude/CLAUDE.md`, not `/home/you/.claude/CLAUDE.md`);
+- **the name of the machine and the name of the person**;
+- **the value of a secret**: the redaction runs before the item is counted,
+ and the store scrubs again everything it receives;
+- **an item whose text is empty** after the redaction;
+- the files themselves, the settings of a tool and the chat history of a tool:
+ the scan never reads them.
+
+### The store
+
+The items go to `POST https://ai.openvibe.services/api/v1/memory/items:batch`
+in batches of 200, with your token as a Bearer. A store that is busy or broken
+is tried again with a wait. The store scrubs the text again, refuses an item
+that is nothing but a secret, and counts what it did:
+
+```
+Sent: 18 new, 2 already there, 0 refused.
+```
+
+An item the store would not take is counted as refused and the rest of the
+batch is sent again: nothing is dropped without being said.
+
+## The status
+
+```sh
+openvibe-node memory status [--show]
+```
+
+It shows what your own store holds: how many items there are, per source and
+per scope. The text of an item is only printed with `--show`.
+
+```
+Your memory store holds 42 items.
+
+Per source:
+ claude-code: 30
+ cursor: 12
+
+Per scope:
+ project:demo: 17
+ user: 25
+```
+
+## Deleting everything from one source
+
+The store has no soft delete: a delete removes the item. To remove everything
+that came from one adapter, ask the store to delete that source:
+
+```sh
+curl -X DELETE "https://ai.openvibe.services/api/v1/memory/items?source=claude-code" \
+ -H "Authorization: Bearer $OPENVIBE_TOKEN"
+```
+
+That removes every item of that source, for you only.
+
## The JSON
## The JSON
```json
```json
@@ -215,3 +333,16 @@ A file that was looked at but could not be used carries a `note`:
@@ -215,3 +333,16 @@ A file that was looked at but could not be used carries a `note`:
- a file over 256 KB and a binary file are skipped;
- a file over 256 KB and a binary file are skipped;
- the package does not import `net/http`: it never opens a network
- the package does not import `net/http`: it never opens a network
connection.
connection.
+
+`internal/memoryupload` and `internal/memorystore` have the tests of the
+upload:
+
+- the scope of a project is the slug of its folder name, never an absolute
+ path;
+- the path that is sent is relative to the folder that was scanned;
+- an item that is empty after the redaction never leaves the machine;
+- a dry run sends nothing, and nothing is sent before the person says yes;
+- the items go in batches of 200, and a busy store is tried again;
+- the request that leaves holds no absolute path, no hostname and no name of
+ the person;
+- the token is never in an error message.
new file mode 100644
new file mode 100644
@@ -0,0 +1,396 @@
@@ -0,0 +1,396 @@
+// Package memorystore is the client of the person's own memory store: the
+// OpenVibe.AI module behind /api/v1/memory.
+//
+// The store is scoped to one person: the person's own Network access token
+// (the audience openvibe.network) is the only thing that opens it, and every
+// query the server runs is scoped by the owner. This client sends the person's
+// token as a Bearer and nothing else about the machine: the text of an item,
+// its kind, its scope, the adapter it came from and where it was written.
+//
+// The token is a secret: it is never printed, never written to an error
+// message and never put in a URL.
+package memorystore
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "math/rand"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+ "time"
+)
+
+// DefaultBaseURL is where the store lives.
+const DefaultBaseURL = "https://ai.openvibe.services"
+
+// BatchSize is how many items one request may carry. The server takes 200.
+const BatchSize = 200
+
+// MaxAttempts is how many times a request is tried when the server is busy or
+// broken.
+const MaxAttempts = 4
+
+// MinTokenLength is the length a token must have before it is taken out of a
+// message. A shorter one would match inside every other word.
+const MinTokenLength = 8
+
+// DefaultTokenEnv is the environment variable the person's access token is
+// read from. It is the same token the OpenVibe.Codes CLI uses.
+const DefaultTokenEnv = "OPENVIBE_TOKEN"
+
+// SourceRef says where an item was written: the path of the file relative to
+// the scanned root, and the heading it was under. It never holds an absolute
+// path.
+type SourceRef struct {
+ Path string `json:"path,omitempty"`
+ Heading string `json:"heading,omitempty"`
+}
+
+// Item is one memory item on the wire.
+type Item struct {
+ // Scope is `user` or `project:<slug>`. Never an absolute path.
+ Scope string `json:"scope"`
+ // Kind is one of preference, fact, project_rule, procedure, decision, lesson.
+ Kind string `json:"kind"`
+ // Text is the item itself, already redacted.
+ Text string `json:"text"`
+ // Source is the id of the adapter that found it.
+ Source string `json:"source"`
+ // SourceRef says where it was written.
+ SourceRef *SourceRef `json:"source_ref,omitempty"`
+ // Confidence is how sure the item is, 0 to 1.
+ Confidence float64 `json:"confidence,omitempty"`
+ // NoLooks says the item was written by an AI, so it is a suggestion and
+ // may not read as a fact about the person.
+ NoLooks bool `json:"no_looks,omitempty"`
+}
+
+// BatchResult is what the store answered about an import.
+type BatchResult struct {
+ // Created is how many items were new.
+ Created int
+ // Deduplicated is how many items were already there.
+ Deduplicated int
+ // Refused is how many items the store would not take.
+ Refused int
+}
+
+// ListItem is one item of the person's own list.
+type ListItem struct {
+ ID string `json:"id"`
+ Scope string `json:"scope"`
+ Kind string `json:"kind"`
+ Text string `json:"text"`
+ Source string `json:"source"`
+ Status string `json:"status"`
+ Ref *SourceRef `json:"source_ref,omitempty"`
+}
+
+// ListResult is the person's own items.
+type ListResult struct {
+ Items []ListItem
+ Count int
+}
+
+// APIError is a refusal of the store.
+type APIError struct {
+ // Status is the HTTP status.
+ Status int
+ // Code is the problem code, e.g. `input.invalid`.
+ Code string
+ // Detail says what the store refused.
+ Detail string
+}
+
+func (e *APIError) Error() string {
+ if e.Code == "" {
+ return fmt.Sprintf("the memory store refused this with HTTP %d", e.Status)
+ }
+ return fmt.Sprintf("the memory store refused this with HTTP %d (%s): %s", e.Status, e.Code, e.Detail)
+}
+
+// Retryable says whether trying again may help: the server is busy or broken.
+func (e *APIError) Retryable() bool {
+ return e.Status == http.StatusTooManyRequests || e.Status >= 500
+}
+
+// ErrNoToken says there is no person token to open the store with.
+var ErrNoToken = errors.New("no person token: set OPENVIBE_TOKEN (or the variable --token-env names) to your OpenVibe access token")
+
+// Client is the store client of one person.
+type Client struct {
+ baseURL string
+ token string
+ http *http.Client
+ // Backoff waits before a retry. It is a field so the tests need not wait.
+ backoff func(attempt int) time.Duration
+}
+
+// Options configure a Client.
+type Options struct {
+ // BaseURL is the origin of the store. Empty means DefaultBaseURL.
+ BaseURL string
+ // Token is the person's access token.
+ Token string
+ // HTTP is the client to use. Empty means a client with a 30 s timeout.
+ HTTP *http.Client
+ // Backoff waits before a retry. Empty means an exponential wait with jitter.
+ Backoff func(attempt int) time.Duration
+}
+
+// New makes a client of the store.
+func New(opts Options) (*Client, error) {
+ base := strings.TrimSuffix(strings.TrimSpace(opts.BaseURL), "/")
+ if base == "" {
+ base = DefaultBaseURL
+ }
+ u, err := url.Parse(base)
+ if err != nil || u.Scheme == "" || u.Host == "" {
+ return nil, fmt.Errorf("the memory store address %q is not an URL", opts.BaseURL)
+ }
+ if u.Scheme != "https" && !(u.Scheme == "http" && isLoopback(u.Hostname())) {
+ return nil, fmt.Errorf("refusing to talk to the memory store over %s", u.Scheme)
+ }
+ httpClient := opts.HTTP
+ if httpClient == nil {
+ httpClient = &http.Client{Timeout: 30 * time.Second}
+ }
+ c := &Client{
+ baseURL: base,
+ token: strings.TrimSpace(opts.Token),
+ http: httpClient,
+ backoff: opts.Backoff,
+ }
+ if c.backoff == nil {
+ c.backoff = defaultBackoff
+ }
+ return c, nil
+}
+
+// defaultBackoff waits a little longer every time, with jitter, so a busy
+// store is not hammered by every machine at once.
+func defaultBackoff(attempt int) time.Duration {
+ base := 500 * time.Millisecond * time.Duration(1<<uint(attempt-1))
+ if base > 8*time.Second {
+ base = 8 * time.Second
+ }
+ return base/2 + time.Duration(rand.Int63n(int64(base/2)+1))
+}
+
+// isLoopback tells whether a host is the machine itself.
+func isLoopback(host string) bool {
+ return host == "127.0.0.1" || host == "::1" || host == "localhost"
+}
+
+// Upload sends the items to the store, in batches of 200.
+//
+// An item the store refuses is counted and the rest of the batch is sent
+// again; a store that is busy or broken is tried again with a wait. The text
+// of an item never appears in an error.
+func (c *Client) Upload(ctx context.Context, items []Item) (BatchResult, error) {
+ var out BatchResult
+ for start := 0; start < len(items); start += BatchSize {
+ end := start + BatchSize
+ if end > len(items) {
+ end = len(items)
+ }
+ res, refused, err := c.uploadChunk(ctx, items[start:end])
+ out.Created += res.Created
+ out.Deduplicated += res.Deduplicated
+ out.Refused += refused
+ if err != nil {
+ return out, err
+ }
+ }
+ return out, nil
+}
+
+// uploadChunk sends one batch and returns what the store made of it. A refused
+// item is dropped and the batch is sent again, at most a few times.
+func (c *Client) uploadChunk(ctx context.Context, chunk []Item) (BatchResult, int, error) {
+ refused := 0
+ for attempt := 0; attempt < 10 && len(chunk) > 0; attempt++ {
+ res, err := c.postBatch(ctx, chunk)
+ if err == nil {
+ res.Refused = refused
+ return res, refused, nil
+ }
+ var apiErr *APIError
+ if !errors.As(err, &apiErr) || !apiErr.Retryable() {
+ index, ok := refusedIndex(apiErr)
+ if !ok {
+ return BatchResult{}, refused, err
+ }
+ // The store named the item it would not take: drop it and send the rest.
+ refused++
+ chunk = append(chunk[:index:index], chunk[index+1:]...)
+ continue
+ }
+ if attempt == MaxAttempts-1 {
+ return BatchResult{}, refused, err
+ }
+ select {
+ case <-ctx.Done():
+ return BatchResult{}, refused, ctx.Err()
+ case <-time.After(c.backoff(attempt + 1)):
+ }
+ }
+ return BatchResult{}, refused, errors.New("the memory store refused too many items")
+}
+
+// refusedIndex finds the item the store named in its refusal.
+func refusedIndex(err *APIError) (int, bool) {
+ if err == nil {
+ return 0, false
+ }
+ const marker = "items["
+ at := strings.Index(err.Detail, marker)
+ if at < 0 {
+ return 0, false
+ }
+ rest := err.Detail[at+len(marker):]
+ end := strings.Index(rest, "]")
+ if end < 0 {
+ return 0, false
+ }
+ index, convErr := strconv.Atoi(rest[:end])
+ if convErr != nil || index < 0 {
+ return 0, false
+ }
+ return index, true
+}
+
+// batchResponse is what the store answers to an import.
+type batchResponse struct {
+ Created int `json:"created"`
+ Deduped int `json:"deduped"`
+ Count int `json:"count"`
+}
+
+// postBatch sends one batch to the store.
+func (c *Client) postBatch(ctx context.Context, chunk []Item) (BatchResult, error) {
+ body, err := json.Marshal(map[string]any{"items": chunk})
+ if err != nil {
+ return BatchResult{}, fmt.Errorf("the items cannot be read: %w", err)
+ }
+ raw, status, err := c.do(ctx, http.MethodPost, "/api/v1/memory/items:batch", body)
+ if err != nil {
+ return BatchResult{}, err
+ }
+ if status/100 != 2 {
+ return BatchResult{}, c.apiError(status, raw)
+ }
+ var res batchResponse
+ if err := json.Unmarshal(raw, &res); err != nil {
+ return BatchResult{}, errors.New("the memory store answered something that is not JSON")
+ }
+ out := BatchResult{Created: res.Created, Deduplicated: res.Deduped}
+ if out.Created == 0 && out.Deduplicated == 0 && res.Count > 0 {
+ // An older store counts the items and names them differently.
+ out.Created = res.Count
+ }
+ return out, nil
+}
+
+// List gives the person's own items.
+func (c *Client) List(ctx context.Context, scope, query string) (ListResult, error) {
+ path := "/api/v1/memory/items"
+ params := url.Values{}
+ if scope != "" {
+ params.Set("scope", scope)
+ }
+ if query != "" {
+ params.Set("q", query)
+ }
+ if len(params) > 0 {
+ path += "?" + params.Encode()
+ }
+ raw, status, err := c.do(ctx, http.MethodGet, path, nil)
+ if err != nil {
+ return ListResult{}, err
+ }
+ if status/100 != 2 {
+ return ListResult{}, c.apiError(status, raw)
+ }
+ var res struct {
+ Items []ListItem `json:"items"`
+ Count int `json:"count"`
+ }
+ if err := json.Unmarshal(raw, &res); err != nil {
+ return ListResult{}, errors.New("the memory store answered something that is not JSON")
+ }
+ return ListResult{Items: res.Items, Count: res.Count}, nil
+}
+
+// do sends one request with the person's token and gives back the body and the
+// status. A network failure is an error that names the cause, never the token.
+func (c *Client) do(ctx context.Context, method, path string, body []byte) ([]byte, int, error) {
+ if c.token == "" {
+ return nil, 0, ErrNoToken
+ }
+ var reader io.Reader
+ if body != nil {
+ reader = bytes.NewReader(body)
+ }
+ req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, reader)
+ if err != nil {
+ return nil, 0, fmt.Errorf("the request to the memory store cannot be made: %w", err)
+ }
+ req.Header.Set("Authorization", "Bearer "+c.token)
+ req.Header.Set("Accept", "application/json")
+ if body != nil {
+ req.Header.Set("Content-Type", "application/json")
+ }
+ resp, err := c.http.Do(req)
+ if err != nil {
+ return nil, 0, fmt.Errorf("the memory store could not be reached: %s", scrub(err.Error(), c.token))
+ }
+ defer resp.Body.Close()
+ raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
+ if err != nil {
+ return nil, resp.StatusCode, fmt.Errorf("the memory store answered badly: %w", err)
+ }
+ return raw, resp.StatusCode, nil
+}
+
+// problem is the RFC 9457 body of a refusal.
+type problem struct {
+ Status int `json:"status"`
+ Code string `json:"code"`
+ Title string `json:"title"`
+ Detail string `json:"detail"`
+}
+
+// apiError turns a refusal into an APIError.
+func (c *Client) apiError(status int, raw []byte) error {
+ var p problem
+ if err := json.Unmarshal(raw, &p); err != nil {
+ p.Detail = strings.TrimSpace(string(raw))
+ }
+ detail := p.Detail
+ if detail == "" {
+ detail = p.Title
+ }
+ if len(detail) > 400 {
+ detail = detail[:400]
+ }
+ return &APIError{Status: status, Code: p.Code, Detail: scrub(detail, c.token)}
+}
+
+// scrub removes the token from a text that is about to be printed.
+//
+// A short token is left alone: it would match inside every other word and
+// turn the message into nonsense. A real access token is long, and a long one
+// is worth taking out.
+func scrub(text, token string) string {
+ if len(token) < MinTokenLength {
+ return text
+ }
+ return strings.ReplaceAll(text, token, "[redacted]")
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,309 @@
@@ -0,0 +1,309 @@
+package memorystore
+
+import (
+ "context"
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+)
+
+// recorded is one request the fake store received.
+type recorded struct {
+ Method string
+ Path string
+ Auth string
+ Body []byte
+}
+
+// fakeStore is a store that answers what the test tells it to.
+type fakeStore struct {
+ mu sync.Mutex
+ requests []recorded
+ // answers is the list of answers, one per request; the last one repeats.
+ answers []func(w http.ResponseWriter, r *http.Request)
+}
+
+// newFakeStore starts a fake store.
+func newFakeStore(t *testing.T, answers ...func(w http.ResponseWriter, r *http.Request)) (*fakeStore, *httptest.Server) {
+ t.Helper()
+ fs := &fakeStore{answers: answers}
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ body := readBody(r)
+ fs.mu.Lock()
+ fs.requests = append(fs.requests, recorded{Method: r.Method, Path: r.URL.Path, Auth: r.Header.Get("Authorization"), Body: body})
+ var answer func(w http.ResponseWriter, r *http.Request)
+ if len(fs.answers) == 0 {
+ answer = okAnswer
+ } else if len(fs.requests) <= len(fs.answers) {
+ answer = fs.answers[len(fs.requests)-1]
+ } else {
+ answer = fs.answers[len(fs.answers)-1]
+ }
+ fs.mu.Unlock()
+ answer(w, r)
+ }))
+ t.Cleanup(srv.Close)
+ return fs, srv
+}
+
+// readBody reads a request body.
+func readBody(r *http.Request) []byte {
+ if r.Body == nil {
+ return nil
+ }
+ raw, _ := io.ReadAll(r.Body)
+ return raw
+}
+
+// okAnswer answers with an empty import.
+func okAnswer(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"items":[],"count":0,"created":0,"deduped":0}`))
+}
+
+// seen gives the requests the fake store received.
+func (fs *fakeStore) seen() []recorded {
+ fs.mu.Lock()
+ defer fs.mu.Unlock()
+ return append([]recorded(nil), fs.requests...)
+}
+
+// TestUploadBatchesInTwoHundred checks that the items go in batches of 200.
+func TestUploadBatchesInTwoHundred(t *testing.T) {
+ fs, srv := newFakeStore(t)
+ client, err := New(Options{BaseURL: srv.URL, Token: "person-token", Backoff: func(int) time.Duration { return 0 }})
+ if err != nil {
+ t.Fatal(err)
+ }
+ items := make([]Item, 0, 401)
+ for i := 0; i < 401; i++ {
+ items = append(items, Item{Scope: "user", Kind: "fact", Text: "item", Source: "claude-code"})
+ }
+ res, err := client.Upload(context.Background(), items)
+ if err != nil {
+ t.Fatalf("the upload failed: %v", err)
+ }
+ reqs := fs.seen()
+ if len(reqs) != 3 {
+ t.Fatalf("the store got %d requests, want 3", len(reqs))
+ }
+ for i, want := range []int{200, 200, 1} {
+ var body struct {
+ Items []Item `json:"items"`
+ }
+ if err := json.Unmarshal(reqs[i].Body, &body); err != nil {
+ t.Fatalf("the body is not JSON: %v", err)
+ }
+ if len(body.Items) != want {
+ t.Errorf("request %d carried %d items, want %d", i, len(body.Items), want)
+ }
+ }
+ if res.Created != 0 {
+ t.Errorf("the store said %d items were created, want 0", res.Created)
+ }
+}
+
+// TestUploadCountsCreatedAndDeduped checks the counts of the store's answer.
+func TestUploadCountsCreatedAndDeduped(t *testing.T) {
+ _, srv := newFakeStore(t, func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"items":[],"count":5,"created":3,"deduped":2}`))
+ })
+ client, err := New(Options{BaseURL: srv.URL, Token: "t", Backoff: func(int) time.Duration { return 0 }})
+ if err != nil {
+ t.Fatal(err)
+ }
+ res, err := client.Upload(context.Background(), []Item{{Scope: "user", Kind: "fact", Text: "x", Source: "claude-code"}})
+ if err != nil {
+ t.Fatalf("the upload failed: %v", err)
+ }
+ if res.Created != 3 || res.Deduplicated != 2 {
+ t.Errorf("the counts are %d created and %d deduped, want 3 and 2", res.Created, res.Deduplicated)
+ }
+}
+
+// TestUploadRetriesOnABusyStore checks that a 429 and a 500 are tried again.
+func TestUploadRetriesOnABusyStore(t *testing.T) {
+ fs, srv := newFakeStore(t,
+ func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusTooManyRequests) },
+ func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) },
+ okAnswer,
+ )
+ client, err := New(Options{BaseURL: srv.URL, Token: "t", Backoff: func(int) time.Duration { return 0 }})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := client.Upload(context.Background(), []Item{{Scope: "user", Kind: "fact", Text: "x", Source: "claude-code"}}); err != nil {
+ t.Fatalf("the upload failed: %v", err)
+ }
+ if len(fs.seen()) != 3 {
+ t.Errorf("the store got %d requests, want 3 (the two refusals and the answer)", len(fs.seen()))
+ }
+}
+
+// TestUploadGivesUpOnABrokenStore checks that a store that stays broken ends
+// the upload with an error.
+func TestUploadGivesUpOnABrokenStore(t *testing.T) {
+ _, srv := newFakeStore(t, func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusBadGateway) })
+ client, err := New(Options{BaseURL: srv.URL, Token: "t", Backoff: func(int) time.Duration { return 0 }})
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = client.Upload(context.Background(), []Item{{Scope: "user", Kind: "fact", Text: "x", Source: "claude-code"}})
+ if err == nil {
+ t.Fatal("the upload said it worked, but the store was broken")
+ }
+ var apiErr *APIError
+ if !asAPIError(err, &apiErr) {
+ t.Fatalf("the error is not an APIError: %v", err)
+ }
+ if !apiErr.Retryable() {
+ t.Errorf("the error is not retryable: %v", err)
+ }
+}
+
+// TestUploadDropsTheItemTheStoreRefuses checks that one refused item does not
+// stop the rest.
+func TestUploadDropsTheItemTheStoreRefuses(t *testing.T) {
+ refuse := func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/problem+json")
+ w.WriteHeader(http.StatusUnprocessableEntity)
+ _, _ = w.Write([]byte(`{"status":422,"code":"memory.secret_only","detail":"items[1] is only a secret and is refused"}`))
+ }
+ fs, srv := newFakeStore(t, refuse, okAnswer)
+ client, err := New(Options{BaseURL: srv.URL, Token: "t", Backoff: func(int) time.Duration { return 0 }})
+ if err != nil {
+ t.Fatal(err)
+ }
+ items := []Item{
+ {Scope: "user", Kind: "fact", Text: "one", Source: "claude-code"},
+ {Scope: "user", Kind: "fact", Text: "two", Source: "claude-code"},
+ }
+ res, err := client.Upload(context.Background(), items)
+ if err != nil {
+ t.Fatalf("the upload failed: %v", err)
+ }
+ if res.Refused != 1 {
+ t.Errorf("the client counted %d refused items, want 1", res.Refused)
+ }
+ reqs := fs.seen()
+ if len(reqs) != 2 {
+ t.Fatalf("the store got %d requests, want 2", len(reqs))
+ }
+ var body struct {
+ Items []Item `json:"items"`
+ }
+ if err := json.Unmarshal(reqs[1].Body, &body); err != nil {
+ t.Fatalf("the body is not JSON: %v", err)
+ }
+ if len(body.Items) != 1 || body.Items[0].Text != "one" {
+ t.Errorf("the second request carried %+v, want only the first item", body.Items)
+ }
+}
+
+// TestUploadSendsTheTokenAsABearer checks how the client authenticates.
+func TestUploadSendsTheTokenAsABearer(t *testing.T) {
+ fs, srv := newFakeStore(t)
+ client, err := New(Options{BaseURL: srv.URL, Token: "the-person-token", Backoff: func(int) time.Duration { return 0 }})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := client.Upload(context.Background(), []Item{{Scope: "user", Kind: "fact", Text: "x", Source: "claude-code"}}); err != nil {
+ t.Fatalf("the upload failed: %v", err)
+ }
+ reqs := fs.seen()
+ if len(reqs) != 1 {
+ t.Fatalf("the store got %d requests, want 1", len(reqs))
+ }
+ if reqs[0].Auth != "Bearer the-person-token" {
+ t.Errorf("the request carried %q, want the person's token", reqs[0].Auth)
+ }
+ if reqs[0].Path != "/api/v1/memory/items:batch" {
+ t.Errorf("the request went to %q, want the batch path", reqs[0].Path)
+ }
+}
+
+// TestUploadWithoutATokenSaysSo checks that a client with no token does not
+// call the store at all.
+func TestUploadWithoutATokenSaysSo(t *testing.T) {
+ fs, srv := newFakeStore(t)
+ client, err := New(Options{BaseURL: srv.URL, Token: " "})
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = client.Upload(context.Background(), []Item{{Scope: "user", Kind: "fact", Text: "x", Source: "claude-code"}})
+ if err == nil {
+ t.Fatal("the upload worked without a token")
+ }
+ if !strings.Contains(err.Error(), "OPENVIBE_TOKEN") {
+ t.Errorf("the error does not say which variable is missing: %v", err)
+ }
+ if len(fs.seen()) != 0 {
+ t.Errorf("the store got %d requests, want none", len(fs.seen()))
+ }
+}
+
+// TestTheTokenNeverAppearsInAnError checks that the token is not in the
+// message of a refusal.
+func TestTheTokenNeverAppearsInAnError(t *testing.T) {
+ _, srv := newFakeStore(t, func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/problem+json")
+ w.WriteHeader(http.StatusUnauthorized)
+ _, _ = w.Write([]byte(`{"status":401,"code":"token.invalid","detail":"the token the-person-token is not valid"}`))
+ })
+ client, err := New(Options{BaseURL: srv.URL, Token: "the-person-token", Backoff: func(int) time.Duration { return 0 }})
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = client.Upload(context.Background(), []Item{{Scope: "user", Kind: "fact", Text: "x", Source: "claude-code"}})
+ if err == nil {
+ t.Fatal("the upload worked with a refused token")
+ }
+ if strings.Contains(err.Error(), "the-person-token") {
+ t.Errorf("the error holds the token: %v", err)
+ }
+}
+
+// TestListGivesTheItems checks that the list is asked for and read.
+func TestListGivesTheItems(t *testing.T) {
+ _, srv := newFakeStore(t, func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"items":[{"id":"mem_1","scope":"user","kind":"fact","text":"hello","source":"claude-code","status":"active"}],"count":1}`))
+ })
+ client, err := New(Options{BaseURL: srv.URL, Token: "t"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ res, err := client.List(context.Background(), "", "")
+ if err != nil {
+ t.Fatalf("the list failed: %v", err)
+ }
+ if res.Count != 1 || len(res.Items) != 1 || res.Items[0].Text != "hello" {
+ t.Errorf("the list is %+v, want one item with the text hello", res)
+ }
+}
+
+// TestNewRefusesAPlainAddress checks that the client does not talk to the
+// store over plain http.
+func TestNewRefusesAPlainAddress(t *testing.T) {
+ if _, err := New(Options{BaseURL: "http://ai.example.com", Token: "t"}); err == nil {
+ t.Error("the client accepted a plain http store")
+ }
+ if _, err := New(Options{BaseURL: "https://ai.example.com", Token: "t"}); err != nil {
+ t.Errorf("the client refused an https store: %v", err)
+ }
+}
+
+// asAPIError is errors.As without the import, kept short for the tests.
+func asAPIError(err error, target **APIError) bool {
+ if e, ok := err.(*APIError); ok {
+ *target = e
+ return true
+ }
+ return false
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,196 @@
@@ -0,0 +1,196 @@
+// Package memoryupload turns a memory scan into the items that go to the
+// person's own memory store.
+//
+// The scan found the items on the machine; this package decides what may leave
+// it. Only the text of an item, its kind, its scope, the adapter it came from
+// and where it was written (relative to the scanned root) ever leave. The
+// absolute path of the file, the name of the machine and the name of the
+// person stay behind.
+package memoryupload
+
+import (
+ "path/filepath"
+ "sort"
+ "strings"
+
+ "github.com/OpenVibers/OpenVibe.Node/internal/memory"
+ "github.com/OpenVibers/OpenVibe.Node/internal/memorystore"
+)
+
+// MaxSlug is the length of a project slug. The store takes at most 64.
+const MaxSlug = 64
+
+// FileSummary says what one file of a tool gave.
+type FileSummary struct {
+ // Path is the file, as the scan found it.
+ Path string
+ // Items is how many items it gave.
+ Items int
+ // Note says why the file was skipped, when it was.
+ Note string
+ // Found is true when the file was there.
+ Found bool
+}
+
+// ToolSummary says what one tool gave.
+type ToolSummary struct {
+ // ID is the id of the adapter.
+ ID string
+ // Name is how the tool is called.
+ Name string
+ // Files are the files that were looked at.
+ Files []FileSummary
+ // Items is how many items the tool gave.
+ Items int
+ // Redacted is how many of them held a secret.
+ Redacted int
+ // Redactions is how many secrets were removed.
+ Redactions int
+}
+
+// Summary is what the person is shown before anything is sent.
+type Summary struct {
+ // Tools are the tools that were read, in the order of the scan.
+ Tools []ToolSummary
+ // Scopes are the scopes the items go to, sorted.
+ Scopes []string
+ // Items is how many items will be sent.
+ Items int
+ // Redacted is how many items held a secret.
+ Redacted int
+ // Redactions is how many secrets were removed.
+ Redactions int
+ // Skipped is how many items stay behind because they are empty.
+ Skipped int
+}
+
+// Plan is what will be sent, and what the person is shown first.
+type Plan struct {
+ // Items are the items to send, in the order of the scan.
+ Items []memorystore.Item
+ // Summary is what the person is shown.
+ Summary Summary
+}
+
+// Build turns a scan into a plan.
+//
+// An item whose text is empty after the redaction never leaves the machine, so
+// it is not in the plan and is counted as skipped. The scope of a project item
+// is the slug of the project's folder name: never an absolute path.
+func Build(res memory.Result) Plan {
+ plan := Plan{Items: []memorystore.Item{}, Summary: Summary{Tools: []ToolSummary{}}}
+ scopes := map[string]bool{}
+ redactedBySource := map[string]int{}
+ redactionsBySource := map[string]int{}
+
+ for _, ad := range res.Adapters {
+ ts := ToolSummary{ID: ad.ID, Name: ad.Name, Files: []FileSummary{}}
+ for _, f := range ad.Files {
+ ts.Files = append(ts.Files, FileSummary{Path: f.Path, Items: f.Items, Note: f.Note, Found: f.Found})
+ }
+ ts.Items = ad.Items
+ plan.Summary.Tools = append(plan.Summary.Tools, ts)
+ }
+
+ for _, it := range res.Items {
+ if it.Redacted {
+ redactedBySource[it.Source]++
+ redactionsBySource[it.Source] += it.Redactions
+ }
+ text := strings.TrimSpace(it.Text)
+ if text == "" {
+ plan.Summary.Skipped++
+ continue
+ }
+ scope, root := scopeOf(it, res)
+ scopes[scope] = true
+ kind := string(it.Kind)
+ if kind == "" {
+ kind = string(memory.KindFact)
+ }
+ item := memorystore.Item{
+ Scope: scope,
+ Kind: kind,
+ Text: text,
+ Source: it.Source,
+ Confidence: 1,
+ // What a coding tool wrote is a suggestion until the person accepts it.
+ NoLooks: true,
+ }
+ if ref := sourceRef(it, root); ref != nil {
+ item.SourceRef = ref
+ }
+ plan.Items = append(plan.Items, item)
+ }
+
+ for i := range plan.Summary.Tools {
+ plan.Summary.Tools[i].Redacted = redactedBySource[plan.Summary.Tools[i].ID]
+ plan.Summary.Tools[i].Redactions = redactionsBySource[plan.Summary.Tools[i].ID]
+ }
+ plan.Summary.Items = len(plan.Items)
+ for _, ts := range plan.Summary.Tools {
+ plan.Summary.Redacted += ts.Redacted
+ plan.Summary.Redactions += ts.Redactions
+ }
+ for scope := range scopes {
+ plan.Summary.Scopes = append(plan.Summary.Scopes, scope)
+ }
+ sort.Strings(plan.Summary.Scopes)
+ return plan
+}
+
+// scopeOf gives the scope an item goes to and the root its path is relative
+// to: the home for the person, the project folder for a project.
+func scopeOf(it memory.Item, res memory.Result) (string, string) {
+ scope := string(it.Scope)
+ if scope == "" || scope == string(memory.ScopeUser) {
+ return string(memory.ScopeUser), res.Home
+ }
+ if abs, ok := strings.CutPrefix(scope, "project:"); ok {
+ return "project:" + Slug(abs), abs
+ }
+ return string(memory.ScopeUser), res.Home
+}
+
+// sourceRef says where the item was written: the path relative to the scanned
+// root and the heading it was under. A path that stays inside the root is
+// never absolute.
+func sourceRef(it memory.Item, root string) *memorystore.SourceRef {
+ ref := &memorystore.SourceRef{Heading: it.Heading}
+ if root != "" && it.Path != "" {
+ if rel, err := filepath.Rel(root, it.Path); err == nil && !strings.HasPrefix(rel, "..") {
+ ref.Path = filepath.ToSlash(rel)
+ }
+ }
+ if ref.Path == "" && ref.Heading == "" {
+ return nil
+ }
+ return ref
+}
+
+// Slug is the name a project is known by in the store: the folder's name,
+// lowercased, with everything that is not a letter, a digit, a dot, a dash or
+// an underscore turned into a dash. It is never an absolute path.
+func Slug(absPath string) string {
+ base := filepath.Base(filepath.Clean(absPath))
+ var b strings.Builder
+ for _, r := range strings.ToLower(base) {
+ switch {
+ case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '.', r == '-', r == '_':
+ b.WriteRune(r)
+ default:
+ b.WriteRune('-')
+ }
+ }
+ slug := b.String()
+ // The store wants a slug that starts with a letter or a digit.
+ slug = strings.TrimLeft(slug, "._-")
+ if slug == "" {
+ return "project"
+ }
+ if len(slug) > MaxSlug {
+ slug = slug[:MaxSlug]
+ slug = strings.TrimRight(slug, "._-")
+ }
+ return slug
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,153 @@
@@ -0,0 +1,153 @@
+package memoryupload
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/OpenVibers/OpenVibe.Node/internal/memory"
+)
+
+// TestSlugIsTheFolderName checks the name a project is known by.
+func TestSlugIsTheFolderName(t *testing.T) {
+ cases := map[string]string{
+ "/home/you/work/My Project": "my-project",
+ "/home/you/work/demo": "demo",
+ "/home/you/work/Ünïcode": "n-code",
+ "/home/you/work/": "work",
+ "/": "project",
+ "/home/you/....": "project",
+ }
+ for path, want := range cases {
+ if got := Slug(path); got != want {
+ t.Errorf("the slug of %q is %q, want %q", path, got, want)
+ }
+ }
+}
+
+// TestSlugFitsTheStore checks that a slug is what the store accepts: it
+// starts with a letter or a digit and is not longer than 64 characters.
+func TestSlugFitsTheStore(t *testing.T) {
+ long := strings.Repeat("a", 200)
+ slug := Slug("/home/you/" + long)
+ if len(slug) > MaxSlug {
+ t.Errorf("the slug is %d characters long, want at most %d", len(slug), MaxSlug)
+ }
+ for _, r := range slug {
+ if !(r >= 'a' && r <= 'z' || r >= '0' && r <= '9' || r == '.' || r == '-' || r == '_') {
+ t.Errorf("the slug holds %q, want only letters, digits, dots, dashes and underscores", r)
+ }
+ }
+ if slug[0] == '.' || slug[0] == '-' || slug[0] == '_' {
+ t.Errorf("the slug starts with %q, want a letter or a digit", slug[0])
+ }
+}
+
+// result is a small scan with a user item, a project item and an empty one.
+func result() memory.Result {
+ return memory.Result{
+ Home: "/home/you",
+ Projects: []string{"/home/you/work/My Project"},
+ Adapters: []memory.AdapterReport{
+ {ID: "claude-code", Name: "Claude Code", Items: 3, Files: []memory.FileReport{
+ {Path: "/home/you/.claude/CLAUDE.md", Found: true, Items: 2},
+ {Path: "/home/you/work/My Project/CLAUDE.md", Found: true, Items: 1},
+ }},
+ },
+ Items: []memory.Item{
+ {Text: "Prefer tabs", Kind: memory.KindPreference, Scope: memory.ScopeUser, Source: "claude-code",
+ Path: "/home/you/.claude/CLAUDE.md", Heading: "Setup"},
+ {Text: "The build takes 12 minutes", Kind: memory.KindProjectRule, Scope: memory.ScopeProject("/home/you/work/My Project"),
+ Source: "claude-code", Path: "/home/you/work/My Project/CLAUDE.md", Heading: "Build"},
+ {Text: " ", Kind: memory.KindFact, Scope: memory.ScopeUser, Source: "claude-code", Path: "/home/you/.claude/CLAUDE.md"},
+ },
+ }
+}
+
+// TestBuildMakesTheScopes checks the scopes of the plan: the person, and a
+// project by the slug of its folder name.
+func TestBuildMakesTheScopes(t *testing.T) {
+ plan := Build(result())
+ if len(plan.Items) != 2 {
+ t.Fatalf("the plan holds %d items, want 2 (the empty one stays behind)", len(plan.Items))
+ }
+ if plan.Items[0].Scope != "user" {
+ t.Errorf("the first item has the scope %q, want user", plan.Items[0].Scope)
+ }
+ if plan.Items[1].Scope != "project:my-project" {
+ t.Errorf("the second item has the scope %q, want project:my-project", plan.Items[1].Scope)
+ }
+ if plan.Summary.Skipped != 1 {
+ t.Errorf("the plan skipped %d items, want 1", plan.Summary.Skipped)
+ }
+}
+
+// TestBuildNeverHoldsAnAbsolutePath checks that no path of the machine is in
+// the plan: the paths are relative to the folder they were scanned from.
+func TestBuildNeverHoldsAnAbsolutePath(t *testing.T) {
+ plan := Build(result())
+ if plan.Items[0].SourceRef == nil || plan.Items[0].SourceRef.Path != ".claude/CLAUDE.md" {
+ t.Errorf("the first item has the path %+v, want .claude/CLAUDE.md", plan.Items[0].SourceRef)
+ }
+ if plan.Items[1].SourceRef == nil || plan.Items[1].SourceRef.Path != "CLAUDE.md" {
+ t.Errorf("the second item has the path %+v, want CLAUDE.md", plan.Items[1].SourceRef)
+ }
+ if plan.Items[0].SourceRef.Heading != "Setup" {
+ t.Errorf("the first item has the heading %q, want Setup", plan.Items[0].SourceRef.Heading)
+ }
+ for _, item := range plan.Items {
+ if strings.Contains(item.SourceRef.Path, "/home/") {
+ t.Errorf("the path %q holds the home folder", item.SourceRef.Path)
+ }
+ }
+}
+
+// TestBuildCountsWhatWasRedacted checks the counts of the summary.
+func TestBuildCountsWhatWasRedacted(t *testing.T) {
+ res := result()
+ res.Items[0].Redacted = true
+ res.Items[0].Redactions = 2
+ plan := Build(res)
+ if plan.Summary.Redacted != 1 {
+ t.Errorf("the summary counts %d redacted items, want 1", plan.Summary.Redacted)
+ }
+ if plan.Summary.Redactions != 2 {
+ t.Errorf("the summary counts %d secrets, want 2", plan.Summary.Redactions)
+ }
+ if len(plan.Summary.Tools) != 1 || plan.Summary.Tools[0].Redacted != 1 {
+ t.Errorf("the tool of the summary counts %+v, want one redacted item", plan.Summary.Tools)
+ }
+ if plan.Summary.Items != 2 {
+ t.Errorf("the summary counts %d items to send, want 2", plan.Summary.Items)
+ }
+}
+
+// TestBuildMarksWhatAToolWrote checks that the items a coding tool wrote are
+// marked as written by an AI, so the store keeps them as suggestions.
+func TestBuildMarksWhatAToolWrote(t *testing.T) {
+ plan := Build(result())
+ for _, item := range plan.Items {
+ if !item.NoLooks {
+ t.Errorf("the item %q is not marked as written by a tool", item.Text)
+ }
+ if item.Confidence != 1 {
+ t.Errorf("the item %q has the confidence %v, want 1", item.Text, item.Confidence)
+ }
+ if item.Source != "claude-code" {
+ t.Errorf("the item %q has the source %q, want the adapter id", item.Text, item.Source)
+ }
+ }
+}
+
+// TestBuildSortsTheScopes checks that the scopes of the summary are sorted.
+func TestBuildSortsTheScopes(t *testing.T) {
+ plan := Build(result())
+ want := []string{"project:my-project", "user"}
+ if len(plan.Summary.Scopes) != len(want) {
+ t.Fatalf("the summary holds the scopes %v, want %v", plan.Summary.Scopes, want)
+ }
+ for i, scope := range want {
+ if plan.Summary.Scopes[i] != scope {
+ t.Errorf("the scopes are %v, want %v", plan.Summary.Scopes, want)
+ }
+ }
+}