openvibers/openvibe.ai Commits

Read Google AI Studio models in owner settings and daily sync (#465)

OP OpenVibers committed 2 days ago · fb31212593 · 7 files, +98 −12

* Read native Google AI Studio model lists and show owner key access

* Check pasted owner keys before storing them

* Parse pasted key check requests explicitly

Changes

ddiff --git a/server/api/console-data.js b/server/api/console-data.js
iindex 22118f7..fa05ab1 100644
---- a/server/api/console-data.js
++++ b/server/api/console-data.js
@@@ -49,6 +49,7 @@ function consoleDataRouter({ db, registry, pool, quotas, cache, runs, ops, auth,
              const hints = vault && vault.hints ? await vault.hints() : new Map();
              const rows = (await Promise.all((await registry.listProviders()).map(ops.providerView))).map((p) => ({
                  ...p, key_present: p.credentials === 'configured', preference_order: p.priority,
++                keys_url: require('../providers/pool-providers').byKey.get(p.key)?.keys_url || null,
                  key_state: p.auth_mode === 'none' ? { source: 'not_required', hint: null, set_at: null }
                      : { source: hints.has(p.key) ? 'stored' : p.credentials === 'configured' ? 'environment' : 'missing',
                          hint: hints.get(p.key)?.hint || (providerSecret(p, env) ? `…${providerSecret(p, env).slice(-4)}` : null),
@@@ -168,15 +169,18 @@ function consoleDataRouter({ db, registry, pool, quotas, cache, runs, ops, auth,
          await registry.audit(who.actor, 'provider.key_removed', 'provider', provider.key, { trace: who.trace });
          return { ok: true, provider: provider.key };
      }));
--    r.post('/api/v1/admin/console/providers/:key/check', write, wrap(async (req) => {
++    r.post('/api/v1/admin/console/providers/:key/check', json, write, wrap(async (req) => {
          const provider = await registry.getProvider(req.params.key);
          if (!provider) throw new AiError(404, 'ai.not_found', 'no such provider');
++        const keyOverride = (req.body || {}).api_key;
++        if (keyOverride != null && (typeof keyOverride !== 'string' || keyOverride.length < 8 || keyOverride.length > 400))
++            throw new AiError(422, 'ai.invalid', 'the provider key must be 8-400 characters');
          const { checkProvider } = require('../providers/check');
--        const result = await checkProvider(provider, { env, now: clock.now,
++        const result = await checkProvider(provider, { env, now: clock.now, keyOverride,
              fetchImpl: fetchImpl === globalThis.fetch ? require('../providers/guarded-fetch').guardedFetch : fetchImpl });
          // A check does not enable a provider or overwrite settings changed while its request was in flight.
          const current = await registry.getProvider(provider.key);
--        if (current.updated_at === provider.updated_at) await registry.upsertProvider({ key: provider.key, metadata: { ...current.metadata, last_check: result } }, { actor: actor(req), origin: current.origin });
++        if (!keyOverride && current.updated_at === provider.updated_at) await registry.upsertProvider({ key: provider.key, metadata: { ...current.metadata, last_check: result } }, { actor: actor(req), origin: current.origin });
          return { provider: provider.key, ...result };
      }));
      r.post('/api/v1/admin/console/providers/:key/sync', write, wrap(async (req) => {
ddiff --git a/server/owner-inputs.js b/server/owner-inputs.js
iindex 1f3f63c..349665e 100644
---- a/server/owner-inputs.js
++++ b/server/owner-inputs.js
@@@ -111,13 +111,13 @@ async function testProviderKey(provider, key, { fetchImpl }) {
      if (provider.kind === 'anthropic') {
          headers = /anthropic\.com/i.test(base) ? { 'x-api-key': key, 'anthropic-version': '2023-06-01' } : { 'x-api-key': key, Authorization: `Bearer ${key}`, 'anthropic-version': '2023-06-01' };
          if (!/\/v1$/.test(base) && /anthropic\.com/i.test(base)) url = `${base}/v1/models`;
--    } else if (provider.kind === 'gemini') {
--        url = `${base}/models?pageSize=1`;
++    } else if (provider.key === 'gemini' || provider.kind === 'gemini' || provider.metadata?.api_family === 'google') {
++        url = 'https://generativelanguage.googleapis.com/v1beta/models';
          headers = { 'x-goog-api-key': key };
      }
      if (provider.metadata && provider.metadata.models_endpoint) url = provider.metadata.models_endpoint;
      if (provider.auth_mode === 'none') headers = {};
--    else if (provider.kind !== 'anthropic' && provider.kind !== 'gemini' && provider.auth_mode === 'x-api-key') headers = { 'x-api-key': key };
++    else if (provider.kind !== 'anthropic' && provider.key !== 'gemini' && provider.kind !== 'gemini' && provider.metadata?.api_family !== 'google' && provider.auth_mode === 'x-api-key') headers = { 'x-api-key': key };
      // A configured models endpoint receives a key only on the origin the owner entered it for.
      try { if (new URL(url).origin !== new URL(base).origin) return { ok: false, message: `${name} has a model-list address on a different origin.` }; }
      catch { return { ok: false, message: `${name} has an invalid model-list address.` }; }
@@@ -125,7 +125,7 @@ async function testProviderKey(provider, key, { fetchImpl }) {
      if (r.status >= 200 && r.status < 300) {
          const list = r.json && (Array.isArray(r.json.data) ? r.json.data : Array.isArray(r.json.models) ? r.json.models : null);
          if (!list) return { ok: false, message: `${name} answered without a model list.` };
--        return { ok: true, message: `${name} accepted the key${list ? ` and lists ${list.length} model${list.length === 1 ? '' : 's'}` : ''}.` };
++        return { ok: true, message: `${name} accepted the key and lists ${list.length} model${list.length === 1 ? '' : 's'}.`, models: list.map(m => m.id || m.name).filter(x => typeof x === 'string') };
      }
      return { ok: false, message: refused(name, r) };
  }
ddiff --git a/server/providers/catalog.js b/server/providers/catalog.js
iindex c7fbda1..b403aae 100644
---- a/server/providers/catalog.js
++++ b/server/providers/catalog.js
@@@ -95,6 +95,22 @@ async function listRemote(provider, apiKey, { fetchImpl = globalThis.fetch } = {
      if (!base) throw new ProviderError('the provider has no base URL', { code: 'provider.config' });
      const endpoint = provider.metadata?.models_endpoint || `${base}/models`;
      if (new URL(endpoint).origin !== new URL(base).origin) throw new ProviderError('model list must use the provider origin', { code: 'provider.config' });
++    if (provider.key === 'gemini' || provider.metadata?.api_family === 'google') {
++        const out = [];
++        const tokens = new Set();
++        const url = new URL(endpoint);
++        for (let page = 0; page < 100; page++) {
++            const j = await getJson(url.href, { 'x-goog-api-key': apiKey }, { fetchImpl });
++            if (!Array.isArray(j.models) || j.models.some(m => !m || typeof m.name !== 'string' || !m.name.startsWith('models/')))
++                throw new ProviderError('invalid model list', { code: 'provider.catalog_invalid' });
++            out.push(...j.models.map(remoteModel));
++            if (!j.nextPageToken) return out;
++            if (typeof j.nextPageToken !== 'string' || tokens.has(j.nextPageToken) || page === 99)
++                throw new ProviderError('incomplete model list', { code: 'provider.catalog_invalid' });
++            tokens.add(j.nextPageToken);
++            url.searchParams.set('pageToken', j.nextPageToken);
++        }
++    }
      if (provider.kind === 'anthropic') {
          // Anthropic's Models API pages by after_id; its Anthropic-format look-alikes (DeepSeek, MiniMax) have no list.
          if (!/anthropic\.com/i.test(base) && !provider.metadata?.models_endpoint) return [];
@@@ -143,10 +159,14 @@ function remoteModel(m) {
      const params = Array.isArray(m.supported_parameters) ? m.supported_parameters : null;
      const type = m.type === 'language' ? 'chat' : m.type;
      return {
--        id, name: m.name && m.name !== m.id ? String(m.name).replace(/^models\//, '') : (m.display_name || id),
++        id, name: m.displayName || m.display_name || (m.name && m.name !== m.id ? String(m.name).replace(/^models\//, '') : id),
++        ...(typeof m.version === 'string' ? { version: m.version } : {}),
++        ...(Array.isArray(m.supportedGenerationMethods) ? { generationMethods: m.supportedGenerationMethods.filter(x => typeof x === 'string') } : {}),
          hints: {
              contextWindow: m.context_length || m.context_window || m.inputTokenLimit || (m.top_provider && m.top_provider.context_length),
              maxOutput: m.outputTokenLimit || (m.top_provider && m.top_provider.max_completion_tokens),
++            ...(Array.isArray(m.supportedGenerationMethods) ? { type: m.supportedGenerationMethods.includes('embedContent') && !m.supportedGenerationMethods.includes('generateContent') ? 'embedding' : 'chat' } : {}),
++            ...(m.thinking != null ? { reasoning: Boolean(m.thinking) } : {}),
              vision: Array.isArray(arch.input_modalities) ? arch.input_modalities.includes('image') : null,
              tools: params ? params.includes('tools') : null,
              reasoning: params ? params.includes('reasoning') || params.includes('include_reasoning') : null,
@@@ -178,6 +198,8 @@ async function sync({ registry, provider, apiKey, fetchImpl = globalThis.fetch,
          const uncensored = require('./pool-providers').isUncensored(provider, { model_key: m.id, metadata: prev?.metadata || {} });
          const contributor = provider.key === 'meta' && /-contributor$/i.test(m.id);
          const metadata = { tier: d.tier, reasoning: d.reasoning, note: d.note, ...prev?.metadata, synced_at: at, listed: true,
++            ...(m.version ? { version: m.version } : {}),
++            ...(m.generationMethods ? { generation_methods: m.generationMethods } : {}),
              ...(contributor ? { routing_excluded: true, routing_excluded_reason: 'Meta Contributor mode permits training on prompts and is not allowed in OpenVibe routing.' } : {}),
              ...(uncensored ? { uncensored: true, min_age: 18 } : {}),
              ...(extra.source ? { catalogue_enrichment: { source: extra.source, checked_at: extra.checked_at } } : {}) };
ddiff --git a/server/providers/check.js b/server/providers/check.js
iindex 9a98c9d..00a15dd 100644
---- a/server/providers/check.js
++++ b/server/providers/check.js
@@@ -5,15 +5,17 @@ const { testProviderKey } = require('../owner-inputs');
  const { guardedFetch } = require('./guarded-fetch');
  
  /** One explicit owner check, with no inference, retries, redirects or fallback. Never retain upstream text. */
--async function checkProvider(provider, { env = process.env, fetchImpl = guardedFetch, now = Date.now } = {}) {
++async function checkProvider(provider, { env = process.env, fetchImpl = guardedFetch, now = Date.now, keyOverride = null } = {}) {
      const started = now();
      let calls = 0;
      let httpStatus = null;
++    let models = [];
      const finish = (ok, error = null, method = null) => ({
          ok, error, method, http_status: httpStatus, requests: calls,
++        models_count: models.length, models: models.filter(id => !String(id).includes(key)),
          latency_ms: Math.max(0, now() - started), checked_at: new Date(started).toISOString(),
      });
--    const key = providerSecret(provider, env);
++    const key = keyOverride || providerSecret(provider, env);
      if (provider.auth_mode !== 'none' && !key) return finish(false, 'key_not_set');
      if (!provider.base_url) return finish(false, 'base_url_not_set');
      let origin;
@@@ -58,7 +60,25 @@ async function checkProvider(provider, { env = process.env, fetchImpl = guardedF
              });
              return finish(true, null, method);
          }
++        if (provider.key === 'gemini' || provider.metadata?.api_family === 'google') {
++            const { listRemote } = require('./catalog');
++            // Google pages its native list; every page uses the bound origin and key header.
++            const remote = await listRemote(provider, key, { fetchImpl: async (url, init) => {
++                const target = new URL(String(url));
++                if (target.origin !== origin || target.protocol !== 'https:' || target.username || target.password || calls >= 100) {
++                    failure = 'endpoint_origin_mismatch'; throw new Error(failure);
++                }
++                calls++;
++                const response = await fetchImpl(url, { ...init, redirect: 'error', signal: AbortSignal.timeout(10000) });
++                httpStatus = response.status;
++                if (!response.ok) failure = `http_${response.status}`;
++                return response;
++            } });
++            models = remote.map(m => m.id);
++            return finish(true, null, method);
++        }
          const result = await testProviderKey(provider, key, { fetchImpl: once });
++        models = result.models || [];
          return finish(result.ok, result.ok ? null : httpStatus >= 200 && httpStatus < 300 ? 'invalid_response' : failure, method);
      } catch {
          return finish(false, failure, method);
ddiff --git a/server/providers/pool-providers.js b/server/providers/pool-providers.js
iindex ca74433..8ca899b 100644
---- a/server/providers/pool-providers.js
++++ b/server/providers/pool-providers.js
@@@ -742,7 +742,8 @@ const PROVIDERS = [
          docs_url: 'https://platform.claude.com/docs/en/api/overview', capabilities: ['chat', 'messages', 'vision', 'tools'] },
      { key: 'openai', keys_url: 'https://platform.openai.com/api-keys', display_name: 'OpenAI', base_url: 'https://api.openai.com/v1', default_model: 'gpt-5-mini',
          docs_url: 'https://platform.openai.com/docs/api-reference/models/list', capabilities: ['chat', 'vision', 'tools', 'image', 'video', 'speech', 'transcribe', 'embed'] },
--    { key: 'gemini', models_dev_id: 'google', keys_url: 'https://aistudio.google.com/apikey', display_name: 'Google Gemini', base_url: 'https://generativelanguage.googleapis.com/v1beta/openai',
++    { key: 'gemini', models_dev_id: 'google', keys_url: 'https://aistudio.google.com/apikey', display_name: 'Google AI Studio (Gemini API)', base_url: 'https://generativelanguage.googleapis.com/v1beta/openai',
++        models_endpoint: 'https://generativelanguage.googleapis.com/v1beta/models',
          api_family: 'google', docs_url: 'https://ai.google.dev/gemini-api/docs/openai', capabilities: ['chat', 'vision', 'tools', 'embed'] },
      { key: 'xai', keys_url: 'https://console.x.ai/', display_name: 'xAI', base_url: 'https://api.x.ai/v1',
          docs_url: 'https://docs.x.ai/docs/api-reference', capabilities: ['chat', 'vision', 'tools', 'image', 'video'] },
ddiff --git a/server/providers/pool-seed.js b/server/providers/pool-seed.js
iindex 7e4bad8..4c0d89c 100644
---- a/server/providers/pool-seed.js
++++ b/server/providers/pool-seed.js
@@@ -29,6 +29,7 @@ function recordOf(p) {
          metadata: {
              catalog: 'pool-providers',
              api_family: p.api_family, models_endpoint: p.models_endpoint,
++            model_sync_interval_minutes: p.models_endpoint ? 1440 : 0,
              check_endpoint: p.check_endpoint || null,
              models_public: p.models_public === true, models_dev_id: p.models_dev_id || null,
              auth_header: p.auth_header || null, docs_url: p.docs_url || null,
@@@ -111,6 +112,8 @@ async function seedPoolProviders({ registry, actor = 'system', env = process.env
                  defaults.metadata.check_endpoint = null;
              }
              const metadata = { ...defaults.metadata, ...prev.metadata, search_adapters: [...new Set([...(prev.metadata.search_adapters || []), ...(defaults.metadata.search_adapters || [])])] };
++            if (p.key === 'gemini' && prev.base_url === p.base_url && prev.metadata.models_endpoint === `${p.base_url}/models`)
++                metadata.models_endpoint = defaults.metadata.models_endpoint;
              if (!configured && prev.base_url !== p.base_url) {
                  metadata.models_endpoint = defaults.metadata.models_endpoint;
                  metadata.api_family = defaults.metadata.api_family;
@@@ -132,7 +135,9 @@ async function seedPoolProviders({ registry, actor = 'system', env = process.env
                  protocol.models_endpoint = prev.base_url ? `${prev.base_url.replace(/\/+$/, '')}/models` : null;
                  protocol.check_endpoint = null;
              }
--            const additions = Object.fromEntries(Object.entries(protocol).filter(([k]) => ['api_family', 'models_endpoint', 'check_endpoint', 'models_public', 'models_dev_id', 'auth_header', 'docs_url', 'search_adapters'].includes(k) && prev.metadata[k] === undefined));
++            const additions = Object.fromEntries(Object.entries(protocol).filter(([k]) => ['api_family', 'models_endpoint', 'model_sync_interval_minutes', 'check_endpoint', 'models_public', 'models_dev_id', 'auth_header', 'docs_url', 'search_adapters'].includes(k) && prev.metadata[k] === undefined));
++            if (p.key === 'gemini' && prev.base_url === p.base_url && prev.metadata.models_endpoint === `${p.base_url}/models`)
++                additions.models_endpoint = protocol.models_endpoint;
              const engines = [...new Set([...(prev.metadata.search_adapters || []), ...(protocol.search_adapters || [])])];
              if (engines.length !== (prev.metadata.search_adapters || []).length) additions.search_adapters = engines;
              if (Object.keys(additions).length) await registry.upsertProvider({ key: p.key, metadata: { ...prev.metadata, ...additions } }, { actor, origin: prev.origin });
ddiff --git a/test/provider-check.test.js b/test/provider-check.test.js
iindex f30b06c..9c52905 100644
---- a/test/provider-check.test.js
++++ b/test/provider-check.test.js
@@@ -32,6 +32,30 @@ t.test('a model-list check is one free read at the registered endpoint with late
      assert.strictEqual(result.latency_ms, 25);
  });
  
++t.test('AI Studio check and sync read every native Gemini page with the Google header', async () => {
++    const { listRemote } = require('../server/providers/catalog');
++    const gemini = { key: 'gemini', kind: 'openai', auth_mode: 'bearer', base_url: 'https://generativelanguage.googleapis.com/v1beta/openai',
++        metadata: { api_family: 'google', models_endpoint: 'https://generativelanguage.googleapis.com/v1beta/models' }, secret_ref: 'env:CHECK_KEY' };
++    const seen = [];
++    const fetchImpl = async (url, init) => {
++        seen.push(String(url));
++        assert.strictEqual(init.headers['x-goog-api-key'], key);
++        assert.strictEqual(init.headers.Authorization, undefined);
++        return response(200, seen.length % 2 ? { models: [{ name: 'models/gemini-a', displayName: 'Gemini A', version: 'a-1', inputTokenLimit: 1000, outputTokenLimit: 100, supportedGenerationMethods: ['generateContent'] }], nextPageToken: 'next' }
++            : { models: [{ name: 'models/gemini-b', displayName: 'Gemini B', supportedGenerationMethods: ['generateContent'] }] });
++    };
++    const checked = await checkProvider(gemini, { env, fetchImpl });
++    assert.strictEqual(checked.ok, true);
++    assert.strictEqual(checked.models_count, 2);
++    assert.deepStrictEqual(checked.models, ['gemini-a', 'gemini-b']);
++    assert.strictEqual(checked.requests, 2);
++    seen.length = 0;
++    const remote = await listRemote(gemini, key, { fetchImpl });
++    assert.strictEqual(remote[0].name, 'Gemini A');
++    assert.strictEqual(remote[0].hints.contextWindow, 1000);
++    assert.deepStrictEqual(remote.map(m => m.id), checked.models);
++});
++
  t.test('upstream refusals and thrown errors cannot echo the key; no retries', async () => {
      for (const failure of [401, 429, 500, 'throw']) {
          let calls = 0;
@@@ -125,6 +149,16 @@ t.test('the check route makes one request and records only its safe result', asy
      assert.ok(!JSON.stringify(stored.metadata).includes(replacement));
  });
  
++t.test('a pasted key can be checked before save without changing the stored credential', async () => {
++    const result = await api('POST', `${path}/check`, { api_key: key });
++    assert.strictEqual(result.status, 200, result.text);
++    assert.strictEqual(result.body.ok, true);
++    assert.strictEqual(authUsed, `Bearer ${key}`);
++    assert.strictEqual(providerSecret(await h.registry.getProvider('keycheck'), h.env), replacement);
++    assert.ok(!result.text.includes(key));
++    assert.ok(!JSON.stringify((await h.registry.getProvider('keycheck')).metadata).includes(key));
++});
++
  t.test('a service without owner capabilities gets 403 for every key operation', async () => {
      const before = requests;
      for (const [method, suffix] of [['POST', 'key'], ['DELETE', 'key'], ['POST', 'check']]) {