openvibers/openvibe.bot Commits
The sitemap leaves out /install, a 302 to the installer script and not a page (#141)
Changes
@@@ -51,8 +51,9 @@ const TRY_PROFILE = loadProfiles().get('sim.rover');// The tagline stated for openvibe.bot (openvibe-contracts manifests/services/bot.json site.tagline).const SITE_TAGLINE = 'An open control panel for robots';// Every public GET page, for the sitemap. The interactive pages (/robots, /pair/, /panel/) need a session and--// are Disallowed in robots.txt; the API and sign-in are not pages.--const PUBLIC_PAGES = ['/', '/try', '/robots', '/models', '/install', '/docs', '/docs/drivers', '/docs/profiles', '/docs/automation', '/docs/computers', '/updates', '/terms', '/privacy', '/dmca'];++// are Disallowed in robots.txt; the API and sign-in are not pages, and neither is /install (a 302 to the installer++// script: a sitemap that lists a redirect wastes the crawl on it).++const PUBLIC_PAGES = ['/', '/try', '/robots', '/models', '/docs', '/docs/drivers', '/docs/profiles', '/docs/automation', '/docs/computers', '/updates', '/terms', '/privacy', '/dmca'];// The public pages an AI crawler can read, for /llms.txt and /llms-full.txt (llmstxt.org). Same pages as the// sitemap minus /install (a 302 to the installer script, not a page); no signed-in or per-robot page, nothing// under the API or /internal. `note` is the one-line llms.txt entry, `text` the page's full text in llms-full.txt,@@@ -42,10 +42,11 @@ const { boot, check, done } = require('./helpers/app');assert.strictEqual(r.status, 200);assert.match(r.headers.get('content-type'), /xml/);const xml = await r.text();-- for (const p of ['/', '/install', '/terms', '/privacy', '/dmca']) {++ for (const p of ['/', '/terms', '/privacy', '/dmca']) {assert.ok(xml.includes(`<loc>http://bot.test${p}</loc>`), `sitemap lists ${p}`);}-- assert.ok(assertSitemapLastmods(xml).length >= 5);++ assert.ok(!xml.includes('<loc>http://bot.test/install</loc>'), 'not /install, a redirect to the installer script');++ assert.ok(assertSitemapLastmods(xml).length >= 4);});await check('GET /favicon.ico: the app icon as SVG (browsers ask for it even on a 404 page)', async () => {
@@ -51,8 +51,9 @@ const TRY_PROFILE = loadProfiles().get('sim.rover');
@@ -51,8 +51,9 @@ const TRY_PROFILE = loadProfiles().get('sim.rover');
// The tagline stated for openvibe.bot (openvibe-contracts manifests/services/bot.json site.tagline).
// The tagline stated for openvibe.bot (openvibe-contracts manifests/services/bot.json site.tagline).
const SITE_TAGLINE = 'An open control panel for robots';
const SITE_TAGLINE = 'An open control panel for robots';
// Every public GET page, for the sitemap. The interactive pages (/robots, /pair/, /panel/) need a session and
// Every public GET page, for the sitemap. The interactive pages (/robots, /pair/, /panel/) need a session and
-// are Disallowed in robots.txt; the API and sign-in are not pages.
-const PUBLIC_PAGES = ['/', '/try', '/robots', '/models', '/install', '/docs', '/docs/drivers', '/docs/profiles', '/docs/automation', '/docs/computers', '/updates', '/terms', '/privacy', '/dmca'];
+// are Disallowed in robots.txt; the API and sign-in are not pages, and neither is /install (a 302 to the installer
+// script: a sitemap that lists a redirect wastes the crawl on it).
+const PUBLIC_PAGES = ['/', '/try', '/robots', '/models', '/docs', '/docs/drivers', '/docs/profiles', '/docs/automation', '/docs/computers', '/updates', '/terms', '/privacy', '/dmca'];
// The public pages an AI crawler can read, for /llms.txt and /llms-full.txt (llmstxt.org). Same pages as the
// The public pages an AI crawler can read, for /llms.txt and /llms-full.txt (llmstxt.org). Same pages as the
// sitemap minus /install (a 302 to the installer script, not a page); no signed-in or per-robot page, nothing
// sitemap minus /install (a 302 to the installer script, not a page); no signed-in or per-robot page, nothing
// under the API or /internal. `note` is the one-line llms.txt entry, `text` the page's full text in llms-full.txt,
// under the API or /internal. `note` is the one-line llms.txt entry, `text` the page's full text in llms-full.txt,
@@ -42,10 +42,11 @@ const { boot, check, done } = require('./helpers/app');
@@ -42,10 +42,11 @@ const { boot, check, done } = require('./helpers/app');
assert.strictEqual(r.status, 200);
assert.strictEqual(r.status, 200);
assert.match(r.headers.get('content-type'), /xml/);
assert.match(r.headers.get('content-type'), /xml/);
const xml = await r.text();
const xml = await r.text();
- for (const p of ['/', '/install', '/terms', '/privacy', '/dmca']) {
+ for (const p of ['/', '/terms', '/privacy', '/dmca']) {
assert.ok(xml.includes(`<loc>http://bot.test${p}</loc>`), `sitemap lists ${p}`);
assert.ok(xml.includes(`<loc>http://bot.test${p}</loc>`), `sitemap lists ${p}`);
}
}
- assert.ok(assertSitemapLastmods(xml).length >= 5);
+ assert.ok(!xml.includes('<loc>http://bot.test/install</loc>'), 'not /install, a redirect to the installer script');
+ assert.ok(assertSitemapLastmods(xml).length >= 4);
});
});
await check('GET /favicon.ico: the app icon as SVG (browsers ask for it even on a 404 page)', async () => {
await check('GET /favicon.ico: the app icon as SVG (browsers ask for it even on a 404 page)', async () => {