openvibers/openvibe.connect Commits
/sitemap.xml: the front door, named by robots.txt (#30)
Changes
@@@ -305,8 +305,14 @@ function createPageRoutes(ctx) {}});++ const site = String(config.baseUrl).replace(/\/+$/, '');r.get('/robots.txt', (req, res) => {-- res.type('text/plain').set('Cache-Control', 'public, max-age=3600').send(`User-agent: *\nAllow: /$\nDisallow: /shares\nDisallow: /grant\nDisallow: /auth\nDisallow: /api\n`);++ res.type('text/plain').set('Cache-Control', 'public, max-age=3600').send(`User-agent: *\nAllow: /$\nDisallow: /shares\nDisallow: /grant\nDisallow: /auth\nDisallow: /api\n\nSitemap: ${site}/sitemap.xml\n`);++ });++ // GET /sitemap.xml: the one public page (the front door). agent.json names /sitemap.xml, which answered 404 (11-03).++ r.get('/sitemap.xml', (req, res) => {++ res.type('application/xml').set('Cache-Control', 'public, max-age=3600')++ .send(`<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n <url><loc>${site}/</loc></url>\n</urlset>\n`);});// What llms.txt says (SEO-33: /llms-full.txt adds the README, Connect's full documentation, after it).const LLMS_LINES = [@@@ -99,6 +99,10 @@ const { check, done } = require('./helpers/check');await check('robots, llms.txt and the client download', async () => {assert.match((await t.get('/robots.txt')).text, /Disallow: \/grant/);++ assert.match((await t.get('/robots.txt')).text, /\nSitemap: https?:\/\/[^\n]+\/sitemap\.xml\n/, 'robots names the sitemap');++ const map = await t.get('/sitemap.xml');++ assert.strictEqual(map.status, 200, 'the sitemap agent.json names answers');++ assert.match(map.text, /<urlset xmlns="http:\/\/www\.sitemaps\.org\/schemas\/sitemap\/0\.9">\n <url><loc>https?:\/\/[^<]+\/<\/loc><\/url>\n<\/urlset>/);assert.match((await t.get('/llms.txt')).text, /OpenVibe Connect/);// SEO-2: the agent manifest is served (11-03: the mount had landed inside the share-host middleware, after its// return, so it never ran while the source check passed).
@@ -305,8 +305,14 @@ function createPageRoutes(ctx) {
@@ -305,8 +305,14 @@ function createPageRoutes(ctx) {
}
}
});
});
+ const site = String(config.baseUrl).replace(/\/+$/, '');
r.get('/robots.txt', (req, res) => {
r.get('/robots.txt', (req, res) => {
- res.type('text/plain').set('Cache-Control', 'public, max-age=3600').send(`User-agent: *\nAllow: /$\nDisallow: /shares\nDisallow: /grant\nDisallow: /auth\nDisallow: /api\n`);
+ res.type('text/plain').set('Cache-Control', 'public, max-age=3600').send(`User-agent: *\nAllow: /$\nDisallow: /shares\nDisallow: /grant\nDisallow: /auth\nDisallow: /api\n\nSitemap: ${site}/sitemap.xml\n`);
+ });
+ // GET /sitemap.xml: the one public page (the front door). agent.json names /sitemap.xml, which answered 404 (11-03).
+ r.get('/sitemap.xml', (req, res) => {
+ res.type('application/xml').set('Cache-Control', 'public, max-age=3600')
+ .send(`<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n <url><loc>${site}/</loc></url>\n</urlset>\n`);
});
});
// What llms.txt says (SEO-33: /llms-full.txt adds the README, Connect's full documentation, after it).
// What llms.txt says (SEO-33: /llms-full.txt adds the README, Connect's full documentation, after it).
const LLMS_LINES = [
const LLMS_LINES = [
@@ -99,6 +99,10 @@ const { check, done } = require('./helpers/check');
@@ -99,6 +99,10 @@ const { check, done } = require('./helpers/check');
await check('robots, llms.txt and the client download', async () => {
await check('robots, llms.txt and the client download', async () => {
assert.match((await t.get('/robots.txt')).text, /Disallow: \/grant/);
assert.match((await t.get('/robots.txt')).text, /Disallow: \/grant/);
+ assert.match((await t.get('/robots.txt')).text, /\nSitemap: https?:\/\/[^\n]+\/sitemap\.xml\n/, 'robots names the sitemap');
+ const map = await t.get('/sitemap.xml');
+ assert.strictEqual(map.status, 200, 'the sitemap agent.json names answers');
+ assert.match(map.text, /<urlset xmlns="http:\/\/www\.sitemaps\.org\/schemas\/sitemap\/0\.9">\n <url><loc>https?:\/\/[^<]+\/<\/loc><\/url>\n<\/urlset>/);
assert.match((await t.get('/llms.txt')).text, /OpenVibe Connect/);
assert.match((await t.get('/llms.txt')).text, /OpenVibe Connect/);
// SEO-2: the agent manifest is served (11-03: the mount had landed inside the share-host middleware, after its
// SEO-2: the agent manifest is served (11-03: the mount had landed inside the share-host middleware, after its
// return, so it never ran while the source check passed).
// return, so it never ran while the source check passed).