openvibers/openvibe.media Commits
Recordings are virtual too: one rule (readiness isVirtual/virtualSql) for every object with no bytes of its own, so the verification job, the no-good-copy report and playableSql skip a recording over its stream's parts as they skip a virtual clip (three held /api/ready at degraded) (#202)
Changes
@@@ -3,7 +3,8 @@** A ready object has NO GOOD COPY when none of its locations is `present` (verified there) or* `pending` (not verified yet, so possibly fine). That covers objects whose every copy was found-- * missing/corrupt, and ready objects with no location row at all, except virtual clips (a window over their VOD).++ * missing/corrupt, and ready objects with no location row at all, except virtual objects (readiness.js isVirtual:++ * a clip over its VOD, a recording over its stream's parts).** Pure SQL over a small `q` interface — { all(sql, params), get(sql, params) } — so the service* (server/db/database.js), the metrics/readiness checks and the read-only report script@@@ -12,12 +13,13 @@*/'use strict';--// A virtual clip (clip-jobs makeVirtual: a window over its VOD's timeline) has no copy of its own by design: its bytes--// are its VOD's, whose object is checked on its own. The same test as verify-job's selection; five of them held--// /api/ready at "degraded" (2026-10-17).++const { virtualSql } = require('./readiness');++++// A virtual object has no copy of its own by design: its bytes are its sources', each checked on its own. The same++// test as verify-job's selection (readiness.js virtualSql).const NO_GOOD_COPY_WHERE = `o.lifecycle_status = 'ready'AND NOT EXISTS (SELECT 1 FROM media_locations l WHERE l.object_id = o.id AND l.state IN ('present', 'pending'))-- AND NOT (o.kind = 'clip' AND (CASE WHEN json_valid(o.metadata) THEN json_extract(o.metadata, '$.virtual') END) IS NOT NULL)`;++ AND NOT ${virtualSql('o')}`;async function countNoGoodCopy(q, { appId = null } = {}) {const r = await q.get(`SELECT COUNT(*) AS n FROM media_objects o WHERE ${NO_GOOD_COPY_WHERE}${appId ? ' AND o.app_id = ?' : ''}`, appId ? [appId] : []);@@@ -42,18 +42,30 @@ function isVerifiedCopy(obj, l) {/** A ready clip object with no file of its own, played as a window of its source's timeline (model.js clipProjection). */function isVirtualClip(obj, md = metadataOf(obj)) {-- return obj.kind === 'clip' && obj.lifecycle_status === 'ready' && !!md && md.virtual === true;++ return obj.kind === 'clip' && isVirtual(obj, md);}++/**++ * A ready object with no bytes of its own, of any kind: a virtual clip (a window over its VOD's timeline) or a++ * recording (an asset over its stream's parts, docs/recordings.md). Its bytes are other objects' segments, each++ * checked as its own object, so it never has a copy to verify. virtualSql() is the same test in SQL, shared by the++ * verification job's selection, the "no good copy" report and playableSql(): recordings were not in it, and three++ * held /api/ready at "degraded" (2026-10-28) as the five virtual clips had (2026-10-17).++ */++function isVirtual(obj, md = metadataOf(obj)) {++ return obj.lifecycle_status === 'ready' && !!md && md.virtual === true;++}++const virtualSql = (alias) => `(CASE WHEN json_valid(${alias}.metadata) THEN json_extract(${alias}.metadata, '$.virtual') END) IS NOT NULL`;++/** Readiness of one object row (null → nothing known). `locations` may be passed when already loaded. */async function compute(obj, locations = null) {if (!obj) return { metadata: false, bytes_verified: false, playable: false, hash_verified: false, verified_copies: [], reason: 'no_object' };const locs = locations || await db.all('SELECT provider, state, checksum, verified_at FROM media_locations WHERE object_id = ? ORDER BY id', [obj.id]);const md = metadataOf(obj);const verified = locs.filter(l => isVerifiedCopy(obj, l));-- // A virtual clip (F3.5) has no copy of its own: its bytes are its source's timeline segments, served under the-- // source's rules, so a ready one counts as having its bytes.-- const bytesVerified = verified.length > 0 || isVirtualClip(obj, md);++ // A virtual object (a clip over its VOD, a recording over its stream's parts) has no copy of its own: its bytes are++ // its sources' timeline segments, served under their rules, so a ready one counts as having its bytes.++ const bytesVerified = verified.length > 0 || isVirtual(obj, md);let reason = null;if (obj.lifecycle_status === 'deleted') reason = 'deleted';else if (!md) reason = 'metadata_unreadable';@@@ -84,7 +96,7 @@ function playableSql(ref) {AND ro.kind IN (${PLAYBACK_KINDS.map(k => `'${k}'`).join(', ')}) AND (CASE WHEN json_valid(ro.metadata) THEN json_type(ro.metadata) END) = 'object'AND (EXISTS (SELECT 1 FROM media_locations rl WHERE rl.object_id = ro.id AND rl.state = 'present' AND rl.verified_at IS NOT NULLAND (rl.checksum IS NULL OR ro.content_hash IS NULL OR rl.checksum = ro.content_hash))-- OR (ro.kind = 'clip' AND (CASE WHEN json_valid(ro.metadata) THEN json_extract(ro.metadata, '$.virtual') END) IS NOT NULL)))`;++ OR ${virtualSql('ro')}))`;}--module.exports = { compute, forRow, playableSql, isVerifiedCopy, isVirtualClip, PLAYBACK_KINDS, REASONS };++module.exports = { compute, forRow, playableSql, isVerifiedCopy, isVirtualClip, isVirtual, virtualSql, PLAYBACK_KINDS, REASONS };@@@ -24,6 +24,7 @@ const crypto = require('crypto');const db = require('../db/database');const config = require('../config');const copyReport = require('./copy-report');++const { virtualSql } = require('./readiness');const MB = 1024 * 1024;// Every remote tier the fabric knows — the canonical list lives in vod-storage (REMOTE_PROVIDERS);@@@ -157,7 +158,7 @@ async function runOnce({ batch = config.verify.batch, head = defaultHead, uploadconst n = Math.max(1, Number(batch) || 1);const objects = await db.all(`SELECT o.* FROM media_objects o LEFT JOIN media_verifications v ON v.object_id = o.idWHERE o.lifecycle_status = 'ready'-- AND NOT (o.kind = 'clip' AND (CASE WHEN json_valid(o.metadata) THEN json_extract(o.metadata, '$.virtual') END) IS NOT NULL)++ AND NOT ${virtualSql('o')}ORDER BY (v.verified_at IS NOT NULL), v.verified_at, o.id LIMIT ?`, [n]);const budget = { left: Math.max(0, Number(maxReuploads) || 0) };for (const obj of objects) {@@@ -160,7 +160,10 @@ const crypto = require('crypto');// A virtual clip (clip-jobs makeVirtual) has no copy of its own by design: its bytes are its VOD's.await model.createObject({ app_id: 'live', kind: 'clip', lifecycle_status: 'ready', size_bytes: 0,owner_user_id: 7, owner_subject: 'usr_01JAB2C3D4E5F6G7H8J9K0MNPQ', mime_type: 'video/mp4', metadata: { virtual: true } });-- assert.strictEqual(await copyReport.countNoGoodCopy(db), 3, 'c (remote corrupt), d (local gone), h (no locations); not the virtual clip');++ // Nor has a recording (docs/recordings.md: an asset over its stream's parts, an HLS playlist of their segments).++ await model.createObject({ app_id: 'live', kind: 'asset', lifecycle_status: 'ready', size_bytes: 0,++ owner_user_id: 7, owner_subject: 'usr_01JAB2C3D4E5F6G7H8J9K0MNPQ', mime_type: 'application/vnd.apple.mpegurl', metadata: { recording: true, virtual: true, group_key: 'stream:1' } });++ assert.strictEqual(await copyReport.countNoGoodCopy(db), 3, 'c (remote corrupt), d (local gone), h (no locations); not the virtual clip or the recording');const st = await job.status();assert.strictEqual(st.no_good_copy, 3);assert.strictEqual(st.verified_objects, READY);
@@ -3,7 +3,8 @@
@@ -3,7 +3,8 @@
*
*
* A ready object has NO GOOD COPY when none of its locations is `present` (verified there) or
* A ready object has NO GOOD COPY when none of its locations is `present` (verified there) or
* `pending` (not verified yet, so possibly fine). That covers objects whose every copy was found
* `pending` (not verified yet, so possibly fine). That covers objects whose every copy was found
- * missing/corrupt, and ready objects with no location row at all, except virtual clips (a window over their VOD).
+ * missing/corrupt, and ready objects with no location row at all, except virtual objects (readiness.js isVirtual:
+ * a clip over its VOD, a recording over its stream's parts).
*
*
* Pure SQL over a small `q` interface — { all(sql, params), get(sql, params) } — so the service
* Pure SQL over a small `q` interface — { all(sql, params), get(sql, params) } — so the service
* (server/db/database.js), the metrics/readiness checks and the read-only report script
* (server/db/database.js), the metrics/readiness checks and the read-only report script
@@ -12,12 +13,13 @@
@@ -12,12 +13,13 @@
*/
*/
'use strict';
'use strict';
-// A virtual clip (clip-jobs makeVirtual: a window over its VOD's timeline) has no copy of its own by design: its bytes
-// are its VOD's, whose object is checked on its own. The same test as verify-job's selection; five of them held
-// /api/ready at "degraded" (2026-10-17).
+const { virtualSql } = require('./readiness');
+
+// A virtual object has no copy of its own by design: its bytes are its sources', each checked on its own. The same
+// test as verify-job's selection (readiness.js virtualSql).
const NO_GOOD_COPY_WHERE = `o.lifecycle_status = 'ready'
const NO_GOOD_COPY_WHERE = `o.lifecycle_status = 'ready'
AND NOT EXISTS (SELECT 1 FROM media_locations l WHERE l.object_id = o.id AND l.state IN ('present', 'pending'))
AND NOT EXISTS (SELECT 1 FROM media_locations l WHERE l.object_id = o.id AND l.state IN ('present', 'pending'))
- AND NOT (o.kind = 'clip' AND (CASE WHEN json_valid(o.metadata) THEN json_extract(o.metadata, '$.virtual') END) IS NOT NULL)`;
+ AND NOT ${virtualSql('o')}`;
async function countNoGoodCopy(q, { appId = null } = {}) {
async function countNoGoodCopy(q, { appId = null } = {}) {
const r = await q.get(`SELECT COUNT(*) AS n FROM media_objects o WHERE ${NO_GOOD_COPY_WHERE}${appId ? ' AND o.app_id = ?' : ''}`, appId ? [appId] : []);
const r = await q.get(`SELECT COUNT(*) AS n FROM media_objects o WHERE ${NO_GOOD_COPY_WHERE}${appId ? ' AND o.app_id = ?' : ''}`, appId ? [appId] : []);
@@ -42,18 +42,30 @@ function isVerifiedCopy(obj, l) {
@@ -42,18 +42,30 @@ function isVerifiedCopy(obj, l) {
/** A ready clip object with no file of its own, played as a window of its source's timeline (model.js clipProjection). */
/** A ready clip object with no file of its own, played as a window of its source's timeline (model.js clipProjection). */
function isVirtualClip(obj, md = metadataOf(obj)) {
function isVirtualClip(obj, md = metadataOf(obj)) {
- return obj.kind === 'clip' && obj.lifecycle_status === 'ready' && !!md && md.virtual === true;
+ return obj.kind === 'clip' && isVirtual(obj, md);
}
}
+/**
+ * A ready object with no bytes of its own, of any kind: a virtual clip (a window over its VOD's timeline) or a
+ * recording (an asset over its stream's parts, docs/recordings.md). Its bytes are other objects' segments, each
+ * checked as its own object, so it never has a copy to verify. virtualSql() is the same test in SQL, shared by the
+ * verification job's selection, the "no good copy" report and playableSql(): recordings were not in it, and three
+ * held /api/ready at "degraded" (2026-10-28) as the five virtual clips had (2026-10-17).
+ */
+function isVirtual(obj, md = metadataOf(obj)) {
+ return obj.lifecycle_status === 'ready' && !!md && md.virtual === true;
+}
+const virtualSql = (alias) => `(CASE WHEN json_valid(${alias}.metadata) THEN json_extract(${alias}.metadata, '$.virtual') END) IS NOT NULL`;
+
/** Readiness of one object row (null → nothing known). `locations` may be passed when already loaded. */
/** Readiness of one object row (null → nothing known). `locations` may be passed when already loaded. */
async function compute(obj, locations = null) {
async function compute(obj, locations = null) {
if (!obj) return { metadata: false, bytes_verified: false, playable: false, hash_verified: false, verified_copies: [], reason: 'no_object' };
if (!obj) return { metadata: false, bytes_verified: false, playable: false, hash_verified: false, verified_copies: [], reason: 'no_object' };
const locs = locations || await db.all('SELECT provider, state, checksum, verified_at FROM media_locations WHERE object_id = ? ORDER BY id', [obj.id]);
const locs = locations || await db.all('SELECT provider, state, checksum, verified_at FROM media_locations WHERE object_id = ? ORDER BY id', [obj.id]);
const md = metadataOf(obj);
const md = metadataOf(obj);
const verified = locs.filter(l => isVerifiedCopy(obj, l));
const verified = locs.filter(l => isVerifiedCopy(obj, l));
- // A virtual clip (F3.5) has no copy of its own: its bytes are its source's timeline segments, served under the
- // source's rules, so a ready one counts as having its bytes.
- const bytesVerified = verified.length > 0 || isVirtualClip(obj, md);
+ // A virtual object (a clip over its VOD, a recording over its stream's parts) has no copy of its own: its bytes are
+ // its sources' timeline segments, served under their rules, so a ready one counts as having its bytes.
+ const bytesVerified = verified.length > 0 || isVirtual(obj, md);
let reason = null;
let reason = null;
if (obj.lifecycle_status === 'deleted') reason = 'deleted';
if (obj.lifecycle_status === 'deleted') reason = 'deleted';
else if (!md) reason = 'metadata_unreadable';
else if (!md) reason = 'metadata_unreadable';
@@ -84,7 +96,7 @@ function playableSql(ref) {
@@ -84,7 +96,7 @@ function playableSql(ref) {
AND ro.kind IN (${PLAYBACK_KINDS.map(k => `'${k}'`).join(', ')}) AND (CASE WHEN json_valid(ro.metadata) THEN json_type(ro.metadata) END) = 'object'
AND ro.kind IN (${PLAYBACK_KINDS.map(k => `'${k}'`).join(', ')}) AND (CASE WHEN json_valid(ro.metadata) THEN json_type(ro.metadata) END) = 'object'
AND (EXISTS (SELECT 1 FROM media_locations rl WHERE rl.object_id = ro.id AND rl.state = 'present' AND rl.verified_at IS NOT NULL
AND (EXISTS (SELECT 1 FROM media_locations rl WHERE rl.object_id = ro.id AND rl.state = 'present' AND rl.verified_at IS NOT NULL
AND (rl.checksum IS NULL OR ro.content_hash IS NULL OR rl.checksum = ro.content_hash))
AND (rl.checksum IS NULL OR ro.content_hash IS NULL OR rl.checksum = ro.content_hash))
- OR (ro.kind = 'clip' AND (CASE WHEN json_valid(ro.metadata) THEN json_extract(ro.metadata, '$.virtual') END) IS NOT NULL)))`;
+ OR ${virtualSql('ro')}))`;
}
}
-module.exports = { compute, forRow, playableSql, isVerifiedCopy, isVirtualClip, PLAYBACK_KINDS, REASONS };
+module.exports = { compute, forRow, playableSql, isVerifiedCopy, isVirtualClip, isVirtual, virtualSql, PLAYBACK_KINDS, REASONS };
@@ -24,6 +24,7 @@ const crypto = require('crypto');
@@ -24,6 +24,7 @@ const crypto = require('crypto');
const db = require('../db/database');
const db = require('../db/database');
const config = require('../config');
const config = require('../config');
const copyReport = require('./copy-report');
const copyReport = require('./copy-report');
+const { virtualSql } = require('./readiness');
const MB = 1024 * 1024;
const MB = 1024 * 1024;
// Every remote tier the fabric knows — the canonical list lives in vod-storage (REMOTE_PROVIDERS);
// Every remote tier the fabric knows — the canonical list lives in vod-storage (REMOTE_PROVIDERS);
@@ -157,7 +158,7 @@ async function runOnce({ batch = config.verify.batch, head = defaultHead, upload
@@ -157,7 +158,7 @@ async function runOnce({ batch = config.verify.batch, head = defaultHead, upload
const n = Math.max(1, Number(batch) || 1);
const n = Math.max(1, Number(batch) || 1);
const objects = await db.all(`SELECT o.* FROM media_objects o LEFT JOIN media_verifications v ON v.object_id = o.id
const objects = await db.all(`SELECT o.* FROM media_objects o LEFT JOIN media_verifications v ON v.object_id = o.id
WHERE o.lifecycle_status = 'ready'
WHERE o.lifecycle_status = 'ready'
- AND NOT (o.kind = 'clip' AND (CASE WHEN json_valid(o.metadata) THEN json_extract(o.metadata, '$.virtual') END) IS NOT NULL)
+ AND NOT ${virtualSql('o')}
ORDER BY (v.verified_at IS NOT NULL), v.verified_at, o.id LIMIT ?`, [n]);
ORDER BY (v.verified_at IS NOT NULL), v.verified_at, o.id LIMIT ?`, [n]);
const budget = { left: Math.max(0, Number(maxReuploads) || 0) };
const budget = { left: Math.max(0, Number(maxReuploads) || 0) };
for (const obj of objects) {
for (const obj of objects) {
@@ -160,7 +160,10 @@ const crypto = require('crypto');
@@ -160,7 +160,10 @@ const crypto = require('crypto');
// A virtual clip (clip-jobs makeVirtual) has no copy of its own by design: its bytes are its VOD's.
// A virtual clip (clip-jobs makeVirtual) has no copy of its own by design: its bytes are its VOD's.
await model.createObject({ app_id: 'live', kind: 'clip', lifecycle_status: 'ready', size_bytes: 0,
await model.createObject({ app_id: 'live', kind: 'clip', lifecycle_status: 'ready', size_bytes: 0,
owner_user_id: 7, owner_subject: 'usr_01JAB2C3D4E5F6G7H8J9K0MNPQ', mime_type: 'video/mp4', metadata: { virtual: true } });
owner_user_id: 7, owner_subject: 'usr_01JAB2C3D4E5F6G7H8J9K0MNPQ', mime_type: 'video/mp4', metadata: { virtual: true } });
- assert.strictEqual(await copyReport.countNoGoodCopy(db), 3, 'c (remote corrupt), d (local gone), h (no locations); not the virtual clip');
+ // Nor has a recording (docs/recordings.md: an asset over its stream's parts, an HLS playlist of their segments).
+ await model.createObject({ app_id: 'live', kind: 'asset', lifecycle_status: 'ready', size_bytes: 0,
+ owner_user_id: 7, owner_subject: 'usr_01JAB2C3D4E5F6G7H8J9K0MNPQ', mime_type: 'application/vnd.apple.mpegurl', metadata: { recording: true, virtual: true, group_key: 'stream:1' } });
+ assert.strictEqual(await copyReport.countNoGoodCopy(db), 3, 'c (remote corrupt), d (local gone), h (no locations); not the virtual clip or the recording');
const st = await job.status();
const st = await job.status();
assert.strictEqual(st.no_good_copy, 3);
assert.strictEqual(st.no_good_copy, 3);
assert.strictEqual(st.verified_objects, READY);
assert.strictEqual(st.verified_objects, READY);