feat(dashboard): ratings + pairing + scoreboard logic
Pure logic (no DOM): axis normalisers, attribution, Bradley-Terry with a ghost prior, pair selection, scoreboard/CSV. node:test coverage. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Jason Hoku
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
1692492eab
commit
02f0766566
@@ -0,0 +1,538 @@
|
||||
/**
|
||||
* PAIRWISE REVIEW - PURE LOGIC (no DOM access)
|
||||
* Axis normalisation, attribution, Bradley-Terry ratings, pair selection, scoreboard.
|
||||
* The dashboard inlines every logic_*.js into ONE <script>, so top-level names here are
|
||||
* prefixed (CMP_ / cmp / rt) or unique. Node tests load this file through the
|
||||
* module.exports guard at the bottom (tests/js/ratings.test.js).
|
||||
*
|
||||
* Vote shape (manifest top-level "comparisons"):
|
||||
* { a, b, winner: 'a'|'b'|'tie', ts, mode, blind? } a = left/top item id
|
||||
*/
|
||||
|
||||
var CMP_AXES = ['model', 'lora', 'sampler', 'scheduler', 'cfg', 'steps', 'denoise', 'resolution', 'prompt', 'upscale'];
|
||||
// 'seed' is a diagnostic axis (noise floor): it participates in axisDiff but not in CMP_AXES
|
||||
var CMP_DIFF_AXES = CMP_AXES.concat(['seed']);
|
||||
|
||||
// Item keys that never make two items "different settings" (deny list on purpose: unknown
|
||||
// keys such as vae / clip_type / use_slg are treated as confounders and block attribution).
|
||||
var CMP_EXCLUDED_KEYS = new Set([
|
||||
'id', 'gen_index', 'file', 'filename', 'duration', 'batch_idx', 'favorited', 'rejected',
|
||||
'note', 'notes', 'positive', 'config_positive', 'negative', 'config_negative', 'seed',
|
||||
'width', 'height', 'base_width', 'base_height', 'model', 'lora', 'lora_expanded',
|
||||
'sampler', 'scheduler', 'cfg', 'steps', 'denoise', 'upscaled', 'source_file',
|
||||
'image_path', 'preview_path', 'video_path', 'hash', 'hires_denoise'
|
||||
]);
|
||||
var CMP_EXCLUDED_RE = /^upscale_|^hires_prompt_|_hash$|^worker|^_/;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Normalisers
|
||||
// ---------------------------------------------------------------------------
|
||||
function fmtNum(v) {
|
||||
var ok = (typeof v === 'number') || (typeof v === 'string' && v.trim() !== '');
|
||||
if (ok && isFinite(+v)) return String(+(+v).toFixed(6));
|
||||
return String(v === undefined || v === null ? '?' : v);
|
||||
}
|
||||
|
||||
function cmpBase(s) {
|
||||
return String(s).replace(/\\/g, '/').split('/').pop().replace(/\.[^.]+$/, '');
|
||||
}
|
||||
|
||||
function cmpProduct(r) {
|
||||
var arr = Array.isArray(r) ? r : [r];
|
||||
var p = 1, any = false;
|
||||
for (var i = 0; i < arr.length; i++) {
|
||||
var n = +arr[i];
|
||||
if (isFinite(n) && n > 0 && arr[i] !== null && arr[i] !== '') { p *= n; any = true; }
|
||||
}
|
||||
return any ? p : 0;
|
||||
}
|
||||
|
||||
function normModel(d, m) {
|
||||
var raw = String(d.model || (m && m.model) || 'Default');
|
||||
return { key: raw.replace(/\\/g, '/'), label: cmpBase(raw) };
|
||||
}
|
||||
|
||||
function normLora(d) {
|
||||
var raw = String(d.lora_expanded || d.lora || 'None');
|
||||
if (raw === 'None' || raw === '') return { key: 'None', label: 'None' };
|
||||
var parts = raw.split(' + ').map(function (p) {
|
||||
var c = p.split(':');
|
||||
var name = c[0].replace(/\\/g, '/');
|
||||
var ms = fmtNum(c[1] !== undefined && c[1] !== '' ? c[1] : 1);
|
||||
var cs = fmtNum(c[2] !== undefined && c[2] !== '' ? c[2] : ms);
|
||||
return { name: name, ms: ms, cs: cs };
|
||||
});
|
||||
parts.sort(function (x, y) {
|
||||
var a = x.name + ':' + x.ms + ':' + x.cs, b = y.name + ':' + y.ms + ':' + y.cs;
|
||||
return a < b ? -1 : a > b ? 1 : 0;
|
||||
});
|
||||
return {
|
||||
key: parts.map(function (p) { return p.name + ':' + p.ms + ':' + p.cs; }).join(' + '),
|
||||
label: parts.map(function (p) {
|
||||
return cmpBase(p.name) + '@' + p.ms + (p.cs !== p.ms ? '/' + p.cs : '');
|
||||
}).join(' + ')
|
||||
};
|
||||
}
|
||||
|
||||
function normPrompt(d, m) {
|
||||
var raw = d.config_positive !== undefined && d.config_positive !== null ? d.config_positive
|
||||
: (d.positive !== undefined && d.positive !== null ? d.positive
|
||||
: ((m && m.positive) || ''));
|
||||
var s = String(raw).replace(/\s+/g, ' ').trim();
|
||||
return { key: s, label: s.length > 60 ? s.slice(0, 60) + '…' : s };
|
||||
}
|
||||
|
||||
function normResolution(d) {
|
||||
var k;
|
||||
if (d.base_width && d.base_height) {
|
||||
k = d.base_width + 'x' + d.base_height;
|
||||
} else if (!d.upscaled) {
|
||||
k = d.width + 'x' + d.height;
|
||||
} else {
|
||||
var r = cmpProduct(d.upscale_ratio);
|
||||
if (r > 0 && d.width > 0 && d.height > 0) {
|
||||
k = (Math.round(d.width / r / 8) * 8) + 'x' + (Math.round(d.height / r / 8) * 8);
|
||||
} else {
|
||||
k = 'AR ' + (d.height > 0 ? (d.width / d.height).toFixed(2) : '?');
|
||||
}
|
||||
}
|
||||
return { key: k, label: k };
|
||||
}
|
||||
|
||||
function normUpscale(d) {
|
||||
if (!d.upscaled) return { key: 'none', label: 'none' };
|
||||
var join = function (v) { return Array.isArray(v) ? v.join('>') : v; };
|
||||
var mode = d.upscale_mode !== undefined && d.upscale_mode !== null ? String(join(d.upscale_mode)) : '';
|
||||
var models = d.upscale_model !== undefined && d.upscale_model !== null && d.upscale_model !== ''
|
||||
? (Array.isArray(d.upscale_model) ? d.upscale_model : [d.upscale_model]).map(cmpBase).join('>') : '';
|
||||
var ratio = d.upscale_ratio !== undefined && d.upscale_ratio !== null
|
||||
? (Array.isArray(d.upscale_ratio) ? d.upscale_ratio.map(fmtNum).join('>') : fmtNum(d.upscale_ratio)) : '';
|
||||
var parts = [];
|
||||
if (mode) parts.push(mode);
|
||||
if (models) parts.push(models);
|
||||
if (ratio) parts.push('x' + ratio);
|
||||
if (d.upscale_denoise !== undefined && d.upscale_denoise !== null) parts.push('d' + fmtNum(d.upscale_denoise));
|
||||
if (d.hires_prompt_behavior) parts.push('hp:' + d.hires_prompt_behavior);
|
||||
if (!parts.length) parts.push('upscaled');
|
||||
var label = [mode, models, ratio ? 'x' + ratio : ''].filter(Boolean).join(' ') || 'upscaled';
|
||||
return { key: parts.join('|'), label: label };
|
||||
}
|
||||
|
||||
function normalizeAxis(axis, d, m) {
|
||||
switch (axis) {
|
||||
case 'model': return normModel(d, m);
|
||||
case 'lora': return normLora(d);
|
||||
case 'sampler': { var s = String(d.sampler == null ? '?' : d.sampler); return { key: s, label: s }; }
|
||||
case 'scheduler': { var c = String(d.scheduler == null ? '?' : d.scheduler); return { key: c, label: c }; }
|
||||
case 'cfg': { var f = fmtNum(d.cfg); return { key: f, label: f }; }
|
||||
case 'steps': { var t = fmtNum(d.steps); return { key: t, label: t }; }
|
||||
case 'denoise': { var n = fmtNum(d.denoise); return { key: n, label: n }; }
|
||||
case 'resolution': return normResolution(d);
|
||||
case 'prompt': return normPrompt(d, m);
|
||||
case 'upscale': return normUpscale(d);
|
||||
case 'seed': { var e = fmtNum(d.seed); return { key: e, label: e }; }
|
||||
}
|
||||
return { key: '?', label: '?' };
|
||||
}
|
||||
|
||||
function cmpStable(v) {
|
||||
if (v === null || typeof v !== 'object') return JSON.stringify(v === undefined ? null : v);
|
||||
if (Array.isArray(v)) return '[' + v.map(cmpStable).join(',') + ']';
|
||||
return '{' + Object.keys(v).sort().map(function (k) { return JSON.stringify(k) + ':' + cmpStable(v[k]); }).join(',') + '}';
|
||||
}
|
||||
|
||||
function cmpCtxEntries(d) {
|
||||
var out = {};
|
||||
var ks = Object.keys(d);
|
||||
for (var i = 0; i < ks.length; i++) {
|
||||
var k = ks[i];
|
||||
if (CMP_EXCLUDED_KEYS.has(k) || CMP_EXCLUDED_RE.test(k)) continue;
|
||||
out[k] = d[k];
|
||||
}
|
||||
out.negative = d.config_negative !== undefined && d.config_negative !== null ? d.config_negative : d.negative;
|
||||
return out;
|
||||
}
|
||||
|
||||
function cmpMediaType(d) { return d.media_type === 'video' ? 'video' : 'image'; }
|
||||
|
||||
var _cmpSigCache = new WeakMap();
|
||||
|
||||
/** {axes:{axis:key}, labels:{axis:label}, ctx} cached per item object (invalidated on w/h/meta change). */
|
||||
function itemSig(d, m) {
|
||||
var hit = _cmpSigCache.get(d);
|
||||
if (hit && hit.w === d.width && hit.h === d.height && hit.m === m) return hit;
|
||||
var axes = {}, labels = {};
|
||||
for (var i = 0; i < CMP_DIFF_AXES.length; i++) {
|
||||
var ax = CMP_DIFF_AXES[i];
|
||||
var r = normalizeAxis(ax, d, m);
|
||||
axes[ax] = r.key; labels[ax] = r.label;
|
||||
}
|
||||
var sig = { axes: axes, labels: labels, ctx: cmpStable(cmpCtxEntries(d)), w: d.width, h: d.height, m: m };
|
||||
_cmpSigCache.set(d, sig);
|
||||
return sig;
|
||||
}
|
||||
|
||||
/** Which axes differ between two items. ctxKeys only when opts.withCtxKeys (diagnostics). */
|
||||
function axisDiff(A, B, opts) {
|
||||
opts = opts || {};
|
||||
var m = opts.meta;
|
||||
var sa = itemSig(A, m), sb = itemSig(B, m);
|
||||
var axes = [];
|
||||
for (var i = 0; i < CMP_DIFF_AXES.length; i++) {
|
||||
var ax = CMP_DIFF_AXES[i];
|
||||
if (sa.axes[ax] !== sb.axes[ax]) axes.push(ax);
|
||||
}
|
||||
var ctxEqual = sa.ctx === sb.ctx;
|
||||
var ctxKeys = [];
|
||||
if (!ctxEqual && opts.withCtxKeys) {
|
||||
var ea = cmpCtxEntries(A), eb = cmpCtxEntries(B);
|
||||
var all = new Set(Object.keys(ea).concat(Object.keys(eb)));
|
||||
all.forEach(function (k) {
|
||||
if (cmpStable(ea[k]) !== cmpStable(eb[k])) ctxKeys.push(k);
|
||||
});
|
||||
ctxKeys.sort();
|
||||
}
|
||||
return { axes: axes, ctxEqual: ctxEqual, ctxKeys: ctxKeys };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Comparison list helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
function cmpVoteKey(c) { return c.ts + ':' + c.a + ':' + c.b; }
|
||||
|
||||
function mergeComparisons(existing, incoming) {
|
||||
var out = [], seen = new Set();
|
||||
var all = (existing || []).concat(incoming || []);
|
||||
for (var i = 0; i < all.length; i++) {
|
||||
var c = all[i];
|
||||
if (!c) continue;
|
||||
var k = cmpVoteKey(c);
|
||||
if (seen.has(k)) continue;
|
||||
seen.add(k);
|
||||
out.push(c);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function cmpPairKey(a, b) { return a < b ? a + '|' + b : b + '|' + a; }
|
||||
|
||||
function cmpValidVote(c, ids) {
|
||||
return c && c.a !== c.b && (c.winner === 'a' || c.winner === 'b' || c.winner === 'tie') &&
|
||||
ids.has(c.a) && ids.has(c.b);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Bradley-Terry (MM, Hunter 2004) with a ghost-player Laplace prior
|
||||
// ---------------------------------------------------------------------------
|
||||
function computeBradleyTerry(comparisons, presentIds, opts) {
|
||||
opts = opts || {};
|
||||
var prior = opts.prior !== undefined ? opts.prior : 1;
|
||||
var maxIter = opts.maxIter || 200;
|
||||
var tol = opts.tol || 1e-6;
|
||||
var ids = presentIds instanceof Set ? presentIds : new Set(presentIds || []);
|
||||
var rec = new Map(); // id -> {w,l,t,W,nb:Map}
|
||||
var get = function (id) {
|
||||
var r = rec.get(id);
|
||||
if (!r) { r = { w: 0, l: 0, t: 0, W: prior, nb: new Map() }; rec.set(id, r); }
|
||||
return r;
|
||||
};
|
||||
for (var i = 0; i < (comparisons || []).length; i++) {
|
||||
var c = comparisons[i];
|
||||
if (!cmpValidVote(c, ids)) continue;
|
||||
var ra = get(c.a), rb = get(c.b);
|
||||
ra.nb.set(c.b, (ra.nb.get(c.b) || 0) + 1);
|
||||
rb.nb.set(c.a, (rb.nb.get(c.a) || 0) + 1);
|
||||
if (c.winner === 'a') { ra.w++; rb.l++; ra.W += 1; }
|
||||
else if (c.winner === 'b') { rb.w++; ra.l++; rb.W += 1; }
|
||||
else { ra.t++; rb.t++; ra.W += 0.5; rb.W += 0.5; }
|
||||
}
|
||||
var out = new Map();
|
||||
if (!rec.size) return out;
|
||||
var p = new Map();
|
||||
rec.forEach(function (_, id) { p.set(id, 1); });
|
||||
var nG = 2 * prior;
|
||||
for (var it = 0; it < maxIter; it++) {
|
||||
var next = new Map(), delta = 0;
|
||||
rec.forEach(function (r, id) {
|
||||
var pi = p.get(id);
|
||||
var denom = nG / (pi + 1);
|
||||
r.nb.forEach(function (n, j) { denom += n / (pi + p.get(j)); });
|
||||
var pn = r.W / denom;
|
||||
next.set(id, pn);
|
||||
var dl = Math.abs(Math.log(pn) - Math.log(pi));
|
||||
if (dl > delta) delta = dl;
|
||||
});
|
||||
p = next;
|
||||
if (delta < tol) break;
|
||||
}
|
||||
rec.forEach(function (r, id) {
|
||||
var pi = p.get(id);
|
||||
out.set(id, { rating: 1500 + 400 * Math.log10(pi), votes: r.w + r.l + r.t, w: r.w, l: r.l, t: r.t, p: pi });
|
||||
});
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Pair source + selection
|
||||
// ---------------------------------------------------------------------------
|
||||
var CMP_MODES = ['matched', 'seed', 'one_axis', 'random'];
|
||||
|
||||
/**
|
||||
* pool: array of item objects. mode: matched | seed | one_axis | random. axisFocus: axis or ''.
|
||||
* opts.meta: manifest meta for fallbacks. Returns {mode, axisFocus, buckets, estimate}.
|
||||
*/
|
||||
function buildPairSource(pool, mode, axisFocus, opts) {
|
||||
opts = opts || {};
|
||||
var m = opts.meta;
|
||||
var intern = new Map();
|
||||
var iid = function (s) {
|
||||
var v = intern.get(s);
|
||||
if (v === undefined) { v = intern.size; intern.set(s, v); }
|
||||
return v;
|
||||
};
|
||||
var entries = [];
|
||||
for (var i = 0; i < pool.length; i++) {
|
||||
var d = pool[i];
|
||||
if (!d) continue;
|
||||
var sg = itemSig(d, m);
|
||||
var vec = new Array(CMP_AXES.length);
|
||||
for (var a = 0; a < CMP_AXES.length; a++) vec[a] = iid(CMP_AXES[a] + '=' + sg.axes[CMP_AXES[a]]);
|
||||
entries.push({
|
||||
d: d, id: d.id, vec: vec, seed: iid('seed=' + sg.axes.seed), ctx: iid('ctx=' + sg.ctx),
|
||||
mt: cmpMediaType(d)
|
||||
});
|
||||
}
|
||||
var promptIdx = CMP_AXES.indexOf('prompt');
|
||||
var buckets = [];
|
||||
var addBuckets = function (keyFn, distinctFn, validFn, axisIdx) {
|
||||
var map = new Map();
|
||||
for (var i = 0; i < entries.length; i++) {
|
||||
var e = entries[i];
|
||||
var k = keyFn(e);
|
||||
var arr = map.get(k);
|
||||
if (!arr) { arr = []; map.set(k, arr); }
|
||||
arr.push(e);
|
||||
}
|
||||
map.forEach(function (items) {
|
||||
if (items.length < 2) return;
|
||||
var first = distinctFn(items[0]), ok = false;
|
||||
for (var j = 1; j < items.length; j++) if (distinctFn(items[j]) !== first) { ok = true; break; }
|
||||
if (!ok) return;
|
||||
buckets.push({ items: items, weight: Math.min(items.length, 50), valid: validFn, axis: axisIdx === undefined ? null : CMP_AXES[axisIdx] });
|
||||
});
|
||||
};
|
||||
var configSig = function (e) {
|
||||
var parts = [];
|
||||
for (var a = 0; a < e.vec.length; a++) if (a !== promptIdx) parts.push(e.vec[a]);
|
||||
parts.push(e.ctx);
|
||||
return parts.join(',');
|
||||
};
|
||||
if (mode === 'matched') {
|
||||
addBuckets(function (e) { return e.vec[promptIdx] + '|' + e.seed + '|' + e.mt; }, configSig,
|
||||
function (x, y) { return configSig(x) !== configSig(y); });
|
||||
} else if (mode === 'seed') {
|
||||
addBuckets(function (e) { return e.vec.join(',') + '|' + e.ctx + '|' + e.mt; },
|
||||
function (e) { return e.seed; }, function (x, y) { return x.seed !== y.seed; });
|
||||
} else if (mode === 'one_axis') {
|
||||
var list = axisFocus && CMP_AXES.indexOf(axisFocus) >= 0 ? [CMP_AXES.indexOf(axisFocus)] : CMP_AXES.map(function (_, i) { return i; });
|
||||
list.forEach(function (ai) {
|
||||
addBuckets(function (e) {
|
||||
var rest = [];
|
||||
for (var a = 0; a < e.vec.length; a++) if (a !== ai) rest.push(e.vec[a]);
|
||||
return ai + '|' + e.ctx + '|' + rest.join(',') + '|' + e.seed + '|' + e.mt;
|
||||
},
|
||||
function (e) { return e.vec[ai]; }, function (x, y) { return x.vec[ai] !== y.vec[ai]; }, ai);
|
||||
});
|
||||
} else {
|
||||
addBuckets(function (e) { return e.mt; }, function (e) { return e.id; }, function () { return true; });
|
||||
}
|
||||
var estimate = 0;
|
||||
for (var b = 0; b < buckets.length; b++) { var k = buckets[b].items.length; estimate += k * (k - 1) / 2; }
|
||||
return { mode: mode, axisFocus: axisFocus || '', buckets: buckets, estimate: estimate, poolSize: entries.length };
|
||||
}
|
||||
|
||||
function cmpPickOne(items, rng) { return items[Math.floor(rng() * items.length)]; }
|
||||
|
||||
/**
|
||||
* state: {votesById: Map id->count, votedPairs: Set(pairKey), skipped: Set(pairKey), allowRepeats}
|
||||
* Returns {a, b (item objects, random left/right), repeating, axis} or null.
|
||||
*/
|
||||
function nextPair(src, state, rng) {
|
||||
rng = rng || Math.random;
|
||||
if (!src || !src.buckets.length) return null;
|
||||
var votes = function (id) { return (state.votesById && state.votesById.get(id)) || 0; };
|
||||
var cum = [], total = 0;
|
||||
for (var i = 0; i < src.buckets.length; i++) { total += src.buckets[i].weight; cum.push(total); }
|
||||
var pickBucket = function () {
|
||||
var r = rng() * total, lo = 0, hi = cum.length - 1;
|
||||
while (lo < hi) { var mid = (lo + hi) >> 1; if (cum[mid] > r) hi = mid; else lo = mid + 1; }
|
||||
return src.buckets[lo];
|
||||
};
|
||||
var attempt = function (allowRepeats) {
|
||||
var best = null;
|
||||
for (var s = 0; s < 64; s++) {
|
||||
var bk = pickBucket();
|
||||
var anchor = null;
|
||||
for (var t = 0; t < 4; t++) {
|
||||
var c = cmpPickOne(bk.items, rng);
|
||||
if (!anchor || votes(c.id) < votes(anchor.id)) anchor = c;
|
||||
}
|
||||
var partner = null;
|
||||
for (var u = 0; u < 8; u++) {
|
||||
var y = cmpPickOne(bk.items, rng);
|
||||
if (y === anchor || y.id === anchor.id || !bk.valid(anchor, y)) continue;
|
||||
var pk = cmpPairKey(anchor.id, y.id);
|
||||
if (state.skipped && state.skipped.has(pk)) continue;
|
||||
if (!allowRepeats && state.votedPairs && state.votedPairs.has(pk)) continue;
|
||||
if (!partner || votes(y.id) < votes(partner.id)) partner = y;
|
||||
}
|
||||
if (!partner) continue;
|
||||
var score = votes(anchor.id) + votes(partner.id) + rng() * 0.5;
|
||||
if (!best || score < best.score) best = { score: score, x: anchor, y: partner, axis: bk.axis };
|
||||
}
|
||||
return best;
|
||||
};
|
||||
var repeating = false;
|
||||
var best = attempt(!!state.allowRepeats);
|
||||
if (!best && !state.allowRepeats) { best = attempt(true); repeating = !!best; }
|
||||
else if (best && state.allowRepeats) repeating = true;
|
||||
if (!best) return null;
|
||||
var swap = rng() < 0.5;
|
||||
return { a: swap ? best.y.d : best.x.d, b: swap ? best.x.d : best.y.d, repeating: repeating, axis: best.axis };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scoreboard
|
||||
// ---------------------------------------------------------------------------
|
||||
function wilsonInterval(s, n, z) {
|
||||
z = z || 1.96;
|
||||
if (!n) return { low: 0, high: 0 };
|
||||
var p = s / n, z2 = z * z;
|
||||
var den = 1 + z2 / n;
|
||||
var centre = (p + z2 / (2 * n)) / den;
|
||||
var half = (z * Math.sqrt(p * (1 - p) / n + z2 / (4 * n * n))) / den;
|
||||
return { low: Math.max(0, centre - half), high: Math.min(1, centre + half) };
|
||||
}
|
||||
|
||||
function cmpItemLookup(itemsById) {
|
||||
return itemsById instanceof Map ? function (id) { return itemsById.get(id); } : function (id) { return itemsById[id]; };
|
||||
}
|
||||
|
||||
function computeScoreboard(comparisons, itemsById, ratings, opts) {
|
||||
opts = opts || {};
|
||||
var minN = opts.minN !== undefined ? opts.minN : 3;
|
||||
var m = opts.meta;
|
||||
var restrict = opts.restrictIds || null;
|
||||
var lookup = cmpItemLookup(itemsById);
|
||||
var axesOut = {};
|
||||
var tallies = {};
|
||||
var order = CMP_AXES.concat(['seed']);
|
||||
order.forEach(function (ax) { tallies[ax] = new Map(); });
|
||||
var bump = function (ax, key, label, s, res) {
|
||||
var t = tallies[ax].get(key);
|
||||
if (!t) { t = { key: key, label: label, n: 0, s: 0, w: 0, l: 0, t: 0 }; tallies[ax].set(key, t); }
|
||||
t.n++; t.s += s;
|
||||
if (res === 'w') t.w++; else if (res === 'l') t.l++; else t.t++;
|
||||
};
|
||||
var total = 0, attributable = 0, nonAttr = 0;
|
||||
var extra = new Map();
|
||||
var matchedVotes = {};
|
||||
for (var i = 0; i < (comparisons || []).length; i++) {
|
||||
var c = comparisons[i];
|
||||
if (!c || c.a === c.b) continue;
|
||||
if (c.winner !== 'a' && c.winner !== 'b' && c.winner !== 'tie') continue;
|
||||
var A = lookup(c.a), B = lookup(c.b);
|
||||
if (!A || !B) continue;
|
||||
if (restrict && !(restrict.has(c.a) && restrict.has(c.b))) continue;
|
||||
total++;
|
||||
var df = axisDiff(A, B, { meta: m });
|
||||
var single = df.ctxEqual && (df.axes.length === 1);
|
||||
if (single) {
|
||||
var ax = df.axes[0];
|
||||
var sa = itemSig(A, m), sb = itemSig(B, m);
|
||||
var va = c.winner === 'a' ? 1 : c.winner === 'tie' ? 0.5 : 0;
|
||||
bump(ax, sa.axes[ax], sa.labels[ax], va, c.winner === 'a' ? 'w' : c.winner === 'b' ? 'l' : 't');
|
||||
bump(ax, sb.axes[ax], sb.labels[ax], 1 - va, c.winner === 'b' ? 'w' : c.winner === 'a' ? 'l' : 't');
|
||||
matchedVotes[ax] = (matchedVotes[ax] || 0) + 1;
|
||||
if (ax !== 'seed') attributable++;
|
||||
} else {
|
||||
nonAttr++;
|
||||
var dd = axisDiff(A, B, { meta: m, withCtxKeys: true });
|
||||
var keys = dd.ctxKeys.slice();
|
||||
if (dd.axes.length > 1) keys = keys.concat(dd.axes);
|
||||
if (!keys.length) keys.push('(identical settings)');
|
||||
keys.forEach(function (k) { extra.set(k, (extra.get(k) || 0) + 1); });
|
||||
}
|
||||
}
|
||||
// mean rating per axis value over rated items
|
||||
var means = {};
|
||||
order.forEach(function (ax) { means[ax] = new Map(); });
|
||||
if (ratings && ratings.size) {
|
||||
var consider = function (d) {
|
||||
var r = ratings.get(d.id);
|
||||
if (!r) return;
|
||||
var sg = itemSig(d, m);
|
||||
order.forEach(function (ax) {
|
||||
var k = sg.axes[ax];
|
||||
var e = means[ax].get(k);
|
||||
if (!e) { e = { sum: 0, n: 0 }; means[ax].set(k, e); }
|
||||
e.sum += r.rating; e.n++;
|
||||
});
|
||||
};
|
||||
if (itemsById instanceof Map) itemsById.forEach(function (d, id) { if (!restrict || restrict.has(id)) consider(d); });
|
||||
else Object.keys(itemsById).forEach(function (id) { var d = itemsById[id]; if (!restrict || restrict.has(d.id)) consider(d); });
|
||||
}
|
||||
order.forEach(function (ax) {
|
||||
var rows = [];
|
||||
tallies[ax].forEach(function (t) {
|
||||
var w = wilsonInterval(t.s, t.n);
|
||||
var mr = means[ax].get(t.key);
|
||||
rows.push({
|
||||
value: t.key, label: t.label, n: t.n, wins: t.w, losses: t.l, ties: t.t,
|
||||
winPct: t.n ? t.s / t.n : 0, wilsonLow: w.low, wilsonHigh: w.high,
|
||||
meanRating: mr && mr.n ? mr.sum / mr.n : null, enough: t.n >= minN
|
||||
});
|
||||
});
|
||||
rows.sort(function (x, y) { return (y.winPct - x.winPct) || (y.n - x.n); });
|
||||
var best = null;
|
||||
rows.forEach(function (r) {
|
||||
if (!r.enough) return;
|
||||
if (!best || r.wilsonLow > best.wilsonLow || (r.wilsonLow === best.wilsonLow && r.winPct > best.winPct)) best = r;
|
||||
});
|
||||
if (best) best.best = true;
|
||||
axesOut[ax] = { rows: rows, best: best ? best.value : null, matchedVotes: matchedVotes[ax] || 0 };
|
||||
});
|
||||
var topExtraKeys = Array.from(extra.entries()).map(function (e) { return { key: e[0], count: e[1] }; })
|
||||
.sort(function (x, y) { return y.count - x.count || (x.key < y.key ? -1 : 1); }).slice(0, 5);
|
||||
return { axes: axesOut, attributable: attributable, total: total, nonAttributable: nonAttr, topExtraKeys: topExtraKeys };
|
||||
}
|
||||
|
||||
function cmpCsvCell(v) {
|
||||
var s = v === null || v === undefined ? '' : String(v);
|
||||
return /[",\r\n]/.test(s) ? '"' + s.replace(/"/g, '""') + '"' : s;
|
||||
}
|
||||
|
||||
function scoreboardToCSV(sb) {
|
||||
var lines = ['axis,value,n,wins,losses,ties,win_pct,wilson_low,wilson_high,mean_rating,best'];
|
||||
Object.keys(sb.axes).forEach(function (ax) {
|
||||
sb.axes[ax].rows.forEach(function (r) {
|
||||
lines.push([
|
||||
ax, r.value, r.n, r.wins, r.losses, r.ties,
|
||||
(r.winPct * 100).toFixed(1), (r.wilsonLow * 100).toFixed(1), (r.wilsonHigh * 100).toFixed(1),
|
||||
r.meanRating === null ? '' : r.meanRating.toFixed(1), r.value === sb.axes[ax].best ? 'true' : 'false'
|
||||
].map(cmpCsvCell).join(','));
|
||||
});
|
||||
});
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
if (typeof module !== 'undefined' && module.exports) {
|
||||
module.exports = {
|
||||
CMP_AXES: CMP_AXES, CMP_DIFF_AXES: CMP_DIFF_AXES, CMP_MODES: CMP_MODES,
|
||||
CMP_EXCLUDED_KEYS: CMP_EXCLUDED_KEYS, CMP_EXCLUDED_RE: CMP_EXCLUDED_RE,
|
||||
fmtNum: fmtNum, normalizeAxis: normalizeAxis, itemSig: itemSig, axisDiff: axisDiff,
|
||||
mergeComparisons: mergeComparisons, cmpVoteKey: cmpVoteKey, cmpPairKey: cmpPairKey,
|
||||
computeBradleyTerry: computeBradleyTerry, buildPairSource: buildPairSource, nextPair: nextPair,
|
||||
wilsonInterval: wilsonInterval, computeScoreboard: computeScoreboard, scoreboardToCSV: scoreboardToCSV
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
'use strict';
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert');
|
||||
const R = require('../../resources/logic_ratings.js');
|
||||
|
||||
function rngSeed(seed) {
|
||||
let a = seed >>> 0;
|
||||
return function () {
|
||||
a |= 0; a = (a + 0x6D2B79F5) | 0;
|
||||
let t = Math.imul(a ^ (a >>> 15), 1 | a);
|
||||
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
let nextId = 1;
|
||||
function item(o) {
|
||||
return Object.assign({
|
||||
id: nextId++, file: '/view?x', model: 'm.safetensors', lora: 'None', sampler: 'euler', scheduler: 'simple',
|
||||
cfg: 7, steps: 20, denoise: 1, seed: 1, width: 1024, height: 1024,
|
||||
config_positive: 'a cat', positive: 'a cat', negative: '', duration: 1, favorited: false
|
||||
}, o);
|
||||
}
|
||||
const idOf = (x) => (typeof x === 'object' ? x.id : x);
|
||||
const vote = (a, b, winner, ts) => ({ a: idOf(a), b: idOf(b), winner, ts: ts || 1, mode: 'matched' });
|
||||
|
||||
// ---- normalizers ----
|
||||
test('fmtNum', () => {
|
||||
assert.strictEqual(R.fmtNum(1.0), '1');
|
||||
assert.strictEqual(R.fmtNum('7.50'), '7.5');
|
||||
assert.strictEqual(R.fmtNum(undefined), '?');
|
||||
assert.strictEqual(R.fmtNum('abc'), 'abc');
|
||||
});
|
||||
test('lora keys are order-insensitive and legacy-equal', () => {
|
||||
const k = (l) => R.normalizeAxis('lora', { lora: l }, {}).key;
|
||||
assert.strictEqual(k('a\\b.safetensors:1.0:1.0 + c:0.5'), k('c:0.5:0.5 + a/b.safetensors:1:1'));
|
||||
assert.strictEqual(k('x'), k('x:1:1'));
|
||||
assert.strictEqual(k('None'), k(''));
|
||||
assert.strictEqual(k(undefined), 'None');
|
||||
assert.strictEqual(R.normalizeAxis('lora', { lora: 'sub/foo.safetensors:0.8:0.5' }, {}).label, 'foo@0.8/0.5');
|
||||
});
|
||||
test('lora prefers lora_expanded', () => {
|
||||
assert.strictEqual(R.normalizeAxis('lora', { lora: 'a', lora_expanded: 'b:1:1' }, {}).key, 'b:1:1');
|
||||
});
|
||||
test('prompt axis uses config_positive (trigger words do not matter)', () => {
|
||||
const a = item({ positive: 'trigger, a cat', config_positive: 'a cat ' });
|
||||
const b = item({ positive: 'a cat', config_positive: 'a cat' });
|
||||
assert.strictEqual(R.normalizeAxis('prompt', a, {}).key, R.normalizeAxis('prompt', b, {}).key);
|
||||
assert.strictEqual(R.normalizeAxis('prompt', { config_positive: 'x'.repeat(100) }, {}).label.length, 61);
|
||||
});
|
||||
test('model falls back to meta and strips path', () => {
|
||||
assert.strictEqual(R.normalizeAxis('model', {}, { model: 'dir\\foo.safetensors' }).label, 'foo');
|
||||
assert.strictEqual(R.normalizeAxis('model', {}, {}).key, 'Default');
|
||||
});
|
||||
test('resolution of an upscaled item maps back to base size', () => {
|
||||
const d = { width: 1536, height: 2304, upscaled: true, upscale_ratio: 1.5 };
|
||||
assert.strictEqual(R.normalizeAxis('resolution', d, {}).key, '1024x1536');
|
||||
const arr = { width: 2304, height: 2304, upscaled: true, upscale_ratio: [1.5, 1.5] };
|
||||
assert.strictEqual(R.normalizeAxis('resolution', arr, {}).key, '1024x1024');
|
||||
assert.strictEqual(R.normalizeAxis('resolution', { width: 100, height: 50, upscaled: true }, {}).key, 'AR 2.00');
|
||||
assert.strictEqual(R.normalizeAxis('resolution', { width: 1, height: 1, base_width: 512, base_height: 768, upscaled: true }, {}).key, '512x768');
|
||||
assert.strictEqual(R.normalizeAxis('resolution', { width: 640, height: 480 }, {}).key, '640x480');
|
||||
});
|
||||
test('upscale key is stable for array modes', () => {
|
||||
const d = { upscaled: true, upscale_mode: ['model_only', 'hires'], upscale_model: ['dir/4x.pth', 'x/y.pth'], upscale_ratio: [2, 1.5], upscale_denoise: 0.35 };
|
||||
const k1 = R.normalizeAxis('upscale', d, {}).key;
|
||||
assert.strictEqual(k1, R.normalizeAxis('upscale', JSON.parse(JSON.stringify(d)), {}).key);
|
||||
assert.ok(k1.includes('model_only>hires') && k1.includes('x2>1.5') && k1.includes('d0.35'));
|
||||
assert.strictEqual(R.normalizeAxis('upscale', {}, {}).key, 'none');
|
||||
});
|
||||
|
||||
// ---- axisDiff ----
|
||||
test('axisDiff single, double, ctx and excluded keys', () => {
|
||||
const a = item({});
|
||||
assert.deepStrictEqual(R.axisDiff(a, item({ sampler: 'dpmpp' })).axes, ['sampler']);
|
||||
assert.deepStrictEqual(R.axisDiff(a, item({ sampler: 'dpmpp', cfg: 5 })).axes, ['sampler', 'cfg'].sort((x, y) => R.CMP_DIFF_AXES.indexOf(x) - R.CMP_DIFF_AXES.indexOf(y)));
|
||||
const v = R.axisDiff(a, item({ vae: 'x.vae' }), { withCtxKeys: true });
|
||||
assert.strictEqual(v.ctxEqual, false);
|
||||
assert.deepStrictEqual(v.ctxKeys, ['vae']);
|
||||
const e = R.axisDiff(a, item({ conditioning_pos_hash: 'zz', duration: 99, favorited: true, gen_index: 5, upscale_x: 1, worker_id: 'w' }));
|
||||
assert.deepStrictEqual(e.axes, []);
|
||||
assert.strictEqual(e.ctxEqual, true);
|
||||
assert.deepStrictEqual(R.axisDiff(a, item({ seed: 2 })).axes, ['seed']);
|
||||
});
|
||||
|
||||
// ---- Bradley-Terry ----
|
||||
test('BT: A beats B 3-1', () => {
|
||||
const A = item({}), B = item({});
|
||||
const cs = [vote(A, B, 'a', 1), vote(A, B, 'a', 2), vote(B, A, 'b', 3), vote(A, B, 'b', 4)];
|
||||
const r = R.computeBradleyTerry(cs, new Set([A.id, B.id]));
|
||||
assert.ok(r.get(A.id).rating > r.get(B.id).rating);
|
||||
assert.deepStrictEqual([r.get(A.id).w, r.get(A.id).l, r.get(A.id).t, r.get(A.id).votes], [3, 1, 0, 4]);
|
||||
assert.deepStrictEqual([r.get(B.id).w, r.get(B.id).l], [1, 3]);
|
||||
});
|
||||
test('BT: unbeaten item is finite', () => {
|
||||
const A = item({}), B = item({});
|
||||
const cs = [1, 2, 3, 4, 5].map((t) => vote(A, B, 'a', t));
|
||||
const r = R.computeBradleyTerry(cs, new Set([A.id, B.id]));
|
||||
assert.ok(Number.isFinite(r.get(A.id).rating) && Number.isFinite(r.get(B.id).rating));
|
||||
assert.ok(r.get(A.id).rating > 1500 && r.get(B.id).rating < 1500);
|
||||
});
|
||||
test('BT: all ties give equal 1500', () => {
|
||||
const [A, B, C] = [item({}), item({}), item({})];
|
||||
const cs = [vote(A, B, 'tie', 1), vote(B, C, 'tie', 2), vote(B, C, 'tie', 3)];
|
||||
const r = R.computeBradleyTerry(cs, new Set([A.id, B.id, C.id]));
|
||||
for (const id of [A.id, B.id, C.id]) assert.ok(Math.abs(r.get(id).rating - 1500) < 1e-6);
|
||||
});
|
||||
test('BT: chain order and order invariance', () => {
|
||||
const [A, B, C] = [item({}), item({}), item({})];
|
||||
const cs = [vote(A, B, 'a', 1), vote(B, C, 'a', 2)];
|
||||
const ids = new Set([A.id, B.id, C.id]);
|
||||
const r = R.computeBradleyTerry(cs, ids);
|
||||
assert.ok(r.get(A.id).rating > r.get(B.id).rating && r.get(B.id).rating > r.get(C.id).rating);
|
||||
const r2 = R.computeBradleyTerry(cs.slice().reverse(), ids);
|
||||
for (const id of ids) assert.ok(Math.abs(r.get(id).rating - r2.get(id).rating) < 1e-9);
|
||||
});
|
||||
test('BT: empty input, dangling ids, convergence', () => {
|
||||
assert.strictEqual(R.computeBradleyTerry([], new Set([1])).size, 0);
|
||||
const [A, B] = [item({}), item({})];
|
||||
const r = R.computeBradleyTerry([vote(A, 99999999, 'a', 1), vote(A, B, 'a', 2), vote(A, A, 'a', 3)], new Set([A.id, B.id]));
|
||||
assert.strictEqual(r.get(A.id).votes, 1);
|
||||
const items = Array.from({ length: 30 }, () => item({}));
|
||||
const rng = rngSeed(7);
|
||||
const cs = [];
|
||||
for (let i = 0; i < 600; i++) {
|
||||
const x = items[Math.floor(rng() * 30)], y = items[Math.floor(rng() * 30)];
|
||||
if (x === y) continue;
|
||||
cs.push(vote(x, y, (x.id % 7) > (y.id % 7) ? 'a' : 'b', i + 1));
|
||||
}
|
||||
const big = R.computeBradleyTerry(cs, new Set(items.map((d) => d.id)), { maxIter: 200 });
|
||||
for (const v of big.values()) assert.ok(Number.isFinite(v.rating));
|
||||
});
|
||||
|
||||
// ---- pairing ----
|
||||
const freshState = () => ({ votesById: new Map(), votedPairs: new Set(), skipped: new Set(), allowRepeats: false });
|
||||
function drawMany(src, n, rng, state) {
|
||||
const out = [];
|
||||
for (let i = 0; i < n; i++) { const p = R.nextPair(src, state || freshState(), rng); if (p) out.push(p); }
|
||||
return out;
|
||||
}
|
||||
function grid() {
|
||||
const items = [];
|
||||
for (const p of ['cat', 'dog']) for (const s of [1, 2]) for (const sm of ['euler', 'dpm', 'heun'])
|
||||
items.push(item({ config_positive: p, positive: p, seed: s, sampler: sm }));
|
||||
return items;
|
||||
}
|
||||
test('matched pairs share prompt+seed and differ in config', () => {
|
||||
const src = R.buildPairSource(grid(), 'matched', '');
|
||||
const ps = drawMany(src, 200, rngSeed(1));
|
||||
assert.ok(ps.length > 0);
|
||||
for (const p of ps) {
|
||||
assert.strictEqual(p.a.config_positive, p.b.config_positive);
|
||||
assert.strictEqual(p.a.seed, p.b.seed);
|
||||
assert.notStrictEqual(p.a.sampler, p.b.sampler);
|
||||
}
|
||||
});
|
||||
test('one_axis pairs differ in exactly the focus axis', () => {
|
||||
const items = grid();
|
||||
items.push(item({ config_positive: 'cat', positive: 'cat', seed: 1, cfg: 9 }));
|
||||
const src = R.buildPairSource(items, 'one_axis', 'sampler');
|
||||
const ps = drawMany(src, 100, rngSeed(2));
|
||||
assert.ok(ps.length > 0);
|
||||
for (const p of ps) assert.deepStrictEqual(R.axisDiff(p.a, p.b).axes, ['sampler']);
|
||||
const src2 = R.buildPairSource(items, 'one_axis', '');
|
||||
for (const p of drawMany(src2, 100, rngSeed(3))) assert.strictEqual(R.axisDiff(p.a, p.b).axes.length, 1);
|
||||
});
|
||||
test('seed mode pairs differ only in seed', () => {
|
||||
const src = R.buildPairSource(grid(), 'seed', '');
|
||||
const ps = drawMany(src, 100, rngSeed(4));
|
||||
assert.ok(ps.length);
|
||||
for (const p of ps) assert.deepStrictEqual(R.axisDiff(p.a, p.b).axes, ['seed']);
|
||||
});
|
||||
test('no valid bucket gives null; media types never mix', () => {
|
||||
const src = R.buildPairSource([item({}), item({ seed: 5 })], 'matched', '');
|
||||
assert.strictEqual(src.buckets.length, 0);
|
||||
assert.strictEqual(R.nextPair(src, freshState(), rngSeed(1)), null);
|
||||
const items = [item({ sampler: 'a' }), item({ sampler: 'b', media_type: 'video' }), item({ sampler: 'c' }), item({ sampler: 'd', media_type: 'video' })];
|
||||
const rs = R.buildPairSource(items, 'random', '');
|
||||
const ps = drawMany(rs, 100, rngSeed(5));
|
||||
assert.ok(ps.length > 0);
|
||||
for (const p of ps) assert.strictEqual(p.a.media_type === 'video', p.b.media_type === 'video');
|
||||
});
|
||||
test('voted pairs are skipped until exhausted, then repeats', () => {
|
||||
const items = [item({ sampler: 'a' }), item({ sampler: 'b' }), item({ sampler: 'c' })];
|
||||
const src = R.buildPairSource(items, 'matched', '');
|
||||
const state = freshState();
|
||||
const rng = rngSeed(11);
|
||||
const seen = new Set();
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const p = R.nextPair(src, state, rng);
|
||||
assert.ok(p && !p.repeating);
|
||||
const k = R.cmpPairKey(p.a.id, p.b.id);
|
||||
assert.ok(!state.votedPairs.has(k));
|
||||
state.votedPairs.add(k); seen.add(k);
|
||||
state.votesById.set(p.a.id, (state.votesById.get(p.a.id) || 0) + 1);
|
||||
state.votesById.set(p.b.id, (state.votesById.get(p.b.id) || 0) + 1);
|
||||
}
|
||||
assert.strictEqual(seen.size, 3);
|
||||
const p = R.nextPair(src, state, rng);
|
||||
assert.ok(p && p.repeating === true);
|
||||
});
|
||||
test('skipped pairs are excluded even with repeats', () => {
|
||||
const items = [item({ sampler: 'a' }), item({ sampler: 'b' })];
|
||||
const src = R.buildPairSource(items, 'matched', '');
|
||||
const st = freshState();
|
||||
st.skipped.add(R.cmpPairKey(items[0].id, items[1].id));
|
||||
st.allowRepeats = true;
|
||||
assert.strictEqual(R.nextPair(src, st, rngSeed(1)), null);
|
||||
});
|
||||
test('low-vote items are preferred', () => {
|
||||
const items = Array.from({ length: 20 }, (_, i) => item({ sampler: 's' + i }));
|
||||
const src = R.buildPairSource(items, 'matched', '');
|
||||
const st = freshState();
|
||||
items.slice(0, 10).forEach((d) => st.votesById.set(d.id, 50));
|
||||
const low = new Set(items.slice(10).map((d) => d.id));
|
||||
let lowCount = 0, tot = 0;
|
||||
for (const p of drawMany(src, 200, rngSeed(9), st)) {
|
||||
tot += 2; lowCount += (low.has(p.a.id) ? 1 : 0) + (low.has(p.b.id) ? 1 : 0);
|
||||
}
|
||||
assert.ok(lowCount / tot > 0.8, 'low-vote share ' + lowCount / tot);
|
||||
});
|
||||
|
||||
// ---- scoreboard ----
|
||||
test('scoreboard: 3-sampler dataset, ties half, best row, n<3', () => {
|
||||
const [E, D, H] = ['euler', 'dpm', 'heun'].map((s) => item({ sampler: s }));
|
||||
const byId = new Map([E, D, H].map((d) => [d.id, d]));
|
||||
const cs = [];
|
||||
let ts = 1;
|
||||
for (let i = 0; i < 4; i++) cs.push(vote(E, D, 'a', ts++));
|
||||
cs.push(vote(D, E, 'tie', ts++));
|
||||
cs.push(vote(H, D, 'a', ts++));
|
||||
const ids = new Set(byId.keys());
|
||||
const sb = R.computeScoreboard(cs, byId, R.computeBradleyTerry(cs, ids), {});
|
||||
const rows = Object.fromEntries(sb.axes.sampler.rows.map((r) => [r.value, r]));
|
||||
assert.strictEqual(rows.euler.n, 5);
|
||||
assert.strictEqual(rows.euler.wins, 4); assert.strictEqual(rows.euler.ties, 1);
|
||||
assert.ok(Math.abs(rows.euler.winPct - 4.5 / 5) < 1e-9);
|
||||
assert.strictEqual(rows.dpm.n, 6);
|
||||
assert.ok(Math.abs(rows.dpm.winPct - 0.5 / 6) < 1e-9);
|
||||
assert.strictEqual(rows.heun.n, 1); assert.strictEqual(rows.heun.enough, false);
|
||||
assert.strictEqual(sb.axes.sampler.best, 'euler');
|
||||
assert.strictEqual(sb.attributable, 6); assert.strictEqual(sb.total, 6);
|
||||
assert.ok(rows.euler.meanRating > rows.dpm.meanRating);
|
||||
assert.ok(rows.euler.wilsonLow < rows.euler.winPct && rows.euler.wilsonHigh > rows.euler.winPct);
|
||||
});
|
||||
test('scoreboard: non-attributable counted, seed is noise floor, restrictIds', () => {
|
||||
const a = item({}), b = item({ sampler: 'x', cfg: 5 }), c = item({ vae: 'v1' }), d = item({ seed: 9 });
|
||||
const byId = new Map([a, b, c, d].map((x) => [x.id, x]));
|
||||
const cs = [vote(a, b, 'a', 1), vote(a, c, 'a', 2), vote(a, c, 'b', 3), vote(a, d, 'tie', 4)];
|
||||
const sb = R.computeScoreboard(cs, byId, new Map(), {});
|
||||
assert.strictEqual(sb.total, 4); assert.strictEqual(sb.attributable, 0); assert.strictEqual(sb.nonAttributable, 3);
|
||||
assert.strictEqual(sb.topExtraKeys[0].key, 'vae'); assert.strictEqual(sb.topExtraKeys[0].count, 2);
|
||||
assert.ok(sb.topExtraKeys.some((k) => k.key === 'cfg') && sb.topExtraKeys.some((k) => k.key === 'sampler'));
|
||||
assert.strictEqual(sb.axes.seed.rows.length, 2);
|
||||
const only = R.computeScoreboard(cs, byId, new Map(), { restrictIds: new Set([a.id, b.id]) });
|
||||
assert.strictEqual(only.total, 1);
|
||||
assert.strictEqual(R.computeScoreboard([vote(a, 424242, 'a', 9)], byId, new Map(), {}).total, 0);
|
||||
});
|
||||
test('scoreboard CSV quoting + wilson', () => {
|
||||
const a = item({ config_positive: 'a, "quoted" line' }), b = item({ config_positive: 'plain' });
|
||||
const byId = new Map([a, b].map((x) => [x.id, x]));
|
||||
const cs = [vote(a, b, 'a', 1), vote(a, b, 'a', 2), vote(b, a, 'b', 3)];
|
||||
const csv = R.scoreboardToCSV(R.computeScoreboard(cs, byId, new Map(), {}));
|
||||
assert.ok(csv.startsWith('axis,value,n,wins,losses,ties,win_pct,wilson_low,wilson_high,mean_rating,best\n'));
|
||||
assert.ok(csv.includes('"a, ""quoted"" line"'));
|
||||
const w = R.wilsonInterval(5, 10);
|
||||
assert.ok(w.low < 0.5 && w.high > 0.5);
|
||||
assert.deepStrictEqual(R.wilsonInterval(0, 0), { low: 0, high: 0 });
|
||||
});
|
||||
test('mergeComparisons dedupes by ts:a:b', () => {
|
||||
const m = R.mergeComparisons([{ a: 1, b: 2, ts: 5 }], [{ a: 1, b: 2, ts: 5 }, { a: 1, b: 2, ts: 6 }, { a: 2, b: 1, ts: 5 }]);
|
||||
assert.strictEqual(m.length, 3);
|
||||
});
|
||||
Reference in New Issue
Block a user