motionqa.mjs 8.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128
  1. #!/usr/bin/env node
  2. /**
  3. * motionqa.mjs — motion/perf/audio gate for Tier-1 scenes
  4. * (scroll state-machine, audio-reactive, 2.5D composite, scrubbed video).
  5. * Screenshots (shoot.mjs) are blind to time; this scrolls the page under CPU throttle and asserts
  6. * FPS, long-tasks, audio gating, and WebGL console health.
  7. * Usage: node motionqa.mjs <url> [--headed] [--dpr 2] [--throttle 4] [--min-fps 50] [--max-longtask 50] [--json]
  8. * --headed is not optional for a real number on any GPU-dependent page, and a headless run
  9. * reporting 0 console errors is not a pass — a headed run has caught a 404 the headless one missed.
  10. * Measure a PRODUCTION BUILD: a dev server costs roughly double per frame (HMR client, unminified
  11. * bundles, no asset pipeline), so its numbers describe a page nobody will ever load.
  12. * ponytail: reuses the playwright install shoot.mjs already needs; no new deps.
  13. */
  14. import { chromium } from 'playwright';
  15. const argv = process.argv.slice(2);
  16. const url = argv.find(a => !a.startsWith('-'));
  17. const opt = (name, def) => { const i = argv.indexOf('--' + name); return i >= 0 && argv[i + 1] ? +argv[i + 1] : def; };
  18. const jsonMode = argv.includes('--json');
  19. if (!url) {
  20. process.stderr.write('Usage: node motionqa.mjs <url> [--headed] [--throttle 4] [--min-fps 50] [--max-longtask 50] [--json]\n');
  21. process.exit(2);
  22. }
  23. const THROTTLE = opt('throttle', 4), MIN_FPS = opt('min-fps', 50), MAX_LT = opt('max-longtask', 50);
  24. const DPR = opt('dpr', 2) || 2; // `|| 2` so a typo'd --dpr becomes the honest default, not a NaN viewport
  25. const headed = argv.includes('--headed'); // headless chromium has NO GPU (software swiftshader) — WebGL FPS there is a floor, not the real number
  26. const browser = await chromium.launch({ headless: !headed });
  27. // 1440x900 @2x = 5.2 megapixels: a retina laptop, which is what a page like this gets judged on.
  28. // The default DPR 1 renders 1.3MP, and fullscreen post-processing (bloom, DoF, grain, any full-frame
  29. // shader) costs per pixel — so at DPR 1 the expensive part of the frame is a quarter of its real cost
  30. // and the gate says 60fps about a page that stutters on the reviewer's MacBook. Fillrate is the budget,
  31. // not geometry. --dpr 1 is for reproducing an old measurement, not for judging a scene.
  32. const context = await browser.newContext({ viewport: { width: 1440, height: 900 }, deviceScaleFactor: DPR });
  33. const page = await context.newPage();
  34. const consoleErrors = [];
  35. page.on('console', m => { if (m.type() === 'error') consoleErrors.push(m.text()); });
  36. page.on('pageerror', e => consoleErrors.push(String(e)));
  37. await page.goto(url, { waitUntil: 'load' });
  38. // audio/video must be silent on load — sound starts on a user gesture, never before
  39. const audioAutoplaying = await page.evaluate(() =>
  40. [...document.querySelectorAll('audio,video')].some(a => !a.paused && !a.muted && a.volume > 0));
  41. const hasCanvas = await page.evaluate(() => !!document.querySelector('canvas'));
  42. // A dev server costs roughly 2x per frame vs its own production build (HMR client, unminified
  43. // bundles, on-the-fly transforms, no image pipeline). That direction only ever produces FALSE
  44. // FAILURES — but a false FAIL sends you optimizing a scene that was already fast, which is how a
  45. // motion budget gets cut for nothing. Detect the three common dev clients and say so out loud.
  46. const devServer = await page.evaluate(() => !!(
  47. document.querySelector('script[src*="/@vite/client"]') ||
  48. window.__vite_plugin_react_preamble_installed__ ||
  49. window.webpackHotUpdate || window.__webpack_hmr ||
  50. window.next?.router?.isDevBuild || window.__NEXT_DATA__?.buildId === 'development'
  51. ));
  52. // warm up: let CDN modules load, shaders compile, first textures upload — we measure STEADY STATE,
  53. // not the cold-start frame. (Headless chromium renders WebGL via software swiftshader, so for WebGL
  54. // scenes also prefer a headed/GPU run or a lower --throttle; software raster is not representative.)
  55. await page.evaluate(() => new Promise(r => {
  56. scrollTo(0, document.body.scrollHeight);
  57. setTimeout(() => { scrollTo(0, 0); setTimeout(r, 500); }, 900);
  58. }));
  59. const cdp = await page.context().newCDPSession(page);
  60. await cdp.send('Emulation.setCPUThrottlingRate', { rate: THROTTLE });
  61. // `buffered: true` replays every long task since navigation — bundle parse, shader compile,
  62. // PMREM prefilter — into a number the rubric defines as "during the scroll". Observe live only,
  63. // after warm-up, and report load-time separately so both are visible and neither is a false FAIL.
  64. await page.evaluate(() => {
  65. window.__lt = 0; window.__ltLoad = 0;
  66. try {
  67. new PerformanceObserver(l => { for (const e of l.getEntries()) window.__ltLoad = Math.max(window.__ltLoad, e.duration); })
  68. .observe({ type: 'longtask', buffered: true });
  69. new PerformanceObserver(l => { for (const e of l.getEntries()) window.__lt = Math.max(window.__lt, e.duration); })
  70. .observe({ type: 'longtask' });
  71. } catch { /* longtask unsupported */ }
  72. });
  73. // slow full-page scroll over ~4s, sampling rAF deltas for the worst (min) FPS
  74. const minFps = await page.evaluate(() => new Promise(res => {
  75. let last = performance.now(), min = 999, end = last + 4000, seen = 0;
  76. (function tick(t) {
  77. const d = t - last; last = t;
  78. if (d > 0) { min = Math.min(min, 1000 / d); seen++; }
  79. scrollBy(0, innerHeight / 60);
  80. t < end ? requestAnimationFrame(tick) : res(seen > 10 ? min : 60);
  81. })(performance.now());
  82. }));
  83. const maxLongTask = await page.evaluate(() => window.__lt || 0);
  84. const loadLongTask = await page.evaluate(() => window.__ltLoad || 0);
  85. await browser.close();
  86. const fails = [], advisories = [];
  87. // headless chromium has no GPU → software swiftshader inflates WebGL FPS + raster long-tasks.
  88. // Under headless+canvas those two are ADVISORIES (run --headed / on-device for a real number); the
  89. // rest (audio gating, console/WebGL errors) are GPU-independent and stay hard fails.
  90. const softWebgl = hasCanvas && !headed;
  91. const perf = (cond, msg) => { if (cond) (softWebgl ? advisories : fails).push(msg + (softWebgl ? ` [headless software ${hasCanvas ? 'WebGL' : 'rasterization'} — run --headed for a real number]` : '')); };
  92. perf(minFps < MIN_FPS, `minFps ${minFps.toFixed(0)} < ${MIN_FPS} @${THROTTLE}x throttle`);
  93. perf(maxLongTask > MAX_LT, `long task ${maxLongTask.toFixed(0)}ms > ${MAX_LT}ms`);
  94. if (audioAutoplaying) fails.push('audio/video playing with sound on load (must be gesture-gated)');
  95. const ctxErr = consoleErrors.filter(e => /WebGL|Context Lost|Too many active/i.test(e));
  96. if (ctxErr.length) fails.push(`WebGL context error in console: ${ctxErr[0].slice(0, 80)}`);
  97. // Load-time long tasks are real information (bundle parse, shader compile, PMREM prefilter) but
  98. // they are not what the rubric row asks about, so they are reported, never failed on.
  99. if (loadLongTask > MAX_LT) advisories.push(`load-time long task ${loadLongTask.toFixed(0)}ms (bundle parse / shader compile) — not a scroll fail, but it delays interactivity`);
  100. if (devServer) advisories.push('measured against a DEV SERVER — a production build runs roughly 2x faster per frame; re-measure on the built output before you cut anything from the scene');
  101. const megapixels = +((1440 * 900 * DPR * DPR) / 1e6).toFixed(1);
  102. const summary = {
  103. minFps: +minFps.toFixed(0), maxLongTask: +maxLongTask.toFixed(0), loadLongTask: +loadLongTask.toFixed(0),
  104. dpr: DPR, megapixels, devServer,
  105. audioAutoplaying, consoleErrors: consoleErrors.length, headed, softWebgl, fails, advisories,
  106. };
  107. if (jsonMode) {
  108. process.stdout.write(JSON.stringify(summary, null, 2) + '\n');
  109. } else {
  110. // DPR belongs in the headline number: "minFps 57" means nothing without the pixel count behind it,
  111. // and this line is what gets quoted verbatim into the QA sheet.
  112. process.stdout.write(`motionqa: minFps ${summary.minFps} @${THROTTLE}x CPU - 1440x900@${DPR}x (${megapixels}MP) - scroll long-task ${summary.maxLongTask}ms (load ${summary.loadLongTask}ms) - audio ${audioAutoplaying ? 'AUTOPLAYING(!)' : 'gesture-gated'} - console errors ${consoleErrors.length}${softWebgl ? ' [headless/software-WebGL]' : ''}${devServer ? ' [DEV SERVER]' : ''}\n`);
  113. for (const a of advisories) process.stdout.write(`ADVISORY ${a}\n`);
  114. for (const f of fails) process.stdout.write(`FAIL ${f}\n`);
  115. if (!fails.length) process.stdout.write(advisories.length ? 'PASS (perf advisories — verify headed)\n' : 'PASS\n');
  116. }
  117. process.exit(fails.length ? 1 : 0);