| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388 |
- #!/usr/bin/env node
- /**
- * source.mjs — find and download licence-clean assets you don't have to generate.
- *
- * Generation is not always the right tool. You cannot generate a glTF mesh, a 16-bit HDRI
- * environment map, or a correctly-hinted variable font — and CC0 versions of all three exist at
- * production quality. This fetches those, records the licence for every file, and refuses to
- * hand you anything whose terms it can't state.
- *
- * Usage:
- * node scripts/source.mjs <kind> "<query>" [options]
- *
- * kind = hdri | model | texture | icon | font | image | video
- *
- * Options:
- * --limit N how many assets (default 5, cap 20)
- * --out DIR output dir (default assets/sourced)
- * --res 1k|2k|4k|8k resolution for hdri/model/texture (default 2k)
- * --list print a shortlist to stdout; download nothing and write no ledger
- *
- * Writes DIR/ASSETS-SOURCED.md — the licence ledger — on every real fetch. Read it before you ship.
- *
- * Sources (all no-key, verified live 2026-08):
- * hdri/model/texture Poly Haven CC0 no attribution required
- * icon Iconify per set licence reported per icon (MIT/Apache/CC-BY/OFL)
- * font Google Fonts OFL/Apache free for commercial use, no attribution in UI
- * image Openverse CC-BY/BY-SA ATTRIBUTION REQUIRED — ledger carries the string
- * video Coverr free-use no redistribution; see the taste warning below
- *
- * The video caveat is not legal boilerplate. Stock footage is generic by construction — the same
- * drone shot is on three thousand landing pages. Auteur's entire thesis is committed, specific
- * assets, so sourced video is allowed as an ambient loop, a texture, or a fallback, and is never
- * the peak scene. If the wow moment is stock, there is no wow moment.
- */
- import { mkdir, writeFile, readFile } from 'fs/promises';
- import { existsSync } from 'fs';
- import { resolve, join, dirname } from 'path';
- import { tmpdir } from 'os';
- const args = process.argv.slice(2);
- const KINDS = ['hdri', 'model', 'texture', 'icon', 'font', 'image', 'video'];
- if (!args.length || args.includes('--help') || !KINDS.includes(args[0])) {
- console.log(`source.mjs — licence-clean asset sourcing
- node scripts/source.mjs <kind> "<query>" [--limit 5] [--out assets/sourced] [--res 2k] [--list]
- kind: ${KINDS.join(' | ')}
- hdri|model|texture Poly Haven, CC0 — what generation cannot make
- icon Iconify, licence per set
- font Google Fonts, OFL/Apache
- image Openverse, CC — attribution REQUIRED, ledger carries it
- video Coverr, free-use — ambient/fallback only, never the peak`);
- process.exit(args.length && !KINDS.includes(args[0]) ? 1 : 0);
- }
- const kind = args[0];
- const get = (f, d) => { const i = args.indexOf(f); return i !== -1 ? args[i + 1] : d; };
- const has = f => args.includes(f);
- const query = (args[1] && !args[1].startsWith('--') ? args[1] : '').trim();
- const limit = Math.min(parseInt(get('--limit', '5'), 10) || 5, 20);
- const outDir = resolve(get('--out', 'assets/sourced'));
- const res = get('--res', '2k');
- const listOnly = has('--list');
- if (!query) { console.error(`[source] no query. e.g. node scripts/source.mjs ${kind} "coastal dusk"`); process.exit(1); }
- const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
- const H = { 'User-Agent': UA, Accept: '*/*' };
- async function json(url) {
- const r = await fetch(url, { headers: H, signal: AbortSignal.timeout(45_000) });
- if (!r.ok) throw new Error(`${r.status} ${url.slice(0, 70)}`);
- return r.json();
- }
- async function text(url) {
- const r = await fetch(url, { headers: H, signal: AbortSignal.timeout(45_000) });
- if (!r.ok) throw new Error(`${r.status} ${url.slice(0, 70)}`);
- return r.text();
- }
- // Keep pure-digit tokens: Poly Haven names most of its library thing_01 … thing_09, so dropping
- // short words made a specific asset unaddressable ("studio small 03" silently returned 09).
- const words = q => q.toLowerCase().split(/[\s,]+/).filter(w => w.length > 2 || /^\d+$/.test(w));
- /** score a Poly Haven asset against the query across name / tags / categories / attributes */
- function phScore(slug, a, ws) {
- const hay = [slug, a.name, ...(a.tags || []), ...(a.categories || []), ...Object.values(a.attributes || {})]
- .join(' ').toLowerCase();
- return ws.reduce((n, w) => n + (hay.includes(w) ? 1 : 0), 0);
- }
- // ---------------------------------------------------------------- Poly Haven (CC0)
- async function polyhaven(type) {
- const all = await json(`https://api.polyhaven.com/assets?t=${type}`);
- const ws = words(query);
- const ranked = Object.entries(all)
- .map(([slug, a]) => ({ slug, a, score: phScore(slug, a, ws) }))
- .filter(x => x.score > 0)
- .sort((x, y) => y.score - x.score || (y.a.download_count || 0) - (x.a.download_count || 0))
- .slice(0, limit);
- if (!ranked.length) { console.error(`[source] Poly Haven has no ${type} matching "${query}". Try broader words (its tags are physical: "coast", "dusk", "concrete", "chair").`); return []; }
- const out = [];
- for (const { slug, a, score } of ranked) {
- const files = await json(`https://api.polyhaven.com/files/${slug}`).catch(() => null);
- if (!files) continue;
- const item = {
- id: slug,
- title: a.name,
- licence: 'CC0',
- attribution: null, // CC0: none required
- credit: Object.keys(a.authors || {}).join(', '),
- landing: `https://polyhaven.com/a/${slug}`,
- note: [a.categories?.join('/'), Object.entries(a.attributes || {}).map(([k, v]) => `${k}=${v}`).join(' ')].filter(Boolean).join(' · '),
- score, files: [],
- };
- if (type === 'hdris') {
- const pick = files.hdri?.[res] || files.hdri?.['2k'] || files.hdri?.['1k'];
- const f = pick?.hdr || pick?.exr;
- if (f) item.files.push({ url: f.url, path: `${slug}_${res}.${f.url.split('.').pop()}`, size: f.size });
- } else if (type === 'models') {
- const g = files.gltf?.[res]?.gltf || files.gltf?.['2k']?.gltf || files.gltf?.['1k']?.gltf;
- if (g) {
- item.files.push({ url: g.url, path: `${slug}/${g.url.split('/').pop()}`, size: g.size });
- // a glTF without its .bin and textures is a broken file, not a smaller one
- for (const [rel, inc] of Object.entries(g.include || {})) item.files.push({ url: inc.url, path: `${slug}/${rel}`, size: inc.size });
- }
- } else { // textures
- const maps = { Diffuse: 'diff', nor_gl: 'nor', Rough: 'rough', arm: 'arm', Displacement: 'disp' };
- for (const [key, tag] of Object.entries(maps)) {
- const f = files[key]?.[res]?.jpg || files[key]?.['2k']?.jpg || files[key]?.['1k']?.jpg;
- if (f) item.files.push({ url: f.url, path: `${slug}/${slug}_${tag}_${res}.jpg`, size: f.size });
- }
- }
- if (item.files.length) out.push(item);
- }
- return out;
- }
- // ---------------------------------------------------------------- Iconify
- async function icons() {
- const collections = await json('https://api.iconify.design/collections').catch(() => ({}));
- // multi-word queries return nothing; the index is single-keyword
- const terms = words(query).length ? words(query) : [query];
- const seen = new Set(), out = [];
- for (const term of terms) {
- if (out.length >= limit) break;
- const r = await json(`https://api.iconify.design/search?query=${encodeURIComponent(term)}&limit=${limit * 4}`).catch(() => null);
- for (const id of r?.icons || []) {
- if (out.length >= limit || seen.has(id)) continue;
- seen.add(id);
- const [set, name] = id.split(':');
- const c = collections[set] || {};
- out.push({
- id, title: `${c.name || set} / ${name}`,
- licence: c.license?.title || 'unknown',
- attribution: /CC.?BY/i.test(c.license?.title || '') ? `Icon "${name}" from ${c.name || set} — ${c.license?.title}` : null,
- credit: c.author?.name || '', landing: c.license?.url || `https://icon-sets.iconify.design/${set}/`,
- note: `set has ${c.total || '?'} icons`,
- files: [{ url: `https://api.iconify.design/${set}/${name}.svg?height=64`, path: `${set}-${name}.svg` }],
- });
- }
- }
- if (!out.length) console.error('[source] Iconify found nothing — its index is single-keyword; try one noun ("bottle", not "whisky bottle").');
- return out;
- }
- // ---------------------------------------------------------------- Google Fonts
- const GF_BANNED = ['Inter', 'Space Grotesk', 'Instrument Serif', 'Playfair Display'];
- async function fonts() {
- // Not inside --out: that just relocates 1.5MB from your cwd into your shipped assets tree.
- const cacheFile = join(tmpdir(), 'auteur-gf-metadata.json');
- let meta;
- if (existsSync(cacheFile)) meta = JSON.parse(await readFile(cacheFile, 'utf8'));
- else {
- meta = JSON.parse((await text('https://fonts.google.com/metadata/fonts')).replace(/^\)\]\}'\n?/, ''));
- // --list promises to write nothing; a 2.6MB metadata cache is still something.
- if (!listOnly) {
- await mkdir(outDir, { recursive: true });
- await writeFile(cacheFile, JSON.stringify(meta)); // fetch it once per project
- }
- }
- // Google's own category names overlap ("Sans Serif" contains "serif"), so a plain substring
- // match on "serif" happily returns Noto Sans Display. Resolve the category first, filter on it,
- // and only then rank the leftover words.
- const CAT = { serif: 'Serif', slab: 'Serif', sans: 'Sans Serif', grotesque: 'Sans Serif', grotesk: 'Sans Serif', mono: 'Monospace', monospace: 'Monospace', display: 'Display', handwriting: 'Handwriting', script: 'Handwriting' };
- const ws = words(query);
- const wantSans = ws.some(w => w === 'sans' || w.startsWith('grotes'));
- const wantCat = wantSans ? 'Sans Serif' : (ws.map(w => CAT[w]).find(Boolean) || null);
- const wantVariable = ws.some(w => w === 'variable' || w === 'vf');
- const rest = ws.filter(w => !CAT[w] && w !== 'variable' && w !== 'vf');
- // "Display", "Mono" and "Sans" are Google categories AND parts of real family names, so a query
- // like "Playfair Display" was read as a category and filtered its own family out. If the category
- // reading finds nothing, the word was part of a name — drop the constraint and search again.
- const inPool = cat => (meta.familyMetadataList || []).filter(f => (!cat || f.category === cat) && (!wantVariable || (f.axes || []).length));
- let pool = inPool(wantCat);
- if (wantCat && !pool.some(f => rest.some(w => f.family.toLowerCase().includes(w)))) {
- const byName = inPool(null).filter(f => rest.some(w => f.family.toLowerCase().includes(w)));
- if (byName.length) pool = byName;
- }
- const ranked = pool
- .map(f => {
- const name = f.family.toLowerCase();
- // A family must actually match something before it is a candidate. Scoring alone did not
- // enforce that — the "keep banned families listed but demoted" filter let every family
- // through, so `font "Bodoni Moda"` downloaded Roboto.
- const hits = rest.reduce((n, w) => n + (name.includes(w) ? 1 : 0), 0);
- const matched = hits > 0 || (wantCat && !rest.length);
- let score = (wantCat ? 1 : 0) + hits * 2;
- if (name === rest.join(' ')) score += 3; // exact family name wins outright
- if ((f.axes || []).length) score += 0.5; // variable axes are worth having
- // Google's popularity ranking is exactly what makes a font the AI default; a banned family
- // must never be the top suggestion, but it stays listed (flagged) rather than hidden.
- if (GF_BANNED.includes(f.family)) score -= 100;
- return { f, score, matched };
- })
- .filter(x => x.matched)
- .sort((a, b) => b.score - a.score || (a.f.popularity || 9999) - (b.f.popularity || 9999))
- .slice(0, limit);
- if (!ranked.length) console.error(`[source] no Google font matched. Say the shape: "serif variable", "mono", "display grotesk".`);
- const out = ranked.map(({ f }) => {
- const axes = (f.axes || []).map(a => `${a.tag} ${a.min}–${a.max}`).join(', ');
- const banned = GF_BANNED.includes(f.family);
- const spec = (f.axes || []).find(a => a.tag === 'wght')
- ? `${f.family.replace(/ /g, '+')}:wght@${(f.axes.find(a => a.tag === 'wght')).min}..${(f.axes.find(a => a.tag === 'wght')).max}`
- : f.family.replace(/ /g, '+');
- return {
- id: f.family, title: `${f.family} (${f.category})`,
- licence: 'OFL / Apache-2.0 (Google Fonts)', attribution: null, credit: '',
- landing: `https://fonts.google.com/specimen/${f.family.replace(/ /g, '+')}`,
- note: `${banned ? '⛔ ON THE AUTEUR BAN LIST — pick something else. ' : ''}popularity #${f.popularity ?? '?'} · weights ${Object.keys(f.fonts || {}).length}${axes ? ` · variable: ${axes}` : ' · static only'}`,
- css: `https://fonts.googleapis.com/css2?family=${spec}&display=swap`,
- files: [], // filled below with the real woff2
- family: f.family,
- };
- });
- // Most of what this skill builds must have zero third-party origins, so linking
- // fonts.googleapis.com is not an answer. Resolve the CSS with a browser UA (an old UA gets you
- // ttf), take the latin block, and fetch the actual woff2 so the page can self-host.
- for (const it of out) {
- try {
- const css = await text(it.css);
- // Google emits one @font-face per unicode subset, each preceded by a `/* latin */` comment.
- const latin = (css.match(/\/\*\s*latin\s*\*\/([\s\S]*?)(?=\/\*|$)/) || [, ''])[1] || css;
- const url = (latin.match(/url\((https:\/\/[^)]+\.woff2)\)/) || [])[1];
- if (url) {
- it.files.push({ url, path: `${it.family.replace(/ /g, '')}.woff2` });
- it.selfHost = `@font-face{font-family:'${it.family}';src:url('fonts/${it.family.replace(/ /g, '')}.woff2') format('woff2');font-display:swap;font-weight:${(latin.match(/font-weight:\s*([^;]+)/) || [, '400'])[1].trim()};font-style:normal}`;
- }
- } catch { /* leave it link-only; the ledger will show no file */ }
- }
- return out;
- }
- // ---------------------------------------------------------------- Openverse (CC)
- async function images() {
- // commercial,modification filter is mandatory: the unfiltered index is full of by-nc-nd
- const r = await json(`https://api.openverse.org/v1/images/?q=${encodeURIComponent(query)}&page_size=${limit}&license_type=commercial,modification`);
- return (r.results || []).map(x => ({
- id: x.id, title: x.title || x.id,
- licence: `CC ${(x.license || '').toUpperCase()} ${x.license_version || ''}`.trim(),
- attribution: x.attribution || `"${x.title}" by ${x.creator} is licensed under CC ${(x.license || '').toUpperCase()} ${x.license_version || ''}`,
- credit: x.creator || '', landing: x.foreign_landing_url, note: `${x.width}×${x.height} ${x.filetype || ''} · ${x.provider || ''}`,
- files: [{ url: x.url, path: `${(x.title || x.id).replace(/[^\w-]+/g, '_').slice(0, 40)}.${x.filetype || 'jpg'}` }],
- }));
- }
- // ---------------------------------------------------------------- stock video (Coverr)
- // Mixkit is deliberately not wired in: its search page lists mp4s, but its CDN throttles a
- // download to ~15KB in 90s, so every fetch dies on timeout. One working source beats two listed.
- const RES_RANK = { '2160p': 4, '1080p': 3, '720p': 2, '360p': 1 };
- async function videos() {
- let html;
- try { html = await text(`https://coverr.co/s?q=${encodeURIComponent(query)}`); }
- catch (e) { console.error(`[source] coverr: ${e.message}`); return []; }
- // Keep only the curated library (slug `coverr-…`): `user-ai-generation-…` are user uploads
- // with unclear provenance, and cdn-staging is not a URL to build on.
- const best = new Map();
- for (const u of new Set(html.match(/https:\/\/cdn\.coverr\.co\/videos\/coverr-[^"' ]+\.mp4/g) || [])) {
- const [, slug, res] = u.match(/videos\/([^/]+)\/(\d+p)\.mp4$/) || [];
- if (!slug) continue;
- if (!best.has(slug) || RES_RANK[res] > RES_RANK[best.get(slug).res]) best.set(slug, { u, res });
- }
- const out = [...best.entries()].slice(0, limit).map(([slug, { u, res }]) => ({
- id: slug,
- title: slug.replace(/^coverr-/, '').replace(/-\d+$/, '').replace(/-/g, ' '),
- licence: 'Coverr License — free commercial use, redistribution prohibited',
- attribution: null, credit: 'Coverr',
- landing: `https://coverr.co/videos/${slug}`,
- note: `${res} · ⚠ stock — ambient loop / texture / fallback only, never the peak scene`,
- files: [{ url: u, path: `${slug}-${res}.mp4` }],
- }));
- if (!out.length) console.error('[source] no stock video matched — Coverr indexes simple nouns ("snow", "rain", "city", "smoke").');
- return out;
- }
- // ---------------------------------------------------------------- run
- const fetchers = { hdri: () => polyhaven('hdris'), model: () => polyhaven('models'), texture: () => polyhaven('textures'), icon: icons, font: fonts, image: images, video: videos };
- let items = [];
- try { items = await fetchers[kind](); }
- catch (e) { console.error(`[source] ${kind} lookup failed: ${e.message}`); process.exit(1); }
- if (!items.length) { console.error('[source] nothing found.'); process.exit(1); }
- // --list is a read loop: print the shortlist and touch nothing. Appending every exploratory
- // search to the ledger buried three shipped assets under 260 lines of search history.
- if (listOnly) {
- console.log(`
- ${items.length} match(es) for "${query}" (${kind}):
- `);
- for (const it of items) {
- console.log(` ${it.id}`);
- console.log(` ${it.title} · ${it.licence}${it.credit ? ` · ${it.credit}` : ''}`);
- if (it.note) console.log(` ${it.note}`);
- if (it.css) console.log(` css: ${it.css}`);
- console.log(` ${it.landing}`);
- if (it.files.length) console.log(` ${it.files.length} file(s), ${Math.round(it.files.reduce((n, f) => n + (f.size || 0), 0) / 1024)}KB`);
- console.log('');
- }
- console.log(`Nothing downloaded and no ledger written (--list). Re-run without --list to fetch.`);
- process.exit(0);
- }
- await mkdir(outDir, { recursive: true });
- let bytes = 0, files = 0;
- {
- for (const it of items) {
- for (const f of it.files) {
- const dest = join(outDir, kind, f.path);
- if (existsSync(dest)) { f.skipped = true; continue; } // never re-download a cached master
- try {
- const r = await fetch(f.url, { headers: { ...H, Referer: it.landing || '' }, signal: AbortSignal.timeout(120_000) });
- if (!r.ok) { f.error = `HTTP ${r.status}`; continue; }
- const buf = Buffer.from(await r.arrayBuffer());
- await mkdir(dirname(dest), { recursive: true });
- await writeFile(dest, buf);
- f.written = buf.length; bytes += buf.length; files++;
- } catch (e) { f.error = e.message.split('\n')[0]; }
- }
- process.stderr.write(`[source] ${it.id} — ${it.files.filter(f => f.written || f.skipped).length}/${it.files.length} files\n`);
- }
- }
- // ---------------------------------------------------------------- ledger
- const ledgerPath = join(outDir, 'ASSETS-SOURCED.md');
- const prev = existsSync(ledgerPath) ? await readFile(ledgerPath, 'utf8') : `# ASSETS-SOURCED — licence ledger
- Every file below came from someone else. This file is the record of what you may do with it.
- **Before shipping:** every entry with an "attribution required" line must have that credit visible
- on the site (a credits block in the footer is fine). CC0 and OFL entries need nothing.
- Sourced assets are for production use; the moodboard in \`design/\` is not — do not confuse them.
- `;
- const block = [
- '',
- `## ${kind} — "${query}" · ${new Date().toISOString().slice(0, 10)}${listOnly ? ' (list only, nothing downloaded)' : ''}`,
- '',
- ...items.flatMap(it => {
- const l = [`### ${it.title}`];
- l.push(`- licence: **${it.licence}**${it.credit ? ` · by ${it.credit}` : ''}`);
- if (it.attribution) l.push(`- ⚠ **attribution required** — put this on the page: \`${it.attribution}\``);
- l.push(`- source: ${it.landing}`);
- if (it.note) l.push(`- ${it.note}`);
- if (it.selfHost) l.push(`- self-host (no third-party origin): \`${it.selfHost}\``);
- else if (it.css) l.push(`- css: \`<link rel="stylesheet" href="${it.css}">\` (woff2 could not be resolved — this links a third-party origin)`);
- for (const f of it.files) l.push(`- file: \`${kind}/${f.path}\`${f.written ? ` (${Math.round(f.written / 1024)}KB)` : f.skipped ? ' (cached)' : f.error ? ` — FAILED ${f.error}` : ' (not downloaded)'}`);
- return [...l, ''];
- }),
- ].join('\n');
- await writeFile(ledgerPath, prev + block, 'utf8');
- console.log(`\n=== source (${kind}) ===`);
- console.log(`found : ${items.length} for "${query}"`);
- const cached = items.reduce((n, it) => n + it.files.filter(f => f.skipped).length, 0);
- if (!listOnly) console.log(`written : ${files} new${cached ? ` + ${cached} already on disk` : ''}, ${(bytes / 1048576).toFixed(1)}MB → ${join(outDir, kind)}`);
- const needsCredit = items.filter(i => i.attribution);
- if (needsCredit.length) console.log(`⚠ credit: ${needsCredit.length} asset(s) REQUIRE visible attribution — see the ledger`);
- if (kind === 'video') console.log(`⚠ stock video is ambient/fallback material. If your peak scene is stock, you have no peak scene.`);
- console.log(`ledger : ${ledgerPath}`);
- process.exit(0);
|