45:["$","title",null,{"children":"Sylvester Manor IIIF Spelunk | marii | Observable"}] 4d:T75d,# Sylvester Manor IIIF Spelunk

## Context / Links - NYU DLTS IIIF Documentation: https://sites.dlib.nyu.edu/viewer/api/v1/#collections-endpoint - NYU DLTS JSON Index of available IIIF Collections: https://sites.dlib.nyu.edu/viewer/api/v1/collections - Sylvester Manor Collection IIIF JSON: https://sites.dlib.nyu.edu/viewer/api/presentation/collections/mss208:fales/manifest.json - First item from SM Collection (manifest.json / IIIF Presentation API): https://sites.dlib.nyu.edu/viewer/api/presentation/photos/fales_mss208_cuid0/manifest.json - First item's first canvas' thumbnail ID (direct image URL): https://sites.dlib.nyu.edu/viewer/api/image/photos/fales_mss208_cuid0/1/full/!300,300/0/default.jpg - Full size + full quality image inferred from thumbnail ID (direct image URL): https://sites.dlib.nyu.edu/viewer/api/image/photos/fales_mss208_cuid0/1/full/full/0/default.jpg ## Objectives - Open the [SM Collection JSON](https://sites.dlib.nyu.edu/viewer/api/presentation/collections/mss208:fales/manifest.json), loop through each of the collection's items in `items` and get the `item.manifest`. - Open each `item.manifest` (e.g., [fales_mss208_cuid0](https://sites.dlib.nyu.edu/viewer/api/presentation/photos/fales_mss208_cuid0/manifest.json)), loop through each of the manifest's items in `items` and get the `item.thumbnail[0].id`, e.g., [fales_mss208_cuid0/1](https://sites.dlib.nyu.edu/viewer/api/image/photos/fales_mss208_cuid0/1/full/!300,300/0/default.jpg) - Replace the suffix `!300,300/0/default.jpg` with desired size, e.g., `800,/0/default.jpg` to get the full image (no crop) at reasonable size for processing. - Output a list of all image URLs paired with their parent digital item/manifest ID and the page number to CSV.4e:Tab2,refresh; // rerun (after cache clear) when the clear button is clicked running; // rerun when stop/start is toggled (stopped = cache-only, no network) const concurrency = 8; // simultaneous requests const politeDelay = 50; // ms pause after each network fetch (not for cache hits) const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); let cancelled = false; invalidation.then(() => (cancelled = true)); // Fetch one manifest (cached in IndexedDB, with retry + backoff). // Cache key includes the URL; only the compact fields we need are stored. // When stopped, only the cache is consulted: uncached manifests are skipped (not counted as failures). async function loadManifest(url) { const hit = await get(url); if (hit) return {data: hit, cached: true}; if (!running) return {data: null, cached: true, skipped: true}; let error; for (let attempt = 0; attempt < 4; attempt++) { try { const m = await d3.json(url, {proxy: true}); const data = { label: m.label.en[0], services: m.items.map((c) => c.thumbnail[0].id.replace(/\/full\/.*$/, "")) }; await set(url, data); return {data, cached: false}; } catch (e) { error = e; await sleep(500 * 2 ** attempt); } } return {data: null, cached: false, error: String(error?.message ?? error)}; } const targets = collection.items.slice(0, limit); const results = new Array(targets.length); const failed = failedStore.rows; // {manifest_url, error}; shared with the failures table failed.length = 0; failedStore.target.dispatchEvent(new Event("change")); let next = 0, done = 0, skipped = 0; const progress = display(html`

`); const report = () => (progress.textContent = `${running ? "Running" : "Stopped (cache only)"}: loaded ${done - skipped - failed.length} / ${targets.length} manifests` + (skipped ? `, ${skipped} not yet fetched` : "") + (failed.length ? `, ${failed.length} failed` : "")); report(); await Promise.all(Array.from({length: concurrency}, async () => { while (!cancelled && next < targets.length) { const i = next++; const url = targets[i].id; const {data, cached, skipped: wasSkipped, error} = await loadManifest(url); if (cancelled) break; if (data) results[i] = data.services.map((service, index) => ({label: data.label, manifest_url: url, page: index + 1, service})); // pages are 1-based else if (wasSkipped) skipped++; else { failed.push({manifest_url: url, error}); failedStore.target.dispatchEvent(new Event("change")); } done++; if (done % 10 === 0 || done === targets.length) report(); if (!cached) await sleep(politeDelay); } })); report(); const images = results.filter(Boolean).flat();4f:[] 49:{"id":"169dbef05ec6df88","slug":"sylvester-manor-iiif-spelunk","title":"Sylvester Manor IIIF Spelunk","thumbnail":"1f848120620a053f091166266c103689cbe995873b77229150c7583711d565ea","access_level":"public","create_time":"$D2026-10-26T18:21:25.855Z","update_time":"$D2026-10-26T21:00:32.727Z","view_time":null,"first_public_time":"$D2026-10-26T19:04:59.595Z","has_unpublished_edits":null,"fork":false,"stars":1,"starred":false,"owner":{"type":"team","id":"03c6cf5622638216","login":"marii","name":"marii","avatar_url":"https://avatars-next.observableusercontent.com/d1ccc6bfa34716481dd92b2ca50c29d028e37616c4bfbe4631d3f6ee1f611a12","tier":"free_2026","bio":"","followed_by_you":false,"follows_you":false},"version":81,"publish_version":81,"role":null,"authored":false,"authors":[{"type":"individual","id":"de05d5a26ac95307","login":"marii","name":"marii","avatar_url":"https://avatars-next.observableusercontent.com/d1ccc6bfa34716481dd92b2ca50c29d028e37616c4bfbe4631d3f6ee1f611a12","tier":null,"bio":"","followed_by_you":false,"follows_you":false}],"members":[{"type":"team","id":"03c6cf5622638216","login":"marii","name":"marii","avatar_url":"https://avatars-next.observableusercontent.com/d1ccc6bfa34716481dd92b2ca50c29d028e37616c4bfbe4631d3f6ee1f611a12","tier":"free_2026","role":"editor"}],"comments":[],"custom_thumbnail":true,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Sylvester Manor IIIF Spelunk","cells":[{"id":1,"value":"$4d","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":3,"value":"const collectionUrl = \"https://sites.dlib.nyu.edu/viewer/api/presentation/collections/mss208:fales/manifest.json\";\nconst collection = await d3.json(collectionUrl, {proxy: true});\ndisplay(Object.keys(collection));\ndisplay(collection.manifests?.slice(0, 2) ?? collection.members?.slice(0, 2));","pinned":false,"hidden":false,"mode":"js","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":4,"value":"const sample = await d3.json(collection.items[0].id, {proxy: true});\ndisplay(sample.label);\ndisplay(sample.items[0]);\ndisplay(sample.items[0].thumbnail);","pinned":false,"hidden":false,"mode":"js","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":9,"value":"import {get, set, clear} from \"npm:idb-keyval\";","pinned":true,"hidden":false,"mode":"js","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":10,"value":"## Run it","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":11,"value":"// Clicking clears the IndexedDB cache, then bumps `refresh`, which reruns the fetch cell.\nconst refresh = view((() => {\n const button = html``;\n const el = html`${button}`;\n el.value = 0;\n button.onclick = async () => {\n button.disabled = true;\n await clear();\n el.value++;\n button.disabled = false;\n el.dispatchEvent(new Event(\"input\", {bubbles: true}));\n };\n return el;\n})());","pinned":false,"hidden":false,"mode":"js","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":14,"value":"// Stop/Start toggle. Stopping reruns the fetch cell in cache-only mode (no network requests),\n// so partial results stay visible; starting resumes fetching whatever isn't cached yet.\nconst running = view((() => {\n const button = html``;\n const el = html`${button}`;\n el.value = true;\n const render = () => (button.textContent = el.value ? \"⏸ Stop fetching\" : \"▶ Start fetching\");\n render();\n button.onclick = () => {\n el.value = !el.value;\n render();\n el.dispatchEvent(new Event(\"input\", {bubbles: true}));\n };\n return el;\n})());","pinned":false,"hidden":false,"mode":"js","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":16,"value":"// Shared, long-lived store for failures. Lives outside the fetch cell so the failures table\n// can update live while fetching is still in progress.\nconst failedStore = {rows: [], target: new EventTarget()};","pinned":false,"hidden":false,"mode":"js","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":5,"value":"$4e","pinned":false,"hidden":false,"mode":"js","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":7,"value":"// Requested image width in pixels. 0 = full size. Height is omitted (\"w,\") so the server keeps the aspect ratio.\nconst width = view(Inputs.number({label: \"Set image width (px, 0 = full)\", value: 800, min: 0, step: 100}));\n\n// limit the number of collection items for performance; set to Infinity for the whole collection\nconst limit = view(Inputs.number({label: \"Limit # of collection items to fetch (set to 6284 for the whole collection)\", value: 20, min: 0, max: 6284})); //","pinned":false,"hidden":false,"mode":"js","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":12,"value":"## Results (success)","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":8,"value":"const size = width > 0 ? `${width},` : \"full\";\nconst rows = images.map(({service, manifest_url, label, page}) => ({\n label,\n manifest_id: manifest_url.match(/([^/]+)\\/manifest\\.json$/)?.[1],\n manifest_url,\n page,\n full_image_url: `${service}/full/${size}/0/default.jpg`\n}));\n\nconst link = (url) => html`${url}`;\ndisplay(Inputs.table(rows, {format: {manifest_url: link, full_image_url: link}}));","pinned":false,"hidden":false,"mode":"js","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":6,"value":"const csv = d3.csvFormat(rows);\ndisplay(html`Download CSV (${rows.length} rows)`);","pinned":false,"hidden":false,"mode":"js","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":13,"value":"## Results (fail)","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":15,"value":"// Live-updating: re-renders whenever the fetch cell records a failure (no need to wait for it to finish).\nconst container = display(html`

`);\nconst renderFailed = () => {\n const rows = failedStore.rows.slice();\n container.replaceChildren(rows.length\n ? Inputs.table(rows, {format: {manifest_url: (url) => html`${url}`}})\n : html`No failed manifests.`);\n};\nrenderFailed();\nfailedStore.target.addEventListener(\"change\", renderFailed);\ninvalidation.then(() => failedStore.target.removeEventListener(\"change\", renderFailed));","pinned":false,"hidden":false,"mode":"js","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W4f","stdlib":"2"}} 44:null