34:["$","title",null,{"children":"Esri Living Atlas Wayback Imagery data investigation | Chris Henrick | Observable"}] 3c:T569,// recursively retrieves release numbers & image sizes for the current map center & zoom level from the wayback tilemap service const waybackItemAvailability = await (async function () { return new Promise((resolve, reject) => { const {x, y, z} = latLonToTile(lat, lon, +zoom.toFixed(0)); const results = []; const getTileMapJSON = async (releaseNumber: number) => { try { const url = `${WAYBACK_SERVICE_BASE_URL}/tilemap/${releaseNumber}/${z}/${y}/${x}`; const res = await fetch(url); const tileMapJson = await res.json() as WaybackTilemap; const {select, data, size, } = tileMapJson || {}; const lastReleaseWithLocalChange = select?.[0] ?? releaseNumber; if (data?.[0]) { results.push({ releaseNumber: lastReleaseWithLocalChange, size: size?.[0] ?? 0, url: "" }); } const previousReleaseNumber = data?.[0] ? getPreviousReleaseNumber( lastReleaseWithLocalChange ) : null; if (typeof previousReleaseNumber === "number") { getTileMapJSON(previousReleaseNumber); } else { resolve(results); } } catch (error) { display(String(error)); reject(error); } }; getTileMapJSON(waybackItems.at(0).releaseNumber); }); })(); display(waybackItemAvailability);3d:[] 38:{"id":"ec4aebe23dc96a6a","slug":"esri-living-atlas-wayback-imagery-data-investigation","title":"Esri Living Atlas Wayback Imagery data investigation","thumbnail":null,"access_level":"public","create_time":"$D2026-10-20T16:57:09.676Z","update_time":"$D2026-11-14T02:06:33.614Z","view_time":null,"first_public_time":"$D2026-11-14T01:36:03.966Z","has_unpublished_edits":null,"fork":false,"stars":1,"starred":false,"owner":{"type":"team","id":"38569aaf9cf67d70","login":"clhenrick","name":"Chris Henrick","avatar_url":"https://avatars-next.observableusercontent.com/d2bdcdbb972dc7a42dd08637fca131b9f4f8f7d26f2896497815c62952cbbabe","tier":"starter_2024","bio":"Maps, Data, Viz, UX, Design","followed_by_you":false,"follows_you":false},"version":534,"publish_version":534,"role":null,"authored":false,"authors":[{"type":"individual","id":"152f84cb3c7f97b6","login":"clhenrick","name":"Chris Henrick","avatar_url":"https://avatars-next.observableusercontent.com/d2bdcdbb972dc7a42dd08637fca131b9f4f8f7d26f2896497815c62952cbbabe","tier":null,"bio":"Maps, Data, Viz, UX, Design","followed_by_you":false,"follows_you":false}],"members":[{"type":"team","id":"38569aaf9cf67d70","login":"clhenrick","name":"Chris Henrick","avatar_url":"https://avatars-next.observableusercontent.com/d2bdcdbb972dc7a42dd08637fca131b9f4f8f7d26f2896497815c62952cbbabe","tier":"starter_2024","role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Esri Living Atlas Wayback Imagery data investigation","cells":[{"id":1,"value":"# Esri Living Atlas Wayback Imagery data investigation\n\nThe [Living Atlas Wayback imagery](https://livingatlas.arcgis.com/wayback/#mapCenter=-122.27796%2C37.80572%2C12&mode=explore&active=43291) contains duplicate images in its releases. This notebook attempts to dedupe them in a way that is similar to the `@esri/wayback-core` library's method. My goal was to understand how the deduping process works and to visualize the output results. I'm not convinced that this approach guarantees that all duplicates are removed so intend to iterate on it further.\n\n**Note that when changing the latitude, longitude, and zoom parameters below the data may take some time to load.**","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":67,"value":"## Sample Release Availability","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":70,"value":"Available wayback releases for **(${lat}, ${lon})** at zoom **${zoom}**:\n\n

${uniqueReleases.join(\", \")}

","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":58,"value":"const lat = view(Inputs.range([-85, 85], { step: 0.1, value: 37.8, label: \"latitude\" }));","pinned":false,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":61,"value":"const lon = view(Inputs.range([-180, 180], { step: 0.1, value: -122.27, label: \"longitude\" }));","pinned":false,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":64,"value":"const zoom = view(Inputs.range([0, 20], { step: 1, value: 12, label: \"zoom level\" }));","pinned":false,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":92,"value":"const tileImageUrls = uniqueReleases.map(\n (releaseNumber) => getTileImageUrl({\n itemURL: waybackItemByReleaseNumber.get(releaseNumber).itemURL,\n column: x,\n row: y,\n zoom,\n }));\nconst results = html`

${tileImageUrls.map((url) => html`

`)}

`\ndisplay(results)","pinned":false,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":54,"value":"## Data","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":80,"value":"// the @esri/wayback-core library's method of deduplicating releases, for comparison with this notebook's results\nconst itemsWithLocalChangesFromWaybackCore = await waybackCore.getWaybackItemsWithLocalChanges({ latitude: lat, longitude: lon }, zoom, {});\ndisplay(itemsWithLocalChangesFromWaybackCore)\ndisplay(itemsWithLocalChangesFromWaybackCore.map((d) => d.releaseNum))","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":90,"value":"// available wayback items for the current map center & zoom\nconst itemsWithLocalChanges = uniqueReleases.map((releaseNumber) => {\n const item = waybackItemByReleaseNumber.get(releaseNumber);\n return item;\n})\ndisplay(itemsWithLocalChanges)\ndisplay(itemsWithLocalChanges.map((d) => d.releaseNumber))","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":86,"value":"const candidatesGrouped = d3.group(candidates, (d) => d.size);\ndisplay(candidatesGrouped)\nlet uniqueReleases = [];\nfor (const [size, items] of candidatesGrouped) {\n if (items.length === 1) {\n uniqueReleases.push(items.at(0).releaseNumber);\n } else {\n // remove duplicates\n const deduped = await removeDuplicatesFromGroup(items);\n uniqueReleases = [...uniqueReleases, ...deduped];\n }\n}\ndisplay(uniqueReleases)","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":77,"value":"const candidates = [];\nconst {x, y, z} = latLonToTile(lat, lon, +zoom.toFixed(0));\nfor (const { releaseNumber, size, url } of waybackItemAvailability) {\n const { itemURL } = waybackItemByReleaseNumber.get(releaseNumber);\n if (!itemURL) continue;\n candidates.push({\n releaseNumber,\n size,\n url: getTileImageUrl({itemURL, column: x, row: y, zoom: z}),\n })\n}\ndisplay(candidates.map((d) => d.releaseNumber))","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":83,"value":"$3c","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":71,"value":"/** wayback item release number -> wayback item array position */\nconst waybackItemsIndexMap = waybackItems.entries().reduce((acc, cur) => {\n const [index, item] = cur;\n acc.set(item.releaseNumber, index);\n return acc;\n}, new Map());\ndisplay(waybackItemsIndexMap)","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":74,"value":"/** wayback release number -> wayback item */\nconst waybackItemByReleaseNumber = waybackItems.reduce((acc, cur) => {\n const { releaseNumber } = cur;\n acc.set(releaseNumber, cur);\n return acc;\n}, new Map());\ndisplay(waybackItemByReleaseNumber);","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":28,"value":"/** all available Living Atlas wayback items / releases */\nconst waybackItems = (async function () {\n const dateRegEx = /[0-9]+-[0-9]+-[0-9]+/;\n const config = await getWaybackConfig();\n const configParsed = Object.entries(config).map((entry): WaybackItem => {\n const [releaseNumber, item] = entry as [string, WaybackConfigItem];\n const releaseDate = dateRegEx.exec(item.itemTitle)?.[0] as string;\n const releaseDatetime = getDateTime(releaseDate);\n return {\n ...item,\n releaseNumber: +releaseNumber,\n releaseDate,\n releaseDatetime\n };\n })\n .filter((item) => item.releaseDate && item.releaseDatetime);\n configParsed.sort((a, b) => b.releaseDatetime - a.releaseDatetime);\n display(configParsed);\n return configParsed;\n})();","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":26,"value":"## Helpers","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":94,"value":"async function removeDuplicatesFromGroup(candidates: { releaseNumber: number; size: number; url: string; }[]) {\n // first = most recent, last = least recent release\n const first = candidates[0];\n const last = candidates[candidates.length - 1];\n const [firstResult, lastResult] = await Promise.all([first, last].map(({ url, releaseNumber }) => getCandidateImageData({ url, releaseNumber })));\n if (isEqual(firstResult.data, lastResult.data)) {\n // since the images are sorted already by release date, if the first image and last image are identical then it's safe to assume all images in the group are identical and we can pick just one of them.\n return [last.releaseNumber];\n } else {\n return [first, last].map((c) => c.releaseNumber);\n }\n}","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":91,"value":"function isEqual(arr1?: Uint8Array, arr2?: Uint8Array) {\n // handle null or undefined arrays\n if (!arr1 || !arr2) {\n return false;\n }\n\n // check type of both inputs\n if (!(arr1 instanceof Uint8Array) || !(arr2 instanceof Uint8Array)) {\n return false;\n }\n\n // if lengths are different, arrays are not equal\n if (arr1.length !== arr2.length) {\n return false;\n }\n\n // start comparison from the end of the array as the first several hundreds of items in those array\n // will likely be identitical for most of tile images\n for (let i = arr1.length - 1; i >= 0; i--) {\n if (arr1[i] !== arr2[i]) {\n return false;\n }\n }\n\n return true;\n}","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":87,"value":"async function getCandidateImageData({\n url,\n releaseNumber\n}: LocalChangeCandidate): Promise<{releaseNumber: number; data: Uint8Array}> {\n try {\n const res = await fetch(url);\n if (!res.ok) {\n return {\n releaseNumber,\n data: new Uint8Array()\n };\n }\n const arrBuffer = await res.arrayBuffer();\n const data = new Uint8Array(arrBuffer);\n return {\n releaseNumber,\n data\n };\n } catch {\n return {\n releaseNumber,\n data: new Uint8Array()\n };\n }\n}","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":82,"value":"function getDateTime(dateStr: string): number {\n const [year, month, day] = dateStr.split(\"-\").map((d) => +d);\n const date = new Date(year, month - 1, day);\n return date.getTime();\n}","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":93,"value":"display(WAYBACK_SERVICE_SUB_DOMAINS_PROD[Math.floor(Math.random() * 3)])","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":78,"value":"function getTileImageUrl({itemURL, column, row, zoom}): string | null {\n const subdomain =\n WAYBACK_SERVICE_SUB_DOMAINS_PROD[Math.floor(Math.random() * 3)];\n return itemURL\n .replace(\"wayback\", subdomain)\n .replace(\"{level}\", String(zoom))\n .replace(\"{row}\", String(row))\n .replace(\"{col}\", String(column));\n}","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":72,"value":"function getPreviousReleaseNumber(startReleaseNumber: number): number | null {\n const index = waybackItemsIndexMap.get(startReleaseNumber);\n if (index === \"undefined\") return null;\n return waybackItems[index + 1]?.releaseNumber ?? null;\n}","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":23,"value":"/** fetches a JSON object consisting of all wayback releases */\nasync function getWaybackConfig(): Promise<\n Record | undefined\n> {\n try {\n const res = await fetch(WAYBACK_CONFIG_FILE_PROD);\n if (res.ok) {\n return (await res.json()) as Record;\n }\n } catch (error) {\n console.log(String(error));\n }\n}","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":40,"value":"/** for a given lat, long coordinate and zoom level, look up the availability for each wayback release at that location */\nasync function* lookupWaybackItemAvailability(\n lat: number,\n long: number,\n zoom: number,\n) {\n for (const item of waybackItems) {\n const { releaseNumber } = item;\n const { x, y, z } = latLonToTile(lat, long, +zoom.toFixed(0));\n const url = `${WAYBACK_SERVICE_BASE_URL}/tilemap/${releaseNumber}/${z}/${x}/${y}`;\n try {\n const res = await fetch(url);\n const data = await res.json();\n yield data;\n } catch (error) {\n display(String(error));\n }\n }\n}","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":38,"value":"/**\n * converts a lat, lon, zoom to a map tile's row, column position\n * - code credit: https://mapmath.dev/blog/lat-lon-to-tile-coordinates\n */\nfunction latLonToTile(\n lat: number,\n lon: number,\n zoom: number,\n): { z: number; x: number; y: number } {\n const n = 2 ** zoom;\n const latRad = (lat * Math.PI) / 180;\n const x = Math.floor((n * (lon + 180)) / 360);\n const y = Math.floor(\n (n * (1 - Math.log(Math.tan(latRad) + 1 / Math.cos(latRad)) / Math.PI)) / 2,\n );\n return { z: zoom, x, y };\n}\n","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":19,"value":"## Types","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":88,"value":"type LocalChangeCandidate = {\n /**\n * release number of a wayback item\n */\n releaseNumber: number;\n /**\n * size of the tile image data associated with this wayback release\n */\n size: number;\n /**\n * url of a tile image from this wayback release\n */\n url: string;\n};","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":84,"value":"/**\n * Response structure for wayback tilemap requests\n */\ntype WaybackTilemap = {\n /**\n * Array indicating whether local changes exist for the requested tile\n */\n data: Array;\n /**\n * Array of release numbers corresponding to the closest wayback versions with local changes\n */\n select: Array;\n valid: boolean;\n location: {\n left: number;\n top: number;\n width: number;\n height: number;\n };\n /**\n * Array of sizes (in bytes) of the tile image data for each release\n */\n size: number[];\n};","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":20,"value":"type WaybackConfigItem = {\n itemID: string;\n itemTitle: string;\n itemURL: string;\n metadataLayerUrl: string;\n metadataLayerItemID: string;\n layerIdentifier: string;\n};","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":21,"value":"type WaybackItem = WaybackConfigItem & {\n releaseNumber: number;\n releaseDatetime: number;\n releaseDate: string;\n};","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":16,"value":"## Constants","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":42,"value":"const WAYBACK_SERVICE_BASE_URL = waybackCore.getWaybackServiceBaseURL();","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2,"value":"const WAYBACK_CONFIG_FILE_PROD =\n \"https://s3-us-west-2.amazonaws.com/config.maptiles.arcgis.com/waybackconfig.json\";","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":3,"value":"const WAYBACK_SERVICE_SUB_DOMAINS_PROD = [\"wayback\", \"wayback-a\", \"wayback-b\"] as const;","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":15,"value":"## Dependencies","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":4,"value":"import * as waybackCore from \"npm:@esri/wayback-core\"","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W3d","stdlib":"2"}} 33:null