28:null 29:["$","title",null,{"children":"OSM Training Set Creation | cguastininlt | Observable"}] 2f:T2416,AOIMap = { d3.select("#mapContainer").remove(); let container = DOM.element('div', { style: `width:${width}px;height:${width/1.5}px` }); container.id = 'mapContainer'; let latbox = container.appendChild(document.createElement('div')); latbox.style.position = 'absolute'; latbox.id = 'coords'; latbox.style.top = '0'; latbox.style.left = '0'; latbox.style.textAlign = 'center'; latbox.style.width = '150px'; latbox.style.height = '25px'; latbox.style.backgroundColor = '#fff'; latbox.style.zIndex = 999; latbox.style.lineHeight = '25px'; let coords = latbox.appendChild(document.createElement('p')); coords.style.fontHeight = '14px'; //yield so that the size can be established on the DOM yield container; //reset feature id count let serialCount = 0; //reset variables or load existing if (previousJson.features.length > 0) { for (let i = 0; i < previousJson.features.length; i++) { outputCells[previousJson.features[i].properties.id] = previousJson.features[i]; } let keys = Object.keys(outputCells); //get serialCount left off on serialCount = d3.max(keys, d => +d) + 1; outputCellsJSON.features = previousJson.features; } else { for (var member in outputCells) delete outputCells[member]; outputCellsJSON.features = []; } mutable numCells = outputCellsJSON.features.length; //create map to be plotted in container let map = new mapboxgl.Map({ container, center: [(bboxTraining[1]+bboxTraining[3])/2.0, (bboxTraining[0]+bboxTraining[2])/2], zoom: 10, style: { 'version':8, 'sources': { 'raster-tiles': { 'type':'raster', 'tiles': [ 'https://server.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/{z}/{y}/{x}' ], 'tileSize':256, 'attribution':'Tiles © Esri — Source: Esri, i-cubed, USDA, USGS, AEX, GeoEye, Getmapping, Aerogrid, IGN, IGP, UPR-EGP, and the GIS User Community' } }, 'layers':[ { 'id':'simple-tiles', 'type':'raster', 'source':'raster-tiles' } ] }, scrollZoom: true, bounds:mapBounds }); //many things have to happen after map loads map.on('load', function() { //will plot the cell that moves with the mouse map.addSource('grid' , { 'type':'geojson', 'data': hoverCell }) //done source //will plot all of the cells that will be processed downstream map.addSource('outGrid' , { 'type':'geojson', 'data': outputCellsJSON }) //done source //will plot the outline of the OSM query area map.addSource('bounding', { 'type':'geojson', 'data':grid }) //buildings that are always plotted (full dataset) map.addSource('buildingsOn' , { 'type':'geojson', 'data':geojson }) //done source buildingsOn //buildings intersecting a given cell map.addSource('buildings' , { 'type':'geojson', 'data':empty }) //done source buildings //adding full building layer map.addLayer({ 'id':'buildingsCellOn', 'type':'fill', 'source':'buildingsOn', 'paint': { 'fill-color':'#ff0000', 'fill-opacity':0.6, 'fill-outline-color':'#8F1919' } }) //end addLayer buildingsCell //grid cell fill following mouse map.addLayer({ 'id':'gridCellsFill', 'type':'fill', 'source':'grid', 'paint': { 'fill-color':'#ffffff', 'fill-opacity':0.3, } }) //end addLayer gridCells //output grid cells map.addLayer({ 'id':'outGridCellsFill', 'type':'fill', 'source':'outGrid', 'paint': { 'fill-color':'#ffffff', 'fill-opacity':0.4, 'fill-outline-color':'#000000' } }) //end addLayer gridCells //grid cell outline following mouse. Unfortunately need two separate layers for fill and outline map.addLayer({ 'id':'gridCells', 'type':'line', 'source':'grid', 'paint': { 'line-color':'#ca11de', 'line-opacity':1.0 } }) //end addLayer gridCells //query area outline layer map.addLayer({ 'id':'outline', 'type':'line', 'source':'bounding', 'paint': { 'line-color':'#ffffff', 'line-opacity':1.0 } }) //end addLayer gridCells let currentID = null; map.on('mousemove', function(e) { //get cell gridResolution by gridResolution using AEQ let point = e.lngLat.wrap(); let boxInfo = corners(point); d3.select("#coords").text('lon: '+Math.round(point.lng*1000)/1000+' lat: '+Math.round(point.lat*1000)/1000); hoverCell.features = [boxInfo.poly]; map.getSource('grid').setData(hoverCell); let ids = tree.search( boxInfo.box[0], boxInfo.box[1], boxInfo.box[2], boxInfo.box[3]); let outFeatures = []; let toAdd; for (let i=0; i\n

Have similar terrain and building types to the area in which you want to determine OSM completeness

Contain buildings that have been mapped

\n\nThe amount of OSM data downloaded can increase rapidly as the bounding box size increases, so beware of specifying a large box. As a rough rule of thumb, start with a small box (roughly equal to 0.5 × 0.5-degrees) and increase from there.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":13,"value":"bboxTraining = {\n //define bounding box [miny, minx, maxy, maxx] in degrees latitude and longitude\n //Sample area in Nepal\n let bboxArray = [26.6040521352746, 86.1658972164169, 26.8602670933179 ,86.6748399415154];\n return bboxArray;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":337,"value":"md`If you are resuming training set creation, attach the JSON file you downloaded previously to this notebook and change the file name in the cell below`","pinned":false,"hidden":false,"mode":"ojs"},{"id":340,"value":"previousJson = {\n try {\n return FileAttachment(\"previous_file.json\").json();\n }\n catch(err) {\n return {\n type:\"FeatureCollection\",\n features:[]\n };\n }\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":313,"value":"md`**Download the cell below as JSON by clicking the three vertical dots to the left of the cell when you have finished creating your training collection.**`","pinned":false,"hidden":false,"mode":"ojs"},{"id":162,"value":"outputCellsJSON = {\n return {type:\"FeatureCollection\",features:[]}\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":318,"value":"md`When creating the training set:\n

Click the map to drop a polygon where building mapping is complete. You will likely have to zoom in for the red building footprints to appear on the map.
Try to vary the built up area type (urban vs. rural) when dropping polygons. More generally, try to vary the percent of polygon area covered by buildings.
Drop polygons in areas that have no buildings (e.g., the middle of a forest, desert, lake, or river) and should not have buildings to help the model learn where building footprint area is 0. There is no definitive number you will need, but aim for around 10 percent of all samples.
There is no magic number of training samples that need to be collected for a skillful model, but odds are you will need at least a few hundred. A skillful model for Nepal was created using approximately 7000 samples across the country.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":168,"value":"mutable numCells = 0;","pinned":false,"hidden":false,"mode":"ojs"},{"id":16,"value":"$2f","pinned":false,"hidden":false,"mode":"ojs"},{"id":354,"value":"md`**Cells needed for calculations**`","pinned":false,"hidden":false,"mode":"ojs"},{"id":60,"value":"myQuery =`\n[out:json]\n[timeout:120];\n( \n way[\"building\"](${bboxTraining.join()});\n relation[\"building\"](${bboxTraining.join()});\n);\nout;\n>;\nout skel qt;\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":111,"value":"gridResolution = 250 //meters","pinned":false,"hidden":false,"mode":"ojs"},{"id":122,"value":"toRadians = Math.PI/180.;","pinned":false,"hidden":false,"mode":"ojs"},{"id":125,"value":"toDegrees = 180./Math.PI;","pinned":false,"hidden":false,"mode":"ojs"},{"id":107,"value":"corners = function(centerPoint) {\n const R = 6371000;\n let phi1 = centerPoint.lat;\n let lambda0 = centerPoint.lng;\n let dx = gridResolution/2;\n let dy = gridResolution/2;\n let rho = Math.sqrt(Math.pow(dx,2) + Math.pow(dy,2));\n let c = rho/R;\n let sinc = Math.sin(c);\n let cosc = Math.cos(c);\n let sinphi1 = Math.sin(phi1*toRadians);\n let cosphi1 = Math.cos(phi1*toRadians);\n \n \n function returnLatLon(x, y) {\n let phi = Math.asin(cosc*sinphi1 + (y*sinc*cosphi1)/rho);\n let lambda = lambda0*toRadians + Math.atan((x*sinc)/(rho*cosphi1*cosc - y*sinphi1*sinc));\n return [phi*toDegrees,lambda*toDegrees];\n }\n \n \n let ne = returnLatLon(dx, dy);\n let sw = returnLatLon(-dx, -dy);\n let box = [sw[1],sw[0],ne[1],ne[0]];\n let poly = turf.bboxPolygon(box);\n //console.log(poly);\n return {poly:poly,\n box:box};\n \n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":103,"value":"hoverCell = {\n return {type:\"FeatureCollection\",features:[]}\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":75,"value":"empty = {\n return {type:\"FeatureCollection\",features:[]}\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":72,"value":"tree = {\n let features = geojson.features;\n let bush = new flatbush(features.length);\n \n for (let i=0; i {return err.response.data;})\n return res.data\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":34,"value":"mapBounds = {\n return [[bboxTraining[1], bboxTraining[0]], [bboxTraining[3], bboxTraining[2]]];\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":177,"value":"//dictionary of output cells\noutputCells = ({})","pinned":false,"hidden":false,"mode":"ojs"},{"id":70,"value":"geojson = osmtogeojson(osmData)","pinned":false,"hidden":false,"mode":"ojs"},{"id":5,"value":"md`**Loaded modules and libraries**`","pinned":false,"hidden":false,"mode":"ojs"},{"id":22,"value":"mapboxgl = {\n const gl = await require(\"mapbox-gl@1.10.0\");\n if (!gl.accessToken) {\n gl.accessToken =\n \"pk.eyJ1IjoiY2d1YXN0aW5pIiwiYSI6ImNrOWxtOXBkejI3NzQzbW53bHd0aGtkMHMifQ.qtHQqud3I6zzP_LNzZ5UlA\";\n const href = await require.resolve(\"mapbox-gl@1.10.0/dist/mapbox-gl.css\");\n document.head.appendChild(html``);\n }\n return gl;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":7,"value":"martinezPolygonClipping = require(\"martinez-polygon-clipping\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":9,"value":"turf = require('@turf/turf')","pinned":false,"hidden":false,"mode":"ojs"},{"id":11,"value":"flatbush = require('https://unpkg.com/flatbush@3.2.1/flatbush.min.js')","pinned":false,"hidden":false,"mode":"ojs"},{"id":61,"value":"osmtogeojson = require('osmtogeojson@3.0.0-beta.4/osmtogeojson.js')","pinned":false,"hidden":false,"mode":"ojs"},{"id":64,"value":"overpass = query => axios.get('https://lz4.overpass-api.de/api/interpreter', {\n params: {\n data: query\n }\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":66,"value":"axios = require('https://bundle.run/axios@0.18.0')","pinned":true,"hidden":false,"mode":"ojs"},{"id":311,"value":"d3 = require(\"https://d3js.org/d3.v5.min.js\")","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W30","stdlib":"1","titling":"auto"}} 2d:"$2c:body"